From ec051190d27ffb24d71121b910f0c6c9ced526bd Mon Sep 17 00:00:00 2001
From: Giovanni <Giovanni Bassi>
Date: Wed, 7 Sep 2022 15:32:34 +0200
Subject: [PATCH 1/5] first implementation

Fixed formatting

patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/24409955
---
 .../options/add_retina_clusters_to_digi.py    | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100755 Hlt/RecoConf/options/add_retina_clusters_to_digi.py

diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
new file mode 100755
index 00000000000..cd78ad806d5
--- /dev/null
+++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
@@ -0,0 +1,59 @@
+###############################################################################
+# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+
+from Moore import options, run_reconstruction
+from Moore.config import Reconstruction, DETECTOR_RAW_BANK_TYPES
+from PyConf.Algorithms import RawEventCombiner, bankKiller
+from RecoConf.hlt1_tracking import make_SPmixed_raw_event, make_RetinaCluster_raw_event
+from PyConf.application import default_raw_event, root_writer
+
+options.input_files = [
+    'root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi'
+]
+options.input_type = "ROOT"
+options.evt_max = -1
+options.dddb_tag = 'dddb-20190223'
+options.conddb_tag = 'sim-20180530-vc-mu100'
+
+options.output_file = 'test.digi'
+options.output_type = 'ROOT'
+
+
+def combiner_digi():
+
+    data = []
+    # mix VELO SPs
+    spmix = make_SPmixed_raw_event()
+    data.append(spmix)
+    # remove VELO SP bank
+    spkiller = bankKiller(BankTypes=["VP"])
+    data.append(spkiller)
+    # create VELO retina clusters
+    vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event)
+    data.append(vpclus)
+
+    detector_parts = [
+        default_raw_event([bt]) for bt in DETECTOR_RAW_BANK_TYPES
+    ]
+    # get only unique elements while preserving order
+    detector_parts = list(dict.fromkeys(detector_parts).keys())
+    # put mixed SPs and clusters together with rest of event
+    combiner = RawEventCombiner(
+        RawEventLocations=detector_parts + [spmix, vpclus])
+
+    data.append(combiner)
+
+    data.append(root_writer(options.output_file, [combiner.RawEvent]))
+
+    return Reconstruction('write_digi', data)
+
+
+run_reconstruction(options, combiner_digi)
-- 
GitLab


From f870661f8e7595c291b73291ac5283234a1f868a Mon Sep 17 00:00:00 2001
From: Giovanni <Giovanni Bassi>
Date: Wed, 7 Sep 2022 21:35:05 +0200
Subject: [PATCH 2/5] update option file

---
 .../options/add_retina_clusters_to_digi.py    | 56 ++++++++-----------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
index cd78ad806d5..5e06a365304 100755
--- a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
+++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
@@ -11,13 +11,11 @@
 
 from Moore import options, run_reconstruction
 from Moore.config import Reconstruction, DETECTOR_RAW_BANK_TYPES
-from PyConf.Algorithms import RawEventCombiner, bankKiller
+from PyConf.Algorithms import RawEventCombiner, bankKiller, RawEventDump, Gaudi__Hive__FetchLeavesFromFile, VPSuperPixelBankEncoder, RawEventSimpleCombiner, VPRetinaClusterCreator
 from RecoConf.hlt1_tracking import make_SPmixed_raw_event, make_RetinaCluster_raw_event
-from PyConf.application import default_raw_event, root_writer
+from PyConf.application import default_raw_event, root_writer, root_copy_input_writer
 
-options.input_files = [
-    'root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi'
-]
+options.input_files = ['root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi']
 options.input_type = "ROOT"
 options.evt_max = -1
 options.dddb_tag = 'dddb-20190223'
@@ -26,34 +24,26 @@ options.conddb_tag = 'sim-20180530-vc-mu100'
 options.output_file = 'test.digi'
 options.output_type = 'ROOT'
 
-
 def combiner_digi():
-
-    data = []
-    # mix VELO SPs
-    spmix = make_SPmixed_raw_event()
-    data.append(spmix)
-    # remove VELO SP bank
-    spkiller = bankKiller(BankTypes=["VP"])
-    data.append(spkiller)
-    # create VELO retina clusters
-    vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event)
-    data.append(vpclus)
-
-    detector_parts = [
-        default_raw_event([bt]) for bt in DETECTOR_RAW_BANK_TYPES
-    ]
-    # get only unique elements while preserving order
-    detector_parts = list(dict.fromkeys(detector_parts).keys())
-    # put mixed SPs and clusters together with rest of event
-    combiner = RawEventCombiner(
-        RawEventLocations=detector_parts + [spmix, vpclus])
-
-    data.append(combiner)
-
-    data.append(root_writer(options.output_file, [combiner.RawEvent]))
-
-    return Reconstruction('write_digi', data)
-
+  
+  data = []
+  # mix VELO SPs
+  spmix = make_SPmixed_raw_event()
+  data.append(spmix)
+  # create VELO retina clusters
+  vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event)
+  data.append(vpclus)
+  # remove VELO SP bank
+  spkiller = bankKiller(BankTypes = ["VP"])
+  data.append(spkiller)
+
+  # put mixed SPs and clusters together with rest of event
+  input_leaves = Gaudi__Hive__FetchLeavesFromFile()
+  combiner = RawEventCombiner(RawEventLocations=[spmix, vpclus])
+  data.append(combiner)
+  
+  data.append(root_copy_input_writer(options.output_file, input_leaves, [combiner.RawEvent]))
+  
+  return Reconstruction('write_digi', data)
 
 run_reconstruction(options, combiner_digi)
-- 
GitLab


From 78a7cbec81bf6a26d3289ea661aa20a63efe320c Mon Sep 17 00:00:00 2001
From: Giovanni <Giovanni Bassi>
Date: Thu, 8 Sep 2022 09:50:34 +0200
Subject: [PATCH 3/5] fixes and add test

Fixed formatting

patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/24422455
---
 .../options/input_add_retina_clusters.py      | 22 ++++++
 .../options/add_retina_clusters_to_digi.py    | 76 +++++++++++--------
 .../qmtest/add_retina_clusters_to_digi.qmt    | 29 +++++++
 3 files changed, 95 insertions(+), 32 deletions(-)
 create mode 100755 Hlt/Moore/tests/options/input_add_retina_clusters.py
 create mode 100755 Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt

diff --git a/Hlt/Moore/tests/options/input_add_retina_clusters.py b/Hlt/Moore/tests/options/input_add_retina_clusters.py
new file mode 100755
index 00000000000..e57d0018116
--- /dev/null
+++ b/Hlt/Moore/tests/options/input_add_retina_clusters.py
@@ -0,0 +1,22 @@
+###############################################################################
+# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+from Moore import options
+
+options.input_files = [
+    'root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi'
+]
+options.input_type = "ROOT"
+options.evt_max = -1
+options.dddb_tag = 'dddb-20190223'
+options.conddb_tag = 'sim-20180530-vc-mu100'
+
+options.output_file = 'test.digi'
+options.output_type = 'ROOT'
diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
index 5e06a365304..375dd929860 100755
--- a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
+++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
@@ -10,40 +10,52 @@
 ###############################################################################
 
 from Moore import options, run_reconstruction
-from Moore.config import Reconstruction, DETECTOR_RAW_BANK_TYPES
-from PyConf.Algorithms import RawEventCombiner, bankKiller, RawEventDump, Gaudi__Hive__FetchLeavesFromFile, VPSuperPixelBankEncoder, RawEventSimpleCombiner, VPRetinaClusterCreator
-from RecoConf.hlt1_tracking import make_SPmixed_raw_event, make_RetinaCluster_raw_event
-from PyConf.application import default_raw_event, root_writer, root_copy_input_writer
+from Moore.config import Reconstruction
+from PyConf.Algorithms import bankKiller, Gaudi__Hive__FetchLeavesFromFile, RawEventSimpleCombiner, VPRetinaClusterCreator, VPRetinaSPmixer
+from RecoConf.hlt1_tracking import make_SPmixed_raw_event
+from PyConf.application import default_raw_event, root_copy_input_writer
+from PyConf.components import force_location
 
-options.input_files = ['root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi']
-options.input_type = "ROOT"
-options.evt_max = -1
-options.dddb_tag = 'dddb-20190223'
-options.conddb_tag = 'sim-20180530-vc-mu100'
-
-options.output_file = 'test.digi'
-options.output_type = 'ROOT'
 
 def combiner_digi():
-  
-  data = []
-  # mix VELO SPs
-  spmix = make_SPmixed_raw_event()
-  data.append(spmix)
-  # create VELO retina clusters
-  vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event)
-  data.append(vpclus)
-  # remove VELO SP bank
-  spkiller = bankKiller(BankTypes = ["VP"])
-  data.append(spkiller)
-
-  # put mixed SPs and clusters together with rest of event
-  input_leaves = Gaudi__Hive__FetchLeavesFromFile()
-  combiner = RawEventCombiner(RawEventLocations=[spmix, vpclus])
-  data.append(combiner)
-  
-  data.append(root_copy_input_writer(options.output_file, input_leaves, [combiner.RawEvent]))
-  
-  return Reconstruction('write_digi', data)
+
+    data = []
+    # mix VELO SPs
+    spmix = VPRetinaSPmixer(
+        RawEventLocation=default_raw_event(["VP"]),
+        outputs={
+            'RawEventLocationMixed': force_location('VeloSPmixed/RawEvent')
+        })
+    data.append(spmix)
+
+    # create VELO retina clusters
+    vpclus = VPRetinaClusterCreator(
+        RawEventLocation=make_SPmixed_raw_event(["VP"]),
+        outputs={
+            'RetinaClusterLocation': force_location('VeloCluster/RawEvent')
+        })
+    data.append(vpclus)
+
+    # remove VELO SP bank
+    spkiller = bankKiller(BankTypes=["VP"])
+    data.append(spkiller)
+
+    # put mixed SPs and clusters together with rest of event
+    input_leaves = Gaudi__Hive__FetchLeavesFromFile()
+    raw_event_combiner = RawEventSimpleCombiner(
+        InputRawEventLocations=[
+            "VeloSPmixed/RawEvent", "VeloCluster/RawEvent"
+        ],
+        EnableIncrementalMode=True,
+        outputs={
+            'OutputRawEventLocation': force_location('/Event/DAQ/RawEvent')
+        })
+    data.append(raw_event_combiner)
+    data.append(
+        root_copy_input_writer(options.output_file, input_leaves,
+                               [raw_event_combiner.OutputRawEventLocation]))
+
+    return Reconstruction('write_digi', data)
+
 
 run_reconstruction(options, combiner_digi)
diff --git a/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt
new file mode 100755
index 00000000000..b722ab0279f
--- /dev/null
+++ b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+    (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration
+
+    This software is distributed under the terms of the GNU General Public
+    Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+
+    In applying this licence, CERN does not waive the privileges and immunities
+    granted to it by virtue of its status as an Intergovernmental Organization
+    or submit itself to any jurisdiction.
+-->
+<!--
+Make sure add_retina_clusters_to_digi does not change.
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+<argument name="program"><text>gaudirun.py</text></argument>
+<argument name="args"><set>
+  <text>$MOOREROOT/tests/options/input_add_retina_clusters.py</text>
+  <text>$RECOCONFROOT/options/add_retina_clusters_to_digi.py</text>
+</set></argument>
+<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
+<argument name="validator"><text>
+
+from Moore.qmtest.exclusions import remove_known_warnings
+countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0},
+                stdout=remove_known_warnings(stdout))
+
+</text></argument>
+</extension>
-- 
GitLab


From 679c0c64742f5ed2ad8c8e2f203ec2f2b86ad87e Mon Sep 17 00:00:00 2001
From: Giovanni <Giovanni Bassi>
Date: Thu, 8 Sep 2022 11:09:36 +0200
Subject: [PATCH 4/5] update copyright

---
 Hlt/Moore/tests/options/input_add_retina_clusters.py      | 2 +-
 Hlt/RecoConf/options/add_retina_clusters_to_digi.py       | 2 +-
 Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Hlt/Moore/tests/options/input_add_retina_clusters.py b/Hlt/Moore/tests/options/input_add_retina_clusters.py
index e57d0018116..e20896541e0 100755
--- a/Hlt/Moore/tests/options/input_add_retina_clusters.py
+++ b/Hlt/Moore/tests/options/input_add_retina_clusters.py
@@ -1,5 +1,5 @@
 ###############################################################################
-# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration           #
 #                                                                             #
 # This software is distributed under the terms of the GNU General Public      #
 # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
index 375dd929860..cd4d89d3a93 100755
--- a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
+++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py
@@ -1,5 +1,5 @@
 ###############################################################################
-# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration           #
+# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration           #
 #                                                                             #
 # This software is distributed under the terms of the GNU General Public      #
 # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
diff --git a/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt
index b722ab0279f..fbd711245a3 100755
--- a/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt
+++ b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
 <!--
-    (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration
+    (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration
 
     This software is distributed under the terms of the GNU General Public
     Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
-- 
GitLab


From abb62a536c496a74ee67b6b746886965fdef31f3 Mon Sep 17 00:00:00 2001
From: Giovanni <Giovanni Bassi>
Date: Thu, 8 Sep 2022 11:36:06 +0200
Subject: [PATCH 5/5] add corresponding script for retina clusters

---
 ...plit_for_standalone_Allen_retinacluster.py | 129 ++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py

diff --git a/Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py b/Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py
new file mode 100644
index 00000000000..7fbc1ec03df
--- /dev/null
+++ b/Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py
@@ -0,0 +1,129 @@
+###############################################################################
+# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+
+# Script to dump larger MC samples into multiple MDF files for Allen standalone processing
+#
+# Variables to possibly modify:
+# key: to choose entry from TestfileDB
+# scifi_v6: True if the sample uses scifi raw bank version 6, False otherwise
+# n_files_per_chunk: number of input files combined into one output MDF file (output files are named with increasing index)
+# base_dir: output directory
+# n_jobs_parallel: number of jobs (writing a single MDF file) launched in parallel
+#
+# author: Dorothea vom Bruch (dorothea.vom.bruch@cern.ch)
+# date:   07/2021
+#
+
+import os
+from multiprocessing import Process
+
+#key = 'Upgrade_BsPhiPhi_MD_FTv4_DIGI'
+#key = 'Upgrade_Ds2KKPi_MD_FTv4_DIGI'
+#key = 'Upgrade_JPsiMuMu_MD_FTv4_DIGI'
+#key = 'Upgrade_KstEE_MD_FTv4_DIGI'
+#key = 'Upgrade_KstMuMu_MD_FTv4_DIGI'
+#key = 'MiniBrunel_2018_MinBias_FTv4_DIGI'
+
+# use Scifi v6 format for the following samples
+#key = 'upgrade_DC19_01_Bs2PhiPhiMD'
+#key = 'upgrade_DC19_01_MinBiasMD'
+
+key = 'SMOG2_pppHe'
+
+scifi_v6 = True  # set to True when using v6 samples
+
+base_dir = "/eos/lhcb/wg/rta/WP6/Allen/mdf_input"  #os.getcwd()
+n_files_per_chunk = 20
+
+
+class FileMerger(object):
+    def __init__(self, input_files, output_file):
+        self.__process = Process(target=self.run)
+        self.__input_files = input_files
+        self.__output_file = output_file
+
+    def run(self):
+        os.environ["TESTFILE_KEY"] = key
+        os.environ["OUTPUT_FILE"] = self.__output_file
+        input_file_string = "!".join(self.__input_files)
+        os.environ["INPUT_FILES"] = input_file_string
+        os.environ["BASE_DIR"] = base_dir
+        if scifi_v6:
+            os.environ["SCIFI_VERSION"] = "6"
+        else:
+            os.environ["SCIFI_VERSION"] = ""
+        os.system(
+            "gaudirun.py Moore/Hlt/RecoConf/options/mdf_for_standalone_Allen_retinacluster.py"
+        )
+
+    def process(self):
+        return self.__process
+
+    def start(self):
+        self.__process.start()
+
+    def join(self):
+        self.__process.join()
+
+
+def chunks(l, n):
+    """ Yield successive n-sized chunks from l.
+    """
+    for i, j in zip(range(0, len(l), n), range(1, len(l) + 1)):
+        yield l[i:i + n], j
+
+
+if key == 'SMOG2_pppHe':
+    file_list = [
+        "/eos/lhcb/wg/IonPhysics/Simulations/SMOGHepp8MB/digi/" + string
+        for string in os.listdir(
+            r"/eos/lhcb/wg/IonPhysics/Simulations/SMOGHepp8MB/digi/")
+    ]
+    files = sorted(file_list)
+    print(files)
+else:
+    from PRConfig.TestFileDB import test_file_db
+    qualifiers = test_file_db[key].qualifiers
+    files = sorted(test_file_db[key].filenames)
+
+output_dir = base_dir + "/" + key + "/mdf"
+if not os.path.exists(base_dir + "/" + key):
+    os.makedirs(base_dir + "/" + key)
+if not os.path.exists(output_dir):
+    os.makedirs(output_dir)
+outfile_pat = output_dir + "/" + key + "_%02d"
+
+for input_files, i in chunks(files, n_files_per_chunk):
+    print("at i = " + str(i))
+    for file_name in input_files:
+        print(file_name)
+
+mergers = [
+    FileMerger(input_files, outfile_pat % i)
+    for input_files, i in chunks(files, n_files_per_chunk)
+]
+#mergers[0].start()
+#mergers[0].join()
+
+n_jobs = len(mergers)
+print("Total number of jobs = " + str(n_jobs))
+n_jobs_parallel = 5
+n_jobs_parallel_chunks = n_jobs // n_jobs_parallel + n_jobs % n_jobs_parallel
+print("Number of 5 jobs launched together = " + str(n_jobs_parallel_chunks))
+
+for i in range(n_jobs_parallel_chunks):
+    start = i * n_jobs_parallel
+    stop = min(start + n_jobs_parallel, n_jobs)
+    print("Launching jobs " + str(start) + " to " + str(stop))
+    for m in mergers[start:stop]:
+        m.start()
+    for m in mergers[start:stop]:
+        m.join()
-- 
GitLab