From ec051190d27ffb24d71121b910f0c6c9ced526bd Mon Sep 17 00:00:00 2001 From: Giovanni <Giovanni Bassi> Date: Wed, 7 Sep 2022 15:32:34 +0200 Subject: [PATCH 1/5] first implementation Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/24409955 --- .../options/add_retina_clusters_to_digi.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 Hlt/RecoConf/options/add_retina_clusters_to_digi.py diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py new file mode 100755 index 00000000000..cd78ad806d5 --- /dev/null +++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py @@ -0,0 +1,59 @@ +############################################################################### +# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +from Moore import options, run_reconstruction +from Moore.config import Reconstruction, DETECTOR_RAW_BANK_TYPES +from PyConf.Algorithms import RawEventCombiner, bankKiller +from RecoConf.hlt1_tracking import make_SPmixed_raw_event, make_RetinaCluster_raw_event +from PyConf.application import default_raw_event, root_writer + +options.input_files = [ + 'root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi' +] +options.input_type = "ROOT" +options.evt_max = -1 +options.dddb_tag = 'dddb-20190223' +options.conddb_tag = 'sim-20180530-vc-mu100' + +options.output_file = 'test.digi' +options.output_type = 'ROOT' + + +def combiner_digi(): + + data = [] + # mix VELO SPs + spmix = make_SPmixed_raw_event() + data.append(spmix) + # remove VELO SP bank + spkiller = bankKiller(BankTypes=["VP"]) + data.append(spkiller) + # create VELO retina clusters + vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event) + data.append(vpclus) + + detector_parts = [ + default_raw_event([bt]) for bt in DETECTOR_RAW_BANK_TYPES + ] + # get only unique elements while preserving order + detector_parts = list(dict.fromkeys(detector_parts).keys()) + # put mixed SPs and clusters together with rest of event + combiner = RawEventCombiner( + RawEventLocations=detector_parts + [spmix, vpclus]) + + data.append(combiner) + + data.append(root_writer(options.output_file, [combiner.RawEvent])) + + return Reconstruction('write_digi', data) + + +run_reconstruction(options, combiner_digi) -- GitLab From f870661f8e7595c291b73291ac5283234a1f868a Mon Sep 17 00:00:00 2001 From: Giovanni <Giovanni Bassi> Date: Wed, 7 Sep 2022 21:35:05 +0200 Subject: [PATCH 2/5] update option file --- .../options/add_retina_clusters_to_digi.py | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py index cd78ad806d5..5e06a365304 100755 --- a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py +++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py @@ -11,13 +11,11 @@ from Moore import options, run_reconstruction from Moore.config import Reconstruction, DETECTOR_RAW_BANK_TYPES -from PyConf.Algorithms import RawEventCombiner, bankKiller +from PyConf.Algorithms import RawEventCombiner, bankKiller, RawEventDump, Gaudi__Hive__FetchLeavesFromFile, VPSuperPixelBankEncoder, RawEventSimpleCombiner, VPRetinaClusterCreator from RecoConf.hlt1_tracking import make_SPmixed_raw_event, make_RetinaCluster_raw_event -from PyConf.application import default_raw_event, root_writer +from PyConf.application import default_raw_event, root_writer, root_copy_input_writer -options.input_files = [ - 'root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi' -] +options.input_files = ['root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi'] options.input_type = "ROOT" options.evt_max = -1 options.dddb_tag = 'dddb-20190223' @@ -26,34 +24,26 @@ options.conddb_tag = 'sim-20180530-vc-mu100' options.output_file = 'test.digi' options.output_type = 'ROOT' - def combiner_digi(): - - data = [] - # mix VELO SPs - spmix = make_SPmixed_raw_event() - data.append(spmix) - # remove VELO SP bank - spkiller = bankKiller(BankTypes=["VP"]) - data.append(spkiller) - # create VELO retina clusters - vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event) - data.append(vpclus) - - detector_parts = [ - default_raw_event([bt]) for bt in DETECTOR_RAW_BANK_TYPES - ] - # get only unique elements while preserving order - detector_parts = list(dict.fromkeys(detector_parts).keys()) - # put mixed SPs and clusters together with rest of event - combiner = RawEventCombiner( - RawEventLocations=detector_parts + [spmix, vpclus]) - - data.append(combiner) - - data.append(root_writer(options.output_file, [combiner.RawEvent])) - - return Reconstruction('write_digi', data) - + + data = [] + # mix VELO SPs + spmix = make_SPmixed_raw_event() + data.append(spmix) + # create VELO retina clusters + vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event) + data.append(vpclus) + # remove VELO SP bank + spkiller = bankKiller(BankTypes = ["VP"]) + data.append(spkiller) + + # put mixed SPs and clusters together with rest of event + input_leaves = Gaudi__Hive__FetchLeavesFromFile() + combiner = RawEventCombiner(RawEventLocations=[spmix, vpclus]) + data.append(combiner) + + data.append(root_copy_input_writer(options.output_file, input_leaves, [combiner.RawEvent])) + + return Reconstruction('write_digi', data) run_reconstruction(options, combiner_digi) -- GitLab From 78a7cbec81bf6a26d3289ea661aa20a63efe320c Mon Sep 17 00:00:00 2001 From: Giovanni <Giovanni Bassi> Date: Thu, 8 Sep 2022 09:50:34 +0200 Subject: [PATCH 3/5] fixes and add test Fixed formatting patch generated by https://gitlab.cern.ch/lhcb/Moore/-/jobs/24422455 --- .../options/input_add_retina_clusters.py | 22 ++++++ .../options/add_retina_clusters_to_digi.py | 76 +++++++++++-------- .../qmtest/add_retina_clusters_to_digi.qmt | 29 +++++++ 3 files changed, 95 insertions(+), 32 deletions(-) create mode 100755 Hlt/Moore/tests/options/input_add_retina_clusters.py create mode 100755 Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt diff --git a/Hlt/Moore/tests/options/input_add_retina_clusters.py b/Hlt/Moore/tests/options/input_add_retina_clusters.py new file mode 100755 index 00000000000..e57d0018116 --- /dev/null +++ b/Hlt/Moore/tests/options/input_add_retina_clusters.py @@ -0,0 +1,22 @@ +############################################################################### +# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +from Moore import options + +options.input_files = [ + 'root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi' +] +options.input_type = "ROOT" +options.evt_max = -1 +options.dddb_tag = 'dddb-20190223' +options.conddb_tag = 'sim-20180530-vc-mu100' + +options.output_file = 'test.digi' +options.output_type = 'ROOT' diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py index 5e06a365304..375dd929860 100755 --- a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py +++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py @@ -10,40 +10,52 @@ ############################################################################### from Moore import options, run_reconstruction -from Moore.config import Reconstruction, DETECTOR_RAW_BANK_TYPES -from PyConf.Algorithms import RawEventCombiner, bankKiller, RawEventDump, Gaudi__Hive__FetchLeavesFromFile, VPSuperPixelBankEncoder, RawEventSimpleCombiner, VPRetinaClusterCreator -from RecoConf.hlt1_tracking import make_SPmixed_raw_event, make_RetinaCluster_raw_event -from PyConf.application import default_raw_event, root_writer, root_copy_input_writer +from Moore.config import Reconstruction +from PyConf.Algorithms import bankKiller, Gaudi__Hive__FetchLeavesFromFile, RawEventSimpleCombiner, VPRetinaClusterCreator, VPRetinaSPmixer +from RecoConf.hlt1_tracking import make_SPmixed_raw_event +from PyConf.application import default_raw_event, root_copy_input_writer +from PyConf.components import force_location -options.input_files = ['root://eoslhcb.cern.ch//eos/lhcb/wg/IonPhysics/Simulations/SMOG2jpsi/digi/SMOG2jpsi_0-Extended.digi'] -options.input_type = "ROOT" -options.evt_max = -1 -options.dddb_tag = 'dddb-20190223' -options.conddb_tag = 'sim-20180530-vc-mu100' - -options.output_file = 'test.digi' -options.output_type = 'ROOT' def combiner_digi(): - - data = [] - # mix VELO SPs - spmix = make_SPmixed_raw_event() - data.append(spmix) - # create VELO retina clusters - vpclus = make_RetinaCluster_raw_event(make_raw=make_SPmixed_raw_event) - data.append(vpclus) - # remove VELO SP bank - spkiller = bankKiller(BankTypes = ["VP"]) - data.append(spkiller) - - # put mixed SPs and clusters together with rest of event - input_leaves = Gaudi__Hive__FetchLeavesFromFile() - combiner = RawEventCombiner(RawEventLocations=[spmix, vpclus]) - data.append(combiner) - - data.append(root_copy_input_writer(options.output_file, input_leaves, [combiner.RawEvent])) - - return Reconstruction('write_digi', data) + + data = [] + # mix VELO SPs + spmix = VPRetinaSPmixer( + RawEventLocation=default_raw_event(["VP"]), + outputs={ + 'RawEventLocationMixed': force_location('VeloSPmixed/RawEvent') + }) + data.append(spmix) + + # create VELO retina clusters + vpclus = VPRetinaClusterCreator( + RawEventLocation=make_SPmixed_raw_event(["VP"]), + outputs={ + 'RetinaClusterLocation': force_location('VeloCluster/RawEvent') + }) + data.append(vpclus) + + # remove VELO SP bank + spkiller = bankKiller(BankTypes=["VP"]) + data.append(spkiller) + + # put mixed SPs and clusters together with rest of event + input_leaves = Gaudi__Hive__FetchLeavesFromFile() + raw_event_combiner = RawEventSimpleCombiner( + InputRawEventLocations=[ + "VeloSPmixed/RawEvent", "VeloCluster/RawEvent" + ], + EnableIncrementalMode=True, + outputs={ + 'OutputRawEventLocation': force_location('/Event/DAQ/RawEvent') + }) + data.append(raw_event_combiner) + data.append( + root_copy_input_writer(options.output_file, input_leaves, + [raw_event_combiner.OutputRawEventLocation])) + + return Reconstruction('write_digi', data) + run_reconstruction(options, combiner_digi) diff --git a/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt new file mode 100755 index 00000000000..b722ab0279f --- /dev/null +++ b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'> +<!-- + (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration + + This software is distributed under the terms of the GNU General Public + Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". + + In applying this licence, CERN does not waive the privileges and immunities + granted to it by virtue of its status as an Intergovernmental Organization + or submit itself to any jurisdiction. +--> +<!-- +Make sure add_retina_clusters_to_digi does not change. +--> +<extension class="GaudiTest.GaudiExeTest" kind="test"> +<argument name="program"><text>gaudirun.py</text></argument> +<argument name="args"><set> + <text>$MOOREROOT/tests/options/input_add_retina_clusters.py</text> + <text>$RECOCONFROOT/options/add_retina_clusters_to_digi.py</text> +</set></argument> +<argument name="use_temp_dir"><enumeral>true</enumeral></argument> +<argument name="validator"><text> + +from Moore.qmtest.exclusions import remove_known_warnings +countErrorLines({"FATAL": 0, "ERROR": 0, "WARNING": 0}, + stdout=remove_known_warnings(stdout)) + +</text></argument> +</extension> -- GitLab From 679c0c64742f5ed2ad8c8e2f203ec2f2b86ad87e Mon Sep 17 00:00:00 2001 From: Giovanni <Giovanni Bassi> Date: Thu, 8 Sep 2022 11:09:36 +0200 Subject: [PATCH 4/5] update copyright --- Hlt/Moore/tests/options/input_add_retina_clusters.py | 2 +- Hlt/RecoConf/options/add_retina_clusters_to_digi.py | 2 +- Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Hlt/Moore/tests/options/input_add_retina_clusters.py b/Hlt/Moore/tests/options/input_add_retina_clusters.py index e57d0018116..e20896541e0 100755 --- a/Hlt/Moore/tests/options/input_add_retina_clusters.py +++ b/Hlt/Moore/tests/options/input_add_retina_clusters.py @@ -1,5 +1,5 @@ ############################################################################### -# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # diff --git a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py index 375dd929860..cd4d89d3a93 100755 --- a/Hlt/RecoConf/options/add_retina_clusters_to_digi.py +++ b/Hlt/RecoConf/options/add_retina_clusters_to_digi.py @@ -1,5 +1,5 @@ ############################################################################### -# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # +# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration # # # # This software is distributed under the terms of the GNU General Public # # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # diff --git a/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt index b722ab0279f..fbd711245a3 100755 --- a/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt +++ b/Hlt/RecoConf/tests/qmtest/add_retina_clusters_to_digi.qmt @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'> <!-- - (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration + (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration This software is distributed under the terms of the GNU General Public Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". -- GitLab From abb62a536c496a74ee67b6b746886965fdef31f3 Mon Sep 17 00:00:00 2001 From: Giovanni <Giovanni Bassi> Date: Thu, 8 Sep 2022 11:36:06 +0200 Subject: [PATCH 5/5] add corresponding script for retina clusters --- ...plit_for_standalone_Allen_retinacluster.py | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py diff --git a/Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py b/Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py new file mode 100644 index 00000000000..7fbc1ec03df --- /dev/null +++ b/Hlt/RecoConf/scripts/mdf_split_for_standalone_Allen_retinacluster.py @@ -0,0 +1,129 @@ +############################################################################### +# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +# Script to dump larger MC samples into multiple MDF files for Allen standalone processing +# +# Variables to possibly modify: +# key: to choose entry from TestfileDB +# scifi_v6: True if the sample uses scifi raw bank version 6, False otherwise +# n_files_per_chunk: number of input files combined into one output MDF file (output files are named with increasing index) +# base_dir: output directory +# n_jobs_parallel: number of jobs (writing a single MDF file) launched in parallel +# +# author: Dorothea vom Bruch (dorothea.vom.bruch@cern.ch) +# date: 07/2021 +# + +import os +from multiprocessing import Process + +#key = 'Upgrade_BsPhiPhi_MD_FTv4_DIGI' +#key = 'Upgrade_Ds2KKPi_MD_FTv4_DIGI' +#key = 'Upgrade_JPsiMuMu_MD_FTv4_DIGI' +#key = 'Upgrade_KstEE_MD_FTv4_DIGI' +#key = 'Upgrade_KstMuMu_MD_FTv4_DIGI' +#key = 'MiniBrunel_2018_MinBias_FTv4_DIGI' + +# use Scifi v6 format for the following samples +#key = 'upgrade_DC19_01_Bs2PhiPhiMD' +#key = 'upgrade_DC19_01_MinBiasMD' + +key = 'SMOG2_pppHe' + +scifi_v6 = True # set to True when using v6 samples + +base_dir = "/eos/lhcb/wg/rta/WP6/Allen/mdf_input" #os.getcwd() +n_files_per_chunk = 20 + + +class FileMerger(object): + def __init__(self, input_files, output_file): + self.__process = Process(target=self.run) + self.__input_files = input_files + self.__output_file = output_file + + def run(self): + os.environ["TESTFILE_KEY"] = key + os.environ["OUTPUT_FILE"] = self.__output_file + input_file_string = "!".join(self.__input_files) + os.environ["INPUT_FILES"] = input_file_string + os.environ["BASE_DIR"] = base_dir + if scifi_v6: + os.environ["SCIFI_VERSION"] = "6" + else: + os.environ["SCIFI_VERSION"] = "" + os.system( + "gaudirun.py Moore/Hlt/RecoConf/options/mdf_for_standalone_Allen_retinacluster.py" + ) + + def process(self): + return self.__process + + def start(self): + self.__process.start() + + def join(self): + self.__process.join() + + +def chunks(l, n): + """ Yield successive n-sized chunks from l. + """ + for i, j in zip(range(0, len(l), n), range(1, len(l) + 1)): + yield l[i:i + n], j + + +if key == 'SMOG2_pppHe': + file_list = [ + "/eos/lhcb/wg/IonPhysics/Simulations/SMOGHepp8MB/digi/" + string + for string in os.listdir( + r"/eos/lhcb/wg/IonPhysics/Simulations/SMOGHepp8MB/digi/") + ] + files = sorted(file_list) + print(files) +else: + from PRConfig.TestFileDB import test_file_db + qualifiers = test_file_db[key].qualifiers + files = sorted(test_file_db[key].filenames) + +output_dir = base_dir + "/" + key + "/mdf" +if not os.path.exists(base_dir + "/" + key): + os.makedirs(base_dir + "/" + key) +if not os.path.exists(output_dir): + os.makedirs(output_dir) +outfile_pat = output_dir + "/" + key + "_%02d" + +for input_files, i in chunks(files, n_files_per_chunk): + print("at i = " + str(i)) + for file_name in input_files: + print(file_name) + +mergers = [ + FileMerger(input_files, outfile_pat % i) + for input_files, i in chunks(files, n_files_per_chunk) +] +#mergers[0].start() +#mergers[0].join() + +n_jobs = len(mergers) +print("Total number of jobs = " + str(n_jobs)) +n_jobs_parallel = 5 +n_jobs_parallel_chunks = n_jobs // n_jobs_parallel + n_jobs % n_jobs_parallel +print("Number of 5 jobs launched together = " + str(n_jobs_parallel_chunks)) + +for i in range(n_jobs_parallel_chunks): + start = i * n_jobs_parallel + stop = min(start + n_jobs_parallel, n_jobs) + print("Launching jobs " + str(start) + " to " + str(stop)) + for m in mergers[start:stop]: + m.start() + for m in mergers[start:stop]: + m.join() -- GitLab