diff --git a/Rich/RichFutureRecSys/examples/RichFuture.py b/Rich/RichFutureRecSys/examples/RichFuture.py index 9e22424349e9441870fc3c6096b387ecfedd8cd3..835a8b1eb49343e7e6e0c482af6d581289888748 100644 --- a/Rich/RichFutureRecSys/examples/RichFuture.py +++ b/Rich/RichFutureRecSys/examples/RichFuture.py @@ -17,7 +17,7 @@ from GaudiKernel.SystemOfUnits import GeV from Configurables import LHCbApp, GaudiSequencer, DDDBConf from Configurables import UnpackMCParticle, UnpackMCVertex from Configurables import FPEAuditor -import os +import os, socket # -------------------------------------------------------------------------------------- @@ -31,6 +31,9 @@ def addInfo(alg, typ, name): # -------------------------------------------------------------------------------------- +# Print hostname +print("Host", socket.gethostname()) + # Use own DB clones #os.environ['GITCONDDBPATH'] = '/usera/jonesc/NFS/GitDB' @@ -52,9 +55,9 @@ rootFileBaseName = "RichFuture-" + myBuild + "-" + myConf + "-" + histos HistogramPersistencySvc().OutputFile = rootFileBaseName + "-Histos.root" # Event numbers -nEvents = (1000 if not batchMode else 10000000) -EventSelector().PrintFreq = (100 if not batchMode else 250) -#LHCbApp().SkipEvents = 7740 +nEvents = (100 if not batchMode else 99999999) +EventSelector().PrintFreq = (10 if not batchMode else 250) +#LHCbApp().SkipEvents = 1184 # Just to initialise DDDBConf() @@ -70,8 +73,8 @@ from Configurables import SequencerTimerTool SequencerTimerTool("ToolSvc.SequencerTimerTool").NameSize = 40 # Auditors -#AuditorSvc().Auditors += ["FPEAuditor"] -#FPEAuditor().ActivateAt = ["Execute"] +AuditorSvc().Auditors += ["FPEAuditor"] +FPEAuditor().ActivateAt = ["Execute"] #AuditorSvc().Auditors += [ "NameAuditor" ] # The overall sequence. Filled below. @@ -102,19 +105,21 @@ if UseDD4Hep: useMCTracks = True useMCHits = False +# Get time windows from environment vars if set +# otherwise use default values as passed. +PixWin = float(os.getenv("PIXWIN", "3.0")) +PhotWin = float(os.getenv("PHOTWIN", "0.25")) + # Enable 4D reco in each RICH -enable4D = (True, True) +is4D = bool(os.getenv("ENABLE4D", "1")) +enable4D = (is4D, is4D) #enable4D = (False, False) # Average expected time for signal avSignalTime = (13.03, 52.94) # Pixel time window size in each RICH (in ns) -#pixTimeWindow = (9e20, 9e20) -pixTimeWindow = (3.0, 3.0) -#pixTimeWindow = (1.0, 1.0) +pixTimeWindow = (PixWin, PixWin) # Photon reco time window size in each RICH (ns) -#photTimeWindow = (9e20, 9e20) -photTimeWindow = (1.0, 1.0) -#photTimeWindow = (0.25, 0.25) +photTimeWindow = (PhotWin, PhotWin) # Min Momentum cuts minP = (1.0 * GeV, 1.0 * GeV, 1.0 * GeV) diff --git a/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/NoSpill/DIGI/30000000.py b/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/NoSpill/DIGI/30000000.py index 867266279df87e2bf90fe4a0d681a620961b56a1..edb6ea62f6a4c620170c29609ba72af412229c51 100644 --- a/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/NoSpill/DIGI/30000000.py +++ b/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/NoSpill/DIGI/30000000.py @@ -18,15 +18,7 @@ from GaudiConf import IOHelper tag = "U2-NoSpill" # Choose sample luminosity -if "LUMI" in os.environ: - lumi = os.getenv("LUMI") -else: - lumi = "2.0e32" - #lumi = "2.0e33" - #lumi = "3.0e33" - #lumi = "1.0e34" - #lumi = "1.2e34" - #lumi = "1.5e34" +lumi = os.getenv("LUMI", "2.0e33") # DetDesc DB tags dbTag = "dddb-20221004" @@ -53,6 +45,25 @@ for path in searchPaths: for f in files: print(f) +# Batch options ? +if "CONDOR_FILE_BATCH" in os.environ: + # Use env vars to define reduced file range for this job + batch = int(os.environ["CONDOR_FILE_BATCH"]) + nfiles = int(os.environ["CONDOR_FILES_PER_BATCH"]) + print("CONDOR_FILE_BATCH", batch) + print("CONDOR_FILES_PER_BATCH", nfiles) + firstFile = batch * nfiles + lastFile = (batch + 1) * nfiles + if firstFile <= len(data): + if lastFile > len(data): lastFile = len(data) + print("Using restricted file range", firstFile, lastFile) + data = data[firstFile:lastFile] + for f in data: + print(" ", f) + else: + print("WARNING: File range outside input data list") + data = [] + IOHelper('ROOT').inputFiles(data, clear=True) FileCatalog().Catalogs = ['xmlcatalog_file:out.xml'] diff --git a/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py b/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py index 43224273fc6977d1a41a02ab7a71087028de8686..96979f3d0914f6c7dd6b94cf42ab1765bea8b2c0 100644 --- a/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py +++ b/Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py @@ -18,15 +18,7 @@ from GaudiConf import IOHelper tag = "U2-WithSpill" # Choose sample luminosity -if "LUMI" in os.environ: - lumi = os.getenv("LUMI") -else: - lumi = "2.0e32" - #lumi = "2.0e33" - #lumi = "3.0e33" - #lumi = "1.0e34" - #lumi = "1.2e34" - #lumi = "1.5e34" +lumi = os.getenv("LUMI", "2.0e33") # DetDesc DB tags dbTag = "dddb-20221004" @@ -53,6 +45,25 @@ for path in searchPaths: for f in files: print(f) +# Batch options ? +if "CONDOR_FILE_BATCH" in os.environ: + # Use env vars to define reduced file range for this job + batch = int(os.environ["CONDOR_FILE_BATCH"]) + nfiles = int(os.environ["CONDOR_FILES_PER_BATCH"]) + print("CONDOR_FILE_BATCH", batch) + print("CONDOR_FILES_PER_BATCH", nfiles) + firstFile = batch * nfiles + lastFile = (batch + 1) * nfiles + if firstFile <= len(data): + if lastFile > len(data): lastFile = len(data) + print("Using restricted file range", firstFile, lastFile) + data = data[firstFile:lastFile] + for f in data: + print(" ", f) + else: + print("WARNING: File range outside input data list") + data = [] + IOHelper('ROOT').inputFiles(data, clear=True) FileCatalog().Catalogs = ['xmlcatalog_file:out.xml'] diff --git a/Rich/RichFutureRecSys/examples/jobs/RunJobs.py b/Rich/RichFutureRecSys/examples/jobs/RunJobs.py new file mode 100755 index 0000000000000000000000000000000000000000..18830d4eaef624675ffb78f2b3a5cc7396637982 --- /dev/null +++ b/Rich/RichFutureRecSys/examples/jobs/RunJobs.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +############################################################################### +# (c) Copyright 2023 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### + +import subprocess, os, argparse + +parser = argparse.ArgumentParser( + description="Submit condor jobs for RICH reco", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument( + "-l", + "--lumis", + help="luminosity value(s)", + default="1.5e34,1.2e34,1.0e34,3.0e33,2.0e33,2.0e32", + type=str) +parser.add_argument("-4d", "--enable4D", help="Enable 4D", action='store_true') +parser.add_argument( + "--pixwins", + help="Pixel time window(s) (only for 4D)", + default="3.0,2.0,1.0", + type=str) +parser.add_argument( + "--photwins", + help="Photon time window(s) (only for 4D)", + default="1.0,0.5,0.25,0.1,0.05", + type=str) +args = parser.parse_args() +config = vars(args) +print(config) + +enable4D = config["enable4D"] +os.environ["ENABLE4D"] = str(enable4D) + +optsroot = "/usera/jonesc/LHCbCMake/Feature/Rec/Rich/RichFutureRecSys/examples/" + +# To Do. Pass these as parameters below. +#taskOpts = optsroot+"/RichFuture.py" +#dataOpts = optsroot+"/data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py" + +rootDir = os.getcwd() + +for Lumi in config["lumis"].split(","): + + os.environ["LUMI"] = Lumi + + # Known number of input files for each lumi data set + nFiles = (40 if float(Lumi) < 1.0e34 else 100) + # files per subjob. Less for the higher lumi (10^34) samples + filesPerJob = (4 if float(Lumi) < 1.0e34 else 2) + + if enable4D: + + for PixWin in config["pixwins"].split(","): + for PhotWin in config["photwins"].split(","): + os.environ["PIXWIN"] = PixWin + os.environ["PHOTWIN"] = PhotWin + jobName = "4D/lumi-" + Lumi + "/PixWin-" + "{:.3f}".format( + float(PixWin)) + "/PhotWin-" + "{:.3f}".format( + float(PhotWin)) + print("Submitting jobs for", jobName) + subprocess.run([ + optsroot + "jobs/submit.sh", jobName, + str(nFiles), + str(filesPerJob) + ]) + + else: + + jobName = "3D/lumi-" + Lumi + print("Submitting jobs for", jobName) + subprocess.run([ + optsroot + "jobs/submit.sh", jobName, + str(nFiles), + str(filesPerJob) + ]) diff --git a/Rich/RichFutureRecSys/examples/jobs/merge.py b/Rich/RichFutureRecSys/examples/jobs/merge.py index 03fe74d2167cf26a8a207fc0f7a89f9fdc41c001..8a5dc1241566708a33af2f2fd15e468e01384402 100755 --- a/Rich/RichFutureRecSys/examples/jobs/merge.py +++ b/Rich/RichFutureRecSys/examples/jobs/merge.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 ############################################################################### # (c) Copyright 2000-2020 CERN for the benefit of the LHCb Collaboration # @@ -22,7 +22,6 @@ def merge(dir): # Find files in sub-jobs files = sorted(glob.glob(dir + "/jobs/job*/*.root")) - print("Found", len(files), "files to merge in", dir) if len(files) > 0: # group files by base name @@ -34,10 +33,15 @@ def merge(dir): # loop over different files for bf in merge_list.keys(): + print("Merging", bf, "in", dir, "with", len(merge_list[bf]), + "files") + # output merged file + merged = dir + "/" + bf + # if merged file present first remove + subprocess.call(["rm", "-f", merged]) # run merge - subprocess.call( - ["nice", "-n10", "hadd", "-v", "0", "-ff", dir + "/" + bf] + - merge_list[bf]) + subprocess.call(["nice", "-n10", "hadd", "-v", "0", "-ff", merged] + + merge_list[bf]) # delete input files subprocess.call(["rm"] + merge_list[bf]) @@ -47,12 +51,28 @@ def merge(dir): ]) subprocess.call(["rm", "-r", dir + "/jobs"]) + else: + + print("Found no files to merge in", dir) + + +def directory_find(atom, root='.'): + + # Find all directories to merge below root + paths = [] + for path, dirs, files in os.walk(root): + if atom in dirs: + paths += [path] + return paths + if __name__ == '__main__': # Find jobs to merge - dirs = sorted(glob.glob("CondorJobs/*")) + #dirs = sorted(glob.glob("CondorJobs/*")) + dirs = directory_find("jobs", "CondorJobs") print("Found", len(dirs), "directories to merge") + #print(dirs) # Get the number of processors available num_cpu = min(8, multiprocessing.cpu_count()) diff --git a/Rich/RichFutureRecSys/examples/jobs/submit.sh b/Rich/RichFutureRecSys/examples/jobs/submit.sh index 07f01be6b813ef08060166163d668d49b59a1799..8935da74b7251f456fca90db21c0bc825f68ccab 100755 --- a/Rich/RichFutureRecSys/examples/jobs/submit.sh +++ b/Rich/RichFutureRecSys/examples/jobs/submit.sh @@ -17,25 +17,32 @@ if [ -z "$1" ] ; then fi JobName=$1 if [ -z "$2" ] ; then - echo "Second argument is number of files" - exit 1 + # echo "Second argument is number of files" + export NumFiles=-1 +else + export NumFiles=$2 fi -NumFiles=$2 if [ -z "$3" ] ; then - echo "Third argument is files per job" - exit 1 + # echo "Third argument is files per job" + export FilesPerJob=-1 +else + export FilesPerJob=$3 fi -FilesPerJob=$3 -# Deduce number of jobs -if [ $((NumFiles%FilesPerJob)) != 0 ] ; then - echo "Number files ("${NumFiles}") is not an exact multiple of batch size ("${FilesPerJob}")" - exit 1 +if [[ ${NumFiles} -gt 0 ]] ; then + # Deduce number of jobs + if [ $((NumFiles%FilesPerJob)) != 0 ] ; then + echo "Number files ("${NumFiles}") is not an exact multiple of batch size ("${FilesPerJob}")" + exit 1 + fi + export NumJobs=$((NumFiles/FilesPerJob)) +else + export NumJobs=1 fi -NumJobs=$((NumFiles/FilesPerJob)) # Job directory -JobDir=`pwd`"/CondorJobs/"${JobName} +#JobDir=`pwd`"/CondorJobs/"${JobName} +JobDir=${HOME}"/LHCb/output/CondorJobs/"${JobName} echo "Job Directory "${JobDir} rm -rf ${JobDir} mkdir -p ${JobDir} @@ -44,7 +51,7 @@ mkdir -p ${JobDir} mkdir -p ${JobDir}"/jobs" OptsRoot="${HOME}/LHCbCMake/Feature/Rec/Rich/RichFutureRecSys/examples/" cp ${OptsRoot}"RichFuture.py" ${JobDir}/"jobs/opts.py" -cp ${OptsRoot}"data/PMTs/RealTel40Format/MCLDstFiles.py" ${JobDir}"/jobs/data.py" +cp ${OptsRoot}"data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py" ${JobDir}"/jobs/data.py" export RunOptions="${JobDir}/jobs/opts.py ${JobDir}/jobs/data.py" # Create run script @@ -67,7 +74,14 @@ LogFile=\${JobDir}"/job.log" export CONDOR_FILE_BATCH=\${JobBatch} export CONDOR_FILES_PER_BATCH=\${FilesPerJob} -gaudirun.py -T \${RunOptions} 2>&1 | cat > \${LogFile} +# Count root files already existing +#rootFileC=\`find . -name "*.root" -printf '.' | wc -m\` +#if [[ ${rootFileC} -gt 0 ]] ; then +# echo "ROOT files already exist. Aborting. +# exit 0 +#fi + +gaudirun.py \${RunOptions} 2>&1 | cat > \${LogFile} exit 0 EOFSCRIPT @@ -85,13 +99,10 @@ copy_to_spool = true should_transfer_files = YES when_to_transfer_output = ON_EXIT_OR_EVICT environment = CONDOR_ID=\$(Cluster).\$(Process) -JobBatchName = ${JobName} +JobBatchName = ${JobName/\//_} # Requirements -#Requirements = ( OSTYPE == "CC7" && ( POOL == "GEN_FARM" || ( POOL == "GENERAL" && Machine >= "pcjn.hep.phy.cam.ac.uk" && ( Name == strcat("slot1@",Machine) || ( Name == strcat("slot3@",Machine) && Machine >= "pclq.hep.phy.cam.ac.uk" ) ) ) ) ) -#Requirements = ( OSTYPE == "CC7" && ( POOL == "GEN_FARM" || POOL == "GENERAL" ) ) -Requirements = ( OSTYPE == "CC7" && ( POOL == "GEN_FARM" || ( POOL == "GENERAL" && Machine >= "pclq.hep.phy.cam.ac.uk" ) ) ) -#Requirements = ( OSTYPE == "CC7" && POOL == "GEN_FARM" ) +Requirements = (POOL != "GENERAL" && HAS_r02 && OpSysAndVer == "CentOS7" && TARGET.has_avx2) # Rank hosts according to floating point speed Rank = kflops