Restore scripts to submit condor jobs for RichFutureRecSys examples

Rich/RichFutureRecSys/examples/RichFuture.py

@@ -17,7 +17,7 @@ from GaudiKernel.SystemOfUnits import GeV
 from Configurables import LHCbApp, GaudiSequencer, DDDBConf
 from Configurables import UnpackMCParticle, UnpackMCVertex
 from Configurables import FPEAuditor
-import os
+import os, socket
 
 # --------------------------------------------------------------------------------------
 
@@ -31,6 +31,9 @@ def addInfo(alg, typ, name):
 
 # --------------------------------------------------------------------------------------
 
+# Print hostname
+print("Host", socket.gethostname())
+
 # Use own DB clones
 #os.environ['GITCONDDBPATH'] = '/usera/jonesc/NFS/GitDB'
 
@@ -52,9 +55,9 @@ rootFileBaseName = "RichFuture-" + myBuild + "-" + myConf + "-" + histos
 HistogramPersistencySvc().OutputFile = rootFileBaseName + "-Histos.root"
 
 # Event numbers
-nEvents = (1000 if not batchMode else 10000000)
-EventSelector().PrintFreq = (100 if not batchMode else 250)
-#LHCbApp().SkipEvents      = 7740
+nEvents = (100 if not batchMode else 99999999)
+EventSelector().PrintFreq = (10 if not batchMode else 250)
+#LHCbApp().SkipEvents      = 1184
 
 # Just to initialise
 DDDBConf()
@@ -70,8 +73,8 @@ from Configurables import SequencerTimerTool
 SequencerTimerTool("ToolSvc.SequencerTimerTool").NameSize = 40
 
 # Auditors
-#AuditorSvc().Auditors += ["FPEAuditor"]
-#FPEAuditor().ActivateAt = ["Execute"]
+AuditorSvc().Auditors += ["FPEAuditor"]
+FPEAuditor().ActivateAt = ["Execute"]
 #AuditorSvc().Auditors += [ "NameAuditor" ]
 
 # The overall sequence. Filled below.
@@ -102,19 +105,21 @@ if UseDD4Hep:
 useMCTracks = True
 useMCHits = False
 
+# Get time windows from environment vars if set
+# otherwise use default values as passed.
+PixWin = float(os.getenv("PIXWIN", "3.0"))
+PhotWin = float(os.getenv("PHOTWIN", "0.25"))
+
 # Enable 4D reco in each RICH
-enable4D = (True, True)
+is4D = bool(os.getenv("ENABLE4D", "1"))
+enable4D = (is4D, is4D)
 #enable4D = (False, False)
 # Average expected time for signal
 avSignalTime = (13.03, 52.94)
 # Pixel time window size in each RICH (in ns)
-#pixTimeWindow = (9e20, 9e20)
-pixTimeWindow = (3.0, 3.0)
-#pixTimeWindow = (1.0, 1.0)
+pixTimeWindow = (PixWin, PixWin)
 # Photon reco time window size in each RICH (ns)
-#photTimeWindow = (9e20, 9e20)
-photTimeWindow = (1.0, 1.0)
-#photTimeWindow = (0.25, 0.25)
+photTimeWindow = (PhotWin, PhotWin)
 
 # Min Momentum cuts
 minP = (1.0 * GeV, 1.0 * GeV, 1.0 * GeV)

Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/NoSpill/DIGI/30000000.py

@@ -18,15 +18,7 @@ from GaudiConf import IOHelper
 tag = "U2-NoSpill"
 
 # Choose sample luminosity
-if "LUMI" in os.environ:
-    lumi = os.getenv("LUMI")
-else:
-    lumi = "2.0e32"
-    #lumi = "2.0e33"
-    #lumi = "3.0e33"
-    #lumi = "1.0e34"
-    #lumi = "1.2e34"
-    #lumi = "1.5e34"
+lumi = os.getenv("LUMI", "2.0e33")
 
 # DetDesc DB tags
 dbTag = "dddb-20221004"
@@ -53,6 +45,25 @@ for path in searchPaths:
     for f in files:
         print(f)
 
+# Batch options ?
+if "CONDOR_FILE_BATCH" in os.environ:
+    # Use env vars to define reduced file range for this job
+    batch = int(os.environ["CONDOR_FILE_BATCH"])
+    nfiles = int(os.environ["CONDOR_FILES_PER_BATCH"])
+    print("CONDOR_FILE_BATCH", batch)
+    print("CONDOR_FILES_PER_BATCH", nfiles)
+    firstFile = batch * nfiles
+    lastFile = (batch + 1) * nfiles
+    if firstFile <= len(data):
+        if lastFile > len(data): lastFile = len(data)
+        print("Using restricted file range", firstFile, lastFile)
+        data = data[firstFile:lastFile]
+        for f in data:
+            print(" ", f)
+    else:
+        print("WARNING: File range outside input data list")
+        data = []
+
 IOHelper('ROOT').inputFiles(data, clear=True)
 FileCatalog().Catalogs = ['xmlcatalog_file:out.xml']
 

Rich/RichFutureRecSys/examples/data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py

@@ -18,15 +18,7 @@ from GaudiConf import IOHelper
 tag = "U2-WithSpill"
 
 # Choose sample luminosity
-if "LUMI" in os.environ:
-    lumi = os.getenv("LUMI")
-else:
-    lumi = "2.0e32"
-    #lumi = "2.0e33"
-    #lumi = "3.0e33"
-    #lumi = "1.0e34"
-    #lumi = "1.2e34"
-    #lumi = "1.5e34"
+lumi = os.getenv("LUMI", "2.0e33")
 
 # DetDesc DB tags
 dbTag = "dddb-20221004"
@@ -53,6 +45,25 @@ for path in searchPaths:
     for f in files:
         print(f)
 
+# Batch options ?
+if "CONDOR_FILE_BATCH" in os.environ:
+    # Use env vars to define reduced file range for this job
+    batch = int(os.environ["CONDOR_FILE_BATCH"])
+    nfiles = int(os.environ["CONDOR_FILES_PER_BATCH"])
+    print("CONDOR_FILE_BATCH", batch)
+    print("CONDOR_FILES_PER_BATCH", nfiles)
+    firstFile = batch * nfiles
+    lastFile = (batch + 1) * nfiles
+    if firstFile <= len(data):
+        if lastFile > len(data): lastFile = len(data)
+        print("Using restricted file range", firstFile, lastFile)
+        data = data[firstFile:lastFile]
+        for f in data:
+            print(" ", f)
+    else:
+        print("WARNING: File range outside input data list")
+        data = []
+
 IOHelper('ROOT').inputFiles(data, clear=True)
 FileCatalog().Catalogs = ['xmlcatalog_file:out.xml']
 

Rich/RichFutureRecSys/examples/jobs/RunJobs.py

+#!/usr/bin/env python3
+###############################################################################
+# (c) Copyright 2023 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+
+import subprocess, os, argparse
+
+parser = argparse.ArgumentParser(
+    description="Submit condor jobs for RICH reco",
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument(
+    "-l",
+    "--lumis",
+    help="luminosity value(s)",
+    default="1.5e34,1.2e34,1.0e34,3.0e33,2.0e33,2.0e32",
+    type=str)
+parser.add_argument("-4d", "--enable4D", help="Enable 4D", action='store_true')
+parser.add_argument(
+    "--pixwins",
+    help="Pixel time window(s) (only for 4D)",
+    default="3.0,2.0,1.0",
+    type=str)
+parser.add_argument(
+    "--photwins",
+    help="Photon time window(s) (only for 4D)",
+    default="1.0,0.5,0.25,0.1,0.05",
+    type=str)
+args = parser.parse_args()
+config = vars(args)
+print(config)
+
+enable4D = config["enable4D"]
+os.environ["ENABLE4D"] = str(enable4D)
+
+optsroot = "/usera/jonesc/LHCbCMake/Feature/Rec/Rich/RichFutureRecSys/examples/"
+
+# To Do. Pass these as parameters below.
+#taskOpts = optsroot+"/RichFuture.py"
+#dataOpts = optsroot+"/data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py"
+
+rootDir = os.getcwd()
+
+for Lumi in config["lumis"].split(","):
+
+    os.environ["LUMI"] = Lumi
+
+    # Known number of input files for each lumi data set
+    nFiles = (40 if float(Lumi) < 1.0e34 else 100)
+    # files per subjob. Less for the higher lumi (10^34) samples
+    filesPerJob = (4 if float(Lumi) < 1.0e34 else 2)
+
+    if enable4D:
+
+        for PixWin in config["pixwins"].split(","):
+            for PhotWin in config["photwins"].split(","):
+                os.environ["PIXWIN"] = PixWin
+                os.environ["PHOTWIN"] = PhotWin
+                jobName = "4D/lumi-" + Lumi + "/PixWin-" + "{:.3f}".format(
+                    float(PixWin)) + "/PhotWin-" + "{:.3f}".format(
+                        float(PhotWin))
+                print("Submitting jobs for", jobName)
+                subprocess.run([
+                    optsroot + "jobs/submit.sh", jobName,
+                    str(nFiles),
+                    str(filesPerJob)
+                ])
+
+    else:
+
+        jobName = "3D/lumi-" + Lumi
+        print("Submitting jobs for", jobName)
+        subprocess.run([
+            optsroot + "jobs/submit.sh", jobName,
+            str(nFiles),
+            str(filesPerJob)
+        ])

Rich/RichFutureRecSys/examples/jobs/merge.py

-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 ###############################################################################
 # (c) Copyright 2000-2020 CERN for the benefit of the LHCb Collaboration      #
@@ -22,7 +22,6 @@ def merge(dir):
 
     # Find files in sub-jobs
     files = sorted(glob.glob(dir + "/jobs/job*/*.root"))
-    print("Found", len(files), "files to merge in", dir)
     if len(files) > 0:
 
         # group files by base name
@@ -34,10 +33,15 @@ def merge(dir):
 
         # loop over different files
         for bf in merge_list.keys():
+            print("Merging", bf, "in", dir, "with", len(merge_list[bf]),
+                  "files")
+            # output merged file
+            merged = dir + "/" + bf
+            # if merged file present first remove
+            subprocess.call(["rm", "-f", merged])
             # run merge
-            subprocess.call(
-                ["nice", "-n10", "hadd", "-v", "0", "-ff", dir + "/" + bf] +
-                merge_list[bf])
+            subprocess.call(["nice", "-n10", "hadd", "-v", "0", "-ff", merged]
+                            + merge_list[bf])
             # delete input files
             subprocess.call(["rm"] + merge_list[bf])
 
@@ -47,12 +51,28 @@ def merge(dir):
         ])
         subprocess.call(["rm", "-r", dir + "/jobs"])
 
+    else:
+
+        print("Found no files to merge in", dir)
+
+
+def directory_find(atom, root='.'):
+
+    # Find all directories to merge below root
+    paths = []
+    for path, dirs, files in os.walk(root):
+        if atom in dirs:
+            paths += [path]
+    return paths
+
 
 if __name__ == '__main__':
 
     # Find jobs to merge
-    dirs = sorted(glob.glob("CondorJobs/*"))
+    #dirs = sorted(glob.glob("CondorJobs/*"))
+    dirs = directory_find("jobs", "CondorJobs")
     print("Found", len(dirs), "directories to merge")
+    #print(dirs)
 
     # Get the number of processors available
     num_cpu = min(8, multiprocessing.cpu_count())

Rich/RichFutureRecSys/examples/jobs/submit.sh

@@ -17,25 +17,32 @@ if [ -z "$1" ] ; then
 fi
 JobName=$1
 if [ -z "$2" ] ; then
-    echo "Second argument is number of files"
-    exit 1
+    # echo "Second argument is number of files"
+    export NumFiles=-1
+else
+    export NumFiles=$2
 fi
-NumFiles=$2
 if [ -z "$3" ] ; then
-    echo "Third argument is files per job"
-    exit 1
+    # echo "Third argument is files per job"
+    export FilesPerJob=-1
+else
+    export FilesPerJob=$3
 fi
-FilesPerJob=$3
 
-# Deduce number of jobs
-if [ $((NumFiles%FilesPerJob)) != 0 ] ; then
-    echo "Number files ("${NumFiles}") is not an exact multiple of batch size ("${FilesPerJob}")"
-    exit 1
+if [[ ${NumFiles} -gt 0 ]] ; then
+    # Deduce number of jobs
+    if [ $((NumFiles%FilesPerJob)) != 0 ] ; then
+        echo "Number files ("${NumFiles}") is not an exact multiple of batch size ("${FilesPerJob}")"
+        exit 1
+    fi
+    export NumJobs=$((NumFiles/FilesPerJob))
+else
+    export NumJobs=1
 fi
-NumJobs=$((NumFiles/FilesPerJob))
 
 # Job directory
-JobDir=`pwd`"/CondorJobs/"${JobName}
+#JobDir=`pwd`"/CondorJobs/"${JobName}
+JobDir=${HOME}"/LHCb/output/CondorJobs/"${JobName}
 echo "Job Directory "${JobDir}
 rm -rf ${JobDir}
 mkdir -p ${JobDir}
@@ -44,7 +51,7 @@ mkdir -p ${JobDir}
 mkdir -p ${JobDir}"/jobs"
 OptsRoot="${HOME}/LHCbCMake/Feature/Rec/Rich/RichFutureRecSys/examples/"
 cp ${OptsRoot}"RichFuture.py" ${JobDir}/"jobs/opts.py"
-cp ${OptsRoot}"data/PMTs/RealTel40Format/MCLDstFiles.py" ${JobDir}"/jobs/data.py"
+cp ${OptsRoot}"data/PMTs/UpgradeII/WithSpill/DIGI/30000000.py" ${JobDir}"/jobs/data.py"
 export RunOptions="${JobDir}/jobs/opts.py ${JobDir}/jobs/data.py"
 
 # Create run script
@@ -67,7 +74,14 @@ LogFile=\${JobDir}"/job.log"
 export CONDOR_FILE_BATCH=\${JobBatch}
 export CONDOR_FILES_PER_BATCH=\${FilesPerJob}
 
-gaudirun.py -T \${RunOptions} 2>&1 | cat > \${LogFile}
+# Count root files already existing
+#rootFileC=\`find . -name "*.root" -printf '.' | wc -m\`
+#if [[ ${rootFileC} -gt 0 ]] ; then
+#   echo "ROOT files already exist. Aborting.
+#   exit 0
+#fi
+
+gaudirun.py \${RunOptions} 2>&1 | cat > \${LogFile}
 
 exit 0
 EOFSCRIPT
@@ -85,13 +99,10 @@ copy_to_spool           = true
 should_transfer_files   = YES
 when_to_transfer_output = ON_EXIT_OR_EVICT
 environment = CONDOR_ID=\$(Cluster).\$(Process)
-JobBatchName            = ${JobName}
+JobBatchName            = ${JobName/\//_}
 
 # Requirements
-#Requirements = ( OSTYPE == "CC7" && ( POOL == "GEN_FARM" || ( POOL == "GENERAL" && Machine >= "pcjn.hep.phy.cam.ac.uk" && ( Name == strcat("slot1@",Machine) || ( Name == strcat("slot3@",Machine) && Machine >= "pclq.hep.phy.cam.ac.uk" ) ) ) ) )
-#Requirements = ( OSTYPE == "CC7" && ( POOL == "GEN_FARM" || POOL == "GENERAL" ) )
-Requirements = ( OSTYPE == "CC7" && ( POOL == "GEN_FARM" || ( POOL == "GENERAL" && Machine >= "pclq.hep.phy.cam.ac.uk" ) ) )
-#Requirements = ( OSTYPE == "CC7" && POOL == "GEN_FARM" )
+Requirements = (POOL != "GENERAL" && HAS_r02 && OpSysAndVer == "CentOS7" && TARGET.has_avx2)
 
 # Rank hosts according to floating point speed
 Rank = kflops