Merge remote-tracking branch 'upstream/master' into mdc_reco

Control/CalypsoExample/Digitization/scripts/submit_faserMDC_digi.sh

@@ -102,7 +102,8 @@ output_directory="$working_directory/$run_number"
 mkdir -p "$output_directory"
 #
 # This magic redirects everything in this script to our log file
-exec >& "$output_directory/$file_stem.log"
+logfile="${file_stem}.rdo.log"
+exec >& "${output_directory}/${logfile}"
 echo `date` - $HOSTNAME
 echo "File: $file_name"
 echo "Release: $release_directory"
@@ -193,14 +194,14 @@ if ! [ -z "$logdest" ]
 then
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $logdest"
+    echo "copy $logfile to $logdest"
     mkdir -p $logdest
-    eos cp "${config_file_stem}-${seg_str}.log" $logdest
+    eos cp $logfile $logdest
 elif ! [ -z "$outdest" ]
 then 
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $outdest"
+    echo "copy $logfile to $outdest"
     mkdir -p $outdest
-    eos cp "${config_file_stem}-${seg_str}.log" $outdest
+    eos cp $logfile $outdest
 fi

Control/CalypsoExample/Digitization/scripts/submit_faserMDC_digi_merge.sh

@@ -123,7 +123,8 @@ mkdir -p "$output_directory"
 file_stem="$faser-$short-$run_number-RDO-merge-$slice"
 #
 # This magic redirects everything in this script to our log file
-exec >& "$output_directory/$file_stem.log"
+logfile="${file_stem}.rdo.log"
+exec >& "$output_directory/$logfile"
 echo `date` - $HOSTNAME
 echo "Directory: $dir_path"
 echo "Slice: $slice"
@@ -216,14 +217,14 @@ if ! [ -z "$logdest" ]
 then
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $logdest"
+    echo "copy $logfile to $logdest"
     mkdir -p $logdest
-    eos cp "${config_file_stem}-${seg_str}.log" $logdest
+    eos cp $logfile $logdest
 elif ! [ -z "$outdest" ]
 then 
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $outdest"
+    echo "copy $logfile to $outdest"
     mkdir -p $outdest
-    eos cp "${config_file_stem}-${seg_str}.log" $outdest
+    eos cp $logfile $outdest
 fi

Control/CalypsoExample/Generation/scripts/submit_faserMDC_foresee.sh

@@ -119,7 +119,8 @@ fi
 mkdir -p "$output_directory"
 #
 # This magic redirects everything in this script to our log file
-exec >& "$output_directory/${config_file_stem}-${seg_str}.log"
+logfile="${config_file_stem}-${seg_str}.gen.log"
+exec >& "$output_directory/${logfile}"
 echo `date` - $HOSTNAME
 echo "File: $config_file"
 echo "Segment: $seg_str"
@@ -209,14 +210,14 @@ if ! [ -z "$logdest" ]
 then
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $logdest"
+    echo "copy $logfile to $logdest"
     mkdir -p $logdest
-    eos cp "${config_file_stem}-${seg_str}.log" $logdest
+    eos cp $logfile $logdest
 elif ! [ -z "$outdest" ]
 then 
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $outdest"
+    echo "copy $logffile to $outdest"
     mkdir -p $outdest
-    eos cp "${config_file_stem}-${seg_str}.log" $outdest
+    eos cp $logfile $outdest
 fi

Control/CalypsoExample/Generation/scripts/submit_faserMDC_particlegun.sh

@@ -117,7 +117,8 @@ fi
 mkdir -p "$output_directory"
 #
 # This magic redirects everything in this script to our log file
-exec >& "$output_directory/${config_file_stem}-${seg_str}.log"
+logfile="${config_file_stem}-${seg_str}.gen.log"
+exec >& "$output_directory/${logfile}"
 echo `date` - $HOSTNAME
 echo "File: $config_file"
 echo "Segment: $seg_str"
@@ -207,14 +208,14 @@ if ! [ -z "$logdest" ]
 then
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $logdest"
+    echo "copy $logfile to $logdest"
     mkdir -p $logdest
-    eos cp "${config_file_stem}-${seg_str}.log" $logdest
+    eos cp $logfile $logdest
 elif ! [ -z "$outdest" ]
 then 
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $outdest"
+    echo "copy $logfile to $outdest"
     mkdir -p $outdest
-    eos cp "${config_file_stem}-${seg_str}.log" $outdest
+    eos cp $logfile $outdest
 fi

Control/CalypsoExample/Reconstruction/CMakeLists.txt

@@ -30,7 +30,12 @@ atlas_add_test( ProdRecoPilotTracks
     SCRIPT scripts/faser_reco.py ${CMAKE_CURRENT_SOURCE_DIR}/../RAWDATA/Faser-Physics-pilot_tracks-filtered.raw TI12Data
     PROPERTIES TIMEOUT 300 )
 
-atlas_add_test( ProdRecoTI12-2022
-    SCRIPT scripts/faser_reco.py ${CMAKE_CURRENT_SOURCE_DIR}/../rawdata/Faser-Physics-006525-filtered.raw TI12Data02
+# Test of TI12Data02 geometry (should auto-select from run number)
+atlas_add_test( ProdRecoTI12-02-2022
+    SCRIPT scripts/faser_reco.py ${CMAKE_CURRENT_SOURCE_DIR}/../rawdata/Faser-Physics-006525-filtered.raw
     PROPERTIES TIMEOUT 300 )
 
+# Test of TI12Data03 geometry (should auto-select from run number)
+atlas_add_test( ProdRecoTI12-03-2022
+    SCRIPT scripts/faser_reco.py ${CMAKE_CURRENT_SOURCE_DIR}/../rawdata/Faser-Physics-007833-00003-TrigMask08.raw
+    PROPERTIES TIMEOUT 300 )

Control/CalypsoExample/Reconstruction/scripts/faser_reco.py

@@ -49,30 +49,29 @@ if len(args.run_type) > 0:
 # Should be directory above run
 # i.e.: TestBeamData/Run-004150/Faser-Physics-004150-00000.raw"
 else:
-    if len(filepath.parts) < 3:
-        print("Can't determine run type from path - guessing TI12Data!")
-        print("If this is not correct, specify on command line")
-        runtype = "TI12Data"
 
-    else:
-        runtype = filepath.parts[-3]
+    runtype = "TI12Data03"
 
-    # Fix TI12 geometry versions as well (needed in production)
-    # Probably better to do this from configuration in upstream production scripts,
+    # Try to pick correct geometry from run number
+    # This won't work for testbeam data, 
     # so lets call this a hack for now
-    if runtype == "TI12Data":
-
-        runname = filepath.parts[-2]
-        try:
-            runnumber = int(runname.split('-')[1])
-        except Exception as e:
-            print(f"Failed to find run number in {filepath}")
-            print(f"Couldn't parse {runname}")
-            print(f"Leave runtype as {runtype}!")
+    runname = filepath.parts[-2]
+    try:
+        runnumber = int(runname.split('-')[1])
+    except Exception as e:
+        print(f"Failed to find run number in {filepath}")
+        print(f"Couldn't parse {runname}")
+        print(f"Leave runtype as {runtype}!")
+    else:
+        if runnumber > 6700: # Not sure if this is exact
+            print(f"Found run number {runnumber}, using TI12 configuration with IFT+faserNu")
+            runtype = "TI12Data03"
+        elif runnumber > 5302: # Last TI12 run on Nov. 23, 2021 without IFT
+            print(f"Found run number {runnumber}, using TI12 configuration with IFT")
+            runtype = "TI12Data02"
         else:
-            if runnumber > 5302: # Last TI12 run on Nov. 23, 2021 without IFT
-                print(f"Found run number {runnumber}, using TI12 configuration with IFT")
-                runtype = "TI12Data02"
+            print(f"Found run number {runnumber}, using original TI12 configuration")
+            runtype = "TI12Data"
 
 print(f"Starting reconstruction of {filepath.name} with type {runtype}")
 if args.nevents > 0:

Control/CalypsoExample/Reconstruction/scripts/submit_faserMDC_reco.sh

@@ -102,7 +102,8 @@ output_directory="$working_directory/${run_number}"
 mkdir -p "$output_directory"
 #
 # This magic redirects everything in this script to our log file
-exec >& "$output_directory/$file_stem.log"
+logfile="${file_stem}.rec.log"
+exec >& "$output_directory/${logfile}"
 echo `date` - $HOSTNAME
 echo "File: $file_name"
 echo "Release: $release_directory"
@@ -182,14 +183,14 @@ if ! [ -z "$logdest" ]
 then
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $logdest"
+    echo "copy $logfile to $logdest"
     mkdir -p $logdest
-    eos cp "${config_file_stem}-${seg_str}.log" $logdest
+    eos cp $logfile $logdest
 elif ! [ -z "$outdest" ]
 then 
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $outdest"
+    echo "copy $logfile to $outdest"
     mkdir -p $outdest
-    eos cp "${config_file_stem}-${seg_str}.log" $outdest
+    eos cp $logfile\ $outdest
 fi

Control/CalypsoExample/Reconstruction/scripts/submit_faser_reco.sh

@@ -4,6 +4,10 @@
 # Usage:
 # submit_faser_reco.sh file_path [release_directory] [working_directory] [nevents]
 # 
+# Options:
+#   --out - specify output location (in EOS) to copy output HITS file
+#   --log - specify output location (in EOS) for log file
+# 
 # file_path - full file name (with path)
 # release_directory - optional path to release install directory (default pwd)
 # working_directory - optional path to output directory location (default pwd)
@@ -16,8 +20,36 @@
 # If this matches reco/r???? it will be passed to the reco job
 #
 #----------------------------------------
+# Keep track of time
+SECONDS=0
 #
 # Parse command-line options
+while [ -n "$1" ]
+do 
+  case "$1" in
+      -l | --log)
+	  logdest="$2";
+	  shift;
+	  shift;; # Must eat 2 options here
+
+      -o | --out)
+	  outdest="$2";
+	  shift;
+	  shift;;
+
+      --) # End of options
+	  shift; # Eat this
+	  break;; # And stop parsing
+
+    -*) 
+	  echo "Unknown option $1"
+	  shift;;
+
+    *) break;;  # Not an option, don't shift
+  esac
+done
+#
+# Parse command-line arguments
 file_path=${1}
 release_directory=${2}
 working_directory=${3}
@@ -66,11 +98,12 @@ read faser type run_number segment <<< "$file_stem"
 IFS=$defaultIFS
 #
 # Make output directory if needed
-output_directory="$working_directory/Run-$run_number"
+output_directory="$working_directory/${run_number}"
 mkdir -p "$output_directory"
 #
 # This magic redirects everything in this script to our log file
-exec >& "$output_directory/$file_stem.log"
+logfile="${file_stem}.rec.log"
+exec >& "$output_directory/$logfile"
 echo `date` - $HOSTNAME
 echo "File: $file_name"
 echo "Release: $release_directory"
@@ -85,12 +118,12 @@ source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh
 # Always go back to the starting directory in case paths are relative
 cd "$starting_directory"
 cd "$release_directory"
-asetup
-source build/x8*/setup.sh
+#asetup 
+#source build/x8*/setup.sh
 #
 # Do this by hand
-# asetup --input="$release_directory/calypso/asetup.faser" Athena,22.0.49
-# source "$release_directory/build/x8*/setup.sh"
+asetup --input=calypso/asetup.faser Athena,22.0.49
+source build/x86*/setup.sh
 #
 #
 # Try to find a release tag
@@ -102,7 +135,7 @@ if [[ "$recotag" == "reco/r"???? ]]; then
 fi
 if [[ "$recotag" == "reco/p"???? ]]; then
   tag=`echo "$recotag" | cut -c 6-11`
-  echo "Found reco tag: $tag"
+  echo "Found proc tag: $tag"
 fi
 #
 # Move to the run directory
@@ -123,12 +156,42 @@ else
 fi
 cd "$file_stem"
 #
-# Run job
-if [[ -z "$rtag" ]]; then
-    faser_reco.py "--nevents=$nevents" "$file_path"
+# Run job, || True ensures script continues even if job fails
+if [[ -z "$tag" ]]; then
+    faser_reco.py "--nevents=$nevents" "$file_path" || True
 else
-    faser_reco.py "--nevents=$nevents" "--reco=$tag" "$file_path"
+    faser_reco.py "--nevents=$nevents" "--reco=$tag" "$file_path" || True
 fi
 #
 # Print out ending time
 date
+echo "Job finished after $SECONDS seconds"
+# 
+# Copy output to EOS if desired
+export EOS_MGM_URL=root://eospublic.cern.ch
+#
+# Now copy output file
+if ! [ -z "$outdest" ]
+then
+    ls -l
+    echo "copy *-RDO.root to $outdest"
+    mkdir -p $outdest
+    eos cp *-xAOD.root $outdest || True
+fi
+#
+# Copy log file second
+if ! [ -z "$logdest" ]
+then
+    cd ..
+    ls -l
+    echo "copy $logfile to $logdest"
+    mkdir -p $logdest
+    eos cp $logfile $logdest
+elif ! [ -z "$outdest" ]
+then 
+    cd ..
+    ls -l
+    echo "copy $logfile to $outdest"
+    mkdir -p $outdest
+    eos cp $logfile\ $outdest
+fi

Control/CalypsoExample/Simulation/scripts/submit_faserMDC_simulate.sh

@@ -133,7 +133,8 @@ fi
 mkdir -p "$output_directory"
 #
 # This magic redirects everything in this script to our log file
-exec >& "$output_directory/${file_stem}.log"
+logfile=${file_stem}.sim.log
+exec >& "$output_directory/$logfile"
 echo `date` - $HOSTNAME
 echo "Input File: $infile"
 echo "Output File: $outfilename"
@@ -206,15 +207,15 @@ if ! [ -z "$logdest" ]
 then
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $logdest"
+    echo "copy $logfile to $logdest"
     mkdir -p $logdest
-    eos cp "${config_file_stem}-${seg_str}.log" $logdest
+    eos cp $logfile $logdest
 elif ! [ -z "$outdest" ]
 then 
     cd ..
     ls -l
-    echo "copy ${config_file_stem}-${seg_str}.log to $outdest"
+    echo "copy $logfile to $outdest"
     mkdir -p $outdest
-    eos cp "${config_file_stem}-${seg_str}.log" $outdest
+    eos cp $logfile $outdest
 fi