Merge branch 'fasermc-dev' into 'master'

Update simulation scripts

See merge request faser/calypso!322

Control/CalypsoExample/Digitization/scripts/faser_digi_merge.py

@@ -29,6 +29,8 @@ parser.add_argument("-s", "--slice", type=int, default=0,
                     help="Specify file slice to produce")
 parser.add_argument("-f", "--files", type=int, default=5,
                     help="Specify number of input files to run in one batch")
+parser.add_argument("--complete", action="store_true",
+                    help="This is the complete run, remove segment numbers")
 parser.add_argument("-t", "--tag", default="",
                     help="Specify digi tag (to append to output filename)")
 parser.add_argument("--digiTag", default="",
@@ -67,24 +69,35 @@ if len(dirlist) == 0:
 print("HITS files available:")
 [print(file) for file in dirlist]
 
+# Get run number for pattern matching
+stem = dirlist[0].stem
+spl = stem.split('-')
+short = spl[1]
+run = spl[2]
+
 for seg in seglist:
-    # Assume these are in numerical order from 0
-    if seg >= len(dirlist):
-        print(f"Requested file segment {seg} but only {len(dirlist)} files found")
-        if args.partial:
-            break
-        else:
-            sys.exit(1)  # Abort this job
+    # No longer assume all HITS files are complete
+    #if seg >= len(dirlist):
+    #    print(f"Requested file segment {seg} but only {len(dirlist)} files found")
+    #    if args.partial:
+    #        break
+    #    else:
+    #        sys.exit(1)  # Abort this job
 
     # Check if segment number exists in hits file (this is not perfect)
-    segstr = f"{seg:05d}"
-    if segstr not in dirlist[seg].name:
-        print(f"Segment {segstr} not in file {dirlist[seg].name}!")
+    segstr = f"FaserMC-{short}-{run}-{seg:05d}*HITS.root"
+    matchlist = list(dirpath.glob(segstr))
+    if len(matchlist) == 0:
+        print(f"Segment {segstr} not found!")
         if not args.partial: sys.exit(1) # abort
+        continue
+    elif len(matchlist) > 1:
+        print(f"Segment {segstr} matched {len(matchlist)} files!")
+        sys.exit(1)  # Should never happen!
     else:
-        print(f"Segment {segstr} found in file {dirlist[seg]}")
+        print(f"Segment {segstr} found!")
 
-    filelist.append(dirlist[seg])  
+    filelist.append(matchlist[0])  
 
 if len(filelist) == 0:
     # Asked for range that doesn't exist
@@ -114,8 +127,11 @@ seghi = int(spl[3])
 if len(args.short) > 0:
     short += args.short
 
+# Shouldn't normally remove segments with MC as we may always add
+# more files later.  --complete overrides this if wanted
+
 # Build output filename
-if seglo == 0 and (seghi+1) == len(dirlist):  # Full run
+if seglo == 0 and (seghi+1) == len(dirlist) and args.complete:  # Full run
     outfile = f"FaserMC-{short}-{run}"
 elif seglo == seghi:  # Single segment
     outfile = f"FaserMC-{short}-{run}-{seglo:05}"

Control/CalypsoExample/Digitization/scripts/submit_faser_digi.sh

@@ -179,6 +179,8 @@ cd "$file_stem"
 # Run job
 #
 faser_digi.py $geomstr $gainstr $tagstr "$file_path"
+digi_code=$?
+echo "Return code: $digi_code"
 #
 # Print out ending time
 date
@@ -189,25 +191,52 @@ export EOS_MGM_URL=root://eospublic.cern.ch
 #
 if ! [ -z "$outdest" ]
 then
+    echo "Output directory:"
     ls -l
-    echo "copy *-RDO.root to $outdest"
-    mkdir -p $outdest
-    eos cp *-RDO.root ${outdest}/ || true
+    thefile=`ls *-RDO.root`
+    if [ $? -eq 0 ]; then
+        echo "copy $thefile to $outdest"
+	eos mkdir -p $outdest
+	eos cp $thefile ${outdest}/${thefile} || true
+
+	# Check that this worked
+	eos ls ${outdest}/${thefile} > /dev/null
+        if [ $? -eq 0 ]; then
+            echo "file $thefile copied to $outdest"
+	    copy_code=0
+	else
+	    echo "didnt find $thefile in $outdest !"
+	    copy_code=1
+	fi
+    else
+        echo "ls *-RDO.root returned nothing!"
+        copy_code=1
+    fi
 fi
 #
 # Also copy log file
 if ! [ -z "$logdest" ]
 then
     cd ..
+    echo "Working directory:"
     ls -l
     echo "copy $logfile to $logdest"
-    mkdir -p $logdest
+    eos mkdir -p $logdest
     eos cp $logfile $logdest/$logfile
 elif ! [ -z "$outdest" ]
 then 
     cd ..
+    echo "Working directory:"
     ls -l
     echo "copy $logfile to $outdest"
-    mkdir -p $outdest
+    eos mkdir -p $outdest
     eos cp $logfile $outdest/$logfile
 fi
+
+# Make sure to return an error is calypso failed                                
+if [ $digi_code -ne 0 ] || [ $copy_code -ne 0 ]; then
+  exit 1
+else
+  exit 0
+fi
+

Control/CalypsoExample/Digitization/scripts/submit_faser_digi_merge.sh

@@ -213,6 +213,8 @@ cd "$file_stem"
 # Run job
 #
 faser_digi_merge.py $partialstr $geomstr $gainstr $tagstr --slice $slice --files $nfiles $dir_path
+digi_code=$?
+echo "Return code: $digi_code"
 #
 # Print out ending time
 date
@@ -223,25 +225,52 @@ export EOS_MGM_URL=root://eospublic.cern.ch
 #
 if ! [ -z "$outdest" ]
 then
+    echo "Output directory:"
     ls -l
-    echo "copy *-RDO.root to $outdest"
-    mkdir -p $outdest
-    eos cp *-RDO.root ${outdest}/ || true
+    thefile=`ls *-RDO.root`
+    if [ $? -eq 0 ]; then
+        echo "copy $thefile to $outdest"
+	eos mkdir -p $outdest
+	eos cp $thefile ${outdest}/${thefile} || true
+
+	# Check that this worked
+	eos ls ${outdest}/${thefile} > /dev/null
+        if [ $? -eq 0 ]; then
+            echo "file $thefile copied to $outdest"
+	    copy_code=0
+	else
+	    echo "didnt find $thefile in $outdest !"
+	    copy_code=1
+	fi
+    else
+        echo "ls *-RDO.root returned nothing!"
+        copy_code=1
+    fi
 fi
 #
 # Also copy log file
 if ! [ -z "$logdest" ]
 then
     cd ..
+    echo "Working directory:"
     ls -l
     echo "copy $logfile to $logdest"
-    mkdir -p $logdest
+    eos mkdir -p $logdest
     eos cp $logfile $logdest/$logfile
 elif ! [ -z "$outdest" ]
 then 
     cd ..
+    echo "Working directory:"
     ls -l
     echo "copy $logfile to $outdest"
-    mkdir -p $outdest
+    eos mkdir -p $outdest
     eos cp $logfile $outdest/$logfile
 fi
+
+# Make sure to return an error is calypso failed                                
+if [ $digi_code -ne 0 ] || [ $copy_code -ne 0 ]; then
+  exit 1
+else
+  exit 0
+fi
+

Control/CalypsoExample/Simulation/scripts/submit_faser_simulate.sh

@@ -183,9 +183,12 @@ cd "${file_stem}"
 #fi
 if [[ -z "$xangle" ]]; then
     faser_simulate.py  --skip "$skip_events" -n "$nevts" "$infile" "$outfile"
+    sim_code=$?
 else
     faser_simulate.py  --yangle -0.000150 --yshift 12.0 --skip "$skip_events" -n "$nevts" "$infile" "$outfile"
+    sim_code=$?
 fi
+echo "Return code: $sim_code"
 #
 # Print out ending time
 date
@@ -196,16 +199,34 @@ export EOS_MGM_URL=root://eospublic.cern.ch
 #
 if ! [ -z "$outdest" ]
 then
+    echo "Output directory:"
     ls -l
-    echo "copy *-HITS.root to $outdest"
-    eos mkdir -p $outdest
-    eos cp *-HITS.root ${outdest}/ || true
+    thefile=`ls *-HITS.root`
+    if [ $? -eq 0 ]; then
+	echo "copy $thefile to $outdest"
+	eos mkdir -p $outdest
+	eos cp $thefile ${outdest}/${thefile} || true
+
+	# Check that this worked
+	eos ls ${outdest}/${thefile} > /dev/null
+        if [ $? -eq 0 ]; then
+            echo "file $thefile copied to $outdest"
+	    copy_code=0
+	else
+	    echo "didnt find $thefile in $outdest !"
+	    copy_code=1
+	fi
+    else
+        echo "ls *-HITS.root returned nothing!"
+        copy_code=1
+    fi
 fi
 #
 # Also copy log file
 if ! [ -z "$logdest" ]
 then
     cd ..
+    echo "Working directory:"
     ls -l
     echo "copy $logfile to $logdest"
     eos mkdir -p $logdest
@@ -213,9 +234,17 @@ then
 elif ! [ -z "$outdest" ]
 then 
     cd ..
+    echo "Working directory:"
     ls -l
     echo "copy $logfile to $outdest"
     eos mkdir -p $outdest
     eos cp $logfile $outdest/$logfile
 fi
 
+# Make sure to return an error is calypso failed                                
+if [ $sim_code -ne 0 ] || [ $copy_code -ne 0 ]; then
+  exit 1
+else
+  exit 0
+fi
+