Merge branch 'cherry-pick-ae0407e8ca-master' into 'master'

Sweeping !16575 from 21.3 to master.
Replace the datasets for the FTK tests

See merge request atlas/athena!16684

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_FTK_beamspot_ttbar.xml

@@ -7,7 +7,7 @@
             <group>TrigInDetValidation</group>
             <queue>long</queue>
 
-#          include "samples/TrigInDetValidation_FTK_ttbar_sample3.xml"
+#          include "samples/TrigInDetValidation_FTK_ttbar_sample4.xml"
 	   
 
 

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_FTK_bjet_ttbar.xml

@@ -7,7 +7,7 @@
             <group>TrigInDetValidation</group>
             <queue>long</queue>
 	   
-#          include "samples/TrigInDetValidation_FTK_ttbar_sample3.xml"
+#          include "samples/TrigInDetValidation_FTK_ttbar_sample4.xml"
 
 
 

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_FTK_bjet_ttbar_offline.xml

@@ -7,7 +7,7 @@
             <group>TrigInDetValidation</group>
             <queue>long</queue>
 	   
-#          include "samples/TrigInDetValidation_FTK_ttbar_sample3.xml"
+#          include "samples/TrigInDetValidation_FTK_ttbar_sample4.xml"
 
 
 

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_FTK_mu_ttbar.xml

@@ -7,7 +7,7 @@
             <group>TrigInDetValidation</group>
             <queue>long</queue>
 
-#          include "samples/TrigInDetValidation_FTK_ttbar_sample3.xml"
+#          include "samples/TrigInDetValidation_FTK_ttbar_sample4.xml"
 
 
 

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_FTK_mu_ttbar_offline.xml

@@ -7,7 +7,7 @@
             <group>TrigInDetValidation</group>
             <queue>long</queue>
 
-#          include "samples/TrigInDetValidation_FTK_ttbar_sample3.xml"
+#          include "samples/TrigInDetValidation_FTK_ttbar_sample4.xml"
 
 
 

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_FTK_tau_tautau.xml

@@ -7,7 +7,7 @@
             <group>TrigInDetValidation</group>
             <queue>long</queue>
 
-#          include "samples/TrigInDetValidation_FTK_Ztautau_sample3.xml"
+#          include "samples/TrigInDetValidation_FTK_Ztautau_sample4.xml"
 
 
 

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/makefile

@@ -84,7 +84,8 @@ SCRIPTDIR = ../scripts
 
 install : all 
 
-all: $(XMLFILE) artfiles checkdata
+# all: $(XMLFILE) artfiles checkdata
+all: artfiles checkdata
 
 $(FILES) : $(JOBS)
 

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/samples/TrigInDetValidation_FTK_Ztautau_sample4.xml

+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.digit.RDO_FTK.e3601_s3126_d1480_d1471/RDO_FTK.15260491._000002.pool.root.1</dataset>
+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.digit.RDO_FTK.e3601_s3126_d1480_d1471/RDO_FTK.15260491._000003.pool.root.1</dataset>
+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.digit.RDO_FTK.e3601_s3126_d1480_d1471/RDO_FTK.15260491._000004.pool.root.1</dataset>
+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.digit.RDO_FTK.e3601_s3126_d1480_d1471/RDO_FTK.15260491._000021.pool.root.1</dataset>
+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.digit.RDO_FTK.e3601_s3126_d1480_d1471/RDO_FTK.15260491._000022.pool.root.1</dataset>

Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/samples/TrigInDetValidation_FTK_ttbar_sample4.xml

+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471/RDO_FTK.15265974._000001.pool.root.1</dataset>
+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471/RDO_FTK.15265974._000003.pool.root.1</dataset>
+ <dataset>/eos/atlas/atlascerngroupdisk/proj-sit/trigindet/mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471/RDO_FTK.15265974._000004.pool.root.1</dataset>

Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_ftk_beamspot_ttbar.sh

@@ -9,7 +9,7 @@
 # art-output: cost-perEvent
 # art-output: cost-perCall-chain
 # art-output: cost-perEvent-chain
-# art-input:  mc16_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.digit.RDO_FTK.e5602_s3126_d1455_d1457
+# art-input:  mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471
 # art-output: *.dat 
 # art-output: *.root
 # art-output: *.log

Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_ftk_bjet_ttbar.sh

@@ -9,7 +9,7 @@
 # art-output: cost-perEvent
 # art-output: cost-perCall-chain
 # art-output: cost-perEvent-chain
-# art-input:  mc16_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.digit.RDO_FTK.e5602_s3126_d1455_d1457
+# art-input:  mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471
 # art-output: *.dat 
 # art-output: *.root
 # art-output: *.log
@@ -324,7 +324,6 @@ get_files -data TIDAdata_cuts.dat
 get_files -data TIDAdata_chains.dat
 get_files -data TIDAbeam.dat
 get_files -data Test_bin.dat
-get_files -data data-FTK_bjet_ttbar-reference.root
 
 for DATFILE in *.dat ; do
     if ( grep -q DataFile $DATFILE ); then
@@ -337,7 +336,7 @@ done
 if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then
 
 
-TIDArdict TIDAdata11-rtt.dat -f data-bjet-FTK.root -b Test_bin.dat data-bjet-FTK.root data-FTK_bjet_ttbar-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_FTKVtx_InDetTrigTrackingxAODCnv_Bjet_FTF HLT_j55_boffperf_split_FTK_InDetTrigTrackingxAODCnv_Bjet_FTK_SplitJet HLT_j55_boffperf_split_FTKRefit_InDetTrigTrackingxAODCnv_Bjet_FTKRefit_SplitJet -d HLTL2-plots  2>&1 | tee TIDArdict_1.log
+TIDArdict TIDAdata11-rtt.dat -f data-bjet-FTK.root -b Test_bin.dat  2>&1 | tee TIDArdict_1.log
 echo "art-result: $? TIDArdict_1"
 
 
@@ -349,7 +348,7 @@ timestamp "TIDArdict"
 fi
 
 
-TIDArun-art.sh data-bjet-FTK.root data-FTK_bjet_ttbar-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_FTKVtx_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_FTK_InDetTrigTrackingxAODCnv_Bjet_FTK_IDTrig_SplitJet HLT_j55_boffperf_split_FTKRefit_InDetTrigTrackingxAODCnv_Bjet_FTKRefit_IDTrig_SplitJet -d HLTEF-plots  2>&1 | tee TIDArun_2.log
+TIDArun-art.sh data-bjet-FTK.root data-FTK_bjet_ttbar-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_FTKVtx_InDetTrigTrackingxAODCnv_Bjet_FTF HLT_j55_boffperf_split_FTK_InDetTrigTrackingxAODCnv_Bjet_FTK_SplitJet HLT_j55_boffperf_split_FTKRefit_InDetTrigTrackingxAODCnv_Bjet_FTKRefit_SplitJet -d HLTL2-plots  2>&1 | tee TIDArun_2.log
 echo "art-result: $? TIDArun_2"
 
 
@@ -358,7 +357,7 @@ timestamp "TIDArun-art.sh"
 
 
 
-TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times  2>&1 | tee TIDArun_3.log
+TIDArun-art.sh data-bjet-FTK.root data-FTK_bjet_ttbar-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_FTKVtx_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_FTK_InDetTrigTrackingxAODCnv_Bjet_FTK_IDTrig_SplitJet HLT_j55_boffperf_split_FTKRefit_InDetTrigTrackingxAODCnv_Bjet_FTKRefit_IDTrig_SplitJet -d HLTEF-plots  2>&1 | tee TIDArun_3.log
 echo "art-result: $? TIDArun_3"
 
 
@@ -367,7 +366,7 @@ timestamp "TIDArun-art.sh"
 
 
 
-TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF  2>&1 | tee TIDArun_4.log
+TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times  2>&1 | tee TIDArun_4.log
 echo "art-result: $? TIDArun_4"
 
 
@@ -376,25 +375,25 @@ timestamp "TIDArun-art.sh"
 
 
 
-RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir  2>&1 | tee RunTrigCostD3PD_5.log
-echo "art-result: $? RunTrigCostD3PD_5"
+TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF  2>&1 | tee TIDArun_5.log
+echo "art-result: $? TIDArun_5"
 
 
 
-timestamp "RunTrigCostD3PD"
+timestamp "TIDArun-art.sh"
 
 
 
-TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall"  2>&1 | tee TIDAcpucost_6.log
-echo "art-result: $? TIDAcpucost_6"
+RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir  2>&1 | tee RunTrigCostD3PD_6.log
+echo "art-result: $? RunTrigCostD3PD_6"
 
 
 
-timestamp "TIDAcpucost"
+timestamp "RunTrigCostD3PD"
 
 
 
-TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent"  2>&1 | tee TIDAcpucost_7.log
+TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall"  2>&1 | tee TIDAcpucost_7.log
 echo "art-result: $? TIDAcpucost_7"
 
 
@@ -403,7 +402,7 @@ timestamp "TIDAcpucost"
 
 
 
-TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall"  2>&1 | tee TIDAcpucost_8.log
+TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent"  2>&1 | tee TIDAcpucost_8.log
 echo "art-result: $? TIDAcpucost_8"
 
 
@@ -412,11 +411,20 @@ timestamp "TIDAcpucost"
 
 
 
-TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent"  2>&1 | tee TIDAcpucost_9.log
+TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall"  2>&1 | tee TIDAcpucost_9.log
 echo "art-result: $? TIDAcpucost_9"
 
 
 
+timestamp "TIDAcpucost"
+
+
+
+TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent"  2>&1 | tee TIDAcpucost_10.log
+echo "art-result: $? TIDAcpucost_10"
+
+
+
 
 timestamp "TIDAcpucost"
 

Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_ftk_bjet_ttbar_offline.sh

@@ -9,7 +9,7 @@
 # art-output: cost-perEvent
 # art-output: cost-perCall-chain
 # art-output: cost-perEvent-chain
-# art-input:  mc16_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.digit.RDO_FTK.e5602_s3126_d1455_d1457
+# art-input:  mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471
 # art-output: *.dat 
 # art-output: *.root
 # art-output: *.log

Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_ftk_mu_ttbar.sh

@@ -11,7 +11,7 @@
 # art-output: cost-perEvent
 # art-output: cost-perCall-chain
 # art-output: cost-perEvent-chain
-# art-input:  mc16_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.digit.RDO_FTK.e5602_s3126_d1455_d1457
+# art-input:  mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471
 # art-output: *.dat 
 # art-output: *.root
 # art-output: *.log

Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_ftk_mu_ttbar_offline.sh

@@ -11,7 +11,7 @@
 # art-output: cost-perEvent
 # art-output: cost-perCall-chain
 # art-output: cost-perEvent-chain
-# art-input:  mc16_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.digit.RDO_FTK.e5602_s3126_d1455_d1457
+# art-input:  mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.digit.RDO_FTK.e6337_e5984_s3126_d1480_d1471
 # art-output: *.dat 
 # art-output: *.root
 # art-output: *.log

Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_ftk_tau_tautau.sh

@@ -9,7 +9,7 @@
 # art-output: cost-perEvent
 # art-output: cost-perCall-chain
 # art-output: cost-perEvent-chain
-# art-input:  mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.digit.RDO_FTK.e3601_s3126_d1455_d1457
+# art-input:  mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.digit.RDO_FTK.e3601_s3126_d1480_d1471
 # art-output: *.dat 
 # art-output: *.root
 # art-output: *.log
@@ -142,8 +142,6 @@ function waitonallproc   {
 
 # run athena  
 
-iathena=0
-
 function runathena { 
    timestamp  "runathena:"
 
@@ -163,9 +161,7 @@ function runathena {
      echo "ARGS: $ARGS"
      echo -e "\nrunning athena in athena-$1\n"
      athena.py  -c "$ARGS"               TrigInDetValidation/TrigInDetValidation_RTT_topOptions_TauSlice.py  &> athena-local-$1.log
-     echo "art-result: $? athena_$iathena"
-
-     ((iathena++))
+     echo "art-result: $? athena_$1"
 
      pwd
      ls -lt
@@ -262,10 +258,13 @@ done
 
 [ -e topp.log ] && rm topp.log
 
-ps -aF --pid $PPROCS | grep $USER >> topp.log
+echo -e "\nUID        PID  PPID  C    SZ   RSS PSR STIME TTY          TIME CMD" >> topp.log
+ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log
 
 echo >> topp.log
 
+sleep 20 
+
 top -b -n1 > top.log
 grep PID top.log >> topp.log
 for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done
@@ -310,7 +309,9 @@ hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log
 # file to the check will fail. This creates a link so this 
 # test will pass
   
-for git in output-dataset/*.root ; do ln -s $git TrkNtuple-0000.root ; break ; done  
+for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done  
+
+[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed"
 
 fi
 

Trigger/TrigValidation/TrigInDetValidation/utils/submit.sh

 #!/bin/bash
 
+echo "version 2"
+
 echo -e "\nChecking for new data sets to be added to the art container ... \n"
 
 DUFF=$(mktemp -u /tmp/.submitDS-XXXX.sh)
 
 !( which rucio >& /dev/null ) && echo "rucio not found" && exit -1
 
+touch $DUFF
+
 rucio list-content user.artprod.input.container | grep -v SCOPE:NAME |  grep -v + | awk '{print $2}' > $DUFF
 
 rm -rf newDS.txt .newDS.txt
 
+DATASETS=$(mktemp -u /tmp/.DS-XXXX.txt)
+touch $DATASETS
+
 for FILE in ../test/test*.sh ; do 
 
   SKIP=1
 
   DS=$(grep "art-input:" $FILE | sed 's|.*art-input:[[:space:]]*||g') 
 
+  NF=$(grep "^# *art-input-nfiles:" $FILE | sed 's|.*art-input-nfiles: *||g') 
+
+#  printf "\n----------------------------\n$FILE  \tDS: $DS  \tNF: $NF\n"
+
   # echo "DS: $DS" 
   if ! ( grep -q $DS $DUFF ); then 
-        echo "dataset not included: $DS"
-  
-        rucio list-files $DS &> .$DS
+
+        if [ -e newDS.txt ]; then 
+	    ( grep -q " $DS " newDS.txt ) || printf "\n new dataset: $DS\n"
+	fi
+
+        [ ! -e .$DS ] && rucio list-files $DS &> .$DS
    
 	( grep -q "Total files" .$DS ) && SKIP=0
 
+	grep -q "Total files" .$DS
 
 	if [ $SKIP -eq 0 ]; then 
 
             NFILES=$(grep "Total files" .$DS | sed 's|.*:[[:space:]]||g')
 
 	    if [ $NFILES -gt 0 ]; then 	
-#		echo "NFILES: $NFILES"
+
 		[ -e newDS.txt ] || echo -e "\nPlease add the following datasets to user.artprod.input.container\n" > newDS.txt
-		echo " * $DS" >>  newDS.txt  
-		echo "$DS"    >> .newDS.txt  
+
+		if ( grep -q " $DS " newDS.txt ); then 
+		    
+		    NOF=$(cat newDS.txt | grep " $DS " | sed 's|.*(||g' | sed 's|files.*||g') 
+		    
+		    if [ $NOF -lt $NF ]; then 
+
+			for NDS in {,.}newDS.txt ; do mv $NDS{,-} ; grep -v " $DS " ${NDS}- > $NDS ; rm ${NDS}- ; done
+			
+			echo " replacing:  $DS   ( $NF files )"
+		       	printf " * $DS   \t( $NF files )\n" >>  newDS.txt
+			printf   " $DS   \t( $NF files )\n" >> .newDS.txt 
+		    fi
+		else
+		    echo $DS >> $DATASETS
+		    echo " adding:     $DS   ( $NF files )"
+      		    printf " * $DS   \t( $NF files )\n" >>  newDS.txt
+		    printf   " $DS   \t( $NF files )\n" >> .newDS.txt 
+		fi
             fi
+
 	else 
 	    if [ $SKIP -eq 1 ]; then 
                 echo "dataset for $FILE does not exists on the grid - disabling test"
@@ -42,20 +75,23 @@ for FILE in ../test/test*.sh ; do
 	    fi
 	fi
 
-        rm .$DS
+#        rm .$DS
   fi
 
 
 done
 
+
 if [ -e newDS.txt ]; then 
-   echo "new datasets to be added:"
+    printf "\nnew datasets to be added:\n\n"
    cat .newDS.txt
    /usr/bin/cern-get-sso-cookie --url https://its.cern.ch/jira/ -o cookiefile.txt  284  11:30 /usr/bin/cern-get-sso-cookie --url https://its.cern.ch/jira/ -o cookiefile.txt
    ../utils/submit.py -f newDS.txt
    rm newDS.txt .newDS.txt
+   rm $DUFF
+   rm $DATASETS
 else
-   echo "no datasets to be added"
+   printf "\nno datasets to be added\n"
 fi