From 2d08e7a296046a5935007e09185f968872dcdcdf Mon Sep 17 00:00:00 2001 From: Tim Adye Date: Wed, 2 Dec 2020 18:46:22 +0000 Subject: [PATCH 1/2] ML tests now generated from template --- ...InDetValidation_TestConfiguration_main.xml | 6 +- ...idation_beamspot_ttbar_pu80_zfinder_ml.xml | 62 ++++++++++ ...dation_beamspot_ttbar_pu80_zfinder_ml2.xml | 62 ++++++++++ .../TrigInDetValidation_bjet_IBL_pu40_ml.xml | 107 ++++++++++++++++++ .../TrigInDetValidation_tau_IBL_pu46_ml.xml | 79 +++++++++++++ .../TrigInDetValidation/TIDAbuild/makefile | 6 +- ...rigindetvalidation_bjet_ibl_pu40_ml_art.sh | 2 +- .../TrigInDetValidation/utils/art-events.dat | 4 + .../TrigInDetValidation/utils/art-input.dat | 4 + .../TrigInDetValidation/utils/art-nfiles.dat | 4 + 10 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml.xml create mode 100644 Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml2.xml create mode 100644 Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_bjet_IBL_pu40_ml.xml create mode 100644 Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_tau_IBL_pu46_ml.xml diff --git a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_TestConfiguration_main.xml b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_TestConfiguration_main.xml index 9e7ef534cca..ef2bf5a1e9d 100644 --- a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_TestConfiguration_main.xml +++ b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_TestConfiguration_main.xml @@ -37,6 +37,8 @@ # include "TrigInDetValidation_beamspot_ttbar_pu40.xml" # include "TrigInDetValidation_beamspot_ttbar_pu40_zfinder.xml" # include "TrigInDetValidation_beamspot_ttbar_pu80_zfinder.xml" +# include "TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml.xml" +# include "TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml2.xml" # include "TrigInDetValidation_beamspot_ttbar_pu80_fs.xml" # include "TrigInDetValidation_beamspot_ttbar_pu80_fsml.xml" @@ -62,11 +64,12 @@ # include "TrigInDetValidation_el_ttbar_pu80.xml" -# include "TrigInDetValidation_cosmic.xml" + # include "TrigInDetValidation_bjet_IBL_pu40.xml" +# include "TrigInDetValidation_bjet_IBL_pu40_ml.xml" # include "TrigInDetValidation_bjet_IBL_pu40_offline.xml" @@ -85,6 +88,7 @@ # include "TrigInDetValidation_tau_IBL_pu46.xml" +# include "TrigInDetValidation_tau_IBL_pu46_ml.xml" # include "TrigInDetValidation_tau_IBL_pu46_NoDoublet.xml" # include "TrigInDetValidation_tau_IBL_pu46_RoiParam.xml" # include "TrigInDetValidation_tau_IBL_pu46_offline.xml" diff --git a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml.xml b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml.xml new file mode 100644 index 00000000000..d36a7279f66 --- /dev/null +++ b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml.xml @@ -0,0 +1,62 @@ + + BeamSpot slice, ttbar events, pu80 + BeamSpot slice, ttbar, pu80 + EventMax=1000;fastZFinder=True;mlExtensions=True;rec.doFloatingPointException.set_Value_and_Lock(False) + TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BeamspotSlice + Trigger/TrigValidation/TrigInDetValidation + TrigInDetValidation + long + +# include "samples/TrigInDetValidation_ttbar_sample3.xml" + + + RttLibraryTools + ExeRunner + + exeName + TIDArdict.exe + + + exeParamString + TIDAdata11-rtt.dat -f data-all.root -b Test_bin.dat + + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-all.root data-beamspot_ttbar_pu80_zfinder_ml-reference.root HLT_beamspot_allTE_trkfast_InDetTrigTrackingxAODCnv_BeamSpot_FTF -d HLTL2-test-plots + + + + + + +# include "TrigInDetValidation_expertMonitoring.xml" +# include "TrigInDetValidation_costMonitoring.xml" +# include "TrigInDetValidation_extra.xml" + + + + + + + RTT:Top + CheckFileRunner0 + + + +# include "TrigInDetValidation_aux.xml" + + + + + + diff --git a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml2.xml b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml2.xml new file mode 100644 index 00000000000..37706daedb2 --- /dev/null +++ b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml2.xml @@ -0,0 +1,62 @@ + + BeamSpot slice, ttbar events, pu80 + BeamSpot slice, ttbar, pu80 + EventMax=1000;fastZFinder=True;mlExtensions=True;seedML=4;rec.doFloatingPointException.set_Value_and_Lock(False) + TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BeamspotSlice + Trigger/TrigValidation/TrigInDetValidation + TrigInDetValidation + long + +# include "samples/TrigInDetValidation_ttbar_sample3.xml" + + + RttLibraryTools + ExeRunner + + exeName + TIDArdict.exe + + + exeParamString + TIDAdata11-rtt.dat -f data-all.root -b Test_bin.dat + + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-all.root data-beamspot_ttbar_pu80_zfinder_ml2-reference.root HLT_beamspot_allTE_trkfast_InDetTrigTrackingxAODCnv_BeamSpot_FTF -d HLTL2-test-plots + + + + + + +# include "TrigInDetValidation_expertMonitoring.xml" +# include "TrigInDetValidation_costMonitoring.xml" +# include "TrigInDetValidation_extra.xml" + + + + + + + RTT:Top + CheckFileRunner0 + + + +# include "TrigInDetValidation_aux.xml" + + + + + + diff --git a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_bjet_IBL_pu40_ml.xml b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_bjet_IBL_pu40_ml.xml new file mode 100644 index 00000000000..f1023167419 --- /dev/null +++ b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_bjet_IBL_pu40_ml.xml @@ -0,0 +1,107 @@ + + Bjetslice, ttbar, pu40, IBL + BjetSlice, ttbar, pu40, IBL + EventMax=900;mlExtensions=True;doBperf=True;globalTag='OFLCOND-RUN12-SDR-17' + TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BjetSlice + Trigger/TrigValidation/TrigInDetValidation + TrigInDetValidation + long + +# include "samples/TrigInDetValidation_ttbar_sample2.xml" + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArdict.exe + + + exeParamString + TIDAdata11-rtt.dat -f data-bjet-pileup-merging.root -b Test_bin.dat + + + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-bjet-pileup-merging.root data-bjet_IBL_pu40_ml-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots + + + + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-bjet-pileup-merging.root data-bjet_IBL_pu40_ml-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-bjet-pileup-merging.root data-bjet_IBL_pu40_ml-reference.root HLT_j55_bperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_bperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID -d HLTEF-plots-comp + + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-bjet-pileup-merging.root data-bjet_IBL_pu40_ml-reference.root HLT_j55_bperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_bperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots-comp + + + + +# include "TrigInDetValidation_expertMonitoring.xml" +# include "TrigInDetValidation_costMonitoring.xml" +# include "TrigInDetValidation_extra.xml" + + + + + RTT:Top + CheckFileRunner0 + + + +# include "TrigInDetValidation_aux.xml" + + + + + + + + + diff --git a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_tau_IBL_pu46_ml.xml b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_tau_IBL_pu46_ml.xml new file mode 100644 index 00000000000..ff935cbae78 --- /dev/null +++ b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/TrigInDetValidation_tau_IBL_pu46_ml.xml @@ -0,0 +1,79 @@ + + TauSlice, H->tautau IBL sample, pu46 + TauSlice, H->tautau IBL sample, pu46 + EventMax=4000;mlExtensions=True;PdgId=15 + TrigInDetValidation/TrigInDetValidation_RTT_topOptions_TauSlice + Trigger/TrigValidation/TrigInDetValidation + TrigInDetValidation + long + +# include "samples/TrigInDetValidation_tau_sample3.xml" + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArdict.exe + + + exeParamString + TIDAdata11-rtt.dat -f data-tau-IBL.root -b Test_bin.dat + + + + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-tau-IBL.root data-tau_IBL_pu46_ml-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots + + + + + + + + RttLibraryTools + ExeRunner + + exeName + TIDArun.sh + + + exeParamString + data-tau-IBL.root data-tau_IBL_pu46_ml-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots + + + + + +# include "TrigInDetValidation_expertMonitoring.xml" +# include "TrigInDetValidation_costMonitoring.xml" +# include "TrigInDetValidation_extra.xml" + + + + + RTT:Top + CheckFileRunner0 + + + +# include "TrigInDetValidation_aux.xml" + + + + + + diff --git a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/makefile b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/makefile index 08ce168cc43..1640e7e0b56 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/makefile +++ b/Trigger/TrigValidation/TrigInDetValidation/TIDAbuild/makefile @@ -25,6 +25,8 @@ JOBS = \ TrigInDetValidation_beamspot_ttbar_pu40.xml \ TrigInDetValidation_beamspot_ttbar_pu40_zfinder.xml \ TrigInDetValidation_beamspot_ttbar_pu80_zfinder.xml \ + TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml.xml \ + TrigInDetValidation_beamspot_ttbar_pu80_zfinder_ml2.xml \ \ TrigInDetValidation_el_Jpsiee_pu40.xml \ TrigInDetValidation_el_single_e_7-80_larged0.xml \ @@ -38,15 +40,15 @@ JOBS = \ TrigInDetValidation_el_ttbar_pu40.xml \ TrigInDetValidation_el_ttbar_pu80.xml \ \ - TrigInDetValidation_cosmic.xml \ - \ TrigInDetValidation_bjet_IBL_pu40.xml \ + TrigInDetValidation_bjet_IBL_pu40_ml.xml \ TrigInDetValidation_bjet_IBL_pu40_offline.xml \ TrigInDetValidation_bjet_IBL_pu80.xml \ TrigInDetValidation_bjet_IBL_pu40_2016config.xml \ TrigInDetValidation_bjet_IBL_pu40_offline_2016config.xml \ \ TrigInDetValidation_tau_IBL_pu46.xml \ + TrigInDetValidation_tau_IBL_pu46_ml.xml \ TrigInDetValidation_tau_IBL_pu46_offline.xml \ \ TrigInDetValidation_mu_single_mu_larged0.xml \ diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml_art.sh index 4657fde41e6..870a2fbce0d 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml_art.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml_art.sh @@ -265,7 +265,7 @@ i=0 for git in $jobList ; do - ARGS="$git;EventMax=1000;doBperf=True;mlExtensions=True;globalTag='OFLCOND-RUN12-SDR-17'" + ARGS="$git;EventMax=1000;mlExtensions=True;doBperf=True;globalTag='OFLCOND-RUN12-SDR-17'" # echo "ARGS: $ARGS" diff --git a/Trigger/TrigValidation/TrigInDetValidation/utils/art-events.dat b/Trigger/TrigValidation/TrigInDetValidation/utils/art-events.dat index 8bef54ca6e4..caa2d70a15c 100644 --- a/Trigger/TrigValidation/TrigInDetValidation/utils/art-events.dat +++ b/Trigger/TrigValidation/TrigInDetValidation/utils/art-events.dat @@ -4,12 +4,15 @@ test_trigindetvalidation_all_ttbar_pu80 1000 test_trigindetvalidation_beamspot_ttbar_pu40 1000 test_trigindetvalidation_beamspot_ttbar_pu40_zfinder 1000 test_trigindetvalidation_beamspot_ttbar_pu80_zfinder 1000 +test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml 1000 +test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2 1000 test_trigindetvalidation_beamspot_ttbar_pu80_fs 1000 test_trigindetvalidation_beamspot_ttbar_pu80_fsml 1000 test_trigindetvalidation_bjet_ibl_pu40_2016config 1000 test_trigindetvalidation_bjet_ibl_pu40_offline_2016config 1000 test_trigindetvalidation_bjet_ibl_pu40_offline 1000 test_trigindetvalidation_bjet_ibl_pu40 1000 +test_trigindetvalidation_bjet_ibl_pu40_ml 1000 test_trigindetvalidation_bjet_ibl_pu80 1000 test_trigindetvalidation_el_jpsiee_pu40 500 test_trigindetvalidation_el_single_e_7-80_ibl_pu40 2000 @@ -39,5 +42,6 @@ test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher 2000 test_trigindetvalidation_mu_zmumu_ibl_pu40 2000 test_trigindetvalidation_tau_ibl_pu46_offline 2000 test_trigindetvalidation_tau_ibl_pu46 2000 +test_trigindetvalidation_tau_ibl_pu46_ml 2000 test_trigindetvalidation_tau_ibl_pu46_nodoublet 2000 test_trigindetvalidation_tau_ibl_pu46_roiparam 2000 diff --git a/Trigger/TrigValidation/TrigInDetValidation/utils/art-input.dat b/Trigger/TrigValidation/TrigInDetValidation/utils/art-input.dat index d0a96c80110..7f58ab3c94e 100644 --- a/Trigger/TrigValidation/TrigInDetValidation/utils/art-input.dat +++ b/Trigger/TrigValidation/TrigInDetValidation/utils/art-input.dat @@ -4,12 +4,15 @@ test_trigindetvalidation_all_ttbar_pu80 mc15_13TeV.410000.PowhegPythiaEvtGen_P20 test_trigindetvalidation_beamspot_ttbar_pu40 mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 test_trigindetvalidation_beamspot_ttbar_pu40_zfinder mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 test_trigindetvalidation_beamspot_ttbar_pu80_zfinder mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 +test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 +test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2 mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 test_trigindetvalidation_beamspot_ttbar_pu80_fs mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 test_trigindetvalidation_beamspot_ttbar_pu80_fsml mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 test_trigindetvalidation_bjet_ibl_pu40_2016config mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 test_trigindetvalidation_bjet_ibl_pu40_offline_2016config mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 test_trigindetvalidation_bjet_ibl_pu40_offline mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 test_trigindetvalidation_bjet_ibl_pu40 mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 +test_trigindetvalidation_bjet_ibl_pu40_ml mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 test_trigindetvalidation_bjet_ibl_pu80 mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 test_trigindetvalidation_el_jpsiee_pu40 mc15_13TeV.129190.Pythia8_AU2CTEQ6L1_ppToJpsie3e3.recon.RDO.e3802_s2608_s2183_r7042 test_trigindetvalidation_el_single_e_7-80_ibl_pu40 mc15_13TeV.159010.ParticleGenerator_e_Et7to80.recon.RDO.e1948_s2726_r7772 @@ -39,5 +42,6 @@ test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher mc15_13TeV.361107.PowhegPyt test_trigindetvalidation_mu_zmumu_ibl_pu40 mc15_13TeV.361107.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zmumu.recon.RDO.e3601_s2576_s2132_r7143 test_trigindetvalidation_tau_ibl_pu46_offline mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 test_trigindetvalidation_tau_ibl_pu46 mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 +test_trigindetvalidation_tau_ibl_pu46_ml mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 test_trigindetvalidation_tau_ibl_pu46_nodoublet mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 test_trigindetvalidation_tau_ibl_pu46_roiparam mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 diff --git a/Trigger/TrigValidation/TrigInDetValidation/utils/art-nfiles.dat b/Trigger/TrigValidation/TrigInDetValidation/utils/art-nfiles.dat index c365aba6f23..9a46535be21 100644 --- a/Trigger/TrigValidation/TrigInDetValidation/utils/art-nfiles.dat +++ b/Trigger/TrigValidation/TrigInDetValidation/utils/art-nfiles.dat @@ -4,12 +4,15 @@ test_trigindetvalidation_all_ttbar_pu80 3 test_trigindetvalidation_beamspot_ttbar_pu40 3 test_trigindetvalidation_beamspot_ttbar_pu40_zfinder 3 test_trigindetvalidation_beamspot_ttbar_pu80_zfinder 3 +test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml 3 +test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2 3 test_trigindetvalidation_beamspot_ttbar_pu80_fs 3 test_trigindetvalidation_beamspot_ttbar_pu80_fsml 3 test_trigindetvalidation_bjet_ibl_pu40_2016config 3 test_trigindetvalidation_bjet_ibl_pu40_offline_2016config 3 test_trigindetvalidation_bjet_ibl_pu40_offline 3 test_trigindetvalidation_bjet_ibl_pu40 3 +test_trigindetvalidation_bjet_ibl_pu40_ml 3 test_trigindetvalidation_bjet_ibl_pu80 3 test_trigindetvalidation_el_jpsiee_pu40 16 test_trigindetvalidation_el_single_e_7-80_ibl_pu40 10 @@ -39,5 +42,6 @@ test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher 4 test_trigindetvalidation_mu_zmumu_ibl_pu40 4 test_trigindetvalidation_tau_ibl_pu46_offline 3 test_trigindetvalidation_tau_ibl_pu46 3 +test_trigindetvalidation_tau_ibl_pu46_ml 3 test_trigindetvalidation_tau_ibl_pu46_nodoublet 3 test_trigindetvalidation_tau_ibl_pu46_roiparam 3 -- GitLab From 11b388b48f2862a4ddfb67d4de97588c760868f4 Mon Sep 17 00:00:00 2001 From: Tim Adye Date: Thu, 3 Dec 2020 16:10:43 +0000 Subject: [PATCH 2/2] remove old test scripts (no longer run). rename test/*_art.sh to replace them, test/*.sh --- ...test_trigindetvalidation_all_ttbar_pu40.sh | 43 +- ..._trigindetvalidation_all_ttbar_pu40_art.sh | 554 ------------------ ...test_trigindetvalidation_all_ttbar_pu60.sh | 43 +- ..._trigindetvalidation_all_ttbar_pu60_art.sh | 554 ------------------ ...test_trigindetvalidation_all_ttbar_pu80.sh | 43 +- ..._trigindetvalidation_all_ttbar_pu80_art.sh | 554 ------------------ ...trigindetvalidation_beamspot_ttbar_pu40.sh | 43 +- ...indetvalidation_beamspot_ttbar_pu40_art.sh | 464 --------------- ...tvalidation_beamspot_ttbar_pu40_zfinder.sh | 43 +- ...idation_beamspot_ttbar_pu40_zfinder_art.sh | 464 --------------- ...gindetvalidation_beamspot_ttbar_pu80_fs.sh | 43 +- ...etvalidation_beamspot_ttbar_pu80_fs_art.sh | 464 --------------- ...ndetvalidation_beamspot_ttbar_pu80_fsml.sh | 43 +- ...validation_beamspot_ttbar_pu80_fsml_art.sh | 464 --------------- ...tvalidation_beamspot_ttbar_pu80_zfinder.sh | 43 +- ...idation_beamspot_ttbar_pu80_zfinder_art.sh | 464 --------------- ...idation_beamspot_ttbar_pu80_zfinder_ml.sh} | 0 ...dation_beamspot_ttbar_pu80_zfinder_ml2.sh} | 0 .../test_trigindetvalidation_bjet_ibl_pu40.sh | 43 +- ...ndetvalidation_bjet_ibl_pu40_2016config.sh | 43 +- ...validation_bjet_ibl_pu40_2016config_art.sh | 474 --------------- ...t_trigindetvalidation_bjet_ibl_pu40_art.sh | 494 ---------------- ...t_trigindetvalidation_bjet_ibl_pu40_ml.sh} | 0 ...igindetvalidation_bjet_ibl_pu40_offline.sh | 43 +- ...dation_bjet_ibl_pu40_offline_2016config.sh | 43 +- ...on_bjet_ibl_pu40_offline_2016config_art.sh | 474 --------------- ...detvalidation_bjet_ibl_pu40_offline_art.sh | 484 --------------- .../test_trigindetvalidation_bjet_ibl_pu80.sh | 43 +- ...t_trigindetvalidation_bjet_ibl_pu80_art.sh | 474 --------------- ...test_trigindetvalidation_el_jpsiee_pu40.sh | 43 +- ..._trigindetvalidation_el_jpsiee_pu40_art.sh | 474 --------------- ...rigindetvalidation_el_single_e_7-80_ibl.sh | 43 +- ...ndetvalidation_el_single_e_7-80_ibl_art.sh | 494 ---------------- ...detvalidation_el_single_e_7-80_ibl_pu40.sh | 43 +- ...alidation_el_single_e_7-80_ibl_pu40_art.sh | 494 ---------------- ...ndetvalidation_el_single_e_7-80_larged0.sh | 43 +- ...validation_el_single_e_7-80_larged0_art.sh | 494 ---------------- ...tvalidation_el_single_e_7-80_larged0_pu.sh | 43 +- ...idation_el_single_e_7-80_larged0_pu_art.sh | 494 ---------------- .../test_trigindetvalidation_el_ttbar_pu40.sh | 43 +- ...t_trigindetvalidation_el_ttbar_pu40_art.sh | 494 ---------------- .../test_trigindetvalidation_el_ttbar_pu80.sh | 43 +- ...t_trigindetvalidation_el_ttbar_pu80_art.sh | 494 ---------------- ...est_trigindetvalidation_el_zee_ibl_pu40.sh | 43 +- ...trigindetvalidation_el_zee_ibl_pu40_art.sh | 494 ---------------- ...indetvalidation_el_zee_ibl_pu40_offline.sh | 43 +- ...tvalidation_el_zee_ibl_pu40_offline_art.sh | 494 ---------------- ...ation_el_zee_ibl_pu40_offline_rzmatcher.sh | 43 +- ...n_el_zee_ibl_pu40_offline_rzmatcher_art.sh | 494 ---------------- ...detvalidation_el_zee_ibl_pu40_rzmatcher.sh | 43 +- ...alidation_el_zee_ibl_pu40_rzmatcher_art.sh | 494 ---------------- .../test/test_trigindetvalidation_minbias.sh | 43 +- .../test_trigindetvalidation_minbias_art.sh | 464 --------------- ...est_trigindetvalidation_minbias_offline.sh | 43 +- ...trigindetvalidation_minbias_offline_art.sh | 464 --------------- .../test_trigindetvalidation_mu_bphys_ibl.sh | 43 +- ...st_trigindetvalidation_mu_bphys_ibl_art.sh | 474 --------------- ...rigindetvalidation_mu_single_mu_larged0.sh | 43 +- ...ndetvalidation_mu_single_mu_larged0_art.sh | 474 --------------- ...indetvalidation_mu_single_mu_larged0_pu.sh | 43 +- ...tvalidation_mu_single_mu_larged0_pu_art.sh | 474 --------------- ...t_trigindetvalidation_mu_zmumu_ibl_pu40.sh | 43 +- ...igindetvalidation_mu_zmumu_ibl_pu40_art.sh | 474 --------------- ...detvalidation_mu_zmumu_ibl_pu40_offline.sh | 43 +- ...alidation_mu_zmumu_ibl_pu40_offline_art.sh | 474 --------------- ...ion_mu_zmumu_ibl_pu40_offline_rzmatcher.sh | 43 +- ...mu_zmumu_ibl_pu40_offline_rzmatcher_art.sh | 474 --------------- ...tvalidation_mu_zmumu_ibl_pu40_rzmatcher.sh | 43 +- ...idation_mu_zmumu_ibl_pu40_rzmatcher_art.sh | 474 --------------- .../test_trigindetvalidation_tau_ibl_pu46.sh | 43 +- ...st_trigindetvalidation_tau_ibl_pu46_art.sh | 474 --------------- ...st_trigindetvalidation_tau_ibl_pu46_ml.sh} | 0 ...gindetvalidation_tau_ibl_pu46_nodoublet.sh | 43 +- ...etvalidation_tau_ibl_pu46_nodoublet_art.sh | 474 --------------- ...rigindetvalidation_tau_ibl_pu46_offline.sh | 43 +- ...ndetvalidation_tau_ibl_pu46_offline_art.sh | 474 --------------- ...igindetvalidation_tau_ibl_pu46_roiparam.sh | 43 +- ...detvalidation_tau_ibl_pu46_roiparam_art.sh | 474 --------------- .../test/trigindetvalidation_cosmic.sh | 329 ----------- .../TrigInDetValidation/utils/mkart-template | 22 +- 80 files changed, 1561 insertions(+), 18319 deletions(-) delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_art.sh rename Trigger/TrigValidation/TrigInDetValidation/test/{test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml_art.sh => test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml.sh} (100%) rename Trigger/TrigValidation/TrigInDetValidation/test/{test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2_art.sh => test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2.sh} (100%) delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_art.sh rename Trigger/TrigValidation/TrigInDetValidation/test/{test_trigindetvalidation_bjet_ibl_pu40_ml_art.sh => test_trigindetvalidation_bjet_ibl_pu40_ml.sh} (100%) delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_art.sh rename Trigger/TrigValidation/TrigInDetValidation/test/{test_trigindetvalidation_tau_ibl_pu46_ml_art.sh => test_trigindetvalidation_tau_ibl_pu46_ml.sh} (100%) delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam_art.sh delete mode 100755 Trigger/TrigValidation/TrigInDetValidation/test/trigindetvalidation_cosmic.sh diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40.sh index 73db89aafdf..61cdae968e9 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for all_ttbar_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots-electron # art-output: HLTL2-plots-electron @@ -25,7 +29,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -57,6 +62,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -205,6 +211,7 @@ export RTTJOBNAME=TrigInDetValidation_all_ttbar_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -212,16 +219,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -286,6 +325,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40_art.sh deleted file mode 100755 index 61cdae968e9..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu40_art.sh +++ /dev/null @@ -1,554 +0,0 @@ -#!/bin/bash -# art-description: art job for all_ttbar_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots-electron -# art-output: HLTL2-plots-electron -# art-output: HLTEF-plots-electron-lowpt -# art-output: HLTL2-plots-electron-lowpt -# art-output: HLTEF-plots-muon -# art-output: HLTL2-plots-muon -# art-output: HLTEF-plots-tau -# art-output: HLTL2-plots-tau -# art-output: HLTEF-plots-bjet -# art-output: HLTL2-plots-bjet -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_AllSlices.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_all_ttbar_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -r Offline -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-electron 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-electron 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-electron-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-electron-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots-muon 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots-muon 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots-tau 2>&1 | tee TIDArun_8.log -echo "art-result: $? TIDArun_8" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots-tau 2>&1 | tee TIDArun_9.log -echo "art-result: $? TIDArun_9" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots-bjet 2>&1 | tee TIDArun_10.log -echo "art-result: $? TIDArun_10" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu40-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots-bjet 2>&1 | tee TIDArun_11.log -echo "art-result: $? TIDArun_11" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_12.log -echo "art-result: $? TIDArun_12" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_13.log -echo "art-result: $? TIDArun_13" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_14.log -echo "art-result: $? RunTrigCostD3PD_14" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_15.log -echo "art-result: $? TIDAcpucost_15" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_16.log -echo "art-result: $? TIDAcpucost_16" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_17.log -echo "art-result: $? TIDAcpucost_17" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_18.log -echo "art-result: $? TIDAcpucost_18" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60.sh index 1450fc98eee..b00e4ccaf4b 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for all_ttbar_pu60 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots-electron # art-output: HLTL2-plots-electron @@ -25,7 +29,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -57,6 +62,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -205,6 +211,7 @@ export RTTJOBNAME=TrigInDetValidation_all_ttbar_pu60 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -212,16 +219,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -286,6 +325,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60_art.sh deleted file mode 100755 index b00e4ccaf4b..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu60_art.sh +++ /dev/null @@ -1,554 +0,0 @@ -#!/bin/bash -# art-description: art job for all_ttbar_pu60 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots-electron -# art-output: HLTL2-plots-electron -# art-output: HLTEF-plots-electron-lowpt -# art-output: HLTL2-plots-electron-lowpt -# art-output: HLTEF-plots-muon -# art-output: HLTL2-plots-muon -# art-output: HLTEF-plots-tau -# art-output: HLTL2-plots-tau -# art-output: HLTEF-plots-bjet -# art-output: HLTL2-plots-bjet -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7194 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_AllSlices.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_all_ttbar_pu60 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -r Offline -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-electron 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-electron 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-electron-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-electron-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots-muon 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots-muon 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots-tau 2>&1 | tee TIDArun_8.log -echo "art-result: $? TIDArun_8" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots-tau 2>&1 | tee TIDArun_9.log -echo "art-result: $? TIDArun_9" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots-bjet 2>&1 | tee TIDArun_10.log -echo "art-result: $? TIDArun_10" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu60-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots-bjet 2>&1 | tee TIDArun_11.log -echo "art-result: $? TIDArun_11" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_12.log -echo "art-result: $? TIDArun_12" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_13.log -echo "art-result: $? TIDArun_13" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_14.log -echo "art-result: $? RunTrigCostD3PD_14" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_15.log -echo "art-result: $? TIDAcpucost_15" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_16.log -echo "art-result: $? TIDAcpucost_16" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_17.log -echo "art-result: $? TIDAcpucost_17" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_18.log -echo "art-result: $? TIDAcpucost_18" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80.sh index 2de2e293b93..2794b53722c 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for all_ttbar_pu80 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots-electron # art-output: HLTL2-plots-electron @@ -25,7 +29,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -57,6 +62,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -205,6 +211,7 @@ export RTTJOBNAME=TrigInDetValidation_all_ttbar_pu80 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -212,16 +219,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -286,6 +325,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80_art.sh deleted file mode 100755 index 2794b53722c..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_all_ttbar_pu80_art.sh +++ /dev/null @@ -1,554 +0,0 @@ -#!/bin/bash -# art-description: art job for all_ttbar_pu80 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots-electron -# art-output: HLTL2-plots-electron -# art-output: HLTEF-plots-electron-lowpt -# art-output: HLTL2-plots-electron-lowpt -# art-output: HLTEF-plots-muon -# art-output: HLTL2-plots-muon -# art-output: HLTEF-plots-tau -# art-output: HLTL2-plots-tau -# art-output: HLTEF-plots-bjet -# art-output: HLTL2-plots-bjet -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_AllSlices.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_all_ttbar_pu80 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;doIDNewTracking=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -r Offline -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-electron 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-electron 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-electron-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-electron-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots-muon 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots-muon 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots-tau 2>&1 | tee TIDArun_8.log -echo "art-result: $? TIDArun_8" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots-tau 2>&1 | tee TIDArun_9.log -echo "art-result: $? TIDArun_9" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots-bjet 2>&1 | tee TIDArun_10.log -echo "art-result: $? TIDArun_10" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-all.root data-all_ttbar_pu80-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots-bjet 2>&1 | tee TIDArun_11.log -echo "art-result: $? TIDArun_11" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_12.log -echo "art-result: $? TIDArun_12" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_13.log -echo "art-result: $? TIDArun_13" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_14.log -echo "art-result: $? RunTrigCostD3PD_14" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_15.log -echo "art-result: $? TIDAcpucost_15" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_16.log -echo "art-result: $? TIDAcpucost_16" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_17.log -echo "art-result: $? TIDAcpucost_17" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_18.log -echo "art-result: $? TIDAcpucost_18" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40.sh index 41c8b400d99..1d4b5b40d4f 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for beamspot_ttbar_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-test-plots # art-output: times @@ -16,7 +20,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -48,6 +53,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -196,6 +202,7 @@ export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -203,16 +210,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -277,6 +316,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_art.sh deleted file mode 100755 index 1d4b5b40d4f..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_art.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash -# art-description: art job for beamspot_ttbar_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-test-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BeamspotSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-beamspot_ttbar_pu40-reference.root HLT_beamspot_allTE_trkfast_InDetTrigTrackingxAODCnv_BeamSpot_FTF -d HLTL2-test-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder.sh index 8810b08cdcd..288a65e34e9 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for beamspot_ttbar_pu40_zfinder +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-test-plots # art-output: times @@ -16,7 +20,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -48,6 +53,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -196,6 +202,7 @@ export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu40_zfinder jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -203,16 +210,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;fastZFinder=True;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -277,6 +316,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder_art.sh deleted file mode 100755 index 288a65e34e9..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu40_zfinder_art.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash -# art-description: art job for beamspot_ttbar_pu40_zfinder -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-test-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BeamspotSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu40_zfinder - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;fastZFinder=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;fastZFinder=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-beamspot_ttbar_pu40_zfinder-reference.root HLT_beamspot_allTE_trkfast_InDetTrigTrackingxAODCnv_BeamSpot_FTF -d HLTL2-test-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs.sh index 65ab5d5beaa..c11906043b3 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for beamspot_ttbar_pu80_fs +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-test-plots # art-output: times @@ -16,7 +20,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -48,6 +53,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -196,6 +202,7 @@ export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu80_fs jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -203,16 +210,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -277,6 +316,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs_art.sh deleted file mode 100755 index c11906043b3..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fs_art.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash -# art-description: art job for beamspot_ttbar_pu80_fs -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-test-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BeamspotSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu80_fs - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-beamspot_ttbar_pu80_fs-reference.root HLT_beamspot_allTE_trkfast_InDetTrigTrackingxAODCnv_BeamSpot_FTF -d HLTL2-test-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml.sh index 8008a2c9330..0ccd1e5cb3e 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for beamspot_ttbar_pu80_fsml +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-test-plots # art-output: times @@ -16,7 +20,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -48,6 +53,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -196,6 +202,7 @@ export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu80_fsml jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -203,16 +210,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;mlExtensions=True;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -277,6 +316,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml_art.sh deleted file mode 100755 index 0ccd1e5cb3e..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_fsml_art.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash -# art-description: art job for beamspot_ttbar_pu80_fsml -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-test-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BeamspotSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu80_fsml - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;mlExtensions=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;mlExtensions=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-beamspot_ttbar_pu80_fsml-reference.root HLT_beamspot_allTE_trkfast_InDetTrigTrackingxAODCnv_BeamSpot_FTF -d HLTL2-test-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder.sh index 55320296723..51aef2d942d 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for beamspot_ttbar_pu80_zfinder +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-test-plots # art-output: times @@ -16,7 +20,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -48,6 +53,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -196,6 +202,7 @@ export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu80_zfinder jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -203,16 +210,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;fastZFinder=True;rec.doFloatingPointException.set_Value_and_Lock(False)" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -277,6 +316,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_art.sh deleted file mode 100755 index 51aef2d942d..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_art.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash -# art-description: art job for beamspot_ttbar_pu80_zfinder -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-test-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BeamspotSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_beamspot_ttbar_pu80_zfinder - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;fastZFinder=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;fastZFinder=True;rec.doFloatingPointException.set_Value_and_Lock(False)" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-all.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-all.root data-beamspot_ttbar_pu80_zfinder-reference.root HLT_beamspot_allTE_trkfast_InDetTrigTrackingxAODCnv_BeamSpot_FTF -d HLTL2-test-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml.sh similarity index 100% rename from Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml_art.sh rename to Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml.sh diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2.sh similarity index 100% rename from Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2_art.sh rename to Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_beamspot_ttbar_pu80_zfinder_ml2.sh diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40.sh index 513c961ac3a..96d5ac8b934 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for bjet_IBL_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;doBperf=True;globalTag='OFLCOND-RUN12-SDR-17'" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config.sh index 3555777ec52..f550bb99ee9 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for bjet_IBL_pu40_2016config +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40_2016config jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;minVtxTrackpT=1000;bjetEtaHalfWidth=0.2;bjetPhiHalfWidth=0.2;splitZHalfWidth=20;globalTag='OFLCOND-RUN12-SDR-17'" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config_art.sh deleted file mode 100755 index f550bb99ee9..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_2016config_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for bjet_IBL_pu40_2016config -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BjetSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40_2016config - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;minVtxTrackpT=1000;bjetEtaHalfWidth=0.2;bjetPhiHalfWidth=0.2;splitZHalfWidth=20;globalTag='OFLCOND-RUN12-SDR-17'" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;minVtxTrackpT=1000;bjetEtaHalfWidth=0.2;bjetPhiHalfWidth=0.2;splitZHalfWidth=20;globalTag='OFLCOND-RUN12-SDR-17'" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-bjet-pileup-merging.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40_2016config-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40_2016config-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_art.sh deleted file mode 100755 index 96d5ac8b934..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for bjet_IBL_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTEF-plots-comp -# art-output: HLTL2-plots-comp -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BjetSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;doBperf=True;globalTag='OFLCOND-RUN12-SDR-17'" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;doBperf=True;globalTag='OFLCOND-RUN12-SDR-17'" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-bjet-pileup-merging.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40-reference.root HLT_j55_bperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_bperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID -d HLTEF-plots-comp 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40-reference.root HLT_j55_bperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_bperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots-comp 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml.sh similarity index 100% rename from Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml_art.sh rename to Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_ml.sh diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline.sh index 762ee073bd4..dffaf68f15e 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for bjet_IBL_pu40_offline +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -18,7 +22,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -50,6 +55,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -198,6 +204,7 @@ export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40_offline jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -205,16 +212,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;doIDNewTracking=True;globalTag='OFLCOND-RUN12-SDR-17'" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -279,6 +318,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config.sh index 7d24078e36d..015ae18761a 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for bjet_IBL_pu40_offline_2016config +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40_offline_2016config jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;doIDNewTracking=True;minVtxTrackpT=1000;bjetEtaHalfWidth=0.2;bjetPhiHalfWidth=0.2;splitZHalfWidth=20;globalTag='OFLCOND-RUN12-SDR-17'" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config_art.sh deleted file mode 100755 index 015ae18761a..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_2016config_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for bjet_IBL_pu40_offline_2016config -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BjetSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40_offline_2016config - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;doIDNewTracking=True;minVtxTrackpT=1000;bjetEtaHalfWidth=0.2;bjetPhiHalfWidth=0.2;splitZHalfWidth=20;globalTag='OFLCOND-RUN12-SDR-17'" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;doIDNewTracking=True;minVtxTrackpT=1000;bjetEtaHalfWidth=0.2;bjetPhiHalfWidth=0.2;splitZHalfWidth=20;globalTag='OFLCOND-RUN12-SDR-17'" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline.dat -f data-bjet-pileup-merging.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40_offline_2016config-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu40_offline_2016config-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi_xPrimVx -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_art.sh deleted file mode 100755 index dffaf68f15e..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu40_offline_art.sh +++ /dev/null @@ -1,484 +0,0 @@ -#!/bin/bash -# art-description: art job for bjet_IBL_pu40_offline -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTL2-vtx -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BjetSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu40_offline - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;doIDNewTracking=True;globalTag='OFLCOND-RUN12-SDR-17'" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;doIDNewTracking=True;globalTag='OFLCOND-RUN12-SDR-17'" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline.dat -f data-bjet-pileup.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-bjet-pileup.root data-bjet_IBL_pu40_offline-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup.root data-bjet_IBL_pu40_offline-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi_xPrimVx -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup.root data-bjet_IBL_pu40_offline-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi_xPrimVx/xPrimVx HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi_EFHistoVtx/EFHistoVtx -c TIDAvertex.dat -d HLTL2-vtx 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_7.log -echo "art-result: $? RunTrigCostD3PD_7" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80.sh index a996c857e7a..a324fc730d4 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for bjet_IBL_pu80 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu80 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000;globalTag='OFLCOND-RUN12-SDR-17'" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80_art.sh deleted file mode 100755 index a324fc730d4..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_bjet_ibl_pu80_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for bjet_IBL_pu80 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_BjetSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_bjet_IBL_pu80 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000;globalTag='OFLCOND-RUN12-SDR-17'" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000;globalTag='OFLCOND-RUN12-SDR-17'" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-bjet-pileup-merging.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu80-reference.root HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_IDTrig HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_IDTrig_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-bjet-pileup-merging.root data-bjet_IBL_pu80-reference.root HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_InDetTrigTrackingxAODCnv_Bjet_FTF_forID HLT_j55_boffperf_split_InDetTrigTrackingxAODCnv_BjetPrmVtx_FTF_SuperRoi -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40.sh index 57974f91335..e2c7174a57d 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_Jpsiee_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 16 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_el_Jpsiee_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=500" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40_art.sh deleted file mode 100755 index e2c7174a57d..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_jpsiee_pu40_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for el_Jpsiee_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.129190.Pythia8_AU2CTEQ6L1_ppToJpsie3e3.recon.RDO.e3802_s2608_s2183_r7042 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 16 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_Jpsiee_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=500" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=500" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-electron-merge.root -p 11 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-merge.root data-el_Jpsiee_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Jpsiee_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl.sh index a85a4e21bb7..0d45bde39df 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_single_e_7-80_IBL +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTEF-plots-lowpt @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 10 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_IBL jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_art.sh deleted file mode 100755 index 0d45bde39df..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_single_e_7-80_IBL -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.159010.ParticleGenerator_e_Et7to80.recon.RDO.e1948_s2726_r7728 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 10 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_IBL - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-electron-IBL.root -p 11 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40.sh index e51ad745c8b..62071b0a867 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_single_e_7-80_IBL_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTEF-plots-lowpt @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 10 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_IBL_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40_art.sh deleted file mode 100755 index 62071b0a867..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_ibl_pu40_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_single_e_7-80_IBL_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.159010.ParticleGenerator_e_Et7to80.recon.RDO.e1948_s2726_r7772 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 10 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_IBL_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-electron-IBL.root -p 11 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_IBL_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0.sh index a660999ede7..3df0bcda404 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_single_e_7-80_larged0 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTEF-plots-lowpt @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 10 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_larged0 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_art.sh deleted file mode 100755 index 3df0bcda404..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_single_e_7-80_larged0 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.159053.ParticleGenerator_e_Et7to80_vertxy20.recon.RDO.e3603_s2726_r7728 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 10 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_larged0 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-larged0-el.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin_larged0.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-larged0-el.dat -f data-electron-IBL.root -p 11 -b Test_bin_larged0.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu.sh index 53f74f03f8b..9ef73fa4fcb 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_single_e_7-80_larged0_PU +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTEF-plots-lowpt @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 10 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_larged0_PU jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu_art.sh deleted file mode 100755 index 9ef73fa4fcb..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_single_e_7-80_larged0_pu_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_single_e_7-80_larged0_PU -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.159053.ParticleGenerator_e_Et7to80_vertxy20.recon.RDO.e3603_s2726_r7772 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 10 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_single_e_7-80_larged0_PU - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-larged0-el.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin_larged0.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-larged0-el.dat -f data-electron-IBL.root -p 11 -b Test_bin_larged0.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0_PU-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0_PU-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0_PU-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-IBL.root data-el_single_e_7-80_larged0_PU-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40.sh index 717b16b4327..787ce9963ba 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_ttbar_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_ttbar_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40_art.sh deleted file mode 100755 index 787ce9963ba..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu40_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_ttbar_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7193 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_ttbar_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-electron-merge.root -p 11 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80.sh index f2be22b4dd4..bb6038bd399 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_ttbar_pu80 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_ttbar_pu80 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=1000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80_art.sh deleted file mode 100755 index bb6038bd399..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_ttbar_pu80_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_ttbar_pu80 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.410000.PowhegPythiaEvtGen_P2012_ttbar_hdamp172p5_nonallhad.recon.RDO.e3698_s2608_s2183_r7195 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_ttbar_pu80 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=1000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=1000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-electron-merge.root -p 11 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu80-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu80-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu80-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_ttbar_pu80-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40.sh index 249743417cf..b08e2962a8b 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_Zee_IBL_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 8 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_art.sh deleted file mode 100755 index b08e2962a8b..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_Zee_IBL_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.recon.RDO.e3601_s2665_s2183_r7191 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 8 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-electron-merge.root -p 11 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline.sh index c52e79c7eb8..abe38d1138a 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_Zee_IBL_pu40_offline +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40_offline jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=11" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_art.sh deleted file mode 100755 index abe38d1138a..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_Zee_IBL_pu40_offline -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.recon.RDO.e3601_s2665_s2183_r7191 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40_offline - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=11" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;doIDNewTracking=True;PdgId=11" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline.dat -f data-electron-merge.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher.sh index 03c5e61e8aa..ab2474e8157 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_Zee_IBL_pu40_offline_rzMatcher +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40_offline_rzMatcher jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=11" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher_art.sh deleted file mode 100755 index ab2474e8157..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_offline_rzmatcher_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_Zee_IBL_pu40_offline_rzMatcher -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.recon.RDO.e3601_s2665_s2183_r7191 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40_offline_rzMatcher - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=11" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;doIDNewTracking=True;PdgId=11" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline-rzMatcher.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline-rzMatcher.dat -f data-electron-merge.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline_rzMatcher-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline_rzMatcher-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline_rzMatcher-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_offline_rzMatcher-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher.sh index dcd3b5dde67..4ddc990ecd2 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for el_Zee_IBL_pu40_rzMatcher +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -19,7 +23,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 8 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -51,6 +56,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -199,6 +205,7 @@ export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40_rzMatcher jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -206,16 +213,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;PdgId=11" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -280,6 +319,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher_art.sh deleted file mode 100755 index 4ddc990ecd2..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_el_zee_ibl_pu40_rzmatcher_art.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# art-description: art job for el_Zee_IBL_pu40_rzMatcher -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: HLTEF-plots-lowpt -# art-output: HLTL2-plots-lowpt -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.recon.RDO.e3601_s2665_s2183_r7191 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 8 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_ElectronSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_el_Zee_IBL_pu40_rzMatcher - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;PdgId=11" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;PdgId=11" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-rzMatcher.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-rzMatcher.dat -f data-electron-merge.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_rzMatcher-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_rzMatcher-reference.root HLT_e24_medium_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_rzMatcher-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_IDTrig -d HLTEF-plots-lowpt 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-electron-merge.root data-el_Zee_IBL_pu40_rzMatcher-reference.root HLT_e5_loose_idperf_InDetTrigTrackingxAODCnv_Electron_FTF -d HLTL2-plots-lowpt 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_6.log -echo "art-result: $? TIDArun_6" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_7.log -echo "art-result: $? TIDArun_7" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_8.log -echo "art-result: $? RunTrigCostD3PD_8" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_11.log -echo "art-result: $? TIDAcpucost_11" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_12.log -echo "art-result: $? TIDAcpucost_12" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias.sh index c1b17912658..6b1fe66fcf0 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for minBias +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: times @@ -16,7 +20,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -48,6 +53,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -196,6 +202,7 @@ export RTTJOBNAME=TrigInDetValidation_minBias jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -203,16 +210,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -277,6 +316,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_art.sh deleted file mode 100755 index 6b1fe66fcf0..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_art.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash -# art-description: art job for minBias -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361036.Pythia8_A2MSTW2008LO_minbias_inelastic.recon.RDO.e3580_s2726_r7011 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MonitorSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_minBias - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-monitor.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-monitor.root data-minBias-reference.root HLT_mb_idperf_L1RD1_FILLED_InDetTrigTrackingxAODCnv_minBias_EFID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline.sh index aa0e86549db..46e6b595ca5 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for minBias_offline +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: times @@ -16,7 +20,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -48,6 +53,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -196,6 +202,7 @@ export RTTJOBNAME=TrigInDetValidation_minBias_offline jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -203,16 +210,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;doIDNewTracking=True" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -277,6 +316,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline_art.sh deleted file mode 100755 index 46e6b595ca5..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_minbias_offline_art.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/bash -# art-description: art job for minBias_offline -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361036.Pythia8_A2MSTW2008LO_minbias_inelastic.recon.RDO.e3580_s2726_r7011 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MonitorSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_minBias_offline - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;doIDNewTracking=True" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;doIDNewTracking=True" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline.dat -f data-minBias.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-minBias.root data-minBias_offline-reference.root HLT_mb_idperf_L1RD1_FILLED_InDetTrigTrackingxAODCnv_minBias_EFID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl.sh index 7a8135243e1..9882977b626 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for mu_bphys_IBL +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-plots # art-output: HLTEF-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_mu_bphys_IBL jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl_art.sh deleted file mode 100755 index 9882977b626..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_bphys_ibl_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for mu_bphys_IBL -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-plots -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.300401.Pythia8BPhotospp_A14_CTEQ6L1_Bs_Jpsimu3p5mu3p5_phi.recon.RDO.e4397_s2608_r6869 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MuonSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_mu_bphys_IBL - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-muon-bphys-merge.root -p 13 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-muon-bphys-merge.root data-mu_bphys_IBL-reference.root HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-muon-bphys-merge.root data-mu_bphys_IBL-reference.root HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0.sh index eb22b69959b..25e13bd63af 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for mu_single_mu_larged0 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 10 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_mu_single_mu_larged0 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_art.sh deleted file mode 100755 index 25e13bd63af..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for mu_single_mu_larged0 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.107237.ParticleGenerator_mu_Pt4to100_vertxy20.recon.RDO.e3603_s2726_r7728 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 10 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MuonSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_mu_single_mu_larged0 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-larged0.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin_larged0.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-larged0.dat -f data-muon.root -p 13 -b Test_bin_larged0.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-muon.root data-mu_single_mu_larged0-reference.root HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-muon.root data-mu_single_mu_larged0-reference.root HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu.sh index f87b6198246..8c0e9a1a371 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for mu_single_mu_larged0_PU +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 10 -# art-ncores: 8 +# art-cores: 8 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_mu_single_mu_larged0_PU jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu_art.sh deleted file mode 100755 index 8c0e9a1a371..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_single_mu_larged0_pu_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for mu_single_mu_larged0_PU -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.107237.ParticleGenerator_mu_Pt4to100_vertxy20.recon.RDO.e3603_s2726_r7772 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 10 -# art-cores: 8 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MuonSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_mu_single_mu_larged0_PU - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-larged0.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin_larged0.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-larged0.dat -f data-muon.root -p 13 -b Test_bin_larged0.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-muon.root data-mu_single_mu_larged0_PU-reference.root HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-muon.root data-mu_single_mu_larged0_PU-reference.root HLT_mu6_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40.sh index 47fc5b97b46..9968bb0c7fa 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for mu_Zmumu_IBL_pu40 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-plots # art-output: HLTEF-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_art.sh deleted file mode 100755 index 9968bb0c7fa..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for mu_Zmumu_IBL_pu40 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-plots -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361107.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zmumu.recon.RDO.e3601_s2576_s2132_r7143 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MuonSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-muon-pileup-merge.root -p 13 -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline.sh index 4844cf08b7f..b6f0ca6e4f8 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for mu_Zmumu_IBL_pu40_offline +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-plots # art-output: HLTEF-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40_offline jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=13" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_art.sh deleted file mode 100755 index b6f0ca6e4f8..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for mu_Zmumu_IBL_pu40_offline -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-plots -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361107.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zmumu.recon.RDO.e3601_s2576_s2132_r7143 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MuonSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40_offline - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=13" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;doIDNewTracking=True;PdgId=13" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline.dat -f data-muon-pileup-merge.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40_offline-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40_offline-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher.sh index 0577cc2879c..90e01f8e08a 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for mu_Zmumu_IBL_pu40_offline_rzMatcher +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-plots # art-output: HLTEF-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40_offline_rzMatcher jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=13" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher_art.sh deleted file mode 100755 index 90e01f8e08a..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_offline_rzmatcher_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for mu_Zmumu_IBL_pu40_offline_rzMatcher -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-plots -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361107.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zmumu.recon.RDO.e3601_s2576_s2132_r7143 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MuonSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40_offline_rzMatcher - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;doIDNewTracking=True;PdgId=13" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;doIDNewTracking=True;PdgId=13" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline-rzMatcher.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline-rzMatcher.dat -f data-muon-pileup-merge.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40_offline_rzMatcher-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40_offline_rzMatcher-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher.sh index f346b8be9e9..74c9efc82a8 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for mu_Zmumu_IBL_pu40_rzMatcher +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTL2-plots # art-output: HLTEF-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 4 -# art-ncores: 4 +# art-cores: 4 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40_rzMatcher jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;PdgId=13" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher_art.sh deleted file mode 100755 index 74c9efc82a8..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_mu_zmumu_ibl_pu40_rzmatcher_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for mu_Zmumu_IBL_pu40_rzMatcher -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTL2-plots -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc15_13TeV.361107.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zmumu.recon.RDO.e3601_s2576_s2132_r7143 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 4 -# art-cores: 4 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_MuonSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_mu_Zmumu_IBL_pu40_rzMatcher - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;PdgId=13" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;PdgId=13" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-rzMatcher.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-rzMatcher.dat -f data-muon-pileup-merge.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40_rzMatcher-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF -d HLTL2-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-muon-pileup-merge.root data-mu_Zmumu_IBL_pu40_rzMatcher-reference.root HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_FTF HLT_mu24_idperf_InDetTrigTrackingxAODCnv_Muon_IDTrig -d HLTEF-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46.sh index ea77d7c71c8..b424ba06027 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for tau_IBL_pu46 +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46 jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;PdgId=15" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_art.sh deleted file mode 100755 index b424ba06027..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for tau_IBL_pu46 -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_TauSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46 - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;PdgId=15" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;PdgId=15" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-tau-IBL.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_ml_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_ml.sh similarity index 100% rename from Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_ml_art.sh rename to Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_ml.sh diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet.sh index 03bd451bf0a..c0a446341dc 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for tau_IBL_pu46_NoDoublet +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46_NoDoublet jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;PdgId=15;TauDoubletFilter=False" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet_art.sh deleted file mode 100755 index c0a446341dc..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_nodoublet_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for tau_IBL_pu46_NoDoublet -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_TauSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46_NoDoublet - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;PdgId=15;TauDoubletFilter=False" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;PdgId=15;TauDoubletFilter=False" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-tau-IBL.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline.sh index 7eeeee85532..36d6fa1c55a 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for tau_IBL_pu46_offline +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46_offline jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;doIDNewTracking=True" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline_art.sh deleted file mode 100755 index 36d6fa1c55a..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_offline_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for tau_IBL_pu46_offline -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_TauSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46_offline - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;doIDNewTracking=True" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;doIDNewTracking=True" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt-offline.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt-offline.dat -f data-tau-IBL.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46_offline-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46_offline-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam.sh index 110a6ae3375..91afca256eb 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam.sh +++ b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam.sh @@ -1,5 +1,9 @@ #!/bin/bash # art-description: art job for tau_IBL_pu46_RoiParam +# art-include: 21.3/Athena +# art-include: 21.9/Athena +# art-include: master/Athena +# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= # art-type: grid # art-output: HLTEF-plots # art-output: HLTL2-plots @@ -17,7 +21,8 @@ # art-output: output-cost # art-output: output-logs # art-input-nfiles: 3 -# art-ncores: 3 +# art-cores: 3 +# art-memory: 4096 @@ -49,6 +54,7 @@ RUNPOST=-1 DIRECTORY= LOCAL=0 FORCE=0 +FORK=1 while [ $# -ge 1 ]; do case "$1" in @@ -197,6 +203,7 @@ export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46_RoiParam jobList= if [ $LOCAL -eq 1 ]; then + echo "running locally" # get number of files NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') @@ -204,16 +211,48 @@ if [ $LOCAL -eq 1 ]; then DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done + +elif [ -n "$ArtProcess" ]; then + + # art-cores specified, so ART is already forking off subprocesses + FORK=0 + case "$ArtProcess" in + start) + timestamp "ART Starting (${ArtCores}-core)" + exit 0 # nothing more to do this time round + ;; + end) + timestamp "ART Ending (${ArtCores}-core)" + ;; # skip to postprocessing + *) + # runathena here and now, no forking + timestamp "ART job $ArtProcess (${ArtCores}-core)" + IFS=',' read -r -a file <<< "${ArtInFile}" + file=${file[${ArtProcess}]} + _jobList="'../$file'" + echo "ART running over $_jobList" + jobList="ARTConfig=[$_jobList]" + if [ $RUNATHENA -eq 1 ]; then + ARGS="$jobList;EventMax=2000;PdgId=15;TauDoubletFilter=False;TauRoiParam=True" + runathena "$ArtProcess" + fi + exit 0 # this thread is done + ;; + esac + else + fileList="['${ArtInFile//,/', '}']" _jobList="'../${ArtInFile//,/' '../}'" echo "List of files = $fileList" for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done + fi if [ $RUNATHENA -eq 1 ]; then +if [ $FORK -eq 1 ]; then # run athena in separate directories @@ -278,6 +317,8 @@ timestamp "waiting on athena jobs ..." waitonallproc +fi + echo "all done ! hooray !" timestamp "forking all athena jobs completed" diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam_art.sh b/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam_art.sh deleted file mode 100755 index 91afca256eb..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/test_trigindetvalidation_tau_ibl_pu46_roiparam_art.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash -# art-description: art job for tau_IBL_pu46_RoiParam -# art-include: 21.3/Athena -# art-include: 21.9/Athena -# art-include: master/Athena -# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir= -# art-type: grid -# art-output: HLTEF-plots -# art-output: HLTL2-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc16_13TeV.361108.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Ztautau.recon.RDO.e3601_s3126_r9546 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: 3 -# art-cores: 3 -# art-memory: 4096 - - - -RED='\033[0;31m' -NC='\033[0m' - - -function usage { - [ $# -gt 1 ] && echo $2 - - echo "Usage: $(basename $0) [args]" - echo - echo "-d, --directory DIRECTORY \t run from the specified directory" - echo "-l, --local \t run locally rather than on the grid" - echo "-x, --exclude \t don't run athena or the post processing, only the plotting stages" - echo "-p, --post \t force running of post processingthe post processing, even if -x is set" - echo "-f, --force \t disable protection against rerunning where you shouldn't be" - echo "-h, --help \t this help" - [ $# -gt 0 ] && exit $1 - exit 0 -} - -args=$(getopt -ql "searchpath:" -o "d:lxph" -- "$@") - -# eval set -- "$args" - -RUNATHENA=1 -RUNPOST=-1 -DIRECTORY= -LOCAL=0 -FORCE=0 -FORK=1 - -while [ $# -ge 1 ]; do - case "$1" in - --) shift ; break ;; - -d | --directory ) if [ $# -lt 2 ]; then usage; fi ; DIRECTORY="$2" ; shift ;; - -x | --exclude ) RUNATHENA=0 ; [ $RUNPOST -eq -1 ] && RUNPOST=0;; - -p | --post ) RUNPOST=1 ;; - -f | --force ) FORCE=1 ;; - -l | --local ) LOCAL=1 ;; - -h | --help ) usage ;; - esac - shift -done - - -[ $RUNPOST -eq 0 ] || RUNPOST=1 - - -# generate a time stamp - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - -# poor quality function to convert seconds to a proper time - -function converttime { - local totes=$1 - [ $totes -lt 0 ] && ((totes=$totes+86400)) - ((H=$totes/3600)) - ((M=($totes%3600)/60)) - ((S=$totes%60)) - printf "%d:%02d:%02d\n" $H $M $S -} - -timestamp "starting" - -# reset seconds counter - not really needed, but hey - -SECONDS=0 - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_TauSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$1" - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - -ls -l - - - -export RTTJOBNAME=TrigInDetValidation_tau_IBL_pu46_RoiParam - -jobList= - -if [ $LOCAL -eq 1 ]; then - - echo "running locally" - # get number of files - NFILES=$(grep "^#[[:space:]]*art-input-nfiles:" $0 | sed 's|.*art-input-nfiles:[[:space:]]*||g') - [ $NFILES -lt 1 ] && echo "not enough files: $NFILES" && exit -1 - DATASET=$(grep "^#[[:space:]]*art-input:" $0 | sed 's|.*art-input:[[:space:]]*||g') - _jobList=$(\ls /eos/atlas/atlascerngroupdisk/proj-sit/trigindet/$DATASET/* ) - for git in $_jobList ; do [ $NFILES -gt 0 ] || break ; jobList="$jobList ARTConfig=['$git']" ; ((NFILES--)) ; echo "running over $git" ; done - -elif [ -n "$ArtProcess" ]; then - - # art-cores specified, so ART is already forking off subprocesses - FORK=0 - case "$ArtProcess" in - start) - timestamp "ART Starting (${ArtCores}-core)" - exit 0 # nothing more to do this time round - ;; - end) - timestamp "ART Ending (${ArtCores}-core)" - ;; # skip to postprocessing - *) - # runathena here and now, no forking - timestamp "ART job $ArtProcess (${ArtCores}-core)" - IFS=',' read -r -a file <<< "${ArtInFile}" - file=${file[${ArtProcess}]} - _jobList="'../$file'" - echo "ART running over $_jobList" - jobList="ARTConfig=[$_jobList]" - if [ $RUNATHENA -eq 1 ]; then - ARGS="$jobList;EventMax=2000;PdgId=15;TauDoubletFilter=False;TauRoiParam=True" - runathena "$ArtProcess" - fi - exit 0 # this thread is done - ;; - esac - -else - - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; echo "ART running over $git" ; done - -fi - - -if [ $RUNATHENA -eq 1 ]; then - -if [ $FORK -eq 1 ]; then - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=2000;PdgId=15;TauDoubletFilter=False;TauRoiParam=True" - -# echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - -[ -e topp.log ] && rm topp.log - -echo -e "\nUID PID PPID C SZ RSS PSR STIME TTY TIME CMD" >> topp.log -ps -aF --pid $PPROCS | grep $USER | grep -v grep | grep -v sed | sed 's| [^[:space:]]*/python | python |g' | sed 's| [^[:space:]]*/athena| athena|g' | sed 's|ARTConfig=.* |ARTConfig=... |g' | sed 's|eos/[^[:space:]]*/trigindet|eos/.../trigindet|g' >> topp.log - -echo >> topp.log - -sleep 20 - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\n" - -timestamp "waiting on athena jobs ..." - -# echo -e "\n\nwaiting on athena jobs...\n" - -waitonallproc - -fi - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -grep ERROR athena-*/ath*.log | grep -v ERRORS | grep -v MATRIX | grep -v DERIV &> errors.log -grep WARNING athena-*/ath*.log &> warnings.log - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root &> hadd.log - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do if [ -e $git ]; then ln -s $git TrkNtuple-0000.root ; break ; fi ; done - -[ -e TrkNtuple-0000.root ] || echo "WARNING: all athena stages failed" - -fi - - -ls -lt - - -get_files -data TIDAdata11-rtt.dat -get_files -data TIDAdata_cuts.dat -get_files -data TIDAdata_chains.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - -if [ $RUNATHENA -eq 1 -o $RUNPOST -eq 1 ]; then - - -TIDArdict TIDAdata11-rtt.dat -f data-tau-IBL.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -fi - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_IDTrig HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_Tau_IDTrig_forID3 -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh data-tau-IBL.root data-tau_IBL_pu46-reference.root HLT_tau25_idperf_track_InDetTrigTrackingxAODCnv_Tau_FTF HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauCore_FTF_forID1 HLT_tau25_idperf_tracktwo_InDetTrigTrackingxAODCnv_TauIso_FTF_forID3 -d HLTL2-plots 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_5.log -echo "art-result: $? TIDArun_5" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_6.log -echo "art-result: $? RunTrigCostD3PD_6" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_10.log -echo "art-result: $? TIDAcpucost_10" - - - - -timestamp "TIDAcpucost" - - - -printf "${RED}done: $SECONDS seconds${NC}\n" - - -printf "${RED}done: job duration: $(converttime $SECONDS)${NC}\n" - diff --git a/Trigger/TrigValidation/TrigInDetValidation/test/trigindetvalidation_cosmic.sh b/Trigger/TrigValidation/TrigInDetValidation/test/trigindetvalidation_cosmic.sh deleted file mode 100755 index f32f905b814..00000000000 --- a/Trigger/TrigValidation/TrigInDetValidation/test/trigindetvalidation_cosmic.sh +++ /dev/null @@ -1,329 +0,0 @@ -#!/bin/bash -# art-description: art job for cosmic -# art-type: grid -# art-output: pid -# art-output: HLTEF-plots -# art-output: times -# art-output: times-FTF -# art-output: cost-perCall -# art-output: cost-perEvent -# art-output: cost-perCall-chain -# art-output: cost-perEvent-chain -# art-input: mc14_cos.108865.CosSimPixVolSolOnTorOn.recon.RDO.s2111_r6258_tid05313290_00 -# art-output: *.dat -# art-output: *.root -# art-output: *.log -# art-output: output-dataset -# art-output: output-cost -# art-output: output-logs -# art-input-nfiles: -# art-ncores: 8 - - - -RED='\033[0;31m' -NC='\033[0m' - -function timestamp { printf "${RED}Timestamp: $1 $(date)${NC}\n" ; } - - -# stuff to control running concurrent -# processes - -PROCESS=0 -BGPROCS= -NJOBS=8 -WAITING=60 - -# is this needed ? or not ? -# export ATHENA_NUM_PROC=$NJOBS -# export ATHENA_NUM_PROC=1 - -unset ATHENA_NUM_PROC -unset ATHENA_PROC_NUMBER - -env | grep ATHENA_NUM_PROC -env | grep ATHENA_PROC_NUMBER - - -# count how many jobs are running ... - -function running { - local IRUNNING=0 - for BGPROC in $BGPROCS; do ( ps -p $BGPROC &> /dev/null ) && ((IRUNNING++)) ; done - echo $IRUNNING -} - - -# wait for an empty slot ... - -function waitonproc { - echo "waitonproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -ge $NJOBS ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - -# wait for all processes to finish ... - -function waitonallproc { - echo "waitonallproc waiting ..." - local IRUNNING=$(running) - while [ $IRUNNING -gt 0 ]; do sleep $WAITING ; IRUNNING=$(running) ; done -} - - - -# run athena - -iathena=0 - -function runathena { - timestamp "runathena:" - - if [ $# -gt 0 ]; then - - echo "input $1" - - pwd - - local DIR=$PWD - - mkdir -p athena-$1 - cd athena-$1 - cp ../*.py . - - pwd - echo "ARGS: $ARGS" - echo -e "\nrunning athena in athena-$1\n" - athena.py -c "$ARGS" TrigInDetValidation/TrigInDetValidation_RTT_topOptions_CosmicSlice.py &> athena-local-$1.log - echo "art-result: $? athena_$iathena" - - ((iathena++)) - - pwd - ls -lt - - cd $DIR - - fi - - timestamp "runathena done:" -} - - - -# function to save the output from the separate -# athena runs - -function saveoutput { - [ $# -lt 2 ] && echo "not enough arguments for saveoutput" && exit -1 - mkdir -p $1 - for git in athena-*/$2 ; do - FILE=$(echo $git | sed 's|/|-|g') - cp $git $1/$FILE - done -} - - - - - - - - -export RTTJOBNAME=TrigInDetValidation_cosmic - -jobList= - -if [ $# -gt 0 -a "x$1" == "x--local" ]; then - jobList="XMLDataSet='$RTTJOBNAME'" -else - fileList="['${ArtInFile//,/', '}']" - _jobList="'../${ArtInFile//,/' '../}'" - echo "List of files = $fileList" - for git in $_jobList ; do jobList="$jobList ARTConfig=[$git]" ; done -fi - -get_files -jo TrigInDetValidation/TrigInDetValidation_RTT_topOptions_CosmicSlice.py - - -# run athena in separate directories - - -# loop over the inputs spawning the separate athena jobs - -i=0 - -for git in $jobList ; do - - ARGS="$git;EventMax=;" - - echo "ARGS: $ARGS" - - waitonproc - - timestamp "forking athena index: $i" - - runathena $i & - PID=$! - BGPROCS="$BGPROCS $PID" - - ((i++)) - -done - - -# wait for all the jobs to finish - -sleep 10 - - -PPROCS=$BGPROCS - -for PID in $BGPROCS ; do - for CHILD in $(ps --ppid $PID -o pid h) ; do PPROCS="$PPROCS $CHILD" ; done -done - - -ps -aF --pid $PPROCS -ps -aF --pid $PPROCS >> topp.log - -echo >> topp.log - -top -b -n1 > top.log -grep PID top.log >> topp.log -for PID in $PPROCS ; do grep $PID top.log >> topp.log ; done - -cat topp.log - -# cat topp.log | mail sutt@cern.ch - - -echo -e "\n\nwaiting on athena jobs\n" - -waitonallproc - -echo "all done ! hooray !" - -timestamp "forking all athena jobs completed" - - -# jobs have all finished, -# copy the output from the athena run subdirectories to -# single output data set directories etc - -saveoutput output-dataset "Trk*.root" -saveoutput output-logs "ath*.log" -saveoutput output-cost trig_cost.root - -timestamp "combining expert histograms ..." - -# add the expert timing histgrams -hadd expert-monitoring.root athena-*/expert-monitoring.root - -# this is a hack to get the script to work without needing to fix -# TIDArun-art.sh which expects TrkNtple-0000.root -# so if we run multiple jobs then we don;t get a Trk*.root -# file to the check will fail. This creates a link so this -# test will pass - -for git in output-dataset/*.root ; do ln -s $git TrkNtuple-0000.root ; break ; done - -ls -lt - - -get_files -data TIDAcosmic-rtt.dat -get_files -data TIDAbeam.dat -get_files -data Test_bin.dat - -for DATFILE in *.dat ; do - if ( grep -q DataFile $DATFILE ); then - mv $DATFILE $DATFILE.bak - grep -v "\(DataSets\|DataFiles\)" $DATFILE.bak > $DATFILE - echo "DataSets = { \"./output-dataset/\" };" >> $DATFILE - fi -done - - -TIDArdict TIDAcosmic-rtt.dat -f data-cosmic.root -b Test_bin.dat 2>&1 | tee TIDArdict_1.log -echo "art-result: $? TIDArdict_1" - - - -timestamp "TIDArdict" - - - -TIDArun-art.sh data-cosmic.root data-cosmic-reference.root HLT_id_cosmic_InDetTrigTrackingxAODCnvIOTRT_CosmicsN_EFID -d HLTEF-plots 2>&1 | tee TIDArun_2.log -echo "art-result: $? TIDArun_2" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -o times 2>&1 | tee TIDArun_3.log -echo "art-result: $? TIDArun_3" - - - -timestamp "TIDArun-art.sh" - - - -TIDArun-art.sh expert-monitoring.root expert-monitoring*-ref.root --auto -p FastTrack -o times-FTF 2>&1 | tee TIDArun_4.log -echo "art-result: $? TIDArun_4" - - - -timestamp "TIDArun-art.sh" - - - -RunTrigCostD3PD --files output-cost/*trig_cost.root --outputTagFromAthena --costMode --linkOutputDir 2>&1 | tee RunTrigCostD3PD_5.log -echo "art-result: $? RunTrigCostD3PD_5" - - - -timestamp "RunTrigCostD3PD" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall --auto -d "/Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_6.log -echo "art-result: $? TIDAcpucost_6" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent --auto -d "/Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_7.log -echo "art-result: $? TIDAcpucost_7" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perCall-chain --auto -d "/Chain_Algorithm" -p "_Time_perCall" 2>&1 | tee TIDAcpucost_8.log -echo "art-result: $? TIDAcpucost_8" - - - -timestamp "TIDAcpucost" - - - -TIDAcpucost costMon/TrigCostRoot_Results.root costMon/TrigCostRoot_Results.root -o cost-perEvent-chain --auto -d "/Chain_Algorithm" -p "_Time_perEvent" 2>&1 | tee TIDAcpucost_9.log -echo "art-result: $? TIDAcpucost_9" - - - - -timestamp "TIDAcpucost" - - diff --git a/Trigger/TrigValidation/TrigInDetValidation/utils/mkart-template b/Trigger/TrigValidation/TrigInDetValidation/utils/mkart-template index 5fd4a548c16..3529b81eb07 100755 --- a/Trigger/TrigValidation/TrigInDetValidation/utils/mkart-template +++ b/Trigger/TrigValidation/TrigInDetValidation/utils/mkart-template @@ -8,7 +8,7 @@ function usage { echo " -n, --noglob do not overwrite existing files" echo " -d, --dataset FILE run on FILE" echo " -c, --cleanup clean up afterwards" - echo " -a, --art create test scripts for official ART" + echo " -a, --art create test scripts with _art suffix" echo " -h, --help this help" if [ $# -lt 1 ]; then exit 0; fi ; exit $1 } @@ -120,12 +120,10 @@ for xmlfile in $FILES; do echo "#!/bin/bash" > $outfile echo "# art-description: art job for $(echo $_file | sed 's|TrigInDetValidation_||g')" >> $outfile - if [ $ART -eq 1 ]; then - echo "# art-include: 21.3/Athena" >> $outfile - echo "# art-include: 21.9/Athena" >> $outfile - echo "# art-include: master/Athena" >> $outfile - echo "# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir=" >> $outfile - fi + echo "# art-include: 21.3/Athena" >> $outfile + echo "# art-include: 21.9/Athena" >> $outfile + echo "# art-include: master/Athena" >> $outfile + echo "# art-html: https://idtrigger-val.web.cern.ch/idtrigger-val/TIDAWeb/TIDAart/?jobdir=" >> $outfile echo "# art-type: grid" >> $outfile echo "# art-input: $(basename $(dirname $DATASET))" >> $outfile echo "# art-output: *.dat " >> $outfile @@ -137,9 +135,7 @@ for xmlfile in $FILES; do echo "# art-input-nfiles: 10" >> $outfile echo "# art-cores: 8" >> $outfile - if [ $ART -eq 1 ]; then - echo "# art-memory: 4096" >> $outfile - fi + echo "# art-memory: 4096" >> $outfile echo "" >> $outfile echo "" >> $outfile @@ -179,11 +175,7 @@ for xmlfile in $FILES; do ncores=8 [ $nfiles -lt 8 ] && ncores=$nfiles - if [ $ART -eq 1 ]; then - cat $outfile | sed "s|# art-cores: .*|# art-cores: $ncores|g" >> $outfile- - else - cat $outfile | sed "s|# art-cores: .*|# art-ncores: $ncores|g" >> $outfile- - fi + cat $outfile | sed "s|# art-cores: .*|# art-cores: $ncores|g" >> $outfile- fi mv $outfile{-,} -- GitLab