From 2381480fb1811073f8f1192a07347719f368dda1 Mon Sep 17 00:00:00 2001
From: Tadej Novak <tadej.novak@cern.ch>
Date: Thu, 13 Aug 2020 11:16:01 +0200
Subject: [PATCH] Data overlay: cleanup zero bias selection scripts
---
.../scripts/0biasselection/cleanup.sh | 7 -
.../0biasselection/lbn_anal_map_splitter.py | 109 -------
.../scripts/0biasselection/lbn_analyze.C | 194 -------------
.../scripts/0biasselection/query.sh | 102 -------
.../scripts/0biasselection/run_lbn_analyze.C | 14 -
.../OverlayByteStreamUtils/CMakeLists.txt | 10 +-
.../scripts/LB_AnalMapSplitter.py | 120 ++++++++
.../scripts/LB_GetNEvents.py | 39 +--
.../scripts/LB_query.sh | 152 ++++++++++
.../scripts/LB_query_DBonly.sh | 20 ++
.../util/LB_analyze.cxx | 271 ++++++++++++++++++
11 files changed, 592 insertions(+), 446 deletions(-)
delete mode 100755 Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/cleanup.sh
delete mode 100644 Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py
delete mode 100644 Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_analyze.C
delete mode 100755 Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh
delete mode 100644 Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/run_lbn_analyze.C
create mode 100755 Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_AnalMapSplitter.py
rename Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/GetNEventsLB.py => Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_GetNEvents.py (71%)
mode change 100644 => 100755
create mode 100755 Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query.sh
create mode 100755 Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query_DBonly.sh
create mode 100644 Simulation/Overlay/OverlayByteStreamUtils/util/LB_analyze.cxx
diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/cleanup.sh b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/cleanup.sh
deleted file mode 100755
index a85b4571ffe..00000000000
--- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/cleanup.sh
+++ /dev/null
@@ -1,7 +0,0 @@
- rm -rf output_stream*
- rm lbn_anal_map_stream*
- rm lbnevents_*
- rm runs_lbn_files.txt
- rm log_lbn_anal*
- rm lbn_anal_map.txt
-
diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py
deleted file mode 100644
index 4aa40c66b17..00000000000
--- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
-
-import sys
-
-#get the list of special run,lbn with more than 1 file
-special_run_lbn = {}
-with open("runs_lbn_files.txt") as f:
- for line in f:
- #print line
- if not line.startswith("root"):
- print "WARNING: line doesn't start with 'root'"
- print line
- continue
- filenm=line.rstrip()
- print filenm
- finfo=filenm.split(".")
- run=finfo[3]
- lbi=4 #index of the lbn info
- while "_lb" not in finfo[lbi] and lbi<20:
- lbi=lbi+1
- if lbi>=20:
- print finfo[lbi], "doesnt contain _lb"
- exit(1)
- lbn=finfo[lbi].replace("_lb","")
- print run+"_"+lbn
- if run+"_"+lbn in special_run_lbn:
- if filenm in special_run_lbn[run+"_"+lbn]:
- print filenm, "already in map for ",run,lbn
- else:
- special_run_lbn[run+"_"+lbn].append(filenm)
- else:
- special_run_lbn[run+"_"+lbn]=[filenm]
- for key in special_run_lbn.keys():
- #if len(special_run_lbn[key])>1: print special_run_lbn[key]
- pass
- print "done with special_run_lbn"
-
-#do the work
-maxstream = int(sys.argv[1])
-print "maxstream=",maxstream
-for s in range(0,maxstream+1): #50
- with open("lbn_anal_map_stream"+str(s)+".txt") as f:
- tot=0
- nfile=1
- of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w")
- of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w")
- for line in f:
- linfo=line.split(" ")
- #print line
- #print linfo[16]
- tot+=int(linfo[16])
- run=linfo[3].replace(",","").zfill(8)
- lbn=linfo[5].replace(",","").zfill(4)
- if tot>100:
- #replace last bit of line and also write it to the new file
- extra=tot-100
- forthisone=int(linfo[16])-extra
- rline=line.replace(linfo[16]+" wanted",str(forthisone)+" wanted")
- of.write(rline)
- if run+"_"+lbn in special_run_lbn:
- for f in special_run_lbn[run+"_"+lbn]:
- #print of2.tell()
- if of2.tell()>0: of2.write(",")
- of2.write(f)
- else:
- print run,lbn," not in files map!"
- #print line, rline
- #print run, lbn
-
- of.close(); of2.close(); nfile+=1
- of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w")
- of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w")
- tot=extra #the number left over from the last set
-
- rline=line.replace(linfo[16]+" wanted",str(extra)+" wanted")
- of.write(rline)
- if run+"_"+lbn in special_run_lbn:
- for f in special_run_lbn[run+"_"+lbn]:
- if of2.tell()>0: of2.write(",")
- of2.write(f)
- else:
- print run,lbn," not in files map!"
-
- elif tot==100:
- of.write(line)
- if run+"_"+lbn in special_run_lbn:
- for f in special_run_lbn[run+"_"+lbn]:
- if of2.tell()>0: of2.write(",")
- of2.write(f)
- else:
- print run,lbn," not in files map!"
-
- of.close(); of2.close(); nfile+=1
- of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w")
- of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w")
- tot=0
- else:
- of.write(line)
- if run+"_"+lbn in special_run_lbn:
- for f in special_run_lbn[run+"_"+lbn]:
- #print of2.tell()
- if of2.tell()>0: of2.write(",")
- of2.write(f)
- else:
- print run,lbn," not in files map!"
-
- print "done ",s
- of.close(); of2.close()
-
diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_analyze.C b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_analyze.C
deleted file mode 100644
index 6a16bff3c6c..00000000000
--- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_analyze.C
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
-*/
-
-#include <map>
-#include "TRandom3.h"
-
-class lbn_info{
-public:
- lbn_info(){nevt=0; intlumi=0; nevtwanted=0;}
- int nevt;
- float intlumi;
- int nevtwanted;
-};
-
-class run_info{
-public:
- run_info(){nevt=0; intlumi=0; nevtwanted=0;}
- int nevt;
- float intlumi;
- std::map<int,lbn_info> lbnmap;
- int nevtwanted;
-};
-
-FILE *outfile=fopen("lbn_anal_map.txt","w");
-int debug=1;
-
-std::map< int, int > eventcounts;
-void readeventcounts(int run){
- eventcounts.clear();
- char buf[50];
- sprintf(buf,"lbnevents_%d.txt",run);
- if (debug) printf("Opening %s, eventcounts size is %lu\n",buf,eventcounts.size());
- FILE *fp = fopen(buf,"r");
- if (!fp) printf("Failed to open %s!!!\n",buf);
- char *line=new char[500];
- while (fgets(line,480,fp)) {
- int lbn,ne,nf,runn;
- float mb;
- int s=sscanf(&line[0]," ... Run %d, LB %d has %d events, %d RAW files and %f MB",&runn,&lbn,&ne,&nf,&mb);
- if (s>4){
- assert(run==runn);
- if (debug) printf("run %d lbn %d has %d events\n",run,lbn,ne);
- eventcounts[lbn]=ne;
- }
- else {
- printf("s=%d, bad read?\n",s);
- assert(false);
- }
- }
- if (debug) printf("Closing %s, eventcounts size is %lu\n",buf,eventcounts.size());
- fclose(fp);
-}
-
-void lbn_analyze(int stream, int nwanted)
-{
- FILE *fp = fopen("lbn","r");
- if (!fp) printf("Failed to open lbn input file!!!\n");
- char *line=new char[500];
- int run,lbn,L1Acc,beforeps,afterps,valid; //L1p,L2p,L3p
- int liveL1,livebp,liveap;
- float instlumi,dt,avevtperbx,live,L1p,HLTp,LARp;
- //float intlumi,intlumitrig;
- int grun=0;
- float gtotaltotallumi=0,gtotaltotallumiprescaled=0;
- float gtotallumi=0, gtotallumiprescaled=0;
-
- TRandom myrand;
- myrand.SetSeed(0);
- printf("- myrand seed is %u\n",myrand.GetSeed());
- myrand.SetSeed(myrand.Integer(10000000)+stream);
- printf("- myrand seed is now %u\n",myrand.GetSeed());
-
- std::map<int,run_info> runmap;
-
- while (fgets(line,480,fp)) {
- if (line[0]!='-') continue;
-
-// int s=sscanf(&line[0],"--- LumiCalculator : %d[%d]: L1Acc: %d, Livetime trigger L1Acc: %d, InstLumi: %f, deltaT: %f, AvEvtsPerBX: %f, BeforePrescale: %d, AfterPrescale: %d, Livetime trigger BeforePrescale: %d Livetime trigger AfterPrescale: %d, Livefrac: %f, L1Presc: %d, L2Presc: %d, L3Presc: %d, Valid: %d", &run,&lbn,&L1Acc,&liveL1,&instlumi,&dt,&avevtperbx,&beforeps,&afterps,&livebp,&liveap,&live,&L1p,&L2p,&L3p,&valid);
- int s=sscanf(&line[0],"--- LumiCalculator : %d[%d]: L1Acc: %d, Livetime trigger L1Acc: %d, InstLumi: %f, deltaT: %f, AvEvtsPerBX: %f, BeforePrescale: %d, AfterPrescale: %d, Livetime trigger BeforePrescale: %d Livetime trigger AfterPrescale: %d, Livefrac: %f, L1Presc: %f, HLTPresc: %f, Valid: %d, LAr ready fraction: %f", &run,&lbn,&L1Acc,&liveL1,&instlumi,&dt,&avevtperbx,&beforeps,&afterps,&livebp,&liveap,&live,&L1p,&HLTp,&valid,&LARp);
-
- if (s>8){
- if (debug) printf("Got %d values : run=%d, lbn=%d, L1Acc=%d, instlumi=%f, L1p=%f, dt=%f, live=%f, afterps=%d \n",s,run,lbn,L1Acc,instlumi,L1p,dt,live,afterps);
-
- if (run!=grun){
- if (grun>0){//change of run
- runmap[grun].intlumi=gtotallumi; printf("Setting lumi for run %d to %f\n",grun,gtotallumi);
- if (debug) printf("Adding to totaltotal lumis\n");
- gtotaltotallumi+=gtotallumi; gtotaltotallumiprescaled+=gtotallumiprescaled;
- gtotallumi=0;gtotallumiprescaled=0;//reset the int lumi calculation for this run
- //runmap[grun].nevt=0;
- }
- grun=run;//save the run number
- printf("Setting grun to %d\n",run);
-
- //read in event counts from runquery file for run
- readeventcounts(run);
-
- }//new run
-
- if (L1p<0) {
- runmap[run].lbnmap[lbn].nevt=0;
- if (eventcounts[lbn]!=0) printf("For lbn %d, L1p<0 but eventcounts is %d\n",lbn,eventcounts[lbn]);
- }
- else{
- int ne = L1Acc;//afterps
- if (ne!=eventcounts[lbn]){
- if (0==stream) printf("For lbn %d, ne from lumicalc is %d but we will trust runquery value of %d\n",lbn,ne,eventcounts[lbn]);
- ne=eventcounts[lbn];
- }
- if (0==ne){dt=0; printf("For lbn %d, setting lumi to 0 since 0 events were recorded\n",lbn);}
- runmap[run].lbnmap[lbn].nevt=ne;
- runmap[run].lbnmap[lbn].intlumi=instlumi*dt*live;
- runmap[run].nevt+=ne;
- gtotallumiprescaled+= instlumi*dt*live/L1p;
- gtotallumi+= instlumi*dt*live;
- }
- if (debug) printf("grun=%d, gtotallumi=%f, gtotallumiprescaled=%f\n",grun,gtotallumi,gtotallumiprescaled);
- }//good line
- }//loop over lines in file
-
- //after last run
- runmap[grun].intlumi=gtotallumi; printf("Setting lumi for run %d to %f\n",grun,gtotallumi);
- if (debug) printf("Adding to totaltotal lumis for last run\n");
- gtotaltotallumi+=gtotallumi; gtotaltotallumiprescaled+=gtotallumiprescaled;
-
- fclose(fp);
- printf("- %lu runs, gtotaltotallumi=%f, gtotaltotallumiprescaled=%f\n",runmap.size(),gtotaltotallumi,gtotaltotallumiprescaled);
-
- if (runmap.size()<1) {printf("- runmap size is %lu, quitting!\n",runmap.size()); return;}
- //return;
-
- //check the total lumi...
- double tempr=0;
- for (std::map<int,run_info>::const_iterator r=runmap.begin(); r!=runmap.end(); ++r){
- tempr += r->second.intlumi;
- }
- if (fabs(tempr-gtotaltotallumi)/tempr>0.001){
- printf("tempr=%f and gtotaltotallumi=%f\n",tempr,gtotaltotallumi);
- assert(false);
- }
-
- for (int e=0;e<nwanted;++e){
-
- //pick a random run, proportional to intlumi
- double rnd = myrand.Uniform(tempr);
- run=-1;
- for (std::map<int,run_info>::const_iterator r=runmap.begin(); r!=runmap.end(); ++r){
- if (rnd<r->second.intlumi) {run=r->first; break;}
- rnd -= r->second.intlumi;
- }
- assert(run>=0);
-
- //check the total run lumi...
- double tempt=0;
- for (std::map<int,lbn_info>::const_iterator l=runmap[run].lbnmap.begin(); l!=runmap[run].lbnmap.end(); ++l){
- tempt += l->second.intlumi;
- }
- if (fabs(tempt-runmap[run].intlumi)/tempt>0.001){
- printf("tempt=%f and runmap[%d].intlumi=%f\n",tempt,run,runmap[run].intlumi);
- assert(false);
- }
-
- //pick a random lbn, proportional to intlumi
- rnd = myrand.Uniform(tempt);
- lbn=-1;
- for (std::map<int,lbn_info>::const_iterator l=runmap[run].lbnmap.begin(); l!=runmap[run].lbnmap.end(); ++l){
- if (rnd<l->second.intlumi) {lbn=l->first; break;}
- rnd -= l->second.intlumi;
- }
- assert(lbn>=0);
-
- runmap[run].nevtwanted++; runmap[run].lbnmap[lbn].nevtwanted++;
- if (debug) printf("- stream %d, run %d, lbn %d, choose %d of out %d\n",stream,run,lbn,runmap[run].lbnmap[lbn].nevtwanted,runmap[run].lbnmap[lbn].nevt);
-
- }//loop over nwanted
-
- for (std::map<int,run_info>::const_iterator r=runmap.begin(); r!=runmap.end(); ++r){
- int totnevt=0; float totintlumi=0;
- printf("stream %d, run %d, has %d events and %f/ub, %f intlumi of total, and %d wanted\n", stream,
- r->first,r->second.nevt,r->second.intlumi,r->second.intlumi/gtotaltotallumi,r->second.nevtwanted);
- for (std::map<int,lbn_info>::const_iterator l=r->second.lbnmap.begin(); l!=r->second.lbnmap.end(); ++l){
- fprintf(outfile,"stream %d, run %d, lbn %d, has %d events and %f/ub, %f intlumi of run, and %d wanted", stream, r->first,
- l->first,l->second.nevt,l->second.intlumi,l->second.intlumi/r->second.intlumi,l->second.nevtwanted);
- if (l->second.nevtwanted>l->second.nevt) fprintf(outfile," : WARNING, more than available, will be duplicates!\n"); else fprintf(outfile,"\n");
- totnevt+=l->second.nevt; totintlumi+=l->second.intlumi;
- }
- if (totnevt!=r->second.nevt) printf(" XXX events do not agree !!! \n");
- if (totintlumi>0.0 && fabs(totintlumi - r->second.intlumi)/totintlumi>.001) printf(" XXX intlumi does not agree !!! %f %f \n",totintlumi,r->second.intlumi);
-
- }
- printf("--\n\n");
-}
-
diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh
deleted file mode 100755
index f59b17565f6..00000000000
--- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/bin/bash
-
-#setupATLAS
-#lsetup rucio
-#voms-proxy-init -voms atlas
-
-#use GRL, from: http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/?C=M;O=D
-#consult: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/GoodRunListsForAnalysisRun2
-#and: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/DataMCForAnalysis
-
-#dq="data15_13TeV.periodAllYear_DetStatus-v73-pro19-08_DQDefects-00-01-02_PHYS_StandardGRL_All_Good.xml"
-#wget -nc http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/$dq
-
-#Can just go here: https://atlas-lumicalc.cern.ch/ ... note use --online lumi tag for HI, unless a better one is available
-#and request --verbose output and use physics trigger HLT_noalg_zb_L1ZB,
-#then wget "Raw iLimuiCalc.exe output", and "mv output.txt lbn_2015; ln -s lbn_2015 lbn"
-#rm -f lbn # if you want to redo this
-if [ ! -f "lbn" ]; then
-iLumiCalc.exe -t L1_ZB -V -x $dq --livetrigger=L1_EM12 > lbn
-fi
-grep ": Total" lbn ; echo
-
-#get info on number of events and files per lumi block in each run
-#rm -f lbnevents* # if you want to redo this
-#rm -f runs_lbn_files.txt # if you want to redo this
-#sstream="physics_ZeroBias" #pp or UPC
-sstream="physics_MinBiasOverlay" #HI
-#suf="merge"
-suf="daq"
-rm -f runs_temp.txt
-for runn in `grep "subchild node value" lbn|cut -f 3 -d : |sed 's/[\x01-\x1F\x7F]//g'|sed 's%\[0m%%g'|sed 's%\[34m%%g'|sed ':a;N;$!ba;s/\n/ /g' `; do
- if [ ! -f lbnevents_${runn}.txt ]; then
- echo "Getting Nevent info for run $runn ";
- python GetNEventsLB.py -r $runn -s $sstream |grep "... Run" > lbnevents_${runn}.txt;# Only works on lxplus now!
- fi
- if [ ! -f runs_lbn_files.txt ]; then
- echo "Getting file info for run $runn "
- rucio list-file-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW | grep "CERN-PROD_TZDISK" | cut -d '|' -f 6|sed "s% CERN-PROD_TZDISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
- cat runs_temp_temp.txt >> runs_temp.txt
- echo -n "Replicated? : "; rucio list-dataset-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW|grep CERN-PROD_TZDISK
- nfound=`grep -c data runs_temp_temp.txt`
- echo "Found $nfound files on TZDISK"
- if [ $nfound -eq 0 ]; then #look also on DATADISK if not found on TZDISK
- echo "Getting file info for run $runn "
- rucio list-file-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW | grep "CERN-PROD_DATADISK" | cut -d '|' -f 6|sed "s% CERN-PROD_DATADISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
- cat runs_temp_temp.txt >> runs_temp.txt
- echo -n "Replicated? : "; rucio list-dataset-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW|grep CERN-PROD_DATADISK
- nfound=`grep -c data runs_temp_temp.txt`
- echo "Found $nfound files on DATADISK"
- fi
- fi
-done #loop over all runs
-rm runs_temp_temp.txt
-if [ ! -f runs_lbn_files.txt ]; then mv runs_temp.txt runs_lbn_files.txt ; fi
-
-#rm -f lbn_anal_map.txt # if you want to redo this
-if [ ! -f lbn_anal_map.txt ]; then root -l -b -q run_lbn_analyze.C > log_lbn_analyze.txt ; fi
-
-echo -n "Total events in dataset before GRL: "
-cat lbnevents_*.txt |tr -s ' ' | cut -d ' ' -f 8 |awk '{total = total + $1}END{print total}'
-echo -n "Total events in dataset from log_lbn_analyze.txt: "
-grep "stream 0" log_lbn_analyze.txt|cut -d ' ' -f 6 |awk '{total = total + $1}END{print total}'
-echo -n "Total events in dataset from lbn_anal_map: "
-grep "stream 1," lbn_anal_map.txt |cut -d ' ' -f 8 |awk '{total = total + $1}END{print total}'
-echo -n "Selected events per stream: "
-grep "stream 8," lbn_anal_map.txt |cut -d ' ' -f 17 |awk '{total = total + $1}END{print total}'
-
-maxstream=19 #0 up to 49 - 19 does 1M events
-#split into all the desired streams
-for s in $(seq 0 $maxstream); do
- grep "stream ${s}," lbn_anal_map.txt | grep -v "and 0 wanted" > lbn_anal_map_stream${s}.txt
- mkdir -p output_stream${s}
-done
-
-#split into jobs for each stream and make filelist and lbn_anal_map files
-rm -f output_stream*/filelist_*.txt output_stream*/lbn_anal_map_*.txt
-python lbn_anal_map_splitter.py $maxstream > log_lbn_anal_map_splitter.txt
-rm output_stream*/*501.txt
-grep "doesnt contain" log_lbn_anal_map_splitter.txt
-if [ $? == 0 ]; then exit; fi
-grep -c "not in files map" log_lbn_anal_map_splitter.txt
-#if [ $? == 0 ]; then exit; fi
-echo "$? files not found in map..."
-
-for s in $(seq 1 $maxstream); do
- echo "renaming stream $s to 0"
- for f in {1..500}; do sed -i -e "s%stream $s,%stream 0,%g" output_stream${s}/lbn_anal_map_${f}.txt; done
-done
-
-it=2016_hip8TeV_1 #just a name to tag this set of files
-for s in $(seq 0 $maxstream); do cd output_stream${s}; tar cfz stream${s}_${it}.tar.gz *.txt; cd ..; done
-mv -v output_stream*/stream*_${it}.tar.gz ~/public/overlay/lists/
-
-#to check the events in each map:
-for s in $(seq 0 $maxstream); do
- echo -n "files with 100 events in stream $s :"
- for f in {1..500}; do cat output_stream${s}/lbn_anal_map_$f.txt | cut -d ' ' -f 17 |awk '{total = total + $1}END{print total}'; done |grep -c 100
-done
-
-#to show number of files in each filelist:
-echo ; echo "files in each filelist (in stream 0)..." ; for f in output_stream0/filelist_*.txt; do echo -n `grep -o $sstream $f | wc -l`; echo " : $f"; done |sort -n
-
diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/run_lbn_analyze.C b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/run_lbn_analyze.C
deleted file mode 100644
index 608315cc64c..00000000000
--- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/run_lbn_analyze.C
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
-*/
-
-{
- gROOT->ProcessLine(".L lbn_analyze.C+");
-
- for (int i=0; i<50; ++i){//number of zerobias streams to make
- //gSystem->Sleep(2000);//to get new random number seed
- lbn_analyze(i,50000);//number of events per zerobias stream
- }
-
-}
-
diff --git a/Simulation/Overlay/OverlayByteStreamUtils/CMakeLists.txt b/Simulation/Overlay/OverlayByteStreamUtils/CMakeLists.txt
index 87f20348895..9ddead6f5fd 100644
--- a/Simulation/Overlay/OverlayByteStreamUtils/CMakeLists.txt
+++ b/Simulation/Overlay/OverlayByteStreamUtils/CMakeLists.txt
@@ -11,4 +11,12 @@ atlas_add_component( OverlayByteStreamUtils
src/*.cxx
src/components/*.cxx
INCLUDE_DIRS ${ROOT_INCLUDE_DIRS}
- LINK_LIBRARIES LINK_LIBRARIES ${ROOT_LIBRARIES} AthenaBaseComps AthenaKernel ByteStreamCnvSvcLib GaudiKernel StoreGateLib TrigConfHLTData TrigConfInterfaces TrigSteeringEvent TrigT1Result )
+ LINK_LIBRARIES ${ROOT_LIBRARIES} AthenaBaseComps AthenaKernel ByteStreamCnvSvcLib GaudiKernel StoreGateLib TrigConfHLTData TrigConfInterfaces TrigSteeringEvent TrigT1Result )
+
+# Utility scripts
+atlas_install_runtime( scripts/*.py )
+atlas_install_runtime( scripts/*.sh )
+
+atlas_add_executable (LB_analyze util/LB_analyze.cxx
+ INCLUDE_DIRS ${ROOT_INCLUDE_DIRS}
+ LINK_LIBRARIES ${ROOT_LIBRARIES} )
diff --git a/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_AnalMapSplitter.py b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_AnalMapSplitter.py
new file mode 100755
index 00000000000..6c451453b65
--- /dev/null
+++ b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_AnalMapSplitter.py
@@ -0,0 +1,120 @@
+#!/bin/env python
+#
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+from __future__ import print_function
+
+import sys
+
+# get the list of special run,lbn with more than 1 file
+special_run_lbn = {}
+with open("runs_lbn_files.txt") as f:
+ for line in f:
+ # print line
+ if not line.startswith("root"):
+ print("WARNING: line doesn't start with 'root'")
+ print(line)
+ continue
+ fileName = line.rstrip()
+ print(fileName)
+ finfo = fileName.split(".")
+ run = finfo[3]
+ lbi = 4 # index of the lbn info
+ while "_lb" not in finfo[lbi] and lbi < 20:
+ lbi += 1
+ if lbi >= 20:
+ print(finfo[lbi] + " doesnt contain _lb")
+ exit(1)
+ lbn = finfo[lbi].replace("_lb", "")
+ print(run+"_"+lbn)
+ if run+"_"+lbn in special_run_lbn:
+ if fileName in special_run_lbn[run+"_"+lbn]:
+ print("%s already in map for %s %s" % (fileName, run, lbn))
+ else:
+ special_run_lbn[run+"_"+lbn].append(fileName)
+ else:
+ special_run_lbn[run+"_"+lbn] = [fileName]
+ for key in special_run_lbn.keys():
+ pass
+ print("done with special_run_lbn")
+
+# do the work
+maxstream = int(sys.argv[1])
+print("maxstream=%d" % maxstream)
+for s in range(0, maxstream+1): # 50
+ with open("lbn_anal_map_stream"+str(s)+".txt") as f:
+ tot = 0
+ nfile = 1
+ of = open("output_stream"+str(s) +
+ "/lbn_anal_map_"+str(nfile)+".txt", "w")
+ of2 = open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt", "w")
+ for line in f:
+ linfo = line.split(" ")
+ tot += int(linfo[16])
+ run = linfo[3].replace(",", "").zfill(8)
+ lbn = linfo[5].replace(",", "").zfill(4)
+ if tot > 100:
+ # replace last bit of line and also write it to the new file
+ extra = tot-100
+ forthisone = int(linfo[16])-extra
+ rline = line.replace(
+ linfo[16]+" wanted", str(forthisone)+" wanted")
+ of.write(rline)
+ if run+"_"+lbn in special_run_lbn:
+ for f in special_run_lbn[run+"_"+lbn]:
+ if of2.tell() > 0:
+ of2.write(",")
+ of2.write(f)
+ else:
+ print("%s %s not in files map!" % (run, lbn))
+
+ of.close()
+ of2.close()
+ nfile += 1
+ of = open("output_stream"+str(s) +
+ "/lbn_anal_map_"+str(nfile)+".txt", "w")
+ of2 = open("output_stream"+str(s) +
+ "/filelist_"+str(nfile)+".txt", "w")
+ tot = extra # the number left over from the last set
+
+ rline = line.replace(linfo[16]+" wanted", str(extra)+" wanted")
+ of.write(rline)
+ if run+"_"+lbn in special_run_lbn:
+ for f in special_run_lbn[run+"_"+lbn]:
+ if of2.tell() > 0:
+ of2.write(",")
+ of2.write(f)
+ else:
+ print("%s %s not in files map!" % (run, lbn))
+
+ elif tot == 100:
+ of.write(line)
+ if run+"_"+lbn in special_run_lbn:
+ for f in special_run_lbn[run+"_"+lbn]:
+ if of2.tell() > 0:
+ of2.write(",")
+ of2.write(f)
+ else:
+ print("%s %s not in files map!" % (run, lbn))
+
+ of.close()
+ of2.close()
+ nfile += 1
+ of = open("output_stream"+str(s) +
+ "/lbn_anal_map_"+str(nfile)+".txt", "w")
+ of2 = open("output_stream"+str(s) +
+ "/filelist_"+str(nfile)+".txt", "w")
+ tot = 0
+ else:
+ of.write(line)
+ if run+"_"+lbn in special_run_lbn:
+ for f in special_run_lbn[run+"_"+lbn]:
+ if of2.tell() > 0:
+ of2.write(",")
+ of2.write(f)
+ else:
+ print("%s %s not in files map!" % (run, lbn))
+
+ print("done %d" % s)
+ of.close()
+ of2.close()
diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/GetNEventsLB.py b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_GetNEvents.py
old mode 100644
new mode 100755
similarity index 71%
rename from Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/GetNEventsLB.py
rename to Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_GetNEvents.py
index 7a6c072b853..cf47e1421f2
--- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/GetNEventsLB.py
+++ b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_GetNEvents.py
@@ -1,6 +1,6 @@
#!/bin/env python
#
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# ----------------------------------------------------------------
# Script : GetNEventsLB.py
# Purpose: Utility to retrieve number of events per LB from SFO DB
@@ -8,6 +8,8 @@
# Created: Aug 17, 2011
# ----------------------------------------------------------------
+from __future__ import print_function
+
import sys, getopt
def GetSFO_LBNEvents( cursor, runno, streamname ):
@@ -26,12 +28,12 @@ def OpenSFOConnection():
return cx_Oracle.connect("ATLAS_SFO_T0_R/readmesfotz2008@atlr")
def usage():
- print ' '
- print 'Usage: python %s [options]' % sys.argv[0]
- print ' -r | --run : run number (REQUIRED)'
- print ' -s | --stream : full stream name (REQUIRED)'
- print ' -h | --help : print this usage message'
- print ' '
+ print(' ')
+ print('Usage: %s [options]' % sys.argv[0])
+ print(' -r | --run : run number (REQUIRED)')
+ print(' -s | --stream : full stream name (REQUIRED)')
+ print(' -h | --help : print this usage message')
+ print(' ')
exit(1)
def main():
@@ -43,7 +45,7 @@ def main():
except getopt.GetoptError:
# print help information and exit:
- print >> sys.stderr, 'ERROR: unknown options in argument %s' % sys.argv[1:]
+ print('ERROR: unknown options in argument %s' % sys.argv[1:], file=sys.stderr)
usage()
runno = None
@@ -62,17 +64,17 @@ def main():
connection = OpenSFOConnection()
cursor = connection.cursor()
- print 'Results for run: %i, stream: "%s"' % (runno, stream)
+ print('Results for run: %i, stream: "%s"' % (runno, stream))
# min/max LB number for given run/stream
# --> this command is slow... don't know why
# minLB, maxLB = GetSFO_LBs( cursor, runno )
# print 'LB range: %i -- %i' % (minLB, maxLB)
- # list with
+ # list with
lblist = GetSFO_LBNEvents( cursor, runno, stream )
- print 'First non-zero LB: ',lblist[0][1]
- print 'Last non-zero LB: ',lblist[-1][1]
+ print('First non-zero LB: ' + str(lblist[0][1]))
+ print('Last non-zero LB: ' + str(lblist[-1][1]))
# access to all LBs
sumnev = 0
@@ -80,18 +82,17 @@ def main():
sumfsize = 0
for (nev,lb,nfiles,fsize) in lblist:
fsize /= 1.e6
- print ' ... Run %i, LB %i has %i events, %i RAW files and %f MB' % (runno,lb,nev,nfiles,fsize)
+ print(' ... Run %i, LB %i has %i events, %i RAW files and %f MB' % (runno,lb,nev,nfiles,fsize))
sumnev += nev
sumnfiles += nfiles
sumfsize += fsize
- print '--------------------------------------------------'
- print 'Total #events : ', sumnev
- print 'Total #files : ', sumnfiles
- print 'Total RAW size: ', sumfsize/1000.0, ' GB'
-
+ print('--------------------------------------------------')
+ print('Total #events : ', sumnev)
+ print('Total #files : ', sumnfiles)
+ print('Total RAW size: ', sumfsize/1000.0, ' GB')
+
cursor.close()
connection.close()
if __name__ == '__main__':
main()
-
diff --git a/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query.sh b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query.sh
new file mode 100755
index 00000000000..fea8dac1e90
--- /dev/null
+++ b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+#
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+# NOTE: Rucio setup is required
+
+set -e
+
+# use GRL, from: http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/?C=M;O=D
+# consult: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/GoodRunListsForAnalysisRun2
+# and: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/DataMCForAnalysis
+
+# TODO: read from args
+#sstream="physics_ZeroBias" #pp or UPC
+sstream="physics_MinBiasOverlay" #HI
+pref="data18_hi"
+#suf="merge"
+suf="daq"
+# just a name to tag this set of files
+out_name=data18_hi_1
+
+# TODO: read from args
+dq="http://atlasdqm.web.cern.ch/atlasdqm/grlgen/HeavyIons/All_Good/data18_hi.periodS_DetStatus-v104-pro22-08_Unknown_PHYS_HeavyIonP_All_Good.xml"
+wget -nc "$dq"
+
+# Can just go here: https://atlas-lumicalc.cern.ch/ ... note use --online lumi tag for HI, unless a better one is available
+# and request --verbose output and use physics trigger HLT_noalg_zb_L1ZB,
+# then wget "Raw iLimuiCalc output", and "mv output.txt lbn_2015; ln -s lbn_2015 lbn"
+
+# if you want to redo this:
+# rm -f lbn
+if [ ! -f "lbn" ]; then
+ # iLumiCalc -t L1_ZB -V -x "$dq" --livetrigger=L1_EM12 > lbn
+ iLumiCalc -t L1_ZB --lumitag OflLumi-HI-009 -V -x "$dq" --livetrigger=L1_EM12 | tee lbn
+fi
+grep ": Total" lbn ; echo
+
+# get info on number of events and files per lumi block in each run
+# if you want to redo this:
+# rm -f lbnevents*
+# rm -f runs_lbn_files.txt
+
+rm -f runs_temp.txt
+runs=$(grep "subchild node value" lbn|cut -f 3 -d : |sed 's/[\x01-\x1F\x7F]//g'|sed 's%\[0m%%g'|sed 's%\[34m%%g'|sed ':a;N;$!ba;s/\n/ /g')
+# loop over all runs
+for runn in $runs; do
+ # Only works on lxplus now!
+ # if [[ ! -f "lbnevents_${runn}.txt" ]]; then
+ # echo "Getting Nevent info for run $runn "
+ # LB_GetNEvents.py -r "$runn" -s "$sstream" | grep "... Run" | tee "lbnevents_${runn}.txt"
+ # fi
+ rm -f runs_temp_temp.txt
+ touch runs_temp_temp.txt
+ if [[ ! -f "runs_lbn_files.txt" ]]; then
+ echo "Getting file info for run $runn "
+ file=$(rucio list-file-replicas "${pref}.00${runn}.${sstream}.${suf}.RAW" || true)
+ # dataset=$(rucio list-dataset-replicas "${pref}.00${runn}.${sstream}.${suf}.RAW")
+ set +e
+ rsefound=$(echo "$file" | grep -c "CERN-PROD_TZDISK")
+ if [[ ! $rsefound -eq 0 ]]; then
+ echo "$file" | (grep "CERN-PROD_TZDISK" || true) | cut -d '|' -f 6 | sed "s% CERN-PROD_TZDISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
+ cat runs_temp_temp.txt >> runs_temp.txt
+ # echo -n "Replicated? : "; (echo "$dataset" | grep CERN-PROD_TZDISK) || echo "false"
+ fi
+ nfound=$(grep -c data runs_temp_temp.txt)
+ echo "Found $nfound files on TZDISK"
+ set -e
+ if [ "$nfound" -eq 0 ]; then
+ # look also on DATADISK if not found on TZDISK
+ set +e
+ rsefound=$(echo "$file" | grep -c "CERN-PROD_DATADISK")
+ if [[ ! $rsefound -eq 0 ]]; then
+ echo "$file" | grep "CERN-PROD_DATADISK" | cut -d '|' -f 6 | sed "s% CERN-PROD_DATADISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
+ cat runs_temp_temp.txt >> runs_temp.txt
+ # echo -n "Replicated? : "; (echo "$dataset" | grep CERN-PROD_DATADISK) || echo "false"
+ fi
+ nfound=$(grep -c data runs_temp_temp.txt)
+ echo "Found $nfound files on DATADISK"
+ set -e
+ fi
+ if [ "$nfound" -eq 0 ]; then
+ # look also on RAW if not found on DATADISK
+ set +e
+ rsefound=$(echo "$file" | grep -c "CERN-PROD_RAW")
+ if [[ ! $rsefound -eq 0 ]]; then
+ echo "$file" | grep "CERN-PROD_RAW" | cut -d '|' -f 6 | sed "s% CERN-PROD_RAW: %%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
+ cat runs_temp_temp.txt >> runs_temp.txt
+ # echo -n "Replicated? : "; (echo "$dataset" | grep CERN-PROD_RAW) || echo "false"
+ fi
+ nfound=$(grep -c data runs_temp_temp.txt)
+ echo "Found $nfound files on RAW"
+ set -e
+ fi
+ if [ "$nfound" -eq 0 ]; then
+ echo "No files found!"
+ fi
+ fi
+done
+rm runs_temp_temp.txt
+
+if [[ ! -f runs_lbn_files.txt ]]; then mv runs_temp.txt runs_lbn_files.txt ; fi
+
+# if you want to redo this:
+# rm -f lbn_anal_map.txt
+if [[ ! -f lbn_anal_map.txt ]]; then LB_analyze | tee log_lbn_analyze.txt ; fi
+
+# Only works on lxplus now!
+# echo -n "Total events in dataset before GRL: "
+# cat lbnevents_*.txt | tr -s ' ' | cut -d ' ' -f 8 | awk '{total = total + $1}END{print total}'
+echo -n "Total events in dataset from log_lbn_analyze.txt: "
+grep "stream 0" log_lbn_analyze.txt | cut -d ' ' -f 6 | awk '{total = total + $1}END{print total}'
+echo -n "Total events in dataset from lbn_anal_map: "
+grep "stream 1," lbn_anal_map.txt | cut -d ' ' -f 8 | awk '{total = total + $1}END{print total}'
+echo -n "Selected events per stream: "
+grep "stream 8," lbn_anal_map.txt | cut -d ' ' -f 17 | awk '{total = total + $1}END{print total}'
+
+maxstream=19 #0 up to 49 - 19 does 1M events
+#split into all the desired streams
+for s in $(seq 0 $maxstream); do
+ grep "stream ${s}," lbn_anal_map.txt | grep -v "and 0 wanted" > "lbn_anal_map_stream${s}.txt"
+ mkdir -p "output_stream${s}"
+done
+
+# split into jobs for each stream and make filelist and lbn_anal_map files
+rm -f output_stream*/filelist_*.txt output_stream*/lbn_anal_map_*.txt
+LB_AnalMapSplitter.py $maxstream | tee log_lbn_anal_map_splitter.txt
+rm output_stream*/*501.txt
+if grep "doesnt contain" log_lbn_anal_map_splitter.txt; then exit; fi
+set +e
+count=$(grep -c "not in files map" log_lbn_anal_map_splitter.txt)
+set -e
+echo "$count files not found in map..."
+
+for s in $(seq 1 $maxstream); do
+ echo "renaming stream $s to 0"
+ for f in {1..500}; do sed -i -e "s%stream $s,%stream 0,%g" "output_stream${s}/lbn_anal_map_${f}.txt"; done
+done
+
+for s in $(seq 0 $maxstream); do cd "output_stream${s}"; tar cfz "stream${s}_${out_name}.tar.gz" *.txt; cd ..; done
+mkdir -p lists
+mv -v output_stream*/stream*_${out_name}.tar.gz lists/
+
+#to check the events in each map:
+for s in $(seq 0 $maxstream); do
+ echo -n "files with 100 events in stream $s :"
+ for f in {1..500}; do cat "output_stream${s}/lbn_anal_map_$f.txt" | cut -d ' ' -f 17 | awk '{total = total + $1}END{print total}'; done | grep -c 100
+done
+
+#to show number of files in each filelist:
+echo ; echo "files in each filelist (in stream 0)..." ; for f in output_stream0/filelist_*.txt; do echo -n $(grep -o "$sstream" "$f" | wc -l); echo " : $f"; done | sort -n
+
+echo "DONE"
diff --git a/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query_DBonly.sh b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query_DBonly.sh
new file mode 100755
index 00000000000..435a1c453b1
--- /dev/null
+++ b/Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query_DBonly.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+# Only works on lxplus now!
+
+set -e
+
+# TODO: read from args
+#sstream="physics_ZeroBias" #pp or UPC
+sstream="physics_MinBiasOverlay" #HI
+
+runs=$(grep "subchild node value" lbn | cut -f 3 -d : | sed 's/[\x01-\x1F\x7F]//g' | sed 's%\[0m%%g' | sed 's%\[34m%%g' | sed ':a;N;$!ba;s/\n/ /g')
+# loop over all runs
+for runn in $runs; do
+ if [[ ! -f "lbnevents_${runn}.txt" ]]; then
+ echo "Getting Nevent info for run $runn "
+ LB_GetNEvents.py -r "$runn" -s "$sstream" | grep "... Run" | tee "lbnevents_${runn}.txt"
+ fi
+done
diff --git a/Simulation/Overlay/OverlayByteStreamUtils/util/LB_analyze.cxx b/Simulation/Overlay/OverlayByteStreamUtils/util/LB_analyze.cxx
new file mode 100644
index 00000000000..c0ef551c92d
--- /dev/null
+++ b/Simulation/Overlay/OverlayByteStreamUtils/util/LB_analyze.cxx
@@ -0,0 +1,271 @@
+/*
+ Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+*/
+
+#include <map>
+#include <TRandom3.h>
+
+
+struct lbn_info
+{
+ int nevt{};
+ float intlumi{};
+ int nevtwanted{};
+};
+
+struct run_info
+{
+ int nevt{};
+ float intlumi{};
+ std::map<int, lbn_info> lbnmap;
+ int nevtwanted{};
+};
+
+
+int readeventcounts(int run,
+ std::map<int, int> &eventcounts)
+{
+ eventcounts.clear();
+ char buf[50];
+ sprintf(buf, "lbnevents_%d.txt", run);
+ printf("Opening %s, eventcounts size is %lu\n", buf, eventcounts.size());
+ FILE *fp = fopen(buf, "r");
+ if (!fp) {
+ fprintf(stderr, "Failed to open %s!!!\n", buf);
+ return 1;
+ }
+
+ char *line = new char[500];
+ while (fgets(line, 480, fp)) {
+ int lbn, ne, nf, runn;
+ float mb;
+ int s = sscanf(&line[0], " ... Run %d, LB %d has %d events, %d RAW files and %f MB", &runn, &lbn, &ne, &nf, &mb);
+ if (s > 4) {
+ if (run != runn) {
+ return 20;
+ }
+ printf("run %d lbn %d has %d events\n", run, lbn, ne);
+ eventcounts[lbn] = ne;
+ } else {
+ fprintf(stderr, "s=%d, bad read?\n", s);
+ return 20;
+ }
+ }
+ delete[] line;
+
+ printf("Closing %s, eventcounts size is %lu\n", buf, eventcounts.size());
+ fclose(fp);
+
+ return 0;
+}
+
+int lbn_analyze(int stream,
+ int nwanted,
+ FILE *outfile)
+{
+ FILE *fp = fopen("lbn", "r");
+ if (!fp) {
+ fprintf(stderr, "Failed to open lbn input file!!!\n");
+ return 1;
+ }
+
+ char *line = new char[500];
+ int run, lbn, L1Acc, beforeps, afterps, valid;
+ int liveL1, livebp, liveap;
+ float instlumi, dt, avevtperbx, live, L1p, HLTp, LARp;
+ int grun{};
+ float gtotaltotallumi{}, gtotaltotallumiprescaled{};
+ float gtotallumi{}, gtotallumiprescaled{};
+
+ TRandom myrand;
+ myrand.SetSeed(0);
+ printf("- myrand seed is %u\n", myrand.GetSeed());
+ myrand.SetSeed(myrand.Integer(10000000) + stream);
+ printf("- myrand seed is now %u\n", myrand.GetSeed());
+
+ std::map<int, run_info> runmap;
+ std::map<int, int> eventcounts;
+ while (fgets(line, 480, fp)) {
+ if (line[0] != '-') {
+ continue;
+ }
+
+ int s = sscanf(&line[0],
+ "--- LumiCalculator : %d[%d]: L1Acc: %d, Livetime trigger L1Acc: %d, InstLumi: %f, deltaT: %f, AvEvtsPerBX: %f, BeforePrescale: %d, AfterPrescale: %d, Livetime trigger BeforePrescale: %d Livetime trigger AfterPrescale: %d, Livefrac: %f, L1Presc: %f, HLTPresc: %f, Valid: %d, LAr ready fraction: %f",
+ &run, &lbn, &L1Acc, &liveL1, &instlumi, &dt, &avevtperbx, &beforeps, &afterps, &livebp, &liveap, &live, &L1p, &HLTp, &valid, &LARp);
+
+ if (s > 8) {
+ printf("Got %d values : run=%d, lbn=%d, L1Acc=%d, instlumi=%f, L1p=%f, dt=%f, live=%f, afterps=%d \n", s, run, lbn, L1Acc, instlumi, L1p, dt, live, afterps);
+
+ if (run != grun) {
+ if (grun > 0) { //change of run
+ runmap[grun].intlumi = gtotallumi;
+ printf("Setting lumi for run %d to %f\n", grun, gtotallumi);
+ gtotaltotallumi += gtotallumi;
+ gtotaltotallumiprescaled += gtotallumiprescaled;
+ // reset the int lumi calculation for this run
+ gtotallumi = 0;
+ gtotallumiprescaled = 0;
+ }
+
+ grun = run; //save the run number
+ printf("Setting grun to %d\n", run);
+
+ // read in event counts from runquery file for run
+ int status = readeventcounts(run, eventcounts);
+ if (status != 0) {
+ return status;
+ }
+ } //new run
+
+ if (L1p < 0) {
+ runmap[run].lbnmap[lbn].nevt = 0;
+ if (eventcounts[lbn] !=0 ) {
+ printf("For lbn %d, L1p<0 but eventcounts is %d\n",lbn,eventcounts[lbn]);
+ }
+ } else {
+ int ne = L1Acc;//afterps
+ if (ne != eventcounts[lbn]) {
+ if (stream == 0) {
+ printf("For lbn %d, ne from lumicalc is %d but we will trust runquery value of %d\n", lbn, ne, eventcounts[lbn]);
+ }
+ ne = eventcounts[lbn];
+ }
+ if (ne == 0) {
+ dt = 0;
+ printf("For lbn %d, setting lumi to 0 since 0 events were recorded\n", lbn);
+ }
+ runmap[run].lbnmap[lbn].nevt = ne;
+ runmap[run].lbnmap[lbn].intlumi = instlumi * dt * live;
+ runmap[run].nevt += ne;
+ gtotallumiprescaled += instlumi * dt * live / L1p;
+ gtotallumi += instlumi * dt * live;
+ }
+ printf("grun=%d, gtotallumi=%f, gtotallumiprescaled=%f\n", grun, gtotallumi, gtotallumiprescaled);
+ } //good line
+ } //loop over lines in file
+ delete[] line;
+
+ // after last run
+ runmap[grun].intlumi = gtotallumi;
+ printf("Setting lumi for run %d to %f\n", grun, gtotallumi);
+ gtotaltotallumi += gtotallumi;
+ gtotaltotallumiprescaled += gtotallumiprescaled;
+
+ fclose(fp);
+
+ printf("- %lu runs, gtotaltotallumi=%f, gtotaltotallumiprescaled=%f\n", runmap.size(), gtotaltotallumi, gtotaltotallumiprescaled);
+
+ if (runmap.size() < 1) {
+ printf("- runmap size is %lu, quitting!\n", runmap.size());
+ return 0;
+ }
+
+ //check the total lumi...
+ double tempr{};
+ for (const auto &[r, info] : runmap) {
+ tempr += info.intlumi;
+ }
+ if (fabs(tempr - gtotaltotallumi) / tempr > 0.001) {
+ fprintf(stderr, "tempr=%f and gtotaltotallumi=%f\n", tempr, gtotaltotallumi);
+ return 3;
+ }
+
+ for (int e = 0; e < nwanted; ++e) {
+ // pick a random run, proportional to intlumi
+ double rnd = myrand.Uniform(tempr);
+ run = -1;
+ for (const auto &[r, info] : runmap) {
+ if (rnd < info.intlumi) {
+ run = r;
+ break;
+ }
+ rnd -= info.intlumi;
+ }
+ if (run < 0) { // error
+ return 4;
+ }
+
+ // check the total run lumi...
+ double tempt{};
+ for (const auto &[l, info] : runmap[run].lbnmap) {
+ tempt += info.intlumi;
+ }
+ if (fabs(tempt - runmap[run].intlumi) / tempt > 0.001) {
+ fprintf(stderr, "tempt=%f and runmap[%d].intlumi=%f\n", tempt, run, runmap[run].intlumi);
+ return 5;
+ }
+
+ //p ick a random lbn, proportional to intlumi
+ rnd = myrand.Uniform(tempt);
+ lbn = -1;
+ for (const auto &[l, info] : runmap[run].lbnmap) {
+ if (rnd < info.intlumi) {
+ lbn = l;
+ break;
+ }
+ rnd -= info.intlumi;
+ }
+ if (lbn < 0) { // error
+ return 6;
+ }
+
+ runmap[run].nevtwanted++;
+ runmap[run].lbnmap[lbn].nevtwanted++;
+ printf("- stream %d, run %d, lbn %d, choose %d of out %d\n", stream, run, lbn, runmap[run].lbnmap[lbn].nevtwanted, runmap[run].lbnmap[lbn].nevt);
+ } //loop over nwanted
+
+ for (const auto &[r, rinfo] : runmap) {
+ int totnevt{};
+ float totintlumi{};
+ printf("stream %d, run %d, has %d events and %f/ub, %f intlumi of total, and %d wanted\n", stream,
+ r, rinfo.nevt, rinfo.intlumi, rinfo.intlumi / gtotaltotallumi, rinfo.nevtwanted);
+ for (const auto &[l, linfo] : rinfo.lbnmap) {
+ fprintf(outfile, "stream %d, run %d, lbn %d, has %d events and %f/ub, %f intlumi of run, and %d wanted", stream, r,
+ l, linfo.nevt, linfo.intlumi, linfo.intlumi / rinfo.intlumi, linfo.nevtwanted);
+ if (linfo.nevtwanted > linfo.nevt) {
+ fprintf(outfile, " : WARNING, more than available, will be duplicates!\n");
+ } else {
+ fprintf(outfile,"\n");
+ }
+ totnevt += linfo.nevt;
+ totintlumi += linfo.intlumi;
+ }
+ if (totnevt != rinfo.nevt) {
+ fprintf(stderr, " XXX events do not agree !!! \n");
+ }
+ if (totintlumi > 0.0 && fabs(totintlumi - rinfo.intlumi) / totintlumi > .001) {
+ fprintf(stderr, " XXX intlumi does not agree !!! %f %f \n", totintlumi, rinfo.intlumi);
+ }
+ }
+
+ printf("--\n\n");
+
+ return 0;
+}
+
+int main(int argc, const char* argv[])
+{
+ int Nstreams{50}; // number of zerobias streams to make
+ int Nwanted{50000}; // number of events per zerobias stream
+
+ if (argc == 3) {
+ Nstreams = std::atoi(argv[1]);
+ Nwanted = std::atoi(argv[2]);
+ }
+
+ printf("Nstreams=%d, Nwanted=%d\n\n", Nstreams, Nwanted);
+
+ FILE *outfile = fopen("lbn_anal_map.txt", "w");
+
+ for (int i = 0; i < Nstreams; ++i) {
+ int status = lbn_analyze(i, Nwanted, outfile);
+ if (status != 0) {
+ return status;
+ }
+ }
+
+ fclose(outfile);
+
+ return 0;
+}
--
GitLab