Skip to content
Snippets Groups Projects
Verified Commit 2381480f authored by Tadej Novak's avatar Tadej Novak
Browse files

Data overlay: cleanup zero bias selection scripts

parent 2b58a449
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 592 additions and 446 deletions
Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/cleanup.sh deleted 100755 → 0
+ 0
− 7
View file @ 2b58a449
rm -rf output_stream*
rm lbn_anal_map_stream*
rm lbnevents_*
rm runs_lbn_files.txt
rm log_lbn_anal*
rm lbn_anal_map.txt
Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py deleted 100644 → 0
+ 0
− 109
View file @ 2b58a449
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import sys
#get the list of special run,lbn with more than 1 file
special_run_lbn = {}
with open("runs_lbn_files.txt") as f:
for line in f:
#print line
if not line.startswith("root"):
print "WARNING: line doesn't start with 'root'"
print line
continue
filenm=line.rstrip()
print filenm
finfo=filenm.split(".")
run=finfo[3]
lbi=4 #index of the lbn info
while "_lb" not in finfo[lbi] and lbi<20:
lbi=lbi+1
if lbi>=20:
print finfo[lbi], "doesnt contain _lb"
exit(1)
lbn=finfo[lbi].replace("_lb","")
print run+"_"+lbn
if run+"_"+lbn in special_run_lbn:
if filenm in special_run_lbn[run+"_"+lbn]:
print filenm, "already in map for ",run,lbn
else:
special_run_lbn[run+"_"+lbn].append(filenm)
else:
special_run_lbn[run+"_"+lbn]=[filenm]
for key in special_run_lbn.keys():
#if len(special_run_lbn[key])>1: print special_run_lbn[key]
pass
print "done with special_run_lbn"
#do the work
maxstream = int(sys.argv[1])
print "maxstream=",maxstream
for s in range(0,maxstream+1): #50
with open("lbn_anal_map_stream"+str(s)+".txt") as f:
tot=0
nfile=1
of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w")
of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w")
for line in f:
linfo=line.split(" ")
#print line
#print linfo[16]
tot+=int(linfo[16])
run=linfo[3].replace(",","").zfill(8)
lbn=linfo[5].replace(",","").zfill(4)
if tot>100:
#replace last bit of line and also write it to the new file
extra=tot-100
forthisone=int(linfo[16])-extra
rline=line.replace(linfo[16]+" wanted",str(forthisone)+" wanted")
of.write(rline)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
#print of2.tell()
if of2.tell()>0: of2.write(",")
of2.write(f)
else:
print run,lbn," not in files map!"
#print line, rline
#print run, lbn
of.close(); of2.close(); nfile+=1
of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w")
of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w")
tot=extra #the number left over from the last set
rline=line.replace(linfo[16]+" wanted",str(extra)+" wanted")
of.write(rline)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
if of2.tell()>0: of2.write(",")
of2.write(f)
else:
print run,lbn," not in files map!"
elif tot==100:
of.write(line)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
if of2.tell()>0: of2.write(",")
of2.write(f)
else:
print run,lbn," not in files map!"
of.close(); of2.close(); nfile+=1
of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w")
of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w")
tot=0
else:
of.write(line)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
#print of2.tell()
if of2.tell()>0: of2.write(",")
of2.write(f)
else:
print run,lbn," not in files map!"
print "done ",s
of.close(); of2.close()
Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_analyze.C deleted 100644 → 0
+ 0
− 194
View file @ 2b58a449
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
#include <map>
#include "TRandom3.h"
class lbn_info{
public:
lbn_info(){nevt=0; intlumi=0; nevtwanted=0;}
int nevt;
float intlumi;
int nevtwanted;
};
class run_info{
public:
run_info(){nevt=0; intlumi=0; nevtwanted=0;}
int nevt;
float intlumi;
std::map<int,lbn_info> lbnmap;
int nevtwanted;
};
FILE *outfile=fopen("lbn_anal_map.txt","w");
int debug=1;
std::map< int, int > eventcounts;
void readeventcounts(int run){
eventcounts.clear();
char buf[50];
sprintf(buf,"lbnevents_%d.txt",run);
if (debug) printf("Opening %s, eventcounts size is %lu\n",buf,eventcounts.size());
FILE *fp = fopen(buf,"r");
if (!fp) printf("Failed to open %s!!!\n",buf);
char *line=new char[500];
while (fgets(line,480,fp)) {
int lbn,ne,nf,runn;
float mb;
int s=sscanf(&line[0]," ... Run %d, LB %d has %d events, %d RAW files and %f MB",&runn,&lbn,&ne,&nf,&mb);
if (s>4){
assert(run==runn);
if (debug) printf("run %d lbn %d has %d events\n",run,lbn,ne);
eventcounts[lbn]=ne;
}
else {
printf("s=%d, bad read?\n",s);
assert(false);
}
}
if (debug) printf("Closing %s, eventcounts size is %lu\n",buf,eventcounts.size());
fclose(fp);
}
void lbn_analyze(int stream, int nwanted)
{
FILE *fp = fopen("lbn","r");
if (!fp) printf("Failed to open lbn input file!!!\n");
char *line=new char[500];
int run,lbn,L1Acc,beforeps,afterps,valid; //L1p,L2p,L3p
int liveL1,livebp,liveap;
float instlumi,dt,avevtperbx,live,L1p,HLTp,LARp;
//float intlumi,intlumitrig;
int grun=0;
float gtotaltotallumi=0,gtotaltotallumiprescaled=0;
float gtotallumi=0, gtotallumiprescaled=0;
TRandom myrand;
myrand.SetSeed(0);
printf("- myrand seed is %u\n",myrand.GetSeed());
myrand.SetSeed(myrand.Integer(10000000)+stream);
printf("- myrand seed is now %u\n",myrand.GetSeed());
std::map<int,run_info> runmap;
while (fgets(line,480,fp)) {
if (line[0]!='-') continue;
// int s=sscanf(&line[0],"--- LumiCalculator : %d[%d]: L1Acc: %d, Livetime trigger L1Acc: %d, InstLumi: %f, deltaT: %f, AvEvtsPerBX: %f, BeforePrescale: %d, AfterPrescale: %d, Livetime trigger BeforePrescale: %d Livetime trigger AfterPrescale: %d, Livefrac: %f, L1Presc: %d, L2Presc: %d, L3Presc: %d, Valid: %d", &run,&lbn,&L1Acc,&liveL1,&instlumi,&dt,&avevtperbx,&beforeps,&afterps,&livebp,&liveap,&live,&L1p,&L2p,&L3p,&valid);
int s=sscanf(&line[0],"--- LumiCalculator : %d[%d]: L1Acc: %d, Livetime trigger L1Acc: %d, InstLumi: %f, deltaT: %f, AvEvtsPerBX: %f, BeforePrescale: %d, AfterPrescale: %d, Livetime trigger BeforePrescale: %d Livetime trigger AfterPrescale: %d, Livefrac: %f, L1Presc: %f, HLTPresc: %f, Valid: %d, LAr ready fraction: %f", &run,&lbn,&L1Acc,&liveL1,&instlumi,&dt,&avevtperbx,&beforeps,&afterps,&livebp,&liveap,&live,&L1p,&HLTp,&valid,&LARp);
if (s>8){
if (debug) printf("Got %d values : run=%d, lbn=%d, L1Acc=%d, instlumi=%f, L1p=%f, dt=%f, live=%f, afterps=%d \n",s,run,lbn,L1Acc,instlumi,L1p,dt,live,afterps);
if (run!=grun){
if (grun>0){//change of run
runmap[grun].intlumi=gtotallumi; printf("Setting lumi for run %d to %f\n",grun,gtotallumi);
if (debug) printf("Adding to totaltotal lumis\n");
gtotaltotallumi+=gtotallumi; gtotaltotallumiprescaled+=gtotallumiprescaled;
gtotallumi=0;gtotallumiprescaled=0;//reset the int lumi calculation for this run
//runmap[grun].nevt=0;
}
grun=run;//save the run number
printf("Setting grun to %d\n",run);
//read in event counts from runquery file for run
readeventcounts(run);
}//new run
if (L1p<0) {
runmap[run].lbnmap[lbn].nevt=0;
if (eventcounts[lbn]!=0) printf("For lbn %d, L1p<0 but eventcounts is %d\n",lbn,eventcounts[lbn]);
}
else{
int ne = L1Acc;//afterps
if (ne!=eventcounts[lbn]){
if (0==stream) printf("For lbn %d, ne from lumicalc is %d but we will trust runquery value of %d\n",lbn,ne,eventcounts[lbn]);
ne=eventcounts[lbn];
}
if (0==ne){dt=0; printf("For lbn %d, setting lumi to 0 since 0 events were recorded\n",lbn);}
runmap[run].lbnmap[lbn].nevt=ne;
runmap[run].lbnmap[lbn].intlumi=instlumi*dt*live;
runmap[run].nevt+=ne;
gtotallumiprescaled+= instlumi*dt*live/L1p;
gtotallumi+= instlumi*dt*live;
}
if (debug) printf("grun=%d, gtotallumi=%f, gtotallumiprescaled=%f\n",grun,gtotallumi,gtotallumiprescaled);
}//good line
}//loop over lines in file
//after last run
runmap[grun].intlumi=gtotallumi; printf("Setting lumi for run %d to %f\n",grun,gtotallumi);
if (debug) printf("Adding to totaltotal lumis for last run\n");
gtotaltotallumi+=gtotallumi; gtotaltotallumiprescaled+=gtotallumiprescaled;
fclose(fp);
printf("- %lu runs, gtotaltotallumi=%f, gtotaltotallumiprescaled=%f\n",runmap.size(),gtotaltotallumi,gtotaltotallumiprescaled);
if (runmap.size()<1) {printf("- runmap size is %lu, quitting!\n",runmap.size()); return;}
//return;
//check the total lumi...
double tempr=0;
for (std::map<int,run_info>::const_iterator r=runmap.begin(); r!=runmap.end(); ++r){
tempr += r->second.intlumi;
}
if (fabs(tempr-gtotaltotallumi)/tempr>0.001){
printf("tempr=%f and gtotaltotallumi=%f\n",tempr,gtotaltotallumi);
assert(false);
}
for (int e=0;e<nwanted;++e){
//pick a random run, proportional to intlumi
double rnd = myrand.Uniform(tempr);
run=-1;
for (std::map<int,run_info>::const_iterator r=runmap.begin(); r!=runmap.end(); ++r){
if (rnd<r->second.intlumi) {run=r->first; break;}
rnd -= r->second.intlumi;
}
assert(run>=0);
//check the total run lumi...
double tempt=0;
for (std::map<int,lbn_info>::const_iterator l=runmap[run].lbnmap.begin(); l!=runmap[run].lbnmap.end(); ++l){
tempt += l->second.intlumi;
}
if (fabs(tempt-runmap[run].intlumi)/tempt>0.001){
printf("tempt=%f and runmap[%d].intlumi=%f\n",tempt,run,runmap[run].intlumi);
assert(false);
}
//pick a random lbn, proportional to intlumi
rnd = myrand.Uniform(tempt);
lbn=-1;
for (std::map<int,lbn_info>::const_iterator l=runmap[run].lbnmap.begin(); l!=runmap[run].lbnmap.end(); ++l){
if (rnd<l->second.intlumi) {lbn=l->first; break;}
rnd -= l->second.intlumi;
}
assert(lbn>=0);
runmap[run].nevtwanted++; runmap[run].lbnmap[lbn].nevtwanted++;
if (debug) printf("- stream %d, run %d, lbn %d, choose %d of out %d\n",stream,run,lbn,runmap[run].lbnmap[lbn].nevtwanted,runmap[run].lbnmap[lbn].nevt);
}//loop over nwanted
for (std::map<int,run_info>::const_iterator r=runmap.begin(); r!=runmap.end(); ++r){
int totnevt=0; float totintlumi=0;
printf("stream %d, run %d, has %d events and %f/ub, %f intlumi of total, and %d wanted\n", stream,
r->first,r->second.nevt,r->second.intlumi,r->second.intlumi/gtotaltotallumi,r->second.nevtwanted);
for (std::map<int,lbn_info>::const_iterator l=r->second.lbnmap.begin(); l!=r->second.lbnmap.end(); ++l){
fprintf(outfile,"stream %d, run %d, lbn %d, has %d events and %f/ub, %f intlumi of run, and %d wanted", stream, r->first,
l->first,l->second.nevt,l->second.intlumi,l->second.intlumi/r->second.intlumi,l->second.nevtwanted);
if (l->second.nevtwanted>l->second.nevt) fprintf(outfile," : WARNING, more than available, will be duplicates!\n"); else fprintf(outfile,"\n");
totnevt+=l->second.nevt; totintlumi+=l->second.intlumi;
}
if (totnevt!=r->second.nevt) printf(" XXX events do not agree !!! \n");
if (totintlumi>0.0 && fabs(totintlumi - r->second.intlumi)/totintlumi>.001) printf(" XXX intlumi does not agree !!! %f %f \n",totintlumi,r->second.intlumi);
}
printf("--\n\n");
}
Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh deleted 100755 → 0
+ 0
− 102
View file @ 2b58a449
#!/bin/bash
#setupATLAS
#lsetup rucio
#voms-proxy-init -voms atlas
#use GRL, from: http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/?C=M;O=D
#consult: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/GoodRunListsForAnalysisRun2
#and: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/DataMCForAnalysis
#dq="data15_13TeV.periodAllYear_DetStatus-v73-pro19-08_DQDefects-00-01-02_PHYS_StandardGRL_All_Good.xml"
#wget -nc http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/$dq
#Can just go here: https://atlas-lumicalc.cern.ch/ ... note use --online lumi tag for HI, unless a better one is available
#and request --verbose output and use physics trigger HLT_noalg_zb_L1ZB,
#then wget "Raw iLimuiCalc.exe output", and "mv output.txt lbn_2015; ln -s lbn_2015 lbn"
#rm -f lbn # if you want to redo this
if [ ! -f "lbn" ]; then
iLumiCalc.exe -t L1_ZB -V -x $dq --livetrigger=L1_EM12 > lbn
fi
grep ": Total" lbn ; echo
#get info on number of events and files per lumi block in each run
#rm -f lbnevents* # if you want to redo this
#rm -f runs_lbn_files.txt # if you want to redo this
#sstream="physics_ZeroBias" #pp or UPC
sstream="physics_MinBiasOverlay" #HI
#suf="merge"
suf="daq"
rm -f runs_temp.txt
for runn in `grep "subchild node value" lbn|cut -f 3 -d : |sed 's/[\x01-\x1F\x7F]//g'|sed 's%\[0m%%g'|sed 's%\[34m%%g'|sed ':a;N;$!ba;s/\n/ /g' `; do
if [ ! -f lbnevents_${runn}.txt ]; then
echo "Getting Nevent info for run $runn ";
python GetNEventsLB.py -r $runn -s $sstream |grep "... Run" > lbnevents_${runn}.txt;# Only works on lxplus now!
fi
if [ ! -f runs_lbn_files.txt ]; then
echo "Getting file info for run $runn "
rucio list-file-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW | grep "CERN-PROD_TZDISK" | cut -d '|' -f 6|sed "s% CERN-PROD_TZDISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
cat runs_temp_temp.txt >> runs_temp.txt
echo -n "Replicated? : "; rucio list-dataset-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW|grep CERN-PROD_TZDISK
nfound=`grep -c data runs_temp_temp.txt`
echo "Found $nfound files on TZDISK"
if [ $nfound -eq 0 ]; then #look also on DATADISK if not found on TZDISK
echo "Getting file info for run $runn "
rucio list-file-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW | grep "CERN-PROD_DATADISK" | cut -d '|' -f 6|sed "s% CERN-PROD_DATADISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
cat runs_temp_temp.txt >> runs_temp.txt
echo -n "Replicated? : "; rucio list-dataset-replicas data16_hip8TeV.00${runn}.${sstream}.${suf}.RAW|grep CERN-PROD_DATADISK
nfound=`grep -c data runs_temp_temp.txt`
echo "Found $nfound files on DATADISK"
fi
fi
done #loop over all runs
rm runs_temp_temp.txt
if [ ! -f runs_lbn_files.txt ]; then mv runs_temp.txt runs_lbn_files.txt ; fi
#rm -f lbn_anal_map.txt # if you want to redo this
if [ ! -f lbn_anal_map.txt ]; then root -l -b -q run_lbn_analyze.C > log_lbn_analyze.txt ; fi
echo -n "Total events in dataset before GRL: "
cat lbnevents_*.txt |tr -s ' ' | cut -d ' ' -f 8 |awk '{total = total + $1}END{print total}'
echo -n "Total events in dataset from log_lbn_analyze.txt: "
grep "stream 0" log_lbn_analyze.txt|cut -d ' ' -f 6 |awk '{total = total + $1}END{print total}'
echo -n "Total events in dataset from lbn_anal_map: "
grep "stream 1," lbn_anal_map.txt |cut -d ' ' -f 8 |awk '{total = total + $1}END{print total}'
echo -n "Selected events per stream: "
grep "stream 8," lbn_anal_map.txt |cut -d ' ' -f 17 |awk '{total = total + $1}END{print total}'
maxstream=19 #0 up to 49 - 19 does 1M events
#split into all the desired streams
for s in $(seq 0 $maxstream); do
grep "stream ${s}," lbn_anal_map.txt | grep -v "and 0 wanted" > lbn_anal_map_stream${s}.txt
mkdir -p output_stream${s}
done
#split into jobs for each stream and make filelist and lbn_anal_map files
rm -f output_stream*/filelist_*.txt output_stream*/lbn_anal_map_*.txt
python lbn_anal_map_splitter.py $maxstream > log_lbn_anal_map_splitter.txt
rm output_stream*/*501.txt
grep "doesnt contain" log_lbn_anal_map_splitter.txt
if [ $? == 0 ]; then exit; fi
grep -c "not in files map" log_lbn_anal_map_splitter.txt
#if [ $? == 0 ]; then exit; fi
echo "$? files not found in map..."
for s in $(seq 1 $maxstream); do
echo "renaming stream $s to 0"
for f in {1..500}; do sed -i -e "s%stream $s,%stream 0,%g" output_stream${s}/lbn_anal_map_${f}.txt; done
done
it=2016_hip8TeV_1 #just a name to tag this set of files
for s in $(seq 0 $maxstream); do cd output_stream${s}; tar cfz stream${s}_${it}.tar.gz *.txt; cd ..; done
mv -v output_stream*/stream*_${it}.tar.gz ~/public/overlay/lists/
#to check the events in each map:
for s in $(seq 0 $maxstream); do
echo -n "files with 100 events in stream $s :"
for f in {1..500}; do cat output_stream${s}/lbn_anal_map_$f.txt | cut -d ' ' -f 17 |awk '{total = total + $1}END{print total}'; done |grep -c 100
done
#to show number of files in each filelist:
echo ; echo "files in each filelist (in stream 0)..." ; for f in output_stream0/filelist_*.txt; do echo -n `grep -o $sstream $f | wc -l`; echo " : $f"; done |sort -n
Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/run_lbn_analyze.C deleted 100644 → 0
+ 0
− 14
View file @ 2b58a449
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
{
gROOT->ProcessLine(".L lbn_analyze.C+");
for (int i=0; i<50; ++i){//number of zerobias streams to make
//gSystem->Sleep(2000);//to get new random number seed
lbn_analyze(i,50000);//number of events per zerobias stream
}
}
Simulation/Overlay/OverlayByteStreamUtils/CMakeLists.txt
+ 9
− 1
View file @ 2381480f
......@@ -11,4 +11,12 @@ atlas_add_component( OverlayByteStreamUtils
src/*.cxx
src/components/*.cxx
INCLUDE_DIRS ${ROOT_INCLUDE_DIRS}
LINK_LIBRARIES LINK_LIBRARIES ${ROOT_LIBRARIES} AthenaBaseComps AthenaKernel ByteStreamCnvSvcLib GaudiKernel StoreGateLib TrigConfHLTData TrigConfInterfaces TrigSteeringEvent TrigT1Result )
LINK_LIBRARIES ${ROOT_LIBRARIES} AthenaBaseComps AthenaKernel ByteStreamCnvSvcLib GaudiKernel StoreGateLib TrigConfHLTData TrigConfInterfaces TrigSteeringEvent TrigT1Result )
# Utility scripts
atlas_install_runtime( scripts/*.py )
atlas_install_runtime( scripts/*.sh )
atlas_add_executable (LB_analyze util/LB_analyze.cxx
INCLUDE_DIRS ${ROOT_INCLUDE_DIRS}
LINK_LIBRARIES ${ROOT_LIBRARIES} )
Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_AnalMapSplitter.py 0 → 100755
+ 120
− 0
View file @ 2381480f
#!/bin/env python
#
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
from __future__ import print_function
import sys
# get the list of special run,lbn with more than 1 file
special_run_lbn = {}
with open("runs_lbn_files.txt") as f:
for line in f:
# print line
if not line.startswith("root"):
print("WARNING: line doesn't start with 'root'")
print(line)
continue
fileName = line.rstrip()
print(fileName)
finfo = fileName.split(".")
run = finfo[3]
lbi = 4 # index of the lbn info
while "_lb" not in finfo[lbi] and lbi < 20:
lbi += 1
if lbi >= 20:
print(finfo[lbi] + " doesnt contain _lb")
exit(1)
lbn = finfo[lbi].replace("_lb", "")
print(run+"_"+lbn)
if run+"_"+lbn in special_run_lbn:
if fileName in special_run_lbn[run+"_"+lbn]:
print("%s already in map for %s %s" % (fileName, run, lbn))
else:
special_run_lbn[run+"_"+lbn].append(fileName)
else:
special_run_lbn[run+"_"+lbn] = [fileName]
for key in special_run_lbn.keys():
pass
print("done with special_run_lbn")
# do the work
maxstream = int(sys.argv[1])
print("maxstream=%d" % maxstream)
for s in range(0, maxstream+1): # 50
with open("lbn_anal_map_stream"+str(s)+".txt") as f:
tot = 0
nfile = 1
of = open("output_stream"+str(s) +
"/lbn_anal_map_"+str(nfile)+".txt", "w")
of2 = open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt", "w")
for line in f:
linfo = line.split(" ")
tot += int(linfo[16])
run = linfo[3].replace(",", "").zfill(8)
lbn = linfo[5].replace(",", "").zfill(4)
if tot > 100:
# replace last bit of line and also write it to the new file
extra = tot-100
forthisone = int(linfo[16])-extra
rline = line.replace(
linfo[16]+" wanted", str(forthisone)+" wanted")
of.write(rline)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
if of2.tell() > 0:
of2.write(",")
of2.write(f)
else:
print("%s %s not in files map!" % (run, lbn))
of.close()
of2.close()
nfile += 1
of = open("output_stream"+str(s) +
"/lbn_anal_map_"+str(nfile)+".txt", "w")
of2 = open("output_stream"+str(s) +
"/filelist_"+str(nfile)+".txt", "w")
tot = extra # the number left over from the last set
rline = line.replace(linfo[16]+" wanted", str(extra)+" wanted")
of.write(rline)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
if of2.tell() > 0:
of2.write(",")
of2.write(f)
else:
print("%s %s not in files map!" % (run, lbn))
elif tot == 100:
of.write(line)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
if of2.tell() > 0:
of2.write(",")
of2.write(f)
else:
print("%s %s not in files map!" % (run, lbn))
of.close()
of2.close()
nfile += 1
of = open("output_stream"+str(s) +
"/lbn_anal_map_"+str(nfile)+".txt", "w")
of2 = open("output_stream"+str(s) +
"/filelist_"+str(nfile)+".txt", "w")
tot = 0
else:
of.write(line)
if run+"_"+lbn in special_run_lbn:
for f in special_run_lbn[run+"_"+lbn]:
if of2.tell() > 0:
of2.write(",")
of2.write(f)
else:
print("%s %s not in files map!" % (run, lbn))
print("done %d" % s)
of.close()
of2.close()
Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/GetNEventsLB.py → Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_GetNEvents.py 100644 → 100755
+ 20
− 19
View file @ 2381480f
#!/bin/env python
#
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# ----------------------------------------------------------------
# Script : GetNEventsLB.py
# Purpose: Utility to retrieve number of events per LB from SFO DB
......@@ -8,6 +8,8 @@
# Created: Aug 17, 2011
# ----------------------------------------------------------------
from __future__ import print_function
import sys, getopt
def GetSFO_LBNEvents( cursor, runno, streamname ):
......@@ -26,12 +28,12 @@ def OpenSFOConnection():
return cx_Oracle.connect("ATLAS_SFO_T0_R/readmesfotz2008@atlr")
def usage():
print ' '
print 'Usage: python %s [options]' % sys.argv[0]
print ' -r | --run : run number (REQUIRED)'
print ' -s | --stream : full stream name (REQUIRED)'
print ' -h | --help : print this usage message'
print ' '
print(' ')
print('Usage: %s [options]' % sys.argv[0])
print(' -r | --run : run number (REQUIRED)')
print(' -s | --stream : full stream name (REQUIRED)')
print(' -h | --help : print this usage message')
print(' ')
exit(1)
def main():
......@@ -43,7 +45,7 @@ def main():
except getopt.GetoptError:
# print help information and exit:
print >> sys.stderr, 'ERROR: unknown options in argument %s' % sys.argv[1:]
print('ERROR: unknown options in argument %s' % sys.argv[1:], file=sys.stderr)
usage()
runno = None
......@@ -62,17 +64,17 @@ def main():
connection = OpenSFOConnection()
cursor = connection.cursor()
print 'Results for run: %i, stream: "%s"' % (runno, stream)
print('Results for run: %i, stream: "%s"' % (runno, stream))
# min/max LB number for given run/stream
# --> this command is slow... don't know why
# minLB, maxLB = GetSFO_LBs( cursor, runno )
# print 'LB range: %i -- %i' % (minLB, maxLB)
# list with
# list with
lblist = GetSFO_LBNEvents( cursor, runno, stream )
print 'First non-zero LB: ',lblist[0][1]
print 'Last non-zero LB: ',lblist[-1][1]
print('First non-zero LB: ' + str(lblist[0][1]))
print('Last non-zero LB: ' + str(lblist[-1][1]))
# access to all LBs
sumnev = 0
......@@ -80,18 +82,17 @@ def main():
sumfsize = 0
for (nev,lb,nfiles,fsize) in lblist:
fsize /= 1.e6
print ' ... Run %i, LB %i has %i events, %i RAW files and %f MB' % (runno,lb,nev,nfiles,fsize)
print(' ... Run %i, LB %i has %i events, %i RAW files and %f MB' % (runno,lb,nev,nfiles,fsize))
sumnev += nev
sumnfiles += nfiles
sumfsize += fsize
print '--------------------------------------------------'
print 'Total #events : ', sumnev
print 'Total #files : ', sumnfiles
print 'Total RAW size: ', sumfsize/1000.0, ' GB'
print('--------------------------------------------------')
print('Total #events : ', sumnev)
print('Total #files : ', sumnfiles)
print('Total RAW size: ', sumfsize/1000.0, ' GB')
cursor.close()
connection.close()
if __name__ == '__main__':
main()
Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query.sh 0 → 100755
+ 152
− 0
View file @ 2381480f
#!/bin/bash
#
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# NOTE: Rucio setup is required
set -e
# use GRL, from: http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/?C=M;O=D
# consult: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/GoodRunListsForAnalysisRun2
# and: https://twiki.cern.ch/twiki/bin/view/AtlasProtected/DataMCForAnalysis
# TODO: read from args
#sstream="physics_ZeroBias" #pp or UPC
sstream="physics_MinBiasOverlay" #HI
pref="data18_hi"
#suf="merge"
suf="daq"
# just a name to tag this set of files
out_name=data18_hi_1
# TODO: read from args
dq="http://atlasdqm.web.cern.ch/atlasdqm/grlgen/HeavyIons/All_Good/data18_hi.periodS_DetStatus-v104-pro22-08_Unknown_PHYS_HeavyIonP_All_Good.xml"
wget -nc "$dq"
# Can just go here: https://atlas-lumicalc.cern.ch/ ... note use --online lumi tag for HI, unless a better one is available
# and request --verbose output and use physics trigger HLT_noalg_zb_L1ZB,
# then wget "Raw iLimuiCalc output", and "mv output.txt lbn_2015; ln -s lbn_2015 lbn"
# if you want to redo this:
# rm -f lbn
if [ ! -f "lbn" ]; then
# iLumiCalc -t L1_ZB -V -x "$dq" --livetrigger=L1_EM12 > lbn
iLumiCalc -t L1_ZB --lumitag OflLumi-HI-009 -V -x "$dq" --livetrigger=L1_EM12 | tee lbn
fi
grep ": Total" lbn ; echo
# get info on number of events and files per lumi block in each run
# if you want to redo this:
# rm -f lbnevents*
# rm -f runs_lbn_files.txt
rm -f runs_temp.txt
runs=$(grep "subchild node value" lbn|cut -f 3 -d : |sed 's/[\x01-\x1F\x7F]//g'|sed 's%\[0m%%g'|sed 's%\[34m%%g'|sed ':a;N;$!ba;s/\n/ /g')
# loop over all runs
for runn in $runs; do
# Only works on lxplus now!
# if [[ ! -f "lbnevents_${runn}.txt" ]]; then
# echo "Getting Nevent info for run $runn "
# LB_GetNEvents.py -r "$runn" -s "$sstream" | grep "... Run" | tee "lbnevents_${runn}.txt"
# fi
rm -f runs_temp_temp.txt
touch runs_temp_temp.txt
if [[ ! -f "runs_lbn_files.txt" ]]; then
echo "Getting file info for run $runn "
file=$(rucio list-file-replicas "${pref}.00${runn}.${sstream}.${suf}.RAW" || true)
# dataset=$(rucio list-dataset-replicas "${pref}.00${runn}.${sstream}.${suf}.RAW")
set +e
rsefound=$(echo "$file" | grep -c "CERN-PROD_TZDISK")
if [[ ! $rsefound -eq 0 ]]; then
echo "$file" | (grep "CERN-PROD_TZDISK" || true) | cut -d '|' -f 6 | sed "s% CERN-PROD_TZDISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
cat runs_temp_temp.txt >> runs_temp.txt
# echo -n "Replicated? : "; (echo "$dataset" | grep CERN-PROD_TZDISK) || echo "false"
fi
nfound=$(grep -c data runs_temp_temp.txt)
echo "Found $nfound files on TZDISK"
set -e
if [ "$nfound" -eq 0 ]; then
# look also on DATADISK if not found on TZDISK
set +e
rsefound=$(echo "$file" | grep -c "CERN-PROD_DATADISK")
if [[ ! $rsefound -eq 0 ]]; then
echo "$file" | grep "CERN-PROD_DATADISK" | cut -d '|' -f 6 | sed "s% CERN-PROD_DATADISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
cat runs_temp_temp.txt >> runs_temp.txt
# echo -n "Replicated? : "; (echo "$dataset" | grep CERN-PROD_DATADISK) || echo "false"
fi
nfound=$(grep -c data runs_temp_temp.txt)
echo "Found $nfound files on DATADISK"
set -e
fi
if [ "$nfound" -eq 0 ]; then
# look also on RAW if not found on DATADISK
set +e
rsefound=$(echo "$file" | grep -c "CERN-PROD_RAW")
if [[ ! $rsefound -eq 0 ]]; then
echo "$file" | grep "CERN-PROD_RAW" | cut -d '|' -f 6 | sed "s% CERN-PROD_RAW: %%" | sed -r 's/\s+//g' | grep "_lb" > runs_temp_temp.txt
cat runs_temp_temp.txt >> runs_temp.txt
# echo -n "Replicated? : "; (echo "$dataset" | grep CERN-PROD_RAW) || echo "false"
fi
nfound=$(grep -c data runs_temp_temp.txt)
echo "Found $nfound files on RAW"
set -e
fi
if [ "$nfound" -eq 0 ]; then
echo "No files found!"
fi
fi
done
rm runs_temp_temp.txt
if [[ ! -f runs_lbn_files.txt ]]; then mv runs_temp.txt runs_lbn_files.txt ; fi
# if you want to redo this:
# rm -f lbn_anal_map.txt
if [[ ! -f lbn_anal_map.txt ]]; then LB_analyze | tee log_lbn_analyze.txt ; fi
# Only works on lxplus now!
# echo -n "Total events in dataset before GRL: "
# cat lbnevents_*.txt | tr -s ' ' | cut -d ' ' -f 8 | awk '{total = total + $1}END{print total}'
echo -n "Total events in dataset from log_lbn_analyze.txt: "
grep "stream 0" log_lbn_analyze.txt | cut -d ' ' -f 6 | awk '{total = total + $1}END{print total}'
echo -n "Total events in dataset from lbn_anal_map: "
grep "stream 1," lbn_anal_map.txt | cut -d ' ' -f 8 | awk '{total = total + $1}END{print total}'
echo -n "Selected events per stream: "
grep "stream 8," lbn_anal_map.txt | cut -d ' ' -f 17 | awk '{total = total + $1}END{print total}'
maxstream=19 #0 up to 49 - 19 does 1M events
#split into all the desired streams
for s in $(seq 0 $maxstream); do
grep "stream ${s}," lbn_anal_map.txt | grep -v "and 0 wanted" > "lbn_anal_map_stream${s}.txt"
mkdir -p "output_stream${s}"
done
# split into jobs for each stream and make filelist and lbn_anal_map files
rm -f output_stream*/filelist_*.txt output_stream*/lbn_anal_map_*.txt
LB_AnalMapSplitter.py $maxstream | tee log_lbn_anal_map_splitter.txt
rm output_stream*/*501.txt
if grep "doesnt contain" log_lbn_anal_map_splitter.txt; then exit; fi
set +e
count=$(grep -c "not in files map" log_lbn_anal_map_splitter.txt)
set -e
echo "$count files not found in map..."
for s in $(seq 1 $maxstream); do
echo "renaming stream $s to 0"
for f in {1..500}; do sed -i -e "s%stream $s,%stream 0,%g" "output_stream${s}/lbn_anal_map_${f}.txt"; done
done
for s in $(seq 0 $maxstream); do cd "output_stream${s}"; tar cfz "stream${s}_${out_name}.tar.gz" *.txt; cd ..; done
mkdir -p lists
mv -v output_stream*/stream*_${out_name}.tar.gz lists/
#to check the events in each map:
for s in $(seq 0 $maxstream); do
echo -n "files with 100 events in stream $s :"
for f in {1..500}; do cat "output_stream${s}/lbn_anal_map_$f.txt" | cut -d ' ' -f 17 | awk '{total = total + $1}END{print total}'; done | grep -c 100
done
#to show number of files in each filelist:
echo ; echo "files in each filelist (in stream 0)..." ; for f in output_stream0/filelist_*.txt; do echo -n $(grep -o "$sstream" "$f" | wc -l); echo " : $f"; done | sort -n
echo "DONE"
Simulation/Overlay/OverlayByteStreamUtils/scripts/LB_query_DBonly.sh 0 → 100755
+ 20
− 0
View file @ 2381480f
#!/bin/bash
#
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# Only works on lxplus now!
set -e
# TODO: read from args
#sstream="physics_ZeroBias" #pp or UPC
sstream="physics_MinBiasOverlay" #HI
runs=$(grep "subchild node value" lbn | cut -f 3 -d : | sed 's/[\x01-\x1F\x7F]//g' | sed 's%\[0m%%g' | sed 's%\[34m%%g' | sed ':a;N;$!ba;s/\n/ /g')
# loop over all runs
for runn in $runs; do
if [[ ! -f "lbnevents_${runn}.txt" ]]; then
echo "Getting Nevent info for run $runn "
LB_GetNEvents.py -r "$runn" -s "$sstream" | grep "... Run" | tee "lbnevents_${runn}.txt"
fi
done
Simulation/Overlay/OverlayByteStreamUtils/util/LB_analyze.cxx 0 → 100644
+ 271
− 0
View file @ 2381480f
/*
Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
*/
#include <map>
#include <TRandom3.h>
struct lbn_info
{
int nevt{};
float intlumi{};
int nevtwanted{};
};
struct run_info
{
int nevt{};
float intlumi{};
std::map<int, lbn_info> lbnmap;
int nevtwanted{};
};
int readeventcounts(int run,
std::map<int, int> &eventcounts)
{
eventcounts.clear();
char buf[50];
sprintf(buf, "lbnevents_%d.txt", run);
printf("Opening %s, eventcounts size is %lu\n", buf, eventcounts.size());
FILE *fp = fopen(buf, "r");
if (!fp) {
fprintf(stderr, "Failed to open %s!!!\n", buf);
return 1;
}
char *line = new char[500];
while (fgets(line, 480, fp)) {
int lbn, ne, nf, runn;
float mb;
int s = sscanf(&line[0], " ... Run %d, LB %d has %d events, %d RAW files and %f MB", &runn, &lbn, &ne, &nf, &mb);
if (s > 4) {
if (run != runn) {
return 20;
}
printf("run %d lbn %d has %d events\n", run, lbn, ne);
eventcounts[lbn] = ne;
} else {
fprintf(stderr, "s=%d, bad read?\n", s);
return 20;
}
}
delete[] line;
printf("Closing %s, eventcounts size is %lu\n", buf, eventcounts.size());
fclose(fp);
return 0;
}
int lbn_analyze(int stream,
int nwanted,
FILE *outfile)
{
FILE *fp = fopen("lbn", "r");
if (!fp) {
fprintf(stderr, "Failed to open lbn input file!!!\n");
return 1;
}
char *line = new char[500];
int run, lbn, L1Acc, beforeps, afterps, valid;
int liveL1, livebp, liveap;
float instlumi, dt, avevtperbx, live, L1p, HLTp, LARp;
int grun{};
float gtotaltotallumi{}, gtotaltotallumiprescaled{};
float gtotallumi{}, gtotallumiprescaled{};
TRandom myrand;
myrand.SetSeed(0);
printf("- myrand seed is %u\n", myrand.GetSeed());
myrand.SetSeed(myrand.Integer(10000000) + stream);
printf("- myrand seed is now %u\n", myrand.GetSeed());
std::map<int, run_info> runmap;
std::map<int, int> eventcounts;
while (fgets(line, 480, fp)) {
if (line[0] != '-') {
continue;
}
int s = sscanf(&line[0],
"--- LumiCalculator : %d[%d]: L1Acc: %d, Livetime trigger L1Acc: %d, InstLumi: %f, deltaT: %f, AvEvtsPerBX: %f, BeforePrescale: %d, AfterPrescale: %d, Livetime trigger BeforePrescale: %d Livetime trigger AfterPrescale: %d, Livefrac: %f, L1Presc: %f, HLTPresc: %f, Valid: %d, LAr ready fraction: %f",
&run, &lbn, &L1Acc, &liveL1, &instlumi, &dt, &avevtperbx, &beforeps, &afterps, &livebp, &liveap, &live, &L1p, &HLTp, &valid, &LARp);
if (s > 8) {
printf("Got %d values : run=%d, lbn=%d, L1Acc=%d, instlumi=%f, L1p=%f, dt=%f, live=%f, afterps=%d \n", s, run, lbn, L1Acc, instlumi, L1p, dt, live, afterps);
if (run != grun) {
if (grun > 0) { //change of run
runmap[grun].intlumi = gtotallumi;
printf("Setting lumi for run %d to %f\n", grun, gtotallumi);
gtotaltotallumi += gtotallumi;
gtotaltotallumiprescaled += gtotallumiprescaled;
// reset the int lumi calculation for this run
gtotallumi = 0;
gtotallumiprescaled = 0;
}
grun = run; //save the run number
printf("Setting grun to %d\n", run);
// read in event counts from runquery file for run
int status = readeventcounts(run, eventcounts);
if (status != 0) {
return status;
}
} //new run
if (L1p < 0) {
runmap[run].lbnmap[lbn].nevt = 0;
if (eventcounts[lbn] !=0 ) {
printf("For lbn %d, L1p<0 but eventcounts is %d\n",lbn,eventcounts[lbn]);
}
} else {
int ne = L1Acc;//afterps
if (ne != eventcounts[lbn]) {
if (stream == 0) {
printf("For lbn %d, ne from lumicalc is %d but we will trust runquery value of %d\n", lbn, ne, eventcounts[lbn]);
}
ne = eventcounts[lbn];
}
if (ne == 0) {
dt = 0;
printf("For lbn %d, setting lumi to 0 since 0 events were recorded\n", lbn);
}
runmap[run].lbnmap[lbn].nevt = ne;
runmap[run].lbnmap[lbn].intlumi = instlumi * dt * live;
runmap[run].nevt += ne;
gtotallumiprescaled += instlumi * dt * live / L1p;
gtotallumi += instlumi * dt * live;
}
printf("grun=%d, gtotallumi=%f, gtotallumiprescaled=%f\n", grun, gtotallumi, gtotallumiprescaled);
} //good line
} //loop over lines in file
delete[] line;
// after last run
runmap[grun].intlumi = gtotallumi;
printf("Setting lumi for run %d to %f\n", grun, gtotallumi);
gtotaltotallumi += gtotallumi;
gtotaltotallumiprescaled += gtotallumiprescaled;
fclose(fp);
printf("- %lu runs, gtotaltotallumi=%f, gtotaltotallumiprescaled=%f\n", runmap.size(), gtotaltotallumi, gtotaltotallumiprescaled);
if (runmap.size() < 1) {
printf("- runmap size is %lu, quitting!\n", runmap.size());
return 0;
}
//check the total lumi...
double tempr{};
for (const auto &[r, info] : runmap) {
tempr += info.intlumi;
}
if (fabs(tempr - gtotaltotallumi) / tempr > 0.001) {
fprintf(stderr, "tempr=%f and gtotaltotallumi=%f\n", tempr, gtotaltotallumi);
return 3;
}
for (int e = 0; e < nwanted; ++e) {
// pick a random run, proportional to intlumi
double rnd = myrand.Uniform(tempr);
run = -1;
for (const auto &[r, info] : runmap) {
if (rnd < info.intlumi) {
run = r;
break;
}
rnd -= info.intlumi;
}
if (run < 0) { // error
return 4;
}
// check the total run lumi...
double tempt{};
for (const auto &[l, info] : runmap[run].lbnmap) {
tempt += info.intlumi;
}
if (fabs(tempt - runmap[run].intlumi) / tempt > 0.001) {
fprintf(stderr, "tempt=%f and runmap[%d].intlumi=%f\n", tempt, run, runmap[run].intlumi);
return 5;
}
//p ick a random lbn, proportional to intlumi
rnd = myrand.Uniform(tempt);
lbn = -1;
for (const auto &[l, info] : runmap[run].lbnmap) {
if (rnd < info.intlumi) {
lbn = l;
break;
}
rnd -= info.intlumi;
}
if (lbn < 0) { // error
return 6;
}
runmap[run].nevtwanted++;
runmap[run].lbnmap[lbn].nevtwanted++;
printf("- stream %d, run %d, lbn %d, choose %d of out %d\n", stream, run, lbn, runmap[run].lbnmap[lbn].nevtwanted, runmap[run].lbnmap[lbn].nevt);
} //loop over nwanted
for (const auto &[r, rinfo] : runmap) {
int totnevt{};
float totintlumi{};
printf("stream %d, run %d, has %d events and %f/ub, %f intlumi of total, and %d wanted\n", stream,
r, rinfo.nevt, rinfo.intlumi, rinfo.intlumi / gtotaltotallumi, rinfo.nevtwanted);
for (const auto &[l, linfo] : rinfo.lbnmap) {
fprintf(outfile, "stream %d, run %d, lbn %d, has %d events and %f/ub, %f intlumi of run, and %d wanted", stream, r,
l, linfo.nevt, linfo.intlumi, linfo.intlumi / rinfo.intlumi, linfo.nevtwanted);
if (linfo.nevtwanted > linfo.nevt) {
fprintf(outfile, " : WARNING, more than available, will be duplicates!\n");
} else {
fprintf(outfile,"\n");
}
totnevt += linfo.nevt;
totintlumi += linfo.intlumi;
}
if (totnevt != rinfo.nevt) {
fprintf(stderr, " XXX events do not agree !!! \n");
}
if (totintlumi > 0.0 && fabs(totintlumi - rinfo.intlumi) / totintlumi > .001) {
fprintf(stderr, " XXX intlumi does not agree !!! %f %f \n", totintlumi, rinfo.intlumi);
}
}
printf("--\n\n");
return 0;
}
int main(int argc, const char* argv[])
{
int Nstreams{50}; // number of zerobias streams to make
int Nwanted{50000}; // number of events per zerobias stream
if (argc == 3) {
Nstreams = std::atoi(argv[1]);
Nwanted = std::atoi(argv[2]);
}
printf("Nstreams=%d, Nwanted=%d\n\n", Nstreams, Nwanted);
FILE *outfile = fopen("lbn_anal_map.txt", "w");
for (int i = 0; i < Nstreams; ++i) {
int status = lbn_analyze(i, Nwanted, outfile);
if (status != 0) {
return status;
}
}
fclose(outfile);
return 0;
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment