diff --git a/DataQuality/DataQualityUtils/scripts/DeMoDaemon.exe b/DataQuality/DataQualityUtils/scripts/DeMoDaemon.exe
index 77f9f175568c4f9e44e7428e3e21af2f54c09053..553ca4fbba55f2b7f12c17b5275ff043ea088a98 100755
--- a/DataQuality/DataQualityUtils/scripts/DeMoDaemon.exe
+++ b/DataQuality/DataQualityUtils/scripts/DeMoDaemon.exe
@@ -4,11 +4,30 @@
# Daemon job to update daily the DQ stats
##################################################################
-source /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/user/atlasLocalSetup.sh
+date
+
cd $1
export AtlasSetup=/afs/cern.ch/atlas/software/dist/AtlasSetup
source $AtlasSetup/scripts/asetup.sh 21.0.66,Athena,gcc62
-python DeMoUpdate.py --weekly -b
-python DeMoUpdate.py --grlUpdate -b
\ No newline at end of file
+
+echo "First, looking for new runs..."
+python DeMoUpdate.py --runListUpdate
+
+systems='TRT LAr Tile CaloCP'
+
+for system in $systems
+do
+echo "====================================================================================="
+echo "====================================================================================="
+echo "Processing "$system
+echo "====================================================================================="
+echo "====================================================================================="
+
+python DeMoUpdate.py -b --onlineLumiNorm -t $2.onlDelivNorm -s $system --weekly --vetoLumiEvol
+python DeMoUpdate.py -b --onlineLumiNorm -t $2.onlDelivNorm -s $system --grlUpdate
+python DeMoScan.py -t $2.onlDelivNorm -s $system --recapDefects
+done
+
+
diff --git a/DataQuality/DataQualityUtils/scripts/DeMoLib.py b/DataQuality/DataQualityUtils/scripts/DeMoLib.py
index ca2149eede4abee8b37fdf7dd3ee1f43002aac80..b419367307e8c31de411e3fd9d8c51e012099988 100644
--- a/DataQuality/DataQualityUtils/scripts/DeMoLib.py
+++ b/DataQuality/DataQualityUtils/scripts/DeMoLib.py
@@ -197,6 +197,15 @@ def initialize(system,yearTag,partitions,defects0,defectVeto,veto,signOff,year =
if system == "TRT":
yearTag["veto"] = {}
+ # Tags below are derived from the regular ones.
+ # They have the same defect/veto tags but have different options/GRL
+ # The defect/veto tags are derived from the ones defined in the standard tags given before "."
+ similarTags = {"Tier0_2018.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)"}
+ for iSimilar in similarTags.keys():
+ baseTag = iSimilar.split(".")[0]
+ yearTag["description"][iSimilar] = similarTags[iSimilar]
+ yearTag["defect"][iSimilar] = yearTag["defect"][baseTag]
+
partitions["color"] = {}
partitions["list"] = partitions["color"].keys()
@@ -211,7 +220,7 @@ def initialize(system,yearTag,partitions,defects0,defectVeto,veto,signOff,year =
# Some defects may not exist in past years. Remove them to avoid crashes
# WARNING: this fix does not work with multiple year plot
- defectVeto["description"] = {"BADGAS":"Bad gas mixture", # First per partition Tile defects
+ defectVeto["description"] = {"BADGAS":"Bad gas mixture", # First per partition TRT defects
"NODATA_06":"Large part of TRT off"}
defectVeto["color"]= {"BADGAS":kBlue-4, # First per partition LAr defects
@@ -240,13 +249,14 @@ def initialize(system,yearTag,partitions,defects0,defectVeto,veto,signOff,year =
# Tags below are derived from the regular ones.
# They have the same defect/veto tags but have different options/GRL
- similarTags = {"Tier0_2015.onlDelivNorm":"/Tier0 (deliv lumi)",
- "Reproc_2016.onlDelivNorm":"/2016 reproc. (deliv lumi)",
- "Tier0_2016.onlDelivNorm":"/Tier0 (deliv lumi)",
- "Reproc_2017.onlDelivNorm":"/2017 reproc. (deliv lumi)",
- "Tier0_2017.onlDelivNorm":"/Tier0 (deliv lumi)",
- "Reproc_2018.onlDelivNorm":"/2016 reproc. (deliv lumi)"}
-
+ # The defect/veto tags are derived from the ones defined in the standard tags given before "."
+ similarTags = {"Tier0_2015.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)",
+ "Reproc_2016.onlDelivNorm":"/2016 reproc. (onl. deliv. lumi.)",
+ "Tier0_2016.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)",
+ "Reproc_2017.onlDelivNorm":"/2017 reproc. (onl. deliv. lumi.)",
+ "Tier0_2017.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)",
+ "Reproc_2018.onlDelivNorm":"/2016 reproc. (onl. deliv. lumi.)",
+ "Tier0_2018.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)"}
for iSimilar in similarTags.keys():
baseTag = iSimilar.split(".")[0]
yearTag["description"][iSimilar] = similarTags[iSimilar]
@@ -319,6 +329,15 @@ def initialize(system,yearTag,partitions,defects0,defectVeto,veto,signOff,year =
if system == "Tile":
yearTag["veto"] = {}
+ # Tags below are derived from the regular ones.
+ # They have the same defect/veto tags but have different options/GRL
+ # The defect/veto tags are derived from the ones defined in the standard tags given before "."
+ similarTags = {"Tier0_2018.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)"}
+ for iSimilar in similarTags.keys():
+ baseTag = iSimilar.split(".")[0]
+ yearTag["description"][iSimilar] = similarTags[iSimilar]
+ yearTag["defect"][iSimilar] = yearTag["defect"][baseTag]
+
partitions["color"] = { 'EBA':kYellow-9,'EBC':kYellow,'LBA':kOrange,'LBC':kOrange-3}
partitions["list"] = partitions["color"].keys()
@@ -343,13 +362,22 @@ def initialize(system,yearTag,partitions,defects0,defectVeto,veto,signOff,year =
veto["COOL"] = {} # Veto name as defined in the COOL database
signOff["EXPR."] = ["TILE_UNCHECKED"]
- signOff["BULK"] = ["TILE_BULK_UNCHECKED"]
+ signOff["BULK"] = ["TILE_UNCHECKED"]
signOff["FINAL"] = []
################################# Jet/MET/EGamma/Tau/CaloGlobal defects
if system == "CaloCP":
yearTag["veto"] = {}
+ # Tags below are derived from the regular ones.
+ # They have the same defect/veto tags but have different options/GRL
+ # The defect/veto tags are derived from the ones defined in the standard tags given before "."
+ similarTags = {"Tier0_2018.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)"}
+ for iSimilar in similarTags.keys():
+ baseTag = iSimilar.split(".")[0]
+ yearTag["description"][iSimilar] = similarTags[iSimilar]
+ yearTag["defect"][iSimilar] = yearTag["defect"][baseTag]
+
partitions["color"] = { 'BARREL':kYellow-9,'CRACK':kRed-3,'ENDCAP':kBlue-3, # EGamma partitions
'B':kYellow-9,'CR':kRed-3,'E':kBlue-3, # Tau partitions
'CALB':kYellow-9,'CALEA':kRed-3,'CALC':kBlue-3} # CaloGlobal partitions
@@ -403,6 +431,15 @@ def initialize(system,yearTag,partitions,defects0,defectVeto,veto,signOff,year =
if system == "BTag":
yearTag["veto"] = {}
+ # Tags below are derived from the regular ones.
+ # They have the same defect/veto tags but have different options/GRL
+ # The defect/veto tags are derived from the ones defined in the standard tags given before "."
+ similarTags = {"Tier0_2018.onlDelivNorm":"/Tier0 (onl. deliv. lumi.)"}
+ for iSimilar in similarTags.keys():
+ baseTag = iSimilar.split(".")[0]
+ yearTag["description"][iSimilar] = similarTags[iSimilar]
+ yearTag["defect"][iSimilar] = yearTag["defect"][baseTag]
+
partitions["color"] = { } # No partition needed
partitions["list"] = partitions["color"].keys()
diff --git a/DataQuality/DataQualityUtils/scripts/DeMoScan.py b/DataQuality/DataQualityUtils/scripts/DeMoScan.py
index 30ad1687d2ea56a72b4ef7bb15e46d7d16e9cd59..7815cef95914928ac1f31f6614bc6b48801f1184 100644
--- a/DataQuality/DataQualityUtils/scripts/DeMoScan.py
+++ b/DataQuality/DataQualityUtils/scripts/DeMoScan.py
@@ -17,13 +17,11 @@ from ROOT import kTeal
from ROOT import gStyle,gROOT,gPad
-sys.path.append("../Misc")
+sys.path.append("/afs/cern.ch/user/l/larmon/public/prod/Misc")
from DeMoLib import strLumi, initialize
from gb import MakeTH1,SetXLabel,MakeLegend
-import runList
-
global debug
debug = False
#debug = True
@@ -36,16 +34,15 @@ import os,sys
from argparse import RawTextHelpFormatter,ArgumentParser
from DQDefects import DefectsDB
-gROOT.LoadMacro("../Misc/AtlasLabels.C")
parser = ArgumentParser(description='',formatter_class=RawTextHelpFormatter)
parser.add_argument('-r','--run',type=int,dest='parser_run',help='Run or run range (relevant only for lossPerRun)',nargs='*',action='store')
parser.add_argument('-y','--year',dest='parser_year',default = ["2018"],nargs='*',help='Year [Default: 2018]',action='store')
parser.add_argument('-t','--tag',dest='parser_tag',default = ["Tier0_2018"],nargs='*',help='Defect tag [Default: "Tier0_2018"]',action='store')
-parser.add_argument('-d','--defect',type=str,dest='parser_defect',default="",help='Defect to consider (if not specified: all)',action='store')
parser.add_argument('-v','--veto',type=str,dest='parser_veto',default="",help='Veto to consider (if not specified: all)',action='store')
parser.add_argument('-s','--system',dest='parser_system',default="LAr",help='System: LAr, CaloCP [Default: "LAr"]',action='store')
-parser.add_argument('-d','--directory',dest='parser_directory',default="/afs/cern.ch/user/l/larmon/public/prod/LADIeS",help='Directory to display',action='store')
+parser.add_argument('-d','--directory',dest='parser_directory',default=".",help='Directory to display',action='store')
+parser.add_argument('--defect',type=str,dest='parser_defect',default="",help='Defect to consider (if not specified: all)',action='store')
parser.add_argument('--noRecovPlot',dest='parser_noRecovPlot',help='Do not plot the recoverable histograms',action='store_false')
parser.add_argument('--lossPerRun',dest='parser_plotLPR',help='Plot the loss run and per type',action='store_true')
parser.add_argument('--diffTwoTags',dest='parser_diff2tags',help='Compare run by run the data losses for two tags (same year necessary)',action='store_true')
@@ -82,7 +79,14 @@ for iYear in args.parser_year:
yearTagList.append(yearTag)
yearTagDir[yearTag] = directory
yearTagTag[yearTag] = iTag # Used only to retrieve comments
- runGRL[yearTag] = runList.runlist["%s_grl"%iYear] # used only to determine if a run belongs to GRL in recap defects - Data in loss*.txt file NOT reliable
+
+ RunListDat = "RunList/grl-%s.dat"%(iYear)
+ if os.path.exists(RunListDat):
+ fRunList = open(RunListDat,'r')
+ runGRL[yearTag] = []
+ for iRun in fRunList.readlines():
+ runGRL[yearTag].append(int(iRun)) # used only to determine if a run belongs to GRL in recap defects - Data in loss*.txt file NOT reliable
+ fRunList.close()
if len(args.parser_year) == 1:
singleYear = True
@@ -187,6 +191,10 @@ for iYT in yearTagList:
stackResults[iYT] = {}
yearStatsArchiveFilename = '%s/TProfiles.root'%(yearTagDir[iYT])
+ if not (os.path.exists(yearStatsArchiveFilename)):
+ print "No %s found - > Skipping"%yearStatsArchiveFilename
+ continue
+
file[iYT] = TFile(yearStatsArchiveFilename)
h1Period_IntLuminosity[iYT] = file[iYT].Get("h1Period_IntLuminosity_archive")
subperiodNb[iYT] = h1Period_IntLuminosity[iYT].GetNbinsX()
diff --git a/DataQuality/DataQualityUtils/scripts/DeMoStatus.py b/DataQuality/DataQualityUtils/scripts/DeMoStatus.py
index 61688225caa9608f8b82935a49477e4ea9ec636a..3cd9671c0ff8b44d58bbf8b7b392b334e56b4eb3 100644
--- a/DataQuality/DataQualityUtils/scripts/DeMoStatus.py
+++ b/DataQuality/DataQualityUtils/scripts/DeMoStatus.py
@@ -13,7 +13,7 @@ from ROOT import TCanvas
from ROOT import gStyle,gPad
from ROOT import kBlack,kOrange,kGreen
-sys.path.append("../Misc")
+sys.path.append("/afs/cern.ch/user/l/larmon/public/prod/Misc")
from gb import MakeTH1,SetXLabel,MakeTProfile
@@ -67,8 +67,6 @@ def ATLASLabel(x,y,text=""):
from argparse import RawTextHelpFormatter,ArgumentParser
from ROOT import gROOT
-#gROOT.LoadMacro("AtlasLabels.C")
-
parser = ArgumentParser(description='',formatter_class=RawTextHelpFormatter)
parser.add_argument('-y','--year',dest='parser_year',default = ["2018"],nargs='*',help='Year [Default: 2018]',action='store')
parser.add_argument('-t','--tag',dest='parser_tag',default = ["Tier0_2018"],nargs='*',help='Defect tag [Default: "Tier0_2018"]',action='store')
diff --git a/DataQuality/DataQualityUtils/scripts/DeMoUpdate.py b/DataQuality/DataQualityUtils/scripts/DeMoUpdate.py
index ac1c580d4ea368f3d0d395f99c5f8a7e10cf5899..93b3c7d659694ad94da82976d0db26acfeacfc00 100644
--- a/DataQuality/DataQualityUtils/scripts/DeMoUpdate.py
+++ b/DataQuality/DataQualityUtils/scripts/DeMoUpdate.py
@@ -17,7 +17,7 @@ from ROOT import gStyle
import xmlrpclib
-sys.path.append("../Misc")
+sys.path.append("/afs/cern.ch/user/l/larmon/public/prod/Misc")
from LArMonCoolLib import GetLBTimeStamps,GetOnlineLumiFromCOOL,GetOfflineLumiFromCOOL,GetLBDuration,GetReadyFlag,GetNumberOfCollidingBunches
from gb import MakeTH1,SetXLabel,MakeTProfile
@@ -118,6 +118,7 @@ parser.add_argument('-y','--year',dest='parser_year',default = "2018",help='Year
parser.add_argument('-t','--tag',dest='parser_tag',default = "Tier0_2018",help='Defect tag [Default: "Tier0_2018"]',action='store')
parser.add_argument('-b','--batch',dest='parser_batchMode',help='Batch mode',action='store_true')
parser.add_argument('-s','--system',dest='parser_system',default="LAr",help='System: LAr, CaloCP [Default: "LAr"]',action='store')
+parser.add_argument('--runListUpdate',dest='parser_runListUpdate',help='Run list update. No other action allowed. Exit when done',action='store_true')
parser.add_argument('--weekly',dest='parser_weekly',help='Weekly report. No run range to specify',action='store_true')
parser.add_argument('--grlUpdate',dest='parser_grlUpdate',help='GRL update. No run range to specify',action='store_true')
parser.add_argument('--noPlot',dest='parser_noPlot',help='Do not plot the results',action='store_false')
@@ -182,14 +183,32 @@ else:
options['vetoTag']=""
options['yearStatsDir'] = "YearStats-%s/%s/%s"%(options['system'],options['year'],args.parser_tag)
+passfile = open("/afs/cern.ch/user/l/larmon/public/atlasdqmpass.txt")
+passwd = passfile.read().strip(); passfile.close()
+s = xmlrpclib.ServerProxy('https://%s@atlasdqm.cern.ch'%(passwd))
-#if not args.parser_run:
-# print "Please specify run number or run range with -r option"
-# os.system("rm -f %s"%tokenName)
-# sys.exit()
+############ Update the RunList/all-[year].dat files
+if args.parser_runListUpdate:
+ allRunListDat = "RunList/all-%s.dat"%(options['year'])
+ if os.path.exists(allRunListDat):
+ latestRun = 0
+ fRunList = open(allRunListDat,'r+')
+ for iRun in fRunList.readlines(): # Look for the latest run in RunList/all.dat
+ if int(iRun)>latestRun:
+ latestRun = int(iRun)
+ recentRuns = s.get_run_beamluminfo({'low_run':"%d"%(latestRun+1),'high_run':"%s"%(latestRun+1000)})
+
+ for iRecentRun in sorted(recentRuns.keys()):
+ if (recentRuns[iRecentRun][2]): # ATLAS ready
+ print "I am adding the new run with ATLAS ready: %s"%iRecentRun
+ fRunList.write("%s\n"%iRecentRun)
+ fRunList.close()
+ else:
+ print "No %s..."%allRunListDat
-if debug:
- print "I am querying DB to retrieve the periods"
+ os.system("rm -f %s"%tokenName)
+ print "I am exiting..."
+ sys.exit()
############ Fill runlist variable and change some options if single run
# available runs are defined in ~larmon/public/prod/LADIeS/RunList directory
@@ -252,10 +271,6 @@ if (options['updateYearStats'] and options['resetYearStats']):
runSpec = {} # Characteristics of each run: start, stop, data period, luminosity, computed inefficiencies
-passfile = open("/afs/cern.ch/user/l/larmon/public/atlasdqmpass.txt")
-passwd = passfile.read().strip(); passfile.close()
-s = xmlrpclib.ServerProxy('https://%s@atlasdqm.cern.ch'%(passwd))
-
# Fill the list of runs to be considered
if args.parser_weekly: # Weekly report - Look for the last 7-days runs + unsigned off
print "I am looking for all runs signed off in the past week and the older ones not yet signed off..."
@@ -889,7 +904,7 @@ if (len(runSpec.keys())>2 and runSpec['AllRuns']['Lumi']!=0):
column[canvasIndex][7].AddText("%s"%(runSpec[runNb]["period"]))
column[canvasIndex][8].AddText("%10s"%(runSpec[runNb]["signoff"]))
lineNb[canvasIndex] += 1
- if (lineNb[canvasIndex]==15 or runNb == "AllRuns"):
+ if (lineNb[canvasIndex]==25 or runNb == "AllRuns"):
for i in xrange(len(column[canvasIndex])):
column[canvasIndex][i].Draw()
c1[canvasIndex].SetWindowSize(1000,lineNb[canvasIndex]*40)
@@ -1036,4 +1051,5 @@ for irun in runSpec.keys():
errorLogFile.close()
os.system("rm -f %s"%tokenName)
-raw_input("I am done. Type <return> to exit...")
+if not options['batchMode']:
+ raw_input("I am done. Type <return> to exit...")