From 8a8b2f18eae89b1972b0899f8a0d12d8c58bdc08 Mon Sep 17 00:00:00 2001
From: Peter Onyisi <ponyisi@utexas.edu>
Date: Tue, 30 Jun 2020 19:37:40 +0200
Subject: [PATCH] Fix Py3 issues
---
.../DataQualityUtils/python/DQHistogramMergeMod.py | 6 +++---
.../DataQualityUtils/python/DQPostProcessMod.py | 2 +-
DataQuality/DataQualityUtils/python/DQProcMonitor.py | 4 ++--
.../DataQualityUtils/python/DQWebDisplayMod.py | 9 +++++----
DataQuality/DataQualityUtils/python/TestCases.py | 8 ++++----
DataQuality/DataQualityUtils/python/detmaskmod.py | 8 ++++----
DataQuality/DataQualityUtils/python/doZLumi.py | 2 +-
.../DataQualityUtils/python/dqu_subprocess.py | 4 ++--
.../DataQualityUtils/python/hancool_histo_mod.py | 4 ++--
DataQuality/DataQualityUtils/python/hancoolmod.py | 12 ++++++------
DataQuality/DataQualityUtils/python/handimod.py | 10 +++++-----
DataQuality/DataQualityUtils/python/hanwriter.py | 4 ++--
.../DataQualityUtils/python/messaging_listen.py | 4 ++--
DataQuality/DataQualityUtils/python/panic.py | 4 ++--
14 files changed, 41 insertions(+), 40 deletions(-)
diff --git a/DataQuality/DataQualityUtils/python/DQHistogramMergeMod.py b/DataQuality/DataQualityUtils/python/DQHistogramMergeMod.py
index f764a21d575..07212e69aff 100644
--- a/DataQuality/DataQualityUtils/python/DQHistogramMergeMod.py
+++ b/DataQuality/DataQualityUtils/python/DQHistogramMergeMod.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
import os
## Needed to correct ROOT behavior; see below
@@ -23,7 +23,7 @@ MODVERSION = '$Id: DQHistogramMergeMod.py,v 1.8 2009-05-12 11:38:35 ponyisi Exp
DoProcMon=False
if DoProcMon:
- import DQProcMonitor
+ from . import DQProcMonitor
DQProcMonitor.startProcMonThread()
def DQHistogramMerge( listFileName, outFileName, runPostProcessing, directoryRegularExpression=".*", histogramRegularExpression=".*", isIncremental=False, compressionLevel=1,debugLevel=0 ):
@@ -38,5 +38,5 @@ def DQHistogramMerge( listFileName, outFileName, runPostProcessing, directoryReg
mf.mergeLBintervals( outFileName )
if runPostProcessing:
- import DQPostProcessMod
+ from . import DQPostProcessMod
DQPostProcessMod.DQPostProcess( outFileName, isIncremental )
diff --git a/DataQuality/DataQualityUtils/python/DQPostProcessMod.py b/DataQuality/DataQualityUtils/python/DQPostProcessMod.py
index 7364c5256ca..17d173638ca 100644
--- a/DataQuality/DataQualityUtils/python/DQPostProcessMod.py
+++ b/DataQuality/DataQualityUtils/python/DQPostProcessMod.py
@@ -3,7 +3,7 @@ from __future__ import print_function
import shutil, re
-from dqu_subprocess import apply as _local_apply
+from .dqu_subprocess import apply as _local_apply
def _dolsrwrapper(fname):
import ROOT
diff --git a/DataQuality/DataQualityUtils/python/DQProcMonitor.py b/DataQuality/DataQualityUtils/python/DQProcMonitor.py
index 897a1e9144d..806e0d6da2e 100644
--- a/DataQuality/DataQualityUtils/python/DQProcMonitor.py
+++ b/DataQuality/DataQualityUtils/python/DQProcMonitor.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
from __future__ import print_function
#python 2.6 is buggy, it can't import modules in deamon threads one has to import them in to global
@@ -11,7 +11,7 @@ def formatRusage(rusage):
fieldnames=["utime","stime","maxrss","shrmem","ushrmem","ushrstk","minflt",
"majflt","nswap","inblock","oublock","msgsnd","msgrcv",
"nsignals","nvcontsw","nivcontsw"]
- return ", ".join(["=".join([n,"%s"%v] if type(v) is int else [n,"%f"%v]) for n,v in zip(fieldnames,rusage)])
+ return ", ".join(["=".join([n,"%s"%v] if isinstance(v, int) else [n,"%f"%v]) for n,v in zip(fieldnames,rusage)])
def DQProcMon(*args,**kwargs):
# from resource import getrusage, RUSAGE_SELF, RUSAGE_CHILDREN
diff --git a/DataQuality/DataQualityUtils/python/DQWebDisplayMod.py b/DataQuality/DataQualityUtils/python/DQWebDisplayMod.py
index 51b1fd0d66e..33d77d0155b 100755
--- a/DataQuality/DataQualityUtils/python/DQWebDisplayMod.py
+++ b/DataQuality/DataQualityUtils/python/DQWebDisplayMod.py
@@ -58,7 +58,7 @@ def DQWebDisplay( inputFilePath, runAccumulating, c ):
# we use the multiprocessing module to isolate code that possibly
# has memory leaks, so they don't build up over time
- if type(c.server) == str:
+ if isinstance(c.server, str):
if c.server == '':
c.server = []
else:
@@ -601,13 +601,14 @@ def findCacheFile( inputFilePath, run, stream, cache ):
def transferDirectoryToHandoffDir( dirName, localDir, targetDir, config ):
- import time, ConfigParser, shutil, glob
+ import time, shutil, glob
+ import six.moves.configparser as configparser
targetfname = repr(int(time.time())) + '-' + repr(os.getpid()) \
+ '-' + os.uname()[1] + '.tgz'
targetfile = os.path.join(targetDir, targetfname)
print('Creating tarball', targetfname, '...')
- parser = ConfigParser.ConfigParser()
+ parser = configparser.ConfigParser()
parser.set('DEFAULT', 'target', config.htmlDir)
parser.add_section('execute')
parser.set('execute', 'command', config.htmlDir + '/generateDQIndexFiles.py')
@@ -762,7 +763,7 @@ def importConfiguration(modname):
def email(msg, subject, whofrom, addressees):
import smtplib
from email.mime.text import MIMEText
- if type(addressees) == str:
+ if isinstance(addressees, str):
addressees = [addressees]
email = MIMEText(msg)
email['Subject'] = subject
diff --git a/DataQuality/DataQualityUtils/python/TestCases.py b/DataQuality/DataQualityUtils/python/TestCases.py
index dd7d251bcd6..0bc117b4186 100644
--- a/DataQuality/DataQualityUtils/python/TestCases.py
+++ b/DataQuality/DataQualityUtils/python/TestCases.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# None of this works, but keep it around in case someone wants to resurrect it later...
# - PO 20180419
@@ -23,7 +23,7 @@ class DQUTestCase(unittest.TestCase):
inlist = os.path.join(TESTING_DIR, 'test_merging')
self.outfile = os.path.join(outdir, 'data09_calophys.00128005.physics_CosmicMuons.root')
rv = os.system('cd %s ; DQHistogramMerge.py %s %s True' % (outdir, inlist, self.outfile))
- self.failUnless(rv==0, "DQHistogramMerge.py return code is nonzero")
+ self.assertTrue(rv==0, "DQHistogramMerge.py return code is nonzero")
def test02_WebDisplay(self):
'''Test that a terminal web display job works'''
@@ -34,7 +34,7 @@ class DQUTestCase(unittest.TestCase):
outdir = os.environ.get('TMPDIR', '.')
infile = os.path.join(TESTING_DIR, 'data09_calophys.00128005.physics_CosmicMuons.root')
rv = os.system('cd %s ; DQWebDisplay.py %s TestDisplay 123' % (outdir, infile))
- self.failUnless(rv==0, "DQWebDisplay.py return code is nonzero")
+ self.assertTrue(rv==0, "DQWebDisplay.py return code is nonzero")
def test_03_WebDisplay(self):
'''Test that a terminal web display job works in temporary accumulation mode'''
@@ -55,7 +55,7 @@ class DQUTestCase(unittest.TestCase):
shutil.copy(infile, os.path.join(cachedir, cachefilename))
rv = os.system('cd %s ; DQWebDisplay.py %s TestDisplay 123 True' % (outdir, infile))
- self.failUnless(rv==0, "DQWebDisplay.py return code is nonzero")
+ self.assertTrue(rv==0, "DQWebDisplay.py return code is nonzero")
def tearDown(self):
try:
diff --git a/DataQuality/DataQualityUtils/python/detmaskmod.py b/DataQuality/DataQualityUtils/python/detmaskmod.py
index 3dacb9a9c20..e3a6095728f 100644
--- a/DataQuality/DataQualityUtils/python/detmaskmod.py
+++ b/DataQuality/DataQualityUtils/python/detmaskmod.py
@@ -88,12 +88,12 @@ def decode(mask):
dm = eformat.helper.DetectorMask(mask)
rv = []
for keys, value in detmaskmap.items():
- if type(keys) == str:
+ if isinstance(keys, str):
keys = [keys]
if reduce(operator.or_,
[dm.is_set(getSubDetectorObj(key)) for key in keys]):
flags = value
- if type(flags) == str:
+ if isinstance(flags, str):
flags = [flags]
rv += list(flags)
@@ -104,12 +104,12 @@ def decodeBlack(mask, defects=False):
rv = []
dmap = detmaskmap if not defects else detmaskmap_defects
for keys, value in sorted(dmap.items()):
- if type(keys) == str:
+ if isinstance(keys, str):
keys = [keys]
if reduce(operator.and_,
[not dm.is_set(getSubDetectorObj(key)) for key in keys]):
flags = value
- if type(flags) == str:
+ if isinstance(flags, str):
flags = [flags]
rv += list(flags)
diff --git a/DataQuality/DataQualityUtils/python/doZLumi.py b/DataQuality/DataQualityUtils/python/doZLumi.py
index 605c4a763fe..617008858c9 100644
--- a/DataQuality/DataQualityUtils/python/doZLumi.py
+++ b/DataQuality/DataQualityUtils/python/doZLumi.py
@@ -45,7 +45,7 @@ def makeGRL(run, defect, fname):
print('done')
print('Query defects...', end='')
ddb = DQDefects.DefectsDB('COOLOFL_GLOBAL/%s' % dbinstance, tag=tag)
- ignores = set([_ for _ in ddb.defect_names if 'UNCHECKED' in _])
+ ignores = {_ for _ in ddb.defect_names if 'UNCHECKED' in _}
try:
defectiovs = ddb.retrieve(since = min(runs) << 32 | 1,
until = max(runs) << 32 | 0xffffffff,
diff --git a/DataQuality/DataQualityUtils/python/dqu_subprocess.py b/DataQuality/DataQualityUtils/python/dqu_subprocess.py
index 08b0d793483..6b3e2eecabf 100644
--- a/DataQuality/DataQualityUtils/python/dqu_subprocess.py
+++ b/DataQuality/DataQualityUtils/python/dqu_subprocess.py
@@ -10,7 +10,7 @@ def _local_apply_core(func, args, q):
os._exit(1)
def apply(func, args):
- from Queue import Empty
+ from six.moves.queue import Empty
from multiprocessing import Process
from multiprocessing.managers import SyncManager
import tempfile
@@ -34,7 +34,7 @@ def apply(func, args):
try:
rv = q.get(False)
except Empty:
- raise RuntimeError('daughter died while trying to execute %s%s' % (func.func_name, args))
+ raise RuntimeError('daughter died while trying to execute %s%s' % (func.__name__, args))
if isinstance(rv, BaseException):
if isinstance(rv, SystemExit):
print('SystemExit raised by daughter; ignoring')
diff --git a/DataQuality/DataQualityUtils/python/hancool_histo_mod.py b/DataQuality/DataQualityUtils/python/hancool_histo_mod.py
index 572064dc18f..5d3aa1db401 100644
--- a/DataQuality/DataQualityUtils/python/hancool_histo_mod.py
+++ b/DataQuality/DataQualityUtils/python/hancool_histo_mod.py
@@ -82,7 +82,7 @@ def stringGetInfo(file, rootFolder):
def ListHistoAssessments(xml, channel):
the_result_histo = "Undefined"
g = open(xml, 'r')
- while 1:
+ while True:
line = g.readline()
if not line:
break
@@ -160,7 +160,7 @@ def hancool_histo(inputFilePath="", input_run=-1, dbConnectionHisto="", dqmfOflH
filename = "run_"+str(run)+"_han.root"
# filling OFLH DB for histos (global run)
- for pair in folderMapHisto.iteritems():
+ for pair in folderMapHisto.items():
i = 0
number = 0
print(pair[0], pair[1])
diff --git a/DataQuality/DataQualityUtils/python/hancoolmod.py b/DataQuality/DataQualityUtils/python/hancoolmod.py
index 02073e0d605..d53a947a4e5 100644
--- a/DataQuality/DataQualityUtils/python/hancoolmod.py
+++ b/DataQuality/DataQualityUtils/python/hancoolmod.py
@@ -136,7 +136,7 @@ intervalType = {
def getLimits(name):
try:
import re
- import detmaskmod
+ from . import detmaskmod
runNumber = re.match(r'run_(\d+)_.*han.root', name).group(1)
max_hi_limit = detmaskmod.getNumLumiBlocks(int(runNumber))+1
if (name.find('minutes10_') > -1):
@@ -287,7 +287,7 @@ def hancool(runNumber=3070,
def detmask_defects(runNumber, ddb):
- import detmaskmod
+ from . import detmaskmod
blacks = detmaskmod.decodeBlack(detmaskmod.getRunMask(runNumber),
defects=True)
nlbs = detmaskmod.getNumLumiBlocks(runNumber)
@@ -340,7 +340,7 @@ def ctp_defects(d, i, runNumber):
rv.append(defect_iov(mapping[defect], message, False, lb, lb+1))
if overflow_bad_lbs[defect]:
message += '; defect occurred past end of monitoring histogram, marking end of run as bad'
- import detmaskmod # ugly: could happen for more than one defect - should be cheap though
+ from . import detmaskmod # ugly: could happen for more than one defect - should be cheap though
nlbs = detmaskmod.getNumLumiBlocks(runNumber)
rv.append(defect_iov(defect, message,
False, when.GetNbinsX(), nlbs+1))
@@ -413,7 +413,7 @@ def sct_perlb_defects(d, i, runNumber):
rv.append(defect_iov(dname, message, False, lb, lb+1))
if overflow_bad_lbs[dname]:
message += '; defect occurred past end of monitoring histogram, marking end of run as bad'
- import detmaskmod # ugly: could happen for more than one defect - should be cheap though
+ from . import detmaskmod # ugly: could happen for more than one defect - should be cheap though
nlbs = detmaskmod.getNumLumiBlocks(runNumber)
rv.append(defect_iov(dname, message,
False, when.GetNbinsX(), nlbs+1))
@@ -452,7 +452,7 @@ def sct_eff_defect(d, i, runNumber):
h2 = d.Get('InnerDetector/SCT/Summary/SctTotalEff_/Results/Status')
if not h1 or not h2:
return None
- badstatuses = set(['Yellow', 'Red'])
+ badstatuses = {'Yellow', 'Red'}
statuscheck = []
for h in h1, h2:
status = set(x.GetName() for x in h.GetListOfKeys())
@@ -486,7 +486,7 @@ def dqmf_node_defect(node, defect, badstatuses=['Red']):
def hancool_defects(runNumber, filePath="./", dbConnection="", db_tag='HEAD', isESn=True):
- import pix_defect
+ from . import pix_defect
analyzers = []
if isESn:
# CTP
diff --git a/DataQuality/DataQualityUtils/python/handimod.py b/DataQuality/DataQualityUtils/python/handimod.py
index 35535407313..94189b976b7 100644
--- a/DataQuality/DataQualityUtils/python/handimod.py
+++ b/DataQuality/DataQualityUtils/python/handimod.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
from __future__ import print_function
import os
@@ -606,7 +606,7 @@ def makeOneHistFile(htmlDir, name, subname, sp, runlistLoc, compare, jsRoot):
else:
name = ' '.join([namecache[-1]])
namecache = []
- import urllib
+ from six.moves import urllib
resultname = name.rsplit(':', 1)[0]
resultval = sp[cc-1]
if algorithm == 'RepeatAlgorithm' and resultname.endswith('|Status'):
@@ -615,15 +615,15 @@ def makeOneHistFile(htmlDir, name, subname, sp, runlistLoc, compare, jsRoot):
if compare and run is not None:
if period_type == 'run':
queryurl = 'http://atlasdqm.cern.ch:8080/dqmfquery/query?histogram=%s&result=%s&error=&stream=%s&period_type=%s&source=tier0&proc_ver=%s&low_run=%s&high_run=&low_y=&high_y=&outputtype=png' % (
- urllib.quote_plus(subname+'/'+sp[0]), urllib.quote_plus(resultname), stream.strip(), period_type, proc_ver, int(run)-1000)
+ urllib.parse.quote_plus(subname+'/'+sp[0]), urllib.parse.quote_plus(resultname), stream.strip(), period_type, proc_ver, int(run)-1000)
k.write(
'<tr><td align="right"><b><a href="%s">%s</a>:</b></td>' % (queryurl, resultname))
k.write('<td>'+resultval+'</td></tr>\n')
else:
queryurl1 = 'http://atlasdqm.cern.ch:8080/dqmfquery/query?histogram=%s&result=%s&error=&stream=%s&period_type=%s&source=tier0&proc_ver=%s&low_run=%s&high_run=&low_y=&high_y=&outputtype=png' % (
- urllib.quote_plus(subname+'/'+sp[0]), urllib.quote_plus(resultname), stream.strip(), period_type, proc_ver, int(run)-1000)
+ urllib.parse.quote_plus(subname+'/'+sp[0]), urllib.parse.quote_plus(resultname), stream.strip(), period_type, proc_ver, int(run)-1000)
queryurl2 = 'http://atlasdqm.cern.ch:8080/dqmfquery/query?histogram=%s&result=%s&error=&stream=%s&period_type=%s&source=tier0&proc_ver=%s&low_run=%s&high_run=%s&low_y=&high_y=&outputtype=png' % (
- urllib.quote_plus(subname+'/'+sp[0]), urllib.quote_plus(resultname), stream.strip(), period_type, proc_ver, run, run)
+ urllib.parse.quote_plus(subname+'/'+sp[0]), urllib.parse.quote_plus(resultname), stream.strip(), period_type, proc_ver, run, run)
k.write(
'<tr><td align="right"><b><a href="%s">%s</a>:</b></td>' % (queryurl1, resultname))
k.write(
diff --git a/DataQuality/DataQualityUtils/python/hanwriter.py b/DataQuality/DataQualityUtils/python/hanwriter.py
index 3995982cf99..66d2677c1ef 100755
--- a/DataQuality/DataQualityUtils/python/hanwriter.py
+++ b/DataQuality/DataQualityUtils/python/hanwriter.py
@@ -114,7 +114,7 @@ class Node(DQHanConfMaker.Node):
if self.nodeType != Node.DOCUMENT:
writer.write(" { %s" % (newl))
if self.attributes:
- for key, attribute in self.attributes.iteritems():
+ for key, attribute in self.attributes.items():
writer.write("%s %s = %s%s" % (indent, key, attribute, newl))
if self.subnodes:
for node in self.subnodes:
@@ -707,7 +707,7 @@ def _findAllDQBaseObjects(rootlist):
if not isinstance(dqbase, DQBase):
raise ValueError(
'%s is not a valid DQBase object; this should never happen' % dqbase)
- retset = set([dqbase])
+ retset = {dqbase}
for rel in dqbase.getAllRelatedObjects():
retset |= recurse(rel)
return retset
diff --git a/DataQuality/DataQualityUtils/python/messaging_listen.py b/DataQuality/DataQualityUtils/python/messaging_listen.py
index 5137d0fd293..6b1080ef905 100644
--- a/DataQuality/DataQualityUtils/python/messaging_listen.py
+++ b/DataQuality/DataQualityUtils/python/messaging_listen.py
@@ -28,7 +28,7 @@ class ATLASDQMListener(object):
socket.AF_INET,
socket.SOCK_STREAM)]
- import stompconfig
+ from . import stompconfig
self.conns = []
if hasattr(self.listener, 'conn'):
self.listener.conn=[]
@@ -66,7 +66,7 @@ class ATLASDQMListener(object):
socket.AF_INET,
socket.SOCK_STREAM)]
- import stompconfig
+ from . import stompconfig
self.conns = []
if hasattr(self.listener, 'conn'):
self.listener.conn=[]
diff --git a/DataQuality/DataQualityUtils/python/panic.py b/DataQuality/DataQualityUtils/python/panic.py
index b92e312d49b..b0efb85a371 100644
--- a/DataQuality/DataQualityUtils/python/panic.py
+++ b/DataQuality/DataQualityUtils/python/panic.py
@@ -1,11 +1,11 @@
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
DEST='/queue/atlas.dqm.panic'
def panic(msg):
import stomp
import json
- import stompconfig
+ from . import stompconfig
import sys, os
import traceback
import time
--
GitLab