Commit ccadf402 authored by Remi Mommsen's avatar Remi Mommsen
Browse files

references #105: add option to read plain text file defining FEDbuilders and...

references #105: add option to read plain text file defining FEDbuilders and use fits to estimate maximum throughput per FB
parent bb1cbeac
......@@ -2,6 +2,7 @@
import collections
import datetime
import functools
import gzip
import os
import re
......@@ -13,16 +14,58 @@ from xml.etree.ElementTree import QName as QN
import XMLtools
import FitModels
class CheckFBthroughput:
def __init__(self,args):
self.args = vars(args)
self.config = self.readConfigFile(self.args['xmlConfigFile'])
if self.args['plain']:
self.RUs = self.readPlainFile(self.args['configFile'])
else:
config = self.readXMLFile(self.args['configFile'])
self.RUs = self.getRUs(config)
self.params = self.readParamFile(self.args['paramFile'])
#self.maxThroughputModels = {
# 8 : functools.partial(FitModels.linearPlateauModel,parameters=(0,0.79978,12149.5)),
# 10 : functools.partial(FitModels.linearPlateauModel,parameters=(0,0.99971, 8703.38)),
# 12 : functools.partial(FitModels.linearPlateauModel,parameters=(0,1.19966, 7789.65)),
# 16 : functools.partial(FitModels.linearPlateauModel,parameters=(0,1.59955, 5635.41)),
# 20 : functools.partial(FitModels.linearPlateauModel,parameters=(0,1.99939, 4334.02)),
# 27 : functools.partial(FitModels.linearPlateauModel,parameters=(0,2.69930, 3024.79)),
# }
self.maxThroughputModels = {
8 : functools.partial(FitModels.linearPlateauModel, parameters=(5.13191,2.31395,4193.95)),
10 : functools.partial(FitModels.linearPlateauModel, parameters=(11.1087,2.85291,3115.77)),
12 : functools.partial(FitModels.linearPlateauModel, parameters=(74.2754,3.02603,2991.47)),
16 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.17331,4388.68)),
20 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.54145,3774.19)),
27 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,5.06345,3206.14)),
}
self.maxFragmentSizes = {
100 : {
8 : 12000,
10 : 8200,
12 : 7000,
16 : 5000,
20 : 4100,
27 : 2800,
32 : 1900,
},
50 : {
8 : 12000,
10 : 12000,
12 : 12000,
16 : 11000,
20 : 8200,
27 : 6000,
32 : 3500,
}
}
def readConfigFile(self,configFile):
def readXMLFile(self,configFile):
if os.path.exists(configFile):
ETroot = ET.parse(configFile).getroot()
elif os.path.exists(configFile+".gz"):
......@@ -34,6 +77,17 @@ class CheckFBthroughput:
return ETroot
def readPlainFile(self,configFile):
rus = []
with open(configFile) as file:
for line in file:
result = re.match(r'.*FEDBuilder:.*id=(\d+).*name=([\w+-]+).*srcIds: ([\d\(\),; ]+)',line)
if result:
fedIds = re.findall(r'[\(\),; ]*(\d+)',result.group(3))
rus.append((["fb_%02d"%int(result.group(1)),result.group(2)],fedIds))
return rus
def readParamFile(self,paramFile):
params = {}
with open(paramFile) as file:
......@@ -41,22 +95,20 @@ class CheckFBthroughput:
for line in file:
try:
(fedIds,a,b,c,rms) = line.split(',',5)
params[fedIds] = (float(a),float(b),float(c),None)
except ValueError:
(fedIds,a,b,c,rms,_) = line.split(',',6)
try:
params[fedIds] = (float(a),float(b),float(c))
except ValueError:
pass
(fedIds,a,b,c,d,rms) = line.split(',',6)
params[fedIds] = (float(a),float(b),float(c),float(d))
except Exception as e:
print("Failed to parse: "+line)
raise e
return params
def getRUs(self):
def getRUs(self,config):
rus = []
xcns = re.match(r'\{(.*?)\}Partition',self.config.tag).group(1) ## Extract xdaq namespace
for context in self.config.getiterator(str(QN(xcns,'Context'))):
xcns = re.match(r'\{(.*?)\}Partition',config.tag).group(1) ## Extract xdaq namespace
for context in config.getiterator(str(QN(xcns,'Context'))):
for app in context.getiterator(str(QN(xcns,'Application'))):
if app.attrib['class'] in ('evb::EVM','evb::RU'):
fedIds = []
......@@ -67,39 +119,47 @@ class CheckFBthroughput:
if 'fedSourceIds' in prop.tag:
for item in prop:
fedIds.append(item.text)
rus.append((group,fedIds))
rus.append((group[1:],fedIds))
return rus
def calculateThroughput(self):
eventSize = 0
tooLargeFB = []
for ru in self.getRUs():
for ru in self.RUs:
superFragmentSize = 0.
for fed in ru[1]:
try:
param = self.params[fed]
fedSize = param[0] + param[1]*self.args['relEventSize'] + param[2]*self.args['relEventSize']**2
if param[3]:
fedSize += param[3]*self.args['relEventSize']**3
except KeyError:
fedSize = 2048
print("WARNING: FED %s not found in paramFile. Using %d Bytes" % (fed,fedSize))
superFragmentSize += int(fedSize+4)&~0x7
eventSize += superFragmentSize/1e6
throughput = (superFragmentSize*self.args['triggerRate'])/1e6
if throughput > self.args['maxThroughput']:
tooLargeFB.append((ru[0][1:],throughput,ru[1]))
triggerRate = self.args['triggerRate']
throughput = (superFragmentSize*triggerRate)/1e6
fbSize = len(ru[1])
averageFedSize = superFragmentSize/fbSize
maxFragmentSizes = self.maxFragmentSizes[triggerRate] if triggerRate in self.maxFragmentSizes else self.maxFragmentSizes[min(self.maxFragmentSizes.keys(), key=lambda k: abs(k-triggerRate-1))]
maxFedSize = maxFragmentSizes[fbSize] if fbSize in maxFragmentSizes else maxFragmentSizes[min(maxFragmentSizes.keys(), key=lambda k: abs(k-fbSize-1))]
maxThroughputModel = self.maxThroughputModels[fbSize] if fbSize in self.maxThroughputModels else self.maxThroughputModels[min(self.maxThroughputModels.keys(), key=lambda k: abs(k-fbSize-1))]
maxThroughput = maxThroughputModel((averageFedSize,))/1e3
if self.args['printAll'] or throughput > maxThroughput or averageFedSize > maxFedSize:
#print(" %1.2f GB/s (< %1.2f GB/s) from %2d FEDs with average %4d Bytes (< %d Bytes) : %s %s" % (throughput,maxThroughput,fbSize,averageFedSize,maxFedSize,ru[0],ru[1]))
print(" %1.2f GB/s (< %1.2f GB/s) from %2d FEDs with average %4d Bytes : %s %s" % (throughput,maxThroughput,fbSize,averageFedSize,ru[0],ru[1]))
print("Total event size: %1.2f MB"%(eventSize))
for fb in tooLargeFB:
print(" %1.2f GB/s from %d FEDs : %s %s" % (fb[1],len(fb[2]),fb[0],fb[2]))
if __name__ == "__main__":
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("xmlConfigFile",help="configuration file")
parser.add_argument("configFile",help="configuration file")
parser.add_argument("paramFile",help="calculate the FED fragment sizes using the parameters in the given file")
parser.add_argument("relEventSize",type=float,help="scale the event size by this factor")
parser.add_argument("-a","--printAll",action='store_true',help="print the throughput of all FB irrespectively if they exceed the maximum or not")
parser.add_argument("-p","--plain",action='store_true',help="use plain text file instead of XML configuration")
parser.add_argument("-t","--triggerRate",default=100,type=int,help="L1 trigger rate in kHz [default: %(default)s]")
parser.add_argument("-m","--maxThroughput",default=3.3,type=float,help="report FB which have a throughput exceeding this value in GB/s [default: %(default)s]")
checkFBthroughput = CheckFBthroughput( parser.parse_args() )
checkFBthroughput.calculateThroughput()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment