Commit 95f5f2ed authored by Remi Mommsen's avatar Remi Mommsen Committed by Dainius Simelevicius
Browse files

references #158: recent improvements to EvB scripts

parent 9d2dcf64
from TestCase import TestCase
from Context import RU,BU
class case_2x1_bigBlockSize(TestCase):
def runTest(self):
self.configureEvB()
self.enableEvB()
self.checkEVM(12*4096,10)
self.checkRU(12*4096)
self.checkBU(24*4096,10)
self.haltEvB()
def fillConfiguration(self,symbolMap):
blockSize='0xfffc0'
ruParams = [
('inputSource','string','Local'),
('blockSize','unsignedInt',blockSize)]
ruParams.extend(self.getFedParams(range(13,25),4096,0))
self._config.add( RU(symbolMap,ruParams,blockSize) )
self._config.add( RU(symbolMap,ruParams,blockSize) )
self._config.add( BU(symbolMap,[
('dropEventData','boolean','true'),
('lumiSectionTimeout','unsignedInt','0'),
('eventsPerRequest','unsignedInt','64'),
('maxEvtsUnderConstruction','unsignedInt','256')
],blockSize) )
This diff is collapsed.
{
"TRANSFER_MODE": "TIER0_TRANSFER_OFF",
"CMSSW_VERSION": "CMSSW_10_4_0",
"SCRAM_ARCH": "slc7_amd64_gcc700"
"CMSSW_VERSION": "CMSSW_11_0_0",
"SCRAM_ARCH": "slc7_amd64_gcc820"
}
......@@ -7,8 +7,8 @@ from TestCase import *
class ConfigCase(TestCase):
def __init__(self,config,stdout,fedSizeScaleFactors,defaultFedSize,afterStartupCallback = None):
TestCase.__init__(self,config,stdout, afterStartupCallback = afterStartupCallback)
def __init__(self,config,stdout,fedSizeScaleFactors,defaultFedSize,afterStartupCallback=None,waitForStateTransitions=False):
TestCase.__init__(self,config,stdout,afterStartupCallback,waitForStateTransitions)
self.fedSizeScaleFactors = fedSizeScaleFactors
self.defaultFedSize = defaultFedSize
......@@ -20,13 +20,12 @@ class ConfigCase(TestCase):
def calculateFedSize(self,fedId,fragSize,fragSizeRMS):
if len(self.fedSizeScaleFactors):
try:
(a,b,c,rms) = self.fedSizeScaleFactors[fedId]
(a,b,c,d,rms) = self.fedSizeScaleFactors[fedId]
relSize = fragSize / self.defaultFedSize
fedSize = a + b*relSize + c*relSize*relSize
fedSize = a + b*relSize + c*relSize*relSize + d*relSize*relSize*relSize
fedSizeRMS = fedSize * rms
return (int(fedSize+4)&~0x7,int(fedSizeRMS+4)&~0x7)
except KeyError:
if fedId != '0xffffffff':
if fedId not in ('0xffffffff','9999'):
print("Missing scale factor for FED id "+fedId)
else:
return TestCase.calculateFedSize(self,fedId,fragSize,fragSizeRMS)
return TestCase.calculateFedSize(self,fedId,fragSize,fragSizeRMS)
This diff is collapsed.
......@@ -41,12 +41,12 @@ class Context:
return context
def addPeerTransport(self):
def addPeerTransport(self,maxMessageSize):
try:
if 'd3v' in self.hostinfo['i2oHostname'] or 'rbs1v0' in self.hostinfo['i2oHostname'] or 'ebs0v0' in self.hostinfo['i2oHostname'] or 'ebs1v0' in self.hostinfo['i2oHostname']:
app = self.getPtIbvApplication()
app = self.getPtIbvApplication(maxMessageSize)
else:
app = self.getPtUtcpApplication()
app = self.getPtUtcpApplication(maxMessageSize)
app.params['i2oHostname'] = self.hostinfo['i2oHostname']
app.params['i2oPort'] = self.hostinfo['i2oPort']
app.params['network'] = 'evb'
......@@ -56,7 +56,7 @@ class Context:
raise KeyError("Cannot find key "+str(e)+" for "+self.hostinfo['soapHostname'])
def getPtUtcpApplication(self):
def getPtUtcpApplication(self,maxMessageSize):
properties = [
('protocol','string','atcp'),
('maxClients','unsignedInt','9'),
......@@ -64,14 +64,14 @@ class Context:
('ioQueueSize','unsignedInt','65536'),
('eventQueueSize','unsignedInt','65536'),
('maxReceiveBuffers','unsignedInt','12'),
('maxBlockSize','unsignedInt','65537')
('maxBlockSize','unsignedInt',maxMessageSize)
]
app = Application.Application('pt::utcp::Application',Context.ptInstance,properties)
app.params['protocol'] = 'atcp'
return app
def getPtIbvApplication(self):
def getPtIbvApplication(self,maxMessageSize):
interface = SymbolMap.getI2OInterfaceName(self.hostinfo['i2oHostname'])
properties = [
('iaName','string',interface),
......@@ -81,7 +81,7 @@ class Context:
('memAllocTimeout','string','PT1S'),
('sendWithTimeout','boolean','true'),
('useRelay','boolean','false'),
('maxMessageSize','unsignedInt','0x3fff0')
('maxMessageSize','unsignedInt',maxMessageSize)
]
if self.role is 'EVM':
properties.extend([
......@@ -195,13 +195,13 @@ class FEROL(Context):
class RU(Context):
instance = 0
def __init__(self,symbolMap,properties=[]):
def __init__(self,symbolMap,properties=[],maxMessageSize='0x10000'):
if RU.instance == 0:
role = 'EVM'
else:
role = 'RU'
Context.__init__(self,role,symbolMap.getHostInfo('RU'+str(RU.instance)))
self.addPeerTransport()
self.addPeerTransport(maxMessageSize)
self.addRuApplication(properties)
RU.instance += 1
......@@ -307,9 +307,9 @@ class RU(Context):
class BU(Context):
instance = 0
def __init__(self,symbolMap,properties=[]):
def __init__(self,symbolMap,properties=[],maxMessageSize='0x10000'):
Context.__init__(self,'BU',symbolMap.getHostInfo('BU'+str(BU.instance)))
self.addPeerTransport()
self.addPeerTransport(maxMessageSize)
self.addBuApplication(properties)
BU.instance += 1
......@@ -360,13 +360,13 @@ class BU(Context):
class RUBU(RU):
def __init__(self,symbolMap,ruProperties=[],buProperties=[]):
def __init__(self,symbolMap,ruProperties=[],buProperties=[],maxMessageSize='0x10000'):
if RU.instance == 0:
role = 'EVMBU'
else:
role = 'RUBU'
Context.__init__(self,role,symbolMap.getHostInfo('RU'+str(RU.instance)))
self.addPeerTransport()
self.addPeerTransport(maxMessageSize)
self.addRuApplication(ruProperties)
self.addBuApplication(buProperties)
RU.instance += 1
......
......@@ -15,7 +15,14 @@ except ImportError:
def getI2OInterfaceName(i2oHostname):
""" return the IBV interface name as function of the hostname """
if 'd3vrubu-c2e3' in i2oHostname:
if 'd3vrubu-c2e34-20-01' in i2oHostname or 'd3vrubu-c2e34-27-01' in i2oHostname:
if 'd3vfus1v0' in i2oHostname:
interface='mlx5_0'
elif 'd3vfbs1v0' in i2oHostname:
interface='mlx5_1'
else:
interface='mlx5_2'
elif 'd3vrubu-c2e3' in i2oHostname:
if 'd3vfus1v0' in i2oHostname:
interface='mlx5_1'
elif 'd3vfbs1v0' in i2oHostname:
......@@ -64,6 +71,7 @@ class SymbolMap:
val = self._hostname
if useRoCE and 'd3vrbs1v0' in str(val):
val = val.replace('d3vrbs1v0','d3vfbs1v0')
#val = val.replace('d3vrbs1v0','d3vfus1v0')
self._map[key] = val
if key == 'LAUNCHER_BASE_PORT':
launcherPort = int(val)
......@@ -113,6 +121,7 @@ class SymbolMap:
return rawMap
def __readFromPythonFile(self, symbolMapFile):
"""reads the SymbolMap from a python code file"""
......@@ -133,6 +142,10 @@ class SymbolMap:
hostInfo = {'launcherPort':self._map[hostType + '_LAUNCHER_PORT'],
'soapHostname':self._map[hostType + '_SOAP_HOST_NAME'],
'soapPort':self._map[hostType + '_SOAP_PORT']}
try:
hostInfo['whitelist'] = "["+self._map[hostType + '_FUS']+"]"
except KeyError:
hostInfo['whitelist'] = "[]"
try:
hostInfo['i2oHostname'] = self._map[hostType + '_I2O_HOST_NAME']
hostInfo['i2oPort'] = self._map[hostType + '_I2O_PORT']
......@@ -173,3 +186,4 @@ if __name__ == "__main__":
symbolMap = SymbolMap(os.environ["EVB_TESTER_HOME"]+"/cases/standaloneSymbolMap.txt")
print(symbolMap._map)
print(symbolMap.launchers)
print(symbolMap.getHostInfo('RU0'))
......@@ -100,7 +100,7 @@ def init_worker():
class TestCase:
def __init__(self,config,stdout,afterStartupCallback = None):
def __init__(self,config,stdout,afterStartupCallback=None,waitForStateTransitions=False):
"""
@param afterStartupCallback is a callback which will be called
after the EVB has been enabled when nbMeasurements is zero.
......@@ -114,6 +114,7 @@ class TestCase:
self._config = config
self._pool = mp.Pool(20,init_worker)
self.afterStartupCallback = afterStartupCallback
self.waitForStateTransitions = waitForStateTransitions
# standard multiprocessing pool does not work well with exceptions
# over xmlrpc, so we use a pool based on threads (instead of procecesses)
......@@ -529,7 +530,7 @@ class TestCase:
def checkEVM(self,superFragmentSize,eventRate=100):
self.checkAppParam("superFragmentSize","unsignedInt",superFragmentSize,operator.eq,"EVM")
self.checkAppParam("eventCount","unsignedLong",500,operator.ge,"EVM")
self.checkAppParam("eventCount","unsignedLong",5*eventRate,operator.ge,"EVM")
self.checkAppParam("eventRate","unsignedInt",eventRate,operator.ge,"EVM")
......@@ -768,15 +769,23 @@ class TestCase:
"""
runNumber=time.strftime("%s",time.localtime())
if self.waitForStateTransitions:
raw_input("Press Enter to configure...")
self.configureEvB(maxTries=30)
if self.waitForStateTransitions:
raw_input("Press Enter to enable...")
self.enableEvB(maxTries=30,runNumber=runNumber)
self.checkRate()
def prepare(self,testname,maxTries=10):
self.startXDAQs(testname)
if self.waitForStateTransitions:
raw_input("Press Enter to initialize...")
self.sendCmdToExecutive()
self.waitForState(('Halted','uninitialized','Initialized'),maxTries)
if self.waitForStateTransitions:
raw_input("Press Enter to connect...")
self.startPt()
......@@ -787,14 +796,14 @@ class TestCase:
multiple times in case of problems.
"""
if not args['verbose']:
if not args['verbose'] and not args['waitForStateTransitions']:
self._origStdout.write("%dB:"%(fragSize))
self._origStdout.flush()
tries = 0
while tries < 10:
try:
dataPoints = self.doIt(fragSize,fragSizeRMS,args)
if not args['verbose']:
if not args['verbose'] and not args['waitForStateTransitions']:
self._origStdout.write("%dMB/s "%(self.getThroughputMB(dataPoints)))
self._origStdout.flush()
else:
......@@ -804,7 +813,8 @@ class TestCase:
return dataPoints
except (FailedState,StateException,ValueException) as e:
print(" failed: "+str(e))
#raw_input("Press Enter to retry...")
if self.waitForStateTransitions:
raw_input("Press Enter to retry...")
while tries < 10:
tries += 1
try:
......@@ -849,4 +859,8 @@ class TestCase:
dataPoints.append( self.getDataPoint() )
self.haltEvB()
if self.waitForStateTransitions:
raw_input("Press Enter to destroy...")
return dataPoints
......@@ -3,6 +3,7 @@
import argparse
import datetime
import os
import socket
import subprocess
import sys
import time
......@@ -15,6 +16,9 @@ from TestCase import TestCase
class BadConfig(Exception):
pass
class Launcher(Exception):
pass
class Tee(object):
def __init__(self, *files):
......@@ -164,4 +168,7 @@ class TestRunner:
import xmlrpclib
server = xmlrpclib.ServerProxy("http://%s:%s" % (launcher[0],launcher[1]))
server.stopLauncher()
try:
server.stopLauncher()
except socket.error as e:
raise Launcher("Cannot contact launcher on "+launcher[0]+":"+str(launcher[1])+": "+str(e))
......@@ -17,16 +17,29 @@ import XMLtools
import FitModels
def readPlainFile(configFile):
rus = []
with open(configFile) as file:
for line in file:
result = re.match(r'.*FEDBuilder:.*id=(\d+).*name=([\w+-]+).*srcIds: ([\d\(\),; ]+)',line)
if result:
fedIds = re.findall(r'[\(\),; ]*(\d+)',result.group(3))
rus.append((["fb_%02d"%int(result.group(1)),result.group(2)],fedIds))
return rus
class CheckFBthroughput:
def __init__(self,args):
self.args = vars(args)
if self.args['plain']:
self.RUs = self.readPlainFile(self.args['configFile'])
self.RUs = readPlainFile(self.args['configFile'])
else:
config = self.readXMLFile(self.args['configFile'])
self.RUs = self.getRUs(config)
self.params = self.readParamFile(self.args['paramFile'])
#self.getModelParamsSkylake()
self.getModelParamsAMD()
#self.maxThroughputModels = {
# 8 : functools.partial(FitModels.linearPlateauModel,parameters=(0,0.79978,12149.5)),
# 10 : functools.partial(FitModels.linearPlateauModel,parameters=(0,0.99971, 8703.38)),
......@@ -35,32 +48,82 @@ class CheckFBthroughput:
# 20 : functools.partial(FitModels.linearPlateauModel,parameters=(0,1.99939, 4334.02)),
# 27 : functools.partial(FitModels.linearPlateauModel,parameters=(0,2.69930, 3024.79)),
# }
def getModelParamsSkylake(self):
#CHEP2019
self.maxThroughputModels = {
8 : functools.partial(FitModels.linearPlateauModel, parameters=(5.13191,2.31395,4193.95)),
10 : functools.partial(FitModels.linearPlateauModel, parameters=(11.1087,2.85291,3115.77)),
12 : functools.partial(FitModels.linearPlateauModel, parameters=(74.2754,3.02603,2991.47)),
16 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.17331,4388.68)),
20 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.54145,3774.19)),
27 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,5.06345,3206.14)),
8 : functools.partial(FitModels.linearPlateauModel, parameters=(0,2.24928,4316.72)),
10 : functools.partial(FitModels.linearPlateauModel, parameters=(29.8127,2.73777,3168.96)),
12 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,3.44992,5752.32)),
16 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,3.94864,4944.05)),
20 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.59320,3892.66)),
27 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,5.13860,3238.54)),
36 : functools.partial(FitModels.quadraticPlateauModel,parameters=(37.5104,5.20777,2989.20)),
44 : functools.partial(FitModels.quadraticPlateauModel,parameters=(177.452,4.82780,3098.82)),
55 : functools.partial(FitModels.quadraticPlateauModel,parameters=(289.654,4.47295,3394.14)),
}
self.maxFragmentSizes = {
100 : {
8 : 12000,
10 : 8200,
12 : 7000,
16 : 5000,
20 : 4100,
27 : 2800,
32 : 1900,
12 : 7500,
16 : 5500,
20 : 4300,
27 : 3000,
36 : 2000,
44 : 1000,
55 : 200,
},
50 : {
8 : 12000,
10 : 12000,
12 : 12000,
16 : 11000,
20 : 8200,
27 : 6000,
32 : 3500,
20 : 8000,
27 : 5500,
36 : 4200,
44 : 3300,
55 : 1500,
}
}
def getModelParamsAMD(self):
#20191120
self.maxThroughputModels = {
8 : functools.partial(FitModels.linearPlateauModel, parameters=(143.961,2.82612,2125.97)),
10 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,3.82341,3415.73)),
12 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.17365,3398.74)),
16 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.58240,3232.70)),
20 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,4.88082,3211.46)),
27 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,5.23743,3052.89)),
36 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,5.36374,2730.50)),
44 : functools.partial(FitModels.quadraticPlateauModel,parameters=(0,5.33543,2736.68)),
55 : functools.partial(FitModels.quadraticPlateauModel,parameters=(13.5097,5.20221,2695.82)),
}
self.maxFragmentSizes = {
100 : {
8 : 7500,
10 : 6000,
12 : 5600,
16 : 4600,
20 : 3700,
27 : 2800,
36 : 1800,
44 : 1000,
55 : 0,
},
50 : {
8 : 12000,
10 : 12000,
12 : 12000,
16 : 10000,
20 : 8000,
27 : 5500,
36 : 3200,
44 : 3000,
55 : 1500,
}
}
......@@ -77,17 +140,6 @@ class CheckFBthroughput:
return ETroot
def readPlainFile(self,configFile):
rus = []
with open(configFile) as file:
for line in file:
result = re.match(r'.*FEDBuilder:.*id=(\d+).*name=([\w+-]+).*srcIds: ([\d\(\),; ]+)',line)
if result:
fedIds = re.findall(r'[\(\),; ]*(\d+)',result.group(3))
rus.append((["fb_%02d"%int(result.group(1)),result.group(2)],fedIds))
return rus
def readParamFile(self,paramFile):
params = {}
with open(paramFile) as file:
......@@ -125,6 +177,8 @@ class CheckFBthroughput:
def calculateThroughput(self):
eventSize = 0
fedCount = 0
fbCount = 0
for ru in self.RUs:
superFragmentSize = 0.
for fed in ru[1]:
......@@ -141,6 +195,8 @@ class CheckFBthroughput:
triggerRate = self.args['triggerRate']
throughput = (superFragmentSize*triggerRate)/1e6
fbSize = len(ru[1])
fedCount += fbSize
fbCount += 1
averageFedSize = superFragmentSize/fbSize
maxFragmentSizes = self.maxFragmentSizes[triggerRate] if triggerRate in self.maxFragmentSizes else self.maxFragmentSizes[min(self.maxFragmentSizes.keys(), key=lambda k: abs(k-triggerRate-1))]
maxFedSize = maxFragmentSizes[fbSize] if fbSize in maxFragmentSizes else maxFragmentSizes[min(maxFragmentSizes.keys(), key=lambda k: abs(k-fbSize-1))]
......@@ -149,7 +205,9 @@ class CheckFBthroughput:
if self.args['printAll'] or throughput > maxThroughput or averageFedSize > maxFedSize:
#print(" %1.2f GB/s (< %1.2f GB/s) from %2d FEDs with average %4d Bytes (< %d Bytes) : %s %s" % (throughput,maxThroughput,fbSize,averageFedSize,maxFedSize,ru[0],ru[1]))
print(" %1.2f GB/s (< %1.2f GB/s) from %2d FEDs with average %4d Bytes : %s %s" % (throughput,maxThroughput,fbSize,averageFedSize,ru[0],ru[1]))
print("============================")
print("Total event size: %1.2f MB"%(eventSize))
print(str(fedCount)+" FEDs in "+str(fbCount)+" FEDbuilders")
if __name__ == "__main__":
......
#!/usr/bin/env python
import gzip
import os
import re
import xml.etree.ElementTree as ET
from xml.etree.ElementTree import QName as QN
from checkFBthroughput import readPlainFile
class MakeFBconfig:
def __init__(self,args):
self.args = vars(args)
if os.path.exists(self.args['templateFile']):
self.template = ET.parse(self.args['templateFile']).getroot()
elif os.path.exists(self.args['templateFile']+".gz"):
f = gzip.open(self.args['templateFile']+".gz") #gzip.open() doesn't support the context manager protocol needed for it to be used in a 'with' statement.
self.template = ET.parse(f).getroot()
f.close()
else:
raise IOError(self.args['templateFile']+"(.gz) does not exist")
self.xcns = re.match(r'\{(.*?)\}Partition',self.template.tag).group(1) ## Extract xdaq namespace
self.fedIDs = self.getFedIDs()
self.setFedIdsOnFEROLs()
self.setFedIdsOnRUs()
self.writeConfig()
def writeConfig(self):
outputFile = self.args['fb']+"_"+str(len(self.fedIDs))+"streams.xml"
with open(outputFile,'w') as output:
output.write(ET.tostring(self.template)+'\n')
def getFedIDs(self):
for fb in readPlainFile(self.args['configFile']):
if self.args['fb'] in fb[0]:
return fb[1]
raise Exception("FB "+self.args['fb']+" not found in "+self.args['configFile'])
def getValueForFEDid(self,placeholder):
val = int(placeholder[-2:])
if val > 0:
return str(self.fedIDs[(val-1)])
else:
return placeholder
def setFedIdsOnFEROL40(self):
for ferol40 in self.template.findall('.//'+QN(self.xcns,'Application').text+'[@class="ferol40::Ferol40Controller"]'):
propNS = re.match(r'\{(.*?)\}properties',ferol40[0].tag).group(1) ## Extract namespace of properties
for item in ferol40.findall('.//*'+QN(propNS,'item').text):
for value in item:
if 'expectedFedId' in value.tag:
try:
value.text = self.getValueForFEDid(value.text)
enabled = True
except IndexError:
value.text = '9999'
enabled = False
for value in item:
if 'enable' in value.tag:
if enabled:
value.text = 'true'
else:
value.text = 'false'
def setFedIdsOnFEROL(self):
for ferol in self.template.findall('.//'+QN(self.xcns,'Application').text+'[@class="ferol::FerolController"]'):
propNS = re.match(r'\{(.*?)\}properties',ferol[0].tag).group(1) ## Extract namespace of properties
for item in ferol:
for value in item:
if 'expectedFedId_0' in value.tag:
try:
value.text = self.getValueForFEDid(value.text)
enabled_0 = True
except IndexError:
value.text = '9999'
enabled_0 = False
if 'expectedFedId_1' in value.tag:
try:
value.text = self.getValueForFEDid(value.text)
enabled_1 = True
except IndexError:
value.text = '9999'
enabled_1 = False
for value in item:
if 'enableStream0' in value.tag:
if enabled_0:
value.text = 'true'
else:
value.text = 'false'
if 'enableStream1' in value.tag:
if enabled_1:
value.text = 'true'
else:
value.text = 'false'