Skip to content

GitLab

Commit cc1ef9f4 authored by Marian Babik's avatar Marian Babik
Browse files

Merge branch 'master' of ssh://gitlab.cern.ch:7999/etf/cmssam

parents 45c3b184 3eb359e4
Hide whitespace changes
Inline Side-by-side
SiteTests/FroNtier/tests/CE-cms-frontier
View file @ cc1ef9f4
......@@ -17,31 +17,29 @@ if [ -n "$OSG_GRID" ] ; then
elif [ -n "$VO_CMS_SW_DIR" ] ; then
SW_DIR=$VO_CMS_SW_DIR
else
echo "ERROR: Neither VO_CMS_SW_DIR nor OSG_APP defined"
echo "summary: SW_DIR_UNDEF"
exit $SAME_ERROR
SW_DIR=/cvmfs/cms.cern.ch
fi
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
# check for SL5
libc25=1
if ! [ -f $SW_DIR/common/cmsos ] ; then
echo "$SW_DIR/common/cmsos not found - force to SL5"
slVersion="slc5"
osVersion=`$SW_DIR/common/cmsos`
echo "osVersion: " $osVersion
slVersion=`echo $cmsarch|cut -d'_' -f1`
if [[ $osVersion == slc6* ]]; then
cmsarch=slc6_amd64_gcc530
elif [[ $osVersion == slc7* ]] ; then
cmsarch=slc7_amd64_gcc530
else
osVersion=`$SW_DIR/common/cmsos`
echo "osVersion: " $osVersion
cmsarch=`$SW_DIR/common/cmsarch`
echo "scram_arch: " $cmsarch
slVersion=`echo $cmsarch|cut -d'_' -f1`
echo "ERROR: unknown OS"
echo "summary: UNKN_OS"
exit $SAME_ERROR
fi
echo "scram_arch: " $cmsarch
export SCRAM_ARCH=$cmsarch
# this test requires SLC6 arch, currently
export SCRAM_ARCH=slc6_amd64_gcc491
export BUILD_ARCH=$SCRAM_ARCH
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
result=$?
export BUILD_ARCH=$SCRAM_ARCH
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
rm -f $tmpfile
......
SiteTests/FroNtier/tests/CE-cms-frontier.sing 0 → 100755
View file @ cc1ef9f4
#!/bin/bash
TEST_SCRIPT=`basename $0 | sed 's/\.sing//'`
echo "Will run $TEST_SCRIPT with Singularity if available"
source ${SAME_SENSOR_HOME}/tests/CE-cms-singularity-wrapper ${TEST_SCRIPT}
SiteTests/FroNtier/tests/CE-cms-squid
View file @ cc1ef9f4
#!/bin/bash
# Source the CMS environment
export SCRAM_ARCH=slc6_amd64_gcc491
if [ -n "$OSG_GRID" ] ; then
[ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
if [ -d $OSG_APP/cmssoft/cms ] ;then
......@@ -18,9 +17,7 @@ if [ -n "$OSG_GRID" ] ; then
elif [ -n "$VO_CMS_SW_DIR" ] ; then
SW_DIR=$VO_CMS_SW_DIR
else
echo "ERROR: Neither VO_CMS_SW_DIR nor OSG_APP defined"
echo "summary: SW_DIR_UNDEF"
exit $SAME_ERROR
SW_DIR=/cvmfs/cms.cern.ch
fi
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
......
SiteTests/FroNtier/tests/CMSSW_frontier.sh
View file @ cc1ef9f4
......@@ -14,9 +14,9 @@ printf "node: $node\n"
#
# Choose CMSSW version
#
cmsswvsn=CMSSW_7_5_2
cmsswvsn=CMSSW_9_0_0
#
export SCRAM_ARCH=slc6_amd64_gcc491
#export SCRAM_ARCH=slc6_amd64_gcc491
# Check that environmental variable SAME_OK is set
#
if [ ! "${SAME_OK}" ]
......
SiteTests/MonteCarlo/cms-MC-test/RuntimeSAMStageOut.py
View file @ cc1ef9f4
......@@ -10,6 +10,8 @@ at a site
import time
import os
import sys
import random
import string
from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
......@@ -37,7 +39,8 @@ class StageOutDiagnostic:
self.tfc = None
self.datestamp = time.asctime(time.localtime(time.time()))
self.datestamp = self.datestamp.replace(" ", "-").replace(":", "_")
self.testLFN = "/store/unmerged/SAM/StageOutTest-%i-%s" % (os.getpid(), self.datestamp)
self.rndname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(2))
self.testLFN = "/store/unmerged/SAM/StageOutTest-%s-%i-%s" % (self.rndname, os.getpid(), self.datestamp)
def __call__(self):
......@@ -79,6 +82,8 @@ class StageOutDiagnostic:
self.complete()
return
self.complete()
def complete(self):
"""
_complete_
......@@ -117,16 +122,16 @@ class StageOutDiagnostic:
self.summary['SiteConf'] = "Failed: Cannot load SiteConf"
raise RuntimeError, msg
if self.siteConf.localStageOut['command'] == None:
if self.siteConf.localStageOutCommand() == None:
msg = "LocalStageOut Command is not set"
self.summary['SiteConf'] = \
"Failed: local-stage-out command not set"
raise RuntimeError, msg
if self.siteConf.localStageOut['se-name'] == None:
msg = "LocalStageOut SE Name is not set"
if self.siteConf.localStageOutPNN() == None:
msg = "LocalStageOut PhEDEx node is not set"
self.summary['SiteConf'] = \
"Failed: local-stage-out se-name not set"
"Failed: local-stage-out phedex-node not set"
raise RuntimeError, msg
if self.siteConf.localStageOut['catalog'] == None:
......@@ -136,6 +141,7 @@ class StageOutDiagnostic:
raise RuntimeError, msg
msg = "\nSiteConf test successful:\n"
self.summary['SiteConf'] = "OK"
for key, val in self.siteConf.localStageOut.items():
msg += " %s = %s\n" % (key, val)
print msg
......@@ -167,9 +173,8 @@ class StageOutDiagnostic:
raise RuntimeError, msg
msg = "TFC test successful:\n"
msg += "Mapped LFN: %s\n To PFN: %s\n" % (sampleLFN, samplePFN)
# msg += "Using Catalog Rules:\n"
# msg += str(self.tfc)
self.summary['TFC'] = "OK"
msg += "Mapped LFN: %s\n To PFN: %s\n" % (sampleLFN, samplePFN)
print msg
return
......@@ -191,8 +196,8 @@ class StageOutDiagnostic:
sourcePFN = os.path.join(os.getcwd(), "TEST-FILE")
seName = self.siteConf.localStageOut['se-name']
command = self.siteConf.localStageOut['command']
seName = self.siteConf.localStageOutPNN()
command = self.siteConf.localStageOutCommand()
options = self.siteConf.localStageOut.get('option', None)
protocol = self.tfc.preferredProtocol
......@@ -205,13 +210,14 @@ class StageOutDiagnostic:
except Exception, ex:
msg += "Unable to retrieve impl for local stage out:\n"
msg += "Error retrieving StageOutImpl for command named: %s\n" % (
command,)
command)
self.summary['LocalStageOut'] = \
"Failure: Cant retrieve StageOut Impl"
raise RuntimeError, msg
try:
impl.retryPause = 15
impl.numRetries = 2
impl(protocol, sourcePFN, targetPFN, options)
wasSuccessful = True
except Exception, ex:
......@@ -221,6 +227,7 @@ class StageOutDiagnostic:
"Failure: Local Stage Out Failed"
raise RuntimeError, msg
if wasSuccessful:
self.summary['LocalStageOut'] = "OK"
return
except RuntimeError, ex:
......@@ -228,7 +235,7 @@ class StageOutDiagnostic:
### FALLBACK ###
### there are N fallbacks in a list called fallbackStageOut ###
for fallbackCount in range(len(self.siteConf.fallbackStageOut)):
seName = self.siteConf.fallbackStageOut[fallbackCount]['se-name']
seName = self.siteConf.fallbackStageOut[fallbackCount]['phedex-node']
command = self.siteConf.fallbackStageOut[fallbackCount]['command']
options = self.siteConf.fallbackStageOut[fallbackCount].get('option', None)
try:
......@@ -258,7 +265,7 @@ class StageOutDiagnostic:
raise RuntimeError, msg
if wasSuccessful:
self.summary['LocalStageOut'] = ""
self.summary['LocalStageOut'] = "OK (fallback)"
return
# If we got here, nothing worked
......@@ -273,7 +280,7 @@ class StageOutDiagnostic:
"""
os.remove( "TEST-FILE" )
commandList = [ self.siteConf.localStageOut[ 'command' ] ]
commandList = [ self.siteConf.localStageOutCommand() ]
pfnList = [ self.tfc.matchLFN(self.tfc.preferredProtocol, self.testLFN) ]
for fallback in self.siteConf.fallbackStageOut:
......@@ -309,13 +316,15 @@ class StageOutDiagnostic:
self.summary['CleanUp'] = "Failure: Cleanup operation Failed"
raise RuntimeError, msg
wasSuccessful = True
self.summary['CleanUp'] = "OK"
return
except: # except for outer try
wasSuccessful = False
# See if this fallback worked
if wasSuccessful:
self.summary['CleanUp'] = ""
self.summary['CleanUp'] = "OK (fallback)"
return
else:
msg += "Trying Fallback...\n"
......
SiteTests/MonteCarlo/cms-MC-test/lib/python2.6/site-packages/PSetTweaks/PSetTweak.py
View file @ cc1ef9f4
......@@ -7,19 +7,14 @@ independent python structure
"""
import inspect
import pickle
import StringIO
import imp
import inspect
import json
import pickle
import sys
from functools import reduce
#py2.6 compatibility
try:
import json
except ImportError as ex:
import simplejson as json
class PSetHolder(object):
"""
......@@ -145,7 +140,7 @@ class JSONiser:
for param in params:
self.parameters["%s.%s" % (queue, param)] = dictionary[param]
for key, value in dictionary.items():
if type(value) == type(dict()):
if isinstance(value, dict):
self.queue.append(key)
self.dejson(dictionary[key])
self.queue.pop(-1)
......@@ -180,7 +175,7 @@ class PSetTweak:
"""
currentPSet = None
paramList = attrName.split(".")
for i in range(0, len(paramList)):
for _ in range(0, len(paramList)):
param = paramList.pop(0)
if param == "process":
currentPSet = self.process
......@@ -204,7 +199,7 @@ class PSetTweak:
if not paramName.startswith("process"):
msg = "Invalid Parameter Name: %s\n" % paramName
msg += "Parameter must start with process"
raise RuntimeError, msg
raise RuntimeError(msg)
return recursiveGetattr(self, paramName)
......@@ -261,7 +256,7 @@ class PSetTweak:
current = None
last = None
psets = psetPath.split(".")
for i in range(0, len(psets)):
for _ in range(0, len(psets)):
pset = psets.pop(0)
last = current
if current == None:
......@@ -293,8 +288,7 @@ class PSetTweak:
setattrCalls = {}
for pset in self.psets():
setattrCalls.update(self.setattrCalls(pset))
order = setattrCalls.keys()
order.sort()
order = sorted(setattrCalls.keys())
for call in order:
if call == "process": continue
result += "%s\n" % setattrCalls[call]
......@@ -303,7 +297,7 @@ class PSetTweak:
for param, value in self:
psetName = param.rsplit(".", 1)[0]
paramName = param.rsplit(".", 1)[1]
if type(value) == type("string"):
if isinstance(value, basestring):
value = "\"%s\"" % value
result += "setattr(%s, \"%s\", %s)\n" % (
psetName, paramName, value)
......@@ -336,33 +330,33 @@ class PSetTweak:
return jsoniser.json
def persist(self, filename, format = "python"):
def persist(self, filename, formatting="python"):
"""
_persist_
Save this object as either python, json or pickle
"""
if format not in ("python", "json", "pickle"):
msg = "Unsupported Format: %s" % format
raise RuntimeError, msg
if formatting not in ("python", "json", "pickle"):
msg = "Unsupported Format: %s" % formatting
raise RuntimeError(msg)
if format == "python":
if formatting == "python":
handle = open(filename, 'w')
handle.write(self.pythonise())
handle.close()
if format == "json":
if formatting == "json":
handle = open(filename, "w")
handle.write(self.jsonise())
handle.close()
if format == "pickle":
if formatting == "pickle":
handle = open(filename, "w")
pickle.dump(self, handle)
handle.close()
return
def unpersist(self, filename, format = None):
def unpersist(self, filename, formatting=None):
"""
_unpersist_
......@@ -370,26 +364,26 @@ class PSetTweak:
it based on file extension
"""
if format == None:
if formatting == None:
fileSuffix = filename.rsplit(".", 1)[1]
if fileSuffix == "py":
format = "python"
formatting = "python"
if fileSuffix == "pkl":
format = "pickle"
formatting = "pickle"
if fileSuffix == "json":
format = "json"
formatting = "json"
if format not in ("python", "json", "pickle"):
msg = "Unsupported Format: %s" % format
raise RuntimeError, msg
if formatting not in ("python", "json", "pickle"):
msg = "Unsupported Format: %s" % formatting
raise RuntimeError(msg)
if format == "pickle":
if formatting == "pickle":
handle = open(filename, 'r')
unpickle = pickle.load(handle)
handle.close()
self.process.__dict__.update(unpickle.__dict__)
if format == "python":
if formatting == "python":
modRef = imp.load_source('tempTweak', filename)
lister = PSetLister()
lister(modRef.process)
......@@ -399,7 +393,7 @@ class PSetTweak:
del modRef, sys.modules['tempTweak']
if format == "json":
if formatting == "json":
handle = open(filename, 'r')
jsonContent = handle.read()
handle.close()
......
SiteTests/MonteCarlo/cms-MC-test/lib/python2.6/site-packages/PSetTweaks/WMTweak.py
View file @ cc1ef9f4
......@@ -7,8 +7,11 @@ Note: This can be used within the CMSSW environment to act on a
process/config but does not depend on any CMSSW libraries. It needs to stay like this.
"""
from __future__ import print_function
import logging
import pickle
import traceback
import os
from PSetTweaks.PSetTweak import PSetTweak
from PSetTweaks.PSetTweak import parameterIterator, psetIterator
......@@ -25,14 +28,13 @@ _TweakOutputModules = [
"fastCloning",
"sortBaskets",
"dropMetaData",
#"outputCommands", #this is just a huge pile of stuff which we probably shouldnt be setting anyways
# "outputCommands", #this is just a huge pile of stuff which we probably shouldnt be setting anyways
"SelectEvents.SelectEvents",
"dataset.dataTier",
"dataset.filterName",
# TODO: support dataset.* here
]
]
_TweakParams = [
# options
......@@ -46,13 +48,11 @@ _TweakParams = [
"process.options.FailModule",
"process.options.IgnoreCompletely",
#config metadata
# config metadata
"process.configurationMetadata.name",
"process.configurationMetadata.version",
"process.configurationMetadata.annotation",
# source
"process.source.maxEvents",
"process.source.skipEvents",
......@@ -98,17 +98,19 @@ _TweakParams = [
"process.RandomNumberGeneratorService.*.initialSeed",
"process.GlobalTag.globaltag",
]
]
class WMTweakMaskError(Exception):
def __init__(self, mask = None, msg = "Cannot set process from job mask"):
class WMTweakMaskError(Exception):
def __init__(self, mask=None, msg="Cannot set process from job mask"):
super(WMTweakMaskError, self).__init__()
self.mask = mask
self.message = msg
def __str__(self):
return "Error: %s \n Mask: %s" % (self.message, str(self.mask))
def lfnGroup(job):
"""
_lfnGroup_
......@@ -121,7 +123,8 @@ def lfnGroup(job):
lfnGroup = modifier + str(job.get("counter", 0) / 1000).zfill(4)
return lfnGroup
def hasParameter(pset, param, nopop = False):
def hasParameter(pset, param, nopop=False):
"""
_hasParameter_
......@@ -135,7 +138,7 @@ def hasParameter(pset, param, nopop = False):
"""
params = param.split(".")
if not nopop:
params.pop(0) # first param is the pset we have the reference to
params.pop(0) # first param is the pset we have the reference to
lastParam = pset
for param in params:
lastParam = getattr(lastParam, param, None)
......@@ -145,7 +148,8 @@ def hasParameter(pset, param, nopop = False):
return True
return False
def getParameter(pset, param, nopop = False):
def getParameter(pset, param, nopop=False):
"""
_getParameter_
......@@ -157,7 +161,7 @@ def getParameter(pset, param, nopop = False):
"""
params = param.split(".")
if not nopop:
params.pop(0) # first param is the pset we have the reference to
params.pop(0) # first param is the pset we have the reference to
lastParam = pset
for param in params:
lastParam = getattr(lastParam, param, None)
......@@ -165,6 +169,7 @@ def getParameter(pset, param, nopop = False):
return None
return lastParam.value()
def setParameter(process, param, value):
"""
_setParameter_
......@@ -179,22 +184,20 @@ def setParameter(process, param, value):
"""
params = param.split('.')
params.pop(0) # first is process object
params.pop(0) # first is process object
lastPSet = process
for pset in params:
lastPSet = getattr(lastPSet, pset, None)
if lastPSet == None:
msg = "Cannot find attribute named: %s\n" % pset
msg += "Cannot set value: %s" % param
print msg
print(msg)
return
lastPSet.setValue(value)
return
def expandParameter(process, param):
"""
_expandParameter_
......@@ -205,9 +208,9 @@ def expandParameter(process, param):
"""
params = param.split('.')
params.pop(0)
lastResults = {"process" : process}
lastResults = {"process": process}
finalResults = {}
for i in range(0, len(params)):
for _ in range(0, len(params)):
pset = params.pop(0)
if pset == "*":
newResults = {}
......@@ -235,11 +238,11 @@ def expandParameter(process, param):
lastResults = newResults
return finalResults
return finalResults
listParams = lambda x: [y for y in x.parameters_()]
listParams = lambda x: [ y for y in x.parameters_() ]
class TweakMaker:
"""
......@@ -251,8 +254,9 @@ class TweakMaker:
within the output modules
"""
def __init__(self, processParams = _TweakParams,
outmodParams = _TweakOutputModules):
def __init__(self, processParams=_TweakParams,
outmodParams=_TweakOutputModules):
self.processLevel = processParams
self.outModLevel = outmodParams
......@@ -261,12 +265,11 @@ class TweakMaker:
tweak = PSetTweak()
# handle process parameters
processParams = []
[ processParams.extend( expandParameter(process, param).keys())
for param in self.processLevel]
[processParams.extend(expandParameter(process, param).keys())
for param in self.processLevel]
[ tweak.addParameter(param, getParameter(process, param))
for param in processParams if hasParameter(process, param) ]
[tweak.addParameter(param, getParameter(process, param))
for param in processParams if hasParameter(process, param)]
# output modules
tweak.addParameter('process.outputModules_', [])
......@@ -282,9 +285,9 @@ class TweakMaker:
param,
True))
return tweak
def makeTweak(process):
"""
_makeTweak_
......@@ -298,8 +301,7 @@ def makeTweak(process):
return maker(process)