Commit 2b846db9 authored by Christian Gutschow's avatar Christian Gutschow
Browse files

first draft for 7-digit DSID blocks [skip modfiles]

parent 29c10bbe
Pipeline #2847719 passed with stages
in 1 minute and 19 seconds
......@@ -6,6 +6,8 @@ import os, re, string, subprocess, sys
sys.path.append("/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/scripts")
import jo_utils
OPEN_BLOCKS = [ 0, 1000000 ]
# Function to check the jO naming
def check_naming(jofile):
joparts = jofile.split(".")
......@@ -106,7 +108,8 @@ def check_naming(jofile):
# function to get list of unique DSIDs from a specifi branch in git remote
def getDSIDInBranch(branch):
command="git ls-tree -r --name-only {branch} | awk -F'/' '/[0-9]{{3}}xxx/ {{print $2}}' | sort -u".format(branch=branch)
command="git ls-tree -r --name-only {branch}".format(branch=branch)
command+=" | awk '{split($0,a,\"[0-9]*xxx/\"); split(a[2],b,\"/\"); print b[1]}' | sort -u"
return list(map(int, filter(None,os.popen(command).read().strip().split("\n"))))
# function to check if a DSID is free
......@@ -125,6 +128,7 @@ def isFreeDSID(dsid):
if int(dsid) in getDSIDInBranch(branch):
print("\tERROR: {dsid} used in branch: {branch}".format(dsid=dsid, branch=branch))
isFree=False
break
return isFree
# function to return list of consecutive DSIDs within a given range
......@@ -146,15 +150,20 @@ def makeDSIDblocks(freeDSIDs, minDSID, maxDSID):
# function to find list of free DSIDs
# returns a dictionary of {DSID, numberFreeElements}
def findSmallestFreeDSIDblock(minDSID,maxDSID,length):
command="for branch in $(git for-each-ref --format='%(refname)' refs/remotes/origin/); do git ls-tree -r --name-only $branch ; done | \
awk -F'/' -v min=\"{minDSID}\" -v max=\"{maxDSID}\" '/[0-9]{{3}}xxx/ {{if ($2 >= min && $2 <= max) print $2}}' | sort -u".format(minDSID=minDSID, maxDSID=maxDSID)
occupiedDSIDs=list(map(int, filter(None,os.popen(command).read().strip().split("\n"))))
# Get list of free DSIDs
freeDSIDs = [i for i in range(minDSID,maxDSID+1) if i not in occupiedDSIDs]
# Assemple the free DSIDs in consecutive blocks
freeDSIDblocks = makeDSIDblocks(freeDSIDs, minDSID, maxDSID)
freeDSIDblocks = [ ]
for offset in OPEN_BLOCKS:
command="for branch in $(git for-each-ref --format='%(refname)' refs/remotes/origin/); do git ls-tree -r --name-only $branch; done"
command+=" | awk -v min=\"{minDSID}\" -v max=\"{maxDSID}\"".format(minDSID=minDSID+offset, maxDSID=maxDSID+offset)
command+=" '{split($0,a,\"[0-9]*xxx/\"); split(a[2],b,\"/\"); {if (b[1] >= min && b[1] <= max) print b[1]}}'"
occupiedDSIDs=list(map(int, filter(None,os.popen(command).read().strip().split("\n"))))
# Get list of free DSIDs
freeDSIDs = [i for i in range(minDSID+offset,maxDSID+offset+1) if i not in occupiedDSIDs]
# Assemple the free DSIDs in consecutive blocks
freeDSIDblocks += [block for block in makeDSIDblocks(freeDSIDs, minDSID+offset, maxDSID+offset) if len(block)>=length]
if len(freeDSIDblocks):
break
# Return the smallest DSID block that is of length >= length
return [block for block in freeDSIDblocks if len(block)>=length][0]
return freeDSIDblocks[0]
# function to check the DSID range
# increment is used to handle several DSIDs
......@@ -184,7 +193,12 @@ def check_dsidrange(generatorName,dsidList):
for dsid in dsidList:
# Check if the DSID is within the allowed range
# 421xxx is handled as an exception since only ART tests should go there
if dsid.isdigit() and int(dsid) >= minDSID and int(dsid) <= maxDSID or dsid.startswith("421"):
rangeOK = dsid.startswith("421")
if not rangeOK and dsid.isdigit():
for offset in OPEN_BLOCKS:
rangeOK |= int(dsid) >= (minDSID + offset) and int(dsid) <= (maxDSID + offset)
if rangeOK:
print("\tOK: correct DSID range")
# If the DSID is already in the master branch we don't need to test if it's free
if int(dsid) in getDSIDInBranch("refs/remotes/origin/master"):
......@@ -200,9 +214,11 @@ def check_dsidrange(generatorName,dsidList):
print("\tERROR: DSID:{dsid}, generator:{generatorName} => allowed DSID range {minDSID} - {maxDSID}"
.format(dsid=dsid, generatorName=generatorName, minDSID=minDSID, maxDSID=maxDSID))
currentDSIDxxx="{}xxx/".format(dsid[0:3]) if dsid.isdigit() else ""
print("\t\tMove {currentDSIDxxx}{currentDSID} -> {freeDSIDxxx}xxx/{freeDSID}"
.format(currentDSIDxxx=currentDSIDxxx, currentDSID=dsid,
freeDSIDxxx=str(freeDSIDblock[bookedDSIDs])[0:3], freeDSID=freeDSIDblock[bookedDSIDs]))
freeDSID=freeDSIDblock[bookedDSIDs]
prefix = '{block}/{block}'.format(block=str(freeDSID)[0]) if freeDSID > 999999 else ''
print("\t\tMove {currentDSIDxxx}{currentDSID} -> {prefix}{freeDSIDxxx}xxx/{freeDSID}"
.format(currentDSIDxxx=currentDSIDxxx, currentDSID=dsid, prefix=prefix,
freeDSIDxxx=str(freeDSIDblock[bookedDSIDs])[0:3], freeDSID=freeDSID))
bookedDSIDs += 1
if DSIDinWrongRange:
......
......@@ -35,22 +35,33 @@ def main():
sys.exit(1)
try:
# Extract naming
prefix = ''
if file[1] == '/':
prefix = file[:2]
file = file[2:]
dsidxxx, dsiddir, jofile = file.split("/")
dsidxxxPattern = re.compile("[0-9]{3}xxx")
dsidxxxPattern = re.compile("[0-9]{3,4}xxx")
if not (dsidxxxPattern.match(dsidxxx) and dsiddir.isdigit()):
raise ValueError
# Check that the jO lives in a directory named like NNNxxx
if not re.match("[0-9]{3}xxx", dsidxxx):
print("ERROR: modified jO lives in directory " + dsidxxx + ". Need to rename directory like NNNxxx, where N are the first 3 digits of the DSID")
if not re.match("[0-9]{3,4}xxx", dsidxxx):
msg="ERROR: modified jO lives in directory " + dsidxxx + "."
if len(dsidxxx) == 6:
msg+= " Need to rename directory like NNNxxx, where N are the first 3 digits of the DSID"
else:
msg+= " Need to rename directory like NNNNxxx, where N are the first 4 digits of the DSID"
print(msg)
error_DSIDrange=True
# Check that the DSID directory NNNXYZ where the jO lives is inside NNNxxx
if dsidxxx[:3] != dsiddir[:3]:
print("ERROR: modified jO lives in directory " + dsidxxx + "/" + dsiddir + ". Need to put it in " + dsiddir[:3] + "xxx/" + dsiddir)
if (prefix != '' and dsidxxx[:4] != dsiddir[:4]) or dsidxxx[:3] != dsiddir[:3]:
msg = "ERROR: modified jO lives in directory " + prefix + dsidxxx + "/" + dsiddir + "."
msg += " Need to put it in " + prefix + (dsiddir[:4] if prefix != '' else dsiddir[:3]) + "xxx/" + dsiddir
print (msg)
error_DSIDrange=True
except ValueError:
dsiddir = os.path.split(file)[0]
jofile = os.path.split(file)[1]
print("\nModified jO file: {0} - jO file = {1}".format(file, jofile))
print("\nModified jO file: {0} - jO file = {1}".format(prefix + file, jofile))
# Get the name of the first generator
firstGenerator=check_naming(jofile)
if firstGenerator == None:
......@@ -65,11 +76,12 @@ def main():
else: # Everything else
newDSIDs["Misc"].append(dsiddir)
# Check whether the file includes any MC15JobOption includes and if it's in unix format
error_jOcontent += check_jOcontent(file)
error_jOcontent += check_jOcontent(prefix+file)
# Loop over list of DSIDs for each generator and check if they are in the correct DSID range
for key in newDSIDs:
error_DSIDrange += check_dsidrange(key, newDSIDs[key])
if len(newDSIDs[key]):
error_DSIDrange += check_dsidrange(key, newDSIDs[key])
else:
print("OK: no modified jO files")
......
......@@ -285,7 +285,11 @@ jOs=()
for dsid in "${newDSID[@]}" ; do
# Directory should be located in
if [[ $dsid =~ [0-9]{6} && "$dOptionUsed" == "true" ]] ; then
dsiddir=${dsid:0:3}xxx/$dsid
if [ ${#dsid} -eq 6 ]; then
dsiddir=${dsid:0:3}xxx/$dsid
else
dsiddir=${dsid:0:1}/${dsid:0:4}xxx/$dsid
fi
else
dsiddir=$dsid
fi
......@@ -320,7 +324,11 @@ lenDSIDs=${#newDSID[@]}
for dsid in "${newDSID[@]}" ; do
# Directory should be located in
if [[ $dsid =~ [0-9]{6} ]] ; then
dsiddir=${dsid:0:3}xxx/$dsid
if [ ${#dsid} -eq 6 ]; then
dsiddir=${dsid:0:3}xxx/$dsid
else
dsiddir=${dsid:0:1}/${dsid:0:4}xxx/$dsid
fi
else
dsiddir=$dsid
fi
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment