Commit f1074cd6 authored by Spyros Argyropoulos's avatar Spyros Argyropoulos
Browse files

Implementing automatic DSID suggestion [skip ci]

parent b53ad57c
Pipeline #1318374 skipped
......@@ -4,11 +4,14 @@
source scripts/helpers.sh
# Whitelist of objects that can be added to the repo
source scripts/whitelist.sh
# For DSID range suggestion
source scripts/suggestDSID.sh
# Defaults
git=true
newDSID=()
skipAthena=true # default should be false when run_athena is re-activated
suggestDSID=false
logParserOption=""
commitMessage=""
DEFAULTIFS=$IFS
......@@ -27,6 +30,7 @@ printHelp() {
echo -e "-n|--nogit will not commit anything to the git repository. Used when we only want to create the log.generat.short files and check if eveyrthing is ok for committing\n"
echo -e "-t|--testNevents will run logParser with the -t option, i.e. it will not check whether the number of events written in the EVNT file are equal to nEventsPerJob\n"
echo -e "-m \"[skip ...]\" takes as arguments a comma-separated list of strings that will be added to the git commit message to control the execution of the CI jobs. Possible values: [skip ci],[skip modfiles],[skip size],[skip athena],[skip logparser]. These can be combined, e.g. -m=\"[skip modfiles],[skip athena]\"\n"
echo -e "-s|--suggestDSID Suggest DSIDs to use\n"
echo -e "-h|--help prints this help message\n"
}
......@@ -99,6 +103,11 @@ for i in "$@" ; do
logParserOption=" -t "
shift
;;
-s|--suggestDSID) # suggest DSID directories to use
suggestDSID=true
git=false
shift
;;
-n|--nogit) # do not commit anything to git
git=false
shift
......@@ -172,6 +181,58 @@ if (( ${#newDSID[@]} > 0 )) ; then
fi
fi
# Automatically book the correct DSIDs for production
if $suggestDSID && ! $git ; then
# Generators
MG_range=(); Ph_range=(); Sh_range=() ; Py_range=(); Hw_range=()
printInfo "Suggesting DSID ranges ..."
for dsid in "${newDSID[@]}" ; do
# Directory should be located in
dsiddir=${dsid:0:3}xxx
# Check if directory exists
if [ ! -d $dsiddir/$dsid ] ; then
printError "ERROR: Directory $dsiddir/$dsid does not exist"
exit 6
fi
# Get jO and find generator name
# There should only be 1 jO
jO=$(ls $dsiddir/$dsid/mc.*.py)
if [ $(ls $dsiddir/$dsid/mc.*.py | wc -l) -gt 1 ] ; then
printError "ERROR: There should only be 1 jO file in $dsiddir/$dsid. Multiple found:"
ls $dsiddir/$dsid/mc.*.py
exit 7
fi
firstGenerator=$(python scripts/check_jo_consistency_main.py $jO | grep First | awk '{print $NF}')
if [[ "$firstGenerator" == "" ]] ; then
printError "ERROR: jO $jO is malformed. Fix it before continuing"
python scripts/check_jo_consistency_main.py $jO
exit 8
else
echo -e "\tjO: $jO -> generator = $firstGenerator"
if [[ "$firstGenerator" == "MG" || "$firstGenerator" == "aMC" ]] ; then
MG_range+=($dsiddir/$dsid)
elif [[ "$firstGenerator" == "Ph" ]] ; then
Ph_range+=($dsiddir/$dsid)
elif [[ "$firstGenerator" == "Sh" ]] ; then
Sh_range+=($dsiddir/$dsid)
elif [[ "$firstGenerator" == "Py8" || "$firstGenerator" == "P8B" ]] ; then
Py_range+=($dsiddir/$dsid)
elif [[ "$firstGenerator" == "H7" ]] ; then
Hw_range+=($dsiddir/$dsid)
else
echo "Unknown generator: $firstGenerator"
fi
fi
done
# Now suggest DSID ranges to use
if [ ${#MG_range[@]} -gt 0 ] ; then suggestDSIDrange MG ${MG_range[@]} ; fi
if [ ${#Ph_range[@]} -gt 0 ] ; then suggestDSIDrange Ph ${Ph_range[@]} ; fi
if [ ${#Sh_range[@]} -gt 0 ] ; then suggestDSIDrange Sh ${Sh_range[@]} ; fi
if [ ${#Py_range[@]} -gt 0 ] ; then suggestDSIDrange Py ${Py_range[@]} ; fi
if [ ${#Hw_range[@]} -gt 0 ] ; then suggestDSIDrange Hw ${Hw_range[@]} ; fi
exit 0
fi
# Loop over new DSID directories to be added
for dsid in "${newDSID[@]}" ; do
......@@ -205,7 +266,7 @@ for dsid in "${newDSID[@]}" ; do
if (( $? != 0 )); then
cat $tmpLogParserOut
printError -f "\tERROR: logParser run failed."
exit 7
exit 9
fi
# Check if local log.generate file passes the logParser checks
......@@ -214,7 +275,7 @@ for dsid in "${newDSID[@]}" ; do
python scripts/logParser.py $logParserOption -i $dsiddir/$dsid/log.generate
printError -f "\tERROR: log.generate contains errors"
printError -f "\tFix them before committing anything!"
exit 8
exit 10
else
printGood -f "\tOK: log.generate file contains no errors"
fi
......@@ -229,7 +290,7 @@ for dsid in "${newDSID[@]}" ; do
if (( $(echo "$cpu > 1.0" | bc -l) )) ; then
printError -f "\tERROR: CI job is expected to last more than 1h - time estimate: $cpu hours"
printError -f "\tIt is not possible to run this job in the CI. Contact the MC software coordinators"
exit 9
exit 11
else
printGood -f "\tOK: CI job expected to last less than 1h - time estimate: $cpu hours"
fi
......
#!/bin/bash
# For coloured output
source scripts/helpers.sh
getRangeDSID() {
# Generator
Gen=$1
# DSID ranges
if [[ "$Gen" == "MG" ]] ; then
echo 500000 599999
elif [[ "$Gen" == "Ph" ]] ; then
echo 600000 699999
elif [[ "$Gen" == "Sh" ]] ; then
echo 700000 799999
elif [[ "$Gen" == "Py" ]] ; then
echo 800000 829999
elif [[ "$Gen" == "Hw" ]] ; then
echo 830000 859999
fi
}
suggestDSIDrange() {
# IFS should be the original set in commit_new_dsid.sh script
BACKUPIFS=$IFS
IFS=$DEFAULTIFS
# Generator
Gen=$1
# DSIDs
DSIDs=()
for i in "${@:2}" ; do
DSIDs+=($i)
done
# Get minimum and maximum allowed DSID for generator
DSIDrange=$(getRangeDSID $Gen)
minDSID=$(echo $DSIDrange | awk '{print $1}')
maxDSID=$(echo $DSIDrange | awk '{print $2}')
# How many DSIDs we want
NDSID=${#DSIDs[@]}
# Find maximum occupied DSID corresponding to correct DSID range in all branches
maxOccupiedDSID=($(for branch in $(git for-each-ref --format='%(refname)' refs/heads/); do git ls-tree -r --name-only $branch ; done | awk -F'/' -v min="$minDSID" -v max="$maxDSID" '/[0-9]{3}xxx/ {if ($2 > min && $2 < max) print $2}' | sort -u | tail -1))
# Get minimum DSID that is free in the correct range
let minFreeDSID=$maxOccupiedDSID+1
printInfo -f "\tSuggested DSID range for $Gen ..."
for dsid in "${DSIDs[@]}" ; do
echo -e "\t\t$dsid -> ${minFreeDSID:0:3}xxx/$minFreeDSID"
let minFreeDSID=$minFreeDSID+1
done
# Restore IFS
IFS=$BACKUPIFS
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment