Commit a5477283 authored by Andrea Sciaba's avatar Andrea Sciaba
Browse files

Moved to new singularity search algorithm

parent 5c0db4d8
......@@ -14,8 +14,8 @@ export SAME_CRITICAL=60
export SAME_MAINTENANCE=100
# setup grid client software
rhel6_img='/cvmfs/singularity.opensciencegrid.org/bbockelm/cms:rhel6'
rhel7_img='/cvmfs/singularity.opensciencegrid.org/bbockelm/cms:rhel7'
rhel6_img='/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel6-m202001'
rhel7_img='/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel7-m202001'
rhel6_client='/cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/3.4/current/el6-x86_64/setup.sh'
rhel7_client='/cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/3.4/current/el7-x86_64/setup.sh'
......
#!/bin/sh
# default image
export OSG_SINGULARITY_IMAGE_DEFAULT="/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel6-m202001"
# Useful Information in case of a problem:
echo "System: `/bin/uname -a`"
echo "Current working directory: `/bin/pwd`"
......@@ -9,6 +12,8 @@ echo "SAME_SENSOR_HOME = ${SAME_SENSOR_HOME}"
echo "PATH = ${PATH}"
/bin/ls -l /cvmfs/cms.cern.ch/SITECONF/local || true
source ${SAME_SENSOR_HOME}/tests/singularity_lib.sh
echo "Unsetting SCRAM_ARCH..."
unset SCRAM_ARCH
......@@ -27,29 +32,8 @@ for LOCATION in \
fi
done
export PATH
HAS_SINGULARITY="True"
export OSG_SINGULARITY_VERSION=`singularity --version 2>/dev/null`
if [ "x$OSG_SINGULARITY_VERSION" != "x" ]; then
export OSG_SINGULARITY_PATH=`which singularity`
else
# some sites need us to do a module load first
export OSG_SINGULARITY_VERSION=`module load singularity >/dev/null 2>&1; singularity --version 2>/dev/null`
if [ "x$OSG_SINGULARITY_VERSION" != "x" ]; then
export OSG_SINGULARITY_PATH=`module load singularity >/dev/null 2>&1; which singularity`
else
HAS_SINGULARITY='False'
echo "WARNING: Could not find singularity on system"
fi
fi
# default image for this glidein
export OSG_SINGULARITY_IMAGE_DEFAULT="/cvmfs/singularity.opensciencegrid.org/bbockelm/cms:rhel6"
# for now, we will only advertise singularity on nodes which can access cvmfs
if [ ! -e "$OSG_SINGULARITY_IMAGE_DEFAULT" ]; then
HAS_SINGULARITY='False'
echo "WARNING: Could not find singularity image on system"
fi
singularity_locate_bin /usr/bin $OSG_SINGULARITY_IMAGE_DEFAULT
# Grid environment is taken from OSG, check it's available:
if [ ! -e /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client ]; then
......@@ -62,8 +46,8 @@ if [ "$HAS_SINGULARITY" == "False" ] ; then
echo 'summary: SINGULARITY_NOT_FOUND'
exit $SAME_ERROR
else
echo "OSG_SINGULARITY_VERSION = ${OSG_SINGULARITY_VERSION}"
echo "OSG_SINGULARITY_PATH = ${OSG_SINGULARITY_PATH}"
echo "OSG_SINGULARITY_VERSION = ${GWMS_SINGULARITY_VERSION}"
echo "OSG_SINGULARITY_PATH = ${GWMS_SINGULARITY_PATH}"
fi
OSG_SINGULARITY_EXTRA_OPTS="--home ${SAME_SENSOR_HOME}:/srv --bind /cvmfs --contain"
......@@ -86,7 +70,7 @@ fi
export X509_USER_PROXY=/srv/proxy.pem
cd /cvmfs/singularity.opensciencegrid.org
$OSG_SINGULARITY_PATH exec $OSG_SINGULARITY_EXTRA_OPTS \
$GWMS_SINGULARITY_PATH exec $OSG_SINGULARITY_EXTRA_OPTS \
--pwd /srv \
--ipc --pid \
"$OSG_SINGULARITY_IMAGE_DEFAULT" \
......
#!/bin/bash
#
# This script contains some utility functions for the singularity scripts
# The script will be available outside and inside Singularity
#
# Disabling "var is referenced but not assigned". The functions are imported by other scripts defining the variables:
# shellcheck disable=SC2154
#
# This script advertises:
# HAS_SINGULARITY
# SINGULARITY_PATH
# GWMS_SINGULARITY_PATH
# SINGULARITY_VERSION
# GWMS_SINGULARITY_VERSION
# GLIDEIN_DEBUG_OUTPUT
#
# Note that HTCondor has these native attribute names:
# HasSingularity
# SingularityVersion
# Using the above names would interfere and modify HTCondor behavior
# NOTE: HAS_SINGULARITY and HasSingularity are different because of '_'
# TODO: NOTEs for code and check these attrs are handled correctly
# GLIDEIN_SINGULARITY_BINDPATH, GLIDEIN_SINGULARITY_BINDPATH_DEFAULT
# GWMS_ in scripts GLIDEIN_ from attributes in config files and condor
# All output is to stderr
# Functions are using:
# - $glidein_config
# - $GLIDEIN_DEBUG_OUTPUT
# - $GWMS_THIS_SCRIPT
# Singularity images:
# SINGULARITY_IMAGES_DICT
# TODO: Format checkers could be added in python in reconfig
# Singularity images, could be URLs (File path, http://.., docker://.., ...)
# plat1:URL1,plat2:URL2,default:URLd
# No comma is allowed in platform IDs or URLs, no colon is allowed in platform IDs.
# A platform is an arbitrary string, could be the OS name, or a dash separated list (os-arch)
# GWMS will do an exact match with the requested or default ones (rhel7,rhel6,default).
# 'defult' is used for the default platform (no special meaning in reality)
# The legacy variables SINGULARITY_IMAGE_DEFAULT6, SINGULARITY_IMAGE_DEFAULT7 are mapped to rhel6, rhel7
# GLIDEIN_REQUIRED_OS (Factory - OS are allowed on the entry) and REQUIRED_OS (Frontend or Job - OSes the job requires)
# are csv lists of platforms used to request a specific platform. 'any' is the default and means no preference.
# Mount points:
# GLIDEIN_SINGULARITY_BINDPATH
# GLIDEIN_SINGULARITY_BINDPATH_DEFAULT
# Once a mount point is used, following attempts to mount on it are not successful and trigger a warning message
# So the first time a mount point is the one used will determine what is mounted
# SINGULARITY_BINDPATH (left for the host environment) takes precedence (added before the command line options)
# Then come invocation overrides, GLIDEIN_SINGULARITY_BINDPATH, GLIDEIN_SINGULARITY_BINDPATH_DEFAULT and last
# system defaults (e.g. /cvmfs) are added to the command line
# The suggestion is for the Factory to guarantee defaults in GLIDEIN_SINGULARITY_BINDPATH_DEFAULT and let the Frontend
# set or override GLIDEIN_SINGULARITY_BINDPATH
# All lists have the format: src1:dst1,src2:dst2:ro,src3
# Bind w/ non existing sources are removed to prevent Singularity from failing
# GWMS will not do other checks, check your user mount points
# Invocation
# SINGULARITY_BIN path where the singularity binary is located. Can be specified by Factory and/or Frontend and
# will be used before the other possible locations
# Additional options for the Singularity invocation
# GLIDEIN_SINGULARITY_OPTS - options after the exec command
# GLIDEIN_SINGULARITY_GLOBAL_OPTS - singularity options, like debug, silent/verbose, ...
# NOTE: GLIDEIN_SINGULARITY_OPTS and GLIDEIN_SINGULARITY_GLOBAL_OPTS must be expansion/flattening safe because
# is passed as veriable and quoted strings inside it are not preserved
# Reference documentation for the command and env variables:
# https://sylabs.io/guides/3.3/user-guide/cli/singularity.html
# https://sylabs.io/guides/3.3/user-guide/appendix.html
OSG_SINGULARITY_BINARY_DEFAULT="${OSG_SINGULARITY_BINARY:-/cvmfs/oasis.opensciencegrid.org/mis/singularity/bin/singularity}"
# For shell, for HTCondor is the opposite
# 0 = true
# 1 = false
# TODO: Future extensions
# parametrize options:
# GLIDEIN_SINGULARITY_BINDPATH_CHECKS (exist, ...)
# GLIDEIN_SINGULARITY_IMAGE_CHECKS
# GLIDEIN_SINGULARITY_FEATURES
#
# By default Module and Spack are enabled (1=true), MODULE_USE can override this
GWMS_MODULE_USE_DEFAULT=1
# Output log levels:
# WARN used also for error, always to stderr
# INFO if GLIDEIN_QUIET is not set (default)
# DEBUG if GLIDEIN_DEBUG_OUTPUT is set (and GLIDEIN_QUIET is not set)
# GWMS_THIS_SCRIPT should be set to $0 to log the file name
# To increment each time the API changes
export GWMS_SINGULARITY_LIB_VERSION=1
GWMS_SCRIPT_LOG="`dirname "$GWMS_THIS_SCRIPT"`/.LOG_`basename "$GWMS_THIS_SCRIPT"`.$$.txt"
# Change this to enable script log
SCRIPT_LOG=
[[ -n "$GLIDEIN_DEBUG_OUTPUT" ]] && SCRIPT_LOG="$GWMS_SCRIPT_LOG"
info_stdout () {
[[ -z "$GLIDEIN_QUIET" ]] && echo "$@"
true # Needed not to return false if the test if the test above is false
}
info_raw () {
[[ -z "$GLIDEIN_QUIET" ]] && echo "$@" 1>&2
[[ -n "$SCRIPT_LOG" ]] && echo "$@" >> "$GWMS_SCRIPT_LOG"
true # Needed not to return false if the test if the test above is false
}
info () {
info_raw "INFO " "$@"
}
info_dbg () {
if [[ -n "$GLIDEIN_DEBUG_OUTPUT" ]]; then
#local script_txt=''
#[ -n "$GWMS_THIS_SCRIPT" ] && script_txt="(file: $GWMS_THIS_SCRIPT)"
info_raw "DEBUG ${GWMS_THIS_SCRIPT:+"($GWMS_THIS_SCRIPT)"}" "$@"
fi
}
warn () {
warn_raw "WARNING " "$@"
}
warn_raw () {
echo "$@" 1>&2
[[ -n "$SCRIPT_LOG" ]] && echo "$@" >> "$GWMS_SCRIPT_LOG"
true # Needed not to return false if the test if the test above is false
}
######################################################
#
# Dictionary functions
# Dictionaries are strings: key1:val1,key2:val2
# Comma is not allowed in keys or values, colon is not allowed in keys
# Associative dictionaries are OK in bash 4.1. Before then are not or not fully supported
# References (declare -n) are from 4.3.
# TEST: to test dict functions
# my_dict=" key 1:val1:opt1,key2:val2,key3:val3:opt3,key4,key5:,key6 :val6"
#
dict_get_val () {
# Return to stdout the value of the fist key present in the dictionary
# Return true (0) if a value is found and is not empty, 1 otherwise
# Use a regex to extract the values
# $1 dict name
# $2 comma separated list of keys (key can contain a space if you quote it but not a comma)
local IFS=,
local key_list="$2"
for key in $key_list; do
res="$(expr ",${!1}," : ".*,$key:\([^,]*\),.*")"
if [[ -n "$res" ]]; then
echo "$res"
return 0
fi
done
return 1
}
dict_check_key () {
# Return true (0) if the key is in the dict (the value could be empty)
# $1 dict name
# $2 key
#re=*",${2}:"* # bash <= 3.1 needs quoted regex, >=3.2 unquoted, variables are OK with both
[[ ",${!1}," = *",${2}:"* ]] && return 0
[[ ",${!1}," = *",${2},"* ]] && return 0 # could be empty val and no separator
return 1
}
dict_set_val () {
# Echoes a new string including the new key:value. Return is 0 if the key was already there, 1 if new
# $1 dict name
# $2 key
# $3 value (optional)
# Assuming correct use, no check made, at least 2 arguments mandatory
local my_dict=${!1}
local key_found
if [[ ",${my_dict}," = *",${2}:"* || ",${my_dict}," = *",${2},"* ]]; then
my_dict="`echo ",${my_dict}," | sed -E "s/,${2}(,|:[^,]*,)/,/;s/,+/,/g;s/^,//;s/,\$//"`"
key_found=yes
fi
# [ -n "${my_dict}" ] && my_dict="${my_dict},"
# [ -n "$3" ] && echo "${my_dict}$2:$3" || echo "${my_dict}$2"
echo "${my_dict:+"${my_dict},"}$2${3:+":$3"}"
[[ -n "${key_found}" ]] && return 0
return 1
}
# function get_dict_items {} - not needed
# TEST: for iterators tests
# dit () { echo "TEST: <$1> <$2> <$3>"; }
# dict_items_iterator my_dict dit par1
# Make sure that par1 is passed, spaces are preserved, no-val keys are handled correctly and val options are preserved
dict_items_iterator () {
# Split the dict string to list the items and apply the function
# $1 dict
# $2.. $n $2 is the function to apply to all items, $3..$n its parameters (optional), $(n+1) the key, $(n+2) the value
local my_dict=${!1}
shift
local was_ifs=$IFS
IFS=,
local -a arr=($(echo "${my_dict}"))
IFS=$was_ifs
local val
for i in "${arr[@]}" # ${arr[*]} separates also by spaces
do
[[ "$i" = *\:* ]] && val="${i#*:}" || val= # to protect against empty val and no :
# function key value
"$@" "${i%%:*}" "$val"
done
}
dict_keys_iterator () {
# Split the dict string to list the keys and apply the function
# $1 dict
# $2.. $n $2 is the function to apply to all keys, $3..$n its parameters (optional), $(n+1) will be the key
local my_dict=${!1}
shift
local was_ifs=$IFS
IFS=,
local -a arr=($(echo "${my_dict}"))
IFS=$was_ifs
#echo "T:${arr[1]}"
for i in "${arr[@]}"
do
"$@" "${i%%:*}"
done
}
dict_get_keys () {
# Returns a comma separated list of keys (there may be spaces if keys do have spaces)
# Quote the return string and use IFS=, to separate the keys, this way you'll preserve spaces
# Returning the elements would flatten the array and cause problems w/ spaces
# $1 dict
local my_dict=${!1}
local res="`echo "$my_dict," | sed 's/:[^,]*,/,/g; s/,\+/,/g'`"
echo "${res%,}"
}
dict_get_first () {
# Returns the first element of the dictionary (whole item, or key, or value)
# $1 dict
# $2 what to return: item, key, value (default: value)
local my_dict=${!1}
local what=${2:-value}
local res="${my_dict%%,*}"
if [[ -n "$res" ]]; then
# to protect from empty dicts
case $what in
item)
echo "$res"
;;
value)
[[ "$res" = *\:* ]] && echo "${res#*:}"
;;
key)
echo "${res%%:*}"
;;
esac
fi
}
list_get_intersection () {
# Return the intersection of two comma separated lists.
# 'any' in any of the 2 lists, means that the other list is returned (is a wildcard)
# If the Input lists are sorted in order of preference, the result is as well
# In:
# 1: comma separated list of values
# 2: comma separated list of values
# Out:
# intersection returned on stdout, 'any' is returned if both lists are 'any'
# Return 1 if the intersection is empty, 0 otherwise
# This can be used to evaluate the desired OS (platform) that works for both Entry and VO,
# intersection of GLIDEIN_REQUIRED_OS and REQUIRED_OS
# Valid values: rhel6, rhel7, default
local intersection
[[ -z "$1" || -z "$2" ]] && return 1
if [[ "x$1" = "xany" ]]; then
intersection="$2"
else
if [[ "x$2" = "xany" ]]; then
intersection="$1"
else
# desired_os="$(python -c "print sorted(list(set('$2'.split(',')).intersection('$1'.split(','))))[0]" 2>/dev/null)"
intersection="$(python -c "print ','.join(sorted(list(set('$2'.split(',')).intersection('$1'.split(',')))))" 2>/dev/null)"
fi
fi
[[ -z "$intersection" ]] && return 1
echo "$intersection"
}
#######################################
#
# GWMS aux functions
#
get_prop_bool () {
# In:
# $1 the file (for example, $_CONDOR_JOB_AD or $_CONDOR_MACHINE_AD)
# $2 the key
# $3 default value (optional, must be 1->true or 0->false, 0 if unset)
# For HTCondor consider True: true (case insensitive), any integer != 0
# Anything else is False (0, false, undefined, ...)
# This is the default behavior (default=0)
# Out:
# echo "1" for true, "$default" for empty value/undefined, "0" for false/failure (bad invocation, no ClassAd file)
# return the opposite to allow shell truth values true,1->0 , false,0->1
# NOTE Spaces are trimmed, so strings like "T RUE" are true
# TODO: replace grep w/ case insensitive comparison, currently any string containng 'true' case insensitive is
# considered true, e.g. trueval, NotTrue, ...
local default=${3:-0}
local val
if [[ $# -lt 2 || $# -gt 3 ]]; then
val=0
elif [[ "x$1" = "xNONE" ]]; then
val=$default
else
# sed "s/[\"' \t\r\n]//g" not working on OS X, '\040\011\012\015' = ' '$'\t'$'\r'$'\n'
val=`(grep -i "^$2 " $1 | cut -d= -f2 | tr -d '\040\011\012\015') 2>/dev/null`
# Convert variations of true to 1
re="^[0-9]+$" # bash <= 3.1 needs quoted regex, >=3.2 unquoted, variables are OK with both
if (echo "x$val" | grep -i true) >/dev/null 2>&1; then
val=1
elif [[ "$val" =~ $re ]]; then
if [[ $val -eq 0 ]]; then
val=0
else
val=1
fi
elif [[ -z "$val" ]]; then
val=$default
elif (echo "x$val" | grep -i undefined) >/dev/null 2>&1; then
val=$default
else
val=0
fi
fi
# From here on val=0/1
echo $val
# return value accordingly, but backwards (in shell true -> 0, false -> 1)
if [[ "$val" = "1" ]]; then
return 0
else
return 1
fi
}
is_condor_true () {
# Assuming the input is numeric 0->False other->True
if [[ $1 -eq 0 ]]; then
false
else
true
fi
}
get_prop_str () {
# In:
# $1 the file (for example, $_CONDOR_JOB_AD or $_CONDOR_MACHINE_AD)
# $2 the key
# $3 default value (optional)
# Out:
# echo the value (or the default if UNDEFINED) and return 0
# For no ClassAd file, echo the default and return 1
# For bad invocation, return 1
if [[ $# -lt 2 || $# -gt 3 ]]; then
return 1
elif [[ "x$1" = "xNONE" ]]; then
echo "$3"
return 1
fi
val=`(grep -i "^$2 " $1 | cut -d= -f2 | sed -e "s/^[\"' \t\n\r]//g" -e "s/[\"' \t\n\r]$//g" | sed -e "s/^[\"' \t\n\r]//g" ) 2>/dev/null`
[[ -z "$val" ]] && val="$3"
echo "$val"
return 0
}
# $glidein_config from the file importing this
# add_config_line and add_condor_vars_line are in add_config_line.source (ADD_CONFIG_LINE_SOURCE in $glidein_config)
if [[ -e "$glidein_config" ]]; then # was: [ -n "$glidein_config" ] && [ "$glidein_config" != "NONE" ]
error_gen="`grep '^ERROR_GEN_PATH ' "$glidein_config" | cut -d ' ' -f 2-`"
if [[ "x$SOURCED_ADD_CONFIG_LINE" = "x" ]]; then
# import add_config_line and add_condor_vars_line functions used in advertise
if [[ "x$add_config_line_source" = "x" ]]; then
export add_config_line_source="`grep '^ADD_CONFIG_LINE_SOURCE ' $glidein_config | cut -d ' ' -f 2-`"
export condor_vars_file="`grep -i "^CONDOR_VARS_FILE " $glidein_config | cut -d ' ' -f 2-`"
fi
if [[ -e "$add_config_line_source" ]]; then
info "Sourcing add config line: $add_config_line_source"
. "$add_config_line_source"
# make sure we don't source a second time inside the container
export SOURCED_ADD_CONFIG_LINE=1
else
warn "glidein_config defined but add_config_line ($add_config_line_source) not available. Some functions like advertise will be limited." || true
fi
fi
else
# glidein_config not available
warn "glidein_config not defined ($glidein_config) in singularity_lib.sh. Some functions like advertise and error_gen will be limited." || true
[[ -z "$error_gen" ]] && error_gen=warn
glidein_config=NONE
fi
# TODO: should always use add_config_line_safe to avoid 2 functions?
advertise () {
# Add the attribute to glidein_config (if not NONE) and return the string for the HTC ClassAd
# In:
# 1 - key
# 2 - value
# 3 - type, atype is the type of the value as defined by GlideinWMS:
# I - integer
# S - quoted string
# C - unquoted string (i.e. Condor keyword or expression)
# Out:
# string for ClassAd
# Added lines to glidein_config and condor_vars.lst
key="$1"
value="$2"
atype="$3"
if [[ "$glidein_config" != "NONE" ]]; then
add_config_line $key "$value"
add_condor_vars_line $key "$atype" "-" "+" "Y" "Y" "+"
fi
if [[ "$atype" = "S" ]]; then
echo "$key = \"$value\""
else
echo "$key = $value"
fi
}
advertise_safe () {
# Add the attribute to glidein_config (if not NONE) and return the string for the HTC ClassAd
# Thos should be used in periodic scripts or wrappers, because it uses add_config_line_safe
# In:
# 1 - key
# 2 - value
# 3 - type, atype is the type of the value as defined by GlideinWMS:
# I - integer
# S - quoted string
# C - unquoted string (i.e. Condor keyword or expression)
# Out:
# string for ClassAd
# Added lines to glidein_config and condor_vars.lst
local key="$1"
local value="$2"
local atype="$3"
if [[ "$glidein_config" != "NONE" ]]; then
add_config_line_safe $key "$value"
add_condor_vars_line $key "$atype" "-" "+" "Y" "Y" "+"
fi
if [[ "$atype" = "S" ]]; then
echo "$key = \"$value\""
else
echo "$key = $value"
fi
}
# The following four functions (htc_...) are based mostly on Carl Edquist's code to parse the HTCondor file
htc_setmatch () {
local __=("$@")
set -- "${BASH_REMATCH[@]}"
shift
eval "${__[@]}"
}
htc_rematch () {
[[ $1 =~ $2 ]] || return 1
shift 2
htc_setmatch "$@"
}
htc_get_vars_from_env_str () {
local str_arr condor_var_string=""
env_str=${env_str#'"'}
env_str=${env_str%'"'}
# Strip out escaped whitespace
while htc_rematch "$env_str" "(.*)'([[:space:]]+)'(.*)" env_str='$1$3'
do :; done
# Now, split the string on whitespace
read -ra str_arr <<<"${env_str}"
# Finally, parse each element of the array.
# They should each be name=value assignments,
# and we only need to grab the name
vname_regex="(^[_a-zA-Z][_a-zA-Z0-9]*)(=)[.]*"
for assign in "${str_arr[@]}"; do
if [[ "$assign" =~ $vname_regex ]]; then
condor_var_string="$condor_var_string ${BASH_REMATCH[1]}"
fi
done
echo "$condor_var_string"
}
htc_parse_env_file () {
shopt -s nocasematch
while read -r attr eq env_str; do
if [[ $attr = Environment && $eq = '=' ]]; then
htc_get_vars_from_env_str
break
fi
done < "$1"
shopt -u nocasematch
}
env_clear_one () {
# Clear the environment variable and print a info message
# In
# 1 - name of the variable to clear, e.g. LD_LIBRARY_PATH
local varname="GWMS_OLDENV_$1"
if [[ -n "${!1}" ]]; then