added noop backend and very basic test for CI

.gitlab-ci.yml

@@ -17,5 +17,12 @@ compilation:
   tags:
     - cvmfs
   script:
-    - source /cvmfs/atlas.cern.ch/repo/sw/tdaq/tools/cmake_tdaq/bin/cm_setup.sh tdaq-09-04-00
+    - source /cvmfs/atlas.cern.ch/repo/sw/tdaq/tools/cmake_tdaq/bin/cm_setup.sh tdaq-10-00-00
     - python -m compileall .
+
+basictest:
+  image: gitlab-registry.cern.ch/atlas-tdaq-software/tdaq_ci:centos7
+  tags:
+    - cvmfs
+  script:
+    - DeploymentTest/tbed.test.bash || exit 1

DeploymentTest/tbed.test.bash

+#!/usr/bin/env bash
+
+DEFAULT_CONFIG=./tbed_noop.cfg
+
+function usage {
+  echo "usage: $EXECNAME [path to config file; default: $DEFAULT_CONFIG]"
+}
+
+EXECNAME=`basename $0`
+CS_CONFIG_FILE=$DEFAULT_CONFIG
+TESTHOST=localhost
+
+if [ x"$1" != x ]; then
+  CS_CONFIG_FILE=$1
+  shift
+fi
+if [ x"$1" != x ]; then
+  echo "error: too many arguments"
+  usage
+  exit 1
+fi
+
+cd `dirname $0`
+
+running_cs=`ps -ef | grep CastorScript | grep -v -c grep`
+
+if [ "$running_cs" != "0" ]; then
+  echo "error: there seems to be some CastorScript running on $TESTHOST"
+  exit 2
+else
+  echo "OK: no running CastorScript instances on $TESTHOST"
+fi
+
+SOURCE_DIR=`pwd`/..
+REMOTE_DIR=$(sed -rn "s/^CopyDir: '(.*)'/\1/p" $CS_CONFIG_FILE)
+BASEFILENAME="dummy"
+
+COPY_ENABLED=True
+TMP_CE=$(sed -rn "s/^CopyEnabled: (.*)/\1/p" $CS_CONFIG_FILE)
+if [[ "x$TMP_CE" != "x" ]]; then
+	COPY_ENABLED=$TMP_CE
+fi
+
+RECURSIVE_SOURCE=True
+TMP_RSF=$(sed -rn "s/^RecursiveSourceFolder: (.*)/\1/p" $CS_CONFIG_FILE)
+if [[ "x$TMP_RSF" != "x" ]]; then
+	RECURSIVE_SOURCE=$TMP_RSF
+fi
+
+TMP_DIR=/tmp/castorscripttest
+echo "Cleaning tmp directory: $TMP_DIR"
+rm -rf $TMP_DIR
+mkdir -p $TMP_DIR/logs
+
+echo "Creating dummy data files"
+for i in `seq 0 7`; do
+  dd if=/dev/zero of=$TMP_DIR/${BASEFILENAME}$i.data bs=1k count=1 &>/dev/null
+  if [ $? -ne 0 ]; then
+    echo "error creating file: $TMP_DIR/${BASEFILENAME}$i.data"
+  fi
+done
+mkdir -p $TMP_DIR/subdir1/subdir2
+dd if=/dev/zero of=$TMP_DIR/subdir1/${BASEFILENAME}8.data bs=1k count=1 &>/dev/null
+if [ $? -ne 0 ]; then
+  echo "error creating file: $TMP_DIR/${BASEFILENAME}8.data"
+fi
+dd if=/dev/zero of=$TMP_DIR/subdir1/subdir2/${BASEFILENAME}9.data bs=1k count=1 &>/dev/null
+if [ $? -ne 0 ]; then
+  echo "error creating file: $TMP_DIR/${BASEFILENAME}9.data"
+fi
+
+echo "Starting CastorScript with config file: $CS_CONFIG_FILE"
+source $SOURCE_DIR/script_setup.sh
+python -u $SOURCE_DIR/Script/CastorScript.py $CS_CONFIG_FILE &>$TMP_DIR/logs/stdouterr &
+
+echo "Watching files to transfer"
+if [[ $RECURSIVE_SOURCE == True ]]; then
+  LISTING_COMMAND="find $TMP_DIR -name ${BASEFILENAME}* | sort"
+else
+  LISTING_COMMAND="find $TMP_DIR -maxdepth 1 -name ${BASEFILENAME}* | sort"
+fi
+
+LISTING_OUTPUT=`eval $LISTING_COMMAND`
+NL=`echo "$LISTING_OUTPUT" | wc -l`
+for i in `seq 0 $NL`; do echo; done
+for i in `seq 0 $NL`; do tput cuu1; done
+
+while [ "x$LISTING_OUTPUT" != x ]; do
+  echo "$LISTING_OUTPUT"
+  sleep 1
+
+  for i in `seq 0 $NL`; do tput cuu1; done
+  NCOLS=`tput cols`
+  for i in `seq 0 $NL`; do
+    for ((i=0; i<NCOLS; i++));do printf " "; done;
+    echo
+  done
+  for i in `seq 0 $((NL-1))`; do tput cuu1; done
+
+  # Check that the CastorScript instance is still running
+  NB_RUNNING_PROCESSES=`ps -ef | grep "CastorScript.py $CS_CONFIG_FILE" | grep -v grep | wc -l`
+  if [ $NB_RUNNING_PROCESSES -ne 1 ]; then
+    echo "number of running CastorScript: $NB_RUNNING_PROCESSES, =!1 => exiting"
+    exit
+  fi
+
+  LISTING_OUTPUT=`eval $LISTING_COMMAND`
+  NL=`echo "$LISTING_OUTPUT" | wc -l`
+done
+
+echo "Stopping CastorScript"
+$SOURCE_DIR/ProductionTools/installed/castor.signal 12
+
+NB_ERR=$(grep -c -e WARNING -e ERROR -r $TMP_DIR/logs | awk -F: 'BEGIN{sum=0} //{sum += $2} END{print sum}')
+echo "number of errors and warnings in the log files: $NB_ERR"
+
+if [ $NB_ERR -eq 0 ]; then
+  echo "deleting tmp dir (including logs)"
+  #rm -rf $TMP_DIR
+else
+  echo "Check logs: grep -e WARNING -e ERROR -r $TMP_DIR/logs"
+  echo "Don't forget to: rm -rf $TMP_DIR"
+fi

DeploymentTest/tbed_noop.cfg

+LogDir: '/tmp/castorscripttest/logs/'
+LogLevel: 'debug'
+EmailLogList: ['flegoff@cern.ch']
+EmailLogLevel: 'ERROR'
+EmailLogSender: 'testcastorscript@cern.ch'
+
+MainThreadEventTimeout: 1
+ManagerThreadEventTimeout: 1
+CopyThreadEventTimeout: 1
+DeleteThreadEventTimeout: 1
+
+BackendModule: 'noop'
+SrcDirs: ['/tmp/castorscripttest']
+DataFilePattern: ['*.data',]
+CopyDir: '/tmp/castorscripttest'
+EOSInstance: 'unused'
+TransferTimeout: 1
+
+#DBURL: 'oracle://int8r/ATLAS_SFO_T0'
+#DBFileTable: 'SFO_TZ_FILE'
+#DBLBTable: 'SFO_TZ_LUMIBLOCK'
+#DBRunTable: 'SFO_TZ_RUN'

Script/cs/StorageBackends/noop.py

+import subprocess
+
+# if this file runs as a script instead of as a module.
+# add /../../CastorScript/Script to path so imports keeps working
+if __name__=='__main__':
+    from os.path import dirname, abspath, join
+    import sys
+
+    # Very crude implementation
+    SCRIPT_DIR = abspath(join(dirname(__file__), '..','..'))
+    sys.path.append(SCRIPT_DIR)
+
+from cs.Tools.utils import adler32
+
+def sizechecksum(filename, stager, logger=None):
+    """
+    This still needs to return the size and checksum of the original file
+    otherwise transfers fail inevitably.
+    """
+    del stager
+
+    ls = subprocess.Popen(['ls', '-l1', filename],
+                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    ret = ls.wait()
+    ls_output = ls.stdout.read().decode()
+
+    if logger:
+        logger.debug(ls_output)
+
+    size = checksum = None
+    if ret == 0:
+        size = int(ls_output.split()[4])
+        checksum = adler32(filename)
+
+    return size, checksum
+
+def listdir(directory, stager, logger=None):
+    del directory, stager, logger
+    return (True, [])
+
+def migrated(filename, stager, logger=None):
+    del filename, stager, logger
+    return True
+
+def mkdir(directory, stager, logger=None):
+    del directory, stager, logger
+    return 0
+
+def remove(dstfile, stager, logger=None):
+    del dstfile, stager, logger
+    return 0
+
+def backgroundcopy(srcfile, dstfile, stager, logger=None):
+    """
+    This still needs to return a process (subprocess object) so it spawns the
+    simplest process that succeeds.
+    """
+    del srcfile, dstfile, stager, logger # unused args
+    cp = subprocess.Popen(['true'], stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT)
+    return cp, cp.pid
+
+def copystate(proc):
+    return proc.poll()
+
+def copystdout(proc):
+    return proc.stdout.read().decode()
+
+if __name__=='__main__':
+    import sys
+    import cs.StorageBackends.storagetester as storagetester
+    storagetester.execute(sys.modules[__name__])