diff --git a/DeploymentTest/readme b/DeploymentTest/readme deleted file mode 100644 index beb9218524db08c08fccb4ae93e32c842895e880..0000000000000000000000000000000000000000 --- a/DeploymentTest/readme +++ /dev/null @@ -1,29 +0,0 @@ -Point 1 Tests -------------- -Select an unassigned calibration machine (for example): - https://atlasop.cern.ch/twiki/bin/view/Main/CalibrationFarm -Log in to P1 -git clone https://:@gitlab.cern.ch:8443/atlas-tdaq-software/CastorScript.git -cd CastorScript/DeploymentTest -./p1.test.user.bash pc-tdq-calib-XX - -If you need to have an modified authorization file: - source /sw/castor/script_setup.sh - export CORAL_AUTH_PATH="$CORAL_AUTH_PATH:/path/to/CastorScript/coral/" - python -u ../Script/CastorScript.py xxx.cfg - -Testing a new watchdog / Running a non-installed version --------------------------------------------------------- -If the parameter "TEST" is provided to the watchdog it runs version from: -"/atlas-home/1/flegoff/CastorScript" instead of "/sw/castor/" -and look for "*.testcfg" configuration files instead of "*.cfg" - -To test a new version on a production config: change the config file extension, -kill the running instance, and setup a task in atlascdr's crontab on HOST: - mv /mnt/daq_area_rw/castor/HOST/atlascdr/xxx.cfg \ - /mnt/daq_area_rw/castor/HOST/atlascdr/xxx.testcfg - ssh -x HOST /sw/castor/tools/castor.signal 12 - ssh HOST - crontab -e - MAILTO="flegoff@cern.ch" - */5 * * * * /atlas-home/1/flegoff/CastorScript/watchdog.sh TEST >> /logs/castor_watchdog_test.log 2>&1 diff --git a/DeploymentTest/test_config.cfg b/DeploymentTest/test_config.cfg index aaa11fccf4dd9921d2d39e1aef2ae043c53b8865..da970123bdcfd5182a97265060c7d5ebc1e62b70 100644 --- a/DeploymentTest/test_config.cfg +++ b/DeploymentTest/test_config.cfg @@ -1,6 +1,6 @@ LogDir: '/tmp/atlascdr/logs/' LogLevel: 'debug' -EmailLogList: ['flegoff@cern.ch'] +EmailLogList: ['vandelli@cern.ch'] EmailLogLevel: 'ERROR' EmailLogSender: 'testcastorscript@cern.ch' diff --git a/DeploymentTest/test_config_is_ers.cfg b/DeploymentTest/test_config_is_ers.cfg index db8d0e7f829a55fba07b95f8fa1bfd40a5ad38e0..ccd7ae63655a180d7191df1dc46d4b1d4c0bed9d 100644 --- a/DeploymentTest/test_config_is_ers.cfg +++ b/DeploymentTest/test_config_is_ers.cfg @@ -1,6 +1,6 @@ LogDir: '/tmp/atlascdr/logs/' LogLevel: 'debug' -EmailLogList: ['flegoff@cern.ch'] +EmailLogList: ['vandelli@cern.ch'] EmailLogLevel: 'ERROR' EmailLogSender: 'testcastorscript@cern.ch' diff --git a/DeploymentTest/test_config_is_ers_mig.cfg b/DeploymentTest/test_config_is_ers_mig.cfg index 597a0481a9819f027a23e3a13a76f606a831f594..20fe6fe73a56ab8c29ef122c9ba3a913e82e4444 100644 --- a/DeploymentTest/test_config_is_ers_mig.cfg +++ b/DeploymentTest/test_config_is_ers_mig.cfg @@ -1,6 +1,6 @@ LogDir: '/tmp/atlascdr/logs/' LogLevel: 'debug' -EmailLogList: ['flegoff@cern.ch'] +EmailLogList: ['vandelli@cern.ch'] EmailLogLevel: 'ERROR' EmailLogSender: 'testcastorscript@cern.ch' diff --git a/DeploymentTest/test_config_localreader.cfg b/DeploymentTest/test_config_localreader.cfg index 903b571e0c3768f43c3fde5569e4b99ab8bc3ac8..9374c56fcee7a7302612282d470abd37ac865c88 100644 --- a/DeploymentTest/test_config_localreader.cfg +++ b/DeploymentTest/test_config_localreader.cfg @@ -1,6 +1,6 @@ LogDir: '/tmp/atlascdr/logs/' LogLevel: 'debug' -EmailLogList: ['flegoff@cern.ch'] +EmailLogList: ['vandelli@cern.ch'] EmailLogLevel: 'ERROR' EmailLogSender: 'testcastorscript@cern.ch' @@ -8,7 +8,7 @@ MainThreadEventTimeout: 2 FilenameParser: 'TestFileNameParser' BackendModule: 'localreader' -BackendModuleConf: {'READ_SCRIPT': '/atlas-home/1/flegoff/CastorScript/Script/cs/StorageBackends/read_script.sh'} +BackendModuleConf: {'READ_SCRIPT': '/atlas-home/1/vandelli/CastorScript/Script/cs/StorageBackends/read_script.sh'} SrcDirs: ['/tmp/atlascdr/'] DataFilePattern: ['*.data',] diff --git a/DeploymentTest/test_config_localreader_is_ers.cfg b/DeploymentTest/test_config_localreader_is_ers.cfg index 2efc18910c3c7ec1fe962ef0cd7da8faea523890..dcb886bc6f0c22ed195b7d46862ee052267f3e26 100644 --- a/DeploymentTest/test_config_localreader_is_ers.cfg +++ b/DeploymentTest/test_config_localreader_is_ers.cfg @@ -1,6 +1,6 @@ LogDir: '/tmp/atlascdr/logs/' LogLevel: 'debug' -EmailLogList: ['flegoff@cern.ch'] +EmailLogList: ['vandelli@cern.ch'] EmailLogLevel: 'ERROR' EmailLogSender: 'testcastorscript@cern.ch' @@ -24,7 +24,7 @@ MainThreadEventTimeout: 2 FilenameParser: 'TestFileNameParser' BackendModule: 'localreader' -BackendModuleConf: {'READ_SCRIPT': '/atlas-home/1/flegoff/CastorScript/Script/cs/StorageBackends/read_script.sh'} +BackendModuleConf: {'READ_SCRIPT': '/atlas-home/1/vandelli/CastorScript/Script/cs/StorageBackends/read_script.sh'} SrcDirs: ['/tmp/atlascdr/'] DataFilePattern: ['*.data',] diff --git a/DeploymentTest/test_config_mig.cfg b/DeploymentTest/test_config_mig.cfg index aad21430444ffddc3dbf0cd1a388480dca7dadc9..a62c2d070750a2594fcfe1e20cd8edde27c1bf8e 100644 --- a/DeploymentTest/test_config_mig.cfg +++ b/DeploymentTest/test_config_mig.cfg @@ -1,6 +1,6 @@ LogDir: '/tmp/atlascdr/logs/' LogLevel: 'debug' -EmailLogList: ['flegoff@cern.ch'] +EmailLogList: ['vandelli@cern.ch'] EmailLogLevel: 'ERROR' EmailLogSender: 'testcastorscript@cern.ch' diff --git a/readme b/readme deleted file mode 100644 index 7ae77cde39a05cab5421975220229d076f81247a..0000000000000000000000000000000000000000 --- a/readme +++ /dev/null @@ -1,19 +0,0 @@ -This is the software that transfers file from ATLAS online infrastructure to -offline storage. The name, CastorScript, has historical reasons: it used to copy -files directly to the CASTOR system (it does not anymore). - -Contents - Configs: reference, template and test configs - DeploymentTest: deployment test - Script: the actual code - ProductionTools: set of Bash and Python scripts used on the production system to - ease maintenance tasks - - pylintrc: configuration of pylint (used by CI) - readme: this file (are you even following?) - run_docker_container.sh: script to start a docker container in CWD for testing (must be run in CastorScript folder) - run_pylint.sh: script to run pylint on all files in Script - run_unit_tests.sh: script to search and run Python unit tests - script_setup.sh: script used in P1 to setup the running environment - watchdog.sh: script called by sysadmin's cron task that starts and maintains - CastorScript instance up and running. Also handles log turn over. diff --git a/readme.md b/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..7aba7681a1cff927dfbee0445428df63f439e77a --- /dev/null +++ b/readme.md @@ -0,0 +1,51 @@ +This is the software that transfers file from ATLAS online infrastructure to +offline storage. The name, CastorScript, has historical reasons: it used to copy +files directly to the CASTOR system (it does not anymore). + +# Contents + +- Configs: reference, template and test configs +- DeploymentTest: deployment test +- Script: the actual code +- ProductionTools: set of Bash and Python scripts used on the production system + to ease maintenance tasks +- pylintrc: configuration of pylint (used by CI) +- readme: this file (are you even following?) +- run_docker_container.sh: script to start a docker container in CWD for testing (must be run in CastorScript folder) +- run_pylint.sh: script to run pylint on all files in Script +- run_unit_tests.sh: script to search and run Python unit tests +- script_setup.sh: script used in P1 to setup the running environment +- watchdog.sh: script called by sysadmin's cron task that starts and maintains + CastorScript instance up and running. Also handles log turn over. + +# Semi-Automated Tests In P1 + +Choose a "CastorScript compatible", unused computer in P1 (unused calib +machines are a good choice). + +The, in P1, from the code source directory + + ./DeploymentTest/p1.test.user.bash pc-tdq-calib-21 + +The output should be self-explanatory. + +There are a number of options to this test script to test specific features +(online documentation is available): + +- `--isers`: enable IS and ERS publication features; requires running a + partition named 'castorscript'; +- `--mig`: enable migration check before deletion; uses + `/eos/ctaatlaspps/daqtest` as CTA target ; +- `--localreader`: no transfer takes place, files are only read locally. + +The last two options are mutually exclusive. + +# Installing a New Version in P1 + + # Tag the commit you want to install in git + ssh atlasgw + # git clone ssh://git@gitlab.cern.ch:7999/atlas-tdaq-software/CastorScript.git + cd CastorScript + git pull + ./point1.deployment.sh CastorScript-00-00-00 + # Let it flow (and check for errors) diff --git a/watchdog.sh b/watchdog.sh index b194c1812c8fb9ba84f2934d9c8f5a9ffd592a4c..8298bae96bc246c27bf299c43a3a2dfa037bae61 100755 --- a/watchdog.sh +++ b/watchdog.sh @@ -6,7 +6,7 @@ CONFDIR="/daq_area/castor/${SHORTHOSTNAME}/${USER}" CFG_EXTENSION="cfg" if [ x"$1" = xTEST ]; then - SWDIR="/atlas-home/1/flegoff/CastorScript" + SWDIR="/atlas-home/1/vandelli/CastorScript" CFG_EXTENSION="testcfg" echo "running test configurations: swdir=$SWDIR, cfg_ext=$CFG_EXTENSION" fi @@ -27,7 +27,7 @@ for CFG in $CONFDIR/*.$CFG_EXTENSION; do #get logdir from config file (find LogDir: in CFG | pipe 2nd part | del all ' & " | del last char if == / ) logdir=$(grep 'LogDir:' $CFG | awk '{print $2}' | tr -d "'" | tr -d "\"" | sed "s/\/$//") - + # If the CastorScript crashed, especially on uncaught exceptions, this watchdog # restarts it and we won't notice. Uncaught exception messages end up in # stdouterr: mail it to Fabrice if the file has non-zero length. @@ -38,7 +38,7 @@ for CFG in $CONFDIR/*.$CFG_EXTENSION; do cp $logdir/stdouterr $logdir/stdouterr.raw sed '/Partition ".*" does not exist/d;/CORBA system exception "TRANSIENT(1096024066=TRANSIENT_ConnectFailed)" has been raised/d' -i $logdir/stdouterr if [ -s $logdir/stdouterr ]; then - echo "$CFG" | mail -s "CastorScript instance crashed" -a $logdir/stdouterr -r "atlascdr@cern.ch" flegoff@cern.ch + echo "$CFG" | mail -s "CastorScript instance crashed" -a $logdir/stdouterr -r "atlascdr@cern.ch" wainer.vandelli@cern.ch fi fi