diff --git a/DeploymentTest/p1.test.atlascdr.bash b/DeploymentTest/p1.test.atlascdr.bash index 02eee280d29acc86a96ec92b991124f3264b0d65..5c686610e78da83bb0ddc2bd8faa2df59580437b 100755 --- a/DeploymentTest/p1.test.atlascdr.bash +++ b/DeploymentTest/p1.test.atlascdr.bash @@ -107,7 +107,7 @@ while [ "x$LISTING_OUTPUT" != x ]; do done echo "Stopping CastorScript" -~/signal-script.sh 12 +/sw/castor/tools/castor.signal 12 if [[ $COPY_ENABLED == True && $REMOTE_DIR == /eos/* ]]; then echo "Contents of remote directory:" diff --git a/DeploymentTest/stop.all.cs.instances.at.p1.machine.bash b/DeploymentTest/stop.all.cs.instances.at.p1.machine.bash deleted file mode 100755 index 1e04765d066b2662fdb67fb94643f25ebd291543..0000000000000000000000000000000000000000 --- a/DeploymentTest/stop.all.cs.instances.at.p1.machine.bash +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -if [ -z $1 ]; then - echo "usage: $0 {test host}" - exit 1 -fi - -TESTHOST=$1 - - -KEYTAB_FILE="/daq_area/castor/$TESTHOST/atlascdr/atlascdr.keytab" -if ! ssh $TESTHOST "sudo -u atlascdr -i cat $KEYTAB_FILE &>/dev/null"; then - echo "error: keytab file not readble by atlascdr ($KEYTAB_FILE)" - exit 3 -else - echo "OK: readable keytab file: $KEYTAB_FILE" -fi - -SOURCE_DIR=`pwd`/.. -TOOLS_DIR=`pwd`/../../flegoff/castor_tools - -echo "sending signal 12 to CastorScripts at $TESTHOST" -ssh $TESTHOST sudo -u atlascdr -i $TOOLS_DIR/signal-script.sh 12 diff --git a/ProductionTools/castor.eostimeout.freq b/ProductionTools/castor.eostimeout.freq deleted file mode 100755 index ebf039f702ebd8739f11dd623d5b130bf24cbd9b..0000000000000000000000000000000000000000 --- a/ProductionTools/castor.eostimeout.freq +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -grep "taking to long" /atlas/logs/castor/pc-tdq-sfo-0?/CopyLog.out | awk -F':' '{print $2}' | sort | uniq -c - diff --git a/ProductionTools/castor.checklogs b/ProductionTools/installed/castor.checklogs similarity index 75% rename from ProductionTools/castor.checklogs rename to ProductionTools/installed/castor.checklogs index 336c83387063b05b958666a9f3c4a25126baf27e..818c70b66e06e7f516df2998aad7420b057332c7 100755 --- a/ProductionTools/castor.checklogs +++ b/ProductionTools/installed/castor.checklogs @@ -6,11 +6,10 @@ for c in `cat castor.confs`; do logdir=$(grep LogDir $c | awk '{print $NF}' | tr -d "'") logdir=${logdir%/} # remove trailing slash - lsl=$(ls -l `echo "$logdir/*.out"` 2>/dev/null) + lsl=$(ls -l `echo "$logdir/*.log"` 2>/dev/null) if [ $? -eq 0 ]; then # Special case for ManagerLog.out which always contain 80/81 bytes # because it outputs its thread ID even if logged are limited to >info - #echo "$lsl" | awk '/.*ManagerLog.out$/{if ($5 > 82) print $NF ": " $5} ! /.*ManagerLog.out$/ {if ($5 > 0) print $NF ": " $5}' echo "$lsl" | awk '{if ($5 > 0) print $NF ": " $5}' else echo "No log files for $c: $logdir" diff --git a/ProductionTools/castor.greplogs b/ProductionTools/installed/castor.greplogs similarity index 82% rename from ProductionTools/castor.greplogs rename to ProductionTools/installed/castor.greplogs index 493b2fc9133cf102fc3ef2c6e6524f72d570c14c..096f716c300b1c791d5c9dbb3cf4b6c1fbef760c 100755 --- a/ProductionTools/castor.greplogs +++ b/ProductionTools/installed/castor.greplogs @@ -14,7 +14,7 @@ fi for conf in "$CASTOR_CONFS"; do logdir=`grep LogDir $conf | awk '{print $NF}' | tr -d "'"` - for logfile in `find $logdir -maxdepth 1 -name '*.out'`; do #-maxdepth 1 + for logfile in `find $logdir -maxdepth 1 -name '*.log.*'`; do #-maxdepth 1 grep "$TOGREP" $logfile done done diff --git a/ProductionTools/castor.logdir b/ProductionTools/installed/castor.logdir similarity index 100% rename from ProductionTools/castor.logdir rename to ProductionTools/installed/castor.logdir diff --git a/ProductionTools/castor.logsfreq b/ProductionTools/installed/castor.logsfreq similarity index 91% rename from ProductionTools/castor.logsfreq rename to ProductionTools/installed/castor.logsfreq index 7ec1fef2f8f43dc16b06fc0bccf40d38be0bd4ce..b295705d3eb8a7d9358e58a6e8bea1b5026b659e 100755 --- a/ProductionTools/castor.logsfreq +++ b/ProductionTools/installed/castor.logsfreq @@ -26,7 +26,7 @@ for i in `seq -$(($NB_DAYS - 1)) 0`; do for conf in "$CASTOR_CONFS"; do logdir=`grep LogDir $conf | awk '{print $NF}' | tr -d "'"` - for logfile in `find $logdir -name '*.out'`; do #-maxdepth 1 + for logfile in `find $logdir -name '*.log.*'`; do #-maxdepth 1 grep "$TOGREP" $logfile | grep -c $DATE done done | awk 'BEGIN {s=0} {s+=$1} END {print s}' diff --git a/ProductionTools/castor.notrunning b/ProductionTools/installed/castor.notrunning similarity index 100% rename from ProductionTools/castor.notrunning rename to ProductionTools/installed/castor.notrunning diff --git a/ProductionTools/castor.ps b/ProductionTools/installed/castor.ps similarity index 100% rename from ProductionTools/castor.ps rename to ProductionTools/installed/castor.ps diff --git a/ProductionTools/castor.pslocal b/ProductionTools/installed/castor.pslocal similarity index 100% rename from ProductionTools/castor.pslocal rename to ProductionTools/installed/castor.pslocal diff --git a/ProductionTools/sfotzdb_intr_writer.py b/ProductionTools/installed/castor.sfotzdb_intr_writer.py old mode 100644 new mode 100755 similarity index 69% rename from ProductionTools/sfotzdb_intr_writer.py rename to ProductionTools/installed/castor.sfotzdb_intr_writer.py index 3437db71f36bb072aec658d402c2554c26686d10..857da17139bdbba627c8fbf2770ca8f84bf2b5bf --- a/ProductionTools/sfotzdb_intr_writer.py +++ b/ProductionTools/installed/castor.sfotzdb_intr_writer.py @@ -1,3 +1,4 @@ +#!/sw/atlas/sw/lcg/releases/LCG_101/Python/3.9.6/x86_64-centos7-gcc11-opt/bin/python -i import cx_Oracle import coral_auth user,pwd,dbn = coral_auth.get_connection_parameters_from_connection_string('oracle://int8r/ATLAS_SFO_T0') diff --git a/ProductionTools/sfotzdb_reader.py b/ProductionTools/installed/castor.sfotzdb_reader.py old mode 100644 new mode 100755 similarity index 70% rename from ProductionTools/sfotzdb_reader.py rename to ProductionTools/installed/castor.sfotzdb_reader.py index 0d484412d401f2ce83dd6a0d1972c3936c882071..7eb5ff70f62375a98e59cd49253f3281b216422c --- a/ProductionTools/sfotzdb_reader.py +++ b/ProductionTools/installed/castor.sfotzdb_reader.py @@ -1,3 +1,4 @@ +#!/sw/atlas/sw/lcg/releases/LCG_101/Python/3.9.6/x86_64-centos7-gcc11-opt/bin/python -i import cx_Oracle import coral_auth user,pwd,dbn = coral_auth.get_connection_parameters_from_connection_string('oracle://atonr_conf/ATLAS_SFO_T0_R') diff --git a/ProductionTools/sfotzdb_writer.py b/ProductionTools/installed/castor.sfotzdb_writer.py old mode 100644 new mode 100755 similarity index 70% rename from ProductionTools/sfotzdb_writer.py rename to ProductionTools/installed/castor.sfotzdb_writer.py index 2bb7f6ec155692d7a64bf6a35e7028db85926e61..d4c4717bc2de073d285d0cb346ff3186f16e9b23 --- a/ProductionTools/sfotzdb_writer.py +++ b/ProductionTools/installed/castor.sfotzdb_writer.py @@ -1,3 +1,4 @@ +#!/sw/atlas/sw/lcg/releases/LCG_101/Python/3.9.6/x86_64-centos7-gcc11-opt/bin/python -i import cx_Oracle import coral_auth user,pwd,dbn = coral_auth.get_connection_parameters_from_connection_string('oracle://atonr_conf/ATLAS_SFO_T0') diff --git a/ProductionTools/installed/castor.signal b/ProductionTools/installed/castor.signal new file mode 100755 index 0000000000000000000000000000000000000000..e768e1e8048ac595e0999a91ec969ff5e6fd3703 --- /dev/null +++ b/ProductionTools/installed/castor.signal @@ -0,0 +1,9 @@ +#!/bin/bash + +pid=`ps -elf | grep Castor | grep python | grep -v grep | awk '{print \$4}'` +if [ "x$pid" == x ] ; then + echo "cannot send signal: castor script is not running" + exit -1 +else + kill -s $1 $pid +fi diff --git a/ProductionTools/castor.starttime b/ProductionTools/installed/castor.starttime similarity index 100% rename from ProductionTools/castor.starttime rename to ProductionTools/installed/castor.starttime diff --git a/ProductionTools/castor.stopall b/ProductionTools/installed/castor.stopall similarity index 56% rename from ProductionTools/castor.stopall rename to ProductionTools/installed/castor.stopall index 74ba1184ece72d945b9b0f0387d36fba298119e3..dda4758fd9908fb74cf9be8cdcdd1eef5c096024 100755 --- a/ProductionTools/castor.stopall +++ b/ProductionTools/installed/castor.stopall @@ -1,5 +1,5 @@ #!/usr/bin/env bash for c in `cat castor.machines`; do echo "$c" - ssh -x $c "~flegoff/castor_tools/signal-script.sh 12" + ssh -x $c "/sw/castor/tools/castor.signal 12" done diff --git a/ProductionTools/generate_instance_lists.sh b/ProductionTools/installed/castor.updateconflist similarity index 100% rename from ProductionTools/generate_instance_lists.sh rename to ProductionTools/installed/castor.updateconflist diff --git a/ProductionTools/oracle_db_disaster_recovery/oracle_db_disaster_recovery.py b/ProductionTools/oracle_db_disaster_recovery/oracle_db_disaster_recovery.py index 1afb68d76773367486e9515cff815b5cd0e4fd69..7878d3cf5de17b3ca13ba61e88833b4481f386a9 100644 --- a/ProductionTools/oracle_db_disaster_recovery/oracle_db_disaster_recovery.py +++ b/ProductionTools/oracle_db_disaster_recovery/oracle_db_disaster_recovery.py @@ -103,7 +103,8 @@ if not args.dryrun: f'/mnt/daq_area_rw/castor/pc-tdq-sfo-{i}/atlascdr/prod.stopped') if args.verbose: print(f'disabled CS on sfo-{i}') if castorscript_is_running(f'pc-tdq-sfo-{i}', 'prod.cfg'): - stop_cs_cmd = ['ssh', '-x', f'pc-tdq-sfo-{i}', '~/signal-script.sh', '12'] + stop_cs_cmd = ['ssh', '-x', f'pc-tdq-sfo-{i}', + '/sw/castor/tools/castor.signal', '12'] subprocess.run(stop_cs_cmd, check=True) if args.verbose: print(f'sent stop signal to CS on sfo-{i}: waiting for CS to stop') while castorscript_is_running(f'pc-tdq-sfo-{i}', 'prod.cfg'): diff --git a/point1.deployment.sh b/point1.deployment.sh index c6064fefc2c032e6fa4d9ff97bc1ad8246e4645a..c1491397e9cd2762a4fb384212a2ccd1751ec855 100755 --- a/point1.deployment.sh +++ b/point1.deployment.sh @@ -32,6 +32,8 @@ fi # DB disaster recovery are deployed as part of the CastorScript mv ProductionTools/oracle_db_disaster_recovery . || err "cannot mv oracle_db_disaster_recovery" +# operation tools +mv ProductionTools/installed tools || err "cannot mv installed tools" # remove from local directory what we don't want to be deployed rm -rf Configs DeploymentTest ProductionTools UnitTests .git || err "cannot delete"