From 71336fc16f2e8cf0e46364a36ff45494e6a76008 Mon Sep 17 00:00:00 2001
From: Fabrice Le Goff <fabrice.le.goff@cern.ch>
Date: Fri, 23 Jun 2017 13:38:58 +0200
Subject: [PATCH] fix envvar type, fix file permisssions
---
Script/CastorScript.py | 0
Script/Conf.cfg | 0
Script/Conf.py | 0
Script/CopyThread.py | 0
Script/Database.py | 0
Script/DeleteThread.py | 0
Script/ManagerThread.py | 0
Script/eosstorage.py | 2 +-
Test/coral/authentication.xml | 10 ++
Test/readme | 30 +++++
Test/test_config.cfg | 232 ++++++++++++++++++++++++++++++++++
11 files changed, 273 insertions(+), 1 deletion(-)
mode change 100755 => 100644 Script/CastorScript.py
mode change 100755 => 100644 Script/Conf.cfg
mode change 100755 => 100644 Script/Conf.py
mode change 100755 => 100644 Script/CopyThread.py
mode change 100755 => 100644 Script/Database.py
mode change 100755 => 100644 Script/DeleteThread.py
mode change 100755 => 100644 Script/ManagerThread.py
create mode 100644 Test/coral/authentication.xml
create mode 100644 Test/readme
create mode 100644 Test/test_config.cfg
diff --git a/Script/CastorScript.py b/Script/CastorScript.py
old mode 100755
new mode 100644
diff --git a/Script/Conf.cfg b/Script/Conf.cfg
old mode 100755
new mode 100644
diff --git a/Script/Conf.py b/Script/Conf.py
old mode 100755
new mode 100644
diff --git a/Script/CopyThread.py b/Script/CopyThread.py
old mode 100755
new mode 100644
diff --git a/Script/Database.py b/Script/Database.py
old mode 100755
new mode 100644
diff --git a/Script/DeleteThread.py b/Script/DeleteThread.py
old mode 100755
new mode 100644
diff --git a/Script/ManagerThread.py b/Script/ManagerThread.py
old mode 100755
new mode 100644
diff --git a/Script/eosstorage.py b/Script/eosstorage.py
index ead10ad..9dbe136 100644
--- a/Script/eosstorage.py
+++ b/Script/eosstorage.py
@@ -111,7 +111,7 @@ def backgroundcopy(srcfile, dstfile, stager, pool, logger=None):
# Disable default XRootD's writing recovery policy
modified_env = os.environ.copy()
- modified_env['XRD_WRITERECOVERY'] = 0
+ modified_env['XRD_WRITERECOVERY'] = '0'
xrdcp = Popen(cmd, stdout = PIPE, stderr = STDOUT, env=modified_env)
diff --git a/Test/coral/authentication.xml b/Test/coral/authentication.xml
new file mode 100644
index 0000000..794a4bb
--- /dev/null
+++ b/Test/coral/authentication.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" ?>
+<connectionlist>
+
+<connection name="oracle://intr/ATLAS_SFO_T0">
+ <parameter name="user" value="ATLAS_SFO_T0" />
+ <parameter name="password" value="bLX=tIP%PK" />
+</connection>
+
+</connectionlist>
+
diff --git a/Test/readme b/Test/readme
new file mode 100644
index 0000000..f61bac3
--- /dev/null
+++ b/Test/readme
@@ -0,0 +1,30 @@
+Point1 Test Cookbook
+--------------------
+
+Select an unassigned calibration machine (for example):
+ https://atlasop.cern.ch/twiki/bin/view/Main/CalibrationFarm
+Check that no instance of CastorScript is already on this machine
+Copy test version of CastorScript you want to test to P1
+Make it accessible to atlascdr
+Login to P1, then on the chosen machine
+
+sudo su - atlascdr
+mkdir /tmp/atlascdr
+mkdir /tmp/atlascdr/logs
+for i in `seq 0 9`; do dd if=/dev/zero of=/tmp/atlascdr/data17_13TeV.00999999.physics_test.daq.RAW._lb0001._SFO-9._000$i.data bs=1024k count=20; done
+cd CastorScript/Test
+cp test_config.cfg xxx.cfg
+Modify xxx.cfg appropriately
+To check files on EOS: xrd eosatlas dirlist /eos/atlas/atlastier0/rucio/data_test
+
+/sw/castor/script_setup.sh
+export CORAL_AUTH_PATH="$CORAL_AUTH_PATH:/path/to/CastorScript/coral/"
+python -u ../Script/CastorScript.py xxx.cfg
+
+rm -rf /tmp/atlascdr
+To clean EOS data_test, from lxplus as atlascdr:
+ xrd eosatlas dirlist /eos/atlas/atlastier0/rucio/data_test > datatest.ls
+ awk '{ print $NF }' < datatest.ls > datatest.entries
+ for i in `cat datatest.entries`; do eos rm -r $i; done
+ rm datatest.ls datatest.entries
+
diff --git a/Test/test_config.cfg b/Test/test_config.cfg
new file mode 100644
index 0000000..bd00e65
--- /dev/null
+++ b/Test/test_config.cfg
@@ -0,0 +1,232 @@
+########## Common parameters ##########
+
+# Log directory
+LogDir: '/tmp/atlascdr/logs/'
+
+#Lock file
+#lockFile: 'SFO.lock'
+lockFile: 'nolock'
+
+# Log severity level (DEBUG,INFO,WARNING,ERROR,CRITICAL)
+LogLevel: 'debug'
+
+# ERS severity level (DEBUG,INFO,WARNING,ERROR,CRITICAL)
+ERSLogLevel: 'info'
+
+# On / Off switch for ERS
+ERSenabled: False
+
+# Email list which will receive error messages (list of strings)
+#mailList: ['flegoff@cern.ch']
+mailList: []
+
+# Email Log severity level (DEBUG,INFO,WARNING,ERROR,CRITICAL)
+mailLevel: 'ERROR'
+
+# Email sender
+mailSender: 'testcastorscript@cern.ch'
+
+# PYTHON_PATH add-ons
+# e.g. ['/path/to/your/modules','/another/path']
+pythonPathPrepend: []
+
+
+########## VARIABLES FOR ERS ##############
+
+# Partition to log to
+#partition: "part_flegoff"
+
+# Stream ERS messages to
+#ers_info: "mrs"
+#ers_warning: "mrs"
+#ers_error: "mrs"
+#ers_debug: "mrs"
+
+########## METADATA DATABASE ##########
+
+# Oracle connection string
+connection: 'oracle://intr/ATLAS_SFO_T0'
+#Use None to disable DB access
+#connection: None
+
+# File table name
+file_table: "SFO_TZ_FILE"
+
+# Lumiblock table name
+lb_table: "SFO_TZ_LUMIBLOCK"
+
+# Run table name
+run_table: "SFO_TZ_RUN"
+
+
+
+########## MAIN THREAD ##########
+
+# If Oracle database is down, retry to connect to it every DBTimeout (s)
+DBTimeout: 2
+
+# Timeout to refresh the Oracle connection
+DBReconnectTimeout: 3600
+
+# Filename parser module
+Filenameparser: 'SFOFileNameParser'
+
+# Filename parser configuration
+FilenameparserConf: {}
+
+# Module provide backend access functions
+BackendModule: 'eosstorage'
+
+# Backend specific parameters
+BackendModuleConf: {}
+
+# Keytab file for Kerberos auth. None to skip
+#A keytab file can be generated with:
+#$> /usr/kerberos/sbin/ktutil
+#ktutil: add_entry -password -p <username>@CERN.CH -k 1 -e arcfour-hmac-md5
+#ktutil: add_entry -password -p <username>@CERN.CH -k 1 -e aes256-cts
+#ktutil: wkt <filename>
+Keytab: '/daq_area/castor/pc-tdq-calib-12/atlascdr/atlascdr.keytab'
+
+# Kerberos user corresponding to the above keytab
+KrbUser: 'atlascdr'
+
+# Path to custom Kerberos cache file. None to use the default one
+KrbCache: '/tmp/castor_krbcache'
+
+
+#[Manager]
+########## MANAGER THREAD ##########
+
+# Directory List where to get Data Files (list of strings)
+DirList: ['/tmp/atlascdr/']
+
+
+# Unix pathname pattern for file selection
+# FilePattern: ['*.data', '*.out']
+DataFilePattern: ['*.data',]
+
+# Number of files to be deleted before to update the list of files to be copied
+nDel: 1
+
+# Sleep timeout (s) before updating the list of files to be copied
+ManagerTimeout: 2
+
+# Delay (s) before to put .PROBLEMATIC file back into the CopyQueue
+ProblDelay: 3
+
+# At each retry, the delay for .PROBLEMATIC files is scaled
+# using this factor:
+# ActualProblDelay = ProblDelay*e^(ProblScalingFactor*#retries)
+ProblScalingFactor: 0.5
+
+# Actual .PROBLEMATIC delay limit to enable CRITICAL messages
+ProblDelayLimit: 200
+
+# Minimal file size in kB
+MinSizekB: 0.
+
+# If small files should be removed
+RemoveSmallFiles: False
+
+# Directory where to copy the Data Files (string)
+#Keywords can be used here
+#%(year)s
+#%(month)s
+#%(streamtype)s
+#%(streamname)s
+#%(runnumber)s
+#%(filenr)s
+#%(lumiblocknr)s
+#%(projecttag)s
+#%(dataset)s
+#%(applicationid)s
+#%(directory)s
+#%(userdef1)s
+#%(userdef2)s
+#%(userdef3)s
+CopyDir: '/eos/atlas/atlastier0/rucio/data_test/'
+
+# Pool in Castor
+Pool: ''
+
+# Stage Host
+StageHost: 'eosatlas'
+
+drivenPool: []
+
+
+#[Copy]
+########## COPY THREAD ##########
+
+# Max number of concurrent cp processes
+maxCopy: 3
+
+# In case of failure, max number of rfcp retries on the spot (before let the Manager take care of retring)
+maxRetry: 2
+
+# Timeout (s) after which kill and retry an rfcp command in None status
+NoneTimeout: 0
+
+# Sleep timeout (s) before checking again the copy process status
+CopyTimeout: 2
+
+# Kill ongoing transfers if a FS gets locked
+avoidCopyOverlap: False
+
+# Calculate the adler32 checksum from the local file instead of using the one
+# provided by the database (if any)
+computeChecksum: False
+
+# Bandwidth limit (MB/s)
+bwLimit: 0
+
+# Bandwidth limit device
+bwDevice: None
+
+#[Delete]
+########## DELETE THREAD ##########
+
+# Is simultaneous writing and deleting allowed?
+ignoreLock: True
+
+# Get filesystem info after nFiles have been copied
+nFiles: 2
+
+# Do not delete files more recent than minFileTime (s)
+minFileTime: 600
+
+# Is migration required before deleting?
+migFlag: False
+
+# Module that will be used to check the migration state of merged files
+# Will look for the function MergedNameModule.MergedNameModule
+MergedNameModule: None
+
+## Configuration for MergeModule
+MergedModuleConf: {}
+
+# Perform checksum check (for migrated files only)
+checksumCheck: False
+
+# Max total filesystem usage allowed while waiting for migration (in %)
+highCriticMark: 90
+
+# Min total filesystem usage allowed while waiting for migration (in %)
+lowCriticMark: 80
+
+# Low watermark: do not delete if filesystems usage in DirList below threshold (in %)
+lowWaterMark: [1]
+
+# Sleep timeout (s) before deleting next file in DeleteQueue
+DeleteTimeout: 2
+
+# Delay before to check if file is migrated to Castor tape (s)
+MigDelay: 600
+
+########## CHECK THREAD ##########
+# Time (s) between checks for partition
+CheckTimeout: 1
+
+# Timout (s) before adding the ERS handler
+ERSTimeout: 10
--
GitLab