Commit db90e2f0 authored by Stefano Belforte's avatar Stefano Belforte
Browse files

test analysis test with 2_1_12

parent ad7d1ea7
#!/bin/sh
info=0
warning=0
error=0
isEGEE=0
isOSG=0
cat $SAME_SENSOR_HOME/tests/CMS-SAM-Banner.html
echo '<h1><a href="#summary">Jump to test summary</a></h1>'
echo "<h2>Preliminary checks</h2>"
echo "<pre>"
echo "version 23-02-2009 11:00"
uname -a
cat /etc/issue
date
date --utc
voms-proxy-info -identity -fqan
id
workdir=`mktemp -d workdirXXXXXX`
cd $workdir
cp -v $SAME_SENSOR_HOME/tests/cmssw_required_version_list.txt .
cp -v $SAME_SENSOR_HOME/tests/fetch-from-web .
[ -n "$GLITE_WMS_RB_BROKERINFO" ] && cp -v $GLITE_WMS_RB_BROKERINFO .
[ -n "$EDG_WL_RB_BROKERINFO" ] && cp -v $EDG_WL_RB_BROKERINFO .
echo "</pre>"
echo "Memory:"
echo "<pre>"
cat /proc/meminfo
echo "</pre>"
echo "<p>"
echo "<h2>Checking software directory for CMS VO</h2>"
echo "<p>"
if [ -n "$OSG_APP" ] ; then
isOSG=1
SW_DIR=$OSG_APP/cmssoft/cms
echo "Checking variable: \$OSG_APP/cmssoft/cms=$OSG_APP/cmssoft/cms"
echo "</p>"
elif [ -n "$VO_CMS_SW_DIR" ] ; then
isEGEE=1
SW_DIR=$VO_CMS_SW_DIR
echo "Checking variable: \$VO_CMS_SW_DIR=$VO_CMS_SW_DIR"
echo "</p>"
else
echo "<strong>"
echo "ERROR: Neither VO_CMS_SW_DIR nor OSG_APP defined"
echo "</strong></p>"
exit $SAME_ERROR
fi
echo "<pre>"
ls -ld $SW_DIR
result=$?
echo "</pre>"
echo
if [ $result != 0 ]
then
echo "<strong><p>"
echo "ERROR: software $SW_DIR directory non existent or non readable"
echo "</strong></p>"
# echo "summary: NO_SW_DIR"
exit $SAME_ERROR
fi
echo "<h2><a name="DiskSpace"></a>Some trivial checks</h2>"
if [ "`echo $SW_DIR | cut -d / -f 2`" == "afs" ]; then
echo "AFS mode"
SPACE=`fs lq $SW_DIR | tail -1 | awk '{print (\$2-\$3)/1024 }'`
else
SPACE=`df -k -P $SW_DIR | tail -1 | awk '{print \$4/1024}'`
fi
echo "Free disk space in $SW_DIR: $SPACE MB"
cantwrite=0
# on EGEE (but not CERN) check that lcgadmin can write to SW area
if [ $isEGEE == 1 ] && ! ( echo $SW_DIR | grep cern.ch )
then
echo "<pre>"
touch $SW_DIR/.sametest
result=$?
echo "</pre>"
if [ $result != 0 ]
then
cantwrite=1
echo "WARNING: cannot write to software area"
fi
fi
if [ ! -f $SW_DIR/cmsset_default.sh ] ; then
echo "<h3>"
echo "ERROR: cmssw setup file $SW_DIR/cmsset_default.sh not existing"
echo "</h3>"
exit $SAME_ERROR
fi
echo "<h2>Sourcing the CMS environment</h2>"
echo "<pre>"
echo "source $SW_DIR/cmsset_default.sh"
echo "</pre>"
archs_defined='slc4_ia32_gcc345 slc3_ia32_gcc323'
archs_to_test='slc4_ia32_gcc345'
swok=0
for arch in $archs_defined ; do
if [ $swok != 1 ] ; then
export SCRAM_ARCH=$arch
export BUILD_ARCH=$arch
echo "<p>"
echo "Testing with SCRAM_ARCH=$arch ..."
echo "</p>"
echo "<pre>"
rm -f cmsset.log
source $SW_DIR/cmsset_default.sh > cmsset.log 2>&1
result=$?
cat cmsset.log
echo "</pre>"
echo
grep 'No such file or directory' cmsset.log > /dev/null 2>&1
res=$?
if [ $res == 0 ] ; then
result=1
fi
if [ $result != 0 ]
then
echo "<p>"
echo "WARNING: CMS software initialisation script cmsset_default.sh failed for SCRAM_ARCH=$arch"
echo "</p>"
else
swok=1
fi
fi
done
if [ $swok != 1 ]
then
echo "<strong><p>"
echo "ERROR: CMS software initialisation script cmsset_default.sh failed for all architectures"
echo "</strong></p>"
exit $SAME_ERROR
fi
echo "<p>"
echo "OK"
echo "</p>"
echo "<h2>Make sure configuration directory is there</h2>"
echo "<p>"
echo "Checking variable: \$CMS_PATH=$CMS_PATH"
echo "</p>"
if [ -z $CMS_PATH ]; then
echo "<strong><p>"
echo "ERROR: CMS_PATH not defined"
echo "</strong></p>"
exit $SAME_ERROR
fi
if [ ! -d $CMS_PATH ] ; then
echo "<strong><p>"
echo "ERROR: CMS_PATH directory $CMS_PATH not existing"
echo "</strong></p>"
exit $SAME_ERROR
fi
echo "<p>"
echo "OK"
echo "</p>"
echo "<h2>Check that scramv1 command is present</h2>"
echo "<pre>"
echo "scramv1 version"
scramv1 version
result=$?
echo "</pre>"
if [ $result != 0 ]
then
echo "<strong><p>"
echo "ERROR: scramv1 command not found"
echo "</strong></p>"
exit $SAME_ERROR
fi
echo "<p>"
echo "OK"
echo "</p>"
#cd $SAME_SENSOR_WORK
echo "<h2>Retrieve list of CMSSW versions installed</h2>"
for arch in $archs_defined
#
# Choose CMSSW version
#
cmsswvsn=CMSSW_2_1_12
#
# Print CMS_PATH
#
printf "CMS_PATH $CMS_PATH\n"
#
# Check that scramv1 command was defined by . $CMS_PATH/cmsset_default.sh
#
type scramv1 > /dev/null 2>&1
sset=$?
if [ $sset -ne 0 ]
then
printf "AnalysisTest.sh: Error. scramv1 not found\n"
exit $SAME_ERROR
fi
#
# Get Working Directory
#
current=`pwd`
#
# Set up CMSSW
#
tmpfile=`mktemp /tmp/tmp.scram.XXXXXXXXXX`
printf "scramv1 project CMSSW $cmsswvsn ... starting\n"
scramv1 project CMSSW $cmsswvsn > $tmpfile 2>&1
scms=$?
if [ $scms -ne 0 ]
then
cat $tmpfile
printf "AnalysisTest.sh: Error. $cmsswvsn not available\n"
exit $SAME_ERROR
fi
tail -5 $tmpfile
printf "scramv1 project CMSSW $cmsswvsn ... completed\n"
#
cd $cmsswvsn/src
export SCRAM_ARCH=`scramv1 arch`
printf "scramv1 runtime -sh ... starting\n"
eval `scramv1 runtime -sh`
printf "scramv1 runtime -sh ... completed\n"
#
# Return to working directory
#
cd $current
#
# There are all files available in SAM dataset, which is at all sites
#
/bin/cat > possible-input-files <<EOI
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/18A1B60B-A5A5-DD11-84FC-00163E1124E2.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/2C556CAE-ACA5-DD11-A6C7-00163E11248E.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/5A50C03F-ADA5-DD11-9FE8-00163E11247A.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/5E326586-A4A5-DD11-A6AA-0015600E3ECA.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/806CD613-AEA5-DD11-8795-00163E112499.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/8C1E14E9-AFA5-DD11-ACFE-00163E1124AF.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/94AD82A3-AEA5-DD11-9E95-00163E11247A.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/A82F9385-A3A5-DD11-B03E-00163E1124E9.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/BCC46CCA-A4A5-DD11-9CB5-001560AC1D24.root
/store/mc/SAM/QCD_pt_0_15/GEN-SIM-RAW-RECO/IDEAL_V9_SAM/0000/F21EB2B6-ABA5-DD11-87D5-00163E11247C.root
EOI
#
nFiles=`cat possible-input-files | wc -l`
#
# pick a randmon integer in [1,nFiles]
d=`date +%N|tr -d 0`
let r=d%nFiles+1
#
echo "Will use file N. $r fom possible input files"
nl=0
until [ $nl -eq $r ]
do
export SCRAM_ARCH=$arch
export BUILD_ARCH=$arch
echo "<pre>"
echo "Architecture: $arch"
echo "scramv1 list -c CMSSW"
scramv1 list -c CMSSW > scramv1_list_output.txt
listerror=$?
echo "</pre>"
if [ $listerror != 0 ]
then
echo "<p>"
echo "WARNING: scramv1 list error"
echo "</p>"
fi
touch scramv1_list_output.txt
cat scramv1_list_output.txt | tr -s " " | cut -d " " -f2 | sort -u > cmssw_installed_${arch}.txt
cat cmssw_installed_${arch}.txt >> cmssw_installed_version_list.txt
echo "<ol class=CMSSW-list id=\"$arch\">"
cat cmssw_installed_${arch}.txt | while read line
do
echo " <li>"$line"</li>"
done
echo "</ol>"
echo "<p>"
echo "OK"
echo "</p>"
done
echo "<h2>Verify list of required CMSSW versions</h2>"
missing=0
failproject=0
failrunt=0
for cmsver in `cat $SAME_SENSOR_HOME/tests/cmssw_required_version_list.txt`
do
grep -x $cmsver cmssw_installed_version_list.txt >& /dev/null
result=$?
if [ $result != 0 ]
then
echo "<p><strong>"
echo "Required CMSSW version $cmsver not installed"
echo "</strong></p>"
missing=1
else
echo "<p>"
echo "Checking version $cmsver installation"
echo "</p>"
echo "<pre>"
# execute in a subprocess commands that require scram
$SAME_SENSOR_HOME/tests/TestCmsswVersion.sh $cmsver
TestResult=$?
if [ $TestResult != 0 ]
then
failproject=1
fi
fi
done
echo "<h2>Verify installed vs. published CMSSW versions</h2>"
# check that installed version list matches published tags
retrieve_published_list_fail=0
missing_arch_tag=0
missing_version_tag=0
missing_published_version=0
if [ $isEGEE == 1 ]
then
CE=`glite-brokerinfo getCE`
fi
if [ $isOSG == 1 ]
then
CE=`grep 'name.*jobmanager' .BrokerInfo|cut -d'"' -f2`
fi
ceHost=`echo $CE|cut -d':' -f1`
echo "<pre>"
echo "CE is: " $CE
echo "CE host is: " $ceHost
echo "</pre>"
PublishedTagsURL="http://cmsdoc.cern.ch/cms/LCG/SiteComm/published-tags/$ceHost"
echo "<p>Retrieve list of publshed tags from <a href=\"$PublishedTagsURL\">$PublishedTagsURL</a><br>"
./fetch-from-web $PublishedTagsURL publ_tag.txt
rc=$?
echo "rc = " $rc
[ $rc == $SAME_INFO ] && info=1
[ $rc == $SAME_WARNING ] && warning=1
echo "List of published tags<br><pre>"
cat publ_tag.txt
echo "</pre>"
len=`cat publ_tag.txt | wc --chars`
if [ $len -lt 10 ]
then
retrieve_published_list_fail=1
echo "<p><br><strong>"
echo "suspiciously small tag list returned by lcg-info, ignore it"
echo "</p></strong><pre>"
ls -lh publ_tag.txt
echo "</pre>"
fi
if [ $retrieve_published_list_fail == 0 ]
then
for arch in $archs_to_test
do
if ! (grep -q $arch publ_tag.txt) then
echo "<p>ERROR: $arch tag missing</p>"
missing_arch_tag=1
fi
done
TagCollectorURL="https://cmsdoc.cern.ch/swdev/CmsTC/cgi-bin/ReleasesXML"
echo "<p>Retrieve list of production version from <a href=\"$TagCollectorURL\">$TagCollectorURL</a> and ignore releases before 1_5_x<br>"
./fetch-from-web $TagCollectorURL ReleasesXML
rc=$?
echo "rc = " $rc
[ $rc == $SAME_INFO ] && info=1
[ $rc == $SAME_WARNING ] && warning=1
if [ $rc == $SAME_OK -o $rc == $SAME_INFO ] ; then
cat ReleasesXML | grep Announced|cut -d'"' -f 2|sort -u > production_releases.txt
# do not bother with old SL3-only versions (tags are not maintained)
grep -v "CMSSW_0\|CMSSW_1_[0-4]" production_releases.txt > newer_releases
mv newer_releases production_releases.txt
fi
echo "<h3>If a production releease is installed, it should be published</h3>"
for cmsver in `cat cmssw_installed_version_list.txt`
do
if ( grep -qx $cmsver production_releases.txt )
then
if ! (grep -qx $cmsver publ_tag.txt)
then
echo "<p>"
echo "WARNING: CMSSW version $cmsver installed but not published"
echo "</p>"
missing_version_tag=1
fi
fi
done
[ $missing_version_tag == 0 ] && echo "OK<p>"
echo "<h3>If a production release is published, it must be installed</h3>"
for cmsver in `grep CMSSW publ_tag.txt`
do
if ( grep -qx $cmsver production_releases.txt )
then
if ! (grep -qx $cmsver cmssw_installed_version_list.txt)
then
echo "<p>"
echo "ERROR: CMSSW version $cmsver published but not installed"
echo "<pre>"
set -x
cat publ_tag.txt
cat production_releases.txt
set +x
echo "</pre>"
echo "</p>"
missing_published_version=1
fi
fi
done
[ $missing_published_version == 0 ] && echo "OK<p>"
fi
# Print out installed Generator Gridpacks
# We assume that they are stored in $VO_CMS_SW_DIR/slc4_ia32_gcc345/generatorData
# SAM test result does not depend on it (at least for now).
echo "<h2>Installed Generator Gridpacks</h2>"
if [ ! -d $VO_CMS_SW_DIR/slc4_ia32_gcc345/generatorData ]; then
echo "<p>"
echo " WARNING: No generator data installed!"
echo "</p>"
else
cd $VO_CMS_SW_DIR/slc4_ia32_gcc345/generatorData
echo "Following generator data are installed: "
echo "<pre>"
find . -type d -mindepth 4 -maxdepth 4 -print
echo "</pre>"
# Bad permssions - enable later...
# echo "Files or directories with bad permissions"
# find . -type d ! -perm -o+x -print
# find . -type f ! -perm -o+r -print
fi
echo '<h1><a name="summary"> Test Summary</a></h1><p>'
if [ $cantwrite == 1 ]
then
echo "<p>WARNING: Cannot write to CMSSW installation area</p>"
warning=1
fi
if [ $missing == 1 ]
then
echo "<p>"
echo "WARNING: Some of the required CMSSW versions not found"
echo "</p>"
warning=1
fi
if [ $retrieve_published_list_fail == 1 ]
then
echo "<p>WARNING: Fail to retrieve list of published tags from cmsdoc</p>"
info=1
fi
if [ $missing_version_tag == 1 ]
then
echo "<p>"
echo "WARNING: Some CMS VO Tags are missing"
echo "</p>"
warning=1
fi
if [ $missing_arch_tag == 1 ]
then
echo "<p>"
echo "ERROR: Needed CMS VO architecture TAG not published"
echo "</p>"
error=1
fi
if [ $missing_published_version == 1 ]
then
echo "<p>"
echo "ERROR: Some CMSSW version published but not present"
echo "</p>"
error=1
fi
if [[ $failproject == 1 || $failrunt == 1 ]]
then
echo "<p>"
echo "ERROR: Some of the required CMSSW versions not working"
echo "</p>"
error=1
fi
if [[ $error == 1 ]]
then
echo "<p><strong> ERROR </strong></p>"
exit $SAME_ERROR
fi
if [[ $warning == 1 ]]
then
echo "<p><strong> WARNING </strong></p>"
exit $SAME_WARNING
fi
if [ $info == 1 ]
then
echo "<p><strong> INFO </strong></p>"
exit $SAME_INFO
fi
echo "<p><strong> OK </strong></p>"
let nl=nl+1
read line
done < possible-input-files
inputFile=$line
echo inputFile: $inputFile
#
# Create configuration file for cmsRun followin example in
# http://cmslxr.fnal.gov/lxr/source/FWCore/Integration/test/testRunMergeMERGE_cfg.py#006
#
/bin/cat > analysis_test.py <<EOI
import FWCore.ParameterSet.Config as cms
process = cms.Process("CopyJob")
process.load("FWCore.MessageService.MessageLogger_cfi")
process.MessageLogger.cerr.FwkReport.reportEvery = 100
process.source = cms.Source("PoolSource",
fileNames = cms.untracked.vstring("${inputFile}")
)
process.out = cms.OutputModule("PoolOutputModule",
fileName = cms.untracked.string('myout.root'),
outputCommands = cms.untracked.vstring('drop *')
)
process.e = cms.EndPath(process.out)
EOI
echo ""
echo "================= cmsRun configuration file: analysis_test.py ====================="
echo ""
cat analysis_test.py
echo ""
start=`date +%s`
#
# Run cmsRun
#
printf "local file Analysis test\n"
printf "START TIME: `date` --> running cmsRun -j fjr.xml -p analysis_test.py\n"
cmsRun -j fjr.xml -p analysis_test.py
srun=$?
echo ""
echo "================= trimmed framework job report file: fjr.xml ====================="
echo ""
cat fjr.xml | grep -v Branch | grep -v LumiSection | grep -v RunNumber | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'
echo "=================================================================================="
echo ""
printf "`date` --> DONE\n"
stop=`date +%s`
printf "ELAPSED TIME: $[ $stop - $start ] seconds\n"
#
# Check FJR for errors
#
grep -q "FrameworkError " fjr.xml
sfjr=$?
if [ $sfjr -eq 0 ]
then
printf "AnalysisTest.sh: FrameworkError found in Framework Job Report\n"
grep "FrameworkError " fjr.xml | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'
# if possible extract error
which xml_grep > /dev/null 2>1 && xml_grep FrameworkError fjr.xml
fi
#
# Check cmsRun exit status
#
if [ $srun -ne 0 -o $sfjr -eq 0 ]
then
printf "AnalysisTest.sh: Error $srun from cmsRun\n"
exit $SAME_ERROR
fi
#
# Exit
#
printf "OK\n"
exit $SAME_OK
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment