CE-cms-squid 7.9 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
#!/bin/bash

3
4
shopt -s expand_aliases

5
SCRIPT_REVISION="2020-03-20, Dave Dykstra and Edita Kizinevic"
6
7
8
9
10
11
12
13
14
15
16
17
18
19

# Script does not use backupproxyurl if it is SAM test, but script uses backupproxyurl if it is pilot
includebackups=false
while getopts ":b" opt; do
  case ${opt} in
    b)
      includebackups=true
      ;;
    \?)
      echo "Invalid Option: -$OPTARG" 1>&2
      exit 1
      ;;
    esac
done
20

Andrea Sciaba's avatar
Andrea Sciaba committed
21
# Source the CMS environment
Andrea Sciaba's avatar
Andrea Sciaba committed
22
23
if [ -n "$OSG_GRID" ] ; then
    [ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
24
25
    if [ -d $OSG_APP/cmssoft/cms ] ; then
        SW_DIR=$OSG_APP/cmssoft/cms
Andrea Sciaba's avatar
Andrea Sciaba committed
26
    elif [ -d $CVMFS/cms.cern.ch ] ; then
27
        SW_DIR=$CVMFS/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
28
    elif [ -d /cvmfs/cms.cern.ch ] ; then
29
        SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
30
    else
31
32
33
        echo "ERROR: Cannot find CMS software in OSG node"
        echo "summary: SW_DIR_UNDEF"
        exit $SAME_ERROR
Andrea Sciaba's avatar
Andrea Sciaba committed
34
35
    fi
elif [ -n "$VO_CMS_SW_DIR" ] ; then
Andrea Sciaba's avatar
Andrea Sciaba committed
36
37
    SW_DIR=$VO_CMS_SW_DIR
else
38
    SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
39
fi
Andrea Sciaba's avatar
Andrea Sciaba committed
40
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
41
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
Andrea Sciaba's avatar
Andrea Sciaba committed
42
result=$?
43
44
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
45
46
47
48
49
if [ $result != 0 -o $result2 == 0 ] ; then
    cat $tmpfile
    rm -f $tmpfile
    echo "ERROR: CMS software initialisation script cmsset_default.sh failed"
    echo "summary: NO_SETUP_SCRIPT"
50
51
    exit $SAME_ERROR
fi
52
rm -f $tmpfile
Andrea Sciaba's avatar
Andrea Sciaba committed
53

54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# Print out node name
node=`hostname`
echo "node: $node"
echo

# Check that environmental variable SAME_OK is set
if [ -z "$SAME_OK" ] ; then
    echo "ERROR: SAME_OK not defined"
    exit 1
fi

# Check that environmental variable SAME_ERROR is set
if [ -z "$SAME_ERROR" ] ; then
    echo "ERROR: SAME_ERROR not defined"
    exit 1
fi

# Check that environmental variable SAME_WARNING is set
if [ -z "$SAME_WARNING" ] ; then
    echo "ERROR: SAME_WARNING not defined"
    exit 1
fi

# Check that environmental variable CMS_PATH is set
if [ -z "$CMS_PATH" ] ; then
    echo "ERROR: CMS_PATH not defined"
    exit 1
fi

# Print script version information
echo "script version: $SCRIPT_REVISION"
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
87
88
89
90
91
92
93
94
# Get site name
site_local_config_file=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
if [ ! -f "$site_local_config_file" ] ; then
    echo "ERROR: file ${site_local_config_file} does not exist"
    exit $SAME_ERROR
fi
site=`grep -oP '(?<=site name=").*(?=")' $site_local_config_file`

95
96
97
98
99
100
101
# Create Working Directory
mkdir squid
cd squid
current=`pwd`
echo "Current directory is: ${current}"
echo

102
# Script checks failover if it is SAM test, but script does not check failover if it is pilot
Edita Kizinevic's avatar
Edita Kizinevic committed
103
failover=false
104
105
106
107
108
109
110
111
112
113
114
115
if ! $includebackups; then
    if [ ! -z "$site" ] ; then
        site_summary_file=site_summary.txt
        url=http://wlcg-squid-monitor.cern.ch/failover/failoverCMS/$site_summary_file
        wget -q -O $site_summary_file $url
        if [ -f "$site_summary_file" ] ; then
            result="`grep -P "$site[, \t]" $site_summary_file`"
            if [ ! -z "$result"  ] ; then
                failover=true
            fi
        else
            echo "$site_summary_file file is not retrieved from $url"
Edita Kizinevic's avatar
Edita Kizinevic committed
116
117
118
119
        fi
    fi
fi

120
121
122
123
124
125
126
127
# Set up CMSSW
echo "Set up CMSSW ... starting"
CMSREL="`scram l | grep -B 1 "cms.cern.ch" | grep " CMSSW" | tail -1 | awk '{print $2}'`"
scram p $CMSREL
if [ $? -ne 0 ] ; then
    echo "ERROR: $CMSREL not available"
    exit $SAME_WARNING
fi
128
echo "Set up of $CMSREL completed"
129
130
131
132
133
134
echo
cd $CMSREL/src
cmsenv

# Function to check proxies
# $1 - list of proxies
135
# $2 - $includebackups
136
check_proxies() {
137
138
139
140
141
    if $2; then
        backupproxyurl="*(backupproxyurl=*"
    else
        backupproxyurl="NONE"
    fi
142
143
144
    for PART in $1; do
        proxy=`echo $PART | sed 's/^.*(/(/'`
        case $proxy in
145
            *"(proxyurl="* | $backupproxyurl)
146
                found_proxy=true
147
148
                hostname=${proxy#*//}
                hostname=${hostname%:*}
149
150
                # The frontier client does not work with IPv6 addresses in URLs for now
                IP_addresses=`getent ahostsv4 $hostname | awk '/STREAM/{print $1}'`
151
                if [ -z "$IP_addresses" ]; then
152
                    IP_addresses=$hostname
153
                elif getent ahostsv6 $hostname | awk '/STREAM/{print $1}' | grep -qv ^::ffff:; then
154
                    IP_addresses="$IP_addresses $hostname"
155
156
157
158
159
160
161
                fi
                for IP_address in $IP_addresses; do
                    proxy_IP=`echo $proxy | sed "s/$hostname/$IP_address/g"`
                    echo "Query $proxy started:" `date`
                    echo 'select 1 from dual' | FRONTIER_SERVER="$proxy_IP(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $output
                    if grep -q " 1 NUMBER" $output ; then
                        echo "$proxy_IP is OK"
162
                        ever_succeeded=true
163
164
165
166
167
168
169
170
171
172
                    else
                        echo "$proxy_IP is FAILED:"
                        cat $output
                        ever_failed=true
                    fi
                    echo "Query $proxy ended:" `date`
                    echo
                done
                ;;
            *"(proxyconfigurl="*)
173
174
                if [ "$found_proxyconfig" == false ]; then
                    proxy_config_url=$proxy
175
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl started:" `date`
176
                    echo
177
                    echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=debug FRONTIER_SERVER="$proxy(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $debug_output 2>&1
178
                    if grep -q "FindProxyForURL" $debug_output ; then
179
180
                        found_proxyconfig=true
                        PROXYLIST="`sed -n '/returned "PROXY/{s/.*returned "//;s/PROXY /(proxyurl=/g;s/;/)/g;s/"/)/;p;q}' $debug_output`"
181
182
183
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbpfrontier.cern.ch/(backupproxyurl=http:\/\/cmsbpfrontier.cern.ch/g"`
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbproxy.fnal.gov/(backupproxyurl=http:\/\/cmsbproxy.fnal.gov/g"`
                        check_proxies "$PROXYLIST" $2
184
                    fi
185
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl ended:" `date`
186
187
                    echo
                fi
188
189
190
191
192
193
194
195
196
197
198
199
                ;;
        esac
    done
}

# Print out cmsGetFnConnect frontier://FrontierProd
FNCONNECT="`cmsGetFnConnect frontier://FrontierProd | sed 's/)/) /g'`"
echo "Contents of cmsGetFnConnect frontier://FrontierProd are:"
echo "$FNCONNECT" | tr " " "\n"
echo

# Check proxies
200
debug_output="debug_output.txt"
201
202
output="output.txt"
ever_failed=false
203
ever_succeeded=false
204
found_proxyconfig=false
205
found_proxy=false
206
check_proxies "$FNCONNECT" $includebackups
207
208

#Check test results
Edita Kizinevic's avatar
Edita Kizinevic committed
209
if [ "$failover" == true ]; then
Edita Kizinevic's avatar
Edita Kizinevic committed
210
    message=$'many database queries from the site have connected directly to the Frontier servers or backup proxies, with a high rate of queries not going through the local squid(s):\n'$result
Edita Kizinevic's avatar
Edita Kizinevic committed
211
212
213
214
215
216
217
218
219
220
221
222
223
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK, however $message"
        exit $SAME_WARNING
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED and $message"
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found and $message"
        exit $SAME_WARNING
    else
        echo "Proxy test is FAILED and $message"
        exit $SAME_ERROR
    fi
224
else
Edita Kizinevic's avatar
Edita Kizinevic committed
225
226
227
228
229
230
231
232
233
234
235
236
237
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK."
        exit $SAME_OK
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED."
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found."
        exit $SAME_OK
    else
        echo "Proxy test is FAILED."
        exit $SAME_ERROR
    fi
238
fi