CE-cms-squid 10.2 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
#!/bin/bash

3
4
shopt -s expand_aliases

5
SCRIPT_REVISION="2020-12-03"
6
7
MAINTAINERS="Dave Dykstra and Edita Kizinevic"
SCRIPT_NAME=`basename "$0"`
8

9
10
11
# Script has some differences when it runs as SAM or pilot
pilot=false
while getopts ":p" opt; do
12
  case ${opt} in
13
14
    p)
      pilot=true
15
16
17
18
19
20
21
      ;;
    \?)
      echo "Invalid Option: -$OPTARG" 1>&2
      exit 1
      ;;
    esac
done
22

Andrea Sciaba's avatar
Andrea Sciaba committed
23
# Source the CMS environment
Andrea Sciaba's avatar
Andrea Sciaba committed
24
25
if [ -n "$OSG_GRID" ] ; then
    [ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
26
27
    if [ -d $OSG_APP/cmssoft/cms ] ; then
        SW_DIR=$OSG_APP/cmssoft/cms
Andrea Sciaba's avatar
Andrea Sciaba committed
28
    elif [ -d $CVMFS/cms.cern.ch ] ; then
29
        SW_DIR=$CVMFS/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
30
    elif [ -d /cvmfs/cms.cern.ch ] ; then
31
        SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
32
    else
33
34
35
        echo "ERROR: Cannot find CMS software in OSG node"
        echo "summary: SW_DIR_UNDEF"
        exit $SAME_ERROR
Andrea Sciaba's avatar
Andrea Sciaba committed
36
37
    fi
elif [ -n "$VO_CMS_SW_DIR" ] ; then
Andrea Sciaba's avatar
Andrea Sciaba committed
38
39
    SW_DIR=$VO_CMS_SW_DIR
else
40
    SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
41
fi
Andrea Sciaba's avatar
Andrea Sciaba committed
42
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
43
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
Andrea Sciaba's avatar
Andrea Sciaba committed
44
result=$?
45
46
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
47
48
49
50
51
if [ $result != 0 -o $result2 == 0 ] ; then
    cat $tmpfile
    rm -f $tmpfile
    echo "ERROR: CMS software initialisation script cmsset_default.sh failed"
    echo "summary: NO_SETUP_SCRIPT"
52
53
    exit $SAME_ERROR
fi
54
rm -f $tmpfile
Andrea Sciaba's avatar
Andrea Sciaba committed
55

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# Print out node name
node=`hostname`
echo "node: $node"
echo

# Check that environmental variable SAME_OK is set
if [ -z "$SAME_OK" ] ; then
    echo "ERROR: SAME_OK not defined"
    exit 1
fi

# Check that environmental variable SAME_ERROR is set
if [ -z "$SAME_ERROR" ] ; then
    echo "ERROR: SAME_ERROR not defined"
    exit 1
fi

# Check that environmental variable SAME_WARNING is set
if [ -z "$SAME_WARNING" ] ; then
    echo "ERROR: SAME_WARNING not defined"
    exit 1
fi

# Check that environmental variable CMS_PATH is set
if [ -z "$CMS_PATH" ] ; then
    echo "ERROR: CMS_PATH not defined"
    exit 1
fi

# Print script version information
86
echo "script version: $SCRIPT_REVISION, $MAINTAINERS"
87
88
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
89
90
91
92
93
94
95
96
# Get site name
site_local_config_file=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
if [ ! -f "$site_local_config_file" ] ; then
    echo "ERROR: file ${site_local_config_file} does not exist"
    exit $SAME_ERROR
fi
site=`grep -oP '(?<=site name=").*(?=")' $site_local_config_file`

97
98
99
100
101
102
103
# Create Working Directory
mkdir squid
cd squid
current=`pwd`
echo "Current directory is: ${current}"
echo

104
# Script checks failover if it is SAM test, but script does not check failover if it is pilot
Edita Kizinevic's avatar
Edita Kizinevic committed
105
failover=false
106
if ! $pilot; then
107
108
109
110
111
112
113
114
115
116
117
    if [ ! -z "$site" ] ; then
        site_summary_file=site_summary.txt
        url=http://wlcg-squid-monitor.cern.ch/failover/failoverCMS/$site_summary_file
        wget -q -O $site_summary_file $url
        if [ -f "$site_summary_file" ] ; then
            result="`grep -P "$site[, \t]" $site_summary_file`"
            if [ ! -z "$result"  ] ; then
                failover=true
            fi
        else
            echo "$site_summary_file file is not retrieved from $url"
Edita Kizinevic's avatar
Edita Kizinevic committed
118
119
120
121
        fi
    fi
fi

122
123
124
125
126
127
128
129
# Set up CMSSW
echo "Set up CMSSW ... starting"
CMSREL="`scram l | grep -B 1 "cms.cern.ch" | grep " CMSSW" | tail -1 | awk '{print $2}'`"
scram p $CMSREL
if [ $? -ne 0 ] ; then
    echo "ERROR: $CMSREL not available"
    exit $SAME_WARNING
fi
130
echo "Set up of $CMSREL completed"
131
132
133
134
echo
cd $CMSREL/src
cmsenv

Edita Kizinevic's avatar
Edita Kizinevic committed
135
# Use IPv6 addresses if frontier client version is 2.9 or greater
136
137
138
139
frontier_client=`echo $FRONTIER_CLIENT | sed 's/.*frontier_client\///'`
frontier_major_version=${frontier_client%%.*}
frontier_minor_version=${frontier_client#*.}
frontier_minor_version=${frontier_minor_version%%.*}
Edita Kizinevic's avatar
Edita Kizinevic committed
140
use_IPv6_addresses=false
141
142
143
re='^[0-9]+$'
if [[ $frontier_major_version =~ $re ]] && [[ $frontier_minor_version =~ $re ]]; then
    if [ "$frontier_major_version" -gt "2" ]; then
Edita Kizinevic's avatar
Edita Kizinevic committed
144
        use_IPv6_addresses=true
145
    elif [ "$frontier_major_version" -eq "2" ] && [ "$frontier_minor_version" -ge "9" ]; then
Edita Kizinevic's avatar
Edita Kizinevic committed
146
        use_IPv6_addresses=true
147
148
149
    fi
fi

150
151
# Function to check proxies
# $1 - list of proxies
152
# $2 - $pilot
Edita Kizinevic's avatar
Edita Kizinevic committed
153
# $3 - $use_IPv6_addresses
154
check_proxies() {
155
    # Script does not use backupproxyurl if it is SAM test, but script uses backupproxyurl if it is pilot
156
157
    if $2; then
        backupproxyurl="*(backupproxyurl=*"
158
        FRONTIER_ID="${SCRIPT_NAME}_pilot_${SCRIPT_REVISION}"
159
160
    else
        backupproxyurl="NONE"
161
        FRONTIER_ID="${SCRIPT_NAME}_SAM_${SCRIPT_REVISION}"
162
    fi
163
164
165
    for PART in $1; do
        proxy=`echo $PART | sed 's/^.*(/(/'`
        case $proxy in
166
            *"(proxyurl="* | $backupproxyurl)
167
                found_proxy=true
168
169
                hostname=${proxy#*//}
                hostname=${hostname%:*}
170
171
                IPv4_addresses=`getent ahostsv4 $hostname | awk '/STREAM/{print $1}'`
                IPv6_addresses=`getent ahostsv6 $hostname | awk '/STREAM/{print $1}' | grep -v ^::ffff:`
172
173
174
175
                IPv4_and_IPv6_addresses_exist=false
                if [ ! -z "$IPv4_addresses" ] && [ ! -z "$IPv6_addresses" ]; then
                    IPv4_and_IPv6_addresses_exist=true
                fi
176
177
                if $3; then
                  if [ -z "$IPv4_addresses" ] && [ -z "$IPv6_addresses" ]; then
178
                    IP_addresses=$hostname
179
180
181
182
183
184
185
186
187
188
189
190
191
                  else
                    IP_addresses=$IPv4_addresses
                    for IPv6_address in $IPv6_addresses; do
                        IP_addresses="$IP_addresses [$IPv6_address]"
                    done
                  fi
                else
                  IP_addresses=$IPv4_addresses
                  if [ -z "$IP_addresses" ]; then
                      IP_addresses=$hostname
                  elif [ ! -z "$IPv6_addresses" ]; then
                      IP_addresses="$IP_addresses $hostname"
                  fi
192
193
                fi
                for IP_address in $IP_addresses; do
194
195
196
                    if ($2 && ! $ever_succeeded) || ! $2; then
                        proxy_IP=`echo $proxy | sed "s/$hostname/$IP_address/g"`
                        echo "Query $proxy started:" `date`
Edita Kizinevic's avatar
Edita Kizinevic committed
197
198
199
200
201
202
203
                        echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=warning FRONTIER_SERVER="$proxy_IP
                            (serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier1.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier2.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier3.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier4.cern.ch:8000/FrontierProd)
                            (failovertoserver=no)" FRONTIER_ID=$FRONTIER_ID fn-req > $output
204
205
206
207
208
209
                        if grep -q " 1 NUMBER" $output ; then
                            echo "$proxy_IP is OK"
                            ever_succeeded=true
                        else
                            echo "$proxy_IP is FAILED:"
                            cat $output
210
211
212
213
214
215
216
217
218
                            if $IPv4_and_IPv6_addresses_exist; then
                                if [[ $IP_address =~ .*:.* ]]; then
                                    IPv6_ever_failed=true
                                else
                                    ever_failed=true
                                fi
                            else
                                ever_failed=true
                            fi
219
220
221
                        fi
                        echo "Query $proxy ended:" `date`
                        echo
222
                    else
223
                        return
224
225
226
227
                    fi
                done
                ;;
            *"(proxyconfigurl="*)
228
229
                if [ "$found_proxyconfig" == false ]; then
                    proxy_config_url=$proxy
230
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl started:" `date`
231
                    echo
232
                    echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=debug FRONTIER_SERVER="$proxy(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" FRONTIER_ID=$FRONTIER_ID fn-req > $debug_output 2>&1
233
                    if grep -q "FindProxyForURL" $debug_output ; then
234
235
                        found_proxyconfig=true
                        PROXYLIST="`sed -n '/returned "PROXY/{s/.*returned "//;s/PROXY /(proxyurl=/g;s/;/)/g;s/"/)/;p;q}' $debug_output`"
236
237
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbpfrontier.cern.ch/(backupproxyurl=http:\/\/cmsbpfrontier.cern.ch/g"`
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbproxy.fnal.gov/(backupproxyurl=http:\/\/cmsbproxy.fnal.gov/g"`
238
                        check_proxies "$PROXYLIST" $2 $3
239
                    fi
240
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl ended:" `date`
241
242
                    echo
                fi
243
244
245
246
247
248
249
250
251
252
253
254
                ;;
        esac
    done
}

# Print out cmsGetFnConnect frontier://FrontierProd
FNCONNECT="`cmsGetFnConnect frontier://FrontierProd | sed 's/)/) /g'`"
echo "Contents of cmsGetFnConnect frontier://FrontierProd are:"
echo "$FNCONNECT" | tr " " "\n"
echo

# Check proxies
255
debug_output="debug_output.txt"
256
257
output="output.txt"
ever_failed=false
258
ever_succeeded=false
259
IPv6_ever_failed=false
260
found_proxyconfig=false
261
found_proxy=false
Edita Kizinevic's avatar
Edita Kizinevic committed
262
check_proxies "$FNCONNECT" $pilot $use_IPv6_addresses
263
264

#Check test results
265
266
267
268
269
if [ "$IPv6_ever_failed" == false ]; then
    IPv6_message=""
else
    IPv6_message=" for IPv4 addresses, although at least one of proxies is FAILED for IPv6 addresses"
fi
Edita Kizinevic's avatar
Edita Kizinevic committed
270
if [ "$failover" == true ]; then
271
272
    failover_message=$' and many database queries from the site have connected directly to the Frontier servers or backup proxies, with a high rate of queries not going through the local squid(s):\n'$result
    exit_code=$SAME_WARNING
273
else
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
    failover_message=""
    exit_code=$SAME_OK
fi
if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
    echo "Proxy test is OK${IPv6_message}${failover_message}"
    exit $exit_code
elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
    echo "At least one of proxies is FAILED${failover_message}"
    exit $SAME_WARNING
elif [ "$found_proxy" == false ]; then
    echo "No proxies are found${failover_message}"
    exit $exit_code
else
    echo "Proxy test is FAILED${failover_message}"
    exit $SAME_ERROR
289
fi