CE-cms-squid 9.81 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
#!/bin/bash

3
4
shopt -s expand_aliases

5
SCRIPT_REVISION="2020-11-26"
6
7
MAINTAINERS="Dave Dykstra and Edita Kizinevic"
SCRIPT_NAME=`basename "$0"`
8

9
10
11
# Script has some differences when it runs as SAM or pilot
pilot=false
while getopts ":p" opt; do
12
  case ${opt} in
13
14
    p)
      pilot=true
15
16
17
18
19
20
21
      ;;
    \?)
      echo "Invalid Option: -$OPTARG" 1>&2
      exit 1
      ;;
    esac
done
22

Andrea Sciaba's avatar
Andrea Sciaba committed
23
# Source the CMS environment
Andrea Sciaba's avatar
Andrea Sciaba committed
24
25
if [ -n "$OSG_GRID" ] ; then
    [ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
26
27
    if [ -d $OSG_APP/cmssoft/cms ] ; then
        SW_DIR=$OSG_APP/cmssoft/cms
Andrea Sciaba's avatar
Andrea Sciaba committed
28
    elif [ -d $CVMFS/cms.cern.ch ] ; then
29
        SW_DIR=$CVMFS/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
30
    elif [ -d /cvmfs/cms.cern.ch ] ; then
31
        SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
32
    else
33
34
35
        echo "ERROR: Cannot find CMS software in OSG node"
        echo "summary: SW_DIR_UNDEF"
        exit $SAME_ERROR
Andrea Sciaba's avatar
Andrea Sciaba committed
36
37
    fi
elif [ -n "$VO_CMS_SW_DIR" ] ; then
Andrea Sciaba's avatar
Andrea Sciaba committed
38
39
    SW_DIR=$VO_CMS_SW_DIR
else
40
    SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
41
fi
Andrea Sciaba's avatar
Andrea Sciaba committed
42
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
43
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
Andrea Sciaba's avatar
Andrea Sciaba committed
44
result=$?
45
46
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
47
48
49
50
51
if [ $result != 0 -o $result2 == 0 ] ; then
    cat $tmpfile
    rm -f $tmpfile
    echo "ERROR: CMS software initialisation script cmsset_default.sh failed"
    echo "summary: NO_SETUP_SCRIPT"
52
53
    exit $SAME_ERROR
fi
54
rm -f $tmpfile
Andrea Sciaba's avatar
Andrea Sciaba committed
55

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# Print out node name
node=`hostname`
echo "node: $node"
echo

# Check that environmental variable SAME_OK is set
if [ -z "$SAME_OK" ] ; then
    echo "ERROR: SAME_OK not defined"
    exit 1
fi

# Check that environmental variable SAME_ERROR is set
if [ -z "$SAME_ERROR" ] ; then
    echo "ERROR: SAME_ERROR not defined"
    exit 1
fi

# Check that environmental variable SAME_WARNING is set
if [ -z "$SAME_WARNING" ] ; then
    echo "ERROR: SAME_WARNING not defined"
    exit 1
fi

# Check that environmental variable CMS_PATH is set
if [ -z "$CMS_PATH" ] ; then
    echo "ERROR: CMS_PATH not defined"
    exit 1
fi

# Print script version information
86
echo "script version: $SCRIPT_REVISION, $MAINTAINERS"
87
88
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
89
90
91
92
93
94
95
96
# Get site name
site_local_config_file=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
if [ ! -f "$site_local_config_file" ] ; then
    echo "ERROR: file ${site_local_config_file} does not exist"
    exit $SAME_ERROR
fi
site=`grep -oP '(?<=site name=").*(?=")' $site_local_config_file`

97
98
99
100
101
102
103
# Create Working Directory
mkdir squid
cd squid
current=`pwd`
echo "Current directory is: ${current}"
echo

104
# Script checks failover if it is SAM test, but script does not check failover if it is pilot
Edita Kizinevic's avatar
Edita Kizinevic committed
105
failover=false
106
if ! $pilot; then
107
108
109
110
111
112
113
114
115
116
117
    if [ ! -z "$site" ] ; then
        site_summary_file=site_summary.txt
        url=http://wlcg-squid-monitor.cern.ch/failover/failoverCMS/$site_summary_file
        wget -q -O $site_summary_file $url
        if [ -f "$site_summary_file" ] ; then
            result="`grep -P "$site[, \t]" $site_summary_file`"
            if [ ! -z "$result"  ] ; then
                failover=true
            fi
        else
            echo "$site_summary_file file is not retrieved from $url"
Edita Kizinevic's avatar
Edita Kizinevic committed
118
119
120
121
        fi
    fi
fi

122
123
124
125
126
127
128
129
# Set up CMSSW
echo "Set up CMSSW ... starting"
CMSREL="`scram l | grep -B 1 "cms.cern.ch" | grep " CMSSW" | tail -1 | awk '{print $2}'`"
scram p $CMSREL
if [ $? -ne 0 ] ; then
    echo "ERROR: $CMSREL not available"
    exit $SAME_WARNING
fi
130
echo "Set up of $CMSREL completed"
131
132
133
134
echo
cd $CMSREL/src
cmsenv

135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
# Use IPv6 if frontier client version is 2.9 or greater
frontier_client=`echo $FRONTIER_CLIENT | sed 's/.*frontier_client\///'`
frontier_major_version=${frontier_client%%.*}
frontier_minor_version=${frontier_client#*.}
frontier_minor_version=${frontier_minor_version%%.*}
use_IPv6=false
re='^[0-9]+$'
if [[ $frontier_major_version =~ $re ]] && [[ $frontier_minor_version =~ $re ]]; then
    if [ "$frontier_major_version" -gt "2" ]; then
        use_IPv6=true
    elif [ "$frontier_major_version" -eq "2" ] && [ "$frontier_minor_version" -ge "9" ]; then
        use_IPv6=true
    fi
fi

150
151
# Function to check proxies
# $1 - list of proxies
152
# $2 - $pilot
153
# $3 - $use_IPv6
154
check_proxies() {
155
    # Script does not use backupproxyurl if it is SAM test, but script uses backupproxyurl if it is pilot
156
157
    if $2; then
        backupproxyurl="*(backupproxyurl=*"
158
        FRONTIER_ID="${SCRIPT_NAME}_pilot_${SCRIPT_REVISION}"
159
160
    else
        backupproxyurl="NONE"
161
        FRONTIER_ID="${SCRIPT_NAME}_SAM_${SCRIPT_REVISION}"
162
    fi
163
164
165
    for PART in $1; do
        proxy=`echo $PART | sed 's/^.*(/(/'`
        case $proxy in
166
            *"(proxyurl="* | $backupproxyurl)
167
                found_proxy=true
168
169
                hostname=${proxy#*//}
                hostname=${hostname%:*}
170
171
172
173
                IPv4_addresses=`getent ahostsv4 $hostname | awk '/STREAM/{print $1}'`
                IPv6_addresses=`getent ahostsv6 $hostname | awk '/STREAM/{print $1}' | grep -v ^::ffff:`
                if $3; then
                  if [ -z "$IPv4_addresses" ] && [ -z "$IPv6_addresses" ]; then
174
                    IP_addresses=$hostname
175
176
177
178
179
180
181
182
183
184
185
186
187
                  else
                    IP_addresses=$IPv4_addresses
                    for IPv6_address in $IPv6_addresses; do
                        IP_addresses="$IP_addresses [$IPv6_address]"
                    done
                  fi
                else
                  IP_addresses=$IPv4_addresses
                  if [ -z "$IP_addresses" ]; then
                      IP_addresses=$hostname
                  elif [ ! -z "$IPv6_addresses" ]; then
                      IP_addresses="$IP_addresses $hostname"
                  fi
188
189
                fi
                for IP_address in $IP_addresses; do
190
191
192
                    if ($2 && ! $ever_succeeded) || ! $2; then
                        proxy_IP=`echo $proxy | sed "s/$hostname/$IP_address/g"`
                        echo "Query $proxy started:" `date`
Edita Kizinevic's avatar
Edita Kizinevic committed
193
194
195
196
197
198
199
                        echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=warning FRONTIER_SERVER="$proxy_IP
                            (serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier1.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier2.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier3.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier4.cern.ch:8000/FrontierProd)
                            (failovertoserver=no)" FRONTIER_ID=$FRONTIER_ID fn-req > $output
200
201
202
203
204
205
206
207
208
209
                        if grep -q " 1 NUMBER" $output ; then
                            echo "$proxy_IP is OK"
                            ever_succeeded=true
                        else
                            echo "$proxy_IP is FAILED:"
                            cat $output
                            ever_failed=true
                        fi
                        echo "Query $proxy ended:" `date`
                        echo
210
                    else
211
                        return
212
213
214
215
                    fi
                done
                ;;
            *"(proxyconfigurl="*)
216
217
                if [ "$found_proxyconfig" == false ]; then
                    proxy_config_url=$proxy
218
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl started:" `date`
219
                    echo
220
                    echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=debug FRONTIER_SERVER="$proxy(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" FRONTIER_ID=$FRONTIER_ID fn-req > $debug_output 2>&1
221
                    if grep -q "FindProxyForURL" $debug_output ; then
222
223
                        found_proxyconfig=true
                        PROXYLIST="`sed -n '/returned "PROXY/{s/.*returned "//;s/PROXY /(proxyurl=/g;s/;/)/g;s/"/)/;p;q}' $debug_output`"
224
225
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbpfrontier.cern.ch/(backupproxyurl=http:\/\/cmsbpfrontier.cern.ch/g"`
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbproxy.fnal.gov/(backupproxyurl=http:\/\/cmsbproxy.fnal.gov/g"`
226
                        check_proxies "$PROXYLIST" $2 $3
227
                    fi
228
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl ended:" `date`
229
230
                    echo
                fi
231
232
233
234
235
236
237
238
239
240
241
242
                ;;
        esac
    done
}

# Print out cmsGetFnConnect frontier://FrontierProd
FNCONNECT="`cmsGetFnConnect frontier://FrontierProd | sed 's/)/) /g'`"
echo "Contents of cmsGetFnConnect frontier://FrontierProd are:"
echo "$FNCONNECT" | tr " " "\n"
echo

# Check proxies
243
debug_output="debug_output.txt"
244
245
output="output.txt"
ever_failed=false
246
ever_succeeded=false
247
found_proxyconfig=false
248
found_proxy=false
249
check_proxies "$FNCONNECT" $pilot $use_IPv6
250
251

#Check test results
Edita Kizinevic's avatar
Edita Kizinevic committed
252
if [ "$failover" == true ]; then
Edita Kizinevic's avatar
Edita Kizinevic committed
253
    message=$'many database queries from the site have connected directly to the Frontier servers or backup proxies, with a high rate of queries not going through the local squid(s):\n'$result
Edita Kizinevic's avatar
Edita Kizinevic committed
254
255
256
257
258
259
260
261
262
263
264
265
266
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK, however $message"
        exit $SAME_WARNING
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED and $message"
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found and $message"
        exit $SAME_WARNING
    else
        echo "Proxy test is FAILED and $message"
        exit $SAME_ERROR
    fi
267
else
Edita Kizinevic's avatar
Edita Kizinevic committed
268
269
270
271
272
273
274
275
276
277
278
279
280
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK."
        exit $SAME_OK
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED."
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found."
        exit $SAME_OK
    else
        echo "Proxy test is FAILED."
        exit $SAME_ERROR
    fi
281
fi