CE-cms-squid 8.75 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
#!/bin/bash

3
4
shopt -s expand_aliases

5
SCRIPT_REVISION="2020-07-17"
6
7
MAINTAINERS="Dave Dykstra and Edita Kizinevic"
SCRIPT_NAME=`basename "$0"`
8

9
10
11
# Script has some differences when it runs as SAM or pilot
pilot=false
while getopts ":p" opt; do
12
  case ${opt} in
13
14
    p)
      pilot=true
15
16
17
18
19
20
21
      ;;
    \?)
      echo "Invalid Option: -$OPTARG" 1>&2
      exit 1
      ;;
    esac
done
22

Andrea Sciaba's avatar
Andrea Sciaba committed
23
# Source the CMS environment
Andrea Sciaba's avatar
Andrea Sciaba committed
24
25
if [ -n "$OSG_GRID" ] ; then
    [ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
26
27
    if [ -d $OSG_APP/cmssoft/cms ] ; then
        SW_DIR=$OSG_APP/cmssoft/cms
Andrea Sciaba's avatar
Andrea Sciaba committed
28
    elif [ -d $CVMFS/cms.cern.ch ] ; then
29
        SW_DIR=$CVMFS/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
30
    elif [ -d /cvmfs/cms.cern.ch ] ; then
31
        SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
32
    else
33
34
35
        echo "ERROR: Cannot find CMS software in OSG node"
        echo "summary: SW_DIR_UNDEF"
        exit $SAME_ERROR
Andrea Sciaba's avatar
Andrea Sciaba committed
36
37
    fi
elif [ -n "$VO_CMS_SW_DIR" ] ; then
Andrea Sciaba's avatar
Andrea Sciaba committed
38
39
    SW_DIR=$VO_CMS_SW_DIR
else
40
    SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
41
fi
Andrea Sciaba's avatar
Andrea Sciaba committed
42
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
43
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
Andrea Sciaba's avatar
Andrea Sciaba committed
44
result=$?
45
46
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
47
48
49
50
51
if [ $result != 0 -o $result2 == 0 ] ; then
    cat $tmpfile
    rm -f $tmpfile
    echo "ERROR: CMS software initialisation script cmsset_default.sh failed"
    echo "summary: NO_SETUP_SCRIPT"
52
53
    exit $SAME_ERROR
fi
54
rm -f $tmpfile
Andrea Sciaba's avatar
Andrea Sciaba committed
55

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# Print out node name
node=`hostname`
echo "node: $node"
echo

# Check that environmental variable SAME_OK is set
if [ -z "$SAME_OK" ] ; then
    echo "ERROR: SAME_OK not defined"
    exit 1
fi

# Check that environmental variable SAME_ERROR is set
if [ -z "$SAME_ERROR" ] ; then
    echo "ERROR: SAME_ERROR not defined"
    exit 1
fi

# Check that environmental variable SAME_WARNING is set
if [ -z "$SAME_WARNING" ] ; then
    echo "ERROR: SAME_WARNING not defined"
    exit 1
fi

# Check that environmental variable CMS_PATH is set
if [ -z "$CMS_PATH" ] ; then
    echo "ERROR: CMS_PATH not defined"
    exit 1
fi

# Print script version information
86
echo "script version: $SCRIPT_REVISION, $MAINTAINERS"
87
88
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
89
90
91
92
93
94
95
96
# Get site name
site_local_config_file=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
if [ ! -f "$site_local_config_file" ] ; then
    echo "ERROR: file ${site_local_config_file} does not exist"
    exit $SAME_ERROR
fi
site=`grep -oP '(?<=site name=").*(?=")' $site_local_config_file`

97
98
99
100
101
102
103
# Create Working Directory
mkdir squid
cd squid
current=`pwd`
echo "Current directory is: ${current}"
echo

104
# Script checks failover if it is SAM test, but script does not check failover if it is pilot
Edita Kizinevic's avatar
Edita Kizinevic committed
105
failover=false
106
if ! $pilot; then
107
108
109
110
111
112
113
114
115
116
117
    if [ ! -z "$site" ] ; then
        site_summary_file=site_summary.txt
        url=http://wlcg-squid-monitor.cern.ch/failover/failoverCMS/$site_summary_file
        wget -q -O $site_summary_file $url
        if [ -f "$site_summary_file" ] ; then
            result="`grep -P "$site[, \t]" $site_summary_file`"
            if [ ! -z "$result"  ] ; then
                failover=true
            fi
        else
            echo "$site_summary_file file is not retrieved from $url"
Edita Kizinevic's avatar
Edita Kizinevic committed
118
119
120
121
        fi
    fi
fi

122
123
124
125
126
127
128
129
# Set up CMSSW
echo "Set up CMSSW ... starting"
CMSREL="`scram l | grep -B 1 "cms.cern.ch" | grep " CMSSW" | tail -1 | awk '{print $2}'`"
scram p $CMSREL
if [ $? -ne 0 ] ; then
    echo "ERROR: $CMSREL not available"
    exit $SAME_WARNING
fi
130
echo "Set up of $CMSREL completed"
131
132
133
134
135
136
echo
cd $CMSREL/src
cmsenv

# Function to check proxies
# $1 - list of proxies
137
# $2 - $pilot
138
check_proxies() {
139
    # Script does not use backupproxyurl if it is SAM test, but script uses backupproxyurl if it is pilot
140
141
    if $2; then
        backupproxyurl="*(backupproxyurl=*"
142
        FRONTIER_ID="${SCRIPT_NAME}_pilot_${SCRIPT_REVISION}"
143
144
    else
        backupproxyurl="NONE"
145
        FRONTIER_ID="${SCRIPT_NAME}_SAM_${SCRIPT_REVISION}"
146
    fi
147
148
149
    for PART in $1; do
        proxy=`echo $PART | sed 's/^.*(/(/'`
        case $proxy in
150
            *"(proxyurl="* | $backupproxyurl)
151
                found_proxy=true
152
153
                hostname=${proxy#*//}
                hostname=${hostname%:*}
154
155
                # The frontier client does not work with IPv6 addresses in URLs for now
                IP_addresses=`getent ahostsv4 $hostname | awk '/STREAM/{print $1}'`
156
                if [ -z "$IP_addresses" ]; then
157
                    IP_addresses=$hostname
158
                elif getent ahostsv6 $hostname | awk '/STREAM/{print $1}' | grep -qv ^::ffff:; then
159
                    IP_addresses="$IP_addresses $hostname"
160
161
                fi
                for IP_address in $IP_addresses; do
162
163
164
                    if ($2 && ! $ever_succeeded) || ! $2; then
                        proxy_IP=`echo $proxy | sed "s/$hostname/$IP_address/g"`
                        echo "Query $proxy started:" `date`
Edita Kizinevic's avatar
Edita Kizinevic committed
165
166
167
168
169
170
171
                        echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=warning FRONTIER_SERVER="$proxy_IP
                            (serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier1.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier2.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier3.cern.ch:8000/FrontierProd)
                            (serverurl=http://cmsfrontier4.cern.ch:8000/FrontierProd)
                            (failovertoserver=no)" FRONTIER_ID=$FRONTIER_ID fn-req > $output
172
173
174
175
176
177
178
179
180
181
                        if grep -q " 1 NUMBER" $output ; then
                            echo "$proxy_IP is OK"
                            ever_succeeded=true
                        else
                            echo "$proxy_IP is FAILED:"
                            cat $output
                            ever_failed=true
                        fi
                        echo "Query $proxy ended:" `date`
                        echo
182
                    else
183
                        return
184
185
186
187
                    fi
                done
                ;;
            *"(proxyconfigurl="*)
188
189
                if [ "$found_proxyconfig" == false ]; then
                    proxy_config_url=$proxy
190
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl started:" `date`
191
                    echo
192
                    echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=debug FRONTIER_SERVER="$proxy(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" FRONTIER_ID=$FRONTIER_ID fn-req > $debug_output 2>&1
193
                    if grep -q "FindProxyForURL" $debug_output ; then
194
195
                        found_proxyconfig=true
                        PROXYLIST="`sed -n '/returned "PROXY/{s/.*returned "//;s/PROXY /(proxyurl=/g;s/;/)/g;s/"/)/;p;q}' $debug_output`"
196
197
198
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbpfrontier.cern.ch/(backupproxyurl=http:\/\/cmsbpfrontier.cern.ch/g"`
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbproxy.fnal.gov/(backupproxyurl=http:\/\/cmsbproxy.fnal.gov/g"`
                        check_proxies "$PROXYLIST" $2
199
                    fi
200
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl ended:" `date`
201
202
                    echo
                fi
203
204
205
206
207
208
209
210
211
212
213
214
                ;;
        esac
    done
}

# Print out cmsGetFnConnect frontier://FrontierProd
FNCONNECT="`cmsGetFnConnect frontier://FrontierProd | sed 's/)/) /g'`"
echo "Contents of cmsGetFnConnect frontier://FrontierProd are:"
echo "$FNCONNECT" | tr " " "\n"
echo

# Check proxies
215
debug_output="debug_output.txt"
216
217
output="output.txt"
ever_failed=false
218
ever_succeeded=false
219
found_proxyconfig=false
220
found_proxy=false
221
check_proxies "$FNCONNECT" $pilot
222
223

#Check test results
Edita Kizinevic's avatar
Edita Kizinevic committed
224
if [ "$failover" == true ]; then
Edita Kizinevic's avatar
Edita Kizinevic committed
225
    message=$'many database queries from the site have connected directly to the Frontier servers or backup proxies, with a high rate of queries not going through the local squid(s):\n'$result
Edita Kizinevic's avatar
Edita Kizinevic committed
226
227
228
229
230
231
232
233
234
235
236
237
238
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK, however $message"
        exit $SAME_WARNING
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED and $message"
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found and $message"
        exit $SAME_WARNING
    else
        echo "Proxy test is FAILED and $message"
        exit $SAME_ERROR
    fi
239
else
Edita Kizinevic's avatar
Edita Kizinevic committed
240
241
242
243
244
245
246
247
248
249
250
251
252
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK."
        exit $SAME_OK
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED."
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found."
        exit $SAME_OK
    else
        echo "Proxy test is FAILED."
        exit $SAME_ERROR
    fi
253
fi