CE-cms-squid 7.69 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
#!/bin/bash

3
4
shopt -s expand_aliases

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
SCRIPT_REVISION="2020-02-17, Dave Dykstra and Edita Kizinevic"

# Script does not use backupproxyurl if it is SAM test, but script uses backupproxyurl if it is pilot
includebackups=false
while getopts ":b" opt; do
  case ${opt} in
    b)
      includebackups=true
      ;;
    \?)
      echo "Invalid Option: -$OPTARG" 1>&2
      exit 1
      ;;
    esac
done
20

Andrea Sciaba's avatar
Andrea Sciaba committed
21
# Source the CMS environment
Andrea Sciaba's avatar
Andrea Sciaba committed
22
23
if [ -n "$OSG_GRID" ] ; then
    [ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
24
25
    if [ -d $OSG_APP/cmssoft/cms ] ; then
        SW_DIR=$OSG_APP/cmssoft/cms
Andrea Sciaba's avatar
Andrea Sciaba committed
26
    elif [ -d $CVMFS/cms.cern.ch ] ; then
27
        SW_DIR=$CVMFS/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
28
    elif [ -d /cvmfs/cms.cern.ch ] ; then
29
        SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
30
    else
31
32
33
        echo "ERROR: Cannot find CMS software in OSG node"
        echo "summary: SW_DIR_UNDEF"
        exit $SAME_ERROR
Andrea Sciaba's avatar
Andrea Sciaba committed
34
35
    fi
elif [ -n "$VO_CMS_SW_DIR" ] ; then
Andrea Sciaba's avatar
Andrea Sciaba committed
36
37
    SW_DIR=$VO_CMS_SW_DIR
else
38
    SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
39
fi
Andrea Sciaba's avatar
Andrea Sciaba committed
40
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
41
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
Andrea Sciaba's avatar
Andrea Sciaba committed
42
result=$?
43
44
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
45
46
47
48
49
if [ $result != 0 -o $result2 == 0 ] ; then
    cat $tmpfile
    rm -f $tmpfile
    echo "ERROR: CMS software initialisation script cmsset_default.sh failed"
    echo "summary: NO_SETUP_SCRIPT"
50
51
    exit $SAME_ERROR
fi
52
rm -f $tmpfile
Andrea Sciaba's avatar
Andrea Sciaba committed
53

54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# Print out node name
node=`hostname`
echo "node: $node"
echo

# Check that environmental variable SAME_OK is set
if [ -z "$SAME_OK" ] ; then
    echo "ERROR: SAME_OK not defined"
    exit 1
fi

# Check that environmental variable SAME_ERROR is set
if [ -z "$SAME_ERROR" ] ; then
    echo "ERROR: SAME_ERROR not defined"
    exit 1
fi

# Check that environmental variable SAME_WARNING is set
if [ -z "$SAME_WARNING" ] ; then
    echo "ERROR: SAME_WARNING not defined"
    exit 1
fi

# Check that environmental variable CMS_PATH is set
if [ -z "$CMS_PATH" ] ; then
    echo "ERROR: CMS_PATH not defined"
    exit 1
fi

# Print script version information
echo "script version: $SCRIPT_REVISION"
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
87
88
89
90
91
92
93
94
# Get site name
site_local_config_file=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
if [ ! -f "$site_local_config_file" ] ; then
    echo "ERROR: file ${site_local_config_file} does not exist"
    exit $SAME_ERROR
fi
site=`grep -oP '(?<=site name=").*(?=")' $site_local_config_file`

95
96
97
98
99
100
101
# Create Working Directory
mkdir squid
cd squid
current=`pwd`
echo "Current directory is: ${current}"
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
102
103
104
105
106
# Get information from failover monitor
failover=false
if [ ! -z "$site" ] ; then
    site_summary_file=site_summary.txt
    url=http://wlcg-squid-monitor.cern.ch/failover/failoverCMS/$site_summary_file
Edita Kizinevic's avatar
Edita Kizinevic committed
107
    wget -q -O $site_summary_file $url
Edita Kizinevic's avatar
Edita Kizinevic committed
108
    if [ -f "$site_summary_file" ] ; then
Edita Kizinevic's avatar
Edita Kizinevic committed
109
110
        result="`grep -P "$site[, \t]" $site_summary_file`"
        if [ ! -z "$result"  ] ; then
Edita Kizinevic's avatar
Edita Kizinevic committed
111
112
113
114
115
116
117
            failover=true
        fi
    else
        echo "$site_summary_file file is not retrieved from $url"
    fi
fi

118
119
120
121
122
123
124
125
# Set up CMSSW
echo "Set up CMSSW ... starting"
CMSREL="`scram l | grep -B 1 "cms.cern.ch" | grep " CMSSW" | tail -1 | awk '{print $2}'`"
scram p $CMSREL
if [ $? -ne 0 ] ; then
    echo "ERROR: $CMSREL not available"
    exit $SAME_WARNING
fi
126
echo "Set up of $CMSREL completed"
127
128
129
130
131
132
echo
cd $CMSREL/src
cmsenv

# Function to check proxies
# $1 - list of proxies
133
# $2 - $includebackups
134
check_proxies() {
135
136
137
138
139
    if $2; then
        backupproxyurl="*(backupproxyurl=*"
    else
        backupproxyurl="NONE"
    fi
140
141
142
    for PART in $1; do
        proxy=`echo $PART | sed 's/^.*(/(/'`
        case $proxy in
143
            *"(proxyurl="* | $backupproxyurl)
144
                found_proxy=true
145
146
                hostname=${proxy#*//}
                hostname=${hostname%:*}
147
148
                # The frontier client does not work with IPv6 addresses in URLs for now
                IP_addresses=`getent ahostsv4 $hostname | awk '/STREAM/{print $1}'`
149
                if [ -z "$IP_addresses" ]; then
150
                    IP_addresses=$hostname
151
152
                else
                    IP_addresses="$IP_addresses $hostname"
153
154
155
156
157
158
159
                fi
                for IP_address in $IP_addresses; do
                    proxy_IP=`echo $proxy | sed "s/$hostname/$IP_address/g"`
                    echo "Query $proxy started:" `date`
                    echo 'select 1 from dual' | FRONTIER_SERVER="$proxy_IP(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $output
                    if grep -q " 1 NUMBER" $output ; then
                        echo "$proxy_IP is OK"
160
                        ever_succeeded=true
161
162
163
164
165
166
167
168
169
170
                    else
                        echo "$proxy_IP is FAILED:"
                        cat $output
                        ever_failed=true
                    fi
                    echo "Query $proxy ended:" `date`
                    echo
                done
                ;;
            *"(proxyconfigurl="*)
171
172
                if [ "$found_proxyconfig" == false ]; then
                    proxy_config_url=$proxy
173
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl started:" `date`
174
                    echo
175
                    echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=debug FRONTIER_SERVER="$proxy(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $debug_output 2>&1
176
                    if grep -q "FindProxyForURL" $debug_output ; then
177
178
                        found_proxyconfig=true
                        PROXYLIST="`sed -n '/returned "PROXY/{s/.*returned "//;s/PROXY /(proxyurl=/g;s/;/)/g;s/"/)/;p;q}' $debug_output`"
179
180
181
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbpfrontier.cern.ch/(backupproxyurl=http:\/\/cmsbpfrontier.cern.ch/g"`
                        PROXYLIST=`echo $PROXYLIST | sed "s/(proxyurl=http:\/\/cmsbproxy.fnal.gov/(backupproxyurl=http:\/\/cmsbproxy.fnal.gov/g"`
                        check_proxies "$PROXYLIST" $2
182
                    fi
183
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl/backupproxyurl ended:" `date`
184
185
                    echo
                fi
186
187
188
189
190
191
192
193
194
195
196
197
                ;;
        esac
    done
}

# Print out cmsGetFnConnect frontier://FrontierProd
FNCONNECT="`cmsGetFnConnect frontier://FrontierProd | sed 's/)/) /g'`"
echo "Contents of cmsGetFnConnect frontier://FrontierProd are:"
echo "$FNCONNECT" | tr " " "\n"
echo

# Check proxies
198
debug_output="debug_output.txt"
199
200
output="output.txt"
ever_failed=false
201
ever_succeeded=false
202
found_proxyconfig=false
203
found_proxy=false
204
check_proxies "$FNCONNECT" $includebackups
205
206

#Check test results
Edita Kizinevic's avatar
Edita Kizinevic committed
207
if [ "$failover" == true ]; then
Edita Kizinevic's avatar
Edita Kizinevic committed
208
    message=$'many database queries from the site have connected directly to the Frontier servers or backup proxies, with a high rate of queries not going through the local squid(s):\n'$result
Edita Kizinevic's avatar
Edita Kizinevic committed
209
210
211
212
213
214
215
216
217
218
219
220
221
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK, however $message"
        exit $SAME_WARNING
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED and $message"
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found and $message"
        exit $SAME_WARNING
    else
        echo "Proxy test is FAILED and $message"
        exit $SAME_ERROR
    fi
222
else
Edita Kizinevic's avatar
Edita Kizinevic committed
223
224
225
226
227
228
229
230
231
232
233
234
235
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK."
        exit $SAME_OK
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED."
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found."
        exit $SAME_OK
    else
        echo "Proxy test is FAILED."
        exit $SAME_ERROR
    fi
236
fi