CE-cms-squid 6.91 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
#!/bin/bash

3
4
shopt -s expand_aliases

Edita Kizinevic's avatar
Edita Kizinevic committed
5
SCRIPT_REVISION="2019-09-18, Dave Dykstra and Edita Kizinevic"
6

Andrea Sciaba's avatar
Andrea Sciaba committed
7
# Source the CMS environment
Andrea Sciaba's avatar
Andrea Sciaba committed
8
9
if [ -n "$OSG_GRID" ] ; then
    [ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
10
11
    if [ -d $OSG_APP/cmssoft/cms ] ; then
        SW_DIR=$OSG_APP/cmssoft/cms
Andrea Sciaba's avatar
Andrea Sciaba committed
12
    elif [ -d $CVMFS/cms.cern.ch ] ; then
13
        SW_DIR=$CVMFS/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
14
    elif [ -d /cvmfs/cms.cern.ch ] ; then
15
        SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
16
    else
17
18
19
        echo "ERROR: Cannot find CMS software in OSG node"
        echo "summary: SW_DIR_UNDEF"
        exit $SAME_ERROR
Andrea Sciaba's avatar
Andrea Sciaba committed
20
21
    fi
elif [ -n "$VO_CMS_SW_DIR" ] ; then
Andrea Sciaba's avatar
Andrea Sciaba committed
22
23
    SW_DIR=$VO_CMS_SW_DIR
else
24
    SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
25
fi
Andrea Sciaba's avatar
Andrea Sciaba committed
26
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
27
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
Andrea Sciaba's avatar
Andrea Sciaba committed
28
result=$?
29
30
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
31
32
33
34
35
if [ $result != 0 -o $result2 == 0 ] ; then
    cat $tmpfile
    rm -f $tmpfile
    echo "ERROR: CMS software initialisation script cmsset_default.sh failed"
    echo "summary: NO_SETUP_SCRIPT"
36
37
    exit $SAME_ERROR
fi
38
rm -f $tmpfile
Andrea Sciaba's avatar
Andrea Sciaba committed
39

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# Print out node name
node=`hostname`
echo "node: $node"
echo

# Check that environmental variable SAME_OK is set
if [ -z "$SAME_OK" ] ; then
    echo "ERROR: SAME_OK not defined"
    exit 1
fi

# Check that environmental variable SAME_ERROR is set
if [ -z "$SAME_ERROR" ] ; then
    echo "ERROR: SAME_ERROR not defined"
    exit 1
fi

# Check that environmental variable SAME_WARNING is set
if [ -z "$SAME_WARNING" ] ; then
    echo "ERROR: SAME_WARNING not defined"
    exit 1
fi

# Check that environmental variable CMS_PATH is set
if [ -z "$CMS_PATH" ] ; then
    echo "ERROR: CMS_PATH not defined"
    exit 1
fi

# Print script version information
echo "script version: $SCRIPT_REVISION"
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
73
74
75
76
77
78
79
80
# Get site name
site_local_config_file=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
if [ ! -f "$site_local_config_file" ] ; then
    echo "ERROR: file ${site_local_config_file} does not exist"
    exit $SAME_ERROR
fi
site=`grep -oP '(?<=site name=").*(?=")' $site_local_config_file`

81
82
83
84
85
86
87
# Create Working Directory
mkdir squid
cd squid
current=`pwd`
echo "Current directory is: ${current}"
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
88
89
90
91
92
# Get information from failover monitor
failover=false
if [ ! -z "$site" ] ; then
    site_summary_file=site_summary.txt
    url=http://wlcg-squid-monitor.cern.ch/failover/failoverCMS/$site_summary_file
Edita Kizinevic's avatar
Edita Kizinevic committed
93
    wget -q -O $site_summary_file $url
Edita Kizinevic's avatar
Edita Kizinevic committed
94
    if [ -f "$site_summary_file" ] ; then
Edita Kizinevic's avatar
Edita Kizinevic committed
95
96
        result="`grep -P "$site[, \t]" $site_summary_file`"
        if [ ! -z "$result"  ] ; then
Edita Kizinevic's avatar
Edita Kizinevic committed
97
98
99
100
101
102
103
            failover=true
        fi
    else
        echo "$site_summary_file file is not retrieved from $url"
    fi
fi

104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
# Set up CMSSW
echo "Set up CMSSW ... starting"
CMSREL="`scram l | grep -B 1 "cms.cern.ch" | grep " CMSSW" | tail -1 | awk '{print $2}'`"
scram p $CMSREL
if [ $? -ne 0 ] ; then
    echo "ERROR: $CMSREL not available"
    exit $SAME_WARNING
fi
echo "Set up CMSSW ... completed"
echo
cd $CMSREL/src
cmsenv

# Function to check proxies
# $1 - list of proxies
check_proxies() {
    for PART in $1; do
        proxy=`echo $PART | sed 's/^.*(/(/'`
        case $proxy in
            *"(proxyurl="*)
124
                found_proxy=true
125
126
                hostname=${proxy#*//}
                hostname=${hostname%:*}
127
128
                # The frontier client does not work with IPv6 addresses in URLs for now
                IP_addresses=`getent ahostsv4 $hostname | awk '/STREAM/{print $1}'`
129
                if [ -z "$IP_addresses" ]; then
130
                    IP_addresses=$hostname
131
132
133
134
135
136
137
                fi
                for IP_address in $IP_addresses; do
                    proxy_IP=`echo $proxy | sed "s/$hostname/$IP_address/g"`
                    echo "Query $proxy started:" `date`
                    echo 'select 1 from dual' | FRONTIER_SERVER="$proxy_IP(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $output
                    if grep -q " 1 NUMBER" $output ; then
                        echo "$proxy_IP is OK"
138
                        ever_succeeded=true
139
140
141
142
143
144
145
146
147
148
                    else
                        echo "$proxy_IP is FAILED:"
                        cat $output
                        ever_failed=true
                    fi
                    echo "Query $proxy ended:" `date`
                    echo
                done
                ;;
            *"(proxyconfigurl="*)
149
150
151
                if [ "$found_proxyconfig" == false ]; then
                    proxy_config_url=$proxy
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl started:" `date`
152
                    echo
153
                    echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=debug FRONTIER_SERVER="$proxy(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $debug_output 2>&1
154
                    if grep -q "FindProxyForURL" $debug_output ; then
155
156
157
                        found_proxyconfig=true
                        PROXYLIST="`sed -n '/returned "PROXY/{s/.*returned "//;s/PROXY /(proxyurl=/g;s/;/)/g;s/"/)/;p;q}' $debug_output`"
                        check_proxies "$PROXYLIST"
158
159
160
161
                    fi
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl ended:" `date`
                    echo
                fi
162
163
164
165
166
167
168
169
170
171
172
173
                ;;
        esac
    done
}

# Print out cmsGetFnConnect frontier://FrontierProd
FNCONNECT="`cmsGetFnConnect frontier://FrontierProd | sed 's/)/) /g'`"
echo "Contents of cmsGetFnConnect frontier://FrontierProd are:"
echo "$FNCONNECT" | tr " " "\n"
echo

# Check proxies
174
debug_output="debug_output.txt"
175
176
output="output.txt"
ever_failed=false
177
ever_succeeded=false
178
found_proxyconfig=false
179
found_proxy=false
180
181
182
check_proxies "$FNCONNECT"

#Check test results
Edita Kizinevic's avatar
Edita Kizinevic committed
183
if [ "$failover" == true ]; then
Edita Kizinevic's avatar
Edita Kizinevic committed
184
    message="many database queries from the site have connected directly to the Frontier servers or backup proxies, with a high rate of queries not going through the local squid(s):"
Edita Kizinevic's avatar
Edita Kizinevic committed
185
186
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK, however $message"
Edita Kizinevic's avatar
Edita Kizinevic committed
187
        echo $result
Edita Kizinevic's avatar
Edita Kizinevic committed
188
189
190
        exit $SAME_WARNING
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED and $message"
Edita Kizinevic's avatar
Edita Kizinevic committed
191
        echo $result
Edita Kizinevic's avatar
Edita Kizinevic committed
192
193
194
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found and $message"
Edita Kizinevic's avatar
Edita Kizinevic committed
195
        echo $result
Edita Kizinevic's avatar
Edita Kizinevic committed
196
197
198
        exit $SAME_WARNING
    else
        echo "Proxy test is FAILED and $message"
Edita Kizinevic's avatar
Edita Kizinevic committed
199
        echo $result
Edita Kizinevic's avatar
Edita Kizinevic committed
200
201
        exit $SAME_ERROR
    fi
202
else
Edita Kizinevic's avatar
Edita Kizinevic committed
203
204
205
206
207
208
209
210
211
212
213
214
215
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK."
        exit $SAME_OK
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED."
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found."
        exit $SAME_OK
    else
        echo "Proxy test is FAILED."
        exit $SAME_ERROR
    fi
216
fi