CE-cms-squid 6.75 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
#!/bin/bash

3
4
shopt -s expand_aliases

Edita Kizinevic's avatar
Edita Kizinevic committed
5
SCRIPT_REVISION="2019-09-17, Dave Dykstra and Edita Kizinevic"
6

Andrea Sciaba's avatar
Andrea Sciaba committed
7
# Source the CMS environment
Andrea Sciaba's avatar
Andrea Sciaba committed
8
9
if [ -n "$OSG_GRID" ] ; then
    [ -f $OSG_GRID/setup.sh ] && source $OSG_GRID/setup.sh
10
11
    if [ -d $OSG_APP/cmssoft/cms ] ; then
        SW_DIR=$OSG_APP/cmssoft/cms
Andrea Sciaba's avatar
Andrea Sciaba committed
12
    elif [ -d $CVMFS/cms.cern.ch ] ; then
13
        SW_DIR=$CVMFS/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
14
    elif [ -d /cvmfs/cms.cern.ch ] ; then
15
        SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
16
    else
17
18
19
        echo "ERROR: Cannot find CMS software in OSG node"
        echo "summary: SW_DIR_UNDEF"
        exit $SAME_ERROR
Andrea Sciaba's avatar
Andrea Sciaba committed
20
21
    fi
elif [ -n "$VO_CMS_SW_DIR" ] ; then
Andrea Sciaba's avatar
Andrea Sciaba committed
22
23
    SW_DIR=$VO_CMS_SW_DIR
else
24
    SW_DIR=/cvmfs/cms.cern.ch
Andrea Sciaba's avatar
Andrea Sciaba committed
25
fi
Andrea Sciaba's avatar
Andrea Sciaba committed
26
tmpfile=`mktemp /tmp/tmp.XXXXXXXXXX`
27
source $SW_DIR/cmsset_default.sh > $tmpfile 2>&1
Andrea Sciaba's avatar
Andrea Sciaba committed
28
result=$?
29
30
grep 'Your shell is not able to find' $tmpfile > /dev/null
result2=$?
31
32
33
34
35
if [ $result != 0 -o $result2 == 0 ] ; then
    cat $tmpfile
    rm -f $tmpfile
    echo "ERROR: CMS software initialisation script cmsset_default.sh failed"
    echo "summary: NO_SETUP_SCRIPT"
36
37
    exit $SAME_ERROR
fi
38
rm -f $tmpfile
Andrea Sciaba's avatar
Andrea Sciaba committed
39

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# Print out node name
node=`hostname`
echo "node: $node"
echo

# Check that environmental variable SAME_OK is set
if [ -z "$SAME_OK" ] ; then
    echo "ERROR: SAME_OK not defined"
    exit 1
fi

# Check that environmental variable SAME_ERROR is set
if [ -z "$SAME_ERROR" ] ; then
    echo "ERROR: SAME_ERROR not defined"
    exit 1
fi

# Check that environmental variable SAME_WARNING is set
if [ -z "$SAME_WARNING" ] ; then
    echo "ERROR: SAME_WARNING not defined"
    exit 1
fi

# Check that environmental variable CMS_PATH is set
if [ -z "$CMS_PATH" ] ; then
    echo "ERROR: CMS_PATH not defined"
    exit 1
fi

# Print script version information
echo "script version: $SCRIPT_REVISION"
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
73
74
75
76
77
78
79
80
# Get site name
site_local_config_file=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
if [ ! -f "$site_local_config_file" ] ; then
    echo "ERROR: file ${site_local_config_file} does not exist"
    exit $SAME_ERROR
fi
site=`grep -oP '(?<=site name=").*(?=")' $site_local_config_file`

81
82
83
84
85
86
87
# Create Working Directory
mkdir squid
cd squid
current=`pwd`
echo "Current directory is: ${current}"
echo

Edita Kizinevic's avatar
Edita Kizinevic committed
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# Get information from failover monitor
failover=false
if [ ! -z "$site" ] ; then
    site_summary_file=site_summary.txt
    url=http://wlcg-squid-monitor.cern.ch/failover/failoverCMS/$site_summary_file
    wget -q $url
    if [ -f "$site_summary_file" ] ; then
        if grep -q $site $site_summary_file ; then
            failover=true
        fi
    else
        echo "$site_summary_file file is not retrieved from $url"
    fi
fi

103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# Set up CMSSW
echo "Set up CMSSW ... starting"
CMSREL="`scram l | grep -B 1 "cms.cern.ch" | grep " CMSSW" | tail -1 | awk '{print $2}'`"
scram p $CMSREL
if [ $? -ne 0 ] ; then
    echo "ERROR: $CMSREL not available"
    exit $SAME_WARNING
fi
echo "Set up CMSSW ... completed"
echo
cd $CMSREL/src
cmsenv

# Function to check proxies
# $1 - list of proxies
check_proxies() {
    for PART in $1; do
        proxy=`echo $PART | sed 's/^.*(/(/'`
        case $proxy in
            *"(proxyurl="*)
123
                found_proxy=true
124
125
                hostname=${proxy#*//}
                hostname=${hostname%:*}
126
127
                # The frontier client does not work with IPv6 addresses in URLs for now
                IP_addresses=`getent ahostsv4 $hostname | awk '/STREAM/{print $1}'`
128
                if [ -z "$IP_addresses" ]; then
129
                    IP_addresses=$hostname
130
131
132
133
134
135
136
                fi
                for IP_address in $IP_addresses; do
                    proxy_IP=`echo $proxy | sed "s/$hostname/$IP_address/g"`
                    echo "Query $proxy started:" `date`
                    echo 'select 1 from dual' | FRONTIER_SERVER="$proxy_IP(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $output
                    if grep -q " 1 NUMBER" $output ; then
                        echo "$proxy_IP is OK"
137
                        ever_succeeded=true
138
139
140
141
142
143
144
145
146
147
                    else
                        echo "$proxy_IP is FAILED:"
                        cat $output
                        ever_failed=true
                    fi
                    echo "Query $proxy ended:" `date`
                    echo
                done
                ;;
            *"(proxyconfigurl="*)
148
149
150
                if [ "$found_proxyconfig" == false ]; then
                    proxy_config_url=$proxy
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl started:" `date`
151
                    echo
152
                    echo 'select 1 from dual' | FRONTIER_LOG_LEVEL=debug FRONTIER_SERVER="$proxy(serverurl=http://cmsfrontier.cern.ch:8000/FrontierProd)(failovertoserver=no)" fn-req > $debug_output 2>&1
153
                    if grep -q "FindProxyForURL" $debug_output ; then
154
155
156
                        found_proxyconfig=true
                        PROXYLIST="`sed -n '/returned "PROXY/{s/.*returned "//;s/PROXY /(proxyurl=/g;s/;/)/g;s/"/)/;p;q}' $debug_output`"
                        check_proxies "$PROXYLIST"
157
158
159
160
                    fi
                    echo "Converting $proxy_config_url proxyconfigurl to proxyurl ended:" `date`
                    echo
                fi
161
162
163
164
165
166
167
168
169
170
171
172
                ;;
        esac
    done
}

# Print out cmsGetFnConnect frontier://FrontierProd
FNCONNECT="`cmsGetFnConnect frontier://FrontierProd | sed 's/)/) /g'`"
echo "Contents of cmsGetFnConnect frontier://FrontierProd are:"
echo "$FNCONNECT" | tr " " "\n"
echo

# Check proxies
173
debug_output="debug_output.txt"
174
175
output="output.txt"
ever_failed=false
176
ever_succeeded=false
177
found_proxyconfig=false
178
found_proxy=false
179
180
181
check_proxies "$FNCONNECT"

#Check test results
Edita Kizinevic's avatar
Edita Kizinevic committed
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
if [ "$failover" == true ]; then
    message="many database queries from your site have connected directly to the Frontier Server Groups, with a high rate of queries not going through your local squid(s)."
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK, however $message"
        exit $SAME_WARNING
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED and $message"
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found and $message"
        exit $SAME_WARNING
    else
        echo "Proxy test is FAILED and $message"
        exit $SAME_ERROR
    fi
197
else
Edita Kizinevic's avatar
Edita Kizinevic committed
198
199
200
201
202
203
204
205
206
207
208
209
210
    if [ "$ever_failed" == false ] && [ "$ever_succeeded" == true ]; then
        echo "Proxy test is OK."
        exit $SAME_OK
    elif [ "$ever_failed" == true ] && [ "$ever_succeeded" == true ]; then
        echo "At least one of proxies is FAILED."
        exit $SAME_WARNING
    elif [ "$found_proxy" == false ]; then
        echo "No proxies are found."
        exit $SAME_OK
    else
        echo "Proxy test is FAILED."
        exit $SAME_ERROR
    fi
211
fi