test_squid.py 12.1 KB
Newer Older
ericw's avatar
ericw committed
1
2
3
4
5
6
7
#!/usr/bin/env python
"""
Tests the local FroNtier squid
"""
#
# Assumes:
#
wangw's avatar
wangw committed
8
#   1) environmental variables SAME_OK,SAME_WARNING and SAME_ERROR are defined
ericw's avatar
ericw committed
9
10
11
12
#   2) environmental variable $CMS_PATH is defined
#   3) file $CMS_PATH/SITECONF/local/JobConfig/site-local-config.xml
#      contains the location of the local FroNtier squid server
#
Andrea Sciaba's avatar
Andrea Sciaba committed
13
__revision__ = "$Id: test_squid.py,v 1.30 2012/10/24 13:30:07 asciaba Exp $"
ericw's avatar
ericw committed
14
15
16
17
18
19
20
21
22
23
import os
import sys
import urllib2
from xml.dom.minidom import parseString
from xml.dom.minidom import parse
import base64
import zlib 
import curses.ascii
import time
#
wangw's avatar
wangw committed
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
#---------------- define timeout on urllib2 socket ops -------------#
#  Adapted from http://code.google.com/p/timeout-urllib2/

from httplib import HTTPConnection as _HC
import socket
from urllib2 import HTTPHandler as _H

def sethttptimeout(timeout):
  """Use TimeoutHTTPHandler and set the timeout value.
  
  Args:
    timeout: the socket connection timeout value.
  """
  if _under_26():
    opener = urllib2.build_opener(TimeoutHTTPHandler(timeout))
    urllib2.install_opener(opener)
  else:
    raise Error("This python version has timeout builtin")

def _clear(sock):
  sock.close()
  return None

def _under_24():
  import sys
  if sys.version_info[0] < 2: return True
  if sys.version_info[0] == 2:
    return sys.version_info[1] < 4
  return False

def _under_26():
  import sys
  if sys.version_info[0] < 2: return True
  if sys.version_info[0] == 2:
    return sys.version_info[1] < 6
  return False

class Error(Exception): pass

class HTTPConnectionTimeoutError(Error): pass

class TimeoutHTTPConnection(_HC):
  """A timeout control enabled HTTPConnection.
  
  Inherit httplib.HTTPConnection class and provide the socket timeout
  control.
  """
  _timeout = None

  def __init__(self, host, port=None, strict=None, timeout=None):
    """Initialize the object.

    Args:
      host: the connection host.
      port: optional port.
      strict: strict connection.
      timeout: socket connection timeout value.
    """
    _HC.__init__(self, host, port, strict)
    self._timeout = timeout or TimeoutHTTPConnection._timeout
    if self._timeout: self._timeout = float(self._timeout)

  def connect(self):
    """Perform the socket level connection.

    A new socket object will get built everytime. If the connection
    object has _timeout attribute, it will be set as the socket
    timeout value.

    Raises:
      HTTPConnectionTimeoutError: when timeout is hit
      socket.error: when other general socket errors encountered.
    """
    msg = "getaddrinfo returns an empty list"
    err = socket.error
    for res in socket.getaddrinfo(self.host, self.port, 0,
                                  socket.SOCK_STREAM):
      af, socktype, proto, canonname, sa = res
      try:
        try:
          self.sock = socket.socket(af, socktype, proto)
          if self._timeout: self.sock.settimeout(self._timeout)
          if self.debuglevel > 0:
            print "connect: (%s, %s)" % (self.host, self.port)
          self.sock.connect(sa)
        except socket.timeout, msg:
          err = socket.timeout
          if self.debuglevel > 0:
            print 'connect timeout:', (self.host, self.port)
          self.sock = _clear(self.sock)
          continue
        break
      except socket.error, msg:
        if self.debuglevel > 0:
          print 'general connect fail:', (self.host, self.port)
        self.sock = _clear(self.sock)
        continue
      break
    if not self.sock:
      if err == socket.timeout:
        raise HTTPConnectionTimeoutError, msg
      raise err, msg

class TimeoutHTTPHandler(_H):
  """A timeout enabled HTTPHandler for urllib2."""
  def __init__(self, timeout=None, debuglevel=0):
    """Initialize the object.

    Args:
      timeout: the socket connect timeout value.
      debuglevel: the debuglevel level.
    """
    _H.__init__(self, debuglevel)
    TimeoutHTTPConnection._timeout = timeout

  def http_open(self, req):
    """Use TimeoutHTTPConnection to perform the http_open"""
    return self.do_open(TimeoutHTTPConnection, req)

#---------------- end timeout on socket ops ----------------#
#
ericw's avatar
ericw committed
145
146
# Print out node name
#
ericw's avatar
ericw committed
147
print "node: " + os.uname()[1] 
ericw's avatar
ericw committed
148
#
ericw's avatar
ericw committed
149
150
151
# Check that environmental variable SAME_OK is set
#
if not os.environ.has_key("SAME_OK"):
Dong Liang's avatar
Dong Liang committed
152
153
	print "test_squid.py: Error. SAME_OK not defined"
	sys.exit(1)
ericw's avatar
ericw committed
154
155
156
157
158
same_ok = int(os.environ["SAME_OK"])    
#
# Check that environmental variable SAME_ERROR is set
#
if not os.environ.has_key("SAME_ERROR"):
Dong Liang's avatar
Dong Liang committed
159
160
	print "test_squid.py: Error. SAME_ERROR not defined"
	sys.exit(1)
ericw's avatar
ericw committed
161
162
same_error = int(os.environ["SAME_ERROR"])    
#
Dong Liang's avatar
Dong Liang committed
163
164
165
166
167
168
169
# Check that envrionmental variable SAME_WARNING is set
#
if not os.environ.has_key("SAME_WARNING"):
	print "test_squid.py: Error. SAME_WARNING not defined"
	sys.exit(1)
same_warning = int(os.environ["SAME_WARNING"])
#
ericw's avatar
ericw committed
170
171
172
# Check that environmental variable CMS_PATH is set
#
if not os.environ.has_key("CMS_PATH"):
Dong Liang's avatar
Dong Liang committed
173
174
	print "test_squid.py: Error. CMS_PATH not defined"
	sys.exit(same_error)
ericw's avatar
ericw committed
175
176
177
178
#
# Check that file $CMS_PATH/SITECONF/local/JobConfig/site-local-config.xml
# exists
#
ericw's avatar
ericw committed
179
180
slcfil = os.environ["CMS_PATH"] + \
         "/SITECONF/local/JobConfig/site-local-config.xml"
Andrea Sciaba's avatar
Andrea Sciaba committed
181
print 'SiteLocalConfig: ' + slcfil
ericw's avatar
ericw committed
182
if not os.path.exists(slcfil):
Dong Liang's avatar
Dong Liang committed
183
184
	print "test_squid.py: Error. file " + slcfil + " does not exist"
	sys.exit(same_error)
ericw's avatar
ericw committed
185
#
ericw's avatar
ericw committed
186
187
# Print out site-local-config.xml
#
Andrea Sciaba's avatar
Andrea Sciaba committed
188
print "\nContents of site-local-config.xml are:"
ericw's avatar
ericw committed
189
190
191
192
193
194
fileobj = open(slcfil,'r')
slcprint = fileobj.read()
slcprint = slcprint.replace('<','&lt;')
print slcprint
fileobj.close()
#
ericw's avatar
ericw committed
195
# Read and parse site-local-config.xml into a dom
ericw's avatar
ericw committed
196
197
# See http://docs.python.org/lib/module-xml.dom.minidom.html
#
ericw's avatar
ericw committed
198
199
200
201
202
203
slcdom = parse(slcfil)
#
# Work out site name from site-local-config.xml
#
silist = slcdom.getElementsByTagName("site")
if len(silist) == 0:
Dong Liang's avatar
Dong Liang committed
204
	site = "UNKNOWN"
ericw's avatar
ericw committed
205
else:
Dong Liang's avatar
Dong Liang committed
206
207
	stag = silist[0]
	site = stag.getAttribute("name")
ericw's avatar
ericw committed
208
print "site: " + site    
ericw's avatar
ericw committed
209
210
211
#
# Work out local FroNtier squid server from site-local-config.xml
#
ericw's avatar
ericw committed
212
213
214
#
# Check for at least one proxy tag
#
Dong Liang's avatar
Dong Liang committed
215
prlist = [x.getAttribute('url') for x in slcdom.getElementsByTagName("proxy")]
ericw's avatar
ericw committed
216
if len(prlist) == 0:
Dong Liang's avatar
Dong Liang committed
217
218
219
220
221
222
223
224
225
226
227
	print "test_squid.py: Error. no proxy tag in file " + slcfil
	sys.exit(same_error)    
# 
# Check whether has load balance proxies from site-local-config.xml
# 
load = slcdom.getElementsByTagName("load")
if len(load) == 0:
	loadtag = "None"
else:
	loadtag = load[0].getAttribute("balance")
print "loadtag: " + loadtag
ericw's avatar
ericw committed
228
#
ericw's avatar
ericw committed
229
230
# Print script version information
#
Andrea Sciaba's avatar
Andrea Sciaba committed
231
print "script version: " + __revision__
ericw's avatar
ericw committed
232
#
ericw's avatar
ericw committed
233
234
# Set server for urllib2
#
ericw's avatar
ericw committed
235
frontierUrl = "http://cmsfrontier.cern.ch:8080/FrontierProd/Frontier"
ericw's avatar
ericw committed
236
#
ericw's avatar
ericw committed
237
# Set frontierId for later use
ericw's avatar
ericw committed
238
239
240
#
tvers = __revision__.split()[2]
# print "tvers: " + tvers
ericw's avatar
ericw committed
241
frontierId = "test_squid.py " + tvers
ericw's avatar
ericw committed
242
# print "frontierId: " + frontierId
ericw's avatar
ericw committed
243
#
ericw's avatar
ericw committed
244
# The following follows code from Sinisa Veseli 
ericw's avatar
ericw committed
245
246
247
248
249
250
251
252
#
# Set parameters
#
frontierQuery = "SELECT 1 FROM DUAL"
decodeFlag = True
refreshFlag = True
retrieveZiplevel = ""
#
253
# Print parameters
ericw's avatar
ericw committed
254
#  
255
256
print "Using Frontier URL: " + frontierUrl
print "Query: " + frontierQuery
ericw's avatar
ericw committed
257
258
# print "Decode results: " + str(decodeFlag)
# print "Refresh cache: " + str(refreshFlag)
ericw's avatar
ericw committed
259
260
261
262
263
264
265
266
267
#
# Encode query
#
compQuery = zlib.compress(frontierQuery, 9)
encQuery = base64.binascii.b2a_base64(compQuery).replace("+", ".")
#
# Set up FroNtier request
#
format = "%s?type=frontier_request:1:DEFAULT&encoding=BLOB%s&p1=%s"
ericw's avatar
ericw committed
268
#
ericw's avatar
ericw committed
269
270
271
# Start and time query
#
queryStart = time.localtime()
272
print "\nQuery started: " + time.strftime("%m/%d/%y %H:%M:%S %Z", queryStart)
ericw's avatar
ericw committed
273
t1 = time.time()
Dong Liang's avatar
Dong Liang committed
274
test_result = "Failed"
Dong Liang's avatar
Dong Liang committed
275
ever_failed = "False"
ericw's avatar
ericw committed
276
#
Dong Liang's avatar
Dong Liang committed
277
278
279
280
281
282
283
284
285
286
for squid in prlist:
    #
    # Set proxy server
    #
	print "squid: " + squid   
	os.environ["http_proxy"] = squid
    #
    # Create request
    #
	frontierRequest = format % (frontierUrl, retrieveZiplevel, encQuery)
Andrea Sciaba's avatar
Andrea Sciaba committed
287
	print "\nFrontier Request:\n", frontierRequest.replace('&','&amp;') 
Dong Liang's avatar
Dong Liang committed
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
    #
    # Add refresh header if needed
    #
	proxy_support = urllib2.ProxyHandler({'http': squid})
	opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)
	urllib2.install_opener(opener)
	request = urllib2.Request(frontierRequest)
	if refreshFlag:
		request.add_header("pragma", "no-cache")
    #
    # Add Frontier-ID header
    #
	request.add_header("X-Frontier-Id", frontierId)
    #
	try:
wangw's avatar
wangw committed
303
304
305
306
307
308
309
310
		if _under_24():
                	print >> sys.stderr, "*WARNING:* no timeout available in python older than 2.4"
            		result = urllib2.urlopen(request).read()
		elif _under_26():
			sethttptimeout(10)
			result = urllib2.urlopen(request).read()
		else:
			result = urllib2.urlopen(request,None,10).read()
Dong Liang's avatar
Dong Liang committed
311
312
313
314
315
	except urllib2.HTTPError, e1:
		message1 = e1.msg
		if message1 == "Forbidden":
			print "test_squid.py: Error. squid " + squid + " refused request."
			print "                      urllib2.HTTPError: " + message1
Dong Liang's avatar
Dong Liang committed
316
			ever_failed = "True"
Dong Liang's avatar
Dong Liang committed
317
318
319
320
321
			continue
		else:
			print "test_squid.py: Error. squid " + squid + " is down, "
			print "                      unreachable or will not reply."
			print "                      urllib2.HTTPError: " + message1
Dong Liang's avatar
Dong Liang committed
322
			ever_failed = "True"
Dong Liang's avatar
Dong Liang committed
323
324
			continue
	except urllib2.URLError, e2:
325
		message2 = e2.reason
Dong Liang's avatar
Dong Liang committed
326
327
		print "test_squid.py: Error. squid " + squid + " is down, "
		print "                      unreachable or will not reply."
328
329
330
		print "                      urllib2.URLError: "
		print message2
                ever_failed = "True"
Dong Liang's avatar
Dong Liang committed
331
332
333
		continue
	except:
		print "test_squid.py: Error."
Dong Liang's avatar
Dong Liang committed
334
		ever_failed = "True"
Dong Liang's avatar
Dong Liang committed
335
336
337
338
339
340
341
342
343
344
345
		continue
    #
	t2 = time.time()
	queryEnd = time.localtime()
    #
    # Check on length of output
    # 
	size = len(result)
    # print "size: " + str(size)
	if size == 0:
		print "test_squid.py: Error. no output"
Dong Liang's avatar
Dong Liang committed
346
		ever_failed = "True"
Dong Liang's avatar
Dong Liang committed
347
348
349
350
351
		continue
	print "Query ended: " + time.strftime("%m/%d/%y %H:%M:%S %Z", queryEnd)
	print "Query time: %0.2f [seconds]\n" % (t2-t1)
    # duration = (t2-t1)
    # print duration, size, size/duration
352
    #
Dong Liang's avatar
Dong Liang committed
353
    # Print out result
354
    #
Dong Liang's avatar
Dong Liang committed
355
356
357
358
359
360
361
362
363
364
365
366
367
368
	if decodeFlag:
		webprint = result.replace('<','&lt;')
		print "Query result:\n" + webprint
		dom = parseString(result)
        #
        # Check error code in result
        #
		qlist = dom.getElementsByTagName("quality")
		qtag = qlist[0]
		ecode = int(qtag.getAttribute("error"))
        # print "ecode: " + str(ecode)
		if ecode != 0:
			message3 = qtag.getAttribute("message")
			print "test_squid.py: Error. " + message3
Dong Liang's avatar
Dong Liang committed
369
			ever_failed = "True"
Dong Liang's avatar
Dong Liang committed
370
371
372
373
374
375
376
			continue
        #
        # Decode result
        #      
		dataList = dom.getElementsByTagName("data")
        # Control characters represent records, but I won't bother with that now,
        # and will simply replace those by space.
wangw's avatar
wangw committed
377
		keepalives = 0
Dong Liang's avatar
Dong Liang committed
378
		for data in dataList:
wangw's avatar
wangw committed
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
			for node in data.childNodes:
				# <keepalive /> elements may be present, combined with whitespace text
				if node.nodeName == "keepalive":
				# this is of type Element
					keepalives += 1
        				continue
      				# else assume of type Text
      				if node.data.strip() == "":
        				continue
      				if keepalives > 0:
					print keepalives, "keepalives received\n"
					keepalives = 0


				row = base64.decodestring(node.data)
Dong Liang's avatar
Dong Liang committed
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
				if retrieveZiplevel != "":
					row = zlib.decompress(row)
				control = [ '\x00', '\x01', '\x02', '\x03', '\x04',
                            '\x05', '\x06', '\x08', '\x09', '\x0a',
                            '\x0b', '\x0c', '\x0d', '\x1b', '\x17'  ]    
				for c in control:
					row = row.replace(c, ' ')
				print "\nFields: "
				endFirstRow = row.find('\x07')
				firstRow = row[:endFirstRow]
				for c in firstRow:
					if curses.ascii.isctrl(c):
						firstRow = firstRow.replace(c, '\n')
				print firstRow
				print "\nRecords:"
				pos = endFirstRow + 1
				while True:
					newrow = row[pos:]
					endRow = newrow.find('\x07')
					if endRow < 0:
						break
					fixedRow = newrow[:endRow]
					pos = pos + endRow + 1
					fixedRow = fixedRow.replace('\n', '')
					print fixedRow
		test_result = "OK"
#
# chek test_result here
# test_result = OK - one of the load balance proxies success
# test_result = Failed - all squids failed
#
Dong Liang's avatar
Dong Liang committed
425
if test_result == "OK" and ever_failed == "False":
Dong Liang's avatar
Dong Liang committed
426
	print "OK"
Dong Liang's avatar
Dong Liang committed
427
	sys.exit(same_ok)
Dong Liang's avatar
Dong Liang committed
428
if test_result == "OK" and ever_failed == "True":
Dong Liang's avatar
Dong Liang committed
429
430
431
432
	print "test_squid.py: One of the load balance Squid proxies " + test_result
	sys.exit(same_warning)
print "test_squid.py: All squid(s) failed"
sys.exit(same_error)