Skip to content
Snippets Groups Projects
Commit 41017eae authored by Chris Burr's avatar Chris Burr
Browse files

Merge branch 'cherry-pick-2-b571608e-devel' into 'devel'

Sweeping !980 from master to devel. [master] Stop using XM0LRPC for getting list of application platforms

See merge request lhcb-dirac/LHCbDIRAC!987
parents ede2b16b eccff2ef
No related branches found
No related tags found
No related merge requests found
......@@ -16,7 +16,6 @@ import json
import ssl
import LbPlatformUtils
from six.moves import xmlrpc_client
from DIRAC import S_OK, S_ERROR, gLogger
import DIRAC.ConfigurationSystem.Client.Helpers.Resources
......@@ -27,12 +26,16 @@ try:
except NameError:
FileNotFoundError = IOError
try:
from json import JSONDecodeError
except ImportError:
JSONDecodeError = ValueError
getQueues = DIRAC.ConfigurationSystem.Client.Helpers.Resources.getQueues
getDIRACPlatforms = DIRAC.ConfigurationSystem.Client.Helpers.Resources.getDIRACPlatforms
getCompatiblePlatforms = DIRAC.ConfigurationSystem.Client.Helpers.Resources.getCompatiblePlatforms
DEFAULT_XMLRPCURL = "https://lbsoftdb.cern.ch/read/"
DEFAULT_FALLBACKCACHEPATH = '/cvmfs/lhcb.cern.ch/lib/var/lib/softmetadata/project-platforms.json'
DEFAULT_CACHEPATH = '/cvmfs/lhcb.cern.ch/lib/var/lib/softmetadata/project-platforms.json'
def getDIRACPlatform(platform):
......@@ -118,7 +121,7 @@ def _findBinaryTags(wf):
continue
platforms = _listPlatforms(applicationName.value, applicationVersion.value,
DEFAULT_XMLRPCURL, DEFAULT_FALLBACKCACHEPATH)
DEFAULT_CACHEPATH)
if platforms:
binaryTags.add(frozenset(platforms))
......@@ -126,7 +129,7 @@ def _findBinaryTags(wf):
return binaryTags
def _listPlatforms(applicationName, applicationVersion, xmlrpcUrl, fallbackPath):
def _listPlatforms(applicationName, applicationVersion, cachePath):
""" developer function
:returns: set of binary tags found for a given application and version
"""
......@@ -134,28 +137,16 @@ def _listPlatforms(applicationName, applicationVersion, xmlrpcUrl, fallbackPath)
applicationVersion = applicationVersion.lower()
platforms = None
context = ssl.create_default_context(capath=getCAsLocation())
proxy = xmlrpc_client.ServerProxy(xmlrpcUrl, allow_none=True, context=context)
try:
platforms = proxy.listPlatforms(applicationName, applicationVersion)
except xmlrpc_client.Fault as e:
gLogger.error("Failed to find platform in SoftConfDB for", "%s/%s %s" %
(applicationName, applicationVersion, e))
except Exception as e:
gLogger.error("Unknown exception when querying SoftConfDB", repr(e))
# If the XML RPC endpoint is down, try to use the cache on CVMFS
if platforms is None:
with open(cachePath, 'rt') as fp:
projectsMetadata = json.load(fp)
except (FileNotFoundError, JSONDecodeError):
gLogger.error("SoftConfDB JSON cache not found in", cachePath)
else:
try:
with open(fallbackPath, 'rt') as fp:
fallbackCache = json.load(fp)
except FileNotFoundError:
gLogger.error("SoftConfDB JSON cache not found in", fallbackPath)
else:
try:
platforms = fallbackCache[applicationName][applicationVersion]
except KeyError as e:
gLogger.error("Failed to find platform in cache for", "%s/%s in %s (%s)" %
(applicationName, applicationVersion, fallbackPath, e))
platforms = projectsMetadata[applicationName][applicationVersion]
except KeyError as e:
gLogger.error("Failed to find platform in cache for", "%s/%s in %s (%s)" %
(applicationName, applicationVersion, cachePath, e))
return platforms
......@@ -121,42 +121,14 @@ def test_listPlatforms(applicationName, applicationVersion, expected):
if not os.path.isdir('/cvmfs/lhcb.cern.ch'):
pytest.skip('CVMFS is required')
from six.moves import xmlrpc_client
import ssl
context = ssl.create_default_context(capath=getCAsLocation())
try:
xmlrpc_client.ServerProxy(
'https://lbsoftdb.cern.ch/read/',
allow_none=True,
context=context,
).listApplications()
except ssl.SSLError:
pytest.skip('CERN certificate authority must be trusted')
# Good RPC and good fallback
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
moduleTested.DEFAULT_XMLRPCURL,
moduleTested.DEFAULT_FALLBACKCACHEPATH)
assert result is None is expected or set(result) == set(expected)
# Bad RPC and good fallback
# Good cache path
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
'https://lbsoftdb.cern.invalid/read/',
moduleTested.DEFAULT_FALLBACKCACHEPATH)
assert result is None is expected or set(result) == set(expected)
# Good RPC and bad fallback
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
moduleTested.DEFAULT_XMLRPCURL,
'/cvmfs/lhcb.cern.invalid/lib/var/lib/softmetadata/project-platforms.json')
moduleTested.DEFAULT_CACHEPATH)
assert result is None is expected or set(result) == set(expected)
# Bad RPC and bad fallback
# Invalid cache path
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
'https://lbsoftdb.cern.invalid/read/',
'/cvmfs/lhcb.cern.invalid/lib/var/lib/softmetadata/project-platforms.json')
assert result is None
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment