Skip to content
Snippets Groups Projects
Commit b571608e authored by Marko Petric's avatar Marko Petric
Browse files

Merge branch 'remove-xmlrpc' into 'master'

[master] Stop using XM0LRPC for getting list of application platforms

See merge request lhcb-dirac/LHCbDIRAC!980
parents 5ce305e8 af1a0238
No related branches found
No related tags found
No related merge requests found
......@@ -12,7 +12,6 @@
import json
import LbPlatformUtils
from six.moves import xmlrpc_client
from DIRAC import S_OK, S_ERROR, gLogger
import DIRAC.ConfigurationSystem.Client.Helpers.Resources
......@@ -22,12 +21,16 @@ try:
except NameError:
FileNotFoundError = IOError
try:
from json import JSONDecodeError
except ImportError:
JSONDecodeError = ValueError
getQueues = DIRAC.ConfigurationSystem.Client.Helpers.Resources.getQueues
getDIRACPlatforms = DIRAC.ConfigurationSystem.Client.Helpers.Resources.getDIRACPlatforms
getCompatiblePlatforms = DIRAC.ConfigurationSystem.Client.Helpers.Resources.getCompatiblePlatforms
DEFAULT_XMLRPCURL = "https://lbsoftdb.cern.ch/read/"
DEFAULT_FALLBACKCACHEPATH = '/cvmfs/lhcb.cern.ch/lib/var/lib/softmetadata/project-platforms.json'
DEFAULT_CACHEPATH = '/cvmfs/lhcb.cern.ch/lib/var/lib/softmetadata/project-platforms.json'
def getDIRACPlatform(platform):
......@@ -113,7 +116,7 @@ def _findBinaryTags(wf):
continue
platforms = _listPlatforms(applicationName.value, applicationVersion.value,
DEFAULT_XMLRPCURL, DEFAULT_FALLBACKCACHEPATH)
DEFAULT_CACHEPATH)
if platforms:
binaryTags.add(frozenset(platforms))
......@@ -121,7 +124,7 @@ def _findBinaryTags(wf):
return binaryTags
def _listPlatforms(applicationName, applicationVersion, xmlrpcUrl, fallbackPath):
def _listPlatforms(applicationName, applicationVersion, cachePath):
""" developer function
:returns: set of binary tags found for a given application and version
"""
......@@ -129,27 +132,16 @@ def _listPlatforms(applicationName, applicationVersion, xmlrpcUrl, fallbackPath)
applicationVersion = applicationVersion.lower()
platforms = None
proxy = xmlrpc_client.ServerProxy(xmlrpcUrl, allow_none=True)
try:
platforms = proxy.listPlatforms(applicationName, applicationVersion)
except xmlrpc_client.Fault as e:
gLogger.error("Failed to find platform in SoftConfDB for", "%s/%s %s" %
(applicationName, applicationVersion, e))
except Exception as e:
gLogger.error("Unknown exception when querying SoftConfDB", repr(e))
# If the XML RPC endpoint is down, try to use the cache on CVMFS
if platforms is None:
with open(cachePath, 'rt') as fp:
projectsMetadata = json.load(fp)
except (FileNotFoundError, JSONDecodeError):
gLogger.error("SoftConfDB JSON cache not found in", cachePath)
else:
try:
with open(fallbackPath, 'rt') as fp:
fallbackCache = json.load(fp)
except FileNotFoundError:
gLogger.error("SoftConfDB JSON cache not found in", fallbackPath)
else:
try:
platforms = fallbackCache[applicationName][applicationVersion]
except KeyError as e:
gLogger.error("Failed to find platform in cache for", "%s/%s in %s (%s)" %
(applicationName, applicationVersion, fallbackPath, e))
platforms = projectsMetadata[applicationName][applicationVersion]
except KeyError as e:
gLogger.error("Failed to find platform in cache for", "%s/%s in %s (%s)" %
(applicationName, applicationVersion, cachePath, e))
return platforms
......@@ -117,37 +117,14 @@ def test_listPlatforms(applicationName, applicationVersion, expected):
if not os.path.isdir('/cvmfs/lhcb.cern.ch'):
pytest.skip('CVMFS is required')
from six.moves import xmlrpc_client
import ssl
try:
xmlrpc_client.ServerProxy('https://lbsoftdb.cern.ch/read/', allow_none=True).listApplications()
except ssl.SSLError:
pytest.skip('CERN certificate authority must be trusted')
# Good RPC and good fallback
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
moduleTested.DEFAULT_XMLRPCURL,
moduleTested.DEFAULT_FALLBACKCACHEPATH)
assert result is None is expected or set(result) == set(expected)
# Bad RPC and good fallback
# Good cache path
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
'https://lbsoftdb.cern.invalid/read/',
moduleTested.DEFAULT_FALLBACKCACHEPATH)
assert result is None is expected or set(result) == set(expected)
# Good RPC and bad fallback
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
moduleTested.DEFAULT_XMLRPCURL,
'/cvmfs/lhcb.cern.invalid/lib/var/lib/softmetadata/project-platforms.json')
moduleTested.DEFAULT_CACHEPATH)
assert result is None is expected or set(result) == set(expected)
# Bad RPC and bad fallback
# Invalid cache path
result = moduleTested._listPlatforms(
applicationName, applicationVersion,
'https://lbsoftdb.cern.invalid/read/',
'/cvmfs/lhcb.cern.invalid/lib/var/lib/softmetadata/project-platforms.json')
assert result is None
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment