Merge branch 'feature/no-cbng-ping-at-startup' into 'master'

Avoid the need to setup the resolver at Manager instantiation, thereby...

See merge request !48

cmmnbuild_dep_manager/__main__.py

@@ -56,6 +56,7 @@ def configure_parser(parser: argparse.ArgumentParser):
         "install",
         "is_installed",
         "is_registered",
+        "is_resolved",
         "jar_path",
         "jars",
         "list",

cmmnbuild_dep_manager/cmmnbuild_dep_manager.py

-# -*- coding: utf-8 -*-
 '''CommonBuild Dependency Manager
 
 Copyright (c) CERN 2015-2020
@@ -29,6 +28,7 @@ Authors:
 '''
 
 from contextlib import contextmanager
+import functools
 import glob
 import importlib
 import json
@@ -40,6 +40,7 @@ import re
 import shutil
 import signal
 import site
+
 import six
 import zipfile
 
@@ -51,14 +52,27 @@ import entrypoints
 PKG_MODULES_JSON = pathlib.Path(__file__).parent / 'modules.json'
 
 
+def _requires_resolver(fn):
+    """A decorator to ensure that a resolver is setup before the function is run."""
+    @functools.wraps(fn)
+    def wrapped(self, *args, **kwargs):
+        if not getattr(self, '_resolver', None):
+            self._setup_resolver()
+        return fn(self, *args, **kwargs)
+    return wrapped
+
+
 class Manager(object):
     def __init__(self, pkg=None, lvl=None):
+        # Note: There are two patterns being used with Manager:
+        #  1: Manager() - a thing that can be manipulated and resolved explicitly
+        #  2: Manager(pkg=...) - a thing that will be resolved automatically during
+        #                        construction.
         logging.basicConfig()
         self.log = logging.getLogger(__package__)
         if lvl is not None:
             self.log.setLevel(lvl)
 
-        self._setup_resolver()
         needs_installation = pkg is not None and not self.is_installed(pkg)
         if needs_installation:
             # Temporarily set the log level to notify the user that this could
@@ -183,6 +197,7 @@ class Manager(object):
 
         return matches[1], matches[2]
 
+    @_requires_resolver
     def class_doc(self, obj_or_string):
         '''Return URLs of the documentation and source code of a class
 
@@ -409,7 +424,7 @@ class Manager(object):
                 self.log.info('using resolver: {0} ({1})'.format(resolver.__name__, resolver.description))
                 break
 
-    def _find_supported_resolvers(self, module):
+    def _find_supported_resolvers_for_module(self, module):
         supported_resolvers = []
         from .resolver import resolvers
         for resolver in resolvers():
@@ -421,7 +436,45 @@ class Manager(object):
                 pass
         return supported_resolvers
 
+    def _find_supported_resolvers(self):
+        """Given the installed modules, return a list of possible resolvers.
+        """
+        from .resolver import resolvers
+
+        # We start with *all* resolvers, then whittle them down as we see each
+        # of the installed modules.
+        resolvers = set(resolvers())
+        for module in self._load_modules():
+            supported_resolvers = self._find_supported_resolvers_for_module(module)
+            resolvers.intersection_update(supported_resolvers)
+
+        if not resolvers:
+            raise ValueError("This environment is not resolvable - there is no resolver that can resolve all of the dependencies.")
+
+        return resolvers
+
+    def is_resolved(self):
+        """Determine whether the JARs needed for this environment have been resolved.
+        """
+        modules = self._load_modules()
+
+        for module_name, module_version in modules.items():
+            if module_version == '':
+                # An empty version indicates that the module has not been resolved yet
+                # (see docs for _load_modules).
+                return False
+            if module_version != self._module_version(module_name):
+                return False
+        return True
+
+    def _module_version(self, module_name):
+        """Return the version of the package which is installed"""
+        module = importlib.import_module(module_name)
+        return module.__version__
+
+    @_requires_resolver
     def _find_module_info(self, module_name):
+        """Return the version and the Java dependencies for the given package"""
         module = importlib.import_module(module_name)
 
         # Check __***_deps__ exists on the module. For example for the cbng_web
@@ -432,7 +485,7 @@ class Manager(object):
             deps = list(getattr(module, var_name_for_dependencies))
 
         if not deps:
-            supported_resolvers = self._find_supported_resolvers(module)
+            supported_resolvers = self._find_supported_resolvers_for_module(module)
             supported_resolvers_explanation = ''
             if supported_resolvers:
                 supported_resolvers_explanation = '\nResolvers compatible with this module:\n{}'.format(
@@ -453,7 +506,7 @@ class Manager(object):
         modules = self._load_modules()
         for name in args:
             try:
-                version, deps = self._find_module_info(name)
+                version = self._module_version(name)
 
                 if name not in modules or modules[name] != version:
                     modules[name] = ''
@@ -502,11 +555,16 @@ class Manager(object):
         return name in modules.keys()
 
     def is_installed(self, name, version=None):
-        '''Check if module is installed'''
+        """Check if module is installed (registered and resolved).
+
+        If no version is passed then the version that comes from the imported
+        module's ``__version__`` attribute will be used.
+
+        """
         modules = self._load_modules()
         if name in modules:
             if version is None:
-                version, _ = self._find_module_info(name)
+                version = self._module_version(name)
             return modules[name] == version
         return False
 
@@ -514,11 +572,12 @@ class Manager(object):
         '''Returns a list of the currently registered modules'''
         return sorted(self._load_modules().keys())
 
+    @_requires_resolver
     def resolve(self, force_resolver=None):
         '''Resolve dependencies for all registered modules using CBNG'''
         if force_resolver is not None:
             self._setup_resolver(force_resolver)
-        self.log.info('resolving dependencies')
+        self.log.info('Resolving JARs for this environment')
         self.log.debug('lib directory is {0}'.format(self.jar_path()))
 
         all_dependencies = []

cmmnbuild_dep_manager/tests/conftest.py

@@ -3,6 +3,7 @@ import shutil
 import pytest
 
 from cmmnbuild_dep_manager.cmmnbuild_dep_manager import PKG_MODULES_JSON
+import cmmnbuild_dep_manager
 
 
 TMP_PKG_MODULES_JSON = (
@@ -41,3 +42,13 @@ def autoclean_modules_json():
             TMP_PKG_MODULES_JSON.rename(PKG_MODULES_JSON)
         if TMP_LIB_DIR.exists():
             TMP_LIB_DIR.rename(LIB_DIR)
+
+
+@pytest.fixture
+def cleaned_modules():
+    # Clear the modules.json before and after the test
+    if PKG_MODULES_JSON.exists():
+        PKG_MODULES_JSON.unlink()
+    yield
+    if PKG_MODULES_JSON.exists():
+        PKG_MODULES_JSON.unlink()

cmmnbuild_dep_manager/tests/test_Manager__load_modules.py

@@ -120,7 +120,8 @@ def test_pkg_needs_update(caplog):
     assert r == {'fake_cmmnbuild_pkg': ""}
     assert caplog.record_tuples == [
         ('cmmnbuild_dep_manager', logging.WARN,
-         'fake_cmmnbuild_pkg is being updated from 1.2.2 to 1.2.3')]
+         'fake_cmmnbuild_pkg is being updated from 1.2.2 to 1.2.3'),
+    ]
 
 
 def test_pkg_no_update_bad_entrypoint(caplog):
@@ -134,4 +135,5 @@ def test_pkg_no_update_bad_entrypoint(caplog):
     assert r == {'fake_cmmnbuild_pkg': ""}
     assert caplog.record_tuples == [
         ('cmmnbuild_dep_manager', logging.WARN,
-         'fake_cmmnbuild_pkg is being updated from 1.2.3 to 1.2.4')]
+         'fake_cmmnbuild_pkg is being updated from 1.2.3 to 1.2.4'),
+    ]

cmmnbuild_dep_manager/tests/test_cmmnbuild_dep_manager.py

@@ -9,6 +9,7 @@ import pytest
 
 import cmmnbuild_dep_manager as cbdm
 from cmmnbuild_dep_manager.resolver import Resolver
+from .test_Manager__load_modules import tmp_modules_content, FakePkgMgr
 
 
 class SimpleResolver(Resolver):
@@ -58,19 +59,18 @@ def tmp_mod(name, dependencies=None, version=None):
         sys.modules.pop(name)
 
 
-def test_no_such_module(caplog, simple_resolver):
+def test_no_such_module(caplog, simple_resolver, cleaned_modules):
     cbdm.Manager('this_mod_doesnt_exist')
     assert caplog.record_tuples == [
         ("cmmnbuild_dep_manager", logging.INFO,
-         'Package "this_mod_doesnt_exist" is not yet set up - '
-         'installing and resolving JARs'),
+         'Package "this_mod_doesnt_exist" is not yet set up - installing and resolving JARs'),
         ("cmmnbuild_dep_manager", logging.ERROR,
          'this_mod_doesnt_exist not found'),
     ]
 
 
-def test_module_missing_special_resolver_attr(caplog, simple_resolver):
-    with tmp_mod('a_cmmnbuild_test_module'):
+def test_module_missing_special_resolver_attr(caplog, simple_resolver, cleaned_modules):
+    with tmp_mod('a_cmmnbuild_test_module', version='1.2.3'):
         cbdm.Manager('a_cmmnbuild_test_module')
 
     warning = (
@@ -81,7 +81,7 @@ def test_module_missing_special_resolver_attr(caplog, simple_resolver):
     assert warning in caplog.record_tuples
 
 
-def test_module_dependencies_no_version(caplog, simple_resolver):
+def test_module_dependencies_no_version(caplog, simple_resolver, cleaned_modules):
     with tmp_mod('a_cmmnbuild_test_module', dependencies=['j1', 'j2']):
         cbdm.Manager('a_cmmnbuild_test_module')
     warning = (
@@ -194,6 +194,7 @@ def test_imports_removed_on_success(mocked_mgr):
         assert java.util in sys.modules.values()
     assert java.util not in sys.modules.values()
 
+
 def test_imports_removed_on_failure(mocked_mgr):
     pytest.importorskip('jpype', minversion='1.0')
     mgr = mocked_mgr[0]
@@ -204,6 +205,7 @@ def test_imports_removed_on_failure(mocked_mgr):
             raise ValueError()
     assert java.util not in sys.modules.values()
 
+
 def test_imports_are_reentrant(mocked_mgr):
     pytest.importorskip('jpype', minversion='1.0')
     mgr = mocked_mgr[0]
@@ -217,3 +219,33 @@ def test_imports_are_reentrant(mocked_mgr):
         assert java.lang not in sys.modules.values()
     assert java.util not in sys.modules.values()
     assert java.lang not in sys.modules.values()
+
+
+@pytest.fixture
+def no_cbng_web_ping_allowed():
+    with unittest.mock.patch(
+        'cmmnbuild_dep_manager.resolver.cbng_web.CbngWebResolver.is_available',
+        side_effect=RuntimeError('The is_available method was called on the CbngWebResolver')
+    ):
+        yield
+
+
+def test_init_manager_no_ping_cbng_web(no_cbng_web_ping_allowed):
+    # We shouldn't need cbng-web to be up in order to initialise a manager for
+    # an already resolved environment.
+    assert cbdm.Manager()
+
+
+def test_init_manager_no_ping_cbng_web_with_package_name_already_resolved(no_cbng_web_ping_allowed):
+    # We shouldn't need cbng-web to be up in order to initialise a manager for
+    # an already resolved environment.
+    with FakePkgMgr.fake_pkg("fake_cmmnbuild_pkg", "1.2.3", entrypoint="1.2.3"):
+        with tmp_modules_content({'fake_cmmnbuild_pkg': '1.2.3'}):
+            assert cbdm.Manager('fake_cmmnbuild_pkg')
+
+
+def test_resolve_manager_pings_cbng_web(no_cbng_web_ping_allowed):
+    # Validate that resolving does indeed hit the cbng-web endpoint,
+    # and that the no_cbng_web_ping_allowed fixture is doing the right thing.
+    with pytest.raises(RuntimeError):
+        assert cbdm.Manager().resolve()