Merge branch 'feature/generate-lockfile' into 'master'

Add functionality to create and install a lockfile of Java dependencies

See merge request !67

cmmnbuild_dep_manager/__main__.py

@@ -61,6 +61,8 @@ def configure_parser(parser: argparse.ArgumentParser):
         "jars",
         "java_requirements",
         "list",
+        "lockfile_generate",
+        "lockfile_install",
         "register",
         "registered_modules",
         "resolve",

cmmnbuild_dep_manager/_manager.py

@@ -28,6 +28,7 @@ Authors:
 '''
 
 from contextlib import contextmanager
+import dataclasses
 import functools
 import importlib
 import json
@@ -35,6 +36,7 @@ import logging
 import os
 import pathlib
 import pickle
+import pprint
 import re
 import shutil
 import signal
@@ -753,6 +755,131 @@ class Manager:
         # Deploy new jars
         resolver.save_jars(self.jar_path())
 
+    @_requires_resolver
+    def lockfile_generate(self, lockfile_path: typing.Optional[str] = None):
+        """
+        Using all of the registered modules, generate a file containing a resolved
+        set of artifacts for later installation.
+
+        See also :meth:`lockfile_install`.
+
+        Parameters
+        ----------
+
+        lockfile_path:
+            The path of the lockfile to install.
+            Defaults to ``cmmnbuild-dep-manager.lock`` in the CWD.
+
+
+        Note that, unlike :meth:`.resolve` this method can raise in the
+        case of incorrectly installed packages.
+
+        """
+        from .resolver import MultiPhaseResolver
+
+        if not issubclass(self._resolver, MultiPhaseResolver):
+            raise TypeError(
+                f"Only multi-phase resolvers such as CbngWebResolver can be "
+                f"used to generate a lockfile. "
+                f"The resolver enabled is {self._resolver}."
+            )
+
+        if lockfile_path is None:
+            lockfile_path = pathlib.Path.cwd() / 'cmmnbuild-dep-manager.lock'
+        else:
+            lockfile_path = pathlib.Path(lockfile_path)
+
+        self.log.info('Resolving JARs for this environment')
+
+        # Run the resolver, possibly raising if there are any issues with the
+        # modules, or the resolution itself.
+        java_reqs = self.java_requirements()
+        resolver = self._resolver(java_reqs)
+
+        # Get the names and versions of the modules which are being
+        # considered for JARs.
+        mod_versions = {
+            module_name: self._module_version(module_name)
+            for module_name in self.registered_modules()
+        }
+
+        with lockfile_path.open('wt') as fh:
+            json.dump(
+                {
+                    'python_modules': mod_versions,
+                    'java_requirements': java_reqs,
+                    'resolved_artifacts':
+                        [dataclasses.asdict(artifact)
+                         for artifact in resolver.resolved_artifacts],
+                },
+                fh,
+                indent=2,
+            )
+        self.log.info(f'Lockfile created at {lockfile_path}')
+
+    @_requires_resolver
+    def lockfile_install(self, lockfile_path: typing.Optional[str] = None):
+        """
+        Install the lockfile by downloading the necessary JARs and marking the
+        environment as resolved.
+
+        Parameters
+        ----------
+
+        lockfile_path:
+            The path of the lockfile to install.
+            Defaults to ``cmmnbuild-dep-manager.lock`` in the CWD.
+
+        """
+        from .resolver import MultiPhaseResolver, Artifact
+
+        if not issubclass(self._resolver, MultiPhaseResolver):
+            raise TypeError(
+                f"Only multi-phase resolvers such as CbngWebResolver can be "
+                f"used to generate a lockfile. "
+                f"The resolver enabled is {self._resolver}."
+            )
+
+        if lockfile_path is None:
+            lockfile_path = pathlib.Path.cwd() / 'cmmnbuild-dep-manager.lock'
+        else:
+            lockfile_path = pathlib.Path(lockfile_path)
+
+        with lockfile_path.open('rt') as fh:
+            lockfile_contents = json.load(fh)
+
+        artifacts = tuple(
+            Artifact(**artifact) for artifact in lockfile_contents['resolved_artifacts']
+        )
+
+        def to_requirements_set(requirements):
+            return {
+                tuple(req.items()) if isinstance(req, dict) else req
+                for req in requirements
+            }
+
+        # Check that the environment's Java requirements match the ones in the file.
+        # Note that this check does not behave like the main `Manager.resolve() method
+        # as that method checks the *python* modules are the same. At the end of the
+        # day though, all we really care about is satisfying the Java dependencies,
+        # and if the Python module version changes in between locking and installation,
+        # we don't need that to be a concern so long as the Java requirements remain the
+        # same.
+        env_java_reqs = to_requirements_set(self.java_requirements())
+        lockfile_reqs = to_requirements_set(lockfile_contents['java_requirements'])
+        if lockfile_reqs != env_java_reqs:
+            raise ValueError(
+                "The Java requirements from the lockfile do not match those for the environment: \n"
+                f"   Lockfile: \n{pprint.pformat(lockfile_reqs, indent=5)}\n"
+                f"   Environment: \n{pprint.pformat(env_java_reqs, indent=5)}\n"
+            )
+
+        destination = pathlib.Path(self.jar_path())
+        destination.mkdir(exist_ok=True, parents=True)
+
+        self._resolver.download_jars(destination, artifacts)
+        self._save_modules(lockfile_contents['python_modules'])
+
     def stubgen(self, output_dir=None):
         """ Generate python type stubs for accessing Java APIs through the JPype import system (see imports()).
 

cmmnbuild_dep_manager/resolver/__init__.py

 import abc as _abc
+import contextlib
+import dataclasses
+import logging
+import pathlib
 import typing
+import warnings
+
+import requests
+import requests.exceptions
+from requests.packages.urllib3 import exceptions as _r_exceptions
 
 
 def resolvers():
@@ -46,3 +55,58 @@ class Resolver(_abc.ABC):
     @_abc.abstractmethod
     def get_help(cls, classnames, class_info):
         pass
+
+
+@contextlib.contextmanager
+def _requests_no_insecure_warning():
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore", category=_r_exceptions.InsecureRequestWarning,
+        )
+        yield
+
+
+LOG = logging.getLogger(__package__)
+
+
+@dataclasses.dataclass(frozen=True)
+class Artifact:
+    artifact_id: str
+    group_id: str
+    version: str
+    uri: str
+
+    @property
+    def filename_from_uri(self):
+        """Guess the name of the JAR based on the URI."""
+        return self.uri.rsplit('/', 1)[1]
+
+    def download_jar(self, destination: pathlib.Path):
+        """Download the JAR into the given directory.
+        """
+        # Function to download a single jar into a given directory
+        with _requests_no_insecure_warning():
+            jar_request = requests.get(self.uri, stream=True, timeout=20.0, verify=False)
+        LOG.info('downloading {group}:{artifact}:{version}'.format(
+            group=self.group_id, artifact=self.artifact_id, version=self.version
+        ))
+        with (destination / self.filename_from_uri).open("wb") as jar_file:
+            for chunk in jar_request.iter_content(chunk_size=1024):
+                jar_file.write(chunk)
+
+
+class MultiPhaseResolver(Resolver, _abc.ABC):
+    """
+    An abstract Resolver which has a strong separation between the "resolve"
+    and the "fetch/download" phases.
+
+    """
+    resolved_artifacts: typing.Tuple[Artifact, ...]
+
+    @_abc.abstractclassmethod
+    def download_jars(cls, destination: pathlib.Path, artifacts: typing.Tuple[Artifact, ...]) -> None:
+        """Download the given :class:`Artifact` instances into the given
+           directory, removing any pre-existing JARs in the directory if necessary.
+
+        """
+        pass

cmmnbuild_dep_manager/resolver/cbng_web.py

 import concurrent.futures
-import contextlib
-import dataclasses
 import logging
 import pathlib
 import platform
 import sys
 import typing
-import warnings
 import xml.etree.ElementTree as ET
 
 import requests
 import requests.exceptions
-from requests.packages.urllib3 import exceptions as _r_exceptions
-
-from . import Resolver
-
-
-@contextlib.contextmanager
-def _requests_no_insecure_warning():
-    with warnings.catch_warnings():
-        warnings.filterwarnings(
-            "ignore", category=_r_exceptions.InsecureRequestWarning,
-        )
-        yield
+from .. import resolver
 
 
 LOG = logging.getLogger(__package__)
 
 
-@dataclasses.dataclass
-class Artifact:
-    artifact_id: str
-    group_id: str
-    version: str
-    uri: str
-
-    @property
-    def filename_from_uri(self):
-        """Guess the name of the JAR based on the URI."""
-        return self.uri.rsplit('/', 1)[1]
-
-    def download_jar(self, destination: pathlib.Path):
-        """Download the JAR into the given directory.
-        """
-        # Function to download a single jar into a given directory
-        with _requests_no_insecure_warning():
-            jar_request = requests.get(self.uri, stream=True, timeout=20.0, verify=False)
-        LOG.info('downloading {group}:{artifact}:{version}'.format(
-            group=self.group_id, artifact=self.artifact_id, version=self.version
-        ))
-        with (destination / self.filename_from_uri).open("wb") as jar_file:
-            for chunk in jar_request.iter_content(chunk_size=1024):
-                jar_file.write(chunk)
-
-
-class CbngWebResolver(Resolver):
+class CbngWebResolver(resolver.MultiPhaseResolver):
     dependency_variable = '__cmmnbuild_deps__'
     description = 'CBNG Web Service'
     CBNG_WEB_ENDPOINT = 'https://acc-py-repo.cern.ch/cbng-web-resolver/resolve/productxml'
@@ -63,7 +23,7 @@ class CbngWebResolver(Resolver):
     def is_available(cls):
         # check if CBNG web service is reachable
         try:
-            with _requests_no_insecure_warning():
+            with resolver._requests_no_insecure_warning():
                 requests.get(cls.CBNG_WEB_ENDPOINT, timeout=5.0, verify=False)
             return True
         except requests.exceptions.RequestException:
@@ -86,7 +46,7 @@ class CbngWebResolver(Resolver):
         # Post product.xml to CBNG web service
         LOG.info('resolving dependencies using CBNG web service')
 
-        with _requests_no_insecure_warning():
+        with resolver._requests_no_insecure_warning():
             response = requests.post(
                 self.CBNG_WEB_ENDPOINT, data=ET.tostring(pxml), verify=False,
             )
@@ -103,7 +63,7 @@ class CbngWebResolver(Resolver):
     def _resolved_artifacts_from_response(
             cls,
             resolved_deps_response: typing.List[typing.Dict[str, str]]
-    ) -> typing.Tuple[Artifact, ...]:
+    ) -> typing.Tuple[resolver.Artifact, ...]:
         result = []
         expected_keys = {'artifactId', 'groupId', 'version', 'uri'}
         for dependency in resolved_deps_response:
@@ -112,7 +72,7 @@ class CbngWebResolver(Resolver):
                 raise ValueError(
                     f"Fields {', '.join(missing_keys)} missing from response"
                 )
-            result.append(Artifact(
+            result.append(resolver.Artifact(
                 artifact_id=dependency['artifactId'],
                 group_id=dependency['groupId'],
                 version=dependency['version'],
@@ -157,11 +117,8 @@ class CbngWebResolver(Resolver):
         self.download_jars(pathlib.Path(dir), self.resolved_artifacts)
 
     @classmethod
-    def download_jars(cls, destination: pathlib.Path, artifacts: typing.Tuple[Artifact, ...]):
-        """Download the given :class:`Artifact` instances into the given
-        directory, removing any pre-existing JARs in the directory if necessary.
-
-        """
+    def download_jars(cls, destination: pathlib.Path,
+                      artifacts: typing.Tuple[resolver.Artifact, ...]):
         existing_jars = {jarfile.name for jarfile in destination.glob('*.jar')}
         desired_jars = {artifact.filename_from_uri for artifact in artifacts}
 

cmmnbuild_dep_manager/resolver/gradle.py

@@ -9,10 +9,10 @@ import subprocess
 import tempfile
 import zipfile
 
-from . import Resolver
+from .. import resolver
 
 
-class GradleResolver(Resolver):
+class GradleResolver(resolver.Resolver):
     dependency_variable = '__gradle_deps__'
     description = 'Pure Gradle - when the CERN CBNG is not available'
 

cmmnbuild_dep_manager/tests/integration/test_Manager_lockfile.py

+import json
+from pathlib import Path
+from unittest import mock
+
+import pytest
+
+from .test_cbng_web_service_resolver import resolver_w_2_artifacts, mocked_artifact_download # noqa
+from ..test_cmmnbuild_dep_manager import mod_w_simple_deps, mod3_w_groupid_deps  # noqa
+
+
+@pytest.fixture
+def manager_w_jar_deps(manager_cls, mod_w_simple_deps, mod3_w_groupid_deps):
+    mgr = manager_cls()
+
+    with mod_w_simple_deps as mod, mod3_w_groupid_deps as mod2:
+        mgr.register('mod_w_simple_deps', 'mod3_w_groupid_deps')
+        mod.__cmmnbuild_deps__ = mod.__test_resolver_deps__
+        mod2.__cmmnbuild_deps__ = mod2.__test_resolver_deps__
+
+        yield mgr
+
+
+def test_empty_environment(manager_w_jar_deps, resolver_w_2_artifacts, tmpdir):
+    mgr = manager_w_jar_deps
+
+    mgr._resolver = resolver_w_2_artifacts
+
+    with mock.patch('pathlib.Path.cwd', return_value=Path(tmpdir)):
+        mgr.lockfile_generate()
+
+    lockfile = tmpdir / 'cmmnbuild-dep-manager.lock'
+    assert lockfile.exists()
+
+    with lockfile.open('rt') as fh:
+        result = json.load(fh)
+
+    expected = {
+        "python_modules": {
+            'mod3_w_groupid_deps': '1.2.3', 'mod_w_simple_deps': '1.2.3'
+        },
+        "java_requirements": [
+            'java_dep1', {'artifactId': 'java_dep2', 'groupId': 'something.else'},
+            'java_dep1', 'java_dep2',
+        ],
+        "resolved_artifacts": [
+            {
+                "artifact_id": "",
+                "group_id": "",
+                "version": "",
+                "uri": "www.some.where/group/atr/todownload.jar"
+            },
+            {
+                "artifact_id": "",
+                "group_id": "",
+                "version": "",
+                "uri": "www.some.where/group/atr/existing.jar"
+            }
+        ]
+    }
+    assert expected == result
+    with mock.patch('pathlib.Path.cwd', return_value=Path(tmpdir)):
+        mgr.lockfile_install()
+
+    assert mgr.is_resolved()
+
+    assert mgr.jars() == [
+        f'{mgr.jar_path()}/{fname}' for fname in ['existing.jar', 'todownload.jar']
+    ]
+
+
+def test_jar_requirements_changed(manager_w_jar_deps, resolver_w_2_artifacts, tmpdir):
+    mgr = manager_w_jar_deps
+    mgr._resolver = resolver_w_2_artifacts
+
+    with mock.patch('pathlib.Path.cwd', return_value=Path(tmpdir)):
+        mgr.lockfile_generate()
+
+    # If we unregister one of the modules which made up the lockfile, we
+    # shouldn't be allowed to install the lockfile.
+    mgr.unregister('mod_w_simple_deps')
+
+    with mock.patch('pathlib.Path.cwd', return_value=Path(tmpdir)):
+        with pytest.raises(ValueError, match="The Java requirements from the lockfile do not match"):
+            mgr.lockfile_install()
+
+    assert not mgr.is_resolved()

cmmnbuild_dep_manager/tests/integration/test_cbng_web_service_resolver.py

-import copy
 import pathlib
 import typing
 
+from unittest import mock
+
 import pytest
 
-from cmmnbuild_dep_manager.resolver.cbng_web import CbngWebResolver, Artifact
+import cmmnbuild_dep_manager.resolver as resolver_mod
+import cmmnbuild_dep_manager.resolver.cbng_web as cbng_web
+
+
+def mock_resolver_cls(
+        resolved_artifacts: typing.List[resolver_mod.Artifact],
+) -> typing.Type[cbng_web.CbngWebResolver]:
+
+    class TestCbngWebResolver(cbng_web.CbngWebResolver):
+        def __init__(self, *args, **kwargs):
+            self.resolved_artifacts = resolved_artifacts
+            self.init_args = args
+            self.init_kwargs = kwargs
 
+    return TestCbngWebResolver
 
-def mock_resolver(resolved_artifacts: typing.List[Artifact]):
-    class TestArtifact(Artifact):
+
+@pytest.fixture
+def mocked_artifact_download():
+    class TestArtifact(resolver_mod.Artifact):
         def download_jar(self, destination: pathlib.Path):
             jar_filename = self.filename_from_uri
             create_mock_jar_file(destination, jar_filename)
 
-    artifacts = ()
-    for artifact in copy.deepcopy(resolved_artifacts):
-        # Fiddle with the given artifacts such that they are using our test infra
-        # and don't really download stuff unintentionally.
-        artifact.__class__ = TestArtifact
-        artifacts += (artifact, )
-
-    class TestCbngWebResolver(CbngWebResolver):
-        def __init__(self):
-            self.resolved_artifacts = artifacts
-
-    return TestCbngWebResolver()
+    with mock.patch('cmmnbuild_dep_manager.resolver.Artifact', TestArtifact):
+        yield
 
 
 @pytest.fixture
-def resolver_w_2_artifacts():
+def resolver_w_2_artifacts(mocked_artifact_download):
     unused_kwargs = {'artifact_id': '', 'group_id': '', 'version': ''}
-    resolver = mock_resolver([
-        Artifact(uri='www.some.where/group/atr/todownload.jar', **unused_kwargs),
-        Artifact(uri='www.some.where/group/atr/existing.jar', **unused_kwargs),
+    resolver = mock_resolver_cls([
+        resolver_mod.Artifact(uri='www.some.where/group/atr/todownload.jar', **unused_kwargs),
+        resolver_mod.Artifact(uri='www.some.where/group/atr/existing.jar', **unused_kwargs),
     ])
-    return resolver
+    yield resolver
 
 
 def create_mock_jar_file(lib_dir, filename, content=""):
@@ -43,7 +49,7 @@ def create_mock_jar_file(lib_dir, filename, content=""):
 
 
 def test_resolve_and_download(tmp_lib_dir, local_cbng_web_endpoint):
-    resolver = CbngWebResolver(['japc'])
+    resolver = cbng_web.CbngWebResolver(['japc'])
     resolver.save_jars(tmp_lib_dir)
 
     for dep in resolver.resolved_artifacts:
@@ -54,23 +60,23 @@ def test_resolve_and_download(tmp_lib_dir, local_cbng_web_endpoint):
 
 
 def test_resolver_download_all_jars_when_libs_empty(tmp_lib_dir, resolver_w_2_artifacts):
-    resolver_w_2_artifacts.save_jars(tmp_lib_dir)
+    resolver_w_2_artifacts().save_jars(tmp_lib_dir)
     assert (tmp_lib_dir / "todownload.jar").exists()
 
 
 def test_resolver_download_only_missing_jars(tmp_lib_dir, resolver_w_2_artifacts):
     create_mock_jar_file(tmp_lib_dir, "existing.jar", content="jar_content")
-    resolver_w_2_artifacts.save_jars(tmp_lib_dir)
+    resolver_w_2_artifacts().save_jars(tmp_lib_dir)
     # Check previously existing file content wasn't edited
     assert (tmp_lib_dir/"existing.jar").read_text() == 'jar_content'
 
 
 def test_resolver_removes_unnecessary_jars(tmp_lib_dir, resolver_w_2_artifacts):
     create_mock_jar_file(tmp_lib_dir, "unwanted.jar")
-    resolver_w_2_artifacts.save_jars(tmp_lib_dir)
+    resolver_w_2_artifacts().save_jars(tmp_lib_dir)
     assert not (tmp_lib_dir / "unwanted.jar").exists()
 
 
 def test_resolved_artifacts__filename_from_uri():
-    artifact = Artifact('pkg', 'group-id', '321.123.55+1', 'https://uri/thing.jar')
+    artifact = resolver_mod.Artifact('pkg', 'group-id', '321.123.55+1', 'https://uri/thing.jar')
     assert artifact.filename_from_uri == 'thing.jar'