Merge branch 'bcouturi_check_individual_files' into 'master'

Allow checking individual files and change nightlies JSON

See merge request !7

.gitlab-ci.yml

@@ -53,7 +53,7 @@ generate_stacks:
       - to_release.json
     expire_in: 1 week
   script:
-    - lb-stacks-to-release ./data/stacks /cvmfs/lhcb.cern.ch/lib/ | tee to_release.json
+    - lb-stacks-to-release /cvmfs/lhcb.cern.ch/lib/ ./data/stacks | tee to_release.json
 
 generate_project_list:
   extends: .setup_environment

src/lhcbstacks/__init__.py

@@ -31,60 +31,55 @@ STACK_SCHEMA = Map(
 )
 
 
-def load_stack_info(repository, name):
+def stack_filename(repository, name):
+    """ Returns the file name for the stack """
+    return Path(repository) / f"{name}.yml"
+
+
+def _load_stack_file(filename):
     """ Load stack information by name from the specified repository """
-    filename = Path(repository) / f"{name}.yml"
     with open(filename) as file:
         info = load(file.read(), STACK_SCHEMA)
         return info.data
 
 
-def load_stack(repository, siteroot, name):
-    """ Factory method returning a stack object of the correct type """
-    info = load_stack_info(repository, name)
-    stack = None
-    stack_type = info["type"].lower()
-    if stack_type == "lhcbcmake":
-        stack = LHCbClassicStack(siteroot, info, "cmake")
-    elif stack_type == "cmt":
-        stack = LHCbClassicStack(siteroot, info, "cmt")
-    else:
-        raise NotImplementedError(f"Stack type {info['type']} not implemented")
-    return stack
-
-
-def list_stacks(repository):
-    """ Returns the list of stack objects as  (name, path) """
-    for path, _, files in os.walk(repository):
+def _list_stacks_in_dir(dirname):
+    """ Returns the list of stack objects as (name, path) in the directory specified"""
+    for path, _, files in os.walk(dirname):
         for f in files:
             if f.endswith(".yml"):
                 name = f[: -len(".yml")]
                 yield (name, Path(path) / f)
 
 
-def incomplete_stacks(repository, siteroot):
+def incomplete_stacks(stack_paths, siteroot):
     """ List all the projects in all the defined stacks in the yaml repository """
     inc_stacks = []
-    for name, _ in list_stacks(repository):
-        s = load_stack(repository, siteroot, name)
-        missing = s.missing_projects()
-        if len(missing) > 0:
-            inc_stacks.append(s)
+    for path in stack_paths:
+        if os.path.isdir(path):
+            for _, filename in _list_stacks_in_dir(path):
+                s = Stack.load(filename, siteroot)
+                if s.has_missing_projects():
+                    inc_stacks.append(s)
+        else:
+            s = Stack.load(path, siteroot)
+            if s.has_missing_projects():
+                inc_stacks.append(s)
+
     return inc_stacks
 
 
-def list_stacks_to_release(repository, siteroot):
+def list_stacks_to_release(stack_paths, siteroot):
     """ Generate the JSON file with the stacks that should be released """
-    # 1. Get the diff between what should be released, and what is in the repository
-    inc_stacks = incomplete_stacks(repository, siteroot)
+    # 1. Get the list of stacks with missing projects
+    inc_stacks = incomplete_stacks(stack_paths, siteroot)
 
     # 2. Iterate to generate the stack in the exported format
     tobuild = []
     for stack in inc_stacks:
         for platform, project_list in stack.builds_to_do():
-
             # First checking if another build has the same projects exactly
-            # Then add platform
+            # Then add platform to it instead of creating a new build
             found_matching = False
             for d, s in tobuild:
                 if set(project_list) == s:
@@ -95,10 +90,19 @@ def list_stacks_to_release(repository, siteroot):
             if found_matching:
                 continue
 
+            # No match, so here we prepare the document listing the work
+            # project list is the list to build
+            # In this file for the nightlies we list the whole stack config
+            # and the one already done (so the complement of the info we have)
             data = dict()
             data["build_tool"] = stack.build_tool()
             data["platforms"] = [platform]
-            data["projects"] = project_list
+            data["projects"] = stack.projects(with_toolchain=True)
+            data["deployed"] = {
+                p: v
+                for p, v in stack.projects(with_toolchain=True).items()
+                if (p, v) not in project_list
+            }
             data["name"] = stack.name()
             tobuild.append((data, set(project_list)))
 
@@ -106,6 +110,20 @@ def list_stacks_to_release(repository, siteroot):
 
 
 class Stack:
+    @classmethod
+    def load(cls, filename, siteroot):
+        """ Factory method returning a stack object of the correct type """
+        info = _load_stack_file(filename)
+        stack = None
+        stack_type = info["type"].lower()
+        if stack_type == "lhcbcmake":
+            stack = LHCbClassicStack(siteroot, info, "cmake")
+        elif stack_type == "cmt":
+            stack = LHCbClassicStack(siteroot, info, "cmt")
+        else:
+            raise NotImplementedError(f"Stack type {info['type']} not implemented")
+        return stack
+
     def __init__(self, siteroot, stack_info):
         self.siteroot_path = Path(siteroot)
         self.info = stack_info
@@ -117,6 +135,13 @@ class Stack:
     def platforms(self):
         return self.info["platforms"]
 
+    def projects(self, with_toolchain=False):
+        tmp = self.info["projects"]
+        if with_toolchain and "gaudi" in [p.lower() for p in tmp]:
+            tctype, tcversion = self.toolchain()
+            tmp[tctype] = tcversion
+        return tmp
+
     def name(self):
         return self.info["name"]
 
@@ -129,7 +154,11 @@ class Stack:
                     missing[binary_tag].append((p, project, version))
         return missing
 
-    def get_toolchain(self):
+    def has_missing_projects(self):
+        """ return true if some projects are missing """
+        return len(self.missing_projects()) > 0
+
+    def toolchain(self):
         """ Accessor to the toolchain information """
         toolchain = self.info["toolchain"]
         return (toolchain["type"], str(toolchain["version"]))
@@ -177,10 +206,6 @@ class LHCbClassicStack(Stack):
         todo = []
         for platform, project_list in missing.items():
             tmplist = [(p, v) for (_, p, v) in project_list]
-            # Unfortunately we need to Hardcode this as we not have the
-            # exact dependencies at this stage
-            if "gaudi" in [p.lower() for p, v in tmplist]:
-                tmplist += [self.get_toolchain()]
             todo.append((platform, tmplist))
         return todo
 

src/lhcbstacks/cli.py

@@ -16,10 +16,10 @@ import lhcbstacks
 
 def stacks_to_release_json():
     parser = argparse.ArgumentParser()
-    parser.add_argument("stack_dir")
     parser.add_argument("siteroot")
+    parser.add_argument("stack_paths", nargs="*")
     args = parser.parse_args()
-    release = lhcbstacks.list_stacks_to_release(args.stack_dir, args.siteroot)
+    release = lhcbstacks.list_stacks_to_release(args.stack_paths, args.siteroot)
     print(json.dumps(release, indent=4, sort_keys=True))
 
 

tests/data/stacks/CMTStack.yml

@@ -12,6 +12,7 @@ name: CMTStack
 type: cmt
 
 projects:
+  Gaudi: v1r1
   LHCb: v1r1
 
 toolchain:

tests/data/stacks/Gaudi_v35r2.yml

+###############################################################################
+# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+name: Gaudi_v35r2
+type: lhcbcmake
+
+projects:
+  Gaudi: v35r2
+
+toolchain:
+  type: LCG
+  version: 97a
+
+platforms:
+  - x86_64+avx2+fma-centos7-gcc9-opt
+  - x86_64-centos7-clang8-dbg
+  - x86_64-centos7-clang8-opt
+  - x86_64-centos7-gcc9-dbg
+  - x86_64-centos7-gcc9-do0
+  - x86_64-centos7-gcc9-opt
+
+tags:
+  - Run3

tests/test_load.py

@@ -37,7 +37,9 @@ TEST_PLATFORM = "x86_64-centos7-gcc9-opt"
 def lhcb_install_area(tmp_path):
     """ Create a fake install with a missing platform and missing projects """
     lhcb_dir = tmp_path
-    s = lhcbstacks.load_stack(STACK_DIR, lhcb_dir, TEST_STACK)
+    s = lhcbstacks.Stack.load(
+        lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), lhcb_dir
+    )
 
     binary_paths = s.binary_paths()
     binary_paths.pop("x86_64+avx2+fma-centos7-gcc9-opt")
@@ -53,12 +55,16 @@ def lhcb_install_area(tmp_path):
 
 
 def test_load_info():
-    stack_info = lhcbstacks.load_stack_info(STACK_DIR, TEST_STACK)
+    stack_info = lhcbstacks._load_stack_file(
+        lhcbstacks.stack_filename(STACK_DIR, TEST_STACK)
+    )
     assert stack_info["type"] == "lhcbcmake"
 
 
 def test_load_stack():
-    a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
+    a = lhcbstacks.Stack.load(
+        lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
+    )
     ps = [p for (p, _, _) in a.project_paths()]
     dv = [p for p in ps if TEST_PROJECT.upper() in str(p)]
     assert (
@@ -68,17 +74,21 @@ def test_load_stack():
 
 
 def test_list_platforms():
-    a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
+    a = lhcbstacks.Stack.load(
+        lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
+    )
     assert a.platforms() == PLATFORMS
 
 
-def test_list_stacks():
-    stack_list = list(lhcbstacks.list_stacks(STACK_DIR))
-    assert len(stack_list) == 2
+def test_list_stacks_in_dir():
+    stack_list = list(lhcbstacks._list_stacks_in_dir(STACK_DIR))
+    assert len(stack_list) == 3
 
 
 def test_list_binary_paths():
-    a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
+    a = lhcbstacks.Stack.load(
+        lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
+    )
     dv = [p for p in a.binary_paths()[TEST_PLATFORM] if TEST_PROJECT.upper() in str(p)]
     assert (
         str(dv[0][0])
@@ -87,7 +97,9 @@ def test_list_binary_paths():
 
 
 def test_missing(lhcb_install_area):
-    a = lhcbstacks.load_stack(STACK_DIR, lhcb_install_area, TEST_STACK)
+    a = lhcbstacks.Stack.load(
+        lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), lhcb_install_area
+    )
     missing = a.missing_projects()
 
     # We should be missing x86_64+avx2+fma-centos7-gcc9-opt, and DV for x86_64-centos7-clang8-dbg
@@ -97,14 +109,14 @@ def test_missing(lhcb_install_area):
 
 
 def test_incomplete_stacks(lhcb_install_area):
-    inc_stacks = lhcbstacks.incomplete_stacks(STACK_DIR, lhcb_install_area)
-    assert len(inc_stacks) == 2
+    inc_stacks = lhcbstacks.incomplete_stacks([STACK_DIR], lhcb_install_area)
+    assert len(inc_stacks) == 3
 
 
 def test_stacks_to_release(lhcb_install_area):
-    release = lhcbstacks.list_stacks_to_release(STACK_DIR, lhcb_install_area)
+    release = lhcbstacks.list_stacks_to_release([STACK_DIR], lhcb_install_area)
     result_json = json.dumps(release, sort_keys=True)
-    expected = '[{"build_tool": "cmt", "name": "CMTStack", "platforms": ["x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": [["LHCb", "v1r1"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-gcc9-do0"], "projects": [["Analysis", "v32r0"], ["Phys", "v31r1"], ["Rec", "v31r1"], ["Lbcom", "v31r1"], ["LHCb", "v51r1"], ["Gaudi", "v33r2"], ["Detector", "v0r5"], ["DaVinci", "v52r0"], ["LCG", "97a"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64-centos7-clang8-dbg"], "projects": [["DaVinci", "v52r0"]]}]'
+    expected = '[{"build_tool": "cmt", "deployed": {"LCG": "60"}, "name": "CMTStack", "platforms": ["x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": {"Gaudi": "v1r1", "LCG": "60", "LHCb": "v1r1"}}, {"build_tool": "cmake", "deployed": {"LCG": "97a"}, "name": "Gaudi_v35r2", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-clang8-dbg", "x86_64-centos7-clang8-opt", "x86_64-centos7-gcc9-dbg", "x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": {"Gaudi": "v35r2", "LCG": "97a"}}, {"build_tool": "cmake", "deployed": {"LCG": "97a"}, "name": "Run3Analysis202008", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-gcc9-do0"], "projects": {"Analysis": "v32r0", "DaVinci": "v52r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}}, {"build_tool": "cmake", "deployed": {"Analysis": "v32r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}, "name": "Run3Analysis202008", "platforms": ["x86_64-centos7-clang8-dbg"], "projects": {"Analysis": "v32r0", "DaVinci": "v52r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}}]'
     assert result_json == expected