Added full list of stacks and list of the ones already deployed

src/lhcbstacks/__init__.py

@@ -62,7 +62,7 @@ def incomplete_stacks(stack_paths, siteroot):
                 if s.has_missing_projects():
                     inc_stacks.append(s)
         else:
-            s = Stack.load(filename, siteroot)
+            s = Stack.load(path, siteroot)
             if s.has_missing_projects():
                 inc_stacks.append(s)
 
@@ -71,16 +71,15 @@ def incomplete_stacks(stack_paths, siteroot):
 
 def list_stacks_to_release(stack_paths, siteroot):
     """ Generate the JSON file with the stacks that should be released """
-    # 1. Get the list of incomplete stacks
+    # 1. Get the list of stacks with missing projects
     inc_stacks = incomplete_stacks(stack_paths, siteroot)
 
     # 2. Iterate to generate the stack in the exported format
     tobuild = []
     for stack in inc_stacks:
         for platform, project_list in stack.builds_to_do():
-
             # First checking if another build has the same projects exactly
-            # Then add platform
+            # Then add platform to it instead of creating a new build
             found_matching = False
             for d, s in tobuild:
                 if set(project_list) == s:
@@ -91,10 +90,19 @@ def list_stacks_to_release(stack_paths, siteroot):
             if found_matching:
                 continue
 
+            # No match, so here we prepare the document listing the work
+            # project list is the list to build
+            # In this file for the nightlies we list the whole stack config
+            # and the one already done (so the complement of the info we have)
             data = dict()
             data["build_tool"] = stack.build_tool()
             data["platforms"] = [platform]
-            data["projects"] = project_list
+            data["projects"] = stack.projects(with_toolchain=True)
+            data["deployed"] = {
+                p: v
+                for p, v in stack.projects(with_toolchain=True).items()
+                if (p, v) not in project_list
+            }
             data["name"] = stack.name()
             tobuild.append((data, set(project_list)))
 
@@ -127,6 +135,13 @@ class Stack:
     def platforms(self):
         return self.info["platforms"]
 
+    def projects(self, with_toolchain=False):
+        tmp = self.info["projects"]
+        if with_toolchain and "gaudi" in [p.lower() for p in tmp]:
+            tctype, tcversion = self.toolchain()
+            tmp[tctype] = tcversion
+        return tmp
+
     def name(self):
         return self.info["name"]
 
@@ -143,7 +158,7 @@ class Stack:
         """ return true if some projects are missing """
         return len(self.missing_projects()) > 0
 
-    def get_toolchain(self):
+    def toolchain(self):
         """ Accessor to the toolchain information """
         toolchain = self.info["toolchain"]
         return (toolchain["type"], str(toolchain["version"]))
@@ -191,10 +206,6 @@ class LHCbClassicStack(Stack):
         todo = []
         for platform, project_list in missing.items():
             tmplist = [(p, v) for (_, p, v) in project_list]
-            # Unfortunately we need to Hardcode this as we not have the
-            # exact dependencies at this stage
-            if "gaudi" in [p.lower() for p, v in tmplist]:
-                tmplist += [self.get_toolchain()]
             todo.append((platform, tmplist))
         return todo
 

src/lhcbstacks/cli.py

@@ -19,7 +19,7 @@ def stacks_to_release_json():
     parser.add_argument("siteroot")
     parser.add_argument("stack_paths", nargs="*")
     args = parser.parse_args()
-    release = lhcbstacks.list_stacks_to_release(args.siteroot, args.stack_paths)
+    release = lhcbstacks.list_stacks_to_release(args.stack_paths, args.siteroot)
     print(json.dumps(release, indent=4, sort_keys=True))
 
 

tests/data/stacks/CMTStack.yml

@@ -12,6 +12,7 @@ name: CMTStack
 type: cmt
 
 projects:
+  Gaudi: v1r1
   LHCb: v1r1
 
 toolchain:

tests/data/stacks/Gaudi_v35r2.yml

+###############################################################################
+# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+name: Gaudi_v35r2
+type: lhcbcmake
+
+projects:
+  Gaudi: v35r2
+
+toolchain:
+  type: LCG
+  version: 97a
+
+platforms:
+  - x86_64+avx2+fma-centos7-gcc9-opt
+  - x86_64-centos7-clang8-dbg
+  - x86_64-centos7-clang8-opt
+  - x86_64-centos7-gcc9-dbg
+  - x86_64-centos7-gcc9-do0
+  - x86_64-centos7-gcc9-opt
+
+tags:
+  - Run3

tests/test_load.py

@@ -82,7 +82,7 @@ def test_list_platforms():
 
 def test_list_stacks_in_dir():
     stack_list = list(lhcbstacks._list_stacks_in_dir(STACK_DIR))
-    assert len(stack_list) == 2
+    assert len(stack_list) == 3
 
 
 def test_list_binary_paths():
@@ -110,13 +110,13 @@ def test_missing(lhcb_install_area):
 
 def test_incomplete_stacks(lhcb_install_area):
     inc_stacks = lhcbstacks.incomplete_stacks([STACK_DIR], lhcb_install_area)
-    assert len(inc_stacks) == 2
+    assert len(inc_stacks) == 3
 
 
 def test_stacks_to_release(lhcb_install_area):
     release = lhcbstacks.list_stacks_to_release([STACK_DIR], lhcb_install_area)
     result_json = json.dumps(release, sort_keys=True)
-    expected = '[{"build_tool": "cmt", "name": "CMTStack", "platforms": ["x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": [["LHCb", "v1r1"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-gcc9-do0"], "projects": [["Analysis", "v32r0"], ["Phys", "v31r1"], ["Rec", "v31r1"], ["Lbcom", "v31r1"], ["LHCb", "v51r1"], ["Gaudi", "v33r2"], ["Detector", "v0r5"], ["DaVinci", "v52r0"], ["LCG", "97a"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64-centos7-clang8-dbg"], "projects": [["DaVinci", "v52r0"]]}]'
+    expected = '[{"build_tool": "cmt", "deployed": {"LCG": "60"}, "name": "CMTStack", "platforms": ["x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": {"Gaudi": "v1r1", "LCG": "60", "LHCb": "v1r1"}}, {"build_tool": "cmake", "deployed": {"LCG": "97a"}, "name": "Gaudi_v35r2", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-clang8-dbg", "x86_64-centos7-clang8-opt", "x86_64-centos7-gcc9-dbg", "x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": {"Gaudi": "v35r2", "LCG": "97a"}}, {"build_tool": "cmake", "deployed": {"LCG": "97a"}, "name": "Run3Analysis202008", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-gcc9-do0"], "projects": {"Analysis": "v32r0", "DaVinci": "v52r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}}, {"build_tool": "cmake", "deployed": {"Analysis": "v32r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}, "name": "Run3Analysis202008", "platforms": ["x86_64-centos7-clang8-dbg"], "projects": {"Analysis": "v32r0", "DaVinci": "v52r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}}]'
     assert result_json == expected