Commit 3d929931 authored by Marco Clemencic's avatar Marco Clemencic
Browse files

Merge branch 'bcouturi_check_individual_files' into 'master'

Allow checking individual files and change nightlies JSON

See merge request !7
parents 11be0d34 8dad8491
Pipeline #2392340 passed with stages
in 1 minute and 49 seconds
......@@ -53,7 +53,7 @@ generate_stacks:
- to_release.json
expire_in: 1 week
script:
- lb-stacks-to-release ./data/stacks /cvmfs/lhcb.cern.ch/lib/ | tee to_release.json
- lb-stacks-to-release /cvmfs/lhcb.cern.ch/lib/ ./data/stacks | tee to_release.json
generate_project_list:
extends: .setup_environment
......
......@@ -31,60 +31,55 @@ STACK_SCHEMA = Map(
)
def load_stack_info(repository, name):
def stack_filename(repository, name):
""" Returns the file name for the stack """
return Path(repository) / f"{name}.yml"
def _load_stack_file(filename):
""" Load stack information by name from the specified repository """
filename = Path(repository) / f"{name}.yml"
with open(filename) as file:
info = load(file.read(), STACK_SCHEMA)
return info.data
def load_stack(repository, siteroot, name):
""" Factory method returning a stack object of the correct type """
info = load_stack_info(repository, name)
stack = None
stack_type = info["type"].lower()
if stack_type == "lhcbcmake":
stack = LHCbClassicStack(siteroot, info, "cmake")
elif stack_type == "cmt":
stack = LHCbClassicStack(siteroot, info, "cmt")
else:
raise NotImplementedError(f"Stack type {info['type']} not implemented")
return stack
def list_stacks(repository):
""" Returns the list of stack objects as (name, path) """
for path, _, files in os.walk(repository):
def _list_stacks_in_dir(dirname):
""" Returns the list of stack objects as (name, path) in the directory specified"""
for path, _, files in os.walk(dirname):
for f in files:
if f.endswith(".yml"):
name = f[: -len(".yml")]
yield (name, Path(path) / f)
def incomplete_stacks(repository, siteroot):
def incomplete_stacks(stack_paths, siteroot):
""" List all the projects in all the defined stacks in the yaml repository """
inc_stacks = []
for name, _ in list_stacks(repository):
s = load_stack(repository, siteroot, name)
missing = s.missing_projects()
if len(missing) > 0:
inc_stacks.append(s)
for path in stack_paths:
if os.path.isdir(path):
for _, filename in _list_stacks_in_dir(path):
s = Stack.load(filename, siteroot)
if s.has_missing_projects():
inc_stacks.append(s)
else:
s = Stack.load(path, siteroot)
if s.has_missing_projects():
inc_stacks.append(s)
return inc_stacks
def list_stacks_to_release(repository, siteroot):
def list_stacks_to_release(stack_paths, siteroot):
""" Generate the JSON file with the stacks that should be released """
# 1. Get the diff between what should be released, and what is in the repository
inc_stacks = incomplete_stacks(repository, siteroot)
# 1. Get the list of stacks with missing projects
inc_stacks = incomplete_stacks(stack_paths, siteroot)
# 2. Iterate to generate the stack in the exported format
tobuild = []
for stack in inc_stacks:
for platform, project_list in stack.builds_to_do():
# First checking if another build has the same projects exactly
# Then add platform
# Then add platform to it instead of creating a new build
found_matching = False
for d, s in tobuild:
if set(project_list) == s:
......@@ -95,10 +90,19 @@ def list_stacks_to_release(repository, siteroot):
if found_matching:
continue
# No match, so here we prepare the document listing the work
# project list is the list to build
# In this file for the nightlies we list the whole stack config
# and the one already done (so the complement of the info we have)
data = dict()
data["build_tool"] = stack.build_tool()
data["platforms"] = [platform]
data["projects"] = project_list
data["projects"] = stack.projects(with_toolchain=True)
data["deployed"] = {
p: v
for p, v in stack.projects(with_toolchain=True).items()
if (p, v) not in project_list
}
data["name"] = stack.name()
tobuild.append((data, set(project_list)))
......@@ -106,6 +110,20 @@ def list_stacks_to_release(repository, siteroot):
class Stack:
@classmethod
def load(cls, filename, siteroot):
""" Factory method returning a stack object of the correct type """
info = _load_stack_file(filename)
stack = None
stack_type = info["type"].lower()
if stack_type == "lhcbcmake":
stack = LHCbClassicStack(siteroot, info, "cmake")
elif stack_type == "cmt":
stack = LHCbClassicStack(siteroot, info, "cmt")
else:
raise NotImplementedError(f"Stack type {info['type']} not implemented")
return stack
def __init__(self, siteroot, stack_info):
self.siteroot_path = Path(siteroot)
self.info = stack_info
......@@ -117,6 +135,13 @@ class Stack:
def platforms(self):
return self.info["platforms"]
def projects(self, with_toolchain=False):
tmp = self.info["projects"]
if with_toolchain and "gaudi" in [p.lower() for p in tmp]:
tctype, tcversion = self.toolchain()
tmp[tctype] = tcversion
return tmp
def name(self):
return self.info["name"]
......@@ -129,7 +154,11 @@ class Stack:
missing[binary_tag].append((p, project, version))
return missing
def get_toolchain(self):
def has_missing_projects(self):
""" return true if some projects are missing """
return len(self.missing_projects()) > 0
def toolchain(self):
""" Accessor to the toolchain information """
toolchain = self.info["toolchain"]
return (toolchain["type"], str(toolchain["version"]))
......@@ -177,10 +206,6 @@ class LHCbClassicStack(Stack):
todo = []
for platform, project_list in missing.items():
tmplist = [(p, v) for (_, p, v) in project_list]
# Unfortunately we need to Hardcode this as we not have the
# exact dependencies at this stage
if "gaudi" in [p.lower() for p, v in tmplist]:
tmplist += [self.get_toolchain()]
todo.append((platform, tmplist))
return todo
......
......@@ -16,10 +16,10 @@ import lhcbstacks
def stacks_to_release_json():
parser = argparse.ArgumentParser()
parser.add_argument("stack_dir")
parser.add_argument("siteroot")
parser.add_argument("stack_paths", nargs="*")
args = parser.parse_args()
release = lhcbstacks.list_stacks_to_release(args.stack_dir, args.siteroot)
release = lhcbstacks.list_stacks_to_release(args.stack_paths, args.siteroot)
print(json.dumps(release, indent=4, sort_keys=True))
......
......@@ -12,6 +12,7 @@ name: CMTStack
type: cmt
projects:
Gaudi: v1r1
LHCb: v1r1
toolchain:
......
###############################################################################
# (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
name: Gaudi_v35r2
type: lhcbcmake
projects:
Gaudi: v35r2
toolchain:
type: LCG
version: 97a
platforms:
- x86_64+avx2+fma-centos7-gcc9-opt
- x86_64-centos7-clang8-dbg
- x86_64-centos7-clang8-opt
- x86_64-centos7-gcc9-dbg
- x86_64-centos7-gcc9-do0
- x86_64-centos7-gcc9-opt
tags:
- Run3
......@@ -37,7 +37,9 @@ TEST_PLATFORM = "x86_64-centos7-gcc9-opt"
def lhcb_install_area(tmp_path):
""" Create a fake install with a missing platform and missing projects """
lhcb_dir = tmp_path
s = lhcbstacks.load_stack(STACK_DIR, lhcb_dir, TEST_STACK)
s = lhcbstacks.Stack.load(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), lhcb_dir
)
binary_paths = s.binary_paths()
binary_paths.pop("x86_64+avx2+fma-centos7-gcc9-opt")
......@@ -53,12 +55,16 @@ def lhcb_install_area(tmp_path):
def test_load_info():
stack_info = lhcbstacks.load_stack_info(STACK_DIR, TEST_STACK)
stack_info = lhcbstacks._load_stack_file(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK)
)
assert stack_info["type"] == "lhcbcmake"
def test_load_stack():
a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
a = lhcbstacks.Stack.load(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
)
ps = [p for (p, _, _) in a.project_paths()]
dv = [p for p in ps if TEST_PROJECT.upper() in str(p)]
assert (
......@@ -68,17 +74,21 @@ def test_load_stack():
def test_list_platforms():
a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
a = lhcbstacks.Stack.load(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
)
assert a.platforms() == PLATFORMS
def test_list_stacks():
stack_list = list(lhcbstacks.list_stacks(STACK_DIR))
assert len(stack_list) == 2
def test_list_stacks_in_dir():
stack_list = list(lhcbstacks._list_stacks_in_dir(STACK_DIR))
assert len(stack_list) == 3
def test_list_binary_paths():
a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
a = lhcbstacks.Stack.load(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
)
dv = [p for p in a.binary_paths()[TEST_PLATFORM] if TEST_PROJECT.upper() in str(p)]
assert (
str(dv[0][0])
......@@ -87,7 +97,9 @@ def test_list_binary_paths():
def test_missing(lhcb_install_area):
a = lhcbstacks.load_stack(STACK_DIR, lhcb_install_area, TEST_STACK)
a = lhcbstacks.Stack.load(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), lhcb_install_area
)
missing = a.missing_projects()
# We should be missing x86_64+avx2+fma-centos7-gcc9-opt, and DV for x86_64-centos7-clang8-dbg
......@@ -97,14 +109,14 @@ def test_missing(lhcb_install_area):
def test_incomplete_stacks(lhcb_install_area):
inc_stacks = lhcbstacks.incomplete_stacks(STACK_DIR, lhcb_install_area)
assert len(inc_stacks) == 2
inc_stacks = lhcbstacks.incomplete_stacks([STACK_DIR], lhcb_install_area)
assert len(inc_stacks) == 3
def test_stacks_to_release(lhcb_install_area):
release = lhcbstacks.list_stacks_to_release(STACK_DIR, lhcb_install_area)
release = lhcbstacks.list_stacks_to_release([STACK_DIR], lhcb_install_area)
result_json = json.dumps(release, sort_keys=True)
expected = '[{"build_tool": "cmt", "name": "CMTStack", "platforms": ["x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": [["LHCb", "v1r1"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-gcc9-do0"], "projects": [["Analysis", "v32r0"], ["Phys", "v31r1"], ["Rec", "v31r1"], ["Lbcom", "v31r1"], ["LHCb", "v51r1"], ["Gaudi", "v33r2"], ["Detector", "v0r5"], ["DaVinci", "v52r0"], ["LCG", "97a"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64-centos7-clang8-dbg"], "projects": [["DaVinci", "v52r0"]]}]'
expected = '[{"build_tool": "cmt", "deployed": {"LCG": "60"}, "name": "CMTStack", "platforms": ["x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": {"Gaudi": "v1r1", "LCG": "60", "LHCb": "v1r1"}}, {"build_tool": "cmake", "deployed": {"LCG": "97a"}, "name": "Gaudi_v35r2", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-clang8-dbg", "x86_64-centos7-clang8-opt", "x86_64-centos7-gcc9-dbg", "x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": {"Gaudi": "v35r2", "LCG": "97a"}}, {"build_tool": "cmake", "deployed": {"LCG": "97a"}, "name": "Run3Analysis202008", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-gcc9-do0"], "projects": {"Analysis": "v32r0", "DaVinci": "v52r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}}, {"build_tool": "cmake", "deployed": {"Analysis": "v32r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}, "name": "Run3Analysis202008", "platforms": ["x86_64-centos7-clang8-dbg"], "projects": {"Analysis": "v32r0", "DaVinci": "v52r0", "Detector": "v0r5", "Gaudi": "v33r2", "LCG": "97a", "LHCb": "v51r1", "Lbcom": "v31r1", "Phys": "v31r1", "Rec": "v31r1"}}]'
assert result_json == expected
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment