Commit f7a1dd52 authored by Ben Couturier's avatar Ben Couturier
Browse files

Allow specifying the stacks to analyze

parent 11be0d34
......@@ -31,52 +31,48 @@ STACK_SCHEMA = Map(
)
def load_stack_info(repository, name):
def stack_filename(repository, name):
""" Returns the file name for the stack """
return Path(repository) / f"{name}.yml"
def load_stack_file(filename):
""" Load stack information by name from the specified repository """
filename = Path(repository) / f"{name}.yml"
with open(filename) as file:
info = load(file.read(), STACK_SCHEMA)
return info.data
def load_stack(repository, siteroot, name):
""" Factory method returning a stack object of the correct type """
info = load_stack_info(repository, name)
stack = None
stack_type = info["type"].lower()
if stack_type == "lhcbcmake":
stack = LHCbClassicStack(siteroot, info, "cmake")
elif stack_type == "cmt":
stack = LHCbClassicStack(siteroot, info, "cmt")
else:
raise NotImplementedError(f"Stack type {info['type']} not implemented")
return stack
def list_stacks(repository):
""" Returns the list of stack objects as (name, path) """
for path, _, files in os.walk(repository):
def _list_stacks_in_dir(dirname):
""" Returns the list of stack objects as (name, path) in the directory specified"""
for path, _, files in os.walk(dirname):
for f in files:
if f.endswith(".yml"):
name = f[: -len(".yml")]
yield (name, Path(path) / f)
def incomplete_stacks(repository, siteroot):
def incomplete_stacks(stack_paths, siteroot):
""" List all the projects in all the defined stacks in the yaml repository """
inc_stacks = []
for name, _ in list_stacks(repository):
s = load_stack(repository, siteroot, name)
missing = s.missing_projects()
if len(missing) > 0:
inc_stacks.append(s)
for path in stack_paths:
if os.path.isdir(path):
for _, filename in _list_stacks_in_dir(path):
s = load_stack(filename, siteroot)
if s.has_missing_projects():
inc_stacks.append(s)
else:
s = load_stack(filename, siteroot)
if s.has_missing_projects():
inc_stacks.append(s)
return inc_stacks
def list_stacks_to_release(repository, siteroot):
def list_stacks_to_release(stack_paths, siteroot):
""" Generate the JSON file with the stacks that should be released """
# 1. Get the diff between what should be released, and what is in the repository
inc_stacks = incomplete_stacks(repository, siteroot)
inc_stacks = incomplete_stacks(stack_paths, siteroot)
# 2. Iterate to generate the stack in the exported format
tobuild = []
......@@ -105,6 +101,20 @@ def list_stacks_to_release(repository, siteroot):
return sorted([d for d, _ in tobuild], key=lambda d: d["name"])
def load_stack(filename, siteroot):
""" Factory method returning a stack object of the correct type """
info = load_stack_file(filename)
stack = None
stack_type = info["type"].lower()
if stack_type == "lhcbcmake":
stack = LHCbClassicStack(siteroot, info, "cmake")
elif stack_type == "cmt":
stack = LHCbClassicStack(siteroot, info, "cmt")
else:
raise NotImplementedError(f"Stack type {info['type']} not implemented")
return stack
class Stack:
def __init__(self, siteroot, stack_info):
self.siteroot_path = Path(siteroot)
......@@ -129,6 +139,10 @@ class Stack:
missing[binary_tag].append((p, project, version))
return missing
def has_missing_projects(self):
""" return true if some projects are missing """
return len(self.missing_projects()) > 0
def get_toolchain(self):
""" Accessor to the toolchain information """
toolchain = self.info["toolchain"]
......
......@@ -16,10 +16,10 @@ import lhcbstacks
def stacks_to_release_json():
parser = argparse.ArgumentParser()
parser.add_argument("stack_dir")
parser.add_argument("siteroot")
parser.add_argument("stack_paths", nargs="*")
args = parser.parse_args()
release = lhcbstacks.list_stacks_to_release(args.stack_dir, args.siteroot)
release = lhcbstacks.list_stacks_to_release(args.siteroot, args.stack_paths)
print(json.dumps(release, indent=4, sort_keys=True))
......
......@@ -37,7 +37,9 @@ TEST_PLATFORM = "x86_64-centos7-gcc9-opt"
def lhcb_install_area(tmp_path):
""" Create a fake install with a missing platform and missing projects """
lhcb_dir = tmp_path
s = lhcbstacks.load_stack(STACK_DIR, lhcb_dir, TEST_STACK)
s = lhcbstacks.load_stack(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), lhcb_dir
)
binary_paths = s.binary_paths()
binary_paths.pop("x86_64+avx2+fma-centos7-gcc9-opt")
......@@ -53,12 +55,16 @@ def lhcb_install_area(tmp_path):
def test_load_info():
stack_info = lhcbstacks.load_stack_info(STACK_DIR, TEST_STACK)
stack_info = lhcbstacks.load_stack_file(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK)
)
assert stack_info["type"] == "lhcbcmake"
def test_load_stack():
a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
a = lhcbstacks.load_stack(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
)
ps = [p for (p, _, _) in a.project_paths()]
dv = [p for p in ps if TEST_PROJECT.upper() in str(p)]
assert (
......@@ -68,17 +74,21 @@ def test_load_stack():
def test_list_platforms():
a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
a = lhcbstacks.load_stack(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
)
assert a.platforms() == PLATFORMS
def test_list_stacks():
stack_list = list(lhcbstacks.list_stacks(STACK_DIR))
def test_list_stacks_in_dir():
stack_list = list(lhcbstacks._list_stacks_in_dir(STACK_DIR))
assert len(stack_list) == 2
def test_list_binary_paths():
a = lhcbstacks.load_stack(STACK_DIR, LHCBSITEROOT, TEST_STACK)
a = lhcbstacks.load_stack(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), LHCBSITEROOT
)
dv = [p for p in a.binary_paths()[TEST_PLATFORM] if TEST_PROJECT.upper() in str(p)]
assert (
str(dv[0][0])
......@@ -87,7 +97,9 @@ def test_list_binary_paths():
def test_missing(lhcb_install_area):
a = lhcbstacks.load_stack(STACK_DIR, lhcb_install_area, TEST_STACK)
a = lhcbstacks.load_stack(
lhcbstacks.stack_filename(STACK_DIR, TEST_STACK), lhcb_install_area
)
missing = a.missing_projects()
# We should be missing x86_64+avx2+fma-centos7-gcc9-opt, and DV for x86_64-centos7-clang8-dbg
......@@ -97,12 +109,12 @@ def test_missing(lhcb_install_area):
def test_incomplete_stacks(lhcb_install_area):
inc_stacks = lhcbstacks.incomplete_stacks(STACK_DIR, lhcb_install_area)
inc_stacks = lhcbstacks.incomplete_stacks([STACK_DIR], lhcb_install_area)
assert len(inc_stacks) == 2
def test_stacks_to_release(lhcb_install_area):
release = lhcbstacks.list_stacks_to_release(STACK_DIR, lhcb_install_area)
release = lhcbstacks.list_stacks_to_release([STACK_DIR], lhcb_install_area)
result_json = json.dumps(release, sort_keys=True)
expected = '[{"build_tool": "cmt", "name": "CMTStack", "platforms": ["x86_64-centos7-gcc9-do0", "x86_64-centos7-gcc9-opt"], "projects": [["LHCb", "v1r1"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64+avx2+fma-centos7-gcc9-opt", "x86_64-centos7-gcc9-do0"], "projects": [["Analysis", "v32r0"], ["Phys", "v31r1"], ["Rec", "v31r1"], ["Lbcom", "v31r1"], ["LHCb", "v51r1"], ["Gaudi", "v33r2"], ["Detector", "v0r5"], ["DaVinci", "v52r0"], ["LCG", "97a"]]}, {"build_tool": "cmake", "name": "Run3Analysis202008", "platforms": ["x86_64-centos7-clang8-dbg"], "projects": [["DaVinci", "v52r0"]]}]'
assert result_json == expected
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment