Skip to content
Snippets Groups Projects

New deployment and update DPA/analysis-productions

Merged Chris Burr requested to merge new-deployment into master
7 files
+ 65
132
Compare changes
  • Side-by-side
  • Inline
Files
7
+ 0
55
#!/usr/bin/env python
import subprocess
import shutil
import os
from os.path import dirname, isdir, isfile, islink, join, relpath
import tempfile
subdir = "linux-64"
cvmfs_base = "/cvmfs/lhcbdev.cern.ch/conda"
pkgs_dir = join(cvmfs_base, "pkgs", subdir)
envs_dir = join(cvmfs_base, "envs")
output_dir = join(os.getcwd(), 'build-artifacts')
TAR_CMD = ["tar", "--zstd", "--hard-dereference", "--recursive", "--create"]
if isdir(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
tarball_paths = set()
# Package cache
for fn in os.listdir(pkgs_dir):
if fn in ["cache", "urls"]:
continue
fn = join(pkgs_dir, fn)
if not islink(fn) and not isdir(fn):
print("Adding:", fn)
tarball_paths.add(fn)
# Environments
for dirpath, dirnames, filenames in os.walk(envs_dir, topdown=True):
print("Trying", dirpath)
if ".cvmfscatalog" in filenames:
print("Found environment:", dirpath)
tarball_paths.add(dirpath)
yaml_fn = dirpath + ".yaml"
if not isfile(yaml_fn):
raise RuntimeError("Failed to find yaml file" + repr(yaml_fn))
tarball_paths.add(yaml_fn)
dirnames[:] = []
if not tarball_paths:
raise RuntimeError("Failed to find anything to upload!")
with tempfile.NamedTemporaryFile(mode="wt") as fp:
fp.write("\n".join([relpath(fn, cvmfs_base) for fn in tarball_paths]))
fp.flush()
tar_fn = join(output_dir, f"{os.environ['CI_PIPELINE_ID']}.tar.ztd")
print("Creating", tar_fn)
subprocess.run(TAR_CMD + [
f"--directory={cvmfs_base}",
f"--file={tar_fn}",
"--files-from",
fp.name
], check=True)
Loading