Commit 01f0178b authored by Marco Clemencic's avatar Marco Clemencic
Browse files

Add function lb.nightly.functions.rpc.checkout

parent e312fa9d
###############################################################################
# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
"""
Entry point functions for nightly build operations.
"""
import os
import pathlib
import json
from lb.nightly.configuration import get, Project, service_config
from lb.nightly.utils import Repository
def checkout(project_id: str, gitlab_token=None):
from .checkout import git, notify_gitlab, update_nightly_git_archive
from .common import Report
from lb.nightly.db import connect
from subprocess import check_output, STDOUT
conf = service_config()
gitlab_token = (
gitlab_token
or conf.get("gitlab", {}).get("token")
or os.environ.get("GITLAB_TOKEN")
)
project = get(project_id)
if not isinstance(project, Project):
raise ValueError(
f"project_id {project_id} does not identify a nightly builds Project instance"
)
db = connect(flavour=project.slot.metadata.get("flavour", "nightly"),)
db.checkout_start(project)
# checkout the project
report = git(project)
if gitlab_token:
report = notify_gitlab(project, report, gitlab_token)
try:
report = update_nightly_git_archive(project, report)
except Exception as err:
report.warning(f"failed to update Git archive: {err}")
# resolve and update dependencies
db.set_dependencies(project)
archive_name = os.path.basename(project.artifacts("checkout"))
report.info(f"packing {project.name} into {archive_name}")
report.log(
"stdout",
"debug",
check_output(["zip", "-r", archive_name, project.name], stderr=STDOUT).decode(),
)
report.info(f"adding logs to {archive_name}")
log_dir = pathlib.Path(project.name) / ".logs" / "checkout"
os.makedirs(log_dir)
with open(log_dir / "report.json", "w") as f:
json.dump(report.to_dict(), f, indent=2)
with open(log_dir / "report.md", "w") as f:
f.write(report.md())
report.log(
"stdout",
"debug",
check_output(["zip", "-r", archive_name, log_dir], stderr=STDOUT).decode(),
)
if "artifacts" in conf:
artifacts_repo = Repository.connect(conf["artifacts"]["uri"])
assert artifacts_repo.push(
open(archive_name, "rb"), project.artifacts("checkout")
), "failed to upload artifacts"
else:
report.warning("artifacts repository not configured: no publishing")
return db.checkout_complete(project, report)
###############################################################################
# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
import pytest
import os
import zipfile
from unittest.mock import patch, MagicMock, Mock
from lb.nightly.functions.rpc import checkout
from lb.nightly.configuration import Slot, Project
from .test_checkout import working_directory
def mock_git(project, *args, **kwargs):
from lb.nightly.functions.common import Report
os.makedirs(project.name)
with open(os.path.join(project.name, "data.txt"), "w") as f:
f.write("data\n")
return Mock()
@patch(
"lb.nightly.functions.rpc.service_config",
return_value={"gitlab": {"token": "some_token"}, "artifacts": {"uri": "artifacts"}},
)
@patch(
"lb.nightly.functions.rpc.get",
return_value=Slot(
"test-slot", build_id=123, projects=[Project("Moore", "HEAD")]
).Moore,
)
@patch("lb.nightly.db.connect")
@patch("lb.nightly.functions.checkout.git")
@patch("lb.nightly.functions.checkout.notify_gitlab")
@patch("lb.nightly.functions.checkout.update_nightly_git_archive")
def test_checkout_full(
update_nightly_git_archive,
notify_gitlab,
git,
connect,
get,
service_config,
tmp_path,
):
git.side_effect = mock_git
update_nightly_git_archive().md.return_value = "md report"
update_nightly_git_archive().to_dict.return_value = {}
update_nightly_git_archive.reset_mock()
with patch("json.dump"), working_directory(tmp_path):
report = checkout("flavour/slot/123/project")
service_config.assert_called_once()
get.assert_called_once()
connect.assert_called_once()
git.assert_called_once()
notify_gitlab.assert_called_once()
update_nightly_git_archive.assert_called_once()
archive_path = tmp_path / os.path.basename(get().artifacts("checkout"))
assert zipfile.is_zipfile(archive_path)
with zipfile.ZipFile(archive_path) as archive:
names = archive.namelist()
assert "Moore/data.txt" in names
assert archive.read("Moore/data.txt").decode() == "data\n"
assert "Moore/.logs/checkout/report.json" in names
assert "Moore/.logs/checkout/report.md" in names
artifact_path = tmp_path / "artifacts" / get().artifacts("checkout")
assert os.path.exists(artifact_path)
assert open(archive_path, "rb").read() == open(artifact_path, "rb").read()
@patch("lb.nightly.functions.rpc.service_config")
@patch(
"lb.nightly.functions.rpc.get", return_value="something",
)
def test_checkout_bad_get_result(get, service_config):
with pytest.raises(ValueError):
checkout("flavour/slot/123/project")
get.assert_called_once()
@patch(
"lb.nightly.functions.rpc.service_config",
return_value={"gitlab": {"token": None}, "artifacts": {"uri": "artifacts"}},
)
@patch(
"lb.nightly.functions.rpc.get",
return_value=Slot(
"test-slot", build_id=123, projects=[Project("Moore", "HEAD")]
).Moore,
)
@patch("lb.nightly.db.connect")
@patch("lb.nightly.functions.checkout.git")
@patch("lb.nightly.functions.checkout.notify_gitlab")
@patch("lb.nightly.functions.checkout.update_nightly_git_archive")
def test_checkout_no_gitlab_notify(
update_nightly_git_archive,
notify_gitlab,
git,
connect,
get,
service_config,
tmp_path,
):
git.side_effect = mock_git
update_nightly_git_archive().md.return_value = "md report"
update_nightly_git_archive().to_dict.return_value = {}
update_nightly_git_archive.reset_mock()
with patch("json.dump"), working_directory(tmp_path), patch.dict(
os.environ, {"GITLAB_TOKEN": ""}
):
report = checkout("flavour/slot/123/project")
service_config.assert_called_once()
get.assert_called_once()
connect.assert_called_once()
git.assert_called_once()
notify_gitlab.assert_not_called()
update_nightly_git_archive.assert_called_once()
archive_path = tmp_path / os.path.basename(get().artifacts("checkout"))
assert zipfile.is_zipfile(archive_path)
with zipfile.ZipFile(archive_path) as archive:
names = archive.namelist()
assert "Moore/data.txt" in names
assert archive.read("Moore/data.txt").decode() == "data\n"
assert "Moore/.logs/checkout/report.json" in names
assert "Moore/.logs/checkout/report.md" in names
artifact_path = tmp_path / "artifacts" / get().artifacts("checkout")
assert os.path.exists(artifact_path)
assert open(archive_path, "rb").read() == open(artifact_path, "rb").read()
@patch(
"lb.nightly.functions.rpc.service_config",
return_value={"gitlab": {"token": "some_token"}, "artifacts": {"uri": "artifacts"}},
)
@patch(
"lb.nightly.functions.rpc.get",
return_value=Slot(
"test-slot", build_id=123, projects=[Project("Moore", "HEAD")]
).Moore,
)
@patch("lb.nightly.db.connect")
@patch("lb.nightly.functions.checkout.git")
@patch("lb.nightly.functions.checkout.notify_gitlab")
@patch("lb.nightly.functions.checkout.update_nightly_git_archive")
def test_checkout_git_archive_fail(
update_nightly_git_archive,
notify_gitlab,
git,
connect,
get,
service_config,
tmp_path,
):
git.side_effect = mock_git
notify_gitlab().md.return_value = "md report"
notify_gitlab().to_dict.return_value = {}
notify_gitlab.reset_mock()
update_nightly_git_archive.side_effect = RuntimeError("some problem")
with patch("json.dump"), working_directory(tmp_path):
report = checkout("flavour/slot/123/project")
service_config.assert_called_once()
get.assert_called_once()
connect.assert_called_once()
git.assert_called_once()
notify_gitlab.assert_called_once()
update_nightly_git_archive.assert_called_once()
notify_gitlab().warning.assert_called_once()
archive_path = tmp_path / os.path.basename(get().artifacts("checkout"))
assert zipfile.is_zipfile(archive_path)
with zipfile.ZipFile(archive_path) as archive:
names = archive.namelist()
assert "Moore/data.txt" in names
assert archive.read("Moore/data.txt").decode() == "data\n"
assert "Moore/.logs/checkout/report.json" in names
assert "Moore/.logs/checkout/report.md" in names
artifact_path = tmp_path / "artifacts" / get().artifacts("checkout")
assert os.path.exists(artifact_path)
assert open(archive_path, "rb").read() == open(artifact_path, "rb").read()
@patch(
"lb.nightly.functions.rpc.service_config",
return_value={"gitlab": {"token": "some_token"}},
)
@patch(
"lb.nightly.functions.rpc.get",
return_value=Slot(
"test-slot", build_id=123, projects=[Project("Moore", "HEAD")]
).Moore,
)
@patch("lb.nightly.db.connect")
@patch("lb.nightly.functions.checkout.git")
@patch("lb.nightly.functions.checkout.notify_gitlab")
@patch("lb.nightly.functions.checkout.update_nightly_git_archive")
def test_checkout_full(
update_nightly_git_archive,
notify_gitlab,
git,
connect,
get,
service_config,
tmp_path,
):
git.side_effect = mock_git
update_nightly_git_archive().md.return_value = "md report"
update_nightly_git_archive().to_dict.return_value = {}
update_nightly_git_archive.reset_mock()
with patch("json.dump"), working_directory(tmp_path):
report = checkout("flavour/slot/123/project")
service_config.assert_called_once()
get.assert_called_once()
connect.assert_called_once()
git.assert_called_once()
notify_gitlab.assert_called_once()
update_nightly_git_archive.assert_called_once()
update_nightly_git_archive().warning.assert_called_once()
archive_path = tmp_path / os.path.basename(get().artifacts("checkout"))
assert zipfile.is_zipfile(archive_path)
with zipfile.ZipFile(archive_path) as archive:
names = archive.namelist()
assert "Moore/data.txt" in names
assert archive.read("Moore/data.txt").decode() == "data\n"
assert "Moore/.logs/checkout/report.json" in names
assert "Moore/.logs/checkout/report.md" in names
artifact_path = tmp_path / "artifacts" / get().artifacts("checkout")
assert not os.path.exists(artifact_path)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment