Commit b37402cb authored by Chris Burr's avatar Chris Burr
Browse files

Initial commit

parents
Pipeline #1729829 failed with stage
in 1 minute and 11 seconds
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
stages:
- test
- deploy
check_black:
stage: test
image: continuumio/miniconda3:latest
before_script:
- eval "$(python -m conda shell.bash hook)" && conda config --add channels conda-forge
- conda install --quiet --yes black
script:
- black --check --diff .
.run_pytest:
stage: test
script:
- python --version
- pip install --upgrade setuptools
- pip install '.[testing]'
- pytest -vvv
slc6:
extends: .run_pytest
image: gitlab-registry.cern.ch/lhcb-docker/python-deployment:slc6
centos7:
extends: .run_pytest
image: gitlab-registry.cern.ch/lhcb-docker/python-deployment:centos7
# Packaging step
deploy-packages:
stage: deploy
only:
- tags
dependencies: []
image: gitlab-registry.cern.ch/lhcb-docker/python-deployment:python-3.7
script:
- python setup.py sdist --dist-dir public/
- python setup.py bdist_wheel --dist-dir public/
- if [ -z "$TWINE_PASSWORD" ] ; then echo "Set TWINE_PASSWORD in CI variables" ; exit 1 ; fi
- twine upload -u __token__ public/*
before_script: []
after_script: []
{
"python.pythonPath": "/home/cburr/miniconda3/envs/analysis-productions-dev/bin/python"
}
\ No newline at end of file
This diff is collapsed.
[flake8]
max_line_length = 120
[tool:pytest]
addopts = --cov=LbAPCommon --cov-report=term-missing
###############################################################################
# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
from setuptools import setup, find_packages
from os.path import abspath, dirname, join
from io import open
here = abspath(dirname(__file__))
# Get the long description from the README file
with open(join(here, "README.md"), encoding="utf-8") as f:
long_description = f.read()
package_data = []
setup(
name="LbAPCommon",
use_scm_version=True,
description="Common utilities used by LHCb DPA WP2 related software",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://gitlab.cern.ch/lhcb-dpa/analysis-productions/LbAPCommon",
author="LHCb",
classifiers=[
"Development Status :: 3 - Alpha",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
],
keywords="LHCb Core task runner",
packages=find_packages("src"),
package_dir={"": "src"},
python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*",
setup_requires=["setuptools_scm"],
install_requires=[
"strictyaml",
],
extras_require={"testing": ["pytest", "pytest-cov"]},
package_data={"LbAPCommon": package_data},
zip_safe=False,
project_urls={
"Bug Reports": "https://gitlab.cern.ch/lhcb-dpa/analysis-productions/LbAPCommon/issues",
"Source": "https://gitlab.cern.ch/lhcb-dpa/analysis-productions/LbAPCommon",
},
)
###############################################################################
# (c) Copyright 2018 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from .parsing import parse_yaml
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
###############################################################################
# (c) Copyright 2018 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import re
from strictyaml import MapPattern, Int, Float, Str, YAMLValidationError, load, Seq, EmptyList
from strictyaml import load, Map, Str, Int, Seq, YAMLError, Regex, Bool, Optional, Any, YAML
RE_JOB_NAME = r"^[a-zA-Z0-9][a-zA-Z0-9_\-]+$"
RE_OUTPUT_FILE_TYPE = r"^([A-Z][A-Z0-9_]+\.)+(ROOT|.?DST)$"
RE_OPTIONS_FN = r"^\$?[a-zA-Z0-9/\.\-\+\=_]+$"
RE_INFORM = r"^(?:[a-zA-Z]{3,}|[^@\s]+@[^@\s]+\.[^@\s]+)$"
BASE_JOB_SCHEMA = {
"application": Str(),
"input": MapPattern(Str(), Any()),
"output": Regex(RE_OUTPUT_FILE_TYPE),
"options": Seq(Regex(RE_OPTIONS_FN)),
"wg": Str(),
"automatically_configure": Bool(),
"inform": EmptyList() | Seq(Regex(RE_INFORM)),
}
DEFAULT_JOB_VALUES = {
"automatically_configure": False,
"inform": [],
}
def render_yaml(rendered_yaml):
data = load(rendered_yaml, schema=MapPattern(
Regex(RE_JOB_NAME),
Any(),
minimum_keys=1
))
if "defaults" in data:
data['defaults'].revalidate(Map({
Optional(k, default=DEFAULT_JOB_VALUES.get(k)): v
for k, v in BASE_JOB_SCHEMA.items()
}))
defaults = data.data['defaults']
# Remove the defaults data from the snippet
del data['defaults']
rendered_yaml = data.as_yaml()
else:
defaults = DEFAULT_JOB_VALUES.copy()
job_name_schema = Regex(r"(" + r"|".join(map(re.escape, data.data.keys())) + r")")
data = load(rendered_yaml, MapPattern(
job_name_schema,
Map({
Optional(k, default=defaults[k]) if k in defaults else k: v
for k, v in BASE_JOB_SCHEMA.items()
}),
minimum_keys=1
))
return data.data
def parse_yaml(rendered_yaml):
data = load(rendered_yaml, schema=MapPattern(
Regex(RE_JOB_NAME),
Any(),
minimum_keys=1
))
if "defaults" in data:
data['defaults'].revalidate(Map({
Optional(k, default=DEFAULT_JOB_VALUES.get(k)): v
for k, v in BASE_JOB_SCHEMA.items()
}))
defaults = data.data['defaults']
# Remove the defaults data from the snippet
del data['defaults']
rendered_yaml = data.as_yaml()
else:
defaults = DEFAULT_JOB_VALUES.copy()
job_name_schema = Regex(r"(" + r"|".join(map(re.escape, data.data.keys())) + r")")
data = load(rendered_yaml, MapPattern(
job_name_schema,
Map({
Optional(k, default=defaults[k]) if k in defaults else k: v
for k, v in BASE_JOB_SCHEMA.items()
}),
minimum_keys=1
))
return data.data
###############################################################################
# (c) Copyright 2018 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from textwrap import dedent
import pytest
import strictyaml
import LbAPCommon
def test_good_no_defaults():
rendered_yaml = dedent("""\
job_1:
application: DaVinci/v45r3
input:
bk_query: /some/query
output: FILETYPE.ROOT
options:
- options.py
- $VAR/a.py
wg: Charm
""")
data = LbAPCommon.parse_yaml(rendered_yaml)
assert len(data) == 1
assert data["job_1"]["application"] == "DaVinci/v45r3"
assert data["job_1"]["input"] == {"bk_query": "/some/query"}
assert data["job_1"]["output"] == "FILETYPE.ROOT"
assert data["job_1"]["options"] == ["options.py", "$VAR/a.py"]
assert data["job_1"]["wg"] == "Charm"
assert data["job_1"]["automatically_configure"] is False
assert data["job_1"]["inform"] == []
def test_good_with_defaults():
rendered_yaml = dedent("""\
defaults:
wg: Charm
automatically_configure: yes
inform:
- name@example.com
job_1:
application: DaVinci/v45r3
input:
bk_query: "/some/query"
output: FILETYPE.ROOT
options:
- options.py
job_2:
application: DaVinci/v44r0
input:
bk_query: "/some/other/query"
output: FILETYPE.ROOT
options:
- other_options.py
wg: B2OC
automatically_configure: false
inform:
- other@example.com
""")
data = LbAPCommon.parse_yaml(rendered_yaml)
assert len(data) == 2
assert data["job_1"]["application"] == "DaVinci/v45r3"
assert data["job_1"]["input"] == {"bk_query": "/some/query"}
assert data["job_1"]["output"] == "FILETYPE.ROOT"
assert data["job_1"]["options"] == ["options.py"]
assert data["job_1"]["wg"] == "Charm"
assert data["job_1"]["automatically_configure"] is True
assert data["job_1"]["inform"] == ["name@example.com"]
assert data["job_2"]["application"] == "DaVinci/v44r0"
assert data["job_2"]["input"] == {"bk_query": "/some/other/query"}
assert data["job_2"]["output"] == "FILETYPE.ROOT"
assert data["job_2"]["options"] == ["other_options.py"]
assert data["job_2"]["wg"] == "B2OC"
assert data["job_2"]["automatically_configure"] is False
assert data["job_2"]["inform"] == ["other@example.com"]
@pytest.mark.parametrize("missing_key", [
"application",
"input",
"output",
"wg",
])
def test_bad_missing_key(missing_key):
data = {
"job_1": {
"application": "DaVinci/v45r3",
"input": {"bk_query": "/some/query"},
"output": "FILETYPE.ROOT",
"options": ["options.py"],
"wg": "Charm",
}
}
del data["job_1"][missing_key]
rendered_yaml = strictyaml.YAML(data).as_yaml()
try:
LbAPCommon.parse_yaml(rendered_yaml)
except strictyaml.YAMLValidationError as e:
assert "required key(s) '"+missing_key+"' not found" in str(e)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment