diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000000000000000000000000000000000000..6c1f185248001af573923d1407a9b4d17035bd5b --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +extend-ignore = E203, E501, E722, B950 +extend-select = B9 +per-file-ignores = + tests/*: T + noxfile.py: T diff --git a/.gitignore b/.gitignore index b370b8fec0d57e752671e5cdc507f83f9eea6d3b..34a292169244adec7b4f3f78935a74d00c96728d 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,6 @@ dmypy.json .DS_Store .auth .webcache + +# setuptools_scm +src/*/_version.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f8bdefad429a1ad64a25a2100d29691ea1a0a02..d67e532d2d5c1a1fd91ef2ec540328f261287e1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,66 +1,101 @@ stages: + - check - test - - coverage - - package + - build - deploy -.tests: - stage: test - before_script: - - pip install --ignore-installed -U -q -e .[complete] - - pip freeze - script: - - flake8 src/ tests/ - - pytest - - if [[ -x $(command -v black) ]]; then black --check --diff --verbose src tests setup.py; fi +variables: + # see https://docs.gitlab.com/ee/ci/caching/#cache-python-dependencies + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit" -.security_checks: - stage: test - before_script: - - pip install --ignore-installed -U -q -e .[complete] - - pip freeze - script: - - bandit -r src/ +cache: + paths: + - .cache/pip + - .cache/pre-commit + - venv/ -tests_python3: - image: python:3.10 - extends: .tests +image: python:3.7-buster +before_script: + # want to set up a virtualenv to cache + - apt-get install -y --no-install-recommends git + - python -V + - git config --global credential.helper 'cache' + - | + echo "protocol=https + host=gitlab.cern.ch + username=gitlab-ci-token + password=${CI_JOB_TOKEN} + " | git credential approve + - python -m venv venv + - source venv/bin/activate + - python -m pip install -U pip pipx + - python -m pipx ensurepath + - python -m pip freeze -security_python3: - image: python:3.10 - extends: .security_checks +lint: + stage: check + script: + - python -m pip install pre-commit + - pre-commit run --all-files + +tests: + stage: test + image: $IMAGE + script: + - python -m pip install .[test] + - python -m pytest + parallel: + matrix: + - IMAGE: + [ + "python:3.7-buster", + "python:3.8-buster", + "python:3.9-buster", + "python:3.10-buster", + "python:3.11-buster", + ] package: - image: python:3.10-buster - stage: package - before_script: - - apt-get install -y --no-install-recommends git - - python -m pip install build --user + stage: build script: - - python -m build --sdist --wheel --outdir dist/ . + - pipx run build + - pipx run twine check dist/* artifacts: paths: - dist/ + expire_in: 1 day .deploy: - image: python:3.10-buster stage: deploy - dependencies: [package] - before_script: - - pip install twine + dependencies: + - package script: - - twine upload --verbose dist/*whl dist/*gz + - pipx run twine upload --verbose dist/*whl dist/*gz deploy_staging: extends: .deploy only: refs: - - master + - $CI_DEFAULT_BRANCH variables: TWINE_REPOSITORY: testpypi TWINE_USERNAME: __token__ TWINE_PASSWORD: $TESTPYPI_TOKEN +deploy_gitlab: + stage: deploy + only: + refs: + - $CI_DEFAULT_BRANCH + variables: + TWINE_PASSWORD: "${CI_JOB_TOKEN}" + TWINE_USERNAME: "gitlab-ci-token" + TWINE_REPOSITORY_URL: "https://gitlab.cern.ch/api/v4/projects/${CI_PROJECT_ID}/packages/pypi" + script: + - ls -lavh dist/ + - pipx run twine upload dist/* + deploy_production: extends: .deploy only: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9a75affdb41a96fd71f4dc663f3c178f0ad171f..3b31551dc6d05b7090186d60369bb4ceec00fdbb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,140 @@ +ci: + autoupdate_commit_msg: "chore: update pre-commit hooks" + autoupdate_schedule: monthly + +exclude: | + (?x)^( + add_attachment.py| + add_comment.py| + generatePlots.py| + getContentSummary.py| + getInventory.py| + registerComponent.py| + tests/integration/cassettes/.*.json + )$ + repos: -- repo: https://github.com/ambv/black - rev: stable + - repo: https://github.com/psf/black + rev: 22.10.0 hooks: - - id: black - language_version: python3.6 + - id: black-jupyter + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: requirements-txt-fixer + - id: trailing-whitespace + + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.9.0 + hooks: + - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore + - id: python-no-eval + - id: python-use-type-annotations + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.0.0-alpha.3" + hooks: + - id: prettier + types_or: [yaml, markdown, html, css, scss, javascript, json] + args: [--prose-wrap=always] + + - repo: https://github.com/asottile/blacken-docs + rev: v1.12.1 + hooks: + - id: blacken-docs + additional_dependencies: [black==22.8.0] + + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + args: ["-a", "from __future__ import annotations"] # Python 3.7+ + + - repo: https://github.com/asottile/pyupgrade + rev: v3.1.0 + hooks: + - id: pyupgrade + args: ["--py37-plus"] + + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v14.0.6 + hooks: + - id: clang-format + types_or: [c++, c, cuda] + + - repo: https://github.com/hadialqattan/pycln + rev: v2.1.1 + hooks: + - id: pycln + args: [--config=pyproject.toml] + + - repo: https://github.com/asottile/yesqa + rev: v1.4.0 + hooks: + - id: yesqa + exclude: docs/conf.py + additional_dependencies: &flake8_dependencies + - flake8-bugbear + - flake8-print + + - repo: https://github.com/pycqa/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + exclude: docs/conf.py + additional_dependencies: *flake8_dependencies + + - repo: https://github.com/pycqa/bandit + rev: 1.7.4 + hooks: + - id: bandit + files: src + + #- repo: https://github.com/pre-commit/mirrors-mypy + # rev: v0.982 + # hooks: + # - id: mypy + # files: src + # args: [--show-error-codes] + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.2 + hooks: + - id: codespell + exclude: | + (?x)^( + .*\.json | + .*\.pem + )$ + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.8.0.4 + hooks: + - id: shellcheck + + - repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache|Github|PyTest + exclude: .pre-commit-config.yaml + + - repo: https://github.com/mgedmin/check-manifest + rev: "0.48" + hooks: + - id: check-manifest + stages: [manual] diff --git a/COPYING b/COPYING index 53d1f3d01864c35841739c55aeba9700657627b7..f288702d2fa16d3cdf0035b15a9fcbc552cd88e7 100644 --- a/COPYING +++ b/COPYING @@ -672,4 +672,3 @@ may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <https://www.gnu.org/licenses/why-not-lgpl.html>. - diff --git a/README.md b/README.md index c592bd3d3343de6b76bc802e0207e81042e50f35..c6c4f9d84b1aaa0769fd0a6ebf1ba12080a831c6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # ITk DB v0.4.0rc6 [](https://badge.fury.io/py/itkdb) -[](https://pepy.tech/project/itkdb) [](https://pepy.tech/project/itkdb) [](https://pepy.tech/project/itkdb) +[](https://pepy.tech/project/itkdb) +[](https://pepy.tech/project/itkdb) +[](https://pepy.tech/project/itkdb) To install as a user @@ -33,20 +35,25 @@ itkdb --help ## Environment Variables -See [itkdb/settings/base.py](src/itkdb/settings/base.py) for all environment variables that can be set. All environment variables for this package are prefixed with `ITKDB_`. As of now, there are: +See [itkdb/settings/base.py](src/itkdb/settings/base.py) for all environment +variables that can be set. All environment variables for this package are +prefixed with `ITKDB_`. As of now, there are: -* `ITKDB_ACCESS_CODE1`: access code #1 for authentication -* `ITKDB_ACCESS_CODE2`: access code #2 for authentication -* `ITKDB_ACCESS_SCOPE`: scope for the access token authentication request -* `ITKDB_AUTH_URL`: authentication server -* `ITKDB_SITE_URL`: API server -* `ITKDB_CASSETTE_LIBRARY_DIR`: for tests, where to store recorded requests/responses +- `ITKDB_ACCESS_CODE1`: access code #1 for authentication +- `ITKDB_ACCESS_CODE2`: access code #2 for authentication +- `ITKDB_ACCESS_SCOPE`: scope for the access token authentication request +- `ITKDB_AUTH_URL`: authentication server +- `ITKDB_SITE_URL`: API server +- `ITKDB_CASSETTE_LIBRARY_DIR`: for tests, where to store recorded + requests/responses ## Develop ### Bump Version -Run `bump2version x.y.z` to bump to the next version. We will always tag any version we bump, and this creates the relevant commits/tags for you. All you need to do is `git push --tags` and that should be it. +Run `bump2version x.y.z` to bump to the next version. We will always tag any +version we bump, and this creates the relevant commits/tags for you. All you +need to do is `git push --tags` and that should be it. # Examples diff --git a/pyproject.toml b/pyproject.toml index 2205b30a7ffd42b5013cc5c1d355005b3a45c7bd..b57a9accc637dc4ca3e99ad1e89180362c0a3156 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,131 @@ [build-system] -# Minimum requirements for the build system to execute. -requires = ["wheel", "setuptools>=30.3.0", "attrs>=17.1", "setuptools_scm"] -build-backend = "setuptools.build_meta" +requires = ["hatchling>=0.7", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "itkdb" +dynamic = ["version"] +authors = [ + { name = "Giordon Stark", email = "kratsg@gmail.com" }, +] +maintainers = [ + { name = "Giordon Stark", email = "kratsg@gmail.com" }, +] + +description = "Python wrapper to interface with ITk DB." +readme = "README.md" + +requires-python = ">=3.7" + +classifiers = [ + "License :: OSI Approved :: BSD License", + "Topic :: Scientific/Engineering", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", + "Development Status :: 1 - Planning", +] + +dependencies = [ + "requests>=1.6.1", # for all HTTP calls to the API + "certifi", # SSL + "cachecontrol[filecache]", # for caching HTTP requests according to spec to local file + "click>=6.0", # for console scripts, + "python-jose", # for id token decoding + "attrs", # for model inflation/deflation + "python-dotenv", # for loading env variables + "simple-settings", # for handling settings more easily + 'importlib_resources; python_version < "3.9"', + "python-magic", # for getting the filetype + "pylibmagic", # for shipping the magic library for python-magic +] + +[project.optional-dependencies] +test = [ + "pytest >=6", + "pytest-cov >=3", + "pytest-mock", + "betamax", # recording api calls for testing + "betamax-serializers", + "requests-mock", +] +dev = [ + "pytest >=6", + "pytest-cov >=3", + "pytest-mock", + "betamax", # recording api calls for testing + "betamax-serializers", + "requests-mock", + "tbump>=6.7.0", +] +docs = [ + "Sphinx>=4.0", + "myst_parser>=0.13", + "sphinx-book-theme>=0.1.0", + "sphinx_copybutton", +] +plotting = [ + "matplotlib" +] + +[project.urls] +Homepage = "https://gitlab.cern.ch/atlas-itk/sw/db/itkdb" +"Bug Tracker" = "https://gitlab.cern.ch/atlas-itk/sw/db/itkdb/issues" +Source = "https://gitlab.cern.ch/atlas-itk/sw/db/itkdb" + +[project.scripts] +itkdb = "itkdb.commandline:itkdb" + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.version.raw-options] +local_scheme = "no-local-version" + +[tool.hatch.build.hooks.vcs] +version-file = "src/itkdb/_version.py" + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config", "--cov-report=term-missing", "--cov-config=.coveragerc", "--cov-report=html", "--doctest-modules", "--doctest-glob='*.rst'"] +xfail_strict = true +filterwarnings = ["error"] +log_cli_level = "INFO" +testpaths = [ + "tests", +] + +[tool.mypy] +files = "src" +python_version = "3.7" +warn_unused_configs = true +strict = true +show_error_codes = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +warn_unreachable = true + +[tool.isort] +profile = "black" + +[tool.pylint] +master.py-version = "3.7" +master.ignore-paths= ["src/itkdb/_version.py"] +reports.output-format = "colorized" +similarities.ignore-imports = "yes" +messages_control.disable = [ + "design", + "fixme", + "line-too-long", + "wrong-import-position", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 309d1c100c4a62afb6ce1d139b9e19fe404d605f..0000000000000000000000000000000000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = --ignore=setup.py --cov=itkdb --cov-report=term-missing --cov-config=.coveragerc --cov-report html --cov-report html --doctest-modules --doctest-glob='*.rst' diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8bbdc21df9c4b8363091cdb946c0b7d0814a2401..0000000000000000000000000000000000000000 --- a/setup.cfg +++ /dev/null @@ -1,17 +0,0 @@ -[bdist_wheel] -universal = 1 - -[metadata] -#license_file = LICENSE - -[options] -setup_requires = - setuptools_scm>=1.15.0 - setuptools_scm_git_archive>=1.0 - -[flake8] -extend-ignore = E203, E501, E722, B950, W503 -select = C,E,F,W,T,B,B9,I -per-file-ignores = - tests/*: T - noxfile.py: T diff --git a/setup.py b/setup.py deleted file mode 100644 index b278d984ccf46254c31830c4113f4695d601c209..0000000000000000000000000000000000000000 --- a/setup.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -from setuptools import setup, find_packages -import os -import sys - -if sys.version_info.major < 3: - from io import open -with open( - os.path.join(os.path.abspath(os.path.dirname(__file__)), "README.md"), - encoding="utf-8", -) as readme_md: - long_description = readme_md.read() - -extras_require = { - "develop": [ - "flake8", - "pytest", - "pytest-cov", - "pytest-mock", - "coverage", - "tbump>=6.7.0", - "pre-commit", - "bandit", - 'black;python_version>="3.6"', # Black is Python3 only - "betamax", # recording api calls for testing - "betamax-serializers", - "twine", # uploading to pypi - "requests-mock", - ], - "plotting": ["matplotlib"], -} -extras_require["complete"] = sorted(set(sum(extras_require.values(), []))) - - -setup( - name="itkdb", - version="0.4.0rc6", - use_scm_version=lambda: {"local_scheme": lambda version: ""}, - package_dir={"": "src"}, - packages=find_packages(where="src", exclude=["tests"]), - include_package_data=True, - description="Python wrapper to interface with ITk DB.", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://gitlab.cern.ch/atlas-itk/sw/db/itkdb", - author="Giordon Stark", - author_email="gstark@cern.ch", - license="", - keywords="physics itk database wrapper", - classifiers=[ - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - ], - python_requires=">=3.7", - install_requires=[ - "requests>=1.6.1", # for all HTTP calls to the API - "certifi", # SSL - "cachecontrol[filecache]", # for caching HTTP requests according to spec to local file - "click>=6.0", # for console scripts, - "python-jose", # for id token decoding - "attrs", # for model inflation/deflation - "python-dotenv", # for loading env variables - "simple-settings", # for handling settings more easily - 'importlib_resources; python_version < "3.9"', - "python-magic", # for getting the filetype - "pylibmagic", # for shipping the magic library for python-magic - ], - extras_require=extras_require, - entry_points={"console_scripts": ["itkdb=itkdb.commandline:itkdb"]}, - dependency_links=[], -) diff --git a/src/itkdb/__init__.py b/src/itkdb/__init__.py index 7a661bb03acdf6ea7798d5d348023df1bda02360..e6f0c220acc9f5233214f9522ada5fd0f082af37 100644 --- a/src/itkdb/__init__.py +++ b/src/itkdb/__init__.py @@ -1,12 +1,12 @@ -from .version import __version__ -from . import core -from . import exceptions -from . import models -from .client import Client -from .settings import settings +from __future__ import annotations import sys +from . import core, exceptions, models +from ._version import __version__ +from .client import Client +from .settings import settings + if sys.version_info >= (3, 9): from importlib import resources else: diff --git a/src/itkdb/caching/__init__.py b/src/itkdb/caching/__init__.py index 64336ee5e39bf83c41a4df0c2d69ac54ee2b850a..6311130845ff1c850cafd7a8e485974dc45ad099 100644 --- a/src/itkdb/caching/__init__.py +++ b/src/itkdb/caching/__init__.py @@ -1,4 +1,6 @@ -from .controller import CacheController +from __future__ import annotations + from .adapter import CacheControlAdapter +from .controller import CacheController __all__ = ["CacheController", "CacheControlAdapter"] diff --git a/src/itkdb/caching/adapter.py b/src/itkdb/caching/adapter.py index f08067fe0dc2abda92125233ed261514a759fe77..2874cf63169de4b2f738d25e744d4db4b483b304 100644 --- a/src/itkdb/caching/adapter.py +++ b/src/itkdb/caching/adapter.py @@ -1,8 +1,11 @@ -import types +from __future__ import annotations + import functools +import types from cachecontrol.adapter import CacheControlAdapter as BaseAdapter from cachecontrol.filewrapper import CallbackFileWrapper + from . import utils @@ -70,7 +73,7 @@ class CacheControlAdapter(BaseAdapter): _update_chunk_length, response ) - resp = super(CacheControlAdapter, self).build_response(request, response) + resp = super().build_response(request, response) # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: diff --git a/src/itkdb/caching/controller.py b/src/itkdb/caching/controller.py index 4dc7d147e7b36a5bfd46804621887e04ad0fdd42..30632696085add304118cb18ad80f9f4daf26cda 100644 --- a/src/itkdb/caching/controller.py +++ b/src/itkdb/caching/controller.py @@ -1,9 +1,13 @@ +from __future__ import annotations + +import calendar import logging import time -import calendar from email.utils import parsedate_tz + from cachecontrol.controller import CacheController as BaseController from requests.structures import CaseInsensitiveDict + from . import utils logger = logging.getLogger(__name__) @@ -204,7 +208,7 @@ class CacheController(BaseController): # Add to the cache any 301s. We do this before looking that # the Date headers. elif response.status == 301: - logger.debug("Caching permanant redirect") + logger.debug("Caching permanent redirect") self.cache.set(cache_url, self.serializer.dumps(request, response)) # Add to the cache if the response headers demand it. If there @@ -251,11 +255,11 @@ class CacheController(BaseController): excluded_headers = ["content-length"] cached_response.headers.update( - dict( - (k, v) + { + k: v for k, v in response.headers.items() if k.lower() not in excluded_headers - ) + } ) # we want a 200 b/c we have content via the cache diff --git a/src/itkdb/caching/utils.py b/src/itkdb/caching/utils.py index a2eed057fc3cf516f8b57e97c7ec1eb5d8f7f79a..4660cda18e810bac7e1ace82d504b68ba0370573 100644 --- a/src/itkdb/caching/utils.py +++ b/src/itkdb/caching/utils.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import json try: # Python 3 - from urllib.parse import urlparse, urlencode + from urllib.parse import urlencode, urlparse except ImportError: # Python 2 - from urlparse import urlparse from urllib import urlencode + from urlparse import urlparse + def build_url(request): if request.body: @@ -18,7 +21,7 @@ def build_url(request): parsed_body = request.body query = "&".join([parsed.query, urlencode({"body": parsed_body})]) # return '{parsed.scheme}://{parsed.netloc}{parsed.path}?{query}' - return "{0:s}://{1:s}{2:s}?{3:s}".format( + return "{:s}://{:s}{:s}?{:s}".format( parsed.scheme, parsed.netloc, parsed.path, query ) return request.url diff --git a/src/itkdb/client.py b/src/itkdb/client.py index 6df774d6419bdd4cc72211fea7c568820de4ead4..490703d6f4183f80cafab024ccd326ae88c94fa5 100644 --- a/src/itkdb/client.py +++ b/src/itkdb/client.py @@ -1,12 +1,13 @@ -from .core import Session -from .responses import PagedResponse -from . import exceptions -from . import models -from . import utilities -import itkdb +from __future__ import annotations -from functools import partial import logging +from functools import partial + +import itkdb + +from . import exceptions, models, utilities +from .core import Session +from .responses import PagedResponse log = logging.getLogger(__name__) @@ -16,7 +17,7 @@ class Client(Session): def __init__(self, use_eos=False, **kwargs): self._use_eos = use_eos - super(Client, self).__init__(**kwargs) + super().__init__(**kwargs) def request(self, method, url, *args, **kwargs): self.limit = kwargs.pop("limit", -1) @@ -26,17 +27,17 @@ class Client(Session): warnings = data.pop("uuAppErrorMap", {}) try: for key, message in warnings.items(): - log.warning("{key}: {message}".format(key=key, message=message)) + log.warning(f"{key}: {message}") except AttributeError: # it's a string like: # 'uuAppErrorMap': '#<UuApp::Oidc::Session:0x00561d53890118>' - log.warning("{message}".format(message=warnings)) + log.warning(f"{warnings}") def upload_to_eos(self, response, *args, eos_file_details=None, **kwargs): log.info("I was able to get a token to upload to EOS. Let me upload.") try: response.raise_for_status() - except: + except BaseException: log.warning("Something went wrong with uploading to EOS.") return response @@ -53,7 +54,7 @@ class Client(Session): headers = {"Content-Type": ft} headers.update(fh) - response = super(Client, self).put(url, data=fp, verify=chain, headers=headers) + response = super().put(url, data=fp, verify=chain, headers=headers) def _request_handler(self, request): if request.url == self._normalize_url("/itkdbPoisonPillTest"): @@ -188,10 +189,10 @@ class Client(Session): def prepare_request(self, request): request.url = self._normalize_url(request.url) self._request_handler(request) - return super(Client, self).prepare_request(request) + return super().prepare_request(request) def send(self, request, **kwargs): - response = super(Client, self).send(request, **kwargs) + response = super().send(request, **kwargs) # sometimes we don't get content-type, so make sure it's a string at least content_type = response.headers.get("content-type") @@ -205,23 +206,19 @@ class Client(Session): try: data = response.json() self._handle_warnings(data) - except ValueError: - raise exceptions.BadJSON(response) + except ValueError as err: + raise exceptions.BadJSON(response) from err limit = self.limit self.limit = -1 # reset the limit again if "pageItemList" in data: - return PagedResponse( - super(Client, self), response, limit=limit, key="pageItemList" - ) + return PagedResponse(super(), response, limit=limit, key="pageItemList") elif "itemList" in data: pageInfo = data.get("pageInfo", None) if pageInfo and ( pageInfo["pageIndex"] * pageInfo["pageSize"] < pageInfo["total"] ): - return PagedResponse( - super(Client, self), response, limit=limit, key="itemList" - ) + return PagedResponse(super(), response, limit=limit, key="itemList") return data["itemList"] elif "testRunList" in data: return data["testRunList"] @@ -238,9 +235,7 @@ class Client(Session): elif content_type == "application/zip": return models.ZipFile.from_response(response) else: - log.warning( - "Do not know how to handle Content-Type: {0:s}.".format(content_type) - ) + log.warning(f"Do not know how to handle Content-Type: {content_type:s}.") return response @property diff --git a/src/itkdb/commandline.py b/src/itkdb/commandline.py index e76e45aeabb702ff38777549ced8797887604bb9..3d417a8376a2f85c3a367592d114437d5b0bd588 100644 --- a/src/itkdb/commandline.py +++ b/src/itkdb/commandline.py @@ -1,13 +1,14 @@ +from __future__ import annotations + +import json import logging +import os import sys + import click -import json -import os -from .version import __version__ -from . import core -from . import settings -from . import utilities +from . import core, settings, utilities +from ._version import __version__ logging.basicConfig(format=utilities.FORMAT_STRING, level=logging.INFO) log = logging.getLogger(__name__) @@ -211,15 +212,16 @@ def add_attachment(component, title, description, file, filename, file_type): "type": "file", "url": filename, } - attachment = {"data": (filename, open(file, "rb"), file_type)} - click.echo( - json.dumps( - _session.post( - "createComponentAttachment", data=data, files=attachment - ).json(), - indent=2, + with open(file, "rb") as fp: + attachment = {"data": (filename, fp, file_type)} + click.echo( + json.dumps( + _session.post( + "createComponentAttachment", data=data, files=attachment + ).json(), + indent=2, + ) ) - ) sys.exit(0) diff --git a/src/itkdb/core.py b/src/itkdb/core.py index 53c6a3bba767f76119ce72832d22ae3e816ba8e9..202ce0188c317d23c6bfd702f2fcf97f6cab2bab 100644 --- a/src/itkdb/core.py +++ b/src/itkdb/core.py @@ -1,23 +1,25 @@ +from __future__ import annotations + import logging -import requests -from requests.status_codes import codes +import os +import pickle # nosec +import time + import cachecontrol.caches.file_cache +import requests from cachecontrol.heuristics import ExpiresAfter -from .caching import CacheControlAdapter, CacheController - from jose import jwt -import time -import os -import pickle # nosec +from requests.status_codes import codes -from .settings import settings from . import exceptions -from .version import __version__ +from ._version import __version__ +from .caching import CacheControlAdapter, CacheController +from .settings import settings log = logging.getLogger(__name__) -class User(object): +class User: def __init__( self, accessCode1=settings.ITKDB_ACCESS_CODE1, @@ -30,7 +32,7 @@ class User(object): # session handling (for injection in tests) self._session = requests.Session() - self._session.headers.update({"User-Agent": "itkdb/{}".format(__version__)}) + self._session.headers.update({"User-Agent": f"itkdb/{__version__}"}) # store last call to authenticate self._response = None self._status_code = None @@ -55,7 +57,8 @@ class User(object): def _load(self): if self._save_auth and os.path.isfile(self._save_auth): try: - saved_user = pickle.load(open(self._save_auth, "rb")) # nosec + with open(self._save_auth, "rb") as _pickle_file: + saved_user = pickle.load(_pickle_file) # nosec if saved_user.is_expired(): log.warning( "Saved user session is expired in {}. Creating a new one.".format( @@ -78,17 +81,16 @@ class User(object): self._save_auth ) ) - return False + return False def _dump(self): if self.is_authenticated() and not self.is_expired() and self._save_auth: - try: - pickle.dump(self, open(self._save_auth, "wb"), pickle.HIGHEST_PROTOCOL) - return True - except (pickle.PicklingError, AttributeError, TypeError): - log.warning( - "Unable to save user session to {}.".format(self._save_auth) - ) + with open(self._save_auth, "wb") as fp: + try: + pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL) + return True + except (pickle.PicklingError, AttributeError, TypeError): + log.warning(f"Unable to save user session to {self._save_auth}.") return False def _load_jwks(self, force=False): @@ -187,7 +189,7 @@ class User(object): return not (self.expires_in > 0) def __repr__(self): - return "{0:s}(name={1:s}, expires_in={2:d}s)".format( + return "{:s}(name={:s}, expires_in={:d}s)".format( self.__class__.__name__, self.name, self.expires_in ) @@ -215,18 +217,18 @@ class Session(requests.Session): user=None, prefix_url=settings.ITKDB_SITE_URL, save_auth=None, - cache=cachecontrol.caches.file_cache.FileCache(".webcache"), + cache=True, expires_after=None, ): """ user (itkdb.core.User): A user object. Create one if not specified. prefix_url (str): The prefix url to use for all requests. save_auth (str): A file path to where to save authentication information. - cache (str): A CacheControl.caches object for cache (default: cachecontrol.caches.file_cache.FileCache) + cache (str): A CacheControl.caches object for cache (default: cachecontrol.caches.file_cache.FileCache). Set to False to disable cache. expires_after (dict): The arguments are the same as the datetime.timedelta object. This will override or add the Expires header and override or set the Cache-Control header to public. """ - super(Session, self).__init__() - self.headers.update({"User-Agent": "itkdb/{}".format(__version__)}) + super().__init__() + self.headers.update({"User-Agent": f"itkdb/{__version__}"}) self.user = user if user else User(save_auth=save_auth) self.auth = self._authorize self.prefix_url = prefix_url @@ -235,13 +237,20 @@ class Session(requests.Session): cache_options = {} if cache: + cache = ( + cachecontrol.caches.file_cache.FileCache(".webcache") + if cache is True + else cache + ) cache_options.update(dict(cache=cache)) + # handle expirations for cache if expires_after and isinstance(expires_after, dict): cache_options.update(dict(heuristic=ExpiresAfter(**expires_after))) + if cache_options: # add caching - super(Session, self).mount( + super().mount( self.prefix_url, CacheControlAdapter(controller_class=CacheController, **cache_options), ) @@ -249,9 +258,7 @@ class Session(requests.Session): def _authorize(self, req): if req.url.startswith(settings.ITKDB_SITE_URL): self.user.authenticate() - req.headers.update( - {"Authorization": "Bearer {0:s}".format(self.user.bearer)} - ) + req.headers.update({"Authorization": f"Bearer {self.user.bearer:s}"}) return req def _normalize_url(self, url): @@ -263,15 +270,15 @@ class Session(requests.Session): try: response.raise_for_status() - except: - raise exceptions.UnhandledResponse(response) + except BaseException as err: + raise exceptions.UnhandledResponse(response) from err def prepare_request(self, request): request.url = self._normalize_url(request.url) - return super(Session, self).prepare_request(request) + return super().prepare_request(request) def send(self, request, **kwargs): - response = super(Session, self).send(request, **kwargs) + response = super().send(request, **kwargs) self._response = response log.debug( "Response: {} ({} bytes)".format( @@ -283,7 +290,7 @@ class Session(requests.Session): def request(self, method, url, *args, **kwargs): url = self._normalize_url(url) - return super(Session, self).request(method, url, *args, **kwargs) + return super().request(method, url, *args, **kwargs) def __call__(self, *args, **kwargs): if len(args) == 1: diff --git a/src/itkdb/data/README.md b/src/itkdb/data/README.md index 420780ea3a389d13bfdc2d1a760242d0ce0ac489..13761dbfdd494e95c53436557f90260f3cc39441 100644 --- a/src/itkdb/data/README.md +++ b/src/itkdb/data/README.md @@ -2,9 +2,17 @@ ## SSL (CERN_chain.pem) -In order to get SSL handshakes working (certificate verification), one needs to make sure we add/trust the CERN Certification Authorities (CA) for both the Root and the Grid CAs. Specifically, we rely on the Root CA to sign/issue the Grid CA. The Grid CA is what's relied on for the SSL chain. To make this happen, we'll need both PEM for each CA combined into a single `CERN_chain.pem` file which is bundled up with this package. - -Going to the [CERN CA Files website](https://cafiles.cern.ch/cafiles/) and downloading the CERN Root Certification Authority 2 (DER file) and CERN Grid Certification Authority (PEM file). We can then convert the DER to PEM as follows (for the Root CA): +In order to get SSL handshakes working (certificate verification), one needs to +make sure we add/trust the CERN Certification Authorities (CA) for both the Root +and the Grid CAs. Specifically, we rely on the Root CA to sign/issue the Grid +CA. The Grid CA is what's relied on for the SSL chain. To make this happen, +we'll need both PEM for each CA combined into a single `CERN_chain.pem` file +which is bundled up with this package. + +Going to the [CERN CA Files website](https://cafiles.cern.ch/cafiles/) and +downloading the CERN Root Certification Authority 2 (DER file) and CERN Grid +Certification Authority (PEM file). We can then convert the DER to PEM as +follows (for the Root CA): ``` openssl x509 -in CERN_ROOT_CA_2.crt -inform der -outform pem -out CERN_ROOT_CA_2.pem @@ -16,12 +24,17 @@ and then combine the two cat CERN_GRID_CA_2.pem CERN_ROOT_CA_2.pem > CERN_chain.pem ``` -This can be passed into any python `requests::Session` via `verify='/path/to/CERN_chain.pem'` and SSL verification should work. +This can be passed into any python `requests::Session` via +`verify='/path/to/CERN_chain.pem'` and SSL verification should work. -[1] [DER vs PEM?](https://support.ssl.com/Knowledgebase/Article/View/19/0/der-vs-crt-vs-cer-vs-pem-certificates-and-how-to-convert-them) +[1] +[DER vs PEM?](https://support.ssl.com/Knowledgebase/Article/View/19/0/der-vs-crt-vs-cer-vs-pem-certificates-and-how-to-convert-them) ## Image -A 1x1 pixel image is used for testing image uploads in the database. This is done via `1x1.sh` and `1x1.jpg` which should be good enough integration testing. +A 1x1 pixel image is used for testing image uploads in the database. This is +done via `1x1.sh` and `1x1.jpg` which should be good enough integration testing. -See [stackoverflow](https://stackoverflow.com/questions/2253404/what-is-the-smallest-valid-jpeg-file-size-in-bytes) for where this came from. +See +[stackoverflow](https://stackoverflow.com/questions/2253404/what-is-the-smallest-valid-jpeg-file-size-in-bytes) +for where this came from. diff --git a/src/itkdb/exceptions.py b/src/itkdb/exceptions.py index 0a1decf6a2725998da73fe5c67a566c051bf7bf0..65f92bcc146daf81d28bc50af9f6cc7cd1afeb3b 100644 --- a/src/itkdb/exceptions.py +++ b/src/itkdb/exceptions.py @@ -1,7 +1,10 @@ """Provide exception classes for the itkdb package.""" -import sys +from __future__ import annotations + import json +import sys + from .utilities import pretty_print try: @@ -45,7 +48,7 @@ class ResponseException(ITkDBException): message = "{}\n\nThe following details may help:\n{}".format( message, additional_message ) - super(ResponseException, self).__init__(message) + super().__init__(message) class BadJSON(ResponseException): @@ -74,7 +77,7 @@ class Forbidden(ResponseException): ) except JSONDecodeError: pass - super(Forbidden, self).__init__(response, additional_message) + super().__init__(response, additional_message) class NotFound(ResponseException): @@ -99,7 +102,7 @@ class Redirect(ResponseException): path = urlparse(response.headers["location"]).path self.path = path[:-5] if path.endswith(".json") else path self.response = response - ITkDBException.__init__(self, "Redirect to {}".format(self.path)) + ITkDBException.__init__(self, f"Redirect to {self.path}") class ServerError(ResponseException): @@ -122,7 +125,7 @@ class SpecialError(ResponseException): self.message = resp_dict.get("message", "") self.reason = resp_dict.get("reason", "") self.special_errors = resp_dict.get("special_errors", []) - ITkDBException.__init__(self, "Special error {!r}".format(self.message)) + ITkDBException.__init__(self, f"Special error {self.message!r}") class TooLarge(ResponseException): @@ -130,7 +133,7 @@ class TooLarge(ResponseException): class UnavailableForLegalReasons(ResponseException): - """Indicate that the requested URL is unavilable due to legal reasons.""" + """Indicate that the requested URL is unavailable due to legal reasons.""" class UnhandledResponse(ResponseException): diff --git a/src/itkdb/models/__init__.py b/src/itkdb/models/__init__.py index 794df282f72096c91f68b60441115e9e0d1d4155..69ee64ce6d04641f35963cd3bb2beec075c4b340 100644 --- a/src/itkdb/models/__init__.py +++ b/src/itkdb/models/__init__.py @@ -1,6 +1,8 @@ +from __future__ import annotations + +from . import institution from .image import Image from .text import Text from .zip import ZipFile -from . import institution __all__ = ["Image", "Text", "ZipFile", "institution"] diff --git a/src/itkdb/models/image.py b/src/itkdb/models/image.py index b37c760dabde1782d4fdc826400301e07c6d3267..82a024c8a0cfe24cf5eb36fd3a3ef6bd77e6f7ee 100644 --- a/src/itkdb/models/image.py +++ b/src/itkdb/models/image.py @@ -1,6 +1,8 @@ -from io import BytesIO -import re +from __future__ import annotations + import logging +import re +from io import BytesIO log = logging.getLogger(__name__) @@ -9,14 +11,14 @@ class Image(BytesIO): def __init__(self, content=None, filename=None): self.filename = filename self.format = filename.split(".")[-1].lower() - super(Image, self).__init__(content) + super().__init__(content) def save(self, filename=None, mode="wb"): filename = filename or self.filename nbytes = len(self.getvalue()) with open(filename, mode) as f: f.write(self.read()) - log.info("Written {0:d} bytes to {1:s}".format(len(self.getvalue()), filename)) + log.info(f"Written {len(self.getvalue()):d} bytes to {filename:s}") return nbytes def _repr_png_(self): diff --git a/src/itkdb/models/institution.py b/src/itkdb/models/institution.py index afa059c78ff98c9997dfb07bcd91e59adcf042e7..b0f76ea0f1c3b8d4e56d0069f2c4cd75145a1ac9 100644 --- a/src/itkdb/models/institution.py +++ b/src/itkdb/models/institution.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import attr @@ -48,28 +50,28 @@ def make_institution_list(data): @attr.s -class Item(object): +class Item: code = attr.ib() id = attr.ib() name = attr.ib() @attr.s -class ComponentType(object): +class ComponentType: code = attr.ib() itemList = attr.ib() name = attr.ib() @attr.s -class Contact(object): +class Contact: email = attr.ib() phone = attr.ib() web = attr.ib() @attr.s -class Address(object): +class Address: building = attr.ib() city = attr.ib() state = attr.ib() @@ -78,7 +80,7 @@ class Address(object): @attr.s -class Institution(object): +class Institution: address = attr.ib() code = attr.ib() componentType = attr.ib() @@ -91,5 +93,5 @@ class Institution(object): @attr.s -class InstitutionList(object): +class InstitutionList: institutions = attr.ib(type=list) diff --git a/src/itkdb/models/text.py b/src/itkdb/models/text.py index 7001895355168d7c1f00228d0b225e7c69f854c5..ebbd47fbe243d7c864e69fac81106874abc1752d 100644 --- a/src/itkdb/models/text.py +++ b/src/itkdb/models/text.py @@ -1,6 +1,8 @@ -from io import BytesIO -import re +from __future__ import annotations + import logging +import re +from io import BytesIO log = logging.getLogger(__name__) @@ -9,14 +11,14 @@ class Text(BytesIO): def __init__(self, content=None, filename=None): self.filename = filename self.format = filename.split(".")[-1].lower() - super(Text, self).__init__(content) + super().__init__(content) def save(self, filename=None, mode="wb"): filename = filename or self.filename nbytes = len(self.getvalue()) with open(filename, mode) as f: f.write(self.read()) - log.info("Written {0:d} bytes to {1:s}".format(len(self.getvalue()), filename)) + log.info(f"Written {len(self.getvalue()):d} bytes to {filename:s}") return nbytes def _repr_html_(self): diff --git a/src/itkdb/models/zip.py b/src/itkdb/models/zip.py index d52d6fc304fce2ed10e645ae3f32a40190b97092..4c5d5ce52b31b4120d6a6cbca7e84f0081e3423e 100644 --- a/src/itkdb/models/zip.py +++ b/src/itkdb/models/zip.py @@ -1,8 +1,10 @@ -from io import BytesIO -import zipfile -import re -import logging +from __future__ import annotations + import html +import logging +import re +import zipfile +from io import BytesIO log = logging.getLogger(__name__) @@ -10,7 +12,7 @@ log = logging.getLogger(__name__) class ZipFile(zipfile.ZipFile): def __init__(self, content=None, filename=None): self._content = content - super(ZipFile, self).__init__(BytesIO(self._content)) + super().__init__(BytesIO(self._content)) self.filename = filename self.format = filename.split(".")[-1].lower() @@ -19,7 +21,7 @@ class ZipFile(zipfile.ZipFile): nbytes = len(self._content) with open(filename, mode) as f: f.write(self._content) - log.info("Written {0:d} bytes to {1:s}".format(nbytes, filename)) + log.info(f"Written {nbytes:d} bytes to {filename:s}") return nbytes def _repr_html_(self): diff --git a/src/itkdb/responses.py b/src/itkdb/responses.py index 5fec8abf19509c60339de35bade4a4a4071fec04..4447f13ffad57c261aab329448dc573da08e8ec9 100644 --- a/src/itkdb/responses.py +++ b/src/itkdb/responses.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import json import logging log = logging.getLogger(__name__) -class PagedResponse(object): +class PagedResponse: def __init__(self, session, response, limit=-1, key="pageItemList"): self._pages = [] self._session = session diff --git a/src/itkdb/settings/__init__.py b/src/itkdb/settings/__init__.py index 30c61493aff5a8305dbf4afd3bf9f40533410116..df4e463f6a32fdf8bf9e9a734a5b632af79fe722 100644 --- a/src/itkdb/settings/__init__.py +++ b/src/itkdb/settings/__init__.py @@ -1,5 +1,8 @@ -from dotenv import load_dotenv +from __future__ import annotations + import os + +from dotenv import load_dotenv from simple_settings import settings load_dotenv(dotenv_path=os.path.join(os.getcwd(), ".env")) diff --git a/src/itkdb/settings/base.py b/src/itkdb/settings/base.py index 0a4344467f66d97c36684765ac51e2902d2a6465..b77f31d4d215bac8b6d38e3b3f0863123875cf5d 100644 --- a/src/itkdb/settings/base.py +++ b/src/itkdb/settings/base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + SIMPLE_SETTINGS = {"OVERRIDE_BY_ENV": True} ITKDB_ACCESS_CODE1 = "" ITKDB_ACCESS_CODE2 = "" diff --git a/src/itkdb/utilities.py b/src/itkdb/utilities.py index d63a961f39722ecd493977b1395fce101b887001..44c55b9f60f8773271ad072b60880e5264db6eaf 100644 --- a/src/itkdb/utilities.py +++ b/src/itkdb/utilities.py @@ -1,13 +1,17 @@ +from __future__ import annotations + import logging -import requests -from urllib.parse import urlencode, urlparse, parse_qs -import pylibmagic # NOQA -import magic import os +from urllib.parse import parse_qs, urlencode, urlparse + +import pylibmagic # noqa: F401 +import requests + +import magic # isort: skip # The background is set with 40 plus the number of the color, and the foreground with 30 -# These are the sequences need to get colored ouput +# These are the sequences need to get colored output def _get_color_seq(i): COLOUR_SEQ = "\033[1;{0:d}m" return COLOUR_SEQ.format(30 + i) diff --git a/src/itkdb/version.py b/src/itkdb/version.py deleted file mode 100644 index 14c9e5d7c7fb9a5f6b0869f9db1bca49835df23d..0000000000000000000000000000000000000000 --- a/src/itkdb/version.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Define itkdb version information.""" - -# Use semantic versioning (https://semver.org/) -# The version number is controlled through bumpversion.cfg -__version__ = "0.4.0rc6" diff --git a/tbump.toml b/tbump.toml index 35ab9363a2dace009501081aa56f257a4464b826..6b69d72b83398d1b1107c1faa506ca2abf66c5ea 100644 --- a/tbump.toml +++ b/tbump.toml @@ -33,12 +33,6 @@ search = "Bump version: {current_version} → " [[file]] src = "README.md" -[[file]] -src = "setup.py" - -[[file]] -src = "src/itkdb/version.py" - [[field]] # the name of the field name = "candidate" diff --git a/tests/conftest.py b/tests/conftest.py index 7950bce690232715b0110495def6de6eb88c1565..bf6df3cb655fa43b376f426ca291066fe0c7c0ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,18 @@ -import socket -from sys import platform -import itkdb +from __future__ import annotations -import betamax -from betamax_serializers import pretty_json import base64 import gzip - import json -import pytest - +import socket import time +from sys import platform +from urllib.parse import parse_qs, quote, urlparse -from urllib.parse import urlparse, parse_qs, quote +import betamax +import pytest +from betamax_serializers import pretty_json + +import itkdb placeholders = { "accessCode1": itkdb.settings.ITKDB_ACCESS_CODE1, @@ -81,7 +81,7 @@ with betamax.Betamax.configure() as config: config.before_record(callback=filter_requests_record) config.before_playback(callback=filter_requests_playback) for key, value in placeholders.items(): - config.define_cassette_placeholder("<{}>".format(key.upper()), replace=value) + config.define_cassette_placeholder(f"<{key.upper()}>", replace=value) @pytest.fixture(scope="session") diff --git a/tests/integration/test_attachments.py b/tests/integration/test_attachments.py index 3bc46b2d14f67e95b227ca821186f26e8ca924c3..3d11dcf4367f7377a0ad67b3223ecabc2637ffb4 100644 --- a/tests/integration/test_attachments.py +++ b/tests/integration/test_attachments.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import betamax diff --git a/tests/integration/test_binaryData.py b/tests/integration/test_binaryData.py index 19925dc0d9f2c0dba745ef5a78a86b6bab62a933..a193fbbcf46e68341b75558baa20724277cf913f 100644 --- a/tests/integration/test_binaryData.py +++ b/tests/integration/test_binaryData.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import betamax + import itkdb diff --git a/tests/integration/test_components.py b/tests/integration/test_components.py index 4a54dd3c8ba080c4849b04179e5903867a45d0ed..1de13c1b0ba147e427267640edf62548f76cecb9 100644 --- a/tests/integration/test_components.py +++ b/tests/integration/test_components.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import betamax + import itkdb @@ -163,16 +166,17 @@ def test_create_attachment_image_eos(tmpdir, auth_client, monkeypatch): "getComponent", json={"component": "7f633f626f5466b2a72c1be7cd4cb8bc"} ) - data = { - "component": "7f633f626f5466b2a72c1be7cd4cb8bc", - "title": "MyTestAttachment", - "description": "This is a test attachment descriptor", - "type": "file", - "url": image, - } - attachment = {"data": (image.name, image.open("rb"), "image/jpeg")} - - auth_client.post("createComponentAttachment", data=data, files=attachment) + with image.open("rb") as fp: + data = { + "component": "7f633f626f5466b2a72c1be7cd4cb8bc", + "title": "MyTestAttachment", + "description": "This is a test attachment descriptor", + "type": "file", + "url": image, + } + attachment = {"data": (image.name, fp, "image/jpeg")} + + auth_client.post("createComponentAttachment", data=data, files=attachment) component_after = auth_client.get( "getComponent", json={"component": "7f633f626f5466b2a72c1be7cd4cb8bc"} diff --git a/tests/integration/test_institution.py b/tests/integration/test_institution.py index 930434cc5842967964a0a669596fe16242a0ce91..a9b0bb706f47df7953726baf5dabe8cb37265e31 100644 --- a/tests/integration/test_institution.py +++ b/tests/integration/test_institution.py @@ -1,6 +1,9 @@ -import itkdb +from __future__ import annotations + import betamax +import itkdb + def test_get(auth_session): with betamax.Betamax(auth_session).use_cassette("test_institution.test_get"): diff --git a/tests/integration/test_projects.py b/tests/integration/test_projects.py index e2c16802357010473460c74dc205abb27824e2e3..e1e99f59724127dbf9c65d5638f3d9e326efc503 100644 --- a/tests/integration/test_projects.py +++ b/tests/integration/test_projects.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import betamax diff --git a/tests/integration/test_session.py b/tests/integration/test_session.py index 0764f0e8c2a7a8cdf796fa737ef20995682d8a6b..f9be09d9e5bf1117d770232d6e759a3739d265e3 100644 --- a/tests/integration/test_session.py +++ b/tests/integration/test_session.py @@ -1,7 +1,10 @@ -import itkdb +from __future__ import annotations + import betamax import pytest +import itkdb + @pytest.mark.xfail def test_fake_route(auth_session): diff --git a/tests/integration/test_shipments.py b/tests/integration/test_shipments.py index 1f957e68cc6ddb27301ae2c06386af948f6045eb..52f2e0f2f71296ca49964dd3535b86d1a214ed55 100644 --- a/tests/integration/test_shipments.py +++ b/tests/integration/test_shipments.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import betamax + import itkdb @@ -14,16 +17,17 @@ def test_create_attachment_image_eos(tmpdir, auth_client, monkeypatch): "getShipment", json={"shipment": "61149203db062f000b98a75a"} ) - data = { - "shipment": "61149203db062f000b98a75a", - "title": "MyTestAttachment", - "description": "This is a test attachment descriptor", - "type": "file", - "url": image, - } - attachment = {"data": (image.name, image.open("rb"), "image/jpeg")} - - auth_client.post("createShipmentAttachment", data=data, files=attachment) + with image.open("rb") as fp: + data = { + "shipment": "61149203db062f000b98a75a", + "title": "MyTestAttachment", + "description": "This is a test attachment descriptor", + "type": "file", + "url": image, + } + attachment = {"data": (image.name, fp, "image/jpeg")} + + auth_client.post("createShipmentAttachment", data=data, files=attachment) shipment_after = auth_client.get( "getShipment", json={"shipment": "61149203db062f000b98a75a"} diff --git a/tests/integration/test_stats.py b/tests/integration/test_stats.py index 1c2873e64af0852fd768356f674bd78c25ae7a0e..3a57ce57b1ba26a3dc0722e0189c9bca2a03eb9e 100644 --- a/tests/integration/test_stats.py +++ b/tests/integration/test_stats.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import betamax diff --git a/tests/integration/test_summary.py b/tests/integration/test_summary.py index e04e690f2e5c5cf2eef14dff9afc27687d93dbc1..e3cdb8fde54214c23aef0062116c08ba4aeff0af 100644 --- a/tests/integration/test_summary.py +++ b/tests/integration/test_summary.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import betamax diff --git a/tests/integration/test_testproperties.py b/tests/integration/test_testproperties.py index fcc2acf156479c5e6424ce559670a2fb0b13cb85..0781daa29471a187b71b3af9ffcbc0112ae60404 100644 --- a/tests/integration/test_testproperties.py +++ b/tests/integration/test_testproperties.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import betamax diff --git a/tests/integration/test_tests.py b/tests/integration/test_tests.py index 3835196c72edec1cb222b43d45ab95946d695c67..e548294a8c4e0be837892ba6a29d8e7b72c75608 100644 --- a/tests/integration/test_tests.py +++ b/tests/integration/test_tests.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import betamax + import itkdb @@ -29,16 +32,17 @@ def test_create_attachment_image_eos(tmpdir, auth_client, monkeypatch): json={"testRun": "5dde2c1279bc5c000a61d5e2", "outputType": "object"}, ) - data = { - "testRun": "5dde2c1279bc5c000a61d5e2", - "title": "MyTestAttachment", - "description": "This is a test attachment descriptor", - "type": "file", - "url": image, - } - attachment = {"data": (image.name, image.open("rb"), "image/jpeg")} - - auth_client.post("createTestRunAttachment", data=data, files=attachment) + with image.open("rb") as fp: + data = { + "testRun": "5dde2c1279bc5c000a61d5e2", + "title": "MyTestAttachment", + "description": "This is a test attachment descriptor", + "type": "file", + "url": image, + } + attachment = {"data": (image.name, fp, "image/jpeg")} + + auth_client.post("createTestRunAttachment", data=data, files=attachment) testRun_after = auth_client.get( "getTestRun", diff --git a/tests/integration/test_user.py b/tests/integration/test_user.py index 17cfc23bc90c353b5d4b6f61da0b2d25ea3d2c10..e9cfcb0f2e2026dcbb3e646b1121a3cac3e09979 100644 --- a/tests/integration/test_user.py +++ b/tests/integration/test_user.py @@ -1,8 +1,12 @@ -import itkdb +from __future__ import annotations + import logging + import betamax import pytest +import itkdb + # because expiration will fail (since we cache this information) skip the verification of expiration jwtOptions = { "verify_signature": False, diff --git a/tests/integration/test_warning.py b/tests/integration/test_warning.py index 33e9cc2165015331be75813c7df6854bc2c56ca3..7830b11b74745acc389331f13a45198e7318a99e 100644 --- a/tests/integration/test_warning.py +++ b/tests/integration/test_warning.py @@ -1,6 +1,9 @@ -import betamax +from __future__ import annotations + import logging +import betamax + def test_get_component(auth_client, caplog): with betamax.Betamax(auth_client).use_cassette("test_warnings.test_get_component"): diff --git a/tests/test_cli.py b/tests/test_cli.py index 86eacbd879bac195fcbc29406ab74e73af558807..b22ce69716880b4d09149922b3260380c25dcfed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,16 +1,18 @@ -import pytest -from click.testing import CliRunner +from __future__ import annotations import time + import betamax +import pytest +from click.testing import CliRunner import itkdb from itkdb import commandline -@pytest.fixture(scope="module") -def recorder_session(auth_user): - commandline._session.user = auth_user +@pytest.fixture(scope="function") +def recorder_session(auth_user, monkeypatch): + monkeypatch.setattr(commandline._session, "user", auth_user) with betamax.Betamax( commandline._session, cassette_library_dir=itkdb.settings.ITKDB_CASSETTE_LIBRARY_DIR, @@ -147,9 +149,9 @@ def test_getSummary(recorder_session): assert result.output -def test_addAttachment(recorder_session, tmpdir): - temp = tmpdir.join("test.txt") - temp.write("this is a fake attachment for testing purposes") +def test_addAttachment(recorder_session, tmp_path): + temp = tmp_path / "test.txt" + temp.write_text("this is a fake attachment for testing purposes") recorder_session.use_cassette("test_attachments.test_add_attachment", record="none") runner = CliRunner() @@ -164,14 +166,14 @@ def test_addAttachment(recorder_session, tmpdir): "-d", '"delete this attachment if you see it"', "-f", - temp.strpath, + temp, ], ) assert result.exit_code == 0 assert result.output -def test_addComment(recorder_session, tmpdir): +def test_addComment(recorder_session): recorder_session.use_cassette("test_components.test_add_comment", record="none") runner = CliRunner() result = runner.invoke( @@ -184,5 +186,6 @@ def test_addComment(recorder_session, tmpdir): '"this is a test message"', ], ) + assert result.exit_code == 0 assert result.output diff --git a/tests/test_client.py b/tests/test_client.py index cb7df96cb670ba81fabbc3cc05b55de3393b38fc..96703c07503af608e44f34f1826f52392f025f8f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,8 +1,12 @@ -import itkdb -import betamax +from __future__ import annotations + import math + +import betamax import requests +import itkdb + def test_client(auth_user): assert itkdb.Client(user=auth_user) @@ -101,22 +105,23 @@ def test_request_handler_noop(auth_client): def test_request_handler_createComponentAttachment_noEOS(auth_client): image = itkdb.data / "1x1.jpg" - req = requests.Request( - "POST", - "createComponentAttachment", - data={ - "component": "7f633f626f5466b2a72c1be7cd4cb8bc", - "title": "MyTestAttachment", - "description": "This is a test attachment descriptor", - "type": "file", - "url": image, - }, - files={"data": (image.name, image.open("rb"), "image/jpeg")}, - ) + with image.open("rb") as fp: + req = requests.Request( + "POST", + "createComponentAttachment", + data={ + "component": "7f633f626f5466b2a72c1be7cd4cb8bc", + "title": "MyTestAttachment", + "description": "This is a test attachment descriptor", + "type": "file", + "url": image, + }, + files={"data": (image.name, fp, "image/jpeg")}, + ) - prepped = auth_client.prepare_request(req) + prepped = auth_client.prepare_request(req) - auth_client._request_handler(prepped) + auth_client._request_handler(prepped) assert prepped.url == auth_client._normalize_url("/createComponentAttachment") assert prepped.headers.get("content-type") != "application/json" @@ -129,23 +134,24 @@ def test_request_handler_createComponentAttachment_noEOS(auth_client): def test_request_handler_createComponentAttachment(auth_client, monkeypatch): image = itkdb.data / "1x1.jpg" - req = requests.Request( - "POST", - "createComponentAttachment", - data={ - "component": "7f633f626f5466b2a72c1be7cd4cb8bc", - "title": "MyTestAttachment", - "description": "This is a test attachment descriptor", - "type": "file", - "url": image, - }, - files={"data": (image.name, image.open("rb"), "image/jpeg")}, - ) + with image.open("rb") as fp: + req = requests.Request( + "POST", + "createComponentAttachment", + data={ + "component": "7f633f626f5466b2a72c1be7cd4cb8bc", + "title": "MyTestAttachment", + "description": "This is a test attachment descriptor", + "type": "file", + "url": image, + }, + files={"data": (image.name, fp, "image/jpeg")}, + ) - monkeypatch.setattr(auth_client, "_use_eos", True) - prepped = auth_client.prepare_request(req) + monkeypatch.setattr(auth_client, "_use_eos", True) + prepped = auth_client.prepare_request(req) - auth_client._request_handler(prepped) + auth_client._request_handler(prepped) assert prepped.url == auth_client._normalize_url("/requestUploadEosFile") assert prepped.headers.get("content-type") == "application/json" diff --git a/tests/test_image.py b/tests/test_image.py index 841cb9e8fd5effb8f5b5f41ef7a279f4007fa00e..d9cc545e1072f3cad80884fa12b05de141337e91 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import pytest + import itkdb diff --git a/tests/test_response.py b/tests/test_response.py index b493d83519d8b56930594046c21fe010cc943132..eb37cac1142e6d1a0f2af1c22cbd4f19380266d5 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import itkdb diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 9413bfd2a2f4c3ed73f3eaf01b566fba6e2f95ee..a464929a78218015ec9ada7ea018cb92768bb624 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -1,16 +1,16 @@ -import pytest +from __future__ import annotations + +import os +import sys +from types import SimpleNamespace + import betamax +import pytest + +import getInventory import itkdb -import sys -import os sys.path.append(os.path.realpath(os.path.dirname(__file__) + "/..")) -import getInventory # NOQA - -try: - from types import SimpleNamespace -except: - from argparse import Namespace as SimpleNamespace @pytest.mark.parametrize( @@ -38,7 +38,7 @@ def test_getInventory(auth_client, param): auth_client, cassette_library_dir=itkdb.settings.ITKDB_CASSETTE_LIBRARY_DIR ) as recorder: recorder.use_cassette( - "test_scripts.test_getInventory.{0:s}".format(command), record="once" + f"test_scripts.test_getInventory.{command:s}", record="once" ) inventory = getInventory.Inventory(args, auth_client) assert inventory.main() diff --git a/tests/test_session.py b/tests/test_session.py index dc4f61df0e41847d2a507c0660959c650022f089..2d52f4890cb67169a48ce9d0405ea4132e37b89f 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,6 +1,9 @@ -import itkdb +from __future__ import annotations + import betamax +import itkdb + def test_urljoin(auth_session): assert ( diff --git a/tests/test_user.py b/tests/test_user.py index 03da3e380d8d67fd2f2853e6916af03f524b3c02..a1605cb052a86a4d5a36fe129d24bb2a655303a2 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -1,10 +1,14 @@ -import itkdb -import time -import pickle +from __future__ import annotations + +import json import logging +import pickle +import time + import pytest import requests -import json + +import itkdb # set some things on user that we expect to persist or not persist _name = "Test User" @@ -18,12 +22,12 @@ _accessCode2 = "4cce$$C0d32" @pytest.fixture -def user_temp(tmpdir): - temp = tmpdir.join("auth.pkl") - assert temp.isfile() is False +def user_temp(tmp_path): + temp = tmp_path / "auth.pkl" + assert temp.exists() is False u = itkdb.core.User( - save_auth=temp.strpath, + save_auth=temp, accessCode1=_accessCode1, accessCode2=_accessCode2, jwtOptions={ @@ -104,7 +108,7 @@ def test_user_accessCodes(user_temp): def test_user_unpicklable(user_temp, caplog): user, temp = user_temp session = itkdb.core.Session(user=user) - assert temp.isfile() is False + assert temp.exists() is False with caplog.at_level(logging.INFO, "itkdb"): # inject an unpicklable object session.user.fake = lambda x: x @@ -116,11 +120,12 @@ def test_user_serialization(user_temp, caplog): user, temp = user_temp # set up first user and check that we can dump session = itkdb.core.Session(user=user) - assert temp.isfile() is False + assert temp.exists() is False assert session.user._dump() - assert temp.isfile() - assert temp.size() - assert pickle.load(open(user._save_auth, "rb")) + assert temp.exists() + assert temp.stat().st_size + with open(user._save_auth, "rb") as fp: + assert pickle.load(fp) # check if we can reload user session.user._id_token = None @@ -129,7 +134,7 @@ def test_user_serialization(user_temp, caplog): del session # check if session can load user - session = itkdb.core.Session(user=user, save_auth=temp.strpath) + session = itkdb.core.Session(user=user, save_auth=temp) assert session.user.name == _name assert session.user._session assert session.user._response == _response @@ -147,13 +152,13 @@ def test_user_serialization(user_temp, caplog): del session # check what happens if corruption - temp.write("fake") + temp.write_text("fake") with caplog.at_level(logging.INFO, "itkdb"): - user = itkdb.core.User(save_auth=temp.strpath) + user = itkdb.core.User(save_auth=temp) assert "Unable to load user session" in caplog.text caplog.clear() assert user._load() is False assert "Unable to load user session" in caplog.text caplog.clear() - itkdb.core.Session(save_auth=temp.strpath) + itkdb.core.Session(save_auth=temp) assert "Unable to load user session" in caplog.text diff --git a/tests/test_utils.py b/tests/test_utils.py index ef78d897caf3d4c987d92b910bd034290b4e1731..b8867916b7523d4a1457f22c106003b304dd8c42 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,12 @@ -import itkdb -import requests -import pytest +from __future__ import annotations + import io +import pytest +import requests + +import itkdb + def test_build_url_utils(mocker): request = mocker.MagicMock() @@ -68,6 +72,7 @@ def test_get_file_components(mocker, tmp_path, fname, ftype): assert isinstance(fp, io.IOBase) assert ft == ftype assert fh == {"a": "b"} + fp.close() fn, fp, ft, fh = itkdb.utilities.get_file_components( {"data": (fname, fpath.open("rb"), ftype)} @@ -76,6 +81,7 @@ def test_get_file_components(mocker, tmp_path, fname, ftype): assert isinstance(fp, io.IOBase) assert ft == ftype assert fh == {} + fp.close() fn, fp, ft, fh = itkdb.utilities.get_file_components( {"data": (fname, fpath.open("rb"))} @@ -84,12 +90,14 @@ def test_get_file_components(mocker, tmp_path, fname, ftype): assert isinstance(fp, io.IOBase) assert ft == ftype assert fh == {} + fp.close() fn, fp, ft, fh = itkdb.utilities.get_file_components({"data": fpath.open("rb")}) assert fn == fname assert isinstance(fp, io.IOBase) assert ft == ftype assert fh == {} + fp.close() def test_get_file_components_too_many(): @@ -99,23 +107,20 @@ def test_get_file_components_too_many(): def test_is_image(): fn = itkdb.data / "1x1.jpg" - fp = fn.open("rb") - - assert itkdb.utilities.is_image(str(fn), fp) + with fn.open("rb") as fp: + assert itkdb.utilities.is_image(str(fn), fp) def test_is_image_bad_path(): fn = itkdb.data / "1x1.jpg" - fp = fn.open("rb") - - assert itkdb.utilities.is_image("/an/absolutely/fake/path", fp) + with fn.open("rb") as fp: + assert itkdb.utilities.is_image("/an/absolutely/fake/path", fp) def test_is_not_image(): fn = itkdb.data / "1x1.sh" - fp = fn.open("rb") - - assert not itkdb.utilities.is_image(str(fn), fp) + with fn.open("rb") as fp: + assert not itkdb.utilities.is_image(str(fn), fp) @pytest.mark.parametrize( @@ -141,9 +146,9 @@ def test_get_mimetype_path(fpath, ftype, mode): ids=["jpeg", "shellscript-binary", "shellscript"], ) def test_get_mimetype_io(fpath, ftype, mode): - fp = fpath.open(mode) - assert itkdb.utilities.get_mimetype("/an/abs/fake/path", fp) == ftype - assert fp.tell() == 0 + with fpath.open(mode) as fp: + assert itkdb.utilities.get_mimetype("/an/abs/fake/path", fp) == ftype + assert fp.tell() == 0 @pytest.mark.parametrize( @@ -156,7 +161,7 @@ def test_get_mimetype_io(fpath, ftype, mode): ids=["jpeg", "shellscript-binary", "shellscript"], ) def test_get_filesize(fpath, fsize, mode): - fp = fpath.open(mode) - assert itkdb.utilities.get_filesize(fpath, None) == fsize - assert itkdb.utilities.get_filesize("/an/abs/fake/path", fp) == fsize - assert fp.tell() == 0 + with fpath.open(mode) as fp: + assert itkdb.utilities.get_filesize(fpath, None) == fsize + assert itkdb.utilities.get_filesize("/an/abs/fake/path", fp) == fsize + assert fp.tell() == 0