diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 000eb1282e2c1cff7116f1125c8a74ccb9b8d64d..dd33f6fa023baf77d70b275e2332f5f934965601 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,20 +1,17 @@ -# The jobs run for every file in src/, even if they haven't been changed. -# Unfortunately Gitlab CI doesn't have any way to know the commit before the push. -# I used to do this to select only the files that had been changed: -# FILES=$(git diff --name-only --diff-filter=ACMR $CI_COMMIT_SHA~1 -- "*.py") -# if [ -z "$FILES" ]; then exit 0; fi -# The problem was that a push with multiple commits might pass the last commit even -# if the previous failed. - image: gitlab-registry.cern.ch/fts/fts-rest-flask:ci - stages: - static_code_analysis - tests - security - build +variables: + MYSQL_DATABASE: ftsflask + MYSQL_ROOT_PASSWORD: asdf + PYTHONPATH: ./src:./src/fts3rest + FTS3TESTCONFIG: ./src/fts3rest/fts3rest/tests/fts3testconfig_ci + black: # Check that every file has been formatted with black stage: static_code_analysis @@ -26,28 +23,25 @@ pylint36: stage: static_code_analysis script: - python --version - - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy pylint37: - # Check that every file doesn't have syntax errors stage: static_code_analysis script: - source ~/.bashrc - pyenv global $PY37 - source $VENV_37/bin/activate - python --version - - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy pylint38: - # Check that every file doesn't have syntax errors stage: static_code_analysis script: - source ~/.bashrc - pyenv global $PY38 - source $VENV_38/bin/activate - python --version - - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy - + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy radon: # Check metrics for every file @@ -55,11 +49,6 @@ radon: script: - source .gitlab-ci/radon.sh - -variables: - MYSQL_DATABASE: ftsflask - MYSQL_ROOT_PASSWORD: asdf - functional_tests36: stage: tests services: @@ -68,8 +57,6 @@ functional_tests36: script: - python --version - source .gitlab-ci/db.sh - - export PYTHONPATH=./src:./src/fts3rest - - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report @@ -84,8 +71,6 @@ functional_tests37: - python --version - source $VENV_37/bin/activate - source .gitlab-ci/db.sh - - export PYTHONPATH=./src:./src/fts3rest - - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report @@ -100,12 +85,9 @@ functional_tests38: - python --version - source $VENV_38/bin/activate - source .gitlab-ci/db.sh - - export PYTHONPATH=./src:./src/fts3rest - - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report - bandit: # Find potential security issues in file. # It's allowed to fail as it may detect false positives. diff --git a/README.md b/README.md index 33893acc3f366246e88c545a10865c025fd9603f..dfcc9a79a380f242f303bdc7a470181e7c429e54 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,30 @@ # About the migration -The migration of [fts-rest](https://gitlab.cern.ch:8443/fts/fts-rest) has started after the decisions made in +The migration of [fts-rest](https://gitlab.cern.ch:8443/fts/fts-rest) started after the decisions made in the [evaluation](https://its.cern.ch/jira/browse/FTS-1496). The development is happening at fts-flask.cern.ch, with the local user ftsflask # Git workflow - `Master` cannot be pushed to directly. -- Create a new branch for each ticket and merge it to master. +- Create a new branch for each ticket and merge it to master through a merge request. # Gitlab CI The current pipeline runs for every push in every branch: - black: fails if the code hasn't been formatted with black - pylint: fails if the code has syntax errors. If you are sure that pylint is mistaken, add `# pylint: skip-file` at - the beginning of the relevant file. + the beginning of the relevant file. Runs for every supported Python3 version - radon: fails if the code complexity is too high +- functional tests: Run for every supported Python3 version - bandit: detects potential security issues in the code, but it's allowed to fail as there may be false positives +- build: sdist and wheel + Merge requests will proceed only if the pipeline succeeds. + In case of emergency the pipeline can be [skipped](https://docs.gitlab.com/ee/ci/yaml/#skipping-jobs). -The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster. +The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the +image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster. +To build and push the image, cd to .gitlab-ci and run .docker_push.sh Developers should add the `pre-commit` hook to their local repository. This scripts does this for every commit: - Runs black to format the changed files. @@ -26,6 +32,10 @@ Developers should add the `pre-commit` hook to their local repository. This scri - Runs radon and bandit only on the changed files. The hook can be skipped, in case bandit detects false positives, with the commit option `--no-verify`. +# Functional tests +Openid tests don't run in CI because the container would need a client registered and this is + difficult to set up. To run these tests in a development environment, the environment variables 'xdc_ClientId' and 'xdc_ClientSecret' must be set. + # Python dependencies This project uses [pip-tools](https://github.com/jazzband/pip-tools) to manage dependencies: - `requirements.in`: list of dependencies for the production app @@ -39,6 +49,7 @@ Because we need mod_wsgi built for Python 3.6, we need to use httpd24-httpd - gfal2-python3 - yum-config-manager --enable centos-sclo-rh - yum install rh-python36-mod_wsgi + # Installation requirements for development To create a development venv: use --system-packages in order to use gfal2-python3 @@ -62,7 +73,6 @@ curl http://localhost:80/hello To access the config page: ``` INSERT INTO t_authz_dn VALUES ('yourdn'); - ``` # Run tests @@ -70,8 +80,5 @@ INSERT INTO t_authz_dn VALUES ('yourdn'); source venv/bin/activate export PYTHONPATH=/home/ftsflask/fts-rest-flask/src:/home/ftsflask/fts-rest-flask/src/fts3rest export FTS3TESTCONFIG=/home/ftsflask/fts-rest-flask/src/fts3rest/fts3rest/tests/fts3testconfig -python3 -m pytest -x src/fts3rest/fts3rest/tests/functional/test_job_submission.py +python3 -m pytest src/fts3rest/fts3rest/tests/ -x ``` -# Migration status -Starting with the client, as it requires small changes only. Will not migrate pycurlrequest.py, as it is not used - anymore. \ No newline at end of file diff --git a/dev-requirements.in b/dev-requirements.in index a0d19a4e8752bd7c39a1aee4898136bf5a4539dd..f0d6e4ee45bdb4e192487deb485df25416d29e5a 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -1,7 +1,7 @@ -c requirements.txt pip-tools black -pylint +pylint==2.4.4 # 2.5 has a bug and fails as of 02/05/20 bandit radon pytest diff --git a/src/fts3/model/__init__.py b/src/fts3/model/__init__.py deleted file mode 100644 index 76dcc79ef4e4e6ba9c9a4499863c6fa1243fd7be..0000000000000000000000000000000000000000 --- a/src/fts3/model/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Members of the EMI Collaboration, 2013. -# Copyright 2020 CERN -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sqlalchemy -from .base import Base -from .banned import * -from .cloudStorage import * -from .config import * -from .credentials import * -from .dm import * -from .file import * -from .job import * -from .oauth2 import * -from .optimizer import * -from .server import * -from .version import * - - -# Convenience method -def connect(connectString): - engine = sqlalchemy.create_engine(connectString, isolation_level="READ-COMMITTED") - Session = sqlalchemy.orm.sessionmaker(bind=engine) - return Session() diff --git a/src/fts3/util/config.py b/src/fts3rest/fts3rest/config/config.py similarity index 67% rename from src/fts3/util/config.py rename to src/fts3rest/fts3rest/config/config.py index fe2262badfd5f5be7d8ad1489c6bd7c25e65264b..644a81b57d797a678b5d2c96e2b5ddd7ea2b85a2 100644 --- a/src/fts3/util/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from configparser import ConfigParser, NoOptionError +from configparser import ConfigParser, NoOptionError, NoSectionError from urllib.parse import quote_plus import os import logging @@ -21,11 +21,10 @@ import logging log = logging.getLogger(__name__) -def fts3_config_load(path="/etc/fts3/fts3config"): +def fts3_config_load(path="/etc/fts3/fts3config", test=False): """ Read the configuration from the FTS3 configuration file """ - log.debug("entered fts3_config_load") fts3cfg = {} parser = ConfigParser() @@ -100,18 +99,37 @@ def fts3_config_load(path="/etc/fts3/fts3config"): fts3cfg["fts3.Roles"][role.lower()][operation.lower()] = level.lower() # Initialize providers - log.debug("initialize providers config in load environment") fts3cfg["fts3.Providers"] = {} - for option in parser.options("providers"): - if "_" not in option: - provider_name = option - provider_url = parser.get("providers", provider_name) - if not provider_url.endswith("/"): - provider_url += "/" - fts3cfg["fts3.Providers"][provider_url] = {} - client_id = parser.get("providers", option + "_ClientId") - fts3cfg["fts3.Providers"][provider_url]["client_id"] = client_id - client_secret = parser.get("providers", option + "_ClientSecret") - fts3cfg["fts3.Providers"][provider_url]["client_secret"] = client_secret - + try: + for option in parser.options("providers"): + if "_" not in option: + provider_name = option + provider_url = parser.get("providers", provider_name) + if not provider_url.endswith("/"): + provider_url += "/" + fts3cfg["fts3.Providers"][provider_url] = {} + client_id = parser.get("providers", option + "_ClientId") + fts3cfg["fts3.Providers"][provider_url]["client_id"] = client_id + client_secret = parser.get("providers", option + "_ClientSecret") + fts3cfg["fts3.Providers"][provider_url]["client_secret"] = client_secret + fts3cfg["fts3.ValidateAccessTokenOffline"] = parser.getboolean( + "fts3", "ValidateAccessTokenOffline", fallback=True + ) + fts3cfg["fts3.JWKCacheSeconds"] = parser.getint( + "fts3", "JWKCacheSeconds", fallback=86400 + ) + fts3cfg["fts3.TokenRefreshDaemonIntervalInSeconds"] = parser.getint( + "fts3", "TokenRefreshDaemonIntervalInSeconds", fallback=600 + ) + except NoSectionError: + pass + if test: # for open id tests + provider_url = "https://iam.extreme-datacloud.eu/" + fts3cfg["fts3.Providers"][provider_url] = {} + fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[ + "xdc_ClientId" + ] + fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[ + "xdc_ClientSecret" + ] return fts3cfg diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index 1aba26108360ecbea213cbe683c195b7595d4a05..248caedac77b6f7686e3187bc05db1fef163fda5 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -1,26 +1,27 @@ -from flask import Flask, jsonify -from werkzeug.exceptions import NotFound -from sqlalchemy import engine_from_config, event -import MySQLdb -import os from io import StringIO +import json import logging.config +import os + +import MySQLdb +from flask import Flask +from sqlalchemy import engine_from_config, event +from werkzeug.exceptions import HTTPException + +from fts3rest.config.config import fts3_config_load from fts3rest.config.routing import base, cstorage -from fts3.util.config import fts3_config_load -from fts3rest.model import init_model +from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher from fts3rest.lib.helpers.connection_validator import ( connection_validator, connection_set_sqlmode, ) from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddleware -from fts3rest.lib.middleware.error_as_json import ErrorAsJson from fts3rest.lib.middleware.timeout import TimeoutHandler -from fts3rest.model.meta import Session -from werkzeug.exceptions import HTTPException -import json +from fts3rest.lib.openidconnect import oidc_manager +from fts3rest.model.meta import init_model, Session -def _load_configuration(config_file): +def _load_configuration(config_file, test): # ConfigParser doesn't handle files without headers. # If the configuration file doesn't start with [fts3], # add it for backwards compatibility, as before migrating to Flask @@ -41,7 +42,7 @@ def _load_configuration(config_file): # Load configuration logging.config.fileConfig(content) content.seek(0) - fts3cfg = fts3_config_load(content) + fts3cfg = fts3_config_load(content, test) content.close() return fts3cfg @@ -90,7 +91,8 @@ def create_app(default_config_file=None, test=False): if not config_file: raise ValueError("The configuration file has not been specified") - fts3cfg = _load_configuration(config_file) + fts3cfg = _load_configuration(config_file, test) + log = logging.getLogger(__name__) # Add configuration app.config.update(fts3cfg) @@ -109,7 +111,6 @@ def create_app(default_config_file=None, test=False): app.wsgi_app = TimeoutHandler(app.wsgi_app, fts3cfg) # Convert errors to JSON - # app.wsgi_app = ErrorAsJson(app.wsgi_app) @app.errorhandler(HTTPException) def handle_exception(e): """Return JSON instead of HTML for HTTP errors.""" @@ -122,10 +123,12 @@ def create_app(default_config_file=None, test=False): response.content_type = "application/json" return response - # @app.errorhandler(NotFound) - # def handle_invalid_usage(error): - # response = jsonify(error=error.code, name=error.name) - # response.status_code = error.code - # return response + # Start OIDC clients + if "fts3.Providers" not in app.config or not app.config["fts3.Providers"]: + oidc_manager.setup(app.config) + if not test: + IAMTokenRefresher("fts_token_refresh_daemon", app.config).start() + else: + log.info("OpenID Connect support disabled. Providers not found in config") return app diff --git a/src/fts3rest/fts3rest/controllers/CSdropbox.py b/src/fts3rest/fts3rest/controllers/CSdropbox.py index e08d19754e73a39ad7686ac3527adf33f04bd500..989e0f886f5c24bf5e4c049ac9c33e4a57dc9fba 100644 --- a/src/fts3rest/fts3rest/controllers/CSdropbox.py +++ b/src/fts3rest/fts3rest/controllers/CSdropbox.py @@ -18,11 +18,11 @@ from werkzeug.exceptions import NotFound, BadRequest, Forbidden from flask import request import urllib from urllib.parse import urlparse, urlencode -from urllib.request import urlopen, Request +from urllib.request import urlopen from urllib.error import HTTPError from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.model.meta import Session -from fts3.model import CloudStorage, CloudStorageUser +from fts3rest.model import CloudStorage, CloudStorageUser from fts3rest.controllers.CSInterface import Connector dropboxEndpoint = "https://www.dropbox.com" diff --git a/src/fts3rest/fts3rest/controllers/api.py b/src/fts3rest/fts3rest/controllers/api.py index ff31b8af1901e7d4b2443dadd3c6e52c1c8e3744..d81db395065f3f9b0678a9029e53ec323c91a5d3 100644 --- a/src/fts3rest/fts3rest/controllers/api.py +++ b/src/fts3rest/fts3rest/controllers/api.py @@ -16,14 +16,13 @@ import glob from flask.views import View -from fts3.model import SchemaVersion +from fts3rest.model import SchemaVersion from fts3rest.model.meta import Session from fts3rest.lib.api.submit_schema import SubmitSchema -from fts3rest.lib.api.introspect import introspect from werkzeug.exceptions import NotFound from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/archive.py b/src/fts3rest/fts3rest/controllers/archive.py index f6958c7ddf4393be0c151e68ee80a835b406c368..68c179db28793ffeba201a50583ef6f7ec1cd0ba 100644 --- a/src/fts3rest/fts3rest/controllers/archive.py +++ b/src/fts3rest/fts3rest/controllers/archive.py @@ -16,7 +16,7 @@ from werkzeug.exceptions import NotFound, Forbidden from fts3rest.lib.helpers.jsonify import jsonify -from fts3.model import ArchivedJob +from fts3rest.model import ArchivedJob from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.authorization import authorized from fts3rest.lib.middleware.fts3auth.constants import * diff --git a/src/fts3rest/fts3rest/controllers/autocomplete.py b/src/fts3rest/fts3rest/controllers/autocomplete.py index 51c9c901bcba898028831fa497aeb922474534f4..7763221b577934d7f8558096840fcedc3858f10e 100644 --- a/src/fts3rest/fts3rest/controllers/autocomplete.py +++ b/src/fts3rest/fts3rest/controllers/autocomplete.py @@ -13,7 +13,7 @@ # limitations under the License. -from fts3.model import Credential, LinkConfig, Job +from fts3rest.model import Credential, LinkConfig, Job from fts3rest.model.meta import Session from flask import request diff --git a/src/fts3rest/fts3rest/controllers/banning.py b/src/fts3rest/fts3rest/controllers/banning.py index b0cab26c08c60d73df15dc8b9a8cdea68805e8ea..46fda523f1a242824c4146802a23ec27eb90be54 100644 --- a/src/fts3rest/fts3rest/controllers/banning.py +++ b/src/fts3rest/fts3rest/controllers/banning.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from werkzeug.exceptions import NotFound, BadRequest, Conflict +from werkzeug.exceptions import BadRequest, Conflict from fts3rest.controllers.config import audit_configuration from flask import request, Response import json @@ -19,9 +19,15 @@ import logging from datetime import datetime from sqlalchemy import distinct, func, and_ -from fts3.model import BannedDN, BannedSE, Job, File, JobActiveStates, FileActiveStates +from fts3rest.model import ( + BannedDN, + BannedSE, + Job, + File, + JobActiveStates, + FileActiveStates, +) from fts3rest.model.meta import Session -from fts3rest.lib.http_exceptions import * from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import * from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/__init__.py b/src/fts3rest/fts3rest/controllers/config/__init__.py index 4e984f2c631efe96a0124947ec55585a648953f9..7e5ed616765df9ccac939ba41e0d171cf30fa5a2 100644 --- a/src/fts3rest/fts3rest/controllers/config/__init__.py +++ b/src/fts3rest/fts3rest/controllers/config/__init__.py @@ -19,7 +19,7 @@ from numbers import Number from flask import request from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.lib.helpers.accept import accept from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import CONFIG diff --git a/src/fts3rest/fts3rest/controllers/config/activities.py b/src/fts3rest/fts3rest/controllers/config/activities.py index 9e5093ec345b02d6daa84ba9b8dc38992f9a405d..3b293d32ff8d7a849408293f0f0038c3536cce57 100644 --- a/src/fts3rest/fts3rest/controllers/config/activities.py +++ b/src/fts3rest/fts3rest/controllers/config/activities.py @@ -18,7 +18,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest, NotFound -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/audit.py b/src/fts3rest/fts3rest/controllers/config/audit.py index fd70bc78612c49bc707936af2750adec38e19561..53b4a85ad0023326ea296e1deea8fd9b52513433 100644 --- a/src/fts3rest/fts3rest/controllers/config/audit.py +++ b/src/fts3rest/fts3rest/controllers/config/audit.py @@ -15,7 +15,7 @@ import logging -from fts3.model import * +from fts3rest.model import * from fts3rest.lib.helpers.accept import accept from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import * diff --git a/src/fts3rest/fts3rest/controllers/config/authz.py b/src/fts3rest/fts3rest/controllers/config/authz.py index eb5c28f52d6c25b26c8192f48c9ce3dc4539e5b0..e7756fbf58766ade513fae65f512e618d8284650 100644 --- a/src/fts3rest/fts3rest/controllers/config/authz.py +++ b/src/fts3rest/fts3rest/controllers/config/authz.py @@ -17,7 +17,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/cloud.py b/src/fts3rest/fts3rest/controllers/config/cloud.py index a63d79c8185ecc384ba3b27fa977ad9316e23145..87a8d30b37d0ddf6e857e5f15cf2133c2a512fc0 100644 --- a/src/fts3rest/fts3rest/controllers/config/cloud.py +++ b/src/fts3rest/fts3rest/controllers/config/cloud.py @@ -17,7 +17,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest, NotFound -from fts3.model import * +from fts3rest.model import * from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.lib.helpers.misc import get_input_as_dict diff --git a/src/fts3rest/fts3rest/controllers/config/drain.py b/src/fts3rest/fts3rest/controllers/config/drain.py index 1bb882b4453ce32720e456abbc69fd7fcc1d4c68..3cea5fc569cb4f51bc2ada3dfb635c090d2dd0b1 100644 --- a/src/fts3rest/fts3rest/controllers/config/drain.py +++ b/src/fts3rest/fts3rest/controllers/config/drain.py @@ -17,7 +17,7 @@ import logging from flask import request from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.lib.helpers.misc import get_input_as_dict diff --git a/src/fts3rest/fts3rest/controllers/config/global_.py b/src/fts3rest/fts3rest/controllers/config/global_.py index 62b9daa12c00cd23ac664cc8d0bd2a6b29870ed2..92e26421042b2dffba5c6da110a3e38a6ac75073 100644 --- a/src/fts3rest/fts3rest/controllers/config/global_.py +++ b/src/fts3rest/fts3rest/controllers/config/global_.py @@ -17,7 +17,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration, validate_type from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify, to_json diff --git a/src/fts3rest/fts3rest/controllers/config/links.py b/src/fts3rest/fts3rest/controllers/config/links.py index d506d9b899fcea2e77ba8e76a64015dd89403c43..804141dfcf63c337a956299b70cb906f7f1d4c27 100644 --- a/src/fts3rest/fts3rest/controllers/config/links.py +++ b/src/fts3rest/fts3rest/controllers/config/links.py @@ -19,7 +19,7 @@ from urllib.parse import unquote from flask import request, Response from werkzeug.exceptions import BadRequest, NotFound -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/se.py b/src/fts3rest/fts3rest/controllers/config/se.py index b55b27796e01c5cac9af2ecbd273a3277780b83a..91e071c9c7f180c22c72e89f30718f233c52107a 100644 --- a/src/fts3rest/fts3rest/controllers/config/se.py +++ b/src/fts3rest/fts3rest/controllers/config/se.py @@ -19,7 +19,7 @@ from flask import Response from flask import request from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/shares.py b/src/fts3rest/fts3rest/controllers/config/shares.py index ae1e9fc814be4ae4efc3a050e67921794f3f38db..4736bc425a4ccfe5c58741a90dff38d7327a08ec 100644 --- a/src/fts3rest/fts3rest/controllers/config/shares.py +++ b/src/fts3rest/fts3rest/controllers/config/shares.py @@ -18,7 +18,7 @@ from urllib.parse import urlparse from flask import request, Response from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.lib.helpers.misc import get_input_as_dict diff --git a/src/fts3rest/fts3rest/controllers/datamanagement.py b/src/fts3rest/fts3rest/controllers/datamanagement.py index 96e270a553aaa87f7148bd1920699b5bdbfb2561..cf96f6b8c6539bdd66d2ce2486fcbd2d35990c82 100644 --- a/src/fts3rest/fts3rest/controllers/datamanagement.py +++ b/src/fts3rest/fts3rest/controllers/datamanagement.py @@ -33,7 +33,7 @@ from urllib.parse import urlparse, unquote_plus import json -from fts3.model import Credential +from fts3rest.model import Credential from fts3rest.model.meta import Session from fts3rest.lib.http_exceptions import HTTPAuthenticationTimeout from fts3rest.lib.gfal2_wrapper import Gfal2Wrapper, Gfal2Error diff --git a/src/fts3rest/fts3rest/controllers/delegation.py b/src/fts3rest/fts3rest/controllers/delegation.py index 3ff251dfa983fbd7f6e8bfca2309f805e86b2d14..0433ab79cfa0104082a6aeb895801ebcba097728 100644 --- a/src/fts3rest/fts3rest/controllers/delegation.py +++ b/src/fts3rest/fts3rest/controllers/delegation.py @@ -25,7 +25,7 @@ import flask from flask import Response from flask import current_app as app from flask.views import View -from fts3.model import CredentialCache, Credential +from fts3rest.model import CredentialCache, Credential from fts3rest.model.meta import Session from fts3rest.lib.helpers.voms import VomsClient, VomsException from fts3rest.lib.middleware.fts3auth.authorization import require_certificate diff --git a/src/fts3rest/fts3rest/controllers/files.py b/src/fts3rest/fts3rest/controllers/files.py index f891cecc484cc87425d915ae5c618659aa7e6a99..93941bad0f899034142e7091141b67f25574a2c2 100644 --- a/src/fts3rest/fts3rest/controllers/files.py +++ b/src/fts3rest/fts3rest/controllers/files.py @@ -16,15 +16,13 @@ from werkzeug.exceptions import Forbidden from datetime import datetime, timedelta from flask import request from urllib.parse import urlparse -import json import logging -from fts3.model import File +from fts3rest.model import File from fts3rest.model.meta import Session from fts3rest.lib.JobBuilder_utils import get_storage_element from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import * -from fts3rest.lib.http_exceptions import * from fts3rest.lib.helpers.jsonify import jsonify log = logging.getLogger(__name__) diff --git a/src/fts3rest/fts3rest/controllers/jobs.py b/src/fts3rest/fts3rest/controllers/jobs.py index 09e480fa50fdb183eed0084776814a71db36b08e..5edf16d944f890a5ffdb82256caa7dde5780f5a0 100644 --- a/src/fts3rest/fts3rest/controllers/jobs.py +++ b/src/fts3rest/fts3rest/controllers/jobs.py @@ -22,9 +22,9 @@ from sqlalchemy.orm import noload import logging -from fts3.model import Job, File, JobActiveStates, FileActiveStates -from fts3.model import DataManagement, DataManagementActiveStates -from fts3.model import Credential, FileRetryLog +from fts3rest.model import Job, File, JobActiveStates, FileActiveStates +from fts3rest.model import DataManagement, DataManagementActiveStates +from fts3rest.model import Credential, FileRetryLog from fts3rest.model.meta import Session from fts3rest.lib.http_exceptions import * diff --git a/src/fts3rest/fts3rest/controllers/optimizer.py b/src/fts3rest/fts3rest/controllers/optimizer.py index 4bd973915387c69b59b1d31fdac4d3701df305ce..65a528fd2f43e394982effc804b363fb52d64715 100644 --- a/src/fts3rest/fts3rest/controllers/optimizer.py +++ b/src/fts3rest/fts3rest/controllers/optimizer.py @@ -18,15 +18,11 @@ Optimizer logging tables from werkzeug.exceptions import BadRequest - -import json -import logging from flask import request, current_app as app from fts3rest.model.meta import Session from fts3rest.lib.helpers.misc import get_input_as_dict -from fts3.model import OptimizerEvolution, Optimizer +from fts3rest.model import OptimizerEvolution, Optimizer from datetime import datetime -from fts3rest.lib.http_exceptions import * from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/serverstatus.py b/src/fts3rest/fts3rest/controllers/serverstatus.py index 5c087d74ef1c8b1879ee30cbe61e803b6c85f021..91466bd16e6f2a2e77e312854440ce988b76a423 100644 --- a/src/fts3rest/fts3rest/controllers/serverstatus.py +++ b/src/fts3rest/fts3rest/controllers/serverstatus.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from werkzeug.exceptions import NotFound -from fts3.model import File from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.authorization import ( authorize, diff --git a/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py new file mode 100644 index 0000000000000000000000000000000000000000..8c19a6a7b3cc2f1adf2c1dea39d7631d14458f5c --- /dev/null +++ b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py @@ -0,0 +1,141 @@ +# Copyright 2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import socket +import time +import random +from datetime import datetime, timedelta +from threading import Thread, current_thread + + +from fts3rest.model.meta import Session +from fts3rest.lib.openidconnect import oidc_manager +from fts3rest.model import Credential, Host + +from sqlalchemy.exc import SQLAlchemyError + +log = logging.getLogger(__name__) + + +class IAMTokenRefresher(Thread): + """ + Daemon thread that refreshes all access tokens in the DB at every interval. + + Keeps running on the background updating the DB, marking the process as alive. + There should be ONLY ONE across all instances. + Keep in mind that with the Apache configuration line + WSGIDaemonProcess fts3rest python-path=... processes=2 threads=15 + there will be 2 instances of the application per server, meaning we need to check that there is only one + IAMTokenRefresher per host, and only one between all hosts. + + The SQLAlchemy scoped_session is thread-safe + """ + + def __init__(self, tag, config): + Thread.__init__(self) + self.daemon = ( + True # The thread will immediately exit when the main thread exits + ) + self.tag = tag + self.refresh_interval = int( + config.get("fts3.TokenRefreshDaemonIntervalInSeconds", 600) + ) + self.config = config + + def _thread_is_inactive(self, thread): + # The thread is considered inactive if it hasn't updated the DB for 3*refresh_interval + log.debug("time since last beat {}".format(datetime.utcnow() - thread.beat)) + if (datetime.utcnow() - thread.beat) > timedelta( + seconds=3 * self.refresh_interval + ): + log.debug("thread is inactive! taking over, beat {}".format(thread.beat)) + return (datetime.utcnow() - thread.beat) > timedelta( + seconds=3 * self.refresh_interval + ) + + def run(self): + """ + Regularly check if there is another active IAMTokenRefresher in the DB. If not, become the active thread. + """ + log.debug("CREATE THREAD ID: {}".format(current_thread().ident)) + # Initial sleep in case all threads started at the same time + time.sleep(random.randint(0, 60)) + # The interval at which the thread will check if there is another active thread. + # It is arbitrary: I chose 2 times the refresh interval, plus a random offset to avoid multiple threads + # checking at the same time (although DB access is transactional) + db_check_interval = 3 * self.refresh_interval + random.randint(0, 120) + while True: + # Check that no other fts-token-refresh-daemon is running + refresher_threads = ( + Session.query(Host).filter(Host.service_name == self.tag).all() + ) + Session.commit() # Close transaction to avoid repeated read + log.debug( + "refresher_threads {}, ID {}".format( + len(refresher_threads), current_thread().ident + ) + ) + if all(self._thread_is_inactive(thread) for thread in refresher_threads): + log.debug("Activating thread") + for thread in refresher_threads: + Session.delete(thread) + log.debug("delete thread") + host = Host(hostname=socket.getfqdn(), service_name=self.tag) + log.debug("host object created") + while True: + host.beat = datetime.utcnow() + log.debug( + "THREAD ID: {}, beat {}".format( + current_thread().ident, host.beat + ) + ) + try: + h2 = Session.merge(host) + Session.commit() + log.debug( + "fts-token-refresh-daemon heartbeat {}".format(h2.beat) + ) + except SQLAlchemyError as ex: + log.warning( + "Failed to update the fts-token-refresh-daemon heartbeat: %s" + % str(ex) + ) + Session.rollback() + raise + + credentials = ( + Session.query(Credential) + .filter(Credential.proxy.notilike("%CERTIFICATE%")) + .all() + ) + log.debug("{} credentials to refresh".format(len(credentials))) + for credential in credentials: + try: + credential = oidc_manager.refresh_access_token(credential) + log.debug("OK refresh_access_token") + Session.merge(credential) + Session.commit() + except Exception as ex: + log.warning( + "Failed to refresh token for dn: %s because: %s" + % (str(credential.dn), str(ex)) + ) + Session.rollback() + raise + time.sleep(self.refresh_interval) + else: + log.debug("THREAD ID: {}".format(current_thread().ident)) + log.debug("Another thread is active -- Going to sleep") + time.sleep(db_check_interval) diff --git a/src/fts3rest/fts3rest/lib/JobBuilder_utils.py b/src/fts3rest/fts3rest/lib/JobBuilder_utils.py index 6580561f887832057af60d04bdc34b84558079d4..b900008b5ec2c5c9fb526fd3be92fcc899a0fa93 100644 --- a/src/fts3rest/fts3rest/lib/JobBuilder_utils.py +++ b/src/fts3rest/fts3rest/lib/JobBuilder_utils.py @@ -9,7 +9,7 @@ import uuid from flask import current_app as app from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError -from fts3.model import BannedSE +from fts3rest.model import BannedSE from fts3rest.model.meta import Session from fts3rest.lib.scheduler.schd import Scheduler diff --git a/src/fts3rest/fts3rest/lib/api/introspect.py b/src/fts3rest/fts3rest/lib/api/introspect.py index b83b9df94020939d3587b4dd29298435bf7c5ca9..fdb3ed5c2f4a00c6c4d8818d5637ae209443eaff 100755 --- a/src/fts3rest/fts3rest/lib/api/introspect.py +++ b/src/fts3rest/fts3rest/lib/api/introspect.py @@ -24,8 +24,8 @@ import logging import imp import itertools import os -import fts3.model -from fts3.model.base import Flag, TernaryFlag, Json +import fts3rest.model +from fts3rest.model.base import Flag, TernaryFlag, Json from fts3rest.config import routing from sqlalchemy import types from sqlalchemy.orm import Mapper @@ -86,7 +86,7 @@ def get_model_fields(model_name, model_list): Get a description of the fields of the model 'model_name' Injects into model_list the additional types that may be needed """ - model = getattr(fts3.model, model_name, None) + model = getattr(fts3rest.model, model_name, None) if not model: return dict() fields = dict() diff --git a/src/fts3rest/fts3rest/lib/helpers/jsonify.py b/src/fts3rest/fts3rest/lib/helpers/jsonify.py index 6cd9c5964aa9ccb0c4264a7cbbf6de136e679847..711ca039638d117c9fbf48cf5e26b2c976c67959 100644 --- a/src/fts3rest/fts3rest/lib/helpers/jsonify.py +++ b/src/fts3rest/fts3rest/lib/helpers/jsonify.py @@ -14,8 +14,7 @@ # limitations under the License. from datetime import datetime -from fts3.model.base import Base -from sqlalchemy.orm.query import Query +from fts3rest.model.base import Base import json import logging import types diff --git a/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py b/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py index a40e8b3cc51574e89b2ba5fef545cb2ed39acea3..0585d58f7940025a338db58e10402257c3dfcaa9 100644 --- a/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py +++ b/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py @@ -18,7 +18,7 @@ import hashlib import logging import re -from fts3.model import AuthorizationByDn +from fts3rest.model import AuthorizationByDn from fts3rest.model.meta import Session from .methods import Authenticator diff --git a/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py b/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py index c4f42bc639501dc00fcb6ac052433bbd0074bdee..a4f04815c53da44d3b0cd354957d258dda9bfc0a 100644 --- a/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py +++ b/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py @@ -16,7 +16,7 @@ import logging from fts3rest.model.meta import Session -from fts3.model import BannedDN +from fts3rest.model import BannedDN from .credentials import UserCredentials, InvalidCredentials from sqlalchemy.exc import DatabaseError from urllib.parse import urlparse diff --git a/src/fts3rest/fts3rest/lib/oauth2provider.py b/src/fts3rest/fts3rest/lib/oauth2provider.py index f22b0784faa86751ef23d2be95241d257e838a97..cd84249fdb45128099f4dc38dbe2385bb35a18f0 100644 --- a/src/fts3rest/fts3rest/lib/oauth2provider.py +++ b/src/fts3rest/fts3rest/lib/oauth2provider.py @@ -28,8 +28,8 @@ from fts3rest.lib.openidconnect import oidc_manager from jwcrypto.jwk import JWK from flask import request -from fts3.model.credentials import Credential, CredentialCache -from fts3.model.oauth2 import OAuth2Application, OAuth2Code, OAuth2Token +from fts3rest.model import Credential, CredentialCache +from fts3rest.model.oauth2 import OAuth2Application, OAuth2Code, OAuth2Token log = logging.getLogger(__name__) diff --git a/src/fts3rest/fts3rest/lib/openidconnect.py b/src/fts3rest/fts3rest/lib/openidconnect.py index 2a48bcf162a005952a3ab62fd164083935e2e433..1024e6ef8b31cdd7aceacd610115993392ae9ab1 100644 --- a/src/fts3rest/fts3rest/lib/openidconnect.py +++ b/src/fts3rest/fts3rest/lib/openidconnect.py @@ -27,7 +27,7 @@ class OIDCmanager: def setup(self, config): self.config = config self._configure_clients(config["fts3.Providers"]) - self._set_keys_cache_time(int(config["fts3.JWKCacheSeconds"])) + self._set_keys_cache_time(config["fts3.JWKCacheSeconds"]) self._retrieve_clients_keys() def _configure_clients(self, providers_config): diff --git a/src/fts3rest/fts3rest/lib/scheduler/db.py b/src/fts3rest/fts3rest/lib/scheduler/db.py index 357cfeeaac55a529b1a6e5d1b27c41ecdc80f4a4..2bb385a73c348409ca0b46abffd60676d803f913 100644 --- a/src/fts3rest/fts3rest/lib/scheduler/db.py +++ b/src/fts3rest/fts3rest/lib/scheduler/db.py @@ -1,9 +1,9 @@ import json import logging -from fts3.model import File -from fts3.model import OptimizerEvolution -from fts3.model import ActivityShare +from fts3rest.model import File +from fts3rest.model import OptimizerEvolution +from fts3rest.model import ActivityShare from sqlalchemy import func diff --git a/src/fts3rest/fts3rest/model/__init__.py b/src/fts3rest/fts3rest/model/__init__.py index 5c0af258304947b0430438651902f01d4086a984..af384e43ab17ac8d322c2d1ab23db57748f2caf0 100644 --- a/src/fts3rest/fts3rest/model/__init__.py +++ b/src/fts3rest/fts3rest/model/__init__.py @@ -12,11 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""The application's model objects""" -from fts3rest.model.meta import Session, Base - - -def init_model(engine): - """Call me before using any of the tables or classes in the model""" - Session.configure(bind=engine) +from .base import Base +from .banned import * +from .cloudStorage import * +from .config import * +from .credentials import * +from .dm import * +from .file import * +from .job import * +from .oauth2 import * +from .optimizer import * +from .server import * +from .version import * diff --git a/src/fts3/model/banned.py b/src/fts3rest/fts3rest/model/banned.py similarity index 100% rename from src/fts3/model/banned.py rename to src/fts3rest/fts3rest/model/banned.py diff --git a/src/fts3/model/base.py b/src/fts3rest/fts3rest/model/base.py similarity index 100% rename from src/fts3/model/base.py rename to src/fts3rest/fts3rest/model/base.py diff --git a/src/fts3/model/cloudStorage.py b/src/fts3rest/fts3rest/model/cloudStorage.py similarity index 100% rename from src/fts3/model/cloudStorage.py rename to src/fts3rest/fts3rest/model/cloudStorage.py diff --git a/src/fts3/model/config.py b/src/fts3rest/fts3rest/model/config.py similarity index 100% rename from src/fts3/model/config.py rename to src/fts3rest/fts3rest/model/config.py diff --git a/src/fts3/model/credentials.py b/src/fts3rest/fts3rest/model/credentials.py similarity index 100% rename from src/fts3/model/credentials.py rename to src/fts3rest/fts3rest/model/credentials.py diff --git a/src/fts3/model/dm.py b/src/fts3rest/fts3rest/model/dm.py similarity index 100% rename from src/fts3/model/dm.py rename to src/fts3rest/fts3rest/model/dm.py diff --git a/src/fts3/model/file.py b/src/fts3rest/fts3rest/model/file.py similarity index 100% rename from src/fts3/model/file.py rename to src/fts3rest/fts3rest/model/file.py diff --git a/src/fts3/model/job.py b/src/fts3rest/fts3rest/model/job.py similarity index 100% rename from src/fts3/model/job.py rename to src/fts3rest/fts3rest/model/job.py diff --git a/src/fts3rest/fts3rest/model/meta.py b/src/fts3rest/fts3rest/model/meta.py index ea55c07b40f8667dbf22096f4e5d7c628f78fc23..c06e9ed5a1c71747bbf72dd14408917f958b1b92 100644 --- a/src/fts3rest/fts3rest/model/meta.py +++ b/src/fts3rest/fts3rest/model/meta.py @@ -19,10 +19,13 @@ import logging import sqlalchemy from sqlalchemy.orm import scoped_session, sessionmaker -from fts3.model.base import Base # The declarative Base - # SQLAlchemy session manager. Updated by model.init_model() Session = scoped_session(sessionmaker()) # Log the version logging.getLogger(__name__).info("Using SQLAlchemy %s" % sqlalchemy.__version__) + + +def init_model(engine): + """Call me before using any of the tables or classes in the model""" + Session.configure(bind=engine) diff --git a/src/fts3/model/oauth2.py b/src/fts3rest/fts3rest/model/oauth2.py similarity index 100% rename from src/fts3/model/oauth2.py rename to src/fts3rest/fts3rest/model/oauth2.py diff --git a/src/fts3/model/optimizer.py b/src/fts3rest/fts3rest/model/optimizer.py similarity index 100% rename from src/fts3/model/optimizer.py rename to src/fts3rest/fts3rest/model/optimizer.py diff --git a/src/fts3/model/server.py b/src/fts3rest/fts3rest/model/server.py similarity index 100% rename from src/fts3/model/server.py rename to src/fts3rest/fts3rest/model/server.py diff --git a/src/fts3/model/version.py b/src/fts3rest/fts3rest/model/version.py similarity index 100% rename from src/fts3/model/version.py rename to src/fts3rest/fts3rest/model/version.py diff --git a/src/fts3rest/fts3rest/tests/__init__.py b/src/fts3rest/fts3rest/tests/__init__.py index ee046677309bce3f771a192723285329aeec28d8..fe9249ee74621432f23c70a895cc2f1aeca13e68 100644 --- a/src/fts3rest/fts3rest/tests/__init__.py +++ b/src/fts3rest/fts3rest/tests/__init__.py @@ -1,18 +1,18 @@ +from datetime import timedelta import os import shutil +import subprocess import time - -from datetime import datetime, timedelta +import unittest from unittest import TestCase + from M2Crypto import ASN1, X509, RSA, EVP from M2Crypto.ASN1 import UTC - +from fts3rest.model import * +from fts3rest.config.middleware import create_app from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials from fts3rest.model.meta import Session -from fts3.model import Credential, CredentialCache, DataManagement -from fts3.model import * -from fts3rest.config.middleware import create_app from .ftstestclient import FTSTestClient, TestResponse @@ -198,3 +198,15 @@ class TestController(TestCase): pass self.flask_app.do_teardown_appcontext() + + def _get_xdc_access_token(self): + command = ( + "eval `oidc-agent` && oidc-add xdctest --pw-cmd=echo && oidc-token xdctest" + ) + try: + output = subprocess.check_output(command, shell=True) + except subprocess.CalledProcessError as ex: + raise unittest.SkipTest("Failed to get access token") from ex + output = str(output).strip() + token = output.split("\n")[2] # The 3rd line is the token + return token diff --git a/src/fts3rest/fts3rest/tests/fts3testconfig b/src/fts3rest/fts3rest/tests/fts3testconfig index a009926838fd6f27e1f818ce171f039cedaf75bd..90334d2f3704912016adeb2ec9ee010002fe0ea0 100644 --- a/src/fts3rest/fts3rest/tests/fts3testconfig +++ b/src/fts3rest/fts3rest/tests/fts3testconfig @@ -1,7 +1,3 @@ -# Running user and group -User=ftsflask -Group=ftsflask - # mysql only DbType=mysql @@ -11,150 +7,27 @@ DbUserName=ftsflask #db password DbPassword= - #For MySQL, it has to follow the format 'host/db' (i.e. "mysql-server.example.com/fts3db") DbConnectString=localhost:3306/ftsflask -#Number of db connections in the pool (use even number, e.g. 2,4,6,8,etc OR 1 for a single connection) -DbThreadsNum=30 - -#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/ -#Alias=fts3-xdc.cern.ch - -#Infosys, either the fqdn:port of a BDII instance or false to disable BDII access -#Infosys=lcg-bdii.cern.ch:2170 - -#Query the info systems specified in the order given, e.g. glue1;glue2 -InfoProviders=glue1 - #List of authorized VOs, separated by ; #Leave * to authorize any VO AuthorizedVO=* -# site name -#SiteName=FTS-DEV-XDC - -#Enable/Disable monitoring using messaging monitoring (disabled=false / enabled=true) -MonitoringMessaging=false - -# Profiling interval in seconds. If set to 0, it will be disabled -Profiling=0 - -# Log directories -TransferLogDirectory=/var/log/fts3/transfers -ServerLogDirectory=/var/log/fts3 - -# Log level. Enables logging for messages of level >= than configured -# Possible values are -# TRACE (every detail), DEBUG (internal behaviour), INFO (normal behaviour), -# NOTICE (final states), WARNING (things worth checking), ERR (internal FTS3 errors, as database connectivity), -# CRIT (fatal errors, as segmentation fault) -# It is recommended to use DEBUG or INFO -LogLevel=DEBUG - -# Check for fts_url_copy processes that do not give their progress back -# CheckStalledTransfers = true -# Stalled timeout, in seconds -# CheckStalledTimeout = 900 -CheckStalledTimeout = 900 - -# Minimum required free RAM (in MB) for FTS3 to work normally -# If the amount of free RAM goes below the limit, FTS3 will enter auto-drain mode -# This is intended to protect against system resource exhaustion -# MinRequiredFreeRAM = 50 -MinRequiredFreeRAM = 50 - -# Maximum number of url copy processes that the node can run -# The RAM limitation may not take into account other node limitations (i.e. IO) -# or, depending on the swapping policy, may not even prevent overloads if the kernel -# starts swapping before the free RAM decreases until it reaches the value of MinRequiredFreeRAM -# 0 disables the check. -# The default is 400. -# MaxUrlCopyProcesses = 400 -MaxUrlCopyProcesses = 400 - -# Parameters for Bring Online -# Maximum bulk size. -# If the size is too large, it will take more resources (memory and CPU) to generate the requests and -# parse the responses. Some servers may reject the requests if they are too big. -# If it is too small, performance will be reduced. -# Keep it to a sensible size (between 100 and 1k) -# StagingBulkSize=400 -# Maximum number of concurrent requests. This gives a maximum of files sent to the storage system -# (StagingBulkSize*StagingConcurrentRequests). The larger the number, the more requests will FTS need to keep track of. -# StagingConcurrentRequests=500 -# Seconds to wait before submitting a bulk request, so FTS can accumulate more files per bulk. -# Note that the resolution is 60 seconds. -# StagingWaitingFactor=300 -# Retry this number of times if a staging poll fails with ECOMM -# StagingPollRetries=3 - -# In seconds, interval between heartbeats -# HeartBeatInterval=60 -# I seconds, after this interval a host is considered down -# HeartBeatGraceInterval=120 - -# Seconds between optimizer runs -# OptimizerInterval = 60 -# After this time without optimizer updates, force a run -# OptimizerSteadyInterval = 300 -# Maximum number of streams per file -# OptimizerMaxStreams = 16 - -# EMA Alpha factor to reduce the influence of fluctuations -# OptimizerEMAAlpha = 0.1 -# Increase step size when the optimizer considers the performance is good -# OptimizerIncreaseStep = 1 -# Increase step size when the optimizer considers the performance is good, and set to aggressive or normal -# OptimizerAggressiveIncreaseStep = 2 -# Decrease step size when the optimizer considers the performance is bad -# OptimizerDecreaseStep = 1 - - -# Set the bulk size, in number of jobs, used for cleaning the old records -#CleanBulkSize=5000 -# In days. Entries older than this will be purged. -#CleanInterval=7 - -## The higher the values for the following parameters, -## the higher the latency for some operations (as cancelations), -## but can also reduce the system and/or database load - -# In seconds, how often to purge the messaging directory -#PurgeMessagingDirectoryInterval = 600 -# In seconds, how often to run sanity checks -#CheckSanityStateInterval = 3600 -# In seconds, how often to check for canceled transfers -#CancelCheckInterval = 10 -# In seconds, how often to check for expired queued transfers -#QueueTimeoutCheckInterval = 300 -# In seconds, how often to check for stalled transfers -#ActiveTimeoutCheckInterval = 300 -# In seconds, how often to schedule new transfers -#SchedulingInterval = 2 -# In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2 -#MessagingConsumeInterval = 1 -#Enable or disable auto session reuse -AutoSessionReuse = true -#Max small file size for session reuse in bytes -AutoSessionReuseMaxSmallFileSize = 104857600 -#Max big file size for session reuse in bytes -AutoSessionReuseMaxBigFileSize = 1073741824 -#Max number of files per session reuse -AutoSessionReuseMaxFiles = 1000 -#Max number of big files per session reuse -AutoSessionReuseMaxBigFiles = 2 -BackupTables=false -OptimizerMaxSuccessRate=100 -OptimizerMedSuccessRate=80 -OptimizerLowSuccessRate=75 -OptimizerBaseSuccessRate=74 -Port=8443 -UseFixedJobPriority=0 - +#OpenID parameters ValidateAccessTokenOffline=True JWKCacheSeconds=86400 TokenRefreshDaemonIntervalInSeconds=600 + +[roles] +Public = vo:transfer;all:datamanagement +lcgadmin = all:config + +[providers] +xdc=https://iam.extreme-datacloud.eu +xdc_ClientId= +xdc_ClientSecret= + # Logging configuration [loggers] keys = root, routes, fts3rest, sqlalchemy @@ -195,8 +68,6 @@ level = NOTSET formatter = generic [handler_log_file] -# See -# http://docs.python.org/2/library/logging.handlers.html class = logging.FileHandler args = ('/var/log/fts3rest/fts3rest.log', 'a') level = NOTSET @@ -205,15 +76,3 @@ formatter = generic [formatter_generic] format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s datefmt = %H:%M:%S - - - -[roles] -Public = vo:transfer;all:datamanagement -lcgadmin = all:config - - -[providers] -xdc=https://iam.extreme-datacloud.eu -xdc_ClientId= -xdc_ClientSecret= diff --git a/src/fts3rest/fts3rest/tests/fts3testconfig_ci b/src/fts3rest/fts3rest/tests/fts3testconfig_ci index f60a400a5ca955b9933eddaf88491ddddb2935d6..ce73e7e459fecced7291bc214230124e2e32bfdd 100644 --- a/src/fts3rest/fts3rest/tests/fts3testconfig_ci +++ b/src/fts3rest/fts3rest/tests/fts3testconfig_ci @@ -1,7 +1,3 @@ -# Running user and group -User=ci -Group=ci - # mysql only DbType=mysql @@ -11,150 +7,27 @@ DbUserName=ci #db password DbPassword=asdf - #For MySQL, it has to follow the format 'host/db' (i.e. "mysql-server.example.com/fts3db") DbConnectString=mariadb:3306/ftsflask -#Number of db connections in the pool (use even number, e.g. 2,4,6,8,etc OR 1 for a single connection) -DbThreadsNum=30 - -#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/ -#Alias=fts3-xdc.cern.ch - -#Infosys, either the fqdn:port of a BDII instance or false to disable BDII access -#Infosys=lcg-bdii.cern.ch:2170 - -#Query the info systems specified in the order given, e.g. glue1;glue2 -InfoProviders=glue1 - #List of authorized VOs, separated by ; #Leave * to authorize any VO AuthorizedVO=* -# site name -#SiteName=FTS-DEV-XDC - -#Enable/Disable monitoring using messaging monitoring (disabled=false / enabled=true) -MonitoringMessaging=false - -# Profiling interval in seconds. If set to 0, it will be disabled -Profiling=0 - -# Log directories -TransferLogDirectory=/var/log/fts3/transfers -ServerLogDirectory=/var/log/fts3 - -# Log level. Enables logging for messages of level >= than configured -# Possible values are -# TRACE (every detail), DEBUG (internal behaviour), INFO (normal behaviour), -# NOTICE (final states), WARNING (things worth checking), ERR (internal FTS3 errors, as database connectivity), -# CRIT (fatal errors, as segmentation fault) -# It is recommended to use DEBUG or INFO -LogLevel=DEBUG - -# Check for fts_url_copy processes that do not give their progress back -# CheckStalledTransfers = true -# Stalled timeout, in seconds -# CheckStalledTimeout = 900 -CheckStalledTimeout = 900 - -# Minimum required free RAM (in MB) for FTS3 to work normally -# If the amount of free RAM goes below the limit, FTS3 will enter auto-drain mode -# This is intended to protect against system resource exhaustion -# MinRequiredFreeRAM = 50 -MinRequiredFreeRAM = 50 - -# Maximum number of url copy processes that the node can run -# The RAM limitation may not take into account other node limitations (i.e. IO) -# or, depending on the swapping policy, may not even prevent overloads if the kernel -# starts swapping before the free RAM decreases until it reaches the value of MinRequiredFreeRAM -# 0 disables the check. -# The default is 400. -# MaxUrlCopyProcesses = 400 -MaxUrlCopyProcesses = 400 - -# Parameters for Bring Online -# Maximum bulk size. -# If the size is too large, it will take more resources (memory and CPU) to generate the requests and -# parse the responses. Some servers may reject the requests if they are too big. -# If it is too small, performance will be reduced. -# Keep it to a sensible size (between 100 and 1k) -# StagingBulkSize=400 -# Maximum number of concurrent requests. This gives a maximum of files sent to the storage system -# (StagingBulkSize*StagingConcurrentRequests). The larger the number, the more requests will FTS need to keep track of. -# StagingConcurrentRequests=500 -# Seconds to wait before submitting a bulk request, so FTS can accumulate more files per bulk. -# Note that the resolution is 60 seconds. -# StagingWaitingFactor=300 -# Retry this number of times if a staging poll fails with ECOMM -# StagingPollRetries=3 - -# In seconds, interval between heartbeats -# HeartBeatInterval=60 -# I seconds, after this interval a host is considered down -# HeartBeatGraceInterval=120 - -# Seconds between optimizer runs -# OptimizerInterval = 60 -# After this time without optimizer updates, force a run -# OptimizerSteadyInterval = 300 -# Maximum number of streams per file -# OptimizerMaxStreams = 16 - -# EMA Alpha factor to reduce the influence of fluctuations -# OptimizerEMAAlpha = 0.1 -# Increase step size when the optimizer considers the performance is good -# OptimizerIncreaseStep = 1 -# Increase step size when the optimizer considers the performance is good, and set to aggressive or normal -# OptimizerAggressiveIncreaseStep = 2 -# Decrease step size when the optimizer considers the performance is bad -# OptimizerDecreaseStep = 1 - - -# Set the bulk size, in number of jobs, used for cleaning the old records -#CleanBulkSize=5000 -# In days. Entries older than this will be purged. -#CleanInterval=7 - -## The higher the values for the following parameters, -## the higher the latency for some operations (as cancelations), -## but can also reduce the system and/or database load - -# In seconds, how often to purge the messaging directory -#PurgeMessagingDirectoryInterval = 600 -# In seconds, how often to run sanity checks -#CheckSanityStateInterval = 3600 -# In seconds, how often to check for canceled transfers -#CancelCheckInterval = 10 -# In seconds, how often to check for expired queued transfers -#QueueTimeoutCheckInterval = 300 -# In seconds, how often to check for stalled transfers -#ActiveTimeoutCheckInterval = 300 -# In seconds, how often to schedule new transfers -#SchedulingInterval = 2 -# In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2 -#MessagingConsumeInterval = 1 -#Enable or disable auto session reuse -AutoSessionReuse = true -#Max small file size for session reuse in bytes -AutoSessionReuseMaxSmallFileSize = 104857600 -#Max big file size for session reuse in bytes -AutoSessionReuseMaxBigFileSize = 1073741824 -#Max number of files per session reuse -AutoSessionReuseMaxFiles = 1000 -#Max number of big files per session reuse -AutoSessionReuseMaxBigFiles = 2 -BackupTables=false -OptimizerMaxSuccessRate=100 -OptimizerMedSuccessRate=80 -OptimizerLowSuccessRate=75 -OptimizerBaseSuccessRate=74 -Port=8443 -UseFixedJobPriority=0 - +#OpenID parameters ValidateAccessTokenOffline=True JWKCacheSeconds=86400 TokenRefreshDaemonIntervalInSeconds=600 + +[roles] +Public = vo:transfer;all:datamanagement +lcgadmin = all:config + +[providers] +xdc=https://iam.extreme-datacloud.eu +xdc_ClientId= +xdc_ClientSecret= + # Logging configuration [loggers] keys = root, routes, fts3rest, sqlalchemy @@ -176,7 +49,7 @@ qualname = routes.middleware # "level = DEBUG" logs the route matched and routing variables. [logger_fts3rest] -level = INFO +level = DEBUG handlers = qualname = fts3rest @@ -195,8 +68,6 @@ level = NOTSET formatter = generic [handler_log_file] -# See -# http://docs.python.org/2/library/logging.handlers.html class = logging.FileHandler args = ('/var/log/fts3rest/fts3rest.log', 'a') level = NOTSET @@ -205,13 +76,3 @@ formatter = generic [formatter_generic] format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s datefmt = %H:%M:%S - -[roles] -Public = vo:transfer;all:datamanagement -lcgadmin = all:config - - -[providers] -xdc=https://iam.extreme-datacloud.eu -xdc_ClientId= -xdc_ClientSecret= diff --git a/src/fts3rest/fts3rest/tests/functional/test_archive.py b/src/fts3rest/fts3rest/tests/functional/test_archive.py index 4e99675ac3b6cc82a8c3ee53afa5338cda8674f0..73af763cdf9d41952dad1eb542fe467ec8e77fe8 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_archive.py +++ b/src/fts3rest/fts3rest/tests/functional/test_archive.py @@ -1,7 +1,7 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials -from fts3.model import ArchivedJob, ArchivedFile +from fts3rest.model import ArchivedJob, ArchivedFile class TestArchive(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_banning.py b/src/fts3rest/fts3rest/tests/functional/test_banning.py index fe1c6d0bda3709bda549f5b4c1793e059504a2db..7ff3f82add7b8d12b77ecfd29ab3cd4bb3d1613d 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_banning.py +++ b/src/fts3rest/fts3rest/tests/functional/test_banning.py @@ -1,7 +1,7 @@ import json from urllib.parse import quote -from fts3.model import BannedDN, BannedSE, Job, File +from fts3rest.model import BannedDN, BannedSE, Job, File from fts3rest.model.meta import Session from fts3rest.tests import TestController diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py b/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py index 2af628206ce204f16191120ff915e74c95ade8df..a3bf56b889199f85ad4fd30d65ecb264b49dd6af 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, ActivityShare +from fts3rest.model import ConfigAudit, ActivityShare class TestConfigActivityShare(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py b/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py index 521f9deea79de7ec9506380b438495a63687cf84..5abf160f3cffa2aa2754589a750d9c503b70daee 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, AuthorizationByDn +from fts3rest.model import ConfigAudit, AuthorizationByDn class TestConfigAuthz(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py index 132404df8f52e764b7435a07c67a358ff64915c6..eb646bb39959580d0ef698bda23d1a034f6a0178 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import CloudStorage, CloudStorageUser +from fts3rest.model import CloudStorage, CloudStorageUser class TestConfigCloud(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_global.py b/src/fts3rest/fts3rest/tests/functional/test_config_global.py index 450cfce67d753c5d424da002f3dbd9dc8c179e5a..b8b3a091fe4a27c88d956f355be295b1f951f592 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_global.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_global.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model.config import ConfigAudit, ServerConfig +from fts3rest.model import ConfigAudit, ServerConfig class TestConfigGlobal(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_links.py b/src/fts3rest/fts3rest/tests/functional/test_config_links.py index 87860caa4032dabb87c19540c516cf27cd9ab1b0..1dd9eff071204b0af1d6d6476e6ff6223b7cf641 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_links.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_links.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, LinkConfig +from fts3rest.model import ConfigAudit, LinkConfig class TestConfigLinks(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_se.py b/src/fts3rest/fts3rest/tests/functional/test_config_se.py index 6ca52993ee64494be557164a7ba4e04cf6eb73a8..b1dcbbd4aabb6bafe6ad860cbdf4908d63ac76cc 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_se.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_se.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, Optimizer, OperationConfig, Se +from fts3rest.model import ConfigAudit, Optimizer, OperationConfig, Se class TestConfigSe(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_shares.py b/src/fts3rest/fts3rest/tests/functional/test_config_shares.py index 3081c3b4311ad986a7bb7b37655220da4cdb916b..1e5dd8bc16b2b874519f5da1e585aaf58ecc11e2 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_shares.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_shares.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model.config import ( +from fts3rest.model import ( ConfigAudit, ServerConfig, OperationConfig, diff --git a/src/fts3rest/fts3rest/tests/functional/test_delegation.py b/src/fts3rest/fts3rest/tests/functional/test_delegation.py index 8dae6d96002e7072b5d4b33bac11954f1093fff2..31f4f6255fa8a1f3510f1f2c8b9eaf13ee433b02 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_delegation.py +++ b/src/fts3rest/fts3rest/tests/functional/test_delegation.py @@ -5,7 +5,7 @@ import time from fts3rest.controllers.delegation import _generate_proxy_request from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Credential, CredentialCache +from fts3rest.model import Credential, CredentialCache class TestDelegation(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_drain.py b/src/fts3rest/fts3rest/tests/functional/test_drain.py index 8324c7fe8db4242172d15f3f8414592409d91c90..a86855c668e633a90f49b6fb8780fe2a9065a1d8 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_drain.py +++ b/src/fts3rest/fts3rest/tests/functional/test_drain.py @@ -2,7 +2,7 @@ from datetime import datetime from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, Host +from fts3rest.model import ConfigAudit, Host class TestDrain(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_dropbox.py b/src/fts3rest/fts3rest/tests/functional/test_dropbox.py index 502186c4ee430f6a4503621d0aee4bd394f19b1a..39932d879c9a225bf7adc9bf519b72b2ab25a181 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_dropbox.py +++ b/src/fts3rest/fts3rest/tests/functional/test_dropbox.py @@ -3,7 +3,7 @@ from unittest.mock import patch from fts3rest.model.meta import Session from fts3rest.tests import TestController from fts3rest.controllers.CSdropbox import DropboxConnector -from fts3.model import CloudStorage, CloudStorageUser +from fts3rest.model import CloudStorage, CloudStorageUser def _oauth_header_dict(raw_header): diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py b/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py index 5ab52a1319cf606b00f1b7a290bbc01e80b45f7f..5067615db42171468752308617407dfa795b9a4d 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ( +from fts3rest.model import ( Job, File, JobActiveStates, diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py b/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py index 6354216b69c5540d6249bce99121fd3b2a7b5049..b072e1a6ba7a12cb90a30af4c4386cc302842277 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job, DataManagement +from fts3rest.model import Job, DataManagement class TestJobDeletion(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_listing.py b/src/fts3rest/fts3rest/tests/functional/test_job_listing.py index ff3824f8bd352e1c19d174b4e8e7e9efce9142fc..c4f691a0eabaeffb1a78187faf4044a662f16078 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_listing.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_listing.py @@ -1,7 +1,7 @@ import json from datetime import datetime, timedelta -from fts3.model import FileRetryLog, Job, File +from fts3rest.model import FileRetryLog, Job, File from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials from fts3rest.lib.middleware.fts3auth import constants diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_modify.py b/src/fts3rest/fts3rest/tests/functional/test_job_modify.py index 449eebd0adac7a091859e1cf9fe0cc26028d8e99..cde2229c8c7a3d383fb3cd7d4f5843e342649e9e 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_modify.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_modify.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job +from fts3rest.model import Job class TestJobModify(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py index 6dc3829c2a1b873e7c75e6858bf992aa866e6260..d882c34620ab5a3114158a363f0866006736503f 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py @@ -4,9 +4,8 @@ import time from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import File, Job +from fts3rest.model import Job import random -import unittest from math import ceil diff --git a/src/fts3rest/fts3rest/tests/functional/test_multiple.py b/src/fts3rest/fts3rest/tests/functional/test_multiple.py index 415a31c824d6a506ec5e0a3b9fc4b1a029bc74e5..81f0ce03d57b96eb3f70c9f336a07d9fd386d47c 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_multiple.py +++ b/src/fts3rest/fts3rest/tests/functional/test_multiple.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job, File +from fts3rest.model import Job, File class TestMultiple(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py new file mode 100644 index 0000000000000000000000000000000000000000..4e254c0281050a8bdba896e999f86687e09dcf62 --- /dev/null +++ b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py @@ -0,0 +1,74 @@ +from fts3rest.lib.oauth2provider import FTS3OAuth2ResourceProvider +from fts3rest.lib.openidconnect import OIDCmanager +from fts3rest.tests import TestController + + +class TestFTS3OAuth2ResourceProvider(TestController): + """ + Test token validation + + To run these tests, the host should have oidc-agent installed, + with an account 'xdctest' + """ + + def setUp(self): + super().setUp() + self.oidc_manager = OIDCmanager() + config = self.flask_app.config + self.issuer = "https://iam.extreme-datacloud.eu/" + self.oidc_manager.setup(config) + self.oauth2_resource_provider = FTS3OAuth2ResourceProvider(dict(), config) + self.expired_token = "eyJraWQiOiJyc2ExIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI5NGQyNTQyOS1mYTZhLTRiYTctOGM0NS1mMTk1YjI3ZWVkNjMiLCJpc3MiOiJodHRwczpcL1wvaWFtLmV4dHJlbWUtZGF0YWNsb3VkLmV1XC8iLCJleHAiOjE1ODAyMjIyMDksImlhdCI6MTU4MDIxODYwOSwianRpIjoiYTI0NDRhYTQtNTE3YS00Y2E0LTgwMTUtY2IyMjc0Nzg4YzlkIn0.hvTjA-Ix_YVxU3HmLB6FQa98eYtUwbw1WcZMO5p_qOjnPwD0OtQViVtV-a5__hLY1_qRFouAzgVvqKnueokh1pmKoI6TJN2KpmybueAZR30lIG_t_aAn4hGQvuVezs_0LLISojQUgprbi2PDsU1q8WTJq1J5mwGwlBijGmHQs60" + + def test_validate_access_token(self): + token = self._get_xdc_access_token() + auth = self.oauth2_resource_provider.authorization_class() + self.oauth2_resource_provider.validate_access_token(token, auth) + self.assertTrue(auth.is_valid) + + def test_validate_token_offline(self): + token = self._get_xdc_access_token() + valid, credential = self.oauth2_resource_provider._validate_token_offline(token) + self.assertTrue(valid) + self.assertEqual(credential["iss"], self.issuer) + + def test_validate_token_online(self): + token = self._get_xdc_access_token() + valid, credential = self.oauth2_resource_provider._validate_token_online(token) + self.assertTrue(valid) + self.assertEqual(credential["iss"], self.issuer) + + def test_validate_access_token_invalid(self): + token = self._get_xdc_access_token() + token += "invalid" + auth = self.oauth2_resource_provider.authorization_class() + self.oauth2_resource_provider.validate_access_token(token, auth) + self.assertFalse(auth.is_valid) + + def test_validate_token_offline_invalid(self): + token = self._get_xdc_access_token() + token += "invalid" + valid, credential = self.oauth2_resource_provider._validate_token_offline(token) + self.assertFalse(valid) + + def test_validate_token_online_invalid(self): + token = self._get_xdc_access_token() + token += "invalid" + valid, credential = self.oauth2_resource_provider._validate_token_online(token) + self.assertFalse(valid) + + def test_validate_access_token_expired(self): + token = self.expired_token + auth = self.oauth2_resource_provider.authorization_class() + self.oauth2_resource_provider.validate_access_token(token, auth) + self.assertFalse(auth.is_valid) + + def test_validate_token_offline_expired(self): + token = self.expired_token + valid, credential = self.oauth2_resource_provider._validate_token_offline(token) + self.assertFalse(valid) + + def test_validate_token_online_expired(self): + token = self.expired_token + valid, credential = self.oauth2_resource_provider._validate_token_online(token) + self.assertFalse(valid) diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py new file mode 100644 index 0000000000000000000000000000000000000000..ad67bd4d9e9b008b82fb92d3bd61a56bff038354 --- /dev/null +++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py @@ -0,0 +1,56 @@ +from fts3rest.model import Credential +from fts3rest.lib.openidconnect import OIDCmanager +from fts3rest.tests import TestController +import unittest + + +class TestOpenidconnect(TestController): + """ + Tests OIDCmanager operations + + To run these tests, the host should have oidc-agent installed, + with an account 'xdctest' + """ + + def setUp(self): + super().setUp() + self.oidc_manager = OIDCmanager() + self.config = self.flask_app.config + self.issuer = "https://iam.extreme-datacloud.eu/" + if "client_id" not in self.config["fts3.Providers"][self.issuer]: + raise unittest.SkipTest("Missing OIDC client configuration data") + + def test_configure_clients(self): + self.oidc_manager._configure_clients(self.config["fts3.Providers"]) + self.assertEqual( + len(self.oidc_manager.clients), len(self.config["fts3.Providers"]) + ) + + def test_introspect(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + response = self.oidc_manager.introspect(self.issuer, access_token) + self.assertTrue(response["active"]) + + def test_generate_refresh_token(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + self.oidc_manager.generate_refresh_token(self.issuer, access_token) + + def test_generate_refresh_token_invalid(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + access_token += "invalid" + with self.assertRaises(Exception): + self.oidc_manager.generate_refresh_token(self.issuer, access_token) + + def test_refresh_access_token(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + refresh_token = self.oidc_manager.generate_refresh_token( + self.issuer, access_token + ) + credential = Credential() + credential.proxy = ":".join([access_token, refresh_token]) + new_credential = self.oidc_manager.refresh_access_token(credential) + self.assertIsNotNone(new_credential.termination_time) diff --git a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py index 66c5ce0157a120de8c882767a83357e06c5ed4f3..05625c4a7f8bb76143aa448762bb797229a0c847 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py +++ b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Optimizer +from fts3rest.model import Optimizer class TestOptimizer(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_scheduler.py b/src/fts3rest/fts3rest/tests/functional/test_scheduler.py index 0328fb9cba9009f3445d05be9f06d605a3c168d2..f98f371c0caba1efcd80c2fb8c263fbc71f60cad 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_scheduler.py +++ b/src/fts3rest/fts3rest/tests/functional/test_scheduler.py @@ -5,7 +5,7 @@ import logging from fts3rest.tests import TestController from fts3rest.model.meta import Session from fts3rest.lib.scheduler.Cache import ThreadLocalCache -from fts3.model import Job, File, OptimizerEvolution, ActivityShare +from fts3rest.model import Job, File, OptimizerEvolution, ActivityShare import random log = logging.getLogger(__name__) diff --git a/src/fts3rest/fts3rest/tests/functional/test_staging.py b/src/fts3rest/fts3rest/tests/functional/test_staging.py index 289c1aff362e083242f3b82ce7904062038ca0fe..8be2502f58d04073b6edbbb14e85dea32bdeea05 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_staging.py +++ b/src/fts3rest/fts3rest/tests/functional/test_staging.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job +from fts3rest.model import Job class TestSubmitToStaging(TestController):