From edaf1ad4b862f1c5500190a4bc7dbf3762a42370 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 11:57:27 +0200 Subject: [PATCH 01/27] Fix tests and improve config parsing --- src/fts3/util/config.py | 37 ++-- src/fts3rest/fts3rest/config/middleware.py | 12 +- .../fts3rest/lib/IAMTokenRefresher.py | 141 +++++++++++++++ src/fts3rest/fts3rest/lib/openidconnect.py | 2 +- src/fts3rest/fts3rest/tests/__init__.py | 24 ++- src/fts3rest/fts3rest/tests/fts3testconfig | 163 ++---------------- .../tests/functional/test_oauth2provider.py | 74 ++++++++ .../tests/functional/test_openidconnect.py | 53 ++++++ 8 files changed, 328 insertions(+), 178 deletions(-) create mode 100644 src/fts3rest/fts3rest/lib/IAMTokenRefresher.py create mode 100644 src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py create mode 100644 src/fts3rest/fts3rest/tests/functional/test_openidconnect.py diff --git a/src/fts3/util/config.py b/src/fts3/util/config.py index fe2262ba..bc26c2be 100644 --- a/src/fts3/util/config.py +++ b/src/fts3/util/config.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from configparser import ConfigParser, NoOptionError +from configparser import ConfigParser, NoOptionError, NoSectionError from urllib.parse import quote_plus import os import logging @@ -100,18 +100,29 @@ def fts3_config_load(path="/etc/fts3/fts3config"): fts3cfg["fts3.Roles"][role.lower()][operation.lower()] = level.lower() # Initialize providers - log.debug("initialize providers config in load environment") fts3cfg["fts3.Providers"] = {} - for option in parser.options("providers"): - if "_" not in option: - provider_name = option - provider_url = parser.get("providers", provider_name) - if not provider_url.endswith("/"): - provider_url += "/" - fts3cfg["fts3.Providers"][provider_url] = {} - client_id = parser.get("providers", option + "_ClientId") - fts3cfg["fts3.Providers"][provider_url]["client_id"] = client_id - client_secret = parser.get("providers", option + "_ClientSecret") - fts3cfg["fts3.Providers"][provider_url]["client_secret"] = client_secret + try: + for option in parser.options("providers"): + if "_" not in option: + provider_name = option + provider_url = parser.get("providers", provider_name) + if not provider_url.endswith("/"): + provider_url += "/" + fts3cfg["fts3.Providers"][provider_url] = {} + client_id = parser.get("providers", option + "_ClientId") + fts3cfg["fts3.Providers"][provider_url]["client_id"] = client_id + client_secret = parser.get("providers", option + "_ClientSecret") + fts3cfg["fts3.Providers"][provider_url]["client_secret"] = client_secret + fts3cfg["fts3.ValidateAccessTokenOffline"] = parser.getboolean( + "fts3", "ValidateAccessTokenOffline", fallback=True + ) + fts3cfg["JWKCacheSeconds"] = parser.getint( + "fts3", "JWKCacheSeconds", fallback=86400 + ) + fts3cfg["TokenRefreshDaemonIntervalInSeconds"] = parser.getint( + "fts3", "TokenRefreshDaemonIntervalInSeconds", fallback=600 + ) + except NoSectionError: + pass return fts3cfg diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index 1aba2610..30b90854 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -16,6 +16,8 @@ from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddlewa from fts3rest.lib.middleware.error_as_json import ErrorAsJson from fts3rest.lib.middleware.timeout import TimeoutHandler from fts3rest.model.meta import Session +from fts3rest.lib.openidconnect import oidc_manager +from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher from werkzeug.exceptions import HTTPException import json @@ -109,7 +111,6 @@ def create_app(default_config_file=None, test=False): app.wsgi_app = TimeoutHandler(app.wsgi_app, fts3cfg) # Convert errors to JSON - # app.wsgi_app = ErrorAsJson(app.wsgi_app) @app.errorhandler(HTTPException) def handle_exception(e): """Return JSON instead of HTML for HTTP errors.""" @@ -122,10 +123,9 @@ def create_app(default_config_file=None, test=False): response.content_type = "application/json" return response - # @app.errorhandler(NotFound) - # def handle_invalid_usage(error): - # response = jsonify(error=error.code, name=error.name) - # response.status_code = error.code - # return response + # Start OIDC clients + if "fts3.Providers" in app.config: + oidc_manager.setup(app.config) + IAMTokenRefresher("fts_token_refresh_daemon", app.config).start() return app diff --git a/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py new file mode 100644 index 00000000..49220094 --- /dev/null +++ b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py @@ -0,0 +1,141 @@ +# Copyright 2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import socket +import time +import random +from datetime import datetime, timedelta +from threading import Thread, current_thread + + +from fts3rest.model.meta import Session +from fts3rest.lib.openidconnect import oidc_manager +from fts3.model import Credential, Host + +from sqlalchemy.exc import SQLAlchemyError + +log = logging.getLogger(__name__) + + +class IAMTokenRefresher(Thread): + """ + Daemon thread that refreshes all access tokens in the DB at every interval. + + Keeps running on the background updating the DB, marking the process as alive. + There should be ONLY ONE across all instances. + Keep in mind that with the Apache configuration line + WSGIDaemonProcess fts3rest python-path=... processes=2 threads=15 + there will be 2 instances of the application per server, meaning we need to check that there is only one + IAMTokenRefresher per host, and only one between all hosts. + + The SQLAlchemy scoped_session is thread-safe + """ + + def __init__(self, tag, config): + Thread.__init__(self) + self.daemon = ( + True # The thread will immediately exit when the main thread exits + ) + self.tag = tag + self.refresh_interval = int( + config.get("fts3.TokenRefreshDaemonIntervalInSeconds", 600) + ) + self.config = config + + def _thread_is_inactive(self, thread): + # The thread is considered inactive if it hasn't updated the DB for 3*refresh_interval + log.debug("time since last beat {}".format(datetime.utcnow() - thread.beat)) + if (datetime.utcnow() - thread.beat) > timedelta( + seconds=3 * self.refresh_interval + ): + log.debug("thread is inactive! taking over, beat {}".format(thread.beat)) + return (datetime.utcnow() - thread.beat) > timedelta( + seconds=3 * self.refresh_interval + ) + + def run(self): + """ + Regularly check if there is another active IAMTokenRefresher in the DB. If not, become the active thread. + """ + log.debug("CREATE THREAD ID: {}".format(current_thread().ident)) + # Initial sleep in case all threads started at the same time + time.sleep(random.randint(0, 60)) + # The interval at which the thread will check if there is another active thread. + # It is arbitrary: I chose 2 times the refresh interval, plus a random offset to avoid multiple threads + # checking at the same time (although DB access is transactional) + db_check_interval = 3 * self.refresh_interval + random.randint(0, 120) + while True: + # Check that no other fts-token-refresh-daemon is running + refresher_threads = ( + Session.query(Host).filter(Host.service_name == self.tag).all() + ) + Session.commit() # Close transaction to avoid repeated read + log.debug( + "refresher_threads {}, ID {}".format( + len(refresher_threads), current_thread().ident + ) + ) + if all(self._thread_is_inactive(thread) for thread in refresher_threads): + log.debug("Activating thread") + for thread in refresher_threads: + Session.delete(thread) + log.debug("delete thread") + host = Host(hostname=socket.getfqdn(), service_name=self.tag) + log.debug("host object created") + while True: + host.beat = datetime.utcnow() + log.debug( + "THREAD ID: {}, beat {}".format( + current_thread().ident, host.beat + ) + ) + try: + h2 = Session.merge(host) + Session.commit() + log.debug( + "fts-token-refresh-daemon heartbeat {}".format(h2.beat) + ) + except SQLAlchemyError as ex: + log.warning( + "Failed to update the fts-token-refresh-daemon heartbeat: %s" + % str(ex) + ) + Session.rollback() + raise + + credentials = ( + Session.query(Credential) + .filter(Credential.proxy.notilike("%CERTIFICATE%")) + .all() + ) + log.debug("{} credentials to refresh".format(len(credentials))) + for credential in credentials: + try: + credential = oidc_manager.refresh_access_token(credential) + log.debug("OK refresh_access_token") + Session.merge(credential) + Session.commit() + except Exception as ex: + log.warning( + "Failed to refresh token for dn: %s because: %s" + % (str(credential.dn), str(ex)) + ) + Session.rollback() + raise + time.sleep(self.refresh_interval) + else: + log.debug("THREAD ID: {}".format(current_thread().ident)) + log.debug("Another thread is active -- Going to sleep") + time.sleep(db_check_interval) diff --git a/src/fts3rest/fts3rest/lib/openidconnect.py b/src/fts3rest/fts3rest/lib/openidconnect.py index 2a48bcf1..1024e6ef 100644 --- a/src/fts3rest/fts3rest/lib/openidconnect.py +++ b/src/fts3rest/fts3rest/lib/openidconnect.py @@ -27,7 +27,7 @@ class OIDCmanager: def setup(self, config): self.config = config self._configure_clients(config["fts3.Providers"]) - self._set_keys_cache_time(int(config["fts3.JWKCacheSeconds"])) + self._set_keys_cache_time(config["fts3.JWKCacheSeconds"]) self._retrieve_clients_keys() def _configure_clients(self, providers_config): diff --git a/src/fts3rest/fts3rest/tests/__init__.py b/src/fts3rest/fts3rest/tests/__init__.py index ee046677..466373d2 100644 --- a/src/fts3rest/fts3rest/tests/__init__.py +++ b/src/fts3rest/fts3rest/tests/__init__.py @@ -1,18 +1,18 @@ +from datetime import timedelta import os import shutil +import subprocess import time - -from datetime import datetime, timedelta +import unittest from unittest import TestCase + from M2Crypto import ASN1, X509, RSA, EVP from M2Crypto.ASN1 import UTC - -from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials -from fts3rest.model.meta import Session -from fts3.model import Credential, CredentialCache, DataManagement from fts3.model import * from fts3rest.config.middleware import create_app +from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials +from fts3rest.model.meta import Session from .ftstestclient import FTSTestClient, TestResponse @@ -198,3 +198,15 @@ class TestController(TestCase): pass self.flask_app.do_teardown_appcontext() + + def _get_xdc_access_token(self): + command = ( + "eval `oidc-agent` && oidc-add xdctest --pw-cmd=echo && oidc-token xdctest" + ) + try: + output = subprocess.check_output(command, shell=True) + except subprocess.CalledProcessError as ex: + raise unittest.SkipTest("Failed to get access token") from ex + output = str(output).strip() + token = output.split("\n")[2] # The 3rd line is the token + return token diff --git a/src/fts3rest/fts3rest/tests/fts3testconfig b/src/fts3rest/fts3rest/tests/fts3testconfig index a0099268..90334d2f 100644 --- a/src/fts3rest/fts3rest/tests/fts3testconfig +++ b/src/fts3rest/fts3rest/tests/fts3testconfig @@ -1,7 +1,3 @@ -# Running user and group -User=ftsflask -Group=ftsflask - # mysql only DbType=mysql @@ -11,150 +7,27 @@ DbUserName=ftsflask #db password DbPassword= - #For MySQL, it has to follow the format 'host/db' (i.e. "mysql-server.example.com/fts3db") DbConnectString=localhost:3306/ftsflask -#Number of db connections in the pool (use even number, e.g. 2,4,6,8,etc OR 1 for a single connection) -DbThreadsNum=30 - -#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/ -#Alias=fts3-xdc.cern.ch - -#Infosys, either the fqdn:port of a BDII instance or false to disable BDII access -#Infosys=lcg-bdii.cern.ch:2170 - -#Query the info systems specified in the order given, e.g. glue1;glue2 -InfoProviders=glue1 - #List of authorized VOs, separated by ; #Leave * to authorize any VO AuthorizedVO=* -# site name -#SiteName=FTS-DEV-XDC - -#Enable/Disable monitoring using messaging monitoring (disabled=false / enabled=true) -MonitoringMessaging=false - -# Profiling interval in seconds. If set to 0, it will be disabled -Profiling=0 - -# Log directories -TransferLogDirectory=/var/log/fts3/transfers -ServerLogDirectory=/var/log/fts3 - -# Log level. Enables logging for messages of level >= than configured -# Possible values are -# TRACE (every detail), DEBUG (internal behaviour), INFO (normal behaviour), -# NOTICE (final states), WARNING (things worth checking), ERR (internal FTS3 errors, as database connectivity), -# CRIT (fatal errors, as segmentation fault) -# It is recommended to use DEBUG or INFO -LogLevel=DEBUG - -# Check for fts_url_copy processes that do not give their progress back -# CheckStalledTransfers = true -# Stalled timeout, in seconds -# CheckStalledTimeout = 900 -CheckStalledTimeout = 900 - -# Minimum required free RAM (in MB) for FTS3 to work normally -# If the amount of free RAM goes below the limit, FTS3 will enter auto-drain mode -# This is intended to protect against system resource exhaustion -# MinRequiredFreeRAM = 50 -MinRequiredFreeRAM = 50 - -# Maximum number of url copy processes that the node can run -# The RAM limitation may not take into account other node limitations (i.e. IO) -# or, depending on the swapping policy, may not even prevent overloads if the kernel -# starts swapping before the free RAM decreases until it reaches the value of MinRequiredFreeRAM -# 0 disables the check. -# The default is 400. -# MaxUrlCopyProcesses = 400 -MaxUrlCopyProcesses = 400 - -# Parameters for Bring Online -# Maximum bulk size. -# If the size is too large, it will take more resources (memory and CPU) to generate the requests and -# parse the responses. Some servers may reject the requests if they are too big. -# If it is too small, performance will be reduced. -# Keep it to a sensible size (between 100 and 1k) -# StagingBulkSize=400 -# Maximum number of concurrent requests. This gives a maximum of files sent to the storage system -# (StagingBulkSize*StagingConcurrentRequests). The larger the number, the more requests will FTS need to keep track of. -# StagingConcurrentRequests=500 -# Seconds to wait before submitting a bulk request, so FTS can accumulate more files per bulk. -# Note that the resolution is 60 seconds. -# StagingWaitingFactor=300 -# Retry this number of times if a staging poll fails with ECOMM -# StagingPollRetries=3 - -# In seconds, interval between heartbeats -# HeartBeatInterval=60 -# I seconds, after this interval a host is considered down -# HeartBeatGraceInterval=120 - -# Seconds between optimizer runs -# OptimizerInterval = 60 -# After this time without optimizer updates, force a run -# OptimizerSteadyInterval = 300 -# Maximum number of streams per file -# OptimizerMaxStreams = 16 - -# EMA Alpha factor to reduce the influence of fluctuations -# OptimizerEMAAlpha = 0.1 -# Increase step size when the optimizer considers the performance is good -# OptimizerIncreaseStep = 1 -# Increase step size when the optimizer considers the performance is good, and set to aggressive or normal -# OptimizerAggressiveIncreaseStep = 2 -# Decrease step size when the optimizer considers the performance is bad -# OptimizerDecreaseStep = 1 - - -# Set the bulk size, in number of jobs, used for cleaning the old records -#CleanBulkSize=5000 -# In days. Entries older than this will be purged. -#CleanInterval=7 - -## The higher the values for the following parameters, -## the higher the latency for some operations (as cancelations), -## but can also reduce the system and/or database load - -# In seconds, how often to purge the messaging directory -#PurgeMessagingDirectoryInterval = 600 -# In seconds, how often to run sanity checks -#CheckSanityStateInterval = 3600 -# In seconds, how often to check for canceled transfers -#CancelCheckInterval = 10 -# In seconds, how often to check for expired queued transfers -#QueueTimeoutCheckInterval = 300 -# In seconds, how often to check for stalled transfers -#ActiveTimeoutCheckInterval = 300 -# In seconds, how often to schedule new transfers -#SchedulingInterval = 2 -# In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2 -#MessagingConsumeInterval = 1 -#Enable or disable auto session reuse -AutoSessionReuse = true -#Max small file size for session reuse in bytes -AutoSessionReuseMaxSmallFileSize = 104857600 -#Max big file size for session reuse in bytes -AutoSessionReuseMaxBigFileSize = 1073741824 -#Max number of files per session reuse -AutoSessionReuseMaxFiles = 1000 -#Max number of big files per session reuse -AutoSessionReuseMaxBigFiles = 2 -BackupTables=false -OptimizerMaxSuccessRate=100 -OptimizerMedSuccessRate=80 -OptimizerLowSuccessRate=75 -OptimizerBaseSuccessRate=74 -Port=8443 -UseFixedJobPriority=0 - +#OpenID parameters ValidateAccessTokenOffline=True JWKCacheSeconds=86400 TokenRefreshDaemonIntervalInSeconds=600 + +[roles] +Public = vo:transfer;all:datamanagement +lcgadmin = all:config + +[providers] +xdc=https://iam.extreme-datacloud.eu +xdc_ClientId= +xdc_ClientSecret= + # Logging configuration [loggers] keys = root, routes, fts3rest, sqlalchemy @@ -195,8 +68,6 @@ level = NOTSET formatter = generic [handler_log_file] -# See -# http://docs.python.org/2/library/logging.handlers.html class = logging.FileHandler args = ('/var/log/fts3rest/fts3rest.log', 'a') level = NOTSET @@ -205,15 +76,3 @@ formatter = generic [formatter_generic] format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s datefmt = %H:%M:%S - - - -[roles] -Public = vo:transfer;all:datamanagement -lcgadmin = all:config - - -[providers] -xdc=https://iam.extreme-datacloud.eu -xdc_ClientId= -xdc_ClientSecret= diff --git a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py new file mode 100644 index 00000000..4e254c02 --- /dev/null +++ b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py @@ -0,0 +1,74 @@ +from fts3rest.lib.oauth2provider import FTS3OAuth2ResourceProvider +from fts3rest.lib.openidconnect import OIDCmanager +from fts3rest.tests import TestController + + +class TestFTS3OAuth2ResourceProvider(TestController): + """ + Test token validation + + To run these tests, the host should have oidc-agent installed, + with an account 'xdctest' + """ + + def setUp(self): + super().setUp() + self.oidc_manager = OIDCmanager() + config = self.flask_app.config + self.issuer = "https://iam.extreme-datacloud.eu/" + self.oidc_manager.setup(config) + self.oauth2_resource_provider = FTS3OAuth2ResourceProvider(dict(), config) + self.expired_token = "eyJraWQiOiJyc2ExIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI5NGQyNTQyOS1mYTZhLTRiYTctOGM0NS1mMTk1YjI3ZWVkNjMiLCJpc3MiOiJodHRwczpcL1wvaWFtLmV4dHJlbWUtZGF0YWNsb3VkLmV1XC8iLCJleHAiOjE1ODAyMjIyMDksImlhdCI6MTU4MDIxODYwOSwianRpIjoiYTI0NDRhYTQtNTE3YS00Y2E0LTgwMTUtY2IyMjc0Nzg4YzlkIn0.hvTjA-Ix_YVxU3HmLB6FQa98eYtUwbw1WcZMO5p_qOjnPwD0OtQViVtV-a5__hLY1_qRFouAzgVvqKnueokh1pmKoI6TJN2KpmybueAZR30lIG_t_aAn4hGQvuVezs_0LLISojQUgprbi2PDsU1q8WTJq1J5mwGwlBijGmHQs60" + + def test_validate_access_token(self): + token = self._get_xdc_access_token() + auth = self.oauth2_resource_provider.authorization_class() + self.oauth2_resource_provider.validate_access_token(token, auth) + self.assertTrue(auth.is_valid) + + def test_validate_token_offline(self): + token = self._get_xdc_access_token() + valid, credential = self.oauth2_resource_provider._validate_token_offline(token) + self.assertTrue(valid) + self.assertEqual(credential["iss"], self.issuer) + + def test_validate_token_online(self): + token = self._get_xdc_access_token() + valid, credential = self.oauth2_resource_provider._validate_token_online(token) + self.assertTrue(valid) + self.assertEqual(credential["iss"], self.issuer) + + def test_validate_access_token_invalid(self): + token = self._get_xdc_access_token() + token += "invalid" + auth = self.oauth2_resource_provider.authorization_class() + self.oauth2_resource_provider.validate_access_token(token, auth) + self.assertFalse(auth.is_valid) + + def test_validate_token_offline_invalid(self): + token = self._get_xdc_access_token() + token += "invalid" + valid, credential = self.oauth2_resource_provider._validate_token_offline(token) + self.assertFalse(valid) + + def test_validate_token_online_invalid(self): + token = self._get_xdc_access_token() + token += "invalid" + valid, credential = self.oauth2_resource_provider._validate_token_online(token) + self.assertFalse(valid) + + def test_validate_access_token_expired(self): + token = self.expired_token + auth = self.oauth2_resource_provider.authorization_class() + self.oauth2_resource_provider.validate_access_token(token, auth) + self.assertFalse(auth.is_valid) + + def test_validate_token_offline_expired(self): + token = self.expired_token + valid, credential = self.oauth2_resource_provider._validate_token_offline(token) + self.assertFalse(valid) + + def test_validate_token_online_expired(self): + token = self.expired_token + valid, credential = self.oauth2_resource_provider._validate_token_online(token) + self.assertFalse(valid) diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py new file mode 100644 index 00000000..74131e92 --- /dev/null +++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py @@ -0,0 +1,53 @@ +from fts3.model import Credential +from fts3rest.lib.openidconnect import OIDCmanager +from fts3rest.tests import TestController + + +class TestOpenidconnect(TestController): + """ + Tests OIDCmanager operations + + To run these tests, the host should have oidc-agent installed, + with an account 'xdctest' + """ + + def setUp(self): + super().setUp() + self.oidc_manager = OIDCmanager() + self.config = self.flask_app.config + self.issuer = "https://iam.extreme-datacloud.eu/" + + def test_configure_clients(self): + self.oidc_manager._configure_clients(self.config["fts3.Providers"]) + self.assertEqual( + len(self.oidc_manager.clients), len(self.config["fts3.Providers"]) + ) + + def test_introspect(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + response = self.oidc_manager.introspect(self.issuer, access_token) + self.assertTrue(response["active"]) + + def test_generate_refresh_token(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + self.oidc_manager.generate_refresh_token(self.issuer, access_token) + + def test_generate_refresh_token_invalid(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + access_token += "invalid" + with self.assertRaises(Exception): + self.oidc_manager.generate_refresh_token(self.issuer, access_token) + + def test_refresh_access_token(self): + self.oidc_manager.setup(self.config) + access_token = self._get_xdc_access_token() + refresh_token = self.oidc_manager.generate_refresh_token( + self.issuer, access_token + ) + credential = Credential() + credential.proxy = ":".join([access_token, refresh_token]) + new_credential = self.oidc_manager.refresh_access_token(credential) + self.assertIsNotNone(new_credential.termination_time) -- GitLab From 5311d4d56e6cdd4005b5ef343b1d537334e921f2 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 12:22:13 +0200 Subject: [PATCH 02/27] fix ci --- .gitlab-ci.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 000eb128..5e1bbc1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,13 +8,18 @@ image: gitlab-registry.cern.ch/fts/fts-rest-flask:ci - stages: - static_code_analysis - tests - security - build +variables: + MYSQL_DATABASE: ftsflask + MYSQL_ROOT_PASSWORD: asdf + PYTHONPATH: ./src:./src/fts3rest + FTS3TESTCONFIG: /src/fts3rest/fts3rest/tests/fts3testconfig_ci + black: # Check that every file has been formatted with black stage: static_code_analysis @@ -56,9 +61,6 @@ radon: - source .gitlab-ci/radon.sh -variables: - MYSQL_DATABASE: ftsflask - MYSQL_ROOT_PASSWORD: asdf functional_tests36: stage: tests @@ -68,8 +70,6 @@ functional_tests36: script: - python --version - source .gitlab-ci/db.sh - - export PYTHONPATH=./src:./src/fts3rest - - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report @@ -84,8 +84,6 @@ functional_tests37: - python --version - source $VENV_37/bin/activate - source .gitlab-ci/db.sh - - export PYTHONPATH=./src:./src/fts3rest - - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report @@ -100,8 +98,6 @@ functional_tests38: - python --version - source $VENV_38/bin/activate - source .gitlab-ci/db.sh - - export PYTHONPATH=./src:./src/fts3rest - - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report -- GitLab From 8b0de3cd16980df51cbc782ab61ec84a8d977609 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 12:30:15 +0200 Subject: [PATCH 03/27] fix ci --- .gitlab-ci.yml | 6 ++--- src/fts3/util/config.py | 1 - src/fts3rest/fts3rest/config/middleware.py | 27 ++++++++++++---------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e1bbc1b..c63f3bb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,7 +31,7 @@ pylint36: stage: static_code_analysis script: - python --version - - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy pylint37: # Check that every file doesn't have syntax errors @@ -41,7 +41,7 @@ pylint37: - pyenv global $PY37 - source $VENV_37/bin/activate - python --version - - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy pylint38: # Check that every file doesn't have syntax errors @@ -51,7 +51,7 @@ pylint38: - pyenv global $PY38 - source $VENV_38/bin/activate - python --version - - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy radon: diff --git a/src/fts3/util/config.py b/src/fts3/util/config.py index bc26c2be..b318d23a 100644 --- a/src/fts3/util/config.py +++ b/src/fts3/util/config.py @@ -25,7 +25,6 @@ def fts3_config_load(path="/etc/fts3/fts3config"): """ Read the configuration from the FTS3 configuration file """ - log.debug("entered fts3_config_load") fts3cfg = {} parser = ConfigParser() diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index 30b90854..db5fcbc5 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -1,25 +1,25 @@ -from flask import Flask, jsonify -from werkzeug.exceptions import NotFound -from sqlalchemy import engine_from_config, event -import MySQLdb -import os from io import StringIO +import json import logging.config -from fts3rest.config.routing import base, cstorage +import os + +import MySQLdb +from flask import Flask +from sqlalchemy import engine_from_config, event +from werkzeug.exceptions import HTTPException + from fts3.util.config import fts3_config_load -from fts3rest.model import init_model +from fts3rest.config.routing import base, cstorage +from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher from fts3rest.lib.helpers.connection_validator import ( connection_validator, connection_set_sqlmode, ) from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddleware -from fts3rest.lib.middleware.error_as_json import ErrorAsJson from fts3rest.lib.middleware.timeout import TimeoutHandler -from fts3rest.model.meta import Session from fts3rest.lib.openidconnect import oidc_manager -from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher -from werkzeug.exceptions import HTTPException -import json +from fts3rest.model import init_model +from fts3rest.model.meta import Session def _load_configuration(config_file): @@ -93,6 +93,7 @@ def create_app(default_config_file=None, test=False): raise ValueError("The configuration file has not been specified") fts3cfg = _load_configuration(config_file) + log = logging.getLogger(__name__) # Add configuration app.config.update(fts3cfg) @@ -127,5 +128,7 @@ def create_app(default_config_file=None, test=False): if "fts3.Providers" in app.config: oidc_manager.setup(app.config) IAMTokenRefresher("fts_token_refresh_daemon", app.config).start() + else: + log.info("OpenID Connect support disabled. Providers not found in config") return app -- GitLab From 9d5c518f320a4229859b097869a818af4f31fd45 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 12:42:04 +0200 Subject: [PATCH 04/27] debug ci --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c63f3bb7..abbeb4f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,7 @@ pylint36: # Check that every file doesn't have syntax errors stage: static_code_analysis script: + - printenv PYTHONPATH - python --version - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy -- GitLab From bccdad4118499f0c8a0ab89244cb4e78b968efee Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 13:22:02 +0200 Subject: [PATCH 05/27] fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index abbeb4f4..f1f35ac1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,7 +32,7 @@ pylint36: script: - printenv PYTHONPATH - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src"); sys.path.append("src/fts3rest")' pylint37: # Check that every file doesn't have syntax errors -- GitLab From ae28292f1b311fe3d964d04054c4fb9554190768 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 13:51:36 +0200 Subject: [PATCH 06/27] fix --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1f35ac1..ed425b7a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,9 +30,9 @@ pylint36: # Check that every file doesn't have syntax errors stage: static_code_analysis script: - - printenv PYTHONPATH + - unset PYTHONPATH - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src"); sys.path.append("src/fts3rest")' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])' pylint37: # Check that every file doesn't have syntax errors -- GitLab From 263e72fc6d65b52b3a866ab098ebc6727bfda381 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 14:39:05 +0200 Subject: [PATCH 07/27] update --- .gitlab-ci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile index 05161ff0..ac3edaf9 100644 --- a/.gitlab-ci/Dockerfile +++ b/.gitlab-ci/Dockerfile @@ -20,7 +20,7 @@ ENV PY37 3.7.7 ENV PY38 3.8.2 # install pyenv, following instructions from https://github.com/pyenv/pyenv -RUN git clone --depth 1 --branch master https://github.com/pyenv/pyenv.git ~/.pyenv && \ +RUN git clone --depth 1 --branch v1.2.17 https://github.com/pyenv/pyenv.git ~/.pyenv && \ echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \ echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \ echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n eval "$(pyenv init -)"\nfi' >> ~/.bashrc && \ -- GitLab From a7511ceb834d09671cf83e68e90f7653581b3f29 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 15:04:09 +0200 Subject: [PATCH 08/27] fix --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed425b7a..2f1c77a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,8 +17,8 @@ stages: variables: MYSQL_DATABASE: ftsflask MYSQL_ROOT_PASSWORD: asdf - PYTHONPATH: ./src:./src/fts3rest - FTS3TESTCONFIG: /src/fts3rest/fts3rest/tests/fts3testconfig_ci + PYTHONPATH: /home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest + FTS3TESTCONFIG: ./src/fts3rest/fts3rest/tests/fts3testconfig_ci black: # Check that every file has been formatted with black @@ -30,9 +30,8 @@ pylint36: # Check that every file doesn't have syntax errors stage: static_code_analysis script: - - unset PYTHONPATH - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask/src","/home/ci/fts-rest-flask/src/fts3rest"])' pylint37: # Check that every file doesn't have syntax errors -- GitLab From cfda2d34b32979437966f8dade5b4e99248ac5fd Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 15:08:16 +0200 Subject: [PATCH 09/27] fix --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f1c77a1..15845808 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ stages: variables: MYSQL_DATABASE: ftsflask MYSQL_ROOT_PASSWORD: asdf - PYTHONPATH: /home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest + PYTHONPATH: /home/ci/fts-rest-flask:/home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest FTS3TESTCONFIG: ./src/fts3rest/fts3rest/tests/fts3testconfig_ci black: @@ -31,7 +31,7 @@ pylint36: stage: static_code_analysis script: - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask/src","/home/ci/fts-rest-flask/src/fts3rest"])' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask","/home/ci/fts-rest-flask/src/fts3rest","/home/ci/fts-rest-flask/src"])' pylint37: # Check that every file doesn't have syntax errors -- GitLab From 2fe875dc39c74aa1a6f43c30159e461f72b4b61d Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 15:45:06 +0200 Subject: [PATCH 10/27] Remove CI for multiple python versions as pyenv causes many problems Later we can change the image to contain multiple python versions installed from source instead of pyev --- .gitlab-ci.yml | 63 ++----------------------------------------- .gitlab-ci/Dockerfile | 36 +++---------------------- 2 files changed, 6 insertions(+), 93 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15845808..f282e423 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,11 +1,3 @@ -# The jobs run for every file in src/, even if they haven't been changed. -# Unfortunately Gitlab CI doesn't have any way to know the commit before the push. -# I used to do this to select only the files that had been changed: -# FILES=$(git diff --name-only --diff-filter=ACMR $CI_COMMIT_SHA~1 -- "*.py") -# if [ -z "$FILES" ]; then exit 0; fi -# The problem was that a push with multiple commits might pass the last commit even -# if the previous failed. - image: gitlab-registry.cern.ch/fts/fts-rest-flask:ci stages: @@ -17,7 +9,7 @@ stages: variables: MYSQL_DATABASE: ftsflask MYSQL_ROOT_PASSWORD: asdf - PYTHONPATH: /home/ci/fts-rest-flask:/home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest + PYTHONPATH: ./src:./src/fts3rest FTS3TESTCONFIG: ./src/fts3rest/fts3rest/tests/fts3testconfig_ci black: @@ -31,27 +23,7 @@ pylint36: stage: static_code_analysis script: - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask","/home/ci/fts-rest-flask/src/fts3rest","/home/ci/fts-rest-flask/src"])' - -pylint37: - # Check that every file doesn't have syntax errors - stage: static_code_analysis - script: - - source ~/.bashrc - - pyenv global $PY37 - - source $VENV_37/bin/activate - - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy - -pylint38: - # Check that every file doesn't have syntax errors - stage: static_code_analysis - script: - - source ~/.bashrc - - pyenv global $PY38 - - source $VENV_38/bin/activate - - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["./src", "./src/fts3rest"])' radon: @@ -60,8 +32,6 @@ radon: script: - source .gitlab-ci/radon.sh - - functional_tests36: stage: tests services: @@ -73,35 +43,6 @@ functional_tests36: - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report -functional_tests37: - stage: tests - services: - - name: centos/mariadb:latest - alias: mariadb - script: - - source ~/.bashrc - - pyenv global $PY37 - - python --version - - source $VENV_37/bin/activate - - source .gitlab-ci/db.sh - - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - - coverage report - -functional_tests38: - stage: tests - services: - - name: centos/mariadb:latest - alias: mariadb - script: - - source ~/.bashrc - - pyenv global $PY38 - - python --version - - source $VENV_38/bin/activate - - source .gitlab-ci/db.sh - - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - - coverage report - - bandit: # Find potential security issues in file. # It's allowed to fail as it may detect false positives. diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile index ac3edaf9..fa20b7f2 100644 --- a/.gitlab-ci/Dockerfile +++ b/.gitlab-ci/Dockerfile @@ -1,9 +1,6 @@ FROM centos:7 -RUN yum install python3 git python3-devel openssl-devel swig gcc gcc-c++ make mysql-devel mariadb -y && \ -yum clean all -# install pyenv dependencies -RUN yum install gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel tk-devel libffi-devel -y && \ +RUN yum install python3 git python3-devel openssl-devel swig gcc-c++ mysql-devel mariadb -y && \ yum clean all RUN useradd --create-home ci @@ -15,19 +12,6 @@ RUN chown ci /var/log/fts3rest WORKDIR /home/ci USER ci -ENV PY36 3.6.10 -ENV PY37 3.7.7 -ENV PY38 3.8.2 - -# install pyenv, following instructions from https://github.com/pyenv/pyenv -RUN git clone --depth 1 --branch v1.2.17 https://github.com/pyenv/pyenv.git ~/.pyenv && \ -echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \ -echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \ -echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n eval "$(pyenv init -)"\nfi' >> ~/.bashrc && \ -source ~/.bashrc && \ -pyenv install $PY36 && \ -pyenv install $PY37 && \ -pyenv install $PY38 # this is needed for pip-tools ENV LC_ALL en_US.utf-8 @@ -38,21 +22,9 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in # Prepare virtual environments ENV VENV_36 /home/ci/venv_36 -ENV VENV_37 /home/ci/venv_37 -ENV VENV_38 /home/ci/venv_38 - -RUN . ~/.bashrc && pyenv global $PY36 && python -m venv $VENV_36 && \ -. $VENV_36/bin/activate && pip install --upgrade pip && pip install pip-tools && \ -. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate - -RUN . ~/.bashrc && pyenv global $PY37 && python -m venv $VENV_37 && \ -. $VENV_37/bin/activate && pip install --upgrade pip && pip install pip-tools && \ -. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate -RUN . ~/.bashrc && pyenv global $PY38 && python -m venv $VENV_38 && \ -. $VENV_38/bin/activate && pip install --upgrade pip && pip install pip-tools && \ -. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate +RUN python -m venv $VENV_36 && $VENV_36/bin/activate && \ + pip install --upgrade pip && pip install pip-tools && \ + . ./pipcompile.sh && . ./pipsyncdev.sh && deactivate -# by default we run on the lowest version supported -RUN . ~/.bashrc && pyenv global $PY36 ENV PATH="$VENV_36/bin:$PATH" -- GitLab From 76a8d0d837b33346ad7b092ba8c71f520e85f18e Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 16:41:54 +0200 Subject: [PATCH 11/27] fix --- .gitlab-ci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile index fa20b7f2..336617b0 100644 --- a/.gitlab-ci/Dockerfile +++ b/.gitlab-ci/Dockerfile @@ -23,7 +23,7 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in # Prepare virtual environments ENV VENV_36 /home/ci/venv_36 -RUN python -m venv $VENV_36 && $VENV_36/bin/activate && \ +RUN python3.6 -m venv $VENV_36 && $VENV_36/bin/activate && \ pip install --upgrade pip && pip install pip-tools && \ . ./pipcompile.sh && . ./pipsyncdev.sh && deactivate -- GitLab From 7b1a1b201a6965025f7e2cfe0e608df8a1d79694 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 16:43:37 +0200 Subject: [PATCH 12/27] fix --- .gitlab-ci/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile index 336617b0..c834d3fe 100644 --- a/.gitlab-ci/Dockerfile +++ b/.gitlab-ci/Dockerfile @@ -23,8 +23,8 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in # Prepare virtual environments ENV VENV_36 /home/ci/venv_36 -RUN python3.6 -m venv $VENV_36 && $VENV_36/bin/activate && \ +RUN python3.6 -m venv $VENV_36 && source $VENV_36/bin/activate && \ pip install --upgrade pip && pip install pip-tools && \ - . ./pipcompile.sh && . ./pipsyncdev.sh && deactivate + source ./pipcompile.sh && source ./pipsyncdev.sh && deactivate ENV PATH="$VENV_36/bin:$PATH" -- GitLab From ad67ac6b41264e8dd8cddd355050d34a0705725e Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 17:03:36 +0200 Subject: [PATCH 13/27] fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f282e423..0ccdba00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ pylint36: stage: static_code_analysis script: - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["./src", "./src/fts3rest"])' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path = ["src", "src/fts3rest"]' radon: -- GitLab From af70748f0b2e109a049b76fc14c28badb0414143 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 17:18:20 +0200 Subject: [PATCH 14/27] fix --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ccdba00..2f0b1b03 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,8 +22,9 @@ pylint36: # Check that every file doesn't have syntax errors stage: static_code_analysis script: + - pwd - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path = ["src", "src/fts3rest"]' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])' radon: -- GitLab From 61a8663ee7914a35718a312f7a3fd5681a278bc3 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 17:24:49 +0200 Subject: [PATCH 15/27] fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f0b1b03..59ad964e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,7 @@ pylint36: # Check that every file doesn't have syntax errors stage: static_code_analysis script: - - pwd + - source /home/ci/venv_36/bin/activate - python --version - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])' -- GitLab From 3e2f57ad21bec108bc9733ea8c4565c247c813e6 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 17:30:55 +0200 Subject: [PATCH 16/27] fix --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59ad964e..73054d35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,7 @@ pylint36: stage: static_code_analysis script: - source /home/ci/venv_36/bin/activate + - echo $PYTHONPATH - python --version - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])' -- GitLab From 10b3def3a5f18d515cbb1e12ea0c6ee9c99f8121 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 17:34:25 +0200 Subject: [PATCH 17/27] fix --- .gitlab-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 73054d35..d0166f4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,16 @@ pylint36: - source /home/ci/venv_36/bin/activate - echo $PYTHONPATH - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")' + +pylint362: + # Check that every file doesn't have syntax errors + stage: static_code_analysis + script: + - source /home/ci/venv_36/bin/activate + - unset PYTHONPATH + - python --version + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")' radon: -- GitLab From d94bf6e2498ad4ccdb7fc5aa13fd677196ebb5af Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Wed, 6 May 2020 18:23:38 +0200 Subject: [PATCH 18/27] pin pylint to fix CI and restore multi versions --- .gitlab-ci.yml | 50 ++++++++++++++++++++++++++++++++++++------- .gitlab-ci/Dockerfile | 36 +++++++++++++++++++++++++++---- dev-requirements.in | 2 +- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0166f4d..dd33f6fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,20 +22,26 @@ pylint36: # Check that every file doesn't have syntax errors stage: static_code_analysis script: - - source /home/ci/venv_36/bin/activate - - echo $PYTHONPATH - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy -pylint362: - # Check that every file doesn't have syntax errors +pylint37: stage: static_code_analysis script: - - source /home/ci/venv_36/bin/activate - - unset PYTHONPATH + - source ~/.bashrc + - pyenv global $PY37 + - source $VENV_37/bin/activate - python --version - - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")' + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy +pylint38: + stage: static_code_analysis + script: + - source ~/.bashrc + - pyenv global $PY38 + - source $VENV_38/bin/activate + - python --version + - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy radon: # Check metrics for every file @@ -54,6 +60,34 @@ functional_tests36: - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x - coverage report +functional_tests37: + stage: tests + services: + - name: centos/mariadb:latest + alias: mariadb + script: + - source ~/.bashrc + - pyenv global $PY37 + - python --version + - source $VENV_37/bin/activate + - source .gitlab-ci/db.sh + - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x + - coverage report + +functional_tests38: + stage: tests + services: + - name: centos/mariadb:latest + alias: mariadb + script: + - source ~/.bashrc + - pyenv global $PY38 + - python --version + - source $VENV_38/bin/activate + - source .gitlab-ci/db.sh + - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x + - coverage report + bandit: # Find potential security issues in file. # It's allowed to fail as it may detect false positives. diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile index c834d3fe..05161ff0 100644 --- a/.gitlab-ci/Dockerfile +++ b/.gitlab-ci/Dockerfile @@ -1,6 +1,9 @@ FROM centos:7 -RUN yum install python3 git python3-devel openssl-devel swig gcc-c++ mysql-devel mariadb -y && \ +RUN yum install python3 git python3-devel openssl-devel swig gcc gcc-c++ make mysql-devel mariadb -y && \ +yum clean all +# install pyenv dependencies +RUN yum install gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel tk-devel libffi-devel -y && \ yum clean all RUN useradd --create-home ci @@ -12,6 +15,19 @@ RUN chown ci /var/log/fts3rest WORKDIR /home/ci USER ci +ENV PY36 3.6.10 +ENV PY37 3.7.7 +ENV PY38 3.8.2 + +# install pyenv, following instructions from https://github.com/pyenv/pyenv +RUN git clone --depth 1 --branch master https://github.com/pyenv/pyenv.git ~/.pyenv && \ +echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \ +echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \ +echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n eval "$(pyenv init -)"\nfi' >> ~/.bashrc && \ +source ~/.bashrc && \ +pyenv install $PY36 && \ +pyenv install $PY37 && \ +pyenv install $PY38 # this is needed for pip-tools ENV LC_ALL en_US.utf-8 @@ -22,9 +38,21 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in # Prepare virtual environments ENV VENV_36 /home/ci/venv_36 +ENV VENV_37 /home/ci/venv_37 +ENV VENV_38 /home/ci/venv_38 + +RUN . ~/.bashrc && pyenv global $PY36 && python -m venv $VENV_36 && \ +. $VENV_36/bin/activate && pip install --upgrade pip && pip install pip-tools && \ +. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate + +RUN . ~/.bashrc && pyenv global $PY37 && python -m venv $VENV_37 && \ +. $VENV_37/bin/activate && pip install --upgrade pip && pip install pip-tools && \ +. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate -RUN python3.6 -m venv $VENV_36 && source $VENV_36/bin/activate && \ - pip install --upgrade pip && pip install pip-tools && \ - source ./pipcompile.sh && source ./pipsyncdev.sh && deactivate +RUN . ~/.bashrc && pyenv global $PY38 && python -m venv $VENV_38 && \ +. $VENV_38/bin/activate && pip install --upgrade pip && pip install pip-tools && \ +. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate +# by default we run on the lowest version supported +RUN . ~/.bashrc && pyenv global $PY36 ENV PATH="$VENV_36/bin:$PATH" diff --git a/dev-requirements.in b/dev-requirements.in index a0d19a4e..f0d6e4ee 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -1,7 +1,7 @@ -c requirements.txt pip-tools black -pylint +pylint==2.4.4 # 2.5 has a bug and fails as of 02/05/20 bandit radon pytest -- GitLab From d0845a13863ffe3b2ee8a6748fc62f56ccedaa68 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 15:06:38 +0200 Subject: [PATCH 19/27] Move config.py to fts3rest, update CI testconfig --- .../fts3rest/config}/config.py | 0 src/fts3rest/fts3rest/config/middleware.py | 2 +- src/fts3rest/fts3rest/tests/fts3testconfig_ci | 163 ++---------------- 3 files changed, 13 insertions(+), 152 deletions(-) rename src/{fts3/util => fts3rest/fts3rest/config}/config.py (100%) diff --git a/src/fts3/util/config.py b/src/fts3rest/fts3rest/config/config.py similarity index 100% rename from src/fts3/util/config.py rename to src/fts3rest/fts3rest/config/config.py diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index db5fcbc5..f9f9eb7f 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -8,7 +8,7 @@ from flask import Flask from sqlalchemy import engine_from_config, event from werkzeug.exceptions import HTTPException -from fts3.util.config import fts3_config_load +from fts3rest.config.config import fts3_config_load from fts3rest.config.routing import base, cstorage from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher from fts3rest.lib.helpers.connection_validator import ( diff --git a/src/fts3rest/fts3rest/tests/fts3testconfig_ci b/src/fts3rest/fts3rest/tests/fts3testconfig_ci index f60a400a..ce73e7e4 100644 --- a/src/fts3rest/fts3rest/tests/fts3testconfig_ci +++ b/src/fts3rest/fts3rest/tests/fts3testconfig_ci @@ -1,7 +1,3 @@ -# Running user and group -User=ci -Group=ci - # mysql only DbType=mysql @@ -11,150 +7,27 @@ DbUserName=ci #db password DbPassword=asdf - #For MySQL, it has to follow the format 'host/db' (i.e. "mysql-server.example.com/fts3db") DbConnectString=mariadb:3306/ftsflask -#Number of db connections in the pool (use even number, e.g. 2,4,6,8,etc OR 1 for a single connection) -DbThreadsNum=30 - -#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/ -#Alias=fts3-xdc.cern.ch - -#Infosys, either the fqdn:port of a BDII instance or false to disable BDII access -#Infosys=lcg-bdii.cern.ch:2170 - -#Query the info systems specified in the order given, e.g. glue1;glue2 -InfoProviders=glue1 - #List of authorized VOs, separated by ; #Leave * to authorize any VO AuthorizedVO=* -# site name -#SiteName=FTS-DEV-XDC - -#Enable/Disable monitoring using messaging monitoring (disabled=false / enabled=true) -MonitoringMessaging=false - -# Profiling interval in seconds. If set to 0, it will be disabled -Profiling=0 - -# Log directories -TransferLogDirectory=/var/log/fts3/transfers -ServerLogDirectory=/var/log/fts3 - -# Log level. Enables logging for messages of level >= than configured -# Possible values are -# TRACE (every detail), DEBUG (internal behaviour), INFO (normal behaviour), -# NOTICE (final states), WARNING (things worth checking), ERR (internal FTS3 errors, as database connectivity), -# CRIT (fatal errors, as segmentation fault) -# It is recommended to use DEBUG or INFO -LogLevel=DEBUG - -# Check for fts_url_copy processes that do not give their progress back -# CheckStalledTransfers = true -# Stalled timeout, in seconds -# CheckStalledTimeout = 900 -CheckStalledTimeout = 900 - -# Minimum required free RAM (in MB) for FTS3 to work normally -# If the amount of free RAM goes below the limit, FTS3 will enter auto-drain mode -# This is intended to protect against system resource exhaustion -# MinRequiredFreeRAM = 50 -MinRequiredFreeRAM = 50 - -# Maximum number of url copy processes that the node can run -# The RAM limitation may not take into account other node limitations (i.e. IO) -# or, depending on the swapping policy, may not even prevent overloads if the kernel -# starts swapping before the free RAM decreases until it reaches the value of MinRequiredFreeRAM -# 0 disables the check. -# The default is 400. -# MaxUrlCopyProcesses = 400 -MaxUrlCopyProcesses = 400 - -# Parameters for Bring Online -# Maximum bulk size. -# If the size is too large, it will take more resources (memory and CPU) to generate the requests and -# parse the responses. Some servers may reject the requests if they are too big. -# If it is too small, performance will be reduced. -# Keep it to a sensible size (between 100 and 1k) -# StagingBulkSize=400 -# Maximum number of concurrent requests. This gives a maximum of files sent to the storage system -# (StagingBulkSize*StagingConcurrentRequests). The larger the number, the more requests will FTS need to keep track of. -# StagingConcurrentRequests=500 -# Seconds to wait before submitting a bulk request, so FTS can accumulate more files per bulk. -# Note that the resolution is 60 seconds. -# StagingWaitingFactor=300 -# Retry this number of times if a staging poll fails with ECOMM -# StagingPollRetries=3 - -# In seconds, interval between heartbeats -# HeartBeatInterval=60 -# I seconds, after this interval a host is considered down -# HeartBeatGraceInterval=120 - -# Seconds between optimizer runs -# OptimizerInterval = 60 -# After this time without optimizer updates, force a run -# OptimizerSteadyInterval = 300 -# Maximum number of streams per file -# OptimizerMaxStreams = 16 - -# EMA Alpha factor to reduce the influence of fluctuations -# OptimizerEMAAlpha = 0.1 -# Increase step size when the optimizer considers the performance is good -# OptimizerIncreaseStep = 1 -# Increase step size when the optimizer considers the performance is good, and set to aggressive or normal -# OptimizerAggressiveIncreaseStep = 2 -# Decrease step size when the optimizer considers the performance is bad -# OptimizerDecreaseStep = 1 - - -# Set the bulk size, in number of jobs, used for cleaning the old records -#CleanBulkSize=5000 -# In days. Entries older than this will be purged. -#CleanInterval=7 - -## The higher the values for the following parameters, -## the higher the latency for some operations (as cancelations), -## but can also reduce the system and/or database load - -# In seconds, how often to purge the messaging directory -#PurgeMessagingDirectoryInterval = 600 -# In seconds, how often to run sanity checks -#CheckSanityStateInterval = 3600 -# In seconds, how often to check for canceled transfers -#CancelCheckInterval = 10 -# In seconds, how often to check for expired queued transfers -#QueueTimeoutCheckInterval = 300 -# In seconds, how often to check for stalled transfers -#ActiveTimeoutCheckInterval = 300 -# In seconds, how often to schedule new transfers -#SchedulingInterval = 2 -# In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2 -#MessagingConsumeInterval = 1 -#Enable or disable auto session reuse -AutoSessionReuse = true -#Max small file size for session reuse in bytes -AutoSessionReuseMaxSmallFileSize = 104857600 -#Max big file size for session reuse in bytes -AutoSessionReuseMaxBigFileSize = 1073741824 -#Max number of files per session reuse -AutoSessionReuseMaxFiles = 1000 -#Max number of big files per session reuse -AutoSessionReuseMaxBigFiles = 2 -BackupTables=false -OptimizerMaxSuccessRate=100 -OptimizerMedSuccessRate=80 -OptimizerLowSuccessRate=75 -OptimizerBaseSuccessRate=74 -Port=8443 -UseFixedJobPriority=0 - +#OpenID parameters ValidateAccessTokenOffline=True JWKCacheSeconds=86400 TokenRefreshDaemonIntervalInSeconds=600 + +[roles] +Public = vo:transfer;all:datamanagement +lcgadmin = all:config + +[providers] +xdc=https://iam.extreme-datacloud.eu +xdc_ClientId= +xdc_ClientSecret= + # Logging configuration [loggers] keys = root, routes, fts3rest, sqlalchemy @@ -176,7 +49,7 @@ qualname = routes.middleware # "level = DEBUG" logs the route matched and routing variables. [logger_fts3rest] -level = INFO +level = DEBUG handlers = qualname = fts3rest @@ -195,8 +68,6 @@ level = NOTSET formatter = generic [handler_log_file] -# See -# http://docs.python.org/2/library/logging.handlers.html class = logging.FileHandler args = ('/var/log/fts3rest/fts3rest.log', 'a') level = NOTSET @@ -205,13 +76,3 @@ formatter = generic [formatter_generic] format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s datefmt = %H:%M:%S - -[roles] -Public = vo:transfer;all:datamanagement -lcgadmin = all:config - - -[providers] -xdc=https://iam.extreme-datacloud.eu -xdc_ClientId= -xdc_ClientSecret= -- GitLab From bae1307c4959a5d0a2bae593910f838f47ddd657 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 15:58:55 +0200 Subject: [PATCH 20/27] Move models to fts3resMove models to fts3rest --- src/fts3/model/__init__.py | 35 ------------------- src/fts3rest/fts3rest/config/middleware.py | 3 +- .../fts3rest/controllers/CSdropbox.py | 4 +-- src/fts3rest/fts3rest/controllers/api.py | 3 +- src/fts3rest/fts3rest/controllers/archive.py | 2 +- .../fts3rest/controllers/autocomplete.py | 2 +- src/fts3rest/fts3rest/controllers/banning.py | 12 +++++-- .../fts3rest/controllers/config/__init__.py | 2 +- .../fts3rest/controllers/config/activities.py | 2 +- .../fts3rest/controllers/config/audit.py | 2 +- .../fts3rest/controllers/config/authz.py | 2 +- .../fts3rest/controllers/config/cloud.py | 2 +- .../fts3rest/controllers/config/drain.py | 2 +- .../fts3rest/controllers/config/global_.py | 2 +- .../fts3rest/controllers/config/links.py | 2 +- .../fts3rest/controllers/config/se.py | 2 +- .../fts3rest/controllers/config/shares.py | 2 +- .../fts3rest/controllers/datamanagement.py | 2 +- .../fts3rest/controllers/delegation.py | 2 +- src/fts3rest/fts3rest/controllers/files.py | 4 +-- src/fts3rest/fts3rest/controllers/jobs.py | 6 ++-- .../fts3rest/controllers/optimizer.py | 6 +--- .../fts3rest/controllers/serverstatus.py | 2 -- .../fts3rest/lib/IAMTokenRefresher.py | 2 +- src/fts3rest/fts3rest/lib/JobBuilder_utils.py | 2 +- src/fts3rest/fts3rest/lib/api/introspect.py | 6 ++-- src/fts3rest/fts3rest/lib/helpers/jsonify.py | 3 +- .../lib/middleware/fts3auth/credentials.py | 2 +- .../middleware/fts3auth/fts3authmiddleware.py | 2 +- src/fts3rest/fts3rest/lib/oauth2provider.py | 4 +-- src/fts3rest/fts3rest/lib/scheduler/db.py | 6 ++-- src/fts3rest/fts3rest/model/__init__.py | 20 ++++++----- .../fts3rest}/model/banned.py | 0 src/{fts3 => fts3rest/fts3rest}/model/base.py | 0 .../fts3rest}/model/cloudStorage.py | 0 .../fts3rest}/model/config.py | 0 .../fts3rest}/model/credentials.py | 0 src/{fts3 => fts3rest/fts3rest}/model/dm.py | 0 src/{fts3 => fts3rest/fts3rest}/model/file.py | 0 src/{fts3 => fts3rest/fts3rest}/model/job.py | 0 src/fts3rest/fts3rest/model/meta.py | 7 ++-- .../fts3rest}/model/oauth2.py | 0 .../fts3rest}/model/optimizer.py | 0 .../fts3rest}/model/server.py | 0 .../fts3rest}/model/version.py | 0 src/fts3rest/fts3rest/tests/__init__.py | 2 +- .../fts3rest/tests/functional/test_banning.py | 2 +- .../functional/test_config_activity_shares.py | 2 +- .../tests/functional/test_config_authz_dn.py | 2 +- .../tests/functional/test_config_cloud.py | 2 +- .../tests/functional/test_config_global.py | 2 +- .../tests/functional/test_config_links.py | 2 +- .../tests/functional/test_config_se.py | 2 +- .../tests/functional/test_config_shares.py | 2 +- .../tests/functional/test_delegation.py | 2 +- .../fts3rest/tests/functional/test_drain.py | 2 +- .../fts3rest/tests/functional/test_dropbox.py | 2 +- .../tests/functional/test_job_cancel.py | 2 +- .../tests/functional/test_job_deletion.py | 2 +- .../tests/functional/test_job_listing.py | 2 +- .../tests/functional/test_job_modify.py | 2 +- .../tests/functional/test_job_submission.py | 3 +- .../tests/functional/test_multiple.py | 2 +- .../tests/functional/test_openidconnect.py | 2 +- .../tests/functional/test_optimizer.py | 2 +- .../tests/functional/test_scheduler.py | 2 +- .../fts3rest/tests/functional/test_staging.py | 2 +- 67 files changed, 84 insertions(+), 118 deletions(-) delete mode 100644 src/fts3/model/__init__.py rename src/{fts3 => fts3rest/fts3rest}/model/banned.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/base.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/cloudStorage.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/config.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/credentials.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/dm.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/file.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/job.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/oauth2.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/optimizer.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/server.py (100%) rename src/{fts3 => fts3rest/fts3rest}/model/version.py (100%) diff --git a/src/fts3/model/__init__.py b/src/fts3/model/__init__.py deleted file mode 100644 index 76dcc79e..00000000 --- a/src/fts3/model/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Members of the EMI Collaboration, 2013. -# Copyright 2020 CERN -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sqlalchemy -from .base import Base -from .banned import * -from .cloudStorage import * -from .config import * -from .credentials import * -from .dm import * -from .file import * -from .job import * -from .oauth2 import * -from .optimizer import * -from .server import * -from .version import * - - -# Convenience method -def connect(connectString): - engine = sqlalchemy.create_engine(connectString, isolation_level="READ-COMMITTED") - Session = sqlalchemy.orm.sessionmaker(bind=engine) - return Session() diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index f9f9eb7f..4dc60197 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -18,8 +18,7 @@ from fts3rest.lib.helpers.connection_validator import ( from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddleware from fts3rest.lib.middleware.timeout import TimeoutHandler from fts3rest.lib.openidconnect import oidc_manager -from fts3rest.model import init_model -from fts3rest.model.meta import Session +from fts3rest.model.meta import init_model, Session def _load_configuration(config_file): diff --git a/src/fts3rest/fts3rest/controllers/CSdropbox.py b/src/fts3rest/fts3rest/controllers/CSdropbox.py index e08d1975..989e0f88 100644 --- a/src/fts3rest/fts3rest/controllers/CSdropbox.py +++ b/src/fts3rest/fts3rest/controllers/CSdropbox.py @@ -18,11 +18,11 @@ from werkzeug.exceptions import NotFound, BadRequest, Forbidden from flask import request import urllib from urllib.parse import urlparse, urlencode -from urllib.request import urlopen, Request +from urllib.request import urlopen from urllib.error import HTTPError from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.model.meta import Session -from fts3.model import CloudStorage, CloudStorageUser +from fts3rest.model import CloudStorage, CloudStorageUser from fts3rest.controllers.CSInterface import Connector dropboxEndpoint = "https://www.dropbox.com" diff --git a/src/fts3rest/fts3rest/controllers/api.py b/src/fts3rest/fts3rest/controllers/api.py index ff31b8af..d81db395 100644 --- a/src/fts3rest/fts3rest/controllers/api.py +++ b/src/fts3rest/fts3rest/controllers/api.py @@ -16,14 +16,13 @@ import glob from flask.views import View -from fts3.model import SchemaVersion +from fts3rest.model import SchemaVersion from fts3rest.model.meta import Session from fts3rest.lib.api.submit_schema import SubmitSchema -from fts3rest.lib.api.introspect import introspect from werkzeug.exceptions import NotFound from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/archive.py b/src/fts3rest/fts3rest/controllers/archive.py index f6958c7d..68c179db 100644 --- a/src/fts3rest/fts3rest/controllers/archive.py +++ b/src/fts3rest/fts3rest/controllers/archive.py @@ -16,7 +16,7 @@ from werkzeug.exceptions import NotFound, Forbidden from fts3rest.lib.helpers.jsonify import jsonify -from fts3.model import ArchivedJob +from fts3rest.model import ArchivedJob from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.authorization import authorized from fts3rest.lib.middleware.fts3auth.constants import * diff --git a/src/fts3rest/fts3rest/controllers/autocomplete.py b/src/fts3rest/fts3rest/controllers/autocomplete.py index 51c9c901..7763221b 100644 --- a/src/fts3rest/fts3rest/controllers/autocomplete.py +++ b/src/fts3rest/fts3rest/controllers/autocomplete.py @@ -13,7 +13,7 @@ # limitations under the License. -from fts3.model import Credential, LinkConfig, Job +from fts3rest.model import Credential, LinkConfig, Job from fts3rest.model.meta import Session from flask import request diff --git a/src/fts3rest/fts3rest/controllers/banning.py b/src/fts3rest/fts3rest/controllers/banning.py index b0cab26c..46fda523 100644 --- a/src/fts3rest/fts3rest/controllers/banning.py +++ b/src/fts3rest/fts3rest/controllers/banning.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from werkzeug.exceptions import NotFound, BadRequest, Conflict +from werkzeug.exceptions import BadRequest, Conflict from fts3rest.controllers.config import audit_configuration from flask import request, Response import json @@ -19,9 +19,15 @@ import logging from datetime import datetime from sqlalchemy import distinct, func, and_ -from fts3.model import BannedDN, BannedSE, Job, File, JobActiveStates, FileActiveStates +from fts3rest.model import ( + BannedDN, + BannedSE, + Job, + File, + JobActiveStates, + FileActiveStates, +) from fts3rest.model.meta import Session -from fts3rest.lib.http_exceptions import * from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import * from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/__init__.py b/src/fts3rest/fts3rest/controllers/config/__init__.py index 4e984f2c..7e5ed616 100644 --- a/src/fts3rest/fts3rest/controllers/config/__init__.py +++ b/src/fts3rest/fts3rest/controllers/config/__init__.py @@ -19,7 +19,7 @@ from numbers import Number from flask import request from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.lib.helpers.accept import accept from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import CONFIG diff --git a/src/fts3rest/fts3rest/controllers/config/activities.py b/src/fts3rest/fts3rest/controllers/config/activities.py index 9e5093ec..3b293d32 100644 --- a/src/fts3rest/fts3rest/controllers/config/activities.py +++ b/src/fts3rest/fts3rest/controllers/config/activities.py @@ -18,7 +18,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest, NotFound -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/audit.py b/src/fts3rest/fts3rest/controllers/config/audit.py index fd70bc78..53b4a85a 100644 --- a/src/fts3rest/fts3rest/controllers/config/audit.py +++ b/src/fts3rest/fts3rest/controllers/config/audit.py @@ -15,7 +15,7 @@ import logging -from fts3.model import * +from fts3rest.model import * from fts3rest.lib.helpers.accept import accept from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import * diff --git a/src/fts3rest/fts3rest/controllers/config/authz.py b/src/fts3rest/fts3rest/controllers/config/authz.py index eb5c28f5..e7756fbf 100644 --- a/src/fts3rest/fts3rest/controllers/config/authz.py +++ b/src/fts3rest/fts3rest/controllers/config/authz.py @@ -17,7 +17,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/cloud.py b/src/fts3rest/fts3rest/controllers/config/cloud.py index a63d79c8..87a8d30b 100644 --- a/src/fts3rest/fts3rest/controllers/config/cloud.py +++ b/src/fts3rest/fts3rest/controllers/config/cloud.py @@ -17,7 +17,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest, NotFound -from fts3.model import * +from fts3rest.model import * from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.lib.helpers.misc import get_input_as_dict diff --git a/src/fts3rest/fts3rest/controllers/config/drain.py b/src/fts3rest/fts3rest/controllers/config/drain.py index 1bb882b4..3cea5fc5 100644 --- a/src/fts3rest/fts3rest/controllers/config/drain.py +++ b/src/fts3rest/fts3rest/controllers/config/drain.py @@ -17,7 +17,7 @@ import logging from flask import request from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.lib.helpers.misc import get_input_as_dict diff --git a/src/fts3rest/fts3rest/controllers/config/global_.py b/src/fts3rest/fts3rest/controllers/config/global_.py index 62b9daa1..92e26421 100644 --- a/src/fts3rest/fts3rest/controllers/config/global_.py +++ b/src/fts3rest/fts3rest/controllers/config/global_.py @@ -17,7 +17,7 @@ import logging from flask import request, Response from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration, validate_type from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify, to_json diff --git a/src/fts3rest/fts3rest/controllers/config/links.py b/src/fts3rest/fts3rest/controllers/config/links.py index d506d9b8..804141df 100644 --- a/src/fts3rest/fts3rest/controllers/config/links.py +++ b/src/fts3rest/fts3rest/controllers/config/links.py @@ -19,7 +19,7 @@ from urllib.parse import unquote from flask import request, Response from werkzeug.exceptions import BadRequest, NotFound -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/se.py b/src/fts3rest/fts3rest/controllers/config/se.py index b55b2779..91e071c9 100644 --- a/src/fts3rest/fts3rest/controllers/config/se.py +++ b/src/fts3rest/fts3rest/controllers/config/se.py @@ -19,7 +19,7 @@ from flask import Response from flask import request from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.accept import accept from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/config/shares.py b/src/fts3rest/fts3rest/controllers/config/shares.py index ae1e9fc8..4736bc42 100644 --- a/src/fts3rest/fts3rest/controllers/config/shares.py +++ b/src/fts3rest/fts3rest/controllers/config/shares.py @@ -18,7 +18,7 @@ from urllib.parse import urlparse from flask import request, Response from werkzeug.exceptions import BadRequest -from fts3.model import * +from fts3rest.model import * from fts3rest.controllers.config import audit_configuration from fts3rest.lib.helpers.jsonify import jsonify from fts3rest.lib.helpers.misc import get_input_as_dict diff --git a/src/fts3rest/fts3rest/controllers/datamanagement.py b/src/fts3rest/fts3rest/controllers/datamanagement.py index 96e270a5..cf96f6b8 100644 --- a/src/fts3rest/fts3rest/controllers/datamanagement.py +++ b/src/fts3rest/fts3rest/controllers/datamanagement.py @@ -33,7 +33,7 @@ from urllib.parse import urlparse, unquote_plus import json -from fts3.model import Credential +from fts3rest.model import Credential from fts3rest.model.meta import Session from fts3rest.lib.http_exceptions import HTTPAuthenticationTimeout from fts3rest.lib.gfal2_wrapper import Gfal2Wrapper, Gfal2Error diff --git a/src/fts3rest/fts3rest/controllers/delegation.py b/src/fts3rest/fts3rest/controllers/delegation.py index 3ff251df..0433ab79 100644 --- a/src/fts3rest/fts3rest/controllers/delegation.py +++ b/src/fts3rest/fts3rest/controllers/delegation.py @@ -25,7 +25,7 @@ import flask from flask import Response from flask import current_app as app from flask.views import View -from fts3.model import CredentialCache, Credential +from fts3rest.model import CredentialCache, Credential from fts3rest.model.meta import Session from fts3rest.lib.helpers.voms import VomsClient, VomsException from fts3rest.lib.middleware.fts3auth.authorization import require_certificate diff --git a/src/fts3rest/fts3rest/controllers/files.py b/src/fts3rest/fts3rest/controllers/files.py index f891cecc..93941bad 100644 --- a/src/fts3rest/fts3rest/controllers/files.py +++ b/src/fts3rest/fts3rest/controllers/files.py @@ -16,15 +16,13 @@ from werkzeug.exceptions import Forbidden from datetime import datetime, timedelta from flask import request from urllib.parse import urlparse -import json import logging -from fts3.model import File +from fts3rest.model import File from fts3rest.model.meta import Session from fts3rest.lib.JobBuilder_utils import get_storage_element from fts3rest.lib.middleware.fts3auth.authorization import authorize from fts3rest.lib.middleware.fts3auth.constants import * -from fts3rest.lib.http_exceptions import * from fts3rest.lib.helpers.jsonify import jsonify log = logging.getLogger(__name__) diff --git a/src/fts3rest/fts3rest/controllers/jobs.py b/src/fts3rest/fts3rest/controllers/jobs.py index 09e480fa..5edf16d9 100644 --- a/src/fts3rest/fts3rest/controllers/jobs.py +++ b/src/fts3rest/fts3rest/controllers/jobs.py @@ -22,9 +22,9 @@ from sqlalchemy.orm import noload import logging -from fts3.model import Job, File, JobActiveStates, FileActiveStates -from fts3.model import DataManagement, DataManagementActiveStates -from fts3.model import Credential, FileRetryLog +from fts3rest.model import Job, File, JobActiveStates, FileActiveStates +from fts3rest.model import DataManagement, DataManagementActiveStates +from fts3rest.model import Credential, FileRetryLog from fts3rest.model.meta import Session from fts3rest.lib.http_exceptions import * diff --git a/src/fts3rest/fts3rest/controllers/optimizer.py b/src/fts3rest/fts3rest/controllers/optimizer.py index 4bd97391..65a528fd 100644 --- a/src/fts3rest/fts3rest/controllers/optimizer.py +++ b/src/fts3rest/fts3rest/controllers/optimizer.py @@ -18,15 +18,11 @@ Optimizer logging tables from werkzeug.exceptions import BadRequest - -import json -import logging from flask import request, current_app as app from fts3rest.model.meta import Session from fts3rest.lib.helpers.misc import get_input_as_dict -from fts3.model import OptimizerEvolution, Optimizer +from fts3rest.model import OptimizerEvolution, Optimizer from datetime import datetime -from fts3rest.lib.http_exceptions import * from fts3rest.lib.helpers.jsonify import jsonify diff --git a/src/fts3rest/fts3rest/controllers/serverstatus.py b/src/fts3rest/fts3rest/controllers/serverstatus.py index 5c087d74..91466bd1 100644 --- a/src/fts3rest/fts3rest/controllers/serverstatus.py +++ b/src/fts3rest/fts3rest/controllers/serverstatus.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from werkzeug.exceptions import NotFound -from fts3.model import File from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.authorization import ( authorize, diff --git a/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py index 49220094..8c19a6a7 100644 --- a/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py +++ b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py @@ -22,7 +22,7 @@ from threading import Thread, current_thread from fts3rest.model.meta import Session from fts3rest.lib.openidconnect import oidc_manager -from fts3.model import Credential, Host +from fts3rest.model import Credential, Host from sqlalchemy.exc import SQLAlchemyError diff --git a/src/fts3rest/fts3rest/lib/JobBuilder_utils.py b/src/fts3rest/fts3rest/lib/JobBuilder_utils.py index 6580561f..b900008b 100644 --- a/src/fts3rest/fts3rest/lib/JobBuilder_utils.py +++ b/src/fts3rest/fts3rest/lib/JobBuilder_utils.py @@ -9,7 +9,7 @@ import uuid from flask import current_app as app from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError -from fts3.model import BannedSE +from fts3rest.model import BannedSE from fts3rest.model.meta import Session from fts3rest.lib.scheduler.schd import Scheduler diff --git a/src/fts3rest/fts3rest/lib/api/introspect.py b/src/fts3rest/fts3rest/lib/api/introspect.py index b83b9df9..fdb3ed5c 100755 --- a/src/fts3rest/fts3rest/lib/api/introspect.py +++ b/src/fts3rest/fts3rest/lib/api/introspect.py @@ -24,8 +24,8 @@ import logging import imp import itertools import os -import fts3.model -from fts3.model.base import Flag, TernaryFlag, Json +import fts3rest.model +from fts3rest.model.base import Flag, TernaryFlag, Json from fts3rest.config import routing from sqlalchemy import types from sqlalchemy.orm import Mapper @@ -86,7 +86,7 @@ def get_model_fields(model_name, model_list): Get a description of the fields of the model 'model_name' Injects into model_list the additional types that may be needed """ - model = getattr(fts3.model, model_name, None) + model = getattr(fts3rest.model, model_name, None) if not model: return dict() fields = dict() diff --git a/src/fts3rest/fts3rest/lib/helpers/jsonify.py b/src/fts3rest/fts3rest/lib/helpers/jsonify.py index 6cd9c596..711ca039 100644 --- a/src/fts3rest/fts3rest/lib/helpers/jsonify.py +++ b/src/fts3rest/fts3rest/lib/helpers/jsonify.py @@ -14,8 +14,7 @@ # limitations under the License. from datetime import datetime -from fts3.model.base import Base -from sqlalchemy.orm.query import Query +from fts3rest.model.base import Base import json import logging import types diff --git a/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py b/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py index a40e8b3c..0585d58f 100644 --- a/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py +++ b/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py @@ -18,7 +18,7 @@ import hashlib import logging import re -from fts3.model import AuthorizationByDn +from fts3rest.model import AuthorizationByDn from fts3rest.model.meta import Session from .methods import Authenticator diff --git a/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py b/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py index c4f42bc6..a4f04815 100644 --- a/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py +++ b/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py @@ -16,7 +16,7 @@ import logging from fts3rest.model.meta import Session -from fts3.model import BannedDN +from fts3rest.model import BannedDN from .credentials import UserCredentials, InvalidCredentials from sqlalchemy.exc import DatabaseError from urllib.parse import urlparse diff --git a/src/fts3rest/fts3rest/lib/oauth2provider.py b/src/fts3rest/fts3rest/lib/oauth2provider.py index f22b0784..cd84249f 100644 --- a/src/fts3rest/fts3rest/lib/oauth2provider.py +++ b/src/fts3rest/fts3rest/lib/oauth2provider.py @@ -28,8 +28,8 @@ from fts3rest.lib.openidconnect import oidc_manager from jwcrypto.jwk import JWK from flask import request -from fts3.model.credentials import Credential, CredentialCache -from fts3.model.oauth2 import OAuth2Application, OAuth2Code, OAuth2Token +from fts3rest.model import Credential, CredentialCache +from fts3rest.model.oauth2 import OAuth2Application, OAuth2Code, OAuth2Token log = logging.getLogger(__name__) diff --git a/src/fts3rest/fts3rest/lib/scheduler/db.py b/src/fts3rest/fts3rest/lib/scheduler/db.py index 357cfeea..2bb385a7 100644 --- a/src/fts3rest/fts3rest/lib/scheduler/db.py +++ b/src/fts3rest/fts3rest/lib/scheduler/db.py @@ -1,9 +1,9 @@ import json import logging -from fts3.model import File -from fts3.model import OptimizerEvolution -from fts3.model import ActivityShare +from fts3rest.model import File +from fts3rest.model import OptimizerEvolution +from fts3rest.model import ActivityShare from sqlalchemy import func diff --git a/src/fts3rest/fts3rest/model/__init__.py b/src/fts3rest/fts3rest/model/__init__.py index 5c0af258..af384e43 100644 --- a/src/fts3rest/fts3rest/model/__init__.py +++ b/src/fts3rest/fts3rest/model/__init__.py @@ -12,11 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""The application's model objects""" -from fts3rest.model.meta import Session, Base - - -def init_model(engine): - """Call me before using any of the tables or classes in the model""" - Session.configure(bind=engine) +from .base import Base +from .banned import * +from .cloudStorage import * +from .config import * +from .credentials import * +from .dm import * +from .file import * +from .job import * +from .oauth2 import * +from .optimizer import * +from .server import * +from .version import * diff --git a/src/fts3/model/banned.py b/src/fts3rest/fts3rest/model/banned.py similarity index 100% rename from src/fts3/model/banned.py rename to src/fts3rest/fts3rest/model/banned.py diff --git a/src/fts3/model/base.py b/src/fts3rest/fts3rest/model/base.py similarity index 100% rename from src/fts3/model/base.py rename to src/fts3rest/fts3rest/model/base.py diff --git a/src/fts3/model/cloudStorage.py b/src/fts3rest/fts3rest/model/cloudStorage.py similarity index 100% rename from src/fts3/model/cloudStorage.py rename to src/fts3rest/fts3rest/model/cloudStorage.py diff --git a/src/fts3/model/config.py b/src/fts3rest/fts3rest/model/config.py similarity index 100% rename from src/fts3/model/config.py rename to src/fts3rest/fts3rest/model/config.py diff --git a/src/fts3/model/credentials.py b/src/fts3rest/fts3rest/model/credentials.py similarity index 100% rename from src/fts3/model/credentials.py rename to src/fts3rest/fts3rest/model/credentials.py diff --git a/src/fts3/model/dm.py b/src/fts3rest/fts3rest/model/dm.py similarity index 100% rename from src/fts3/model/dm.py rename to src/fts3rest/fts3rest/model/dm.py diff --git a/src/fts3/model/file.py b/src/fts3rest/fts3rest/model/file.py similarity index 100% rename from src/fts3/model/file.py rename to src/fts3rest/fts3rest/model/file.py diff --git a/src/fts3/model/job.py b/src/fts3rest/fts3rest/model/job.py similarity index 100% rename from src/fts3/model/job.py rename to src/fts3rest/fts3rest/model/job.py diff --git a/src/fts3rest/fts3rest/model/meta.py b/src/fts3rest/fts3rest/model/meta.py index ea55c07b..c06e9ed5 100644 --- a/src/fts3rest/fts3rest/model/meta.py +++ b/src/fts3rest/fts3rest/model/meta.py @@ -19,10 +19,13 @@ import logging import sqlalchemy from sqlalchemy.orm import scoped_session, sessionmaker -from fts3.model.base import Base # The declarative Base - # SQLAlchemy session manager. Updated by model.init_model() Session = scoped_session(sessionmaker()) # Log the version logging.getLogger(__name__).info("Using SQLAlchemy %s" % sqlalchemy.__version__) + + +def init_model(engine): + """Call me before using any of the tables or classes in the model""" + Session.configure(bind=engine) diff --git a/src/fts3/model/oauth2.py b/src/fts3rest/fts3rest/model/oauth2.py similarity index 100% rename from src/fts3/model/oauth2.py rename to src/fts3rest/fts3rest/model/oauth2.py diff --git a/src/fts3/model/optimizer.py b/src/fts3rest/fts3rest/model/optimizer.py similarity index 100% rename from src/fts3/model/optimizer.py rename to src/fts3rest/fts3rest/model/optimizer.py diff --git a/src/fts3/model/server.py b/src/fts3rest/fts3rest/model/server.py similarity index 100% rename from src/fts3/model/server.py rename to src/fts3rest/fts3rest/model/server.py diff --git a/src/fts3/model/version.py b/src/fts3rest/fts3rest/model/version.py similarity index 100% rename from src/fts3/model/version.py rename to src/fts3rest/fts3rest/model/version.py diff --git a/src/fts3rest/fts3rest/tests/__init__.py b/src/fts3rest/fts3rest/tests/__init__.py index 466373d2..fe9249ee 100644 --- a/src/fts3rest/fts3rest/tests/__init__.py +++ b/src/fts3rest/fts3rest/tests/__init__.py @@ -9,7 +9,7 @@ from unittest import TestCase from M2Crypto import ASN1, X509, RSA, EVP from M2Crypto.ASN1 import UTC -from fts3.model import * +from fts3rest.model import * from fts3rest.config.middleware import create_app from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials from fts3rest.model.meta import Session diff --git a/src/fts3rest/fts3rest/tests/functional/test_banning.py b/src/fts3rest/fts3rest/tests/functional/test_banning.py index fe1c6d0b..7ff3f82a 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_banning.py +++ b/src/fts3rest/fts3rest/tests/functional/test_banning.py @@ -1,7 +1,7 @@ import json from urllib.parse import quote -from fts3.model import BannedDN, BannedSE, Job, File +from fts3rest.model import BannedDN, BannedSE, Job, File from fts3rest.model.meta import Session from fts3rest.tests import TestController diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py b/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py index 2af62820..a3bf56b8 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, ActivityShare +from fts3rest.model import ConfigAudit, ActivityShare class TestConfigActivityShare(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py b/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py index 521f9dee..5abf160f 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, AuthorizationByDn +from fts3rest.model import ConfigAudit, AuthorizationByDn class TestConfigAuthz(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py index 132404df..eb646bb3 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import CloudStorage, CloudStorageUser +from fts3rest.model import CloudStorage, CloudStorageUser class TestConfigCloud(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_global.py b/src/fts3rest/fts3rest/tests/functional/test_config_global.py index 450cfce6..b8b3a091 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_global.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_global.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model.config import ConfigAudit, ServerConfig +from fts3rest.model import ConfigAudit, ServerConfig class TestConfigGlobal(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_links.py b/src/fts3rest/fts3rest/tests/functional/test_config_links.py index 87860caa..1dd9eff0 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_links.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_links.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, LinkConfig +from fts3rest.model import ConfigAudit, LinkConfig class TestConfigLinks(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_se.py b/src/fts3rest/fts3rest/tests/functional/test_config_se.py index 6ca52993..b1dcbbd4 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_se.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_se.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, Optimizer, OperationConfig, Se +from fts3rest.model import ConfigAudit, Optimizer, OperationConfig, Se class TestConfigSe(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_shares.py b/src/fts3rest/fts3rest/tests/functional/test_config_shares.py index 3081c3b4..1e5dd8bc 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_shares.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_shares.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model.config import ( +from fts3rest.model import ( ConfigAudit, ServerConfig, OperationConfig, diff --git a/src/fts3rest/fts3rest/tests/functional/test_delegation.py b/src/fts3rest/fts3rest/tests/functional/test_delegation.py index 8dae6d96..31f4f625 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_delegation.py +++ b/src/fts3rest/fts3rest/tests/functional/test_delegation.py @@ -5,7 +5,7 @@ import time from fts3rest.controllers.delegation import _generate_proxy_request from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Credential, CredentialCache +from fts3rest.model import Credential, CredentialCache class TestDelegation(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_drain.py b/src/fts3rest/fts3rest/tests/functional/test_drain.py index 8324c7fe..a86855c6 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_drain.py +++ b/src/fts3rest/fts3rest/tests/functional/test_drain.py @@ -2,7 +2,7 @@ from datetime import datetime from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ConfigAudit, Host +from fts3rest.model import ConfigAudit, Host class TestDrain(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_dropbox.py b/src/fts3rest/fts3rest/tests/functional/test_dropbox.py index 502186c4..39932d87 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_dropbox.py +++ b/src/fts3rest/fts3rest/tests/functional/test_dropbox.py @@ -3,7 +3,7 @@ from unittest.mock import patch from fts3rest.model.meta import Session from fts3rest.tests import TestController from fts3rest.controllers.CSdropbox import DropboxConnector -from fts3.model import CloudStorage, CloudStorageUser +from fts3rest.model import CloudStorage, CloudStorageUser def _oauth_header_dict(raw_header): diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py b/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py index 5ab52a13..5067615d 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import ( +from fts3rest.model import ( Job, File, JobActiveStates, diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py b/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py index 6354216b..b072e1a6 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job, DataManagement +from fts3rest.model import Job, DataManagement class TestJobDeletion(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_listing.py b/src/fts3rest/fts3rest/tests/functional/test_job_listing.py index ff3824f8..c4f691a0 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_listing.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_listing.py @@ -1,7 +1,7 @@ import json from datetime import datetime, timedelta -from fts3.model import FileRetryLog, Job, File +from fts3rest.model import FileRetryLog, Job, File from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials from fts3rest.lib.middleware.fts3auth import constants diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_modify.py b/src/fts3rest/fts3rest/tests/functional/test_job_modify.py index 449eebd0..cde2229c 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_modify.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_modify.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job +from fts3rest.model import Job class TestJobModify(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py index 6dc3829c..d882c346 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py @@ -4,9 +4,8 @@ import time from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import File, Job +from fts3rest.model import Job import random -import unittest from math import ceil diff --git a/src/fts3rest/fts3rest/tests/functional/test_multiple.py b/src/fts3rest/fts3rest/tests/functional/test_multiple.py index 415a31c8..81f0ce03 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_multiple.py +++ b/src/fts3rest/fts3rest/tests/functional/test_multiple.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job, File +from fts3rest.model import Job, File class TestMultiple(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py index 74131e92..955ccc88 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py +++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py @@ -1,4 +1,4 @@ -from fts3.model import Credential +from fts3rest.model import Credential from fts3rest.lib.openidconnect import OIDCmanager from fts3rest.tests import TestController diff --git a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py index 66c5ce01..05625c4a 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py +++ b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py @@ -1,6 +1,6 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Optimizer +from fts3rest.model import Optimizer class TestOptimizer(TestController): diff --git a/src/fts3rest/fts3rest/tests/functional/test_scheduler.py b/src/fts3rest/fts3rest/tests/functional/test_scheduler.py index 0328fb9c..f98f371c 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_scheduler.py +++ b/src/fts3rest/fts3rest/tests/functional/test_scheduler.py @@ -5,7 +5,7 @@ import logging from fts3rest.tests import TestController from fts3rest.model.meta import Session from fts3rest.lib.scheduler.Cache import ThreadLocalCache -from fts3.model import Job, File, OptimizerEvolution, ActivityShare +from fts3rest.model import Job, File, OptimizerEvolution, ActivityShare import random log = logging.getLogger(__name__) diff --git a/src/fts3rest/fts3rest/tests/functional/test_staging.py b/src/fts3rest/fts3rest/tests/functional/test_staging.py index 289c1aff..8be2502f 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_staging.py +++ b/src/fts3rest/fts3rest/tests/functional/test_staging.py @@ -2,7 +2,7 @@ import json from fts3rest.tests import TestController from fts3rest.model.meta import Session -from fts3.model import Job +from fts3rest.model import Job class TestSubmitToStaging(TestController): -- GitLab From e665c4b6f7e7b2cea7d5aa6c789ebc1565cfbd97 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 16:17:21 +0200 Subject: [PATCH 21/27] Add CI environ variables for OIDC tests --- src/fts3rest/fts3rest/config/config.py | 12 ++++++++++-- src/fts3rest/fts3rest/config/middleware.py | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py index b318d23a..f6b03fe8 100644 --- a/src/fts3rest/fts3rest/config/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -21,7 +21,7 @@ import logging log = logging.getLogger(__name__) -def fts3_config_load(path="/etc/fts3/fts3config"): +def fts3_config_load(path="/etc/fts3/fts3config", test=False): """ Read the configuration from the FTS3 configuration file """ @@ -123,5 +123,13 @@ def fts3_config_load(path="/etc/fts3/fts3config"): ) except NoSectionError: pass - + if test: + provider_url = "https://iam.extreme-datacloud.eu/" + fts3cfg["fts3.Providers"][provider_url] = {} + fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[ + "xdc_ClientId" + ] + fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[ + "xdc_ClientSecret" + ] return fts3cfg diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index 4dc60197..32d30726 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -21,7 +21,7 @@ from fts3rest.lib.openidconnect import oidc_manager from fts3rest.model.meta import init_model, Session -def _load_configuration(config_file): +def _load_configuration(config_file, test): # ConfigParser doesn't handle files without headers. # If the configuration file doesn't start with [fts3], # add it for backwards compatibility, as before migrating to Flask @@ -42,7 +42,7 @@ def _load_configuration(config_file): # Load configuration logging.config.fileConfig(content) content.seek(0) - fts3cfg = fts3_config_load(content) + fts3cfg = fts3_config_load(content, test) content.close() return fts3cfg @@ -91,7 +91,7 @@ def create_app(default_config_file=None, test=False): if not config_file: raise ValueError("The configuration file has not been specified") - fts3cfg = _load_configuration(config_file) + fts3cfg = _load_configuration(config_file, test) log = logging.getLogger(__name__) # Add configuration -- GitLab From f67121cf5e57b4df28721fd0a461127c6276e9a8 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 16:31:14 +0200 Subject: [PATCH 22/27] fix --- src/fts3rest/fts3rest/tests/functional/test_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fts3rest/fts3rest/tests/functional/test_archive.py b/src/fts3rest/fts3rest/tests/functional/test_archive.py index 4e99675a..73af763c 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_archive.py +++ b/src/fts3rest/fts3rest/tests/functional/test_archive.py @@ -1,7 +1,7 @@ from fts3rest.tests import TestController from fts3rest.model.meta import Session from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials -from fts3.model import ArchivedJob, ArchivedFile +from fts3rest.model import ArchivedJob, ArchivedFile class TestArchive(TestController): -- GitLab From 4f5e7b755fd0635d21834768048673d3b9d2fe21 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 16:38:15 +0200 Subject: [PATCH 23/27] fix config --- src/fts3rest/fts3rest/config/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py index f6b03fe8..e01afe79 100644 --- a/src/fts3rest/fts3rest/config/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -115,10 +115,10 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False): fts3cfg["fts3.ValidateAccessTokenOffline"] = parser.getboolean( "fts3", "ValidateAccessTokenOffline", fallback=True ) - fts3cfg["JWKCacheSeconds"] = parser.getint( + fts3cfg["fts3.JWKCacheSeconds"] = parser.getint( "fts3", "JWKCacheSeconds", fallback=86400 ) - fts3cfg["TokenRefreshDaemonIntervalInSeconds"] = parser.getint( + fts3cfg["fts3.TokenRefreshDaemonIntervalInSeconds"] = parser.getint( "fts3", "TokenRefreshDaemonIntervalInSeconds", fallback=600 ) except NoSectionError: -- GitLab From 61fa2078721445fa2f34519db5f19040fd7b2fc9 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 16:52:52 +0200 Subject: [PATCH 24/27] fix --- src/fts3rest/fts3rest/config/config.py | 14 +++++++------- .../tests/functional/test_openidconnect.py | 3 +++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py index e01afe79..89a218cb 100644 --- a/src/fts3rest/fts3rest/config/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -123,13 +123,13 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False): ) except NoSectionError: pass - if test: + if test: # for open id tests provider_url = "https://iam.extreme-datacloud.eu/" fts3cfg["fts3.Providers"][provider_url] = {} - fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[ - "xdc_ClientId" - ] - fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[ - "xdc_ClientSecret" - ] + fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ.get( + "xdc_ClientId", "" + ) + fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ.get( + "xdc_ClientSecret", "" + ) return fts3cfg diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py index 955ccc88..0a0433a7 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py +++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py @@ -1,6 +1,7 @@ from fts3rest.model import Credential from fts3rest.lib.openidconnect import OIDCmanager from fts3rest.tests import TestController +import unittest class TestOpenidconnect(TestController): @@ -16,6 +17,8 @@ class TestOpenidconnect(TestController): self.oidc_manager = OIDCmanager() self.config = self.flask_app.config self.issuer = "https://iam.extreme-datacloud.eu/" + if "client_id" not in self.config["fts3.Providers"][self.issuer]: + raise unittest.SkipTest("Missing OIDC client configurationd data") def test_configure_clients(self): self.oidc_manager._configure_clients(self.config["fts3.Providers"]) -- GitLab From ad83f18ffd142644d56833e6e623a95dc64ff320 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 17:04:44 +0200 Subject: [PATCH 25/27] fix config --- src/fts3rest/fts3rest/config/config.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py index 89a218cb..644a81b5 100644 --- a/src/fts3rest/fts3rest/config/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -126,10 +126,10 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False): if test: # for open id tests provider_url = "https://iam.extreme-datacloud.eu/" fts3cfg["fts3.Providers"][provider_url] = {} - fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ.get( - "xdc_ClientId", "" - ) - fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ.get( - "xdc_ClientSecret", "" - ) + fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[ + "xdc_ClientId" + ] + fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[ + "xdc_ClientSecret" + ] return fts3cfg -- GitLab From 00ddcc8e9807a43d2c28cc0c7f8e938ce777752c Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 7 May 2020 17:21:30 +0200 Subject: [PATCH 26/27] fix config --- src/fts3rest/fts3rest/config/middleware.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index 32d30726..5e622c73 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -126,7 +126,8 @@ def create_app(default_config_file=None, test=False): # Start OIDC clients if "fts3.Providers" in app.config: oidc_manager.setup(app.config) - IAMTokenRefresher("fts_token_refresh_daemon", app.config).start() + if not test: + IAMTokenRefresher("fts_token_refresh_daemon", app.config).start() else: log.info("OpenID Connect support disabled. Providers not found in config") -- GitLab From 2d1896812b099b36b03bf9467416bcf036b9e69c Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Fri, 8 May 2020 12:22:00 +0200 Subject: [PATCH 27/27] Update --- README.md | 25 ++++++++++++------- src/fts3rest/fts3rest/config/middleware.py | 2 +- .../tests/functional/test_openidconnect.py | 2 +- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 33893acc..dfcc9a79 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,30 @@ # About the migration -The migration of [fts-rest](https://gitlab.cern.ch:8443/fts/fts-rest) has started after the decisions made in +The migration of [fts-rest](https://gitlab.cern.ch:8443/fts/fts-rest) started after the decisions made in the [evaluation](https://its.cern.ch/jira/browse/FTS-1496). The development is happening at fts-flask.cern.ch, with the local user ftsflask # Git workflow - `Master` cannot be pushed to directly. -- Create a new branch for each ticket and merge it to master. +- Create a new branch for each ticket and merge it to master through a merge request. # Gitlab CI The current pipeline runs for every push in every branch: - black: fails if the code hasn't been formatted with black - pylint: fails if the code has syntax errors. If you are sure that pylint is mistaken, add `# pylint: skip-file` at - the beginning of the relevant file. + the beginning of the relevant file. Runs for every supported Python3 version - radon: fails if the code complexity is too high +- functional tests: Run for every supported Python3 version - bandit: detects potential security issues in the code, but it's allowed to fail as there may be false positives +- build: sdist and wheel + Merge requests will proceed only if the pipeline succeeds. + In case of emergency the pipeline can be [skipped](https://docs.gitlab.com/ee/ci/yaml/#skipping-jobs). -The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster. +The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the +image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster. +To build and push the image, cd to .gitlab-ci and run .docker_push.sh Developers should add the `pre-commit` hook to their local repository. This scripts does this for every commit: - Runs black to format the changed files. @@ -26,6 +32,10 @@ Developers should add the `pre-commit` hook to their local repository. This scri - Runs radon and bandit only on the changed files. The hook can be skipped, in case bandit detects false positives, with the commit option `--no-verify`. +# Functional tests +Openid tests don't run in CI because the container would need a client registered and this is + difficult to set up. To run these tests in a development environment, the environment variables 'xdc_ClientId' and 'xdc_ClientSecret' must be set. + # Python dependencies This project uses [pip-tools](https://github.com/jazzband/pip-tools) to manage dependencies: - `requirements.in`: list of dependencies for the production app @@ -39,6 +49,7 @@ Because we need mod_wsgi built for Python 3.6, we need to use httpd24-httpd - gfal2-python3 - yum-config-manager --enable centos-sclo-rh - yum install rh-python36-mod_wsgi + # Installation requirements for development To create a development venv: use --system-packages in order to use gfal2-python3 @@ -62,7 +73,6 @@ curl http://localhost:80/hello To access the config page: ``` INSERT INTO t_authz_dn VALUES ('yourdn'); - ``` # Run tests @@ -70,8 +80,5 @@ INSERT INTO t_authz_dn VALUES ('yourdn'); source venv/bin/activate export PYTHONPATH=/home/ftsflask/fts-rest-flask/src:/home/ftsflask/fts-rest-flask/src/fts3rest export FTS3TESTCONFIG=/home/ftsflask/fts-rest-flask/src/fts3rest/fts3rest/tests/fts3testconfig -python3 -m pytest -x src/fts3rest/fts3rest/tests/functional/test_job_submission.py +python3 -m pytest src/fts3rest/fts3rest/tests/ -x ``` -# Migration status -Starting with the client, as it requires small changes only. Will not migrate pycurlrequest.py, as it is not used - anymore. \ No newline at end of file diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index 5e622c73..248caeda 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -124,7 +124,7 @@ def create_app(default_config_file=None, test=False): return response # Start OIDC clients - if "fts3.Providers" in app.config: + if "fts3.Providers" not in app.config or not app.config["fts3.Providers"]: oidc_manager.setup(app.config) if not test: IAMTokenRefresher("fts_token_refresh_daemon", app.config).start() diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py index 0a0433a7..ad67bd4d 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py +++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py @@ -18,7 +18,7 @@ class TestOpenidconnect(TestController): self.config = self.flask_app.config self.issuer = "https://iam.extreme-datacloud.eu/" if "client_id" not in self.config["fts3.Providers"][self.issuer]: - raise unittest.SkipTest("Missing OIDC client configurationd data") + raise unittest.SkipTest("Missing OIDC client configuration data") def test_configure_clients(self): self.oidc_manager._configure_clients(self.config["fts3.Providers"]) -- GitLab