From edaf1ad4b862f1c5500190a4bc7dbf3762a42370 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 11:57:27 +0200
Subject: [PATCH 01/27] Fix tests and improve config parsing

---
 src/fts3/util/config.py                       |  37 ++--
 src/fts3rest/fts3rest/config/middleware.py    |  12 +-
 .../fts3rest/lib/IAMTokenRefresher.py         | 141 +++++++++++++++
 src/fts3rest/fts3rest/lib/openidconnect.py    |   2 +-
 src/fts3rest/fts3rest/tests/__init__.py       |  24 ++-
 src/fts3rest/fts3rest/tests/fts3testconfig    | 163 ++----------------
 .../tests/functional/test_oauth2provider.py   |  74 ++++++++
 .../tests/functional/test_openidconnect.py    |  53 ++++++
 8 files changed, 328 insertions(+), 178 deletions(-)
 create mode 100644 src/fts3rest/fts3rest/lib/IAMTokenRefresher.py
 create mode 100644 src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py
 create mode 100644 src/fts3rest/fts3rest/tests/functional/test_openidconnect.py

diff --git a/src/fts3/util/config.py b/src/fts3/util/config.py
index fe2262ba..bc26c2be 100644
--- a/src/fts3/util/config.py
+++ b/src/fts3/util/config.py
@@ -13,7 +13,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-from configparser import ConfigParser, NoOptionError
+from configparser import ConfigParser, NoOptionError, NoSectionError
 from urllib.parse import quote_plus
 import os
 import logging
@@ -100,18 +100,29 @@ def fts3_config_load(path="/etc/fts3/fts3config"):
             fts3cfg["fts3.Roles"][role.lower()][operation.lower()] = level.lower()
 
     # Initialize providers
-    log.debug("initialize providers config in load environment")
     fts3cfg["fts3.Providers"] = {}
-    for option in parser.options("providers"):
-        if "_" not in option:
-            provider_name = option
-            provider_url = parser.get("providers", provider_name)
-            if not provider_url.endswith("/"):
-                provider_url += "/"
-            fts3cfg["fts3.Providers"][provider_url] = {}
-            client_id = parser.get("providers", option + "_ClientId")
-            fts3cfg["fts3.Providers"][provider_url]["client_id"] = client_id
-            client_secret = parser.get("providers", option + "_ClientSecret")
-            fts3cfg["fts3.Providers"][provider_url]["client_secret"] = client_secret
+    try:
+        for option in parser.options("providers"):
+            if "_" not in option:
+                provider_name = option
+                provider_url = parser.get("providers", provider_name)
+                if not provider_url.endswith("/"):
+                    provider_url += "/"
+                fts3cfg["fts3.Providers"][provider_url] = {}
+                client_id = parser.get("providers", option + "_ClientId")
+                fts3cfg["fts3.Providers"][provider_url]["client_id"] = client_id
+                client_secret = parser.get("providers", option + "_ClientSecret")
+                fts3cfg["fts3.Providers"][provider_url]["client_secret"] = client_secret
+        fts3cfg["fts3.ValidateAccessTokenOffline"] = parser.getboolean(
+            "fts3", "ValidateAccessTokenOffline", fallback=True
+        )
+        fts3cfg["JWKCacheSeconds"] = parser.getint(
+            "fts3", "JWKCacheSeconds", fallback=86400
+        )
+        fts3cfg["TokenRefreshDaemonIntervalInSeconds"] = parser.getint(
+            "fts3", "TokenRefreshDaemonIntervalInSeconds", fallback=600
+        )
+    except NoSectionError:
+        pass
 
     return fts3cfg
diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py
index 1aba2610..30b90854 100644
--- a/src/fts3rest/fts3rest/config/middleware.py
+++ b/src/fts3rest/fts3rest/config/middleware.py
@@ -16,6 +16,8 @@ from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddlewa
 from fts3rest.lib.middleware.error_as_json import ErrorAsJson
 from fts3rest.lib.middleware.timeout import TimeoutHandler
 from fts3rest.model.meta import Session
+from fts3rest.lib.openidconnect import oidc_manager
+from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher
 from werkzeug.exceptions import HTTPException
 import json
 
@@ -109,7 +111,6 @@ def create_app(default_config_file=None, test=False):
     app.wsgi_app = TimeoutHandler(app.wsgi_app, fts3cfg)
 
     # Convert errors to JSON
-    # app.wsgi_app = ErrorAsJson(app.wsgi_app)
     @app.errorhandler(HTTPException)
     def handle_exception(e):
         """Return JSON instead of HTML for HTTP errors."""
@@ -122,10 +123,9 @@ def create_app(default_config_file=None, test=False):
         response.content_type = "application/json"
         return response
 
-    # @app.errorhandler(NotFound)
-    # def handle_invalid_usage(error):
-    #     response = jsonify(error=error.code, name=error.name)
-    #     response.status_code = error.code
-    #     return response
+    # Start OIDC clients
+    if "fts3.Providers" in app.config:
+        oidc_manager.setup(app.config)
+        IAMTokenRefresher("fts_token_refresh_daemon", app.config).start()
 
     return app
diff --git a/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py
new file mode 100644
index 00000000..49220094
--- /dev/null
+++ b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py
@@ -0,0 +1,141 @@
+#   Copyright 2020 CERN
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import logging
+import socket
+import time
+import random
+from datetime import datetime, timedelta
+from threading import Thread, current_thread
+
+
+from fts3rest.model.meta import Session
+from fts3rest.lib.openidconnect import oidc_manager
+from fts3.model import Credential, Host
+
+from sqlalchemy.exc import SQLAlchemyError
+
+log = logging.getLogger(__name__)
+
+
+class IAMTokenRefresher(Thread):
+    """
+    Daemon thread that refreshes all access tokens in the DB at every interval.
+
+    Keeps running on the background updating the DB, marking the process as alive.
+    There should be ONLY ONE across all instances.
+    Keep in mind that with the Apache configuration line
+        WSGIDaemonProcess fts3rest python-path=... processes=2 threads=15
+    there will be 2 instances of the application per server, meaning we need to check that there is only one
+    IAMTokenRefresher per host, and only one between all hosts.
+
+    The SQLAlchemy scoped_session is thread-safe
+    """
+
+    def __init__(self, tag, config):
+        Thread.__init__(self)
+        self.daemon = (
+            True  # The thread will immediately exit when the main thread exits
+        )
+        self.tag = tag
+        self.refresh_interval = int(
+            config.get("fts3.TokenRefreshDaemonIntervalInSeconds", 600)
+        )
+        self.config = config
+
+    def _thread_is_inactive(self, thread):
+        # The thread is considered inactive if it hasn't updated the DB for 3*refresh_interval
+        log.debug("time since last beat {}".format(datetime.utcnow() - thread.beat))
+        if (datetime.utcnow() - thread.beat) > timedelta(
+            seconds=3 * self.refresh_interval
+        ):
+            log.debug("thread is inactive! taking over, beat {}".format(thread.beat))
+        return (datetime.utcnow() - thread.beat) > timedelta(
+            seconds=3 * self.refresh_interval
+        )
+
+    def run(self):
+        """
+        Regularly check if there is another active IAMTokenRefresher in the DB. If not, become the active thread.
+        """
+        log.debug("CREATE THREAD ID: {}".format(current_thread().ident))
+        # Initial sleep in case all threads started at the same time
+        time.sleep(random.randint(0, 60))
+        # The interval at which the thread will check if there is another active thread.
+        # It is arbitrary: I chose 2 times the refresh interval, plus a random offset to avoid multiple threads
+        # checking at the same time (although DB access is transactional)
+        db_check_interval = 3 * self.refresh_interval + random.randint(0, 120)
+        while True:
+            # Check that no other fts-token-refresh-daemon is running
+            refresher_threads = (
+                Session.query(Host).filter(Host.service_name == self.tag).all()
+            )
+            Session.commit()  # Close transaction to avoid repeated read
+            log.debug(
+                "refresher_threads {}, ID {}".format(
+                    len(refresher_threads), current_thread().ident
+                )
+            )
+            if all(self._thread_is_inactive(thread) for thread in refresher_threads):
+                log.debug("Activating thread")
+                for thread in refresher_threads:
+                    Session.delete(thread)
+                    log.debug("delete thread")
+                host = Host(hostname=socket.getfqdn(), service_name=self.tag)
+                log.debug("host object created")
+                while True:
+                    host.beat = datetime.utcnow()
+                    log.debug(
+                        "THREAD ID: {}, beat {}".format(
+                            current_thread().ident, host.beat
+                        )
+                    )
+                    try:
+                        h2 = Session.merge(host)
+                        Session.commit()
+                        log.debug(
+                            "fts-token-refresh-daemon heartbeat {}".format(h2.beat)
+                        )
+                    except SQLAlchemyError as ex:
+                        log.warning(
+                            "Failed to update the fts-token-refresh-daemon heartbeat: %s"
+                            % str(ex)
+                        )
+                        Session.rollback()
+                        raise
+
+                    credentials = (
+                        Session.query(Credential)
+                        .filter(Credential.proxy.notilike("%CERTIFICATE%"))
+                        .all()
+                    )
+                    log.debug("{} credentials to refresh".format(len(credentials)))
+                    for credential in credentials:
+                        try:
+                            credential = oidc_manager.refresh_access_token(credential)
+                            log.debug("OK refresh_access_token")
+                            Session.merge(credential)
+                            Session.commit()
+                        except Exception as ex:
+                            log.warning(
+                                "Failed to refresh token for dn: %s because: %s"
+                                % (str(credential.dn), str(ex))
+                            )
+                            Session.rollback()
+                            raise
+                    time.sleep(self.refresh_interval)
+            else:
+                log.debug("THREAD ID: {}".format(current_thread().ident))
+                log.debug("Another thread is active -- Going to sleep")
+                time.sleep(db_check_interval)
diff --git a/src/fts3rest/fts3rest/lib/openidconnect.py b/src/fts3rest/fts3rest/lib/openidconnect.py
index 2a48bcf1..1024e6ef 100644
--- a/src/fts3rest/fts3rest/lib/openidconnect.py
+++ b/src/fts3rest/fts3rest/lib/openidconnect.py
@@ -27,7 +27,7 @@ class OIDCmanager:
     def setup(self, config):
         self.config = config
         self._configure_clients(config["fts3.Providers"])
-        self._set_keys_cache_time(int(config["fts3.JWKCacheSeconds"]))
+        self._set_keys_cache_time(config["fts3.JWKCacheSeconds"])
         self._retrieve_clients_keys()
 
     def _configure_clients(self, providers_config):
diff --git a/src/fts3rest/fts3rest/tests/__init__.py b/src/fts3rest/fts3rest/tests/__init__.py
index ee046677..466373d2 100644
--- a/src/fts3rest/fts3rest/tests/__init__.py
+++ b/src/fts3rest/fts3rest/tests/__init__.py
@@ -1,18 +1,18 @@
+from datetime import timedelta
 import os
 import shutil
+import subprocess
 import time
-
-from datetime import datetime, timedelta
+import unittest
 from unittest import TestCase
+
 from M2Crypto import ASN1, X509, RSA, EVP
 from M2Crypto.ASN1 import UTC
 
-
-from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials
-from fts3rest.model.meta import Session
-from fts3.model import Credential, CredentialCache, DataManagement
 from fts3.model import *
 from fts3rest.config.middleware import create_app
+from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials
+from fts3rest.model.meta import Session
 from .ftstestclient import FTSTestClient, TestResponse
 
 
@@ -198,3 +198,15 @@ class TestController(TestCase):
                 pass
 
         self.flask_app.do_teardown_appcontext()
+
+    def _get_xdc_access_token(self):
+        command = (
+            "eval `oidc-agent` && oidc-add xdctest --pw-cmd=echo && oidc-token xdctest"
+        )
+        try:
+            output = subprocess.check_output(command, shell=True)
+        except subprocess.CalledProcessError as ex:
+            raise unittest.SkipTest("Failed to get access token") from ex
+        output = str(output).strip()
+        token = output.split("\n")[2]  # The 3rd line is the token
+        return token
diff --git a/src/fts3rest/fts3rest/tests/fts3testconfig b/src/fts3rest/fts3rest/tests/fts3testconfig
index a0099268..90334d2f 100644
--- a/src/fts3rest/fts3rest/tests/fts3testconfig
+++ b/src/fts3rest/fts3rest/tests/fts3testconfig
@@ -1,7 +1,3 @@
-# Running user and group
-User=ftsflask
-Group=ftsflask
-
 # mysql only
 DbType=mysql
 
@@ -11,150 +7,27 @@ DbUserName=ftsflask
 #db password
 DbPassword=
 
-
 #For MySQL, it has to follow the format 'host/db' (i.e. "mysql-server.example.com/fts3db")
 DbConnectString=localhost:3306/ftsflask
 
-#Number of db connections in the pool (use even number, e.g. 2,4,6,8,etc OR 1 for a single connection)
-DbThreadsNum=30
-
-#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/
-#Alias=fts3-xdc.cern.ch
-
-#Infosys, either the fqdn:port of a BDII instance or false to disable BDII access
-#Infosys=lcg-bdii.cern.ch:2170
-
-#Query the info systems specified in the order given, e.g. glue1;glue2
-InfoProviders=glue1
-
 #List of authorized VOs, separated by ;
 #Leave * to authorize any VO
 AuthorizedVO=*
 
-# site name
-#SiteName=FTS-DEV-XDC
-
-#Enable/Disable monitoring using messaging monitoring (disabled=false / enabled=true)
-MonitoringMessaging=false
-
-# Profiling interval in seconds. If set to 0, it will be disabled
-Profiling=0
-
-# Log directories
-TransferLogDirectory=/var/log/fts3/transfers
-ServerLogDirectory=/var/log/fts3
-
-# Log level. Enables logging for messages of level >= than configured
-# Possible values are
-#   TRACE (every detail), DEBUG (internal behaviour), INFO (normal behaviour),
-#   NOTICE (final states), WARNING (things worth checking), ERR (internal FTS3 errors, as database connectivity),
-#   CRIT (fatal errors, as segmentation fault)
-# It is recommended to use DEBUG or INFO
-LogLevel=DEBUG
-
-# Check for fts_url_copy processes that do not give their progress back
-# CheckStalledTransfers = true
-# Stalled timeout, in seconds
-# CheckStalledTimeout = 900
-CheckStalledTimeout = 900
-
-# Minimum required free RAM (in MB) for FTS3 to work normally
-# If the amount of free RAM goes below the limit, FTS3 will enter auto-drain mode
-# This is intended to protect against system resource exhaustion
-# MinRequiredFreeRAM = 50
-MinRequiredFreeRAM = 50
-
-# Maximum number of url copy processes that the node can run
-# The RAM limitation may not take into account other node limitations (i.e. IO)
-# or, depending on the swapping policy, may not even prevent overloads if the kernel
-# starts swapping before the free RAM decreases until it reaches the value of MinRequiredFreeRAM
-# 0 disables the check.
-# The default is 400.
-# MaxUrlCopyProcesses = 400
-MaxUrlCopyProcesses = 400
-
-# Parameters for Bring Online
-# Maximum bulk size.
-# If the size is too large, it will take more resources (memory and CPU) to generate the requests and
-# parse the responses. Some servers may reject the requests if they are too big.
-# If it is too small, performance will be reduced.
-# Keep it to a sensible size (between 100 and 1k)
-# StagingBulkSize=400
-# Maximum number of concurrent requests. This gives a maximum of files sent to the storage system
-# (StagingBulkSize*StagingConcurrentRequests). The larger the number, the more requests will FTS need to keep track of.
-# StagingConcurrentRequests=500
-# Seconds to wait before submitting a bulk request, so FTS can accumulate more files per bulk.
-# Note that the resolution is 60 seconds.
-# StagingWaitingFactor=300
-# Retry this number of times if a staging poll fails with ECOMM
-# StagingPollRetries=3
-
-# In seconds, interval between heartbeats
-# HeartBeatInterval=60
-# I seconds, after this interval a host is considered down
-# HeartBeatGraceInterval=120
-
-# Seconds between optimizer runs
-# OptimizerInterval = 60
-# After this time without optimizer updates, force a run
-# OptimizerSteadyInterval = 300
-# Maximum number of streams per file
-# OptimizerMaxStreams = 16
-
-# EMA Alpha factor to reduce the influence of fluctuations
-# OptimizerEMAAlpha = 0.1
-# Increase step size when the optimizer considers the performance is good
-# OptimizerIncreaseStep = 1
-# Increase step size when the optimizer considers the performance is good, and set to aggressive or normal
-# OptimizerAggressiveIncreaseStep = 2
-# Decrease step size when the optimizer considers the performance is bad
-# OptimizerDecreaseStep = 1
-
-
-# Set the bulk size, in number of jobs, used for cleaning the old records
-#CleanBulkSize=5000
-# In days. Entries older than this will be purged.
-#CleanInterval=7
-
-## The higher the values for the following parameters,
-## the higher the latency for some operations (as cancelations),
-## but can also reduce the system and/or database load
-
-# In seconds, how often to purge the messaging directory
-#PurgeMessagingDirectoryInterval = 600
-# In seconds, how often to run sanity checks
-#CheckSanityStateInterval = 3600
-# In seconds, how often to check for canceled transfers
-#CancelCheckInterval = 10
-# In seconds, how often to check for expired queued transfers
-#QueueTimeoutCheckInterval = 300
-# In seconds, how often to check for stalled transfers
-#ActiveTimeoutCheckInterval = 300
-# In seconds, how often to schedule new transfers
-#SchedulingInterval = 2
-# In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2
-#MessagingConsumeInterval = 1
-#Enable or disable auto session reuse
-AutoSessionReuse = true
-#Max small file size for session reuse in bytes
-AutoSessionReuseMaxSmallFileSize = 104857600
-#Max big file size for session reuse in bytes
-AutoSessionReuseMaxBigFileSize = 1073741824
-#Max number of files per session reuse
-AutoSessionReuseMaxFiles = 1000
-#Max number of big files  per session reuse
-AutoSessionReuseMaxBigFiles = 2
-BackupTables=false
-OptimizerMaxSuccessRate=100
-OptimizerMedSuccessRate=80
-OptimizerLowSuccessRate=75
-OptimizerBaseSuccessRate=74
-Port=8443
-UseFixedJobPriority=0
-
+#OpenID parameters
 ValidateAccessTokenOffline=True
 JWKCacheSeconds=86400
 TokenRefreshDaemonIntervalInSeconds=600
+
+[roles]
+Public = vo:transfer;all:datamanagement
+lcgadmin = all:config
+
+[providers]
+xdc=https://iam.extreme-datacloud.eu
+xdc_ClientId=
+xdc_ClientSecret=
+
 # Logging configuration
 [loggers]
 keys = root, routes, fts3rest, sqlalchemy
@@ -195,8 +68,6 @@ level = NOTSET
 formatter = generic
 
 [handler_log_file]
-# See
-# http://docs.python.org/2/library/logging.handlers.html
 class = logging.FileHandler
 args = ('/var/log/fts3rest/fts3rest.log', 'a')
 level = NOTSET
@@ -205,15 +76,3 @@ formatter = generic
 [formatter_generic]
 format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s
 datefmt = %H:%M:%S
-
-
-
-[roles]
-Public = vo:transfer;all:datamanagement
-lcgadmin = all:config
-
-
-[providers]
-xdc=https://iam.extreme-datacloud.eu
-xdc_ClientId=
-xdc_ClientSecret=
diff --git a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py
new file mode 100644
index 00000000..4e254c02
--- /dev/null
+++ b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py
@@ -0,0 +1,74 @@
+from fts3rest.lib.oauth2provider import FTS3OAuth2ResourceProvider
+from fts3rest.lib.openidconnect import OIDCmanager
+from fts3rest.tests import TestController
+
+
+class TestFTS3OAuth2ResourceProvider(TestController):
+    """
+    Test token validation
+
+    To run these tests, the host should have oidc-agent installed,
+    with an account 'xdctest'
+    """
+
+    def setUp(self):
+        super().setUp()
+        self.oidc_manager = OIDCmanager()
+        config = self.flask_app.config
+        self.issuer = "https://iam.extreme-datacloud.eu/"
+        self.oidc_manager.setup(config)
+        self.oauth2_resource_provider = FTS3OAuth2ResourceProvider(dict(), config)
+        self.expired_token = "eyJraWQiOiJyc2ExIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI5NGQyNTQyOS1mYTZhLTRiYTctOGM0NS1mMTk1YjI3ZWVkNjMiLCJpc3MiOiJodHRwczpcL1wvaWFtLmV4dHJlbWUtZGF0YWNsb3VkLmV1XC8iLCJleHAiOjE1ODAyMjIyMDksImlhdCI6MTU4MDIxODYwOSwianRpIjoiYTI0NDRhYTQtNTE3YS00Y2E0LTgwMTUtY2IyMjc0Nzg4YzlkIn0.hvTjA-Ix_YVxU3HmLB6FQa98eYtUwbw1WcZMO5p_qOjnPwD0OtQViVtV-a5__hLY1_qRFouAzgVvqKnueokh1pmKoI6TJN2KpmybueAZR30lIG_t_aAn4hGQvuVezs_0LLISojQUgprbi2PDsU1q8WTJq1J5mwGwlBijGmHQs60"
+
+    def test_validate_access_token(self):
+        token = self._get_xdc_access_token()
+        auth = self.oauth2_resource_provider.authorization_class()
+        self.oauth2_resource_provider.validate_access_token(token, auth)
+        self.assertTrue(auth.is_valid)
+
+    def test_validate_token_offline(self):
+        token = self._get_xdc_access_token()
+        valid, credential = self.oauth2_resource_provider._validate_token_offline(token)
+        self.assertTrue(valid)
+        self.assertEqual(credential["iss"], self.issuer)
+
+    def test_validate_token_online(self):
+        token = self._get_xdc_access_token()
+        valid, credential = self.oauth2_resource_provider._validate_token_online(token)
+        self.assertTrue(valid)
+        self.assertEqual(credential["iss"], self.issuer)
+
+    def test_validate_access_token_invalid(self):
+        token = self._get_xdc_access_token()
+        token += "invalid"
+        auth = self.oauth2_resource_provider.authorization_class()
+        self.oauth2_resource_provider.validate_access_token(token, auth)
+        self.assertFalse(auth.is_valid)
+
+    def test_validate_token_offline_invalid(self):
+        token = self._get_xdc_access_token()
+        token += "invalid"
+        valid, credential = self.oauth2_resource_provider._validate_token_offline(token)
+        self.assertFalse(valid)
+
+    def test_validate_token_online_invalid(self):
+        token = self._get_xdc_access_token()
+        token += "invalid"
+        valid, credential = self.oauth2_resource_provider._validate_token_online(token)
+        self.assertFalse(valid)
+
+    def test_validate_access_token_expired(self):
+        token = self.expired_token
+        auth = self.oauth2_resource_provider.authorization_class()
+        self.oauth2_resource_provider.validate_access_token(token, auth)
+        self.assertFalse(auth.is_valid)
+
+    def test_validate_token_offline_expired(self):
+        token = self.expired_token
+        valid, credential = self.oauth2_resource_provider._validate_token_offline(token)
+        self.assertFalse(valid)
+
+    def test_validate_token_online_expired(self):
+        token = self.expired_token
+        valid, credential = self.oauth2_resource_provider._validate_token_online(token)
+        self.assertFalse(valid)
diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
new file mode 100644
index 00000000..74131e92
--- /dev/null
+++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
@@ -0,0 +1,53 @@
+from fts3.model import Credential
+from fts3rest.lib.openidconnect import OIDCmanager
+from fts3rest.tests import TestController
+
+
+class TestOpenidconnect(TestController):
+    """
+    Tests OIDCmanager operations
+
+    To run these tests, the host should have oidc-agent installed,
+    with an account 'xdctest'
+    """
+
+    def setUp(self):
+        super().setUp()
+        self.oidc_manager = OIDCmanager()
+        self.config = self.flask_app.config
+        self.issuer = "https://iam.extreme-datacloud.eu/"
+
+    def test_configure_clients(self):
+        self.oidc_manager._configure_clients(self.config["fts3.Providers"])
+        self.assertEqual(
+            len(self.oidc_manager.clients), len(self.config["fts3.Providers"])
+        )
+
+    def test_introspect(self):
+        self.oidc_manager.setup(self.config)
+        access_token = self._get_xdc_access_token()
+        response = self.oidc_manager.introspect(self.issuer, access_token)
+        self.assertTrue(response["active"])
+
+    def test_generate_refresh_token(self):
+        self.oidc_manager.setup(self.config)
+        access_token = self._get_xdc_access_token()
+        self.oidc_manager.generate_refresh_token(self.issuer, access_token)
+
+    def test_generate_refresh_token_invalid(self):
+        self.oidc_manager.setup(self.config)
+        access_token = self._get_xdc_access_token()
+        access_token += "invalid"
+        with self.assertRaises(Exception):
+            self.oidc_manager.generate_refresh_token(self.issuer, access_token)
+
+    def test_refresh_access_token(self):
+        self.oidc_manager.setup(self.config)
+        access_token = self._get_xdc_access_token()
+        refresh_token = self.oidc_manager.generate_refresh_token(
+            self.issuer, access_token
+        )
+        credential = Credential()
+        credential.proxy = ":".join([access_token, refresh_token])
+        new_credential = self.oidc_manager.refresh_access_token(credential)
+        self.assertIsNotNone(new_credential.termination_time)
-- 
GitLab


From 5311d4d56e6cdd4005b5ef343b1d537334e921f2 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 12:22:13 +0200
Subject: [PATCH 02/27] fix ci

---
 .gitlab-ci.yml | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 000eb128..5e1bbc1b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,13 +8,18 @@
 
 image: gitlab-registry.cern.ch/fts/fts-rest-flask:ci
 
-
 stages:
   - static_code_analysis
   - tests
   - security
   - build
 
+variables:
+  MYSQL_DATABASE: ftsflask
+  MYSQL_ROOT_PASSWORD: asdf
+  PYTHONPATH: ./src:./src/fts3rest
+  FTS3TESTCONFIG: /src/fts3rest/fts3rest/tests/fts3testconfig_ci
+
 black:
   # Check that every file has been formatted with black
   stage: static_code_analysis
@@ -56,9 +61,6 @@ radon:
   - source .gitlab-ci/radon.sh
 
 
-variables:
-  MYSQL_DATABASE: ftsflask
-  MYSQL_ROOT_PASSWORD: asdf
 
 functional_tests36:
   stage: tests
@@ -68,8 +70,6 @@ functional_tests36:
   script:
   - python --version
   - source .gitlab-ci/db.sh
-  - export PYTHONPATH=./src:./src/fts3rest
-  - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci
   - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
   - coverage report
 
@@ -84,8 +84,6 @@ functional_tests37:
   - python --version
   - source $VENV_37/bin/activate
   - source .gitlab-ci/db.sh
-  - export PYTHONPATH=./src:./src/fts3rest
-  - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci
   - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
   - coverage report
 
@@ -100,8 +98,6 @@ functional_tests38:
   - python --version
   - source $VENV_38/bin/activate
   - source .gitlab-ci/db.sh
-  - export PYTHONPATH=./src:./src/fts3rest
-  - export FTS3TESTCONFIG=./src/fts3rest/fts3rest/tests/fts3testconfig_ci
   - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
   - coverage report
 
-- 
GitLab


From 8b0de3cd16980df51cbc782ab61ec84a8d977609 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 12:30:15 +0200
Subject: [PATCH 03/27] fix ci

---
 .gitlab-ci.yml                             |  6 ++---
 src/fts3/util/config.py                    |  1 -
 src/fts3rest/fts3rest/config/middleware.py | 27 ++++++++++++----------
 3 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5e1bbc1b..c63f3bb7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,7 +31,7 @@ pylint36:
   stage: static_code_analysis
   script:
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
 
 pylint37:
   # Check that every file doesn't have syntax errors
@@ -41,7 +41,7 @@ pylint37:
   - pyenv global $PY37
   - source $VENV_37/bin/activate
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
 
 pylint38:
   # Check that every file doesn't have syntax errors
@@ -51,7 +51,7 @@ pylint38:
   - pyenv global $PY38
   - source $VENV_38/bin/activate
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --init-hook='import sys; sys.path.append("src/fts3rest")' --ignored-modules=sqlalchemy
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
 
 
 radon:
diff --git a/src/fts3/util/config.py b/src/fts3/util/config.py
index bc26c2be..b318d23a 100644
--- a/src/fts3/util/config.py
+++ b/src/fts3/util/config.py
@@ -25,7 +25,6 @@ def fts3_config_load(path="/etc/fts3/fts3config"):
     """
     Read the configuration from the FTS3 configuration file
     """
-    log.debug("entered fts3_config_load")
     fts3cfg = {}
 
     parser = ConfigParser()
diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py
index 30b90854..db5fcbc5 100644
--- a/src/fts3rest/fts3rest/config/middleware.py
+++ b/src/fts3rest/fts3rest/config/middleware.py
@@ -1,25 +1,25 @@
-from flask import Flask, jsonify
-from werkzeug.exceptions import NotFound
-from sqlalchemy import engine_from_config, event
-import MySQLdb
-import os
 from io import StringIO
+import json
 import logging.config
-from fts3rest.config.routing import base, cstorage
+import os
+
+import MySQLdb
+from flask import Flask
+from sqlalchemy import engine_from_config, event
+from werkzeug.exceptions import HTTPException
+
 from fts3.util.config import fts3_config_load
-from fts3rest.model import init_model
+from fts3rest.config.routing import base, cstorage
+from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher
 from fts3rest.lib.helpers.connection_validator import (
     connection_validator,
     connection_set_sqlmode,
 )
 from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddleware
-from fts3rest.lib.middleware.error_as_json import ErrorAsJson
 from fts3rest.lib.middleware.timeout import TimeoutHandler
-from fts3rest.model.meta import Session
 from fts3rest.lib.openidconnect import oidc_manager
-from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher
-from werkzeug.exceptions import HTTPException
-import json
+from fts3rest.model import init_model
+from fts3rest.model.meta import Session
 
 
 def _load_configuration(config_file):
@@ -93,6 +93,7 @@ def create_app(default_config_file=None, test=False):
         raise ValueError("The configuration file has not been specified")
 
     fts3cfg = _load_configuration(config_file)
+    log = logging.getLogger(__name__)
 
     # Add configuration
     app.config.update(fts3cfg)
@@ -127,5 +128,7 @@ def create_app(default_config_file=None, test=False):
     if "fts3.Providers" in app.config:
         oidc_manager.setup(app.config)
         IAMTokenRefresher("fts_token_refresh_daemon", app.config).start()
+    else:
+        log.info("OpenID Connect support disabled. Providers not found in config")
 
     return app
-- 
GitLab


From 9d5c518f320a4229859b097869a818af4f31fd45 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 12:42:04 +0200
Subject: [PATCH 04/27] debug ci

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c63f3bb7..abbeb4f4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,7 @@ pylint36:
   # Check that every file doesn't have syntax errors
   stage: static_code_analysis
   script:
+  - printenv PYTHONPATH
   - python --version
   - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
 
-- 
GitLab


From bccdad4118499f0c8a0ab89244cb4e78b968efee Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 13:22:02 +0200
Subject: [PATCH 05/27] fix

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index abbeb4f4..f1f35ac1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,7 @@ pylint36:
   script:
   - printenv PYTHONPATH
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src"); sys.path.append("src/fts3rest")'
 
 pylint37:
   # Check that every file doesn't have syntax errors
-- 
GitLab


From ae28292f1b311fe3d964d04054c4fb9554190768 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 13:51:36 +0200
Subject: [PATCH 06/27] fix

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f1f35ac1..ed425b7a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,9 +30,9 @@ pylint36:
   # Check that every file doesn't have syntax errors
   stage: static_code_analysis
   script:
-  - printenv PYTHONPATH
+  - unset PYTHONPATH
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src"); sys.path.append("src/fts3rest")'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])'
 
 pylint37:
   # Check that every file doesn't have syntax errors
-- 
GitLab


From 263e72fc6d65b52b3a866ab098ebc6727bfda381 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 14:39:05 +0200
Subject: [PATCH 07/27] update

---
 .gitlab-ci/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile
index 05161ff0..ac3edaf9 100644
--- a/.gitlab-ci/Dockerfile
+++ b/.gitlab-ci/Dockerfile
@@ -20,7 +20,7 @@ ENV PY37 3.7.7
 ENV PY38 3.8.2
 
 # install pyenv, following instructions from https://github.com/pyenv/pyenv
-RUN git clone --depth 1 --branch master https://github.com/pyenv/pyenv.git ~/.pyenv && \
+RUN git clone --depth 1 --branch v1.2.17 https://github.com/pyenv/pyenv.git ~/.pyenv && \
 echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \
 echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \
 echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n  eval "$(pyenv init -)"\nfi' >> ~/.bashrc && \
-- 
GitLab


From a7511ceb834d09671cf83e68e90f7653581b3f29 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 15:04:09 +0200
Subject: [PATCH 08/27] fix

---
 .gitlab-ci.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ed425b7a..2f1c77a1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,8 +17,8 @@ stages:
 variables:
   MYSQL_DATABASE: ftsflask
   MYSQL_ROOT_PASSWORD: asdf
-  PYTHONPATH: ./src:./src/fts3rest
-  FTS3TESTCONFIG: /src/fts3rest/fts3rest/tests/fts3testconfig_ci
+  PYTHONPATH: /home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest
+  FTS3TESTCONFIG: ./src/fts3rest/fts3rest/tests/fts3testconfig_ci
 
 black:
   # Check that every file has been formatted with black
@@ -30,9 +30,8 @@ pylint36:
   # Check that every file doesn't have syntax errors
   stage: static_code_analysis
   script:
-  - unset PYTHONPATH
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask/src","/home/ci/fts-rest-flask/src/fts3rest"])'
 
 pylint37:
   # Check that every file doesn't have syntax errors
-- 
GitLab


From cfda2d34b32979437966f8dade5b4e99248ac5fd Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 15:08:16 +0200
Subject: [PATCH 09/27] fix

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2f1c77a1..15845808 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,7 +17,7 @@ stages:
 variables:
   MYSQL_DATABASE: ftsflask
   MYSQL_ROOT_PASSWORD: asdf
-  PYTHONPATH: /home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest
+  PYTHONPATH: /home/ci/fts-rest-flask:/home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest
   FTS3TESTCONFIG: ./src/fts3rest/fts3rest/tests/fts3testconfig_ci
 
 black:
@@ -31,7 +31,7 @@ pylint36:
   stage: static_code_analysis
   script:
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask/src","/home/ci/fts-rest-flask/src/fts3rest"])'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask","/home/ci/fts-rest-flask/src/fts3rest","/home/ci/fts-rest-flask/src"])'
 
 pylint37:
   # Check that every file doesn't have syntax errors
-- 
GitLab


From 2fe875dc39c74aa1a6f43c30159e461f72b4b61d Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 15:45:06 +0200
Subject: [PATCH 10/27] Remove CI for multiple python versions as pyenv causes
 many problems

Later we can change the image to contain multiple python versions
installed from source instead of pyev
---
 .gitlab-ci.yml        | 63 ++-----------------------------------------
 .gitlab-ci/Dockerfile | 36 +++----------------------
 2 files changed, 6 insertions(+), 93 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 15845808..f282e423 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,11 +1,3 @@
-# The jobs run for every file in src/, even if they haven't been changed.
-# Unfortunately Gitlab CI doesn't have any way to know the commit before the push.
-# I used to do this to select only the files that had been changed:
-# FILES=$(git diff --name-only --diff-filter=ACMR $CI_COMMIT_SHA~1 -- "*.py")
-# if [ -z "$FILES" ]; then exit 0; fi
-# The problem was that a push with multiple commits might pass the last commit even 
-# if the previous failed.
-
 image: gitlab-registry.cern.ch/fts/fts-rest-flask:ci
 
 stages:
@@ -17,7 +9,7 @@ stages:
 variables:
   MYSQL_DATABASE: ftsflask
   MYSQL_ROOT_PASSWORD: asdf
-  PYTHONPATH: /home/ci/fts-rest-flask:/home/ci/fts-rest-flask/src:/home/ci/fts-rest-flask/src/fts3rest
+  PYTHONPATH: ./src:./src/fts3rest
   FTS3TESTCONFIG: ./src/fts3rest/fts3rest/tests/fts3testconfig_ci
 
 black:
@@ -31,27 +23,7 @@ pylint36:
   stage: static_code_analysis
   script:
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["/home/ci/fts-rest-flask","/home/ci/fts-rest-flask/src/fts3rest","/home/ci/fts-rest-flask/src"])'
-
-pylint37:
-  # Check that every file doesn't have syntax errors
-  stage: static_code_analysis
-  script:
-  - source ~/.bashrc
-  - pyenv global $PY37
-  - source $VENV_37/bin/activate
-  - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
-
-pylint38:
-  # Check that every file doesn't have syntax errors
-  stage: static_code_analysis
-  script:
-  - source ~/.bashrc
-  - pyenv global $PY38
-  - source $VENV_38/bin/activate
-  - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["./src", "./src/fts3rest"])'
 
 
 radon:
@@ -60,8 +32,6 @@ radon:
   script:
   - source .gitlab-ci/radon.sh
 
-
-
 functional_tests36:
   stage: tests
   services:
@@ -73,35 +43,6 @@ functional_tests36:
   - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
   - coverage report
 
-functional_tests37:
-  stage: tests
-  services:
-    - name: centos/mariadb:latest
-      alias: mariadb
-  script:
-  - source ~/.bashrc
-  - pyenv global $PY37
-  - python --version
-  - source $VENV_37/bin/activate
-  - source .gitlab-ci/db.sh
-  - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
-  - coverage report
-
-functional_tests38:
-  stage: tests
-  services:
-    - name: centos/mariadb:latest
-      alias: mariadb
-  script:
-  - source ~/.bashrc
-  - pyenv global $PY38
-  - python --version
-  - source $VENV_38/bin/activate
-  - source .gitlab-ci/db.sh
-  - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
-  - coverage report
-
-
 bandit:
   # Find potential security issues in file. 
   # It's allowed to fail as it may detect false positives.
diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile
index ac3edaf9..fa20b7f2 100644
--- a/.gitlab-ci/Dockerfile
+++ b/.gitlab-ci/Dockerfile
@@ -1,9 +1,6 @@
 FROM centos:7
 
-RUN yum install python3 git python3-devel openssl-devel swig gcc gcc-c++ make mysql-devel mariadb -y && \
-yum clean all
-# install pyenv dependencies
-RUN yum install gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel tk-devel libffi-devel -y && \
+RUN yum install python3 git python3-devel openssl-devel swig gcc-c++ mysql-devel mariadb -y && \
 yum clean all
 
 RUN useradd --create-home ci
@@ -15,19 +12,6 @@ RUN chown ci /var/log/fts3rest
 WORKDIR /home/ci
 USER ci
 
-ENV PY36 3.6.10
-ENV PY37 3.7.7
-ENV PY38 3.8.2
-
-# install pyenv, following instructions from https://github.com/pyenv/pyenv
-RUN git clone --depth 1 --branch v1.2.17 https://github.com/pyenv/pyenv.git ~/.pyenv && \
-echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \
-echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \
-echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n  eval "$(pyenv init -)"\nfi' >> ~/.bashrc && \
-source ~/.bashrc && \
-pyenv install $PY36 && \
-pyenv install $PY37 && \
-pyenv install $PY38
 
 # this is needed for pip-tools
 ENV LC_ALL en_US.utf-8
@@ -38,21 +22,9 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in
 
 # Prepare virtual environments
 ENV VENV_36 /home/ci/venv_36
-ENV VENV_37 /home/ci/venv_37
-ENV VENV_38 /home/ci/venv_38
-
-RUN . ~/.bashrc && pyenv global $PY36 && python -m venv $VENV_36 && \
-. $VENV_36/bin/activate && pip install --upgrade pip && pip install pip-tools && \
-. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
-
-RUN . ~/.bashrc && pyenv global $PY37 && python -m venv $VENV_37 && \
-. $VENV_37/bin/activate && pip install --upgrade pip && pip install pip-tools && \
-. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
 
-RUN . ~/.bashrc && pyenv global $PY38 && python -m venv $VENV_38 && \
-. $VENV_38/bin/activate && pip install --upgrade pip && pip install pip-tools && \
-. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
+RUN python -m venv $VENV_36 && $VENV_36/bin/activate && \
+    pip install --upgrade pip && pip install pip-tools && \
+    . ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
 
-# by default we run on the lowest version supported
-RUN . ~/.bashrc && pyenv global $PY36
 ENV PATH="$VENV_36/bin:$PATH"
-- 
GitLab


From 76a8d0d837b33346ad7b092ba8c71f520e85f18e Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 16:41:54 +0200
Subject: [PATCH 11/27] fix

---
 .gitlab-ci/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile
index fa20b7f2..336617b0 100644
--- a/.gitlab-ci/Dockerfile
+++ b/.gitlab-ci/Dockerfile
@@ -23,7 +23,7 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in
 # Prepare virtual environments
 ENV VENV_36 /home/ci/venv_36
 
-RUN python -m venv $VENV_36 && $VENV_36/bin/activate && \
+RUN python3.6 -m venv $VENV_36 && $VENV_36/bin/activate && \
     pip install --upgrade pip && pip install pip-tools && \
     . ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
 
-- 
GitLab


From 7b1a1b201a6965025f7e2cfe0e608df8a1d79694 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 16:43:37 +0200
Subject: [PATCH 12/27] fix

---
 .gitlab-ci/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile
index 336617b0..c834d3fe 100644
--- a/.gitlab-ci/Dockerfile
+++ b/.gitlab-ci/Dockerfile
@@ -23,8 +23,8 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in
 # Prepare virtual environments
 ENV VENV_36 /home/ci/venv_36
 
-RUN python3.6 -m venv $VENV_36 && $VENV_36/bin/activate && \
+RUN python3.6 -m venv $VENV_36 && source $VENV_36/bin/activate && \
     pip install --upgrade pip && pip install pip-tools && \
-    . ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
+    source ./pipcompile.sh && source ./pipsyncdev.sh && deactivate
 
 ENV PATH="$VENV_36/bin:$PATH"
-- 
GitLab


From ad67ac6b41264e8dd8cddd355050d34a0705725e Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 17:03:36 +0200
Subject: [PATCH 13/27] fix

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f282e423..0ccdba00 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,7 +23,7 @@ pylint36:
   stage: static_code_analysis
   script:
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["./src", "./src/fts3rest"])'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path = ["src", "src/fts3rest"]'
 
 
 radon:
-- 
GitLab


From af70748f0b2e109a049b76fc14c28badb0414143 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 17:18:20 +0200
Subject: [PATCH 14/27] fix

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0ccdba00..2f0b1b03 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,8 +22,9 @@ pylint36:
   # Check that every file doesn't have syntax errors
   stage: static_code_analysis
   script:
+  - pwd
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path = ["src", "src/fts3rest"]'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])'
 
 
 radon:
-- 
GitLab


From 61a8663ee7914a35718a312f7a3fd5681a278bc3 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 17:24:49 +0200
Subject: [PATCH 15/27] fix

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2f0b1b03..59ad964e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,7 @@ pylint36:
   # Check that every file doesn't have syntax errors
   stage: static_code_analysis
   script:
-  - pwd
+  - source /home/ci/venv_36/bin/activate
   - python --version
   - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])'
 
-- 
GitLab


From 3e2f57ad21bec108bc9733ea8c4565c247c813e6 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 17:30:55 +0200
Subject: [PATCH 16/27] fix

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59ad964e..73054d35 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,6 +23,7 @@ pylint36:
   stage: static_code_analysis
   script:
   - source /home/ci/venv_36/bin/activate
+  - echo $PYTHONPATH
   - python --version
   - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])'
 
-- 
GitLab


From 10b3def3a5f18d515cbb1e12ea0c6ee9c99f8121 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 17:34:25 +0200
Subject: [PATCH 17/27] fix

---
 .gitlab-ci.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 73054d35..d0166f4d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,16 @@ pylint36:
   - source /home/ci/venv_36/bin/activate
   - echo $PYTHONPATH
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.extend(["src", "src/fts3rest"])'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")'
+
+pylint362:
+  # Check that every file doesn't have syntax errors
+  stage: static_code_analysis
+  script:
+  - source /home/ci/venv_36/bin/activate
+  - unset PYTHONPATH
+  - python --version
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")'
 
 
 radon:
-- 
GitLab


From d94bf6e2498ad4ccdb7fc5aa13fd677196ebb5af Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Wed, 6 May 2020 18:23:38 +0200
Subject: [PATCH 18/27] pin pylint to fix CI and restore multi versions

---
 .gitlab-ci.yml        | 50 ++++++++++++++++++++++++++++++++++++-------
 .gitlab-ci/Dockerfile | 36 +++++++++++++++++++++++++++----
 dev-requirements.in   |  2 +-
 3 files changed, 75 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d0166f4d..dd33f6fa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,20 +22,26 @@ pylint36:
   # Check that every file doesn't have syntax errors
   stage: static_code_analysis
   script:
-  - source /home/ci/venv_36/bin/activate
-  - echo $PYTHONPATH
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
 
-pylint362:
-  # Check that every file doesn't have syntax errors
+pylint37:
   stage: static_code_analysis
   script:
-  - source /home/ci/venv_36/bin/activate
-  - unset PYTHONPATH
+  - source ~/.bashrc
+  - pyenv global $PY37
+  - source $VENV_37/bin/activate
   - python --version
-  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy --init-hook='import sys; sys.path.append("src/fts3rest")'
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
 
+pylint38:
+  stage: static_code_analysis
+  script:
+  - source ~/.bashrc
+  - pyenv global $PY38
+  - source $VENV_38/bin/activate
+  - python --version
+  - pylint --output-format colorized --disable C,R,W src/ --ignored-modules=sqlalchemy
 
 radon:
   # Check metrics for every file
@@ -54,6 +60,34 @@ functional_tests36:
   - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
   - coverage report
 
+functional_tests37:
+  stage: tests
+  services:
+    - name: centos/mariadb:latest
+      alias: mariadb
+  script:
+  - source ~/.bashrc
+  - pyenv global $PY37
+  - python --version
+  - source $VENV_37/bin/activate
+  - source .gitlab-ci/db.sh
+  - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
+  - coverage report
+
+functional_tests38:
+  stage: tests
+  services:
+    - name: centos/mariadb:latest
+      alias: mariadb
+  script:
+  - source ~/.bashrc
+  - pyenv global $PY38
+  - python --version
+  - source $VENV_38/bin/activate
+  - source .gitlab-ci/db.sh
+  - coverage run --branch --source src/fts3rest/fts3rest/ --omit 'src/fts3rest/fts3rest/tests/*' -m pytest src/ -x
+  - coverage report
+
 bandit:
   # Find potential security issues in file. 
   # It's allowed to fail as it may detect false positives.
diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile
index c834d3fe..05161ff0 100644
--- a/.gitlab-ci/Dockerfile
+++ b/.gitlab-ci/Dockerfile
@@ -1,6 +1,9 @@
 FROM centos:7
 
-RUN yum install python3 git python3-devel openssl-devel swig gcc-c++ mysql-devel mariadb -y && \
+RUN yum install python3 git python3-devel openssl-devel swig gcc gcc-c++ make mysql-devel mariadb -y && \
+yum clean all
+# install pyenv dependencies
+RUN yum install gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel tk-devel libffi-devel -y && \
 yum clean all
 
 RUN useradd --create-home ci
@@ -12,6 +15,19 @@ RUN chown ci /var/log/fts3rest
 WORKDIR /home/ci
 USER ci
 
+ENV PY36 3.6.10
+ENV PY37 3.7.7
+ENV PY38 3.8.2
+
+# install pyenv, following instructions from https://github.com/pyenv/pyenv
+RUN git clone --depth 1 --branch master https://github.com/pyenv/pyenv.git ~/.pyenv && \
+echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \
+echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \
+echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n  eval "$(pyenv init -)"\nfi' >> ~/.bashrc && \
+source ~/.bashrc && \
+pyenv install $PY36 && \
+pyenv install $PY37 && \
+pyenv install $PY38
 
 # this is needed for pip-tools
 ENV LC_ALL en_US.utf-8
@@ -22,9 +38,21 @@ COPY --chown=ci pipcompile.sh pipsyncdev.sh dev-requirements.in requirements.in
 
 # Prepare virtual environments
 ENV VENV_36 /home/ci/venv_36
+ENV VENV_37 /home/ci/venv_37
+ENV VENV_38 /home/ci/venv_38
+
+RUN . ~/.bashrc && pyenv global $PY36 && python -m venv $VENV_36 && \
+. $VENV_36/bin/activate && pip install --upgrade pip && pip install pip-tools && \
+. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
+
+RUN . ~/.bashrc && pyenv global $PY37 && python -m venv $VENV_37 && \
+. $VENV_37/bin/activate && pip install --upgrade pip && pip install pip-tools && \
+. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
 
-RUN python3.6 -m venv $VENV_36 && source $VENV_36/bin/activate && \
-    pip install --upgrade pip && pip install pip-tools && \
-    source ./pipcompile.sh && source ./pipsyncdev.sh && deactivate
+RUN . ~/.bashrc && pyenv global $PY38 && python -m venv $VENV_38 && \
+. $VENV_38/bin/activate && pip install --upgrade pip && pip install pip-tools && \
+. ./pipcompile.sh && . ./pipsyncdev.sh && deactivate
 
+# by default we run on the lowest version supported
+RUN . ~/.bashrc && pyenv global $PY36
 ENV PATH="$VENV_36/bin:$PATH"
diff --git a/dev-requirements.in b/dev-requirements.in
index a0d19a4e..f0d6e4ee 100644
--- a/dev-requirements.in
+++ b/dev-requirements.in
@@ -1,7 +1,7 @@
 -c requirements.txt
 pip-tools
 black
-pylint
+pylint==2.4.4 # 2.5 has a bug and fails as of 02/05/20
 bandit
 radon
 pytest
-- 
GitLab


From d0845a13863ffe3b2ee8a6748fc62f56ccedaa68 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 15:06:38 +0200
Subject: [PATCH 19/27] Move config.py to fts3rest, update CI testconfig

---
 .../fts3rest/config}/config.py                |   0
 src/fts3rest/fts3rest/config/middleware.py    |   2 +-
 src/fts3rest/fts3rest/tests/fts3testconfig_ci | 163 ++----------------
 3 files changed, 13 insertions(+), 152 deletions(-)
 rename src/{fts3/util => fts3rest/fts3rest/config}/config.py (100%)

diff --git a/src/fts3/util/config.py b/src/fts3rest/fts3rest/config/config.py
similarity index 100%
rename from src/fts3/util/config.py
rename to src/fts3rest/fts3rest/config/config.py
diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py
index db5fcbc5..f9f9eb7f 100644
--- a/src/fts3rest/fts3rest/config/middleware.py
+++ b/src/fts3rest/fts3rest/config/middleware.py
@@ -8,7 +8,7 @@ from flask import Flask
 from sqlalchemy import engine_from_config, event
 from werkzeug.exceptions import HTTPException
 
-from fts3.util.config import fts3_config_load
+from fts3rest.config.config import fts3_config_load
 from fts3rest.config.routing import base, cstorage
 from fts3rest.lib.IAMTokenRefresher import IAMTokenRefresher
 from fts3rest.lib.helpers.connection_validator import (
diff --git a/src/fts3rest/fts3rest/tests/fts3testconfig_ci b/src/fts3rest/fts3rest/tests/fts3testconfig_ci
index f60a400a..ce73e7e4 100644
--- a/src/fts3rest/fts3rest/tests/fts3testconfig_ci
+++ b/src/fts3rest/fts3rest/tests/fts3testconfig_ci
@@ -1,7 +1,3 @@
-# Running user and group
-User=ci
-Group=ci
-
 # mysql only
 DbType=mysql
 
@@ -11,150 +7,27 @@ DbUserName=ci
 #db password
 DbPassword=asdf
 
-
 #For MySQL, it has to follow the format 'host/db' (i.e. "mysql-server.example.com/fts3db")
 DbConnectString=mariadb:3306/ftsflask
 
-#Number of db connections in the pool (use even number, e.g. 2,4,6,8,etc OR 1 for a single connection)
-DbThreadsNum=30
-
-#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/
-#Alias=fts3-xdc.cern.ch
-
-#Infosys, either the fqdn:port of a BDII instance or false to disable BDII access
-#Infosys=lcg-bdii.cern.ch:2170
-
-#Query the info systems specified in the order given, e.g. glue1;glue2
-InfoProviders=glue1
-
 #List of authorized VOs, separated by ;
 #Leave * to authorize any VO
 AuthorizedVO=*
 
-# site name
-#SiteName=FTS-DEV-XDC
-
-#Enable/Disable monitoring using messaging monitoring (disabled=false / enabled=true)
-MonitoringMessaging=false
-
-# Profiling interval in seconds. If set to 0, it will be disabled
-Profiling=0
-
-# Log directories
-TransferLogDirectory=/var/log/fts3/transfers
-ServerLogDirectory=/var/log/fts3
-
-# Log level. Enables logging for messages of level >= than configured
-# Possible values are
-#   TRACE (every detail), DEBUG (internal behaviour), INFO (normal behaviour),
-#   NOTICE (final states), WARNING (things worth checking), ERR (internal FTS3 errors, as database connectivity),
-#   CRIT (fatal errors, as segmentation fault)
-# It is recommended to use DEBUG or INFO
-LogLevel=DEBUG
-
-# Check for fts_url_copy processes that do not give their progress back
-# CheckStalledTransfers = true
-# Stalled timeout, in seconds
-# CheckStalledTimeout = 900
-CheckStalledTimeout = 900
-
-# Minimum required free RAM (in MB) for FTS3 to work normally
-# If the amount of free RAM goes below the limit, FTS3 will enter auto-drain mode
-# This is intended to protect against system resource exhaustion
-# MinRequiredFreeRAM = 50
-MinRequiredFreeRAM = 50
-
-# Maximum number of url copy processes that the node can run
-# The RAM limitation may not take into account other node limitations (i.e. IO)
-# or, depending on the swapping policy, may not even prevent overloads if the kernel
-# starts swapping before the free RAM decreases until it reaches the value of MinRequiredFreeRAM
-# 0 disables the check.
-# The default is 400.
-# MaxUrlCopyProcesses = 400
-MaxUrlCopyProcesses = 400
-
-# Parameters for Bring Online
-# Maximum bulk size.
-# If the size is too large, it will take more resources (memory and CPU) to generate the requests and
-# parse the responses. Some servers may reject the requests if they are too big.
-# If it is too small, performance will be reduced.
-# Keep it to a sensible size (between 100 and 1k)
-# StagingBulkSize=400
-# Maximum number of concurrent requests. This gives a maximum of files sent to the storage system
-# (StagingBulkSize*StagingConcurrentRequests). The larger the number, the more requests will FTS need to keep track of.
-# StagingConcurrentRequests=500
-# Seconds to wait before submitting a bulk request, so FTS can accumulate more files per bulk.
-# Note that the resolution is 60 seconds.
-# StagingWaitingFactor=300
-# Retry this number of times if a staging poll fails with ECOMM
-# StagingPollRetries=3
-
-# In seconds, interval between heartbeats
-# HeartBeatInterval=60
-# I seconds, after this interval a host is considered down
-# HeartBeatGraceInterval=120
-
-# Seconds between optimizer runs
-# OptimizerInterval = 60
-# After this time without optimizer updates, force a run
-# OptimizerSteadyInterval = 300
-# Maximum number of streams per file
-# OptimizerMaxStreams = 16
-
-# EMA Alpha factor to reduce the influence of fluctuations
-# OptimizerEMAAlpha = 0.1
-# Increase step size when the optimizer considers the performance is good
-# OptimizerIncreaseStep = 1
-# Increase step size when the optimizer considers the performance is good, and set to aggressive or normal
-# OptimizerAggressiveIncreaseStep = 2
-# Decrease step size when the optimizer considers the performance is bad
-# OptimizerDecreaseStep = 1
-
-
-# Set the bulk size, in number of jobs, used for cleaning the old records
-#CleanBulkSize=5000
-# In days. Entries older than this will be purged.
-#CleanInterval=7
-
-## The higher the values for the following parameters,
-## the higher the latency for some operations (as cancelations),
-## but can also reduce the system and/or database load
-
-# In seconds, how often to purge the messaging directory
-#PurgeMessagingDirectoryInterval = 600
-# In seconds, how often to run sanity checks
-#CheckSanityStateInterval = 3600
-# In seconds, how often to check for canceled transfers
-#CancelCheckInterval = 10
-# In seconds, how often to check for expired queued transfers
-#QueueTimeoutCheckInterval = 300
-# In seconds, how often to check for stalled transfers
-#ActiveTimeoutCheckInterval = 300
-# In seconds, how often to schedule new transfers
-#SchedulingInterval = 2
-# In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2
-#MessagingConsumeInterval = 1
-#Enable or disable auto session reuse
-AutoSessionReuse = true
-#Max small file size for session reuse in bytes
-AutoSessionReuseMaxSmallFileSize = 104857600
-#Max big file size for session reuse in bytes
-AutoSessionReuseMaxBigFileSize = 1073741824
-#Max number of files per session reuse
-AutoSessionReuseMaxFiles = 1000
-#Max number of big files  per session reuse
-AutoSessionReuseMaxBigFiles = 2
-BackupTables=false
-OptimizerMaxSuccessRate=100
-OptimizerMedSuccessRate=80
-OptimizerLowSuccessRate=75
-OptimizerBaseSuccessRate=74
-Port=8443
-UseFixedJobPriority=0
-
+#OpenID parameters
 ValidateAccessTokenOffline=True
 JWKCacheSeconds=86400
 TokenRefreshDaemonIntervalInSeconds=600
+
+[roles]
+Public = vo:transfer;all:datamanagement
+lcgadmin = all:config
+
+[providers]
+xdc=https://iam.extreme-datacloud.eu
+xdc_ClientId=
+xdc_ClientSecret=
+
 # Logging configuration
 [loggers]
 keys = root, routes, fts3rest, sqlalchemy
@@ -176,7 +49,7 @@ qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
 
 [logger_fts3rest]
-level = INFO
+level = DEBUG
 handlers =
 qualname = fts3rest
 
@@ -195,8 +68,6 @@ level = NOTSET
 formatter = generic
 
 [handler_log_file]
-# See
-# http://docs.python.org/2/library/logging.handlers.html
 class = logging.FileHandler
 args = ('/var/log/fts3rest/fts3rest.log', 'a')
 level = NOTSET
@@ -205,13 +76,3 @@ formatter = generic
 [formatter_generic]
 format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s
 datefmt = %H:%M:%S
-
-[roles]
-Public = vo:transfer;all:datamanagement
-lcgadmin = all:config
-
-
-[providers]
-xdc=https://iam.extreme-datacloud.eu
-xdc_ClientId=
-xdc_ClientSecret=
-- 
GitLab


From bae1307c4959a5d0a2bae593910f838f47ddd657 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 15:58:55 +0200
Subject: [PATCH 20/27] Move models to fts3resMove models to fts3rest

---
 src/fts3/model/__init__.py                    | 35 -------------------
 src/fts3rest/fts3rest/config/middleware.py    |  3 +-
 .../fts3rest/controllers/CSdropbox.py         |  4 +--
 src/fts3rest/fts3rest/controllers/api.py      |  3 +-
 src/fts3rest/fts3rest/controllers/archive.py  |  2 +-
 .../fts3rest/controllers/autocomplete.py      |  2 +-
 src/fts3rest/fts3rest/controllers/banning.py  | 12 +++++--
 .../fts3rest/controllers/config/__init__.py   |  2 +-
 .../fts3rest/controllers/config/activities.py |  2 +-
 .../fts3rest/controllers/config/audit.py      |  2 +-
 .../fts3rest/controllers/config/authz.py      |  2 +-
 .../fts3rest/controllers/config/cloud.py      |  2 +-
 .../fts3rest/controllers/config/drain.py      |  2 +-
 .../fts3rest/controllers/config/global_.py    |  2 +-
 .../fts3rest/controllers/config/links.py      |  2 +-
 .../fts3rest/controllers/config/se.py         |  2 +-
 .../fts3rest/controllers/config/shares.py     |  2 +-
 .../fts3rest/controllers/datamanagement.py    |  2 +-
 .../fts3rest/controllers/delegation.py        |  2 +-
 src/fts3rest/fts3rest/controllers/files.py    |  4 +--
 src/fts3rest/fts3rest/controllers/jobs.py     |  6 ++--
 .../fts3rest/controllers/optimizer.py         |  6 +---
 .../fts3rest/controllers/serverstatus.py      |  2 --
 .../fts3rest/lib/IAMTokenRefresher.py         |  2 +-
 src/fts3rest/fts3rest/lib/JobBuilder_utils.py |  2 +-
 src/fts3rest/fts3rest/lib/api/introspect.py   |  6 ++--
 src/fts3rest/fts3rest/lib/helpers/jsonify.py  |  3 +-
 .../lib/middleware/fts3auth/credentials.py    |  2 +-
 .../middleware/fts3auth/fts3authmiddleware.py |  2 +-
 src/fts3rest/fts3rest/lib/oauth2provider.py   |  4 +--
 src/fts3rest/fts3rest/lib/scheduler/db.py     |  6 ++--
 src/fts3rest/fts3rest/model/__init__.py       | 20 ++++++-----
 .../fts3rest}/model/banned.py                 |  0
 src/{fts3 => fts3rest/fts3rest}/model/base.py |  0
 .../fts3rest}/model/cloudStorage.py           |  0
 .../fts3rest}/model/config.py                 |  0
 .../fts3rest}/model/credentials.py            |  0
 src/{fts3 => fts3rest/fts3rest}/model/dm.py   |  0
 src/{fts3 => fts3rest/fts3rest}/model/file.py |  0
 src/{fts3 => fts3rest/fts3rest}/model/job.py  |  0
 src/fts3rest/fts3rest/model/meta.py           |  7 ++--
 .../fts3rest}/model/oauth2.py                 |  0
 .../fts3rest}/model/optimizer.py              |  0
 .../fts3rest}/model/server.py                 |  0
 .../fts3rest}/model/version.py                |  0
 src/fts3rest/fts3rest/tests/__init__.py       |  2 +-
 .../fts3rest/tests/functional/test_banning.py |  2 +-
 .../functional/test_config_activity_shares.py |  2 +-
 .../tests/functional/test_config_authz_dn.py  |  2 +-
 .../tests/functional/test_config_cloud.py     |  2 +-
 .../tests/functional/test_config_global.py    |  2 +-
 .../tests/functional/test_config_links.py     |  2 +-
 .../tests/functional/test_config_se.py        |  2 +-
 .../tests/functional/test_config_shares.py    |  2 +-
 .../tests/functional/test_delegation.py       |  2 +-
 .../fts3rest/tests/functional/test_drain.py   |  2 +-
 .../fts3rest/tests/functional/test_dropbox.py |  2 +-
 .../tests/functional/test_job_cancel.py       |  2 +-
 .../tests/functional/test_job_deletion.py     |  2 +-
 .../tests/functional/test_job_listing.py      |  2 +-
 .../tests/functional/test_job_modify.py       |  2 +-
 .../tests/functional/test_job_submission.py   |  3 +-
 .../tests/functional/test_multiple.py         |  2 +-
 .../tests/functional/test_openidconnect.py    |  2 +-
 .../tests/functional/test_optimizer.py        |  2 +-
 .../tests/functional/test_scheduler.py        |  2 +-
 .../fts3rest/tests/functional/test_staging.py |  2 +-
 67 files changed, 84 insertions(+), 118 deletions(-)
 delete mode 100644 src/fts3/model/__init__.py
 rename src/{fts3 => fts3rest/fts3rest}/model/banned.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/base.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/cloudStorage.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/config.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/credentials.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/dm.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/file.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/job.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/oauth2.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/optimizer.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/server.py (100%)
 rename src/{fts3 => fts3rest/fts3rest}/model/version.py (100%)

diff --git a/src/fts3/model/__init__.py b/src/fts3/model/__init__.py
deleted file mode 100644
index 76dcc79e..00000000
--- a/src/fts3/model/__init__.py
+++ /dev/null
@@ -1,35 +0,0 @@
-#   Copyright  Members of the EMI Collaboration, 2013.
-#   Copyright 2020 CERN
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-import sqlalchemy
-from .base import Base
-from .banned import *
-from .cloudStorage import *
-from .config import *
-from .credentials import *
-from .dm import *
-from .file import *
-from .job import *
-from .oauth2 import *
-from .optimizer import *
-from .server import *
-from .version import *
-
-
-# Convenience method
-def connect(connectString):
-    engine = sqlalchemy.create_engine(connectString, isolation_level="READ-COMMITTED")
-    Session = sqlalchemy.orm.sessionmaker(bind=engine)
-    return Session()
diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py
index f9f9eb7f..4dc60197 100644
--- a/src/fts3rest/fts3rest/config/middleware.py
+++ b/src/fts3rest/fts3rest/config/middleware.py
@@ -18,8 +18,7 @@ from fts3rest.lib.helpers.connection_validator import (
 from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddleware
 from fts3rest.lib.middleware.timeout import TimeoutHandler
 from fts3rest.lib.openidconnect import oidc_manager
-from fts3rest.model import init_model
-from fts3rest.model.meta import Session
+from fts3rest.model.meta import init_model, Session
 
 
 def _load_configuration(config_file):
diff --git a/src/fts3rest/fts3rest/controllers/CSdropbox.py b/src/fts3rest/fts3rest/controllers/CSdropbox.py
index e08d1975..989e0f88 100644
--- a/src/fts3rest/fts3rest/controllers/CSdropbox.py
+++ b/src/fts3rest/fts3rest/controllers/CSdropbox.py
@@ -18,11 +18,11 @@ from werkzeug.exceptions import NotFound, BadRequest, Forbidden
 from flask import request
 import urllib
 from urllib.parse import urlparse, urlencode
-from urllib.request import urlopen, Request
+from urllib.request import urlopen
 from urllib.error import HTTPError
 from fts3rest.lib.helpers.jsonify import jsonify
 from fts3rest.model.meta import Session
-from fts3.model import CloudStorage, CloudStorageUser
+from fts3rest.model import CloudStorage, CloudStorageUser
 from fts3rest.controllers.CSInterface import Connector
 
 dropboxEndpoint = "https://www.dropbox.com"
diff --git a/src/fts3rest/fts3rest/controllers/api.py b/src/fts3rest/fts3rest/controllers/api.py
index ff31b8af..d81db395 100644
--- a/src/fts3rest/fts3rest/controllers/api.py
+++ b/src/fts3rest/fts3rest/controllers/api.py
@@ -16,14 +16,13 @@
 import glob
 
 from flask.views import View
-from fts3.model import SchemaVersion
+from fts3rest.model import SchemaVersion
 
 
 from fts3rest.model.meta import Session
 
 
 from fts3rest.lib.api.submit_schema import SubmitSchema
-from fts3rest.lib.api.introspect import introspect
 from werkzeug.exceptions import NotFound
 from fts3rest.lib.helpers.jsonify import jsonify
 
diff --git a/src/fts3rest/fts3rest/controllers/archive.py b/src/fts3rest/fts3rest/controllers/archive.py
index f6958c7d..68c179db 100644
--- a/src/fts3rest/fts3rest/controllers/archive.py
+++ b/src/fts3rest/fts3rest/controllers/archive.py
@@ -16,7 +16,7 @@
 from werkzeug.exceptions import NotFound, Forbidden
 from fts3rest.lib.helpers.jsonify import jsonify
 
-from fts3.model import ArchivedJob
+from fts3rest.model import ArchivedJob
 from fts3rest.model.meta import Session
 from fts3rest.lib.middleware.fts3auth.authorization import authorized
 from fts3rest.lib.middleware.fts3auth.constants import *
diff --git a/src/fts3rest/fts3rest/controllers/autocomplete.py b/src/fts3rest/fts3rest/controllers/autocomplete.py
index 51c9c901..7763221b 100644
--- a/src/fts3rest/fts3rest/controllers/autocomplete.py
+++ b/src/fts3rest/fts3rest/controllers/autocomplete.py
@@ -13,7 +13,7 @@
 #   limitations under the License.
 
 
-from fts3.model import Credential, LinkConfig, Job
+from fts3rest.model import Credential, LinkConfig, Job
 
 from fts3rest.model.meta import Session
 from flask import request
diff --git a/src/fts3rest/fts3rest/controllers/banning.py b/src/fts3rest/fts3rest/controllers/banning.py
index b0cab26c..46fda523 100644
--- a/src/fts3rest/fts3rest/controllers/banning.py
+++ b/src/fts3rest/fts3rest/controllers/banning.py
@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from werkzeug.exceptions import NotFound, BadRequest, Conflict
+from werkzeug.exceptions import BadRequest, Conflict
 from fts3rest.controllers.config import audit_configuration
 from flask import request, Response
 import json
@@ -19,9 +19,15 @@ import logging
 from datetime import datetime
 from sqlalchemy import distinct, func, and_
 
-from fts3.model import BannedDN, BannedSE, Job, File, JobActiveStates, FileActiveStates
+from fts3rest.model import (
+    BannedDN,
+    BannedSE,
+    Job,
+    File,
+    JobActiveStates,
+    FileActiveStates,
+)
 from fts3rest.model.meta import Session
-from fts3rest.lib.http_exceptions import *
 from fts3rest.lib.middleware.fts3auth.authorization import authorize
 from fts3rest.lib.middleware.fts3auth.constants import *
 from fts3rest.lib.helpers.jsonify import jsonify
diff --git a/src/fts3rest/fts3rest/controllers/config/__init__.py b/src/fts3rest/fts3rest/controllers/config/__init__.py
index 4e984f2c..7e5ed616 100644
--- a/src/fts3rest/fts3rest/controllers/config/__init__.py
+++ b/src/fts3rest/fts3rest/controllers/config/__init__.py
@@ -19,7 +19,7 @@ from numbers import Number
 from flask import request
 from werkzeug.exceptions import BadRequest
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.middleware.fts3auth.authorization import authorize
 from fts3rest.lib.middleware.fts3auth.constants import CONFIG
diff --git a/src/fts3rest/fts3rest/controllers/config/activities.py b/src/fts3rest/fts3rest/controllers/config/activities.py
index 9e5093ec..3b293d32 100644
--- a/src/fts3rest/fts3rest/controllers/config/activities.py
+++ b/src/fts3rest/fts3rest/controllers/config/activities.py
@@ -18,7 +18,7 @@ import logging
 from flask import request, Response
 from werkzeug.exceptions import BadRequest, NotFound
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.controllers.config import audit_configuration
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.helpers.jsonify import jsonify
diff --git a/src/fts3rest/fts3rest/controllers/config/audit.py b/src/fts3rest/fts3rest/controllers/config/audit.py
index fd70bc78..53b4a85a 100644
--- a/src/fts3rest/fts3rest/controllers/config/audit.py
+++ b/src/fts3rest/fts3rest/controllers/config/audit.py
@@ -15,7 +15,7 @@
 
 import logging
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.middleware.fts3auth.authorization import authorize
 from fts3rest.lib.middleware.fts3auth.constants import *
diff --git a/src/fts3rest/fts3rest/controllers/config/authz.py b/src/fts3rest/fts3rest/controllers/config/authz.py
index eb5c28f5..e7756fbf 100644
--- a/src/fts3rest/fts3rest/controllers/config/authz.py
+++ b/src/fts3rest/fts3rest/controllers/config/authz.py
@@ -17,7 +17,7 @@ import logging
 from flask import request, Response
 from werkzeug.exceptions import BadRequest
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.controllers.config import audit_configuration
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.helpers.jsonify import jsonify
diff --git a/src/fts3rest/fts3rest/controllers/config/cloud.py b/src/fts3rest/fts3rest/controllers/config/cloud.py
index a63d79c8..87a8d30b 100644
--- a/src/fts3rest/fts3rest/controllers/config/cloud.py
+++ b/src/fts3rest/fts3rest/controllers/config/cloud.py
@@ -17,7 +17,7 @@ import logging
 from flask import request, Response
 from werkzeug.exceptions import BadRequest, NotFound
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.helpers.jsonify import jsonify
 from fts3rest.lib.helpers.misc import get_input_as_dict
diff --git a/src/fts3rest/fts3rest/controllers/config/drain.py b/src/fts3rest/fts3rest/controllers/config/drain.py
index 1bb882b4..3cea5fc5 100644
--- a/src/fts3rest/fts3rest/controllers/config/drain.py
+++ b/src/fts3rest/fts3rest/controllers/config/drain.py
@@ -17,7 +17,7 @@ import logging
 from flask import request
 from werkzeug.exceptions import BadRequest
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.controllers.config import audit_configuration
 from fts3rest.lib.helpers.jsonify import jsonify
 from fts3rest.lib.helpers.misc import get_input_as_dict
diff --git a/src/fts3rest/fts3rest/controllers/config/global_.py b/src/fts3rest/fts3rest/controllers/config/global_.py
index 62b9daa1..92e26421 100644
--- a/src/fts3rest/fts3rest/controllers/config/global_.py
+++ b/src/fts3rest/fts3rest/controllers/config/global_.py
@@ -17,7 +17,7 @@ import logging
 from flask import request, Response
 from werkzeug.exceptions import BadRequest
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.controllers.config import audit_configuration, validate_type
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.helpers.jsonify import jsonify, to_json
diff --git a/src/fts3rest/fts3rest/controllers/config/links.py b/src/fts3rest/fts3rest/controllers/config/links.py
index d506d9b8..804141df 100644
--- a/src/fts3rest/fts3rest/controllers/config/links.py
+++ b/src/fts3rest/fts3rest/controllers/config/links.py
@@ -19,7 +19,7 @@ from urllib.parse import unquote
 from flask import request, Response
 from werkzeug.exceptions import BadRequest, NotFound
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.controllers.config import audit_configuration
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.helpers.jsonify import jsonify
diff --git a/src/fts3rest/fts3rest/controllers/config/se.py b/src/fts3rest/fts3rest/controllers/config/se.py
index b55b2779..91e071c9 100644
--- a/src/fts3rest/fts3rest/controllers/config/se.py
+++ b/src/fts3rest/fts3rest/controllers/config/se.py
@@ -19,7 +19,7 @@ from flask import Response
 from flask import request
 from werkzeug.exceptions import BadRequest
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.controllers.config import audit_configuration
 from fts3rest.lib.helpers.accept import accept
 from fts3rest.lib.helpers.jsonify import jsonify
diff --git a/src/fts3rest/fts3rest/controllers/config/shares.py b/src/fts3rest/fts3rest/controllers/config/shares.py
index ae1e9fc8..4736bc42 100644
--- a/src/fts3rest/fts3rest/controllers/config/shares.py
+++ b/src/fts3rest/fts3rest/controllers/config/shares.py
@@ -18,7 +18,7 @@ from urllib.parse import urlparse
 from flask import request, Response
 from werkzeug.exceptions import BadRequest
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.controllers.config import audit_configuration
 from fts3rest.lib.helpers.jsonify import jsonify
 from fts3rest.lib.helpers.misc import get_input_as_dict
diff --git a/src/fts3rest/fts3rest/controllers/datamanagement.py b/src/fts3rest/fts3rest/controllers/datamanagement.py
index 96e270a5..cf96f6b8 100644
--- a/src/fts3rest/fts3rest/controllers/datamanagement.py
+++ b/src/fts3rest/fts3rest/controllers/datamanagement.py
@@ -33,7 +33,7 @@ from urllib.parse import urlparse, unquote_plus
 
 import json
 
-from fts3.model import Credential
+from fts3rest.model import Credential
 from fts3rest.model.meta import Session
 from fts3rest.lib.http_exceptions import HTTPAuthenticationTimeout
 from fts3rest.lib.gfal2_wrapper import Gfal2Wrapper, Gfal2Error
diff --git a/src/fts3rest/fts3rest/controllers/delegation.py b/src/fts3rest/fts3rest/controllers/delegation.py
index 3ff251df..0433ab79 100644
--- a/src/fts3rest/fts3rest/controllers/delegation.py
+++ b/src/fts3rest/fts3rest/controllers/delegation.py
@@ -25,7 +25,7 @@ import flask
 from flask import Response
 from flask import current_app as app
 from flask.views import View
-from fts3.model import CredentialCache, Credential
+from fts3rest.model import CredentialCache, Credential
 from fts3rest.model.meta import Session
 from fts3rest.lib.helpers.voms import VomsClient, VomsException
 from fts3rest.lib.middleware.fts3auth.authorization import require_certificate
diff --git a/src/fts3rest/fts3rest/controllers/files.py b/src/fts3rest/fts3rest/controllers/files.py
index f891cecc..93941bad 100644
--- a/src/fts3rest/fts3rest/controllers/files.py
+++ b/src/fts3rest/fts3rest/controllers/files.py
@@ -16,15 +16,13 @@ from werkzeug.exceptions import Forbidden
 from datetime import datetime, timedelta
 from flask import request
 from urllib.parse import urlparse
-import json
 import logging
 
-from fts3.model import File
+from fts3rest.model import File
 from fts3rest.model.meta import Session
 from fts3rest.lib.JobBuilder_utils import get_storage_element
 from fts3rest.lib.middleware.fts3auth.authorization import authorize
 from fts3rest.lib.middleware.fts3auth.constants import *
-from fts3rest.lib.http_exceptions import *
 from fts3rest.lib.helpers.jsonify import jsonify
 
 log = logging.getLogger(__name__)
diff --git a/src/fts3rest/fts3rest/controllers/jobs.py b/src/fts3rest/fts3rest/controllers/jobs.py
index 09e480fa..5edf16d9 100644
--- a/src/fts3rest/fts3rest/controllers/jobs.py
+++ b/src/fts3rest/fts3rest/controllers/jobs.py
@@ -22,9 +22,9 @@ from sqlalchemy.orm import noload
 
 import logging
 
-from fts3.model import Job, File, JobActiveStates, FileActiveStates
-from fts3.model import DataManagement, DataManagementActiveStates
-from fts3.model import Credential, FileRetryLog
+from fts3rest.model import Job, File, JobActiveStates, FileActiveStates
+from fts3rest.model import DataManagement, DataManagementActiveStates
+from fts3rest.model import Credential, FileRetryLog
 from fts3rest.model.meta import Session
 
 from fts3rest.lib.http_exceptions import *
diff --git a/src/fts3rest/fts3rest/controllers/optimizer.py b/src/fts3rest/fts3rest/controllers/optimizer.py
index 4bd97391..65a528fd 100644
--- a/src/fts3rest/fts3rest/controllers/optimizer.py
+++ b/src/fts3rest/fts3rest/controllers/optimizer.py
@@ -18,15 +18,11 @@ Optimizer logging tables
 
 from werkzeug.exceptions import BadRequest
 
-
-import json
-import logging
 from flask import request, current_app as app
 from fts3rest.model.meta import Session
 from fts3rest.lib.helpers.misc import get_input_as_dict
-from fts3.model import OptimizerEvolution, Optimizer
+from fts3rest.model import OptimizerEvolution, Optimizer
 from datetime import datetime
-from fts3rest.lib.http_exceptions import *
 from fts3rest.lib.helpers.jsonify import jsonify
 
 
diff --git a/src/fts3rest/fts3rest/controllers/serverstatus.py b/src/fts3rest/fts3rest/controllers/serverstatus.py
index 5c087d74..91466bd1 100644
--- a/src/fts3rest/fts3rest/controllers/serverstatus.py
+++ b/src/fts3rest/fts3rest/controllers/serverstatus.py
@@ -11,8 +11,6 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from werkzeug.exceptions import NotFound
-from fts3.model import File
 from fts3rest.model.meta import Session
 from fts3rest.lib.middleware.fts3auth.authorization import (
     authorize,
diff --git a/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py
index 49220094..8c19a6a7 100644
--- a/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py
+++ b/src/fts3rest/fts3rest/lib/IAMTokenRefresher.py
@@ -22,7 +22,7 @@ from threading import Thread, current_thread
 
 from fts3rest.model.meta import Session
 from fts3rest.lib.openidconnect import oidc_manager
-from fts3.model import Credential, Host
+from fts3rest.model import Credential, Host
 
 from sqlalchemy.exc import SQLAlchemyError
 
diff --git a/src/fts3rest/fts3rest/lib/JobBuilder_utils.py b/src/fts3rest/fts3rest/lib/JobBuilder_utils.py
index 6580561f..b900008b 100644
--- a/src/fts3rest/fts3rest/lib/JobBuilder_utils.py
+++ b/src/fts3rest/fts3rest/lib/JobBuilder_utils.py
@@ -9,7 +9,7 @@ import uuid
 from flask import current_app as app
 from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError
 
-from fts3.model import BannedSE
+from fts3rest.model import BannedSE
 from fts3rest.model.meta import Session
 
 from fts3rest.lib.scheduler.schd import Scheduler
diff --git a/src/fts3rest/fts3rest/lib/api/introspect.py b/src/fts3rest/fts3rest/lib/api/introspect.py
index b83b9df9..fdb3ed5c 100755
--- a/src/fts3rest/fts3rest/lib/api/introspect.py
+++ b/src/fts3rest/fts3rest/lib/api/introspect.py
@@ -24,8 +24,8 @@ import logging
 import imp
 import itertools
 import os
-import fts3.model
-from fts3.model.base import Flag, TernaryFlag, Json
+import fts3rest.model
+from fts3rest.model.base import Flag, TernaryFlag, Json
 from fts3rest.config import routing
 from sqlalchemy import types
 from sqlalchemy.orm import Mapper
@@ -86,7 +86,7 @@ def get_model_fields(model_name, model_list):
     Get a description of the fields of the model 'model_name'
     Injects into model_list the additional types that may be needed
     """
-    model = getattr(fts3.model, model_name, None)
+    model = getattr(fts3rest.model, model_name, None)
     if not model:
         return dict()
     fields = dict()
diff --git a/src/fts3rest/fts3rest/lib/helpers/jsonify.py b/src/fts3rest/fts3rest/lib/helpers/jsonify.py
index 6cd9c596..711ca039 100644
--- a/src/fts3rest/fts3rest/lib/helpers/jsonify.py
+++ b/src/fts3rest/fts3rest/lib/helpers/jsonify.py
@@ -14,8 +14,7 @@
 #   limitations under the License.
 
 from datetime import datetime
-from fts3.model.base import Base
-from sqlalchemy.orm.query import Query
+from fts3rest.model.base import Base
 import json
 import logging
 import types
diff --git a/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py b/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py
index a40e8b3c..0585d58f 100644
--- a/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py
+++ b/src/fts3rest/fts3rest/lib/middleware/fts3auth/credentials.py
@@ -18,7 +18,7 @@ import hashlib
 import logging
 import re
 
-from fts3.model import AuthorizationByDn
+from fts3rest.model import AuthorizationByDn
 from fts3rest.model.meta import Session
 from .methods import Authenticator
 
diff --git a/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py b/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py
index c4f42bc6..a4f04815 100644
--- a/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py
+++ b/src/fts3rest/fts3rest/lib/middleware/fts3auth/fts3authmiddleware.py
@@ -16,7 +16,7 @@
 import logging
 
 from fts3rest.model.meta import Session
-from fts3.model import BannedDN
+from fts3rest.model import BannedDN
 from .credentials import UserCredentials, InvalidCredentials
 from sqlalchemy.exc import DatabaseError
 from urllib.parse import urlparse
diff --git a/src/fts3rest/fts3rest/lib/oauth2provider.py b/src/fts3rest/fts3rest/lib/oauth2provider.py
index f22b0784..cd84249f 100644
--- a/src/fts3rest/fts3rest/lib/oauth2provider.py
+++ b/src/fts3rest/fts3rest/lib/oauth2provider.py
@@ -28,8 +28,8 @@ from fts3rest.lib.openidconnect import oidc_manager
 from jwcrypto.jwk import JWK
 from flask import request
 
-from fts3.model.credentials import Credential, CredentialCache
-from fts3.model.oauth2 import OAuth2Application, OAuth2Code, OAuth2Token
+from fts3rest.model import Credential, CredentialCache
+from fts3rest.model.oauth2 import OAuth2Application, OAuth2Code, OAuth2Token
 
 log = logging.getLogger(__name__)
 
diff --git a/src/fts3rest/fts3rest/lib/scheduler/db.py b/src/fts3rest/fts3rest/lib/scheduler/db.py
index 357cfeea..2bb385a7 100644
--- a/src/fts3rest/fts3rest/lib/scheduler/db.py
+++ b/src/fts3rest/fts3rest/lib/scheduler/db.py
@@ -1,9 +1,9 @@
 import json
 import logging
 
-from fts3.model import File
-from fts3.model import OptimizerEvolution
-from fts3.model import ActivityShare
+from fts3rest.model import File
+from fts3rest.model import OptimizerEvolution
+from fts3rest.model import ActivityShare
 
 from sqlalchemy import func
 
diff --git a/src/fts3rest/fts3rest/model/__init__.py b/src/fts3rest/fts3rest/model/__init__.py
index 5c0af258..af384e43 100644
--- a/src/fts3rest/fts3rest/model/__init__.py
+++ b/src/fts3rest/fts3rest/model/__init__.py
@@ -12,11 +12,15 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-
-"""The application's model objects"""
-from fts3rest.model.meta import Session, Base
-
-
-def init_model(engine):
-    """Call me before using any of the tables or classes in the model"""
-    Session.configure(bind=engine)
+from .base import Base
+from .banned import *
+from .cloudStorage import *
+from .config import *
+from .credentials import *
+from .dm import *
+from .file import *
+from .job import *
+from .oauth2 import *
+from .optimizer import *
+from .server import *
+from .version import *
diff --git a/src/fts3/model/banned.py b/src/fts3rest/fts3rest/model/banned.py
similarity index 100%
rename from src/fts3/model/banned.py
rename to src/fts3rest/fts3rest/model/banned.py
diff --git a/src/fts3/model/base.py b/src/fts3rest/fts3rest/model/base.py
similarity index 100%
rename from src/fts3/model/base.py
rename to src/fts3rest/fts3rest/model/base.py
diff --git a/src/fts3/model/cloudStorage.py b/src/fts3rest/fts3rest/model/cloudStorage.py
similarity index 100%
rename from src/fts3/model/cloudStorage.py
rename to src/fts3rest/fts3rest/model/cloudStorage.py
diff --git a/src/fts3/model/config.py b/src/fts3rest/fts3rest/model/config.py
similarity index 100%
rename from src/fts3/model/config.py
rename to src/fts3rest/fts3rest/model/config.py
diff --git a/src/fts3/model/credentials.py b/src/fts3rest/fts3rest/model/credentials.py
similarity index 100%
rename from src/fts3/model/credentials.py
rename to src/fts3rest/fts3rest/model/credentials.py
diff --git a/src/fts3/model/dm.py b/src/fts3rest/fts3rest/model/dm.py
similarity index 100%
rename from src/fts3/model/dm.py
rename to src/fts3rest/fts3rest/model/dm.py
diff --git a/src/fts3/model/file.py b/src/fts3rest/fts3rest/model/file.py
similarity index 100%
rename from src/fts3/model/file.py
rename to src/fts3rest/fts3rest/model/file.py
diff --git a/src/fts3/model/job.py b/src/fts3rest/fts3rest/model/job.py
similarity index 100%
rename from src/fts3/model/job.py
rename to src/fts3rest/fts3rest/model/job.py
diff --git a/src/fts3rest/fts3rest/model/meta.py b/src/fts3rest/fts3rest/model/meta.py
index ea55c07b..c06e9ed5 100644
--- a/src/fts3rest/fts3rest/model/meta.py
+++ b/src/fts3rest/fts3rest/model/meta.py
@@ -19,10 +19,13 @@ import logging
 import sqlalchemy
 from sqlalchemy.orm import scoped_session, sessionmaker
 
-from fts3.model.base import Base  # The declarative Base
-
 # SQLAlchemy session manager. Updated by model.init_model()
 Session = scoped_session(sessionmaker())
 
 # Log the version
 logging.getLogger(__name__).info("Using SQLAlchemy %s" % sqlalchemy.__version__)
+
+
+def init_model(engine):
+    """Call me before using any of the tables or classes in the model"""
+    Session.configure(bind=engine)
diff --git a/src/fts3/model/oauth2.py b/src/fts3rest/fts3rest/model/oauth2.py
similarity index 100%
rename from src/fts3/model/oauth2.py
rename to src/fts3rest/fts3rest/model/oauth2.py
diff --git a/src/fts3/model/optimizer.py b/src/fts3rest/fts3rest/model/optimizer.py
similarity index 100%
rename from src/fts3/model/optimizer.py
rename to src/fts3rest/fts3rest/model/optimizer.py
diff --git a/src/fts3/model/server.py b/src/fts3rest/fts3rest/model/server.py
similarity index 100%
rename from src/fts3/model/server.py
rename to src/fts3rest/fts3rest/model/server.py
diff --git a/src/fts3/model/version.py b/src/fts3rest/fts3rest/model/version.py
similarity index 100%
rename from src/fts3/model/version.py
rename to src/fts3rest/fts3rest/model/version.py
diff --git a/src/fts3rest/fts3rest/tests/__init__.py b/src/fts3rest/fts3rest/tests/__init__.py
index 466373d2..fe9249ee 100644
--- a/src/fts3rest/fts3rest/tests/__init__.py
+++ b/src/fts3rest/fts3rest/tests/__init__.py
@@ -9,7 +9,7 @@ from unittest import TestCase
 from M2Crypto import ASN1, X509, RSA, EVP
 from M2Crypto.ASN1 import UTC
 
-from fts3.model import *
+from fts3rest.model import *
 from fts3rest.config.middleware import create_app
 from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials
 from fts3rest.model.meta import Session
diff --git a/src/fts3rest/fts3rest/tests/functional/test_banning.py b/src/fts3rest/fts3rest/tests/functional/test_banning.py
index fe1c6d0b..7ff3f82a 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_banning.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_banning.py
@@ -1,7 +1,7 @@
 import json
 
 from urllib.parse import quote
-from fts3.model import BannedDN, BannedSE, Job, File
+from fts3rest.model import BannedDN, BannedSE, Job, File
 from fts3rest.model.meta import Session
 from fts3rest.tests import TestController
 
diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py b/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py
index 2af62820..a3bf56b8 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_config_activity_shares.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import ConfigAudit, ActivityShare
+from fts3rest.model import ConfigAudit, ActivityShare
 
 
 class TestConfigActivityShare(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py b/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py
index 521f9dee..5abf160f 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_config_authz_dn.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import ConfigAudit, AuthorizationByDn
+from fts3rest.model import ConfigAudit, AuthorizationByDn
 
 
 class TestConfigAuthz(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py
index 132404df..eb646bb3 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import CloudStorage, CloudStorageUser
+from fts3rest.model import CloudStorage, CloudStorageUser
 
 
 class TestConfigCloud(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_global.py b/src/fts3rest/fts3rest/tests/functional/test_config_global.py
index 450cfce6..b8b3a091 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_config_global.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_config_global.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model.config import ConfigAudit, ServerConfig
+from fts3rest.model import ConfigAudit, ServerConfig
 
 
 class TestConfigGlobal(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_links.py b/src/fts3rest/fts3rest/tests/functional/test_config_links.py
index 87860caa..1dd9eff0 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_config_links.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_config_links.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import ConfigAudit, LinkConfig
+from fts3rest.model import ConfigAudit, LinkConfig
 
 
 class TestConfigLinks(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_se.py b/src/fts3rest/fts3rest/tests/functional/test_config_se.py
index 6ca52993..b1dcbbd4 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_config_se.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_config_se.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import ConfigAudit, Optimizer, OperationConfig, Se
+from fts3rest.model import ConfigAudit, Optimizer, OperationConfig, Se
 
 
 class TestConfigSe(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_shares.py b/src/fts3rest/fts3rest/tests/functional/test_config_shares.py
index 3081c3b4..1e5dd8bc 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_config_shares.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_config_shares.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model.config import (
+from fts3rest.model import (
     ConfigAudit,
     ServerConfig,
     OperationConfig,
diff --git a/src/fts3rest/fts3rest/tests/functional/test_delegation.py b/src/fts3rest/fts3rest/tests/functional/test_delegation.py
index 8dae6d96..31f4f625 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_delegation.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_delegation.py
@@ -5,7 +5,7 @@ import time
 from fts3rest.controllers.delegation import _generate_proxy_request
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import Credential, CredentialCache
+from fts3rest.model import Credential, CredentialCache
 
 
 class TestDelegation(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_drain.py b/src/fts3rest/fts3rest/tests/functional/test_drain.py
index 8324c7fe..a86855c6 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_drain.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_drain.py
@@ -2,7 +2,7 @@ from datetime import datetime
 
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import ConfigAudit, Host
+from fts3rest.model import ConfigAudit, Host
 
 
 class TestDrain(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_dropbox.py b/src/fts3rest/fts3rest/tests/functional/test_dropbox.py
index 502186c4..39932d87 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_dropbox.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_dropbox.py
@@ -3,7 +3,7 @@ from unittest.mock import patch
 from fts3rest.model.meta import Session
 from fts3rest.tests import TestController
 from fts3rest.controllers.CSdropbox import DropboxConnector
-from fts3.model import CloudStorage, CloudStorageUser
+from fts3rest.model import CloudStorage, CloudStorageUser
 
 
 def _oauth_header_dict(raw_header):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py b/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py
index 5ab52a13..5067615d 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_job_cancel.py
@@ -2,7 +2,7 @@ import json
 
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import (
+from fts3rest.model import (
     Job,
     File,
     JobActiveStates,
diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py b/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py
index 6354216b..b072e1a6 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_job_deletion.py
@@ -2,7 +2,7 @@ import json
 
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import Job, DataManagement
+from fts3rest.model import Job, DataManagement
 
 
 class TestJobDeletion(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_listing.py b/src/fts3rest/fts3rest/tests/functional/test_job_listing.py
index ff3824f8..c4f691a0 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_job_listing.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_job_listing.py
@@ -1,7 +1,7 @@
 import json
 from datetime import datetime, timedelta
 
-from fts3.model import FileRetryLog, Job, File
+from fts3rest.model import FileRetryLog, Job, File
 from fts3rest.model.meta import Session
 from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials
 from fts3rest.lib.middleware.fts3auth import constants
diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_modify.py b/src/fts3rest/fts3rest/tests/functional/test_job_modify.py
index 449eebd0..cde2229c 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_job_modify.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_job_modify.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import Job
+from fts3rest.model import Job
 
 
 class TestJobModify(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py
index 6dc3829c..d882c346 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py
@@ -4,9 +4,8 @@ import time
 
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import File, Job
+from fts3rest.model import Job
 import random
-import unittest
 from math import ceil
 
 
diff --git a/src/fts3rest/fts3rest/tests/functional/test_multiple.py b/src/fts3rest/fts3rest/tests/functional/test_multiple.py
index 415a31c8..81f0ce03 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_multiple.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_multiple.py
@@ -2,7 +2,7 @@ import json
 
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import Job, File
+from fts3rest.model import Job, File
 
 
 class TestMultiple(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
index 74131e92..955ccc88 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
@@ -1,4 +1,4 @@
-from fts3.model import Credential
+from fts3rest.model import Credential
 from fts3rest.lib.openidconnect import OIDCmanager
 from fts3rest.tests import TestController
 
diff --git a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py
index 66c5ce01..05625c4a 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py
@@ -1,6 +1,6 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import Optimizer
+from fts3rest.model import Optimizer
 
 
 class TestOptimizer(TestController):
diff --git a/src/fts3rest/fts3rest/tests/functional/test_scheduler.py b/src/fts3rest/fts3rest/tests/functional/test_scheduler.py
index 0328fb9c..f98f371c 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_scheduler.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_scheduler.py
@@ -5,7 +5,7 @@ import logging
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
 from fts3rest.lib.scheduler.Cache import ThreadLocalCache
-from fts3.model import Job, File, OptimizerEvolution, ActivityShare
+from fts3rest.model import Job, File, OptimizerEvolution, ActivityShare
 import random
 
 log = logging.getLogger(__name__)
diff --git a/src/fts3rest/fts3rest/tests/functional/test_staging.py b/src/fts3rest/fts3rest/tests/functional/test_staging.py
index 289c1aff..8be2502f 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_staging.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_staging.py
@@ -2,7 +2,7 @@ import json
 
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
-from fts3.model import Job
+from fts3rest.model import Job
 
 
 class TestSubmitToStaging(TestController):
-- 
GitLab


From e665c4b6f7e7b2cea7d5aa6c789ebc1565cfbd97 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 16:17:21 +0200
Subject: [PATCH 21/27] Add CI environ variables for OIDC tests

---
 src/fts3rest/fts3rest/config/config.py     | 12 ++++++++++--
 src/fts3rest/fts3rest/config/middleware.py |  6 +++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py
index b318d23a..f6b03fe8 100644
--- a/src/fts3rest/fts3rest/config/config.py
+++ b/src/fts3rest/fts3rest/config/config.py
@@ -21,7 +21,7 @@ import logging
 log = logging.getLogger(__name__)
 
 
-def fts3_config_load(path="/etc/fts3/fts3config"):
+def fts3_config_load(path="/etc/fts3/fts3config", test=False):
     """
     Read the configuration from the FTS3 configuration file
     """
@@ -123,5 +123,13 @@ def fts3_config_load(path="/etc/fts3/fts3config"):
         )
     except NoSectionError:
         pass
-
+    if test:
+        provider_url = "https://iam.extreme-datacloud.eu/"
+        fts3cfg["fts3.Providers"][provider_url] = {}
+        fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[
+            "xdc_ClientId"
+        ]
+        fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[
+            "xdc_ClientSecret"
+        ]
     return fts3cfg
diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py
index 4dc60197..32d30726 100644
--- a/src/fts3rest/fts3rest/config/middleware.py
+++ b/src/fts3rest/fts3rest/config/middleware.py
@@ -21,7 +21,7 @@ from fts3rest.lib.openidconnect import oidc_manager
 from fts3rest.model.meta import init_model, Session
 
 
-def _load_configuration(config_file):
+def _load_configuration(config_file, test):
     # ConfigParser doesn't handle files without headers.
     # If the configuration file doesn't start with [fts3],
     # add it for backwards compatibility, as before migrating to Flask
@@ -42,7 +42,7 @@ def _load_configuration(config_file):
     # Load configuration
     logging.config.fileConfig(content)
     content.seek(0)
-    fts3cfg = fts3_config_load(content)
+    fts3cfg = fts3_config_load(content, test)
     content.close()
     return fts3cfg
 
@@ -91,7 +91,7 @@ def create_app(default_config_file=None, test=False):
     if not config_file:
         raise ValueError("The configuration file has not been specified")
 
-    fts3cfg = _load_configuration(config_file)
+    fts3cfg = _load_configuration(config_file, test)
     log = logging.getLogger(__name__)
 
     # Add configuration
-- 
GitLab


From f67121cf5e57b4df28721fd0a461127c6276e9a8 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 16:31:14 +0200
Subject: [PATCH 22/27] fix

---
 src/fts3rest/fts3rest/tests/functional/test_archive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fts3rest/fts3rest/tests/functional/test_archive.py b/src/fts3rest/fts3rest/tests/functional/test_archive.py
index 4e99675a..73af763c 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_archive.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_archive.py
@@ -1,7 +1,7 @@
 from fts3rest.tests import TestController
 from fts3rest.model.meta import Session
 from fts3rest.lib.middleware.fts3auth.credentials import UserCredentials
-from fts3.model import ArchivedJob, ArchivedFile
+from fts3rest.model import ArchivedJob, ArchivedFile
 
 
 class TestArchive(TestController):
-- 
GitLab


From 4f5e7b755fd0635d21834768048673d3b9d2fe21 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 16:38:15 +0200
Subject: [PATCH 23/27] fix config

---
 src/fts3rest/fts3rest/config/config.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py
index f6b03fe8..e01afe79 100644
--- a/src/fts3rest/fts3rest/config/config.py
+++ b/src/fts3rest/fts3rest/config/config.py
@@ -115,10 +115,10 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False):
         fts3cfg["fts3.ValidateAccessTokenOffline"] = parser.getboolean(
             "fts3", "ValidateAccessTokenOffline", fallback=True
         )
-        fts3cfg["JWKCacheSeconds"] = parser.getint(
+        fts3cfg["fts3.JWKCacheSeconds"] = parser.getint(
             "fts3", "JWKCacheSeconds", fallback=86400
         )
-        fts3cfg["TokenRefreshDaemonIntervalInSeconds"] = parser.getint(
+        fts3cfg["fts3.TokenRefreshDaemonIntervalInSeconds"] = parser.getint(
             "fts3", "TokenRefreshDaemonIntervalInSeconds", fallback=600
         )
     except NoSectionError:
-- 
GitLab


From 61fa2078721445fa2f34519db5f19040fd7b2fc9 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 16:52:52 +0200
Subject: [PATCH 24/27] fix

---
 src/fts3rest/fts3rest/config/config.py             | 14 +++++++-------
 .../tests/functional/test_openidconnect.py         |  3 +++
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py
index e01afe79..89a218cb 100644
--- a/src/fts3rest/fts3rest/config/config.py
+++ b/src/fts3rest/fts3rest/config/config.py
@@ -123,13 +123,13 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False):
         )
     except NoSectionError:
         pass
-    if test:
+    if test:  # for open id tests
         provider_url = "https://iam.extreme-datacloud.eu/"
         fts3cfg["fts3.Providers"][provider_url] = {}
-        fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[
-            "xdc_ClientId"
-        ]
-        fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[
-            "xdc_ClientSecret"
-        ]
+        fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ.get(
+            "xdc_ClientId", ""
+        )
+        fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ.get(
+            "xdc_ClientSecret", ""
+        )
     return fts3cfg
diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
index 955ccc88..0a0433a7 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
@@ -1,6 +1,7 @@
 from fts3rest.model import Credential
 from fts3rest.lib.openidconnect import OIDCmanager
 from fts3rest.tests import TestController
+import unittest
 
 
 class TestOpenidconnect(TestController):
@@ -16,6 +17,8 @@ class TestOpenidconnect(TestController):
         self.oidc_manager = OIDCmanager()
         self.config = self.flask_app.config
         self.issuer = "https://iam.extreme-datacloud.eu/"
+        if "client_id" not in self.config["fts3.Providers"][self.issuer]:
+            raise unittest.SkipTest("Missing OIDC client configurationd data")
 
     def test_configure_clients(self):
         self.oidc_manager._configure_clients(self.config["fts3.Providers"])
-- 
GitLab


From ad83f18ffd142644d56833e6e623a95dc64ff320 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 17:04:44 +0200
Subject: [PATCH 25/27] fix config

---
 src/fts3rest/fts3rest/config/config.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py
index 89a218cb..644a81b5 100644
--- a/src/fts3rest/fts3rest/config/config.py
+++ b/src/fts3rest/fts3rest/config/config.py
@@ -126,10 +126,10 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False):
     if test:  # for open id tests
         provider_url = "https://iam.extreme-datacloud.eu/"
         fts3cfg["fts3.Providers"][provider_url] = {}
-        fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ.get(
-            "xdc_ClientId", ""
-        )
-        fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ.get(
-            "xdc_ClientSecret", ""
-        )
+        fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[
+            "xdc_ClientId"
+        ]
+        fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[
+            "xdc_ClientSecret"
+        ]
     return fts3cfg
-- 
GitLab


From 00ddcc8e9807a43d2c28cc0c7f8e938ce777752c Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 7 May 2020 17:21:30 +0200
Subject: [PATCH 26/27] fix config

---
 src/fts3rest/fts3rest/config/middleware.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py
index 32d30726..5e622c73 100644
--- a/src/fts3rest/fts3rest/config/middleware.py
+++ b/src/fts3rest/fts3rest/config/middleware.py
@@ -126,7 +126,8 @@ def create_app(default_config_file=None, test=False):
     # Start OIDC clients
     if "fts3.Providers" in app.config:
         oidc_manager.setup(app.config)
-        IAMTokenRefresher("fts_token_refresh_daemon", app.config).start()
+        if not test:
+            IAMTokenRefresher("fts_token_refresh_daemon", app.config).start()
     else:
         log.info("OpenID Connect support disabled. Providers not found in config")
 
-- 
GitLab


From 2d1896812b099b36b03bf9467416bcf036b9e69c Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Fri, 8 May 2020 12:22:00 +0200
Subject: [PATCH 27/27] Update

---
 README.md                                     | 25 ++++++++++++-------
 src/fts3rest/fts3rest/config/middleware.py    |  2 +-
 .../tests/functional/test_openidconnect.py    |  2 +-
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 33893acc..dfcc9a79 100644
--- a/README.md
+++ b/README.md
@@ -1,24 +1,30 @@
 # About the migration
-The migration of [fts-rest](https://gitlab.cern.ch:8443/fts/fts-rest) has started after the decisions made in 
+The migration of [fts-rest](https://gitlab.cern.ch:8443/fts/fts-rest) started after the decisions made in 
 the [evaluation](https://its.cern.ch/jira/browse/FTS-1496).
 
 The development is happening at fts-flask.cern.ch, with the local user ftsflask 
 
 # Git workflow
 - `Master` cannot be pushed to directly.
-- Create a new branch for each ticket and merge it to master.
+- Create a new branch for each ticket and merge it to master through a merge request.
 
 # Gitlab CI
 The current pipeline runs for every push in every branch:
 - black: fails if the code hasn't been formatted with black
 - pylint: fails if the code has syntax errors. If you are sure that pylint is mistaken, add `# pylint: skip-file` at
- the beginning of the relevant file.
+ the beginning of the relevant file. Runs for every supported Python3 version
 - radon: fails if the code complexity is too high
+- functional tests: Run for every supported Python3 version
 - bandit: detects potential security issues in the code, but it's allowed to fail as there may be false positives
+- build: sdist and wheel
+
 Merge requests will proceed only if the pipeline succeeds.
+
 In case of emergency the pipeline can be [skipped](https://docs.gitlab.com/ee/ci/yaml/#skipping-jobs).
 
-The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster.
+The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the 
+image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster.
+To build and push the image, cd to .gitlab-ci and run .docker_push.sh
 
 Developers should add the `pre-commit` hook to their local repository. This scripts does this for every commit:
 - Runs black to format the changed files.
@@ -26,6 +32,10 @@ Developers should add the `pre-commit` hook to their local repository. This scri
 - Runs radon and bandit only on the changed files.
 The hook can be skipped, in case bandit detects false positives, with the commit option `--no-verify`.
 
+# Functional tests
+Openid tests don't run in CI because the container would need a client registered and this is 
+ difficult to set up. To run these tests in a development environment, the environment variables 'xdc_ClientId' and 'xdc_ClientSecret' must be set.
+
 # Python dependencies
 This project uses [pip-tools](https://github.com/jazzband/pip-tools) to manage dependencies:
 - `requirements.in`: list of dependencies for the production app
@@ -39,6 +49,7 @@ Because we need mod_wsgi built for Python 3.6, we need to use httpd24-httpd
 - gfal2-python3
 - yum-config-manager --enable centos-sclo-rh
 - yum install rh-python36-mod_wsgi
+
 # Installation requirements for development
 To create a development venv: use --system-packages in order to use gfal2-python3
 
@@ -62,7 +73,6 @@ curl http://localhost:80/hello
 To access the config page:
 ```
 INSERT INTO t_authz_dn VALUES ('yourdn');
-
 ```
 
 # Run tests 
@@ -70,8 +80,5 @@ INSERT INTO t_authz_dn VALUES ('yourdn');
 source venv/bin/activate
 export PYTHONPATH=/home/ftsflask/fts-rest-flask/src:/home/ftsflask/fts-rest-flask/src/fts3rest 
 export FTS3TESTCONFIG=/home/ftsflask/fts-rest-flask/src/fts3rest/fts3rest/tests/fts3testconfig
-python3 -m pytest -x src/fts3rest/fts3rest/tests/functional/test_job_submission.py 
+python3 -m pytest src/fts3rest/fts3rest/tests/ -x 
 ```
-# Migration status
-Starting with the client, as it requires small changes only. Will not migrate pycurlrequest.py, as it is not used
- anymore. 
\ No newline at end of file
diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py
index 5e622c73..248caeda 100644
--- a/src/fts3rest/fts3rest/config/middleware.py
+++ b/src/fts3rest/fts3rest/config/middleware.py
@@ -124,7 +124,7 @@ def create_app(default_config_file=None, test=False):
         return response
 
     # Start OIDC clients
-    if "fts3.Providers" in app.config:
+    if "fts3.Providers" not in app.config or not app.config["fts3.Providers"]:
         oidc_manager.setup(app.config)
         if not test:
             IAMTokenRefresher("fts_token_refresh_daemon", app.config).start()
diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
index 0a0433a7..ad67bd4d 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
@@ -18,7 +18,7 @@ class TestOpenidconnect(TestController):
         self.config = self.flask_app.config
         self.issuer = "https://iam.extreme-datacloud.eu/"
         if "client_id" not in self.config["fts3.Providers"][self.issuer]:
-            raise unittest.SkipTest("Missing OIDC client configurationd data")
+            raise unittest.SkipTest("Missing OIDC client configuration data")
 
     def test_configure_clients(self):
         self.oidc_manager._configure_clients(self.config["fts3.Providers"])
-- 
GitLab