From f4e90a82508c847e5023f099c55301e199f79a4b Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 14 May 2020 13:36:20 +0200
Subject: [PATCH 1/4] Make openid tests optional

---
 README.md                              | 23 +++++++++++++++++++----
 src/fts3rest/fts3rest/config/config.py |  2 +-
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 336b880f..7cd01bd0 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ The current pipeline runs for every push in every branch:
 - radon: fails if the code complexity is too high
 - functional tests: Run for every supported Python3 version
 - bandit: detects potential security issues in the code, but it's allowed to fail as there may be false positives.
-To ignore a false positive, append "# nosec" to the offending line
+To ignore a false positive, append `# nosec"` to the offending line
 - build: sdist and wheel
 
 Merge requests will proceed only if the pipeline succeeds.
@@ -25,7 +25,7 @@ In case of emergency the pipeline can be [skipped](https://docs.gitlab.com/ee/ci
 
 The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the 
 image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster.
-To build and push the image, cd to .gitlab-ci and run .docker_push.sh
+To build and push the image, cd to .gitlab-ci and run .docker_push.sh. This should be done when new dependencies are added.
 
 Developers should add the `pre-commit` hook to their local repository. This scripts does this for every commit:
 - Runs black to format the changed files.
@@ -45,13 +45,28 @@ This project uses [pip-tools](https://github.com/jazzband/pip-tools) to manage d
 - `pipsyncdev.sh`: run it afterwards to synchronize the virtual environment with the requirements.
 
 # Installation requirements
-Because we need mod_wsgi built for Python 3.6, we need to use httpd24-httpd
+Because we need mod_wsgi built for Python 3.6, we need to use rh-python36-mod_wsgi
 - yum install python3-devel openssl-devel swig gcc gcc-c++ make httpd-devel mysql-devel
 - gfal2-python3
 - yum-config-manager --enable centos-sclo-rh
 - yum install rh-python36-mod_wsgi
 
-# Installation requirements for development
+# Create a development server
+```bash
+ssh garciacc@aiadm.cern.ch
+unset OS_PROJECT_ID;
+unset OS_TENANT_ID;
+unset OS_TENANT_NAME;
+export OS_PROJECT_NAME="IT FTS development";
+ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \
+      --landb-responsible fts-devel --nova-flavor m2.large \
+      fts-flask-02
+ssh root@fts-flask-02
+su ftsflask
+cd
+
+
+```
 To create a development venv: use --system-packages in order to use gfal2-python3
 
 # How to run development server
diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py
index 644a81b5..f24cf461 100644
--- a/src/fts3rest/fts3rest/config/config.py
+++ b/src/fts3rest/fts3rest/config/config.py
@@ -123,7 +123,7 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False):
         )
     except NoSectionError:
         pass
-    if test:  # for open id tests
+    if test and "xdc_ClientId" in os.environ:  # for open id tests
         provider_url = "https://iam.extreme-datacloud.eu/"
         fts3cfg["fts3.Providers"][provider_url] = {}
         fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[
-- 
GitLab


From e52ab53ddedc8d43957bf73af12a65b7ed8b19ea Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 14 May 2020 13:48:17 +0200
Subject: [PATCH 2/4] fix

---
 runtests.sh                                                 | 2 +-
 .../fts3rest/tests/functional/test_oauth2provider.py        | 5 ++++-
 .../fts3rest/tests/functional/test_openidconnect.py         | 6 +++---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/runtests.sh b/runtests.sh
index aa8d32d0..3031a6fd 100644
--- a/runtests.sh
+++ b/runtests.sh
@@ -3,4 +3,4 @@
 source venv/bin/activate
 export PYTHONPATH=/home/ftsflask/fts-rest-flask/src:/home/ftsflask/fts-rest-flask/src/fts3rest
 export FTS3TESTCONFIG=/home/ftsflask/fts-rest-flask/src/fts3rest/fts3rest/tests/fts3testconfig
-python3 -m pytest -x src/fts3rest/fts3rest/tests/functional/test_job_submission.py
+pytest src/fts3rest/fts3rest/tests/functional/ -x
diff --git a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py
index 4e254c02..414b664f 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py
@@ -1,6 +1,7 @@
 from fts3rest.lib.oauth2provider import FTS3OAuth2ResourceProvider
 from fts3rest.lib.openidconnect import OIDCmanager
 from fts3rest.tests import TestController
+import unittest
 
 
 class TestFTS3OAuth2ResourceProvider(TestController):
@@ -13,8 +14,10 @@ class TestFTS3OAuth2ResourceProvider(TestController):
 
     def setUp(self):
         super().setUp()
-        self.oidc_manager = OIDCmanager()
         config = self.flask_app.config
+        if not config["fts3.Providers"]:
+            raise unittest.SkipTest("Missing OIDC client configuration data")
+        self.oidc_manager = OIDCmanager()
         self.issuer = "https://iam.extreme-datacloud.eu/"
         self.oidc_manager.setup(config)
         self.oauth2_resource_provider = FTS3OAuth2ResourceProvider(dict(), config)
diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
index ad67bd4d..3631f1cf 100644
--- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
+++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py
@@ -14,11 +14,11 @@ class TestOpenidconnect(TestController):
 
     def setUp(self):
         super().setUp()
-        self.oidc_manager = OIDCmanager()
         self.config = self.flask_app.config
-        self.issuer = "https://iam.extreme-datacloud.eu/"
-        if "client_id" not in self.config["fts3.Providers"][self.issuer]:
+        if not self.config["fts3.Providers"]:
             raise unittest.SkipTest("Missing OIDC client configuration data")
+        self.oidc_manager = OIDCmanager()
+        self.issuer = "https://iam.extreme-datacloud.eu/"
 
     def test_configure_clients(self):
         self.oidc_manager._configure_clients(self.config["fts3.Providers"])
-- 
GitLab


From 2e9c660f1eaf0a663def5bb17f6c4f91f46f6f45 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Thu, 14 May 2020 13:56:50 +0200
Subject: [PATCH 3/4] fix

---
 README.md                              | 2 +-
 src/fts3rest/fts3rest/config/config.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7cd01bd0..d8f463e8 100644
--- a/README.md
+++ b/README.md
@@ -64,7 +64,7 @@ ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \
 ssh root@fts-flask-02
 su ftsflask
 cd
-
+git clone https://gitlab.cern.ch/fts/fts-rest-flask.git
 
 ```
 To create a development venv: use --system-packages in order to use gfal2-python3
diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py
index f24cf461..6dfdb9df 100644
--- a/src/fts3rest/fts3rest/config/config.py
+++ b/src/fts3rest/fts3rest/config/config.py
@@ -132,4 +132,6 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False):
         fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[
             "xdc_ClientSecret"
         ]
+    else:
+        fts3cfg["fts3.Providers"] = {}
     return fts3cfg
-- 
GitLab


From cb1fad3dd6b3e0f75f8885b882b90c6d271c1fe3 Mon Sep 17 00:00:00 2001
From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch>
Date: Fri, 15 May 2020 11:50:42 +0200
Subject: [PATCH 4/4] Finish documenting dev environment

---
 .gitlab-ci/db.sh |  6 ++----
 README.md        | 46 +++++++++++++++++++++++++++++++++++++++++++++-
 pipcompile.sh    |  4 ++--
 3 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci/db.sh b/.gitlab-ci/db.sh
index e717a9d3..972a3d95 100644
--- a/.gitlab-ci/db.sh
+++ b/.gitlab-ci/db.sh
@@ -1,9 +1,7 @@
 #!/bin/sh
 
-curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-5.0.0.sql
-curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-diff-6.0.0.sql
+curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-6.0.0.sql
 
-mysql --user=root --password=asdf --host=mariadb ftsflask < fts-schema-5.0.0.sql
-mysql --user=root --password=asdf --host=mariadb ftsflask < fts-diff-6.0.0.sql
+mysql --user=root --password=asdf --host=mariadb ftsflask < fts-schema-6.0.0.sql
 echo "CREATE USER 'ci'@'%';" | mysql --user=root --password=asdf --host=mariadb
 echo "GRANT ALL PRIVILEGES ON ftsflask.* TO 'ci'@'%' IDENTIFIED BY 'asdf';" | mysql --user=root --password=asdf --host=mariadb
\ No newline at end of file
diff --git a/README.md b/README.md
index d8f463e8..2f12d498 100644
--- a/README.md
+++ b/README.md
@@ -53,6 +53,7 @@ Because we need mod_wsgi built for Python 3.6, we need to use rh-python36-mod_ws
 
 # Create a development server
 ```bash
+# Create VM
 ssh garciacc@aiadm.cern.ch
 unset OS_PROJECT_ID;
 unset OS_TENANT_ID;
@@ -61,13 +62,56 @@ export OS_PROJECT_NAME="IT FTS development";
 ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \
       --landb-responsible fts-devel --nova-flavor m2.large \
       fts-flask-02
+           
+# Install dependencies
 ssh root@fts-flask-02
+yum install centos-release-scl-rh
+yum-config-manager --enable centos-sclo-rh
+yum install python3-devel openssl-devel swig gcc gcc-c++ make httpd-devel \
+mysql-devel gfal2-python3 gfal2-plugin-mock rh-python36-mod_wsgi \
+git mariadb mariadb-server gridsite -y
+
+# Prepare DB and log directories
+systemctl start mariadb    
+mkdir /var/run/mariadb             
+chown mysql:mysql  /var/run/mariadb
+mkdir /var/log/fts3rest
+chown ftsflask /var/log/fts3rest
+
+# Prepare application and Python dependencies
 su ftsflask
 cd
 git clone https://gitlab.cern.ch/fts/fts-rest-flask.git
+cd fts-rest-flask                 
+# use --system-site-packages in order to use gfal2-python3      
+python3 -m venv venv --system--site-packages
+source venv/bin/activate
+pip install --upgrade pip
+pip install pip-tools
+. ./pipcompile.sh 
+. ./pipsyncdev.sh
+                                            
+# Load DB
+cd ..
+curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-6.0.0.sql
+mysql_secure_installation # put a password for root
+echo "CREATE DATABASE ftsflask;" | mysql --user=root --password
+mysql --user=root --password ftsflask < fts-schema-6.0.0.sql
+echo "CREATE USER ftsflask;" | mysql --user=root --password
+echo "GRANT ALL PRIVILEGES ON ftsflask.* TO 'ftsflask'@'localhost' IDENTIFIED BY 'anotherpassword';" | mysql --user=root --password
+cd fts-rest-flask
+. runtests.sh
+
+# Prepare server
+exit
+cp fts-rest-flask/src/fts3rest/httpd_fts.conf /etc/httpd/conf.d/
+setenforce 0
+chmod o+rx -R /home/ftsflask/
+systemctl restart httpd
+
 
 ```
-To create a development venv: use --system-packages in order to use gfal2-python3
+To create a development venv: 
 
 # How to run development server
 Flask:
diff --git a/pipcompile.sh b/pipcompile.sh
index a0f9ad11..0545ad66 100644
--- a/pipcompile.sh
+++ b/pipcompile.sh
@@ -1,2 +1,2 @@
-pip-compile --generate-hashes requirements.in
-pip-compile --generate-hashes dev-requirements.in
+pip-compile --generate-hashes --upgrade requirements.in
+pip-compile --generate-hashes --upgrade dev-requirements.in
-- 
GitLab