From f4e90a82508c847e5023f099c55301e199f79a4b Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 14 May 2020 13:36:20 +0200 Subject: [PATCH 1/4] Make openid tests optional --- README.md | 23 +++++++++++++++++++---- src/fts3rest/fts3rest/config/config.py | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 336b880f..7cd01bd0 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ The current pipeline runs for every push in every branch: - radon: fails if the code complexity is too high - functional tests: Run for every supported Python3 version - bandit: detects potential security issues in the code, but it's allowed to fail as there may be false positives. -To ignore a false positive, append "# nosec" to the offending line +To ignore a false positive, append `# nosec"` to the offending line - build: sdist and wheel Merge requests will proceed only if the pipeline succeeds. @@ -25,7 +25,7 @@ In case of emergency the pipeline can be [skipped](https://docs.gitlab.com/ee/ci The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster. -To build and push the image, cd to .gitlab-ci and run .docker_push.sh +To build and push the image, cd to .gitlab-ci and run .docker_push.sh. This should be done when new dependencies are added. Developers should add the `pre-commit` hook to their local repository. This scripts does this for every commit: - Runs black to format the changed files. @@ -45,13 +45,28 @@ This project uses [pip-tools](https://github.com/jazzband/pip-tools) to manage d - `pipsyncdev.sh`: run it afterwards to synchronize the virtual environment with the requirements. # Installation requirements -Because we need mod_wsgi built for Python 3.6, we need to use httpd24-httpd +Because we need mod_wsgi built for Python 3.6, we need to use rh-python36-mod_wsgi - yum install python3-devel openssl-devel swig gcc gcc-c++ make httpd-devel mysql-devel - gfal2-python3 - yum-config-manager --enable centos-sclo-rh - yum install rh-python36-mod_wsgi -# Installation requirements for development +# Create a development server +```bash +ssh garciacc@aiadm.cern.ch +unset OS_PROJECT_ID; +unset OS_TENANT_ID; +unset OS_TENANT_NAME; +export OS_PROJECT_NAME="IT FTS development"; +ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \ + --landb-responsible fts-devel --nova-flavor m2.large \ + fts-flask-02 +ssh root@fts-flask-02 +su ftsflask +cd + + +``` To create a development venv: use --system-packages in order to use gfal2-python3 # How to run development server diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py index 644a81b5..f24cf461 100644 --- a/src/fts3rest/fts3rest/config/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -123,7 +123,7 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False): ) except NoSectionError: pass - if test: # for open id tests + if test and "xdc_ClientId" in os.environ: # for open id tests provider_url = "https://iam.extreme-datacloud.eu/" fts3cfg["fts3.Providers"][provider_url] = {} fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[ -- GitLab From e52ab53ddedc8d43957bf73af12a65b7ed8b19ea Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 14 May 2020 13:48:17 +0200 Subject: [PATCH 2/4] fix --- runtests.sh | 2 +- .../fts3rest/tests/functional/test_oauth2provider.py | 5 ++++- .../fts3rest/tests/functional/test_openidconnect.py | 6 +++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/runtests.sh b/runtests.sh index aa8d32d0..3031a6fd 100644 --- a/runtests.sh +++ b/runtests.sh @@ -3,4 +3,4 @@ source venv/bin/activate export PYTHONPATH=/home/ftsflask/fts-rest-flask/src:/home/ftsflask/fts-rest-flask/src/fts3rest export FTS3TESTCONFIG=/home/ftsflask/fts-rest-flask/src/fts3rest/fts3rest/tests/fts3testconfig -python3 -m pytest -x src/fts3rest/fts3rest/tests/functional/test_job_submission.py +pytest src/fts3rest/fts3rest/tests/functional/ -x diff --git a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py index 4e254c02..414b664f 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py +++ b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py @@ -1,6 +1,7 @@ from fts3rest.lib.oauth2provider import FTS3OAuth2ResourceProvider from fts3rest.lib.openidconnect import OIDCmanager from fts3rest.tests import TestController +import unittest class TestFTS3OAuth2ResourceProvider(TestController): @@ -13,8 +14,10 @@ class TestFTS3OAuth2ResourceProvider(TestController): def setUp(self): super().setUp() - self.oidc_manager = OIDCmanager() config = self.flask_app.config + if not config["fts3.Providers"]: + raise unittest.SkipTest("Missing OIDC client configuration data") + self.oidc_manager = OIDCmanager() self.issuer = "https://iam.extreme-datacloud.eu/" self.oidc_manager.setup(config) self.oauth2_resource_provider = FTS3OAuth2ResourceProvider(dict(), config) diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py index ad67bd4d..3631f1cf 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py +++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py @@ -14,11 +14,11 @@ class TestOpenidconnect(TestController): def setUp(self): super().setUp() - self.oidc_manager = OIDCmanager() self.config = self.flask_app.config - self.issuer = "https://iam.extreme-datacloud.eu/" - if "client_id" not in self.config["fts3.Providers"][self.issuer]: + if not self.config["fts3.Providers"]: raise unittest.SkipTest("Missing OIDC client configuration data") + self.oidc_manager = OIDCmanager() + self.issuer = "https://iam.extreme-datacloud.eu/" def test_configure_clients(self): self.oidc_manager._configure_clients(self.config["fts3.Providers"]) -- GitLab From 2e9c660f1eaf0a663def5bb17f6c4f91f46f6f45 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Thu, 14 May 2020 13:56:50 +0200 Subject: [PATCH 3/4] fix --- README.md | 2 +- src/fts3rest/fts3rest/config/config.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7cd01bd0..d8f463e8 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \ ssh root@fts-flask-02 su ftsflask cd - +git clone https://gitlab.cern.ch/fts/fts-rest-flask.git ``` To create a development venv: use --system-packages in order to use gfal2-python3 diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py index f24cf461..6dfdb9df 100644 --- a/src/fts3rest/fts3rest/config/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -132,4 +132,6 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False): fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[ "xdc_ClientSecret" ] + else: + fts3cfg["fts3.Providers"] = {} return fts3cfg -- GitLab From cb1fad3dd6b3e0f75f8885b882b90c6d271c1fe3 Mon Sep 17 00:00:00 2001 From: Carles Garcia Cabot <carles.garcia.cabot@cern.ch> Date: Fri, 15 May 2020 11:50:42 +0200 Subject: [PATCH 4/4] Finish documenting dev environment --- .gitlab-ci/db.sh | 6 ++---- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- pipcompile.sh | 4 ++-- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci/db.sh b/.gitlab-ci/db.sh index e717a9d3..972a3d95 100644 --- a/.gitlab-ci/db.sh +++ b/.gitlab-ci/db.sh @@ -1,9 +1,7 @@ #!/bin/sh -curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-5.0.0.sql -curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-diff-6.0.0.sql +curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-6.0.0.sql -mysql --user=root --password=asdf --host=mariadb ftsflask < fts-schema-5.0.0.sql -mysql --user=root --password=asdf --host=mariadb ftsflask < fts-diff-6.0.0.sql +mysql --user=root --password=asdf --host=mariadb ftsflask < fts-schema-6.0.0.sql echo "CREATE USER 'ci'@'%';" | mysql --user=root --password=asdf --host=mariadb echo "GRANT ALL PRIVILEGES ON ftsflask.* TO 'ci'@'%' IDENTIFIED BY 'asdf';" | mysql --user=root --password=asdf --host=mariadb \ No newline at end of file diff --git a/README.md b/README.md index d8f463e8..2f12d498 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Because we need mod_wsgi built for Python 3.6, we need to use rh-python36-mod_ws # Create a development server ```bash +# Create VM ssh garciacc@aiadm.cern.ch unset OS_PROJECT_ID; unset OS_TENANT_ID; @@ -61,13 +62,56 @@ export OS_PROJECT_NAME="IT FTS development"; ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \ --landb-responsible fts-devel --nova-flavor m2.large \ fts-flask-02 + +# Install dependencies ssh root@fts-flask-02 +yum install centos-release-scl-rh +yum-config-manager --enable centos-sclo-rh +yum install python3-devel openssl-devel swig gcc gcc-c++ make httpd-devel \ +mysql-devel gfal2-python3 gfal2-plugin-mock rh-python36-mod_wsgi \ +git mariadb mariadb-server gridsite -y + +# Prepare DB and log directories +systemctl start mariadb +mkdir /var/run/mariadb +chown mysql:mysql /var/run/mariadb +mkdir /var/log/fts3rest +chown ftsflask /var/log/fts3rest + +# Prepare application and Python dependencies su ftsflask cd git clone https://gitlab.cern.ch/fts/fts-rest-flask.git +cd fts-rest-flask +# use --system-site-packages in order to use gfal2-python3 +python3 -m venv venv --system--site-packages +source venv/bin/activate +pip install --upgrade pip +pip install pip-tools +. ./pipcompile.sh +. ./pipsyncdev.sh + +# Load DB +cd .. +curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-6.0.0.sql +mysql_secure_installation # put a password for root +echo "CREATE DATABASE ftsflask;" | mysql --user=root --password +mysql --user=root --password ftsflask < fts-schema-6.0.0.sql +echo "CREATE USER ftsflask;" | mysql --user=root --password +echo "GRANT ALL PRIVILEGES ON ftsflask.* TO 'ftsflask'@'localhost' IDENTIFIED BY 'anotherpassword';" | mysql --user=root --password +cd fts-rest-flask +. runtests.sh + +# Prepare server +exit +cp fts-rest-flask/src/fts3rest/httpd_fts.conf /etc/httpd/conf.d/ +setenforce 0 +chmod o+rx -R /home/ftsflask/ +systemctl restart httpd + ``` -To create a development venv: use --system-packages in order to use gfal2-python3 +To create a development venv: # How to run development server Flask: diff --git a/pipcompile.sh b/pipcompile.sh index a0f9ad11..0545ad66 100644 --- a/pipcompile.sh +++ b/pipcompile.sh @@ -1,2 +1,2 @@ -pip-compile --generate-hashes requirements.in -pip-compile --generate-hashes dev-requirements.in +pip-compile --generate-hashes --upgrade requirements.in +pip-compile --generate-hashes --upgrade dev-requirements.in -- GitLab