diff --git a/.gitlab-ci/db.sh b/.gitlab-ci/db.sh index e717a9d3268aa0880689baabad9329f36d834941..972a3d95e96473d08714adc264bd4f4b7ae1e9f2 100644 --- a/.gitlab-ci/db.sh +++ b/.gitlab-ci/db.sh @@ -1,9 +1,7 @@ #!/bin/sh -curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-5.0.0.sql -curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-diff-6.0.0.sql +curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-6.0.0.sql -mysql --user=root --password=asdf --host=mariadb ftsflask < fts-schema-5.0.0.sql -mysql --user=root --password=asdf --host=mariadb ftsflask < fts-diff-6.0.0.sql +mysql --user=root --password=asdf --host=mariadb ftsflask < fts-schema-6.0.0.sql echo "CREATE USER 'ci'@'%';" | mysql --user=root --password=asdf --host=mariadb echo "GRANT ALL PRIVILEGES ON ftsflask.* TO 'ci'@'%' IDENTIFIED BY 'asdf';" | mysql --user=root --password=asdf --host=mariadb \ No newline at end of file diff --git a/README.md b/README.md index 336b880f4bb5cc56405cba11dc5851f8ca6cbd88..2f12d4984e60aea4310ed4aabcc2f02c185960f8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ The current pipeline runs for every push in every branch: - radon: fails if the code complexity is too high - functional tests: Run for every supported Python3 version - bandit: detects potential security issues in the code, but it's allowed to fail as there may be false positives. -To ignore a false positive, append "# nosec" to the offending line +To ignore a false positive, append `# nosec"` to the offending line - build: sdist and wheel Merge requests will proceed only if the pipeline succeeds. @@ -25,7 +25,7 @@ In case of emergency the pipeline can be [skipped](https://docs.gitlab.com/ee/ci The pipeline runs in a container from the image tagged as `ci`. The dockerfile is in the .gitlab-ci directory and the image is in the container registry for this project. The image contains the Python tools preinstalled so the CI runs faster. -To build and push the image, cd to .gitlab-ci and run .docker_push.sh +To build and push the image, cd to .gitlab-ci and run .docker_push.sh. This should be done when new dependencies are added. Developers should add the `pre-commit` hook to their local repository. This scripts does this for every commit: - Runs black to format the changed files. @@ -45,14 +45,73 @@ This project uses [pip-tools](https://github.com/jazzband/pip-tools) to manage d - `pipsyncdev.sh`: run it afterwards to synchronize the virtual environment with the requirements. # Installation requirements -Because we need mod_wsgi built for Python 3.6, we need to use httpd24-httpd +Because we need mod_wsgi built for Python 3.6, we need to use rh-python36-mod_wsgi - yum install python3-devel openssl-devel swig gcc gcc-c++ make httpd-devel mysql-devel - gfal2-python3 - yum-config-manager --enable centos-sclo-rh - yum install rh-python36-mod_wsgi -# Installation requirements for development -To create a development venv: use --system-packages in order to use gfal2-python3 +# Create a development server +```bash +# Create VM +ssh garciacc@aiadm.cern.ch +unset OS_PROJECT_ID; +unset OS_TENANT_ID; +unset OS_TENANT_NAME; +export OS_PROJECT_NAME="IT FTS development"; +ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \ + --landb-responsible fts-devel --nova-flavor m2.large \ + fts-flask-02 + +# Install dependencies +ssh root@fts-flask-02 +yum install centos-release-scl-rh +yum-config-manager --enable centos-sclo-rh +yum install python3-devel openssl-devel swig gcc gcc-c++ make httpd-devel \ +mysql-devel gfal2-python3 gfal2-plugin-mock rh-python36-mod_wsgi \ +git mariadb mariadb-server gridsite -y + +# Prepare DB and log directories +systemctl start mariadb +mkdir /var/run/mariadb +chown mysql:mysql /var/run/mariadb +mkdir /var/log/fts3rest +chown ftsflask /var/log/fts3rest + +# Prepare application and Python dependencies +su ftsflask +cd +git clone https://gitlab.cern.ch/fts/fts-rest-flask.git +cd fts-rest-flask +# use --system-site-packages in order to use gfal2-python3 +python3 -m venv venv --system--site-packages +source venv/bin/activate +pip install --upgrade pip +pip install pip-tools +. ./pipcompile.sh +. ./pipsyncdev.sh + +# Load DB +cd .. +curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-6.0.0.sql +mysql_secure_installation # put a password for root +echo "CREATE DATABASE ftsflask;" | mysql --user=root --password +mysql --user=root --password ftsflask < fts-schema-6.0.0.sql +echo "CREATE USER ftsflask;" | mysql --user=root --password +echo "GRANT ALL PRIVILEGES ON ftsflask.* TO 'ftsflask'@'localhost' IDENTIFIED BY 'anotherpassword';" | mysql --user=root --password +cd fts-rest-flask +. runtests.sh + +# Prepare server +exit +cp fts-rest-flask/src/fts3rest/httpd_fts.conf /etc/httpd/conf.d/ +setenforce 0 +chmod o+rx -R /home/ftsflask/ +systemctl restart httpd + + +``` +To create a development venv: # How to run development server Flask: diff --git a/pipcompile.sh b/pipcompile.sh index a0f9ad11a5eaf72f417378c54f61bb1b311a7d98..0545ad665987e092c37844a025a60a1783137c36 100644 --- a/pipcompile.sh +++ b/pipcompile.sh @@ -1,2 +1,2 @@ -pip-compile --generate-hashes requirements.in -pip-compile --generate-hashes dev-requirements.in +pip-compile --generate-hashes --upgrade requirements.in +pip-compile --generate-hashes --upgrade dev-requirements.in diff --git a/runtests.sh b/runtests.sh index aa8d32d0ab75a8901f121ba169091a57e3fc16ec..3031a6fdbc43ed05e7c9c2d6fda6fc1176208186 100644 --- a/runtests.sh +++ b/runtests.sh @@ -3,4 +3,4 @@ source venv/bin/activate export PYTHONPATH=/home/ftsflask/fts-rest-flask/src:/home/ftsflask/fts-rest-flask/src/fts3rest export FTS3TESTCONFIG=/home/ftsflask/fts-rest-flask/src/fts3rest/fts3rest/tests/fts3testconfig -python3 -m pytest -x src/fts3rest/fts3rest/tests/functional/test_job_submission.py +pytest src/fts3rest/fts3rest/tests/functional/ -x diff --git a/src/fts3rest/fts3rest/config/config.py b/src/fts3rest/fts3rest/config/config.py index 644a81b57d797a678b5d2c96e2b5ddd7ea2b85a2..6dfdb9dfd72f7e189bece73196cff04e2a788f96 100644 --- a/src/fts3rest/fts3rest/config/config.py +++ b/src/fts3rest/fts3rest/config/config.py @@ -123,7 +123,7 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False): ) except NoSectionError: pass - if test: # for open id tests + if test and "xdc_ClientId" in os.environ: # for open id tests provider_url = "https://iam.extreme-datacloud.eu/" fts3cfg["fts3.Providers"][provider_url] = {} fts3cfg["fts3.Providers"][provider_url]["client_id"] = os.environ[ @@ -132,4 +132,6 @@ def fts3_config_load(path="/etc/fts3/fts3config", test=False): fts3cfg["fts3.Providers"][provider_url]["client_secret"] = os.environ[ "xdc_ClientSecret" ] + else: + fts3cfg["fts3.Providers"] = {} return fts3cfg diff --git a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py index 4e254c0281050a8bdba896e999f86687e09dcf62..414b664f30532c08fee5ed576048b21d84801668 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py +++ b/src/fts3rest/fts3rest/tests/functional/test_oauth2provider.py @@ -1,6 +1,7 @@ from fts3rest.lib.oauth2provider import FTS3OAuth2ResourceProvider from fts3rest.lib.openidconnect import OIDCmanager from fts3rest.tests import TestController +import unittest class TestFTS3OAuth2ResourceProvider(TestController): @@ -13,8 +14,10 @@ class TestFTS3OAuth2ResourceProvider(TestController): def setUp(self): super().setUp() - self.oidc_manager = OIDCmanager() config = self.flask_app.config + if not config["fts3.Providers"]: + raise unittest.SkipTest("Missing OIDC client configuration data") + self.oidc_manager = OIDCmanager() self.issuer = "https://iam.extreme-datacloud.eu/" self.oidc_manager.setup(config) self.oauth2_resource_provider = FTS3OAuth2ResourceProvider(dict(), config) diff --git a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py index ad67bd4d9e9b008b82fb92d3bd61a56bff038354..3631f1cf37affbbd10f98094e1594196ff067d3b 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py +++ b/src/fts3rest/fts3rest/tests/functional/test_openidconnect.py @@ -14,11 +14,11 @@ class TestOpenidconnect(TestController): def setUp(self): super().setUp() - self.oidc_manager = OIDCmanager() self.config = self.flask_app.config - self.issuer = "https://iam.extreme-datacloud.eu/" - if "client_id" not in self.config["fts3.Providers"][self.issuer]: + if not self.config["fts3.Providers"]: raise unittest.SkipTest("Missing OIDC client configuration data") + self.oidc_manager = OIDCmanager() + self.issuer = "https://iam.extreme-datacloud.eu/" def test_configure_clients(self): self.oidc_manager._configure_clients(self.config["fts3.Providers"])