diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a2a359cc01c3a7f702ccfb735b293d454ed90e47..d4360b2c771ddac5d9e721a299def7ade779d02b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,6 +96,7 @@ bandit: stage: security script: - bandit -r src/ --exclude src/fts3rest/fts3rest/tests/ + needs: [] client_wheel_sdist: stage: build diff --git a/fts3config b/fts3config deleted file mode 100644 index ed43c552fa3fc71bb479a0214eb19565cf6563c3..0000000000000000000000000000000000000000 --- a/fts3config +++ /dev/null @@ -1,167 +0,0 @@ -# Running user and group -User= -Group= - -# mysql only -DbType=mysql - -#db username -DbUserName=ftsflask - -#db password -DbPassword= - - -#For MySQL, it has to follow the format 'host/db' (i.e. "mysql-server.example.com/fts3db") -DbConnectString=localhost:3306/ftsflask - -#Number of db connections in the pool (use even number, e.g. 2,4,6,8,etc OR 1 for a single connection) -DbThreadsNum=30 - -#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/ -#Alias=fts3-xdc.cern.ch - -#Infosys, either the fqdn:port of a BDII instance or false to disable BDII access -#Infosys=lcg-bdii.cern.ch:2170 - -#Query the info systems specified in the order given, e.g. glue1;glue2 -InfoProviders=glue1 - -#List of authorized VOs, separated by ; -#Leave * to authorize any VO -AuthorizedVO=* - -# site name -#SiteName=FTS-DEV-XDC - -#Enable/Disable monitoring using messaging monitoring (disabled=false / enabled=true) -MonitoringMessaging=false - -# Profiling interval in seconds. If set to 0, it will be disabled -Profiling=0 - -# Log directories -TransferLogDirectory=/var/log/fts3/transfers -ServerLogDirectory=/var/log/fts3 - -# Log level. Enables logging for messages of level >= than configured -# Possible values are -# TRACE (every detail), DEBUG (internal behaviour), INFO (normal behaviour), -# NOTICE (final states), WARNING (things worth checking), ERR (internal FTS3 errors, as database connectivity), -# CRIT (fatal errors, as segmentation fault) -# It is recommended to use DEBUG or INFO -LogLevel=DEBUG - -# Check for fts_url_copy processes that do not give their progress back -# CheckStalledTransfers = true -# Stalled timeout, in seconds -# CheckStalledTimeout = 900 -CheckStalledTimeout = 900 - -# Minimum required free RAM (in MB) for FTS3 to work normally -# If the amount of free RAM goes below the limit, FTS3 will enter auto-drain mode -# This is intended to protect against system resource exhaustion -# MinRequiredFreeRAM = 50 -MinRequiredFreeRAM = 50 - -# Maximum number of url copy processes that the node can run -# The RAM limitation may not take into account other node limitations (i.e. IO) -# or, depending on the swapping policy, may not even prevent overloads if the kernel -# starts swapping before the free RAM decreases until it reaches the value of MinRequiredFreeRAM -# 0 disables the check. -# The default is 400. -# MaxUrlCopyProcesses = 400 -MaxUrlCopyProcesses = 400 - -# Parameters for Bring Online -# Maximum bulk size. -# If the size is too large, it will take more resources (memory and CPU) to generate the requests and -# parse the responses. Some servers may reject the requests if they are too big. -# If it is too small, performance will be reduced. -# Keep it to a sensible size (between 100 and 1k) -# StagingBulkSize=400 -# Maximum number of concurrent requests. This gives a maximum of files sent to the storage system -# (StagingBulkSize*StagingConcurrentRequests). The larger the number, the more requests will FTS need to keep track of. -# StagingConcurrentRequests=500 -# Seconds to wait before submitting a bulk request, so FTS can accumulate more files per bulk. -# Note that the resolution is 60 seconds. -# StagingWaitingFactor=300 -# Retry this number of times if a staging poll fails with ECOMM -# StagingPollRetries=3 - -# In seconds, interval between heartbeats -# HeartBeatInterval=60 -# I seconds, after this interval a host is considered down -# HeartBeatGraceInterval=120 - -# Seconds between optimizer runs -# OptimizerInterval = 60 -# After this time without optimizer updates, force a run -# OptimizerSteadyInterval = 300 -# Maximum number of streams per file -# OptimizerMaxStreams = 16 - -# EMA Alpha factor to reduce the influence of fluctuations -# OptimizerEMAAlpha = 0.1 -# Increase step size when the optimizer considers the performance is good -# OptimizerIncreaseStep = 1 -# Increase step size when the optimizer considers the performance is good, and set to aggressive or normal -# OptimizerAggressiveIncreaseStep = 2 -# Decrease step size when the optimizer considers the performance is bad -# OptimizerDecreaseStep = 1 - - -# Set the bulk size, in number of jobs, used for cleaning the old records -#CleanBulkSize=5000 -# In days. Entries older than this will be purged. -#CleanInterval=7 - -## The higher the values for the following parameters, -## the higher the latency for some operations (as cancelations), -## but can also reduce the system and/or database load - -# In seconds, how often to purge the messaging directory -#PurgeMessagingDirectoryInterval = 600 -# In seconds, how often to run sanity checks -#CheckSanityStateInterval = 3600 -# In seconds, how often to check for canceled transfers -#CancelCheckInterval = 10 -# In seconds, how often to check for expired queued transfers -#QueueTimeoutCheckInterval = 300 -# In seconds, how often to check for stalled transfers -#ActiveTimeoutCheckInterval = 300 -# In seconds, how often to schedule new transfers -#SchedulingInterval = 2 -# In seconds, how often to check for messages. Should be less than CheckStalledTimeout/2 -#MessagingConsumeInterval = 1 -#Enable or disable auto session reuse -AutoSessionReuse = true -#Max small file size for session reuse in bytes -AutoSessionReuseMaxSmallFileSize = 104857600 -#Max big file size for session reuse in bytes -AutoSessionReuseMaxBigFileSize = 1073741824 -#Max number of files per session reuse -AutoSessionReuseMaxFiles = 1000 -#Max number of big files per session reuse -AutoSessionReuseMaxBigFiles = 2 -BackupTables=false -OptimizerMaxSuccessRate=100 -OptimizerMedSuccessRate=80 -OptimizerLowSuccessRate=75 -OptimizerBaseSuccessRate=74 -Port=8443 -UseFixedJobPriority=0 - -ValidateAccessTokenOffline=True -JWKCacheSeconds=86400 -TokenRefreshDaemonIntervalInSeconds=600 - -[roles] -Public = all:transfer -lcgadmin = vo:transfer;vo:config -production = all:config - -[providers] -xdc=https://iam.extreme-datacloud.eu -xdc_ClientId= -xdc_ClientSecret= diff --git a/fts_presentation.md b/fts_presentation.md new file mode 100644 index 0000000000000000000000000000000000000000..5d3cdcc5775d7045c9f9a3e21036c663092c5544 --- /dev/null +++ b/fts_presentation.md @@ -0,0 +1,299 @@ +fts-rest migration to Flask and Python3 +=== +Carles Garcia Cabot + +02/10/2020 + +--- + +Source: https://gitlab.cern.ch/fts/fts-rest-flask + +JIRA Epic: https://its.cern.ch/jira/browse/FTS-1501 + +--- + +## About the migration +The migration of [fts-rest](https://gitlab.cern.ch:8443/fts/fts-rest) started after the decisions made in the [evaluation](https://its.cern.ch/jira/browse/FTS-1496) (some of which have changed). + +The evaluation considered four Python 3 Web Frameworks for the migration: +- Django +- Flask +- Pyramid +- FastAPI + +Of those I reached the conclusion that Flask was the best option: +- Many users, active ecosystem. From Jetbrain's The State of Developer Ecosystem 2020: + ![](https://codimd.web.cern.ch/uploads/upload_33952b0b8fb57e7c542567d11281814f.png) +- Its simplicity means that a significant amount of the current code can be reused. +- SQLAlchemy works well with Flask +- We don't need any third-party plugins. For example, for authorization and authentication we can reuse our custom code, we don't need Flask-login. + +## General points +- This is a migration, so the goal has been to copy the structure and code as much as possible to avoid breaking things, even if some parts could have been improved. + +- We will support CentOS 7 Python 3.6 (good enough as .7 and .8 don't bring anything crucial. 3.6 EOL is end of 2021) + +- I updated all Copyright issues that hadn't been updated. At some point I added NOTICE, it's not necessary to have a notice in every file. + +## History +You can see an approximate history of the development in the list of tickets in the JIRA Epic. Briefly: +- Evaluated options for migration +- Created the CI (initially static code analysis) +- Created a minimal Flask app to learn about the framework and do some proofs of concept +- Migrated DB models +- Migrated client +- Created build script for client +- Migrated HTTP exceptions +- Migrated routing +- Migrated controllers +- Migrated the test client + CI +- Migrated authentication and authorization +- Migrated functional tests +- Migrated admin config web pages +- Created RPMs with Apache config + CI +- Configured SELinux +- Configured continuous delivery +- Deploy to fts-flask-03 + +## Development servers +- fts-flask-02: for development, server runs from virtual environment in /home with local DB. local user ftsflask +- fts-flask-03: pre-production environment, runs from RPM with DBOD ftsflask5. local user fts3 + +You can already make successful tranfers with https://fts-flask-03.cern.ch:8446. I suggest you use the new client in the testing repository, you can install it with `yum install fts-rest-client` or alternatively install the wheel in a venv (you can get the [wheel here](https://gitlab.cern.ch/fts/fts-rest-flask/-/jobs/10128577/artifacts/file/dist/fts_client_py3-1-py3-none-any.whl). This server contains fts-rest-server and fts-server connected to DBOD ftsflask5. The installation was done manually, as if done via puppet it fails when including the fts module, as it tries to install fts-rest (python2) and many other things. + +Note: When I installed fts-server manually it failed because missing libzmq.so.5. I solved it by installing yum install zeromq-devel which is not a dependency currently. + +### Create a development server +```bash +# Create VM +ssh garciacc@aiadm.cern.ch +unset OS_PROJECT_ID; +unset OS_TENANT_ID; +unset OS_TENANT_NAME; +export OS_PROJECT_NAME="IT FTS development"; +ai-bs --foreman-hostgroup fts/flask --cc7 --foreman-environment ftsclean \ + --landb-responsible fts-devel --nova-flavor m2.large \ + fts-flask-02 + +# Install dependencies +ssh root@fts-flask-02 +yum install centos-release-scl-rh +yum-config-manager --enable centos-sclo-rh +yum install python3-devel openssl-devel swig gcc gcc-c++ make httpd-devel \ +mysql-devel gfal2-python3 gfal2-plugin-mock rh-python36-mod_wsgi \ +git mariadb mariadb-server gridsite -y + +# Prepare DB and log directories +systemctl start mariadb +mkdir /var/run/mariadb +chown mysql:mysql /var/run/mariadb +mkdir /var/log/fts3rest +chown ftsflask /var/log/fts3rest + +# Prepare application and Python dependencies +su ftsflask +cd +git clone https://gitlab.cern.ch/fts/fts-rest-flask.git +cd fts-rest-flask +# use --system-site-packages in order to use gfal2-python3 +python3 -m venv venv --system--site-packages +source venv/bin/activate +pip install --upgrade pip +pip install pip-tools +. ./pipcompile.sh +. ./pipsyncdev.sh + +# Load DB +cd .. +curl -O https://gitlab.cern.ch/fts/fts3/-/raw/fts-oidc-integration/src/db/schema/mysql/fts-schema-6.0.0.sql +mysql_secure_installation # put a password for root +echo "CREATE DATABASE ftsflask;" | mysql --user=root --password +mysql --user=root --password ftsflask < fts-schema-6.0.0.sql +echo "CREATE USER ftsflask;" | mysql --user=root --password +echo "GRANT ALL PRIVILEGES ON ftsflask.* TO 'ftsflask'@'localhost' IDENTIFIED BY 'anotherpassword';" | mysql --user=root --password +cd fts-rest-flask +. runtests.sh + +# Prepare server +exit +cp fts-rest-flask/src/fts3rest/httpd_fts.conf /etc/httpd/conf.d/ +setenforce 0 +chmod o+rx -R /home/ftsflask/ +systemctl restart httpd +``` +### Create a development environment (e.g. in your PC) +- clone the repository and cd into it +- create a venv and activate it +- run `pip install --upgrade pip` +- run `pip install pip-tools` +- run `source pipcompile.sh` +- run `source pipsyncdev.sh`. If these steps fail, it's because you are missing some system dependencies. +Check the beginning of .gitlab-ci/Dockerfile to see what you need to install. +- run `source precommit_install.sh` + +### How to run development server +Flask: +``` +export PYTHONPATH=/home/ftsflask/fts-rest-flask/src:/home/ftsflask/fts-rest-flask/src/fts3rest +export FLASK_APP=/home/ftsflask/fts-rest-flask/src/fts3rest/fts3restwsgi.py +export FLASK_ENV=development +flask run +curl http://127.0.0.1:5000/hello +``` +httpd: +``` +cp /home/ftsflask/fts-rest-flask/src/fts3rest/httpd_fts.conf /etc/httpd/conf.d/ +systemctl start httpd +curl http://localhost:80/hello +``` + +### Connect to local database +To access the admin config web page: +``` +INSERT INTO t_authz_dn VALUES ('yourdn', 'config'); +``` + +### Run tests +``` +source runtests.sh +``` +--- + +## DB +- It only works with MySQL 5. MySQL 8 doesn't work because of the outdated version of SQLAlchemy in CentOS 7. +- SQLAlchemy models: they are the same, some had to be updated to match the DB schema because they had been oudated for a long time. +- We don't need Flask-SQLAlchemy: https://its.cern.ch/jira/browse/FTS-1538, https://its.cern.ch/jira/browse/FTS-1548 +- Connect to ftsflask5: `mysql -h dbod-ga022 -P 5503 -u admin -p` + +## Git workflow +- `Master` cannot be pushed to directly. +- Create a new branch for each ticket and merge it to master through a merge request. + +## CI +I took the oportunity to start using Gitlab CI instead of Jenkins. With the new CI, we have static code analysis, functional testing, bulding and deployment. + +### Docker image for CI +I created a Docker image containing the necessary tools for CI so they don't have to be installed before every pipeline run, thus saving time. The Dockerfile is in the .gitlab-ci directory and the image is in the container registry for the project. + +To build and push the image, cd to .gitlab-ci and run .docker_push.sh. This should be done when new dependencies are added or they need to be updated. For example, one time the pipeline stopped working because black was failing. It turned out that the local and CI versions of black were different; this was fixed by recreating the CI image, which updated the CI tools. + +### Multiple Python 3 versions + +In the image I have installed Python 3.6, 3.7 and 3.8 so Pylint and the functional tests run with every Python3 version that is currently supported. In FTS we only support the CentOS 7 version: Python3.6. However it's good to have that because the logs show things that will be removed in future versions and things that may break. + +To manage multiple Python versions I considered Tox, but it didn't seem necessary and added complexity. In the end I installed every version with pyenv. Before each stage in a pipeline, the version to use is set accordingly. + +### Pipeline stages + +The current pipeline runs for every push in every branch. These are the stages: +- black: fails if the code hasn't been formatted with black +- pylint: fails if the code has syntax errors. If it fails and you are sure that pylint is mistaken, add `# pylint: skip-file` at the beginning of the relevant file. Runs for every supported Python3 version. +- radon: fails if the code complexity is too high. +- functional tests: Run for every supported Python3 version and calculate code coverage, which we didn't have before. +- bandit: detects potential security issues in the code, but it's allowed to fail as there may be false positives. The logs should be checked regularly to see if there are issues to fix. To ignore a false positive, append `# nosec"` to the offending line. +- build: RPM for the client and server, plus sdist and wheel for the client. +- deploy: upload client and server RPM to the FTS testing repository. + +Merge requests will proceed only if the pipeline succeeds. + +In case of emergency the pipeline can be [skipped](https://docs.gitlab.com/ee/ci/yaml/#skipping-jobs). + +### pre-commit hook +Developers should add the `pre-commit` hook to their local repository. This scripts does this for every commit: +- Runs black to format the changed files. +- Runs pylint only on the changed files (for speed). As pylint works better when it is run on all the project, some rules have been disabled. +- Runs radon and bandit only on the changed files. +The hook can be skipped, in case bandit detects false positives, with the commit option `--no-verify`. + + +## README +https://gitlab.cern.ch/fts/fts-rest-flask/-/blob/master/README.md + +Already integrated in this document. But it need to be updated. + +## Changes in directories and files +- I've removed __init__.py files that were unnecessary +- fts3/model has been moved to fts3rest/fts3rest/model, as it only concerns the server code and there was no reason for it to be there +- fts3/util/config.py has been moved to fts3rest/fts3rest/config for the same reasons +- fts3/rest/client/pycurlRequest.py has been removed. We now only support python-requests +- fts3rest/fts3rest/config/routing/oauth2.py has been removed, as the endpoints were not used +- fts3rest/fts3rest/config/environment.py has been combined with middleware.py +- removed these files for being Pylons specific: + - fts3rest/fts3rest/lib/middleware/request_logger.py + - fts3rest/fts3rest/lib/app_globals.py + - fts3rest/fts3rest/lib/base.py +- fts3rest/fts3rest/lib/JobBuilder.py has been divided in 2 files because the cyclomatic complexity was extremely high. +- fts3rest.lib.base has been replaced by fts3rest.model.meta, which contains Session +- fts3rest/fts3rest/public has been renamed to static +- fts3rest/fts3rest/lib/middleware/error_as_json.py removed as not needed anymore + + +## Miscellany +- ErrorasJson middleware converted to a Flask's error handler (in middleware.py) +- Mako templates migrated by compiling them with the library. The most important difference between Pylons and Flask is that Pylons uses Mako templates and Flask uses Jinja2 templates (we are talking about HTML templates). Fortunately I was able to configure Mako's engine in Flask and so I didn't have to translate the templates. +- Pylon's controller classes are now Flask's view functions. When a controller class had __init__ code and was subclassed, it was been converted to a view class. See for example fts3rest/fts3rest/controllers/delegation.py +- Migrated Pylon's webob exceptions to Flask's werkzeug exceptions +- Renamed fts3config to ftsrestconfig. The problem is fts3config is installed by fts-server, which means that every time the configuration options for fts-rest need a change, the fts-server package has to be updated. This is unnecessary coupling, so now fts-rest has its own independent configuration file. + +## API Documentation +The file controllers/api.py contains code for the api documentation and is not trivial to migrate. We'll need to find a way to migrate documentation. It should be written in the code and then converted to markdown or html with a publickly available tool. Currently the endpoints are documented with decorators that have not been migrated. See also https://its.cern.ch/jira/browse/FTS-1618 and https://its.cern.ch/jira/browse/FTS-1554. + + +## Testing +- In fts-rest, each functional test module was a class that subclassed TestController. This superclass set up a Pylon's WebTest client. However, Flask has it's own test client. In order to be able to reuse all the existing functional tests without changes, I created a test client that subclasses the Flask test client but has the same interface as the Pylon's test client. It adapts the request methods so they can be used with old functional tests created for Pylon's WebTest. +- Selenium tests are in a bad state, so not migrated: https://its.cern.ch/jira/browse/FTS-1613 +- Same with unit tests: https://its.cern.ch/jira/browse/FTS-1614 +- OpenID tests don't run in CI because the container would need a client registered and this is difficult to set up. To run these tests in a development environment, the environment variables 'xdc_ClientId' and 'xdc_ClientSecret' must be set. + +## Packaging and deployment +- We have a python package that can be build with setup.py (sdist and wheel). Eventually it should be uploaded to PyPI. + +- Some dependencies are not found in EPEL or any other community repositories so we have to package them and upload them to our repo. These packages are listed in the spec. There's an easy way to generate an RPM from setup.py: missing rpms can be generated by downloading the latest package from PyPI and running `python3 setup.py bdist_rpm`. This will generate an RPM that should be uploaded to our repo. Note that when you do this, the generated RPM doesn't include the list of dependencies, so you must also do the same for them if they aren't in the official repos. + +- Spec file has been divided in client and server, plus subpackages don't exist anymore. Need to choose the package version number. + +- The package names are + - `fts-rest-server` + - `fts-rest-client` + +- Configured SELinux for the server. Wrote the rules in the RPM scriplets as writing a module is not well documented. + +- Created a docker image, need to test it. Some users use containers such as Fermilab. + +- Apache configuration: mod_wsgi for python3 is needed, and it cannot be found in the default repos in CC7. So we have to use Red Hat repos. + +- Client already advertised to Cristina and DTO (it's beta) + +### Python dependencies +This project uses [pip-tools](https://github.com/jazzband/pip-tools) to manage dependencies: +- `requirements.in`: list of dependencies for the production app +- `dev-requirements.in`: extra list of packages used for development (e.g. static code analysis) +- `pipcompile.sh`: run it in the development venv in order to generate `requirements.txt` +- `pipsyncdev.sh`: run it afterwards to synchronize the virtual environment with the requirements. + +### Installation requirements +Because we need mod_wsgi built for Python 3.6, we need to use rh-python36-mod_wsgi +``` +yum-config-manager --enable centos-sclo-rh +yum install rh-python36-mod_wsgi +``` +All other requirements are specified in the spec files. + +### Build packages +Check .gitlab-ci.yml to see how the packages are built + +## Problems +- One problem is that the development environment and the CI image run the code in a virtual environment with the latest dependencies, while the RPM uses outdated dependencies form the repositories. This means that some bug caused due to old dependencies won't be caught until production. +- Some new commits might have not been migrated to Python3 +- Authentication for WebFTS doesn't work. fts3rest/lib/middleware/fts3auth/methods/http.py. This cannot be migrated because m2ext is a 9 year old obsolete package. Apparently it's used by WebFTS. + +## Probable causes if bugs appear: +- Python2 str is Python3 bytes +- Python2 filter/map return list, Python3 return generator + +## TO DO +- Put fts-flask-03 (or a new server) behind an alias so it runs with real traffic +- Decide version number +- Update Gitlab README +- Add sqlalchemy version which supports mysql8 +- Add CI image with rpms diff --git a/packaging/fts-rest-client.spec b/packaging/fts-rest-client.spec index 302ba9e3f351ed50aab1bd2d1a5450a3297754a7..52f6770b38eaa21c7d71be2ebb4a9b55cb2e37ad 100644 --- a/packaging/fts-rest-client.spec +++ b/packaging/fts-rest-client.spec @@ -24,13 +24,17 @@ File Transfer Service (FTS) -- Python3 Client and CLI %py3_build %install +mkdir -p %{buildroot}%{_sysconfdir}/fts3 +cp src/cli/fts3client.cfg %{buildroot}%{_sysconfdir}/fts3 %py3_install + %files %license LICENSE %{python3_sitelib}/fts3/ %{python3_sitelib}/fts*-*.egg-info/ %{_bindir}/fts-rest-* +%config(noreplace) %{_sysconfdir}/fts3/fts3client.cfg %changelog * Mon May 18 2020 Carles Garcia Cabot - 0.1-1 diff --git a/packaging/fts-rest-server.spec b/packaging/fts-rest-server.spec index 1e9106e718c3310126dffd2259e8d7909ec5e9e6..80364326c633db540181c5941519ec161b9517f4 100644 --- a/packaging/fts-rest-server.spec +++ b/packaging/fts-rest-server.spec @@ -34,7 +34,7 @@ Requires: python36-pycryptodomex Requires: python36-markupsafe ### The packages below are not found in community repos and will have to be packaged by us -# from oic (pyjwkest: six, future? +# from oic (pyjwkest may require six, future...) Requires: pyjwkest Requires: Beaker Requires: typing_extensions @@ -56,6 +56,9 @@ File Transfer Service (FTS) -- Python3 HTTP API Server %build python3 -m compileall fts3rest/fts3rest +# Check https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/ +# Program files go in /usr/lib/python3.6/site-packages +# Where does the WSGI file go? /usr/libexec %install mkdir -p %{buildroot}%{python3_sitelib} mkdir -p %{buildroot}%{_libexecdir}/fts3rest diff --git a/packaging/fts-rest.spec b/packaging/fts-rest.spec deleted file mode 100644 index 755fd888ac02e76cdd5ccdb48c670e410d284978..0000000000000000000000000000000000000000 --- a/packaging/fts-rest.spec +++ /dev/null @@ -1,225 +0,0 @@ -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print (get_python_lib())")} -%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print (get_python_lib(1))")} - -%{!?nosetest_path: %global nosetest_path "/tmp"} - -Name: fts-rest -Version: 3.10.0 -Release: 1%{?dist} -BuildArch: noarch -Summary: FTS3 Rest Interface -Group: Applications/Internet -License: ASL 2.0 -URL: http://fts3-service.web.cern.ch/ - -Source0: %{name}-%{version}.tar.gz - -BuildRequires: gfal2-python -BuildRequires: gfal2-plugin-mock -BuildRequires: cmake -BuildRequires: python-jsonschema - -BuildRequires: python-dateutil -BuildRequires: python-pylons -BuildRequires: m2crypto -BuildRequires: python-mock -BuildRequires: python-m2ext -BuildRequires: python-sqlalchemy -BuildRequires: python-requests -BuildRequires: python-dirq -BuildRequires: python-jwcrypto -BuildRequires: python-jwt -BuildRequires: python-oic -BuildRequires: MySQL-python - -Requires: gridsite%{?_isa} >= 1.7 -Requires: httpd%{?_isa} -Requires: mod_wsgi -Requires: python-fts = %{version}-%{release} -Requires: gfal2-python%{?_isa} -%description -This package provides the FTS3 REST interface - -%if %{?rhel}%{!?rhel:0} >= 7 -%package firewalld -Summary: FTS3 Rest Firewalld -Group: Applications/Internet - -Requires: firewalld-filesystem - -%description firewalld -FTS3 Rest firewalld. -%endif - - -%package cli -Summary: FTS3 Rest Interface CLI -Group: Applications/Internet - -Requires: python-fts = %{version}-%{release} -Requires: python-m2ext - -%description cli -Command line utilities for the FTS3 REST interface - -%package selinux -Summary: SELinux support for fts-rest -Group: Applications/Internet - -Requires: %{name} = %{version}-%{release} - -%description selinux -SELinux support for fts-rest - - -%post -/sbin/service httpd condrestart >/dev/null 2>&1 || : -if [ "$1" -eq "2" ]; then # Upgrade - # 3.5.1 needs owner to be fts3, since fts3rest runs as fts3 - chown fts3.fts3 /var/log/fts3rest - chown fts3.fts3 /var/log/fts3rest/fts3rest.log || true -fi - -%postun -if [ "$1" -eq "0" ] ; then - /sbin/service httpd condrestart >/dev/null 2>&1 || : -fi - -%post selinux -if [ "$1" -le "1" ] ; then # First install -semanage port -a -t http_port_t -p tcp 8446 -setsebool -P httpd_can_network_connect=1 -setsebool -P httpd_setrlimit=1 -semanage fcontext -a -t httpd_log_t "/var/log/fts3rest(/.*)?" -restorecon -R /var/log/fts3rest/ -fi - -%preun selinux -if [ "$1" -lt "1" ] ; then # Final removal -semanage port -d -t http_port_t -p tcp 8446 -setsebool -P httpd_can_network_connect=0 -setsebool -P httpd_setrlimit=0 -fi - -%prep -%setup -q - -%build -# Make sure the version in the spec file and the version used -# for building matches -fts_api_ver=`awk 'match($0, /^API_VERSION = dict\(major=([0-9]+), minor=([0-9]+), patch=([0-9]+)\)/, m) {print m[1]"."m[2]"."m[3]; }' src/fts3rest/fts3rest/controllers/api.py` -fts_spec_ver=`expr "%{version}" : '\([0-9]*\\.[0-9]*\\.[0-9]*\)'` -if [ "$fts_api_ver" != "$fts_spec_ver" ]; then - echo "The version in the spec file does not match the api.py version!" - echo "$fts_api_ver != %{version}" - exit 1 -fi - -%cmake . -DCMAKE_INSTALL_PREFIX=/ -DPYTHON_SITE_PACKAGES=%{python_sitelib} - -make %{?_smp_mflags} - -%check -pushd src/fts3rest -PYTHONPATH=../ nosetests --with-xunit --xunit-file=%{?nosetest_path}/nosetests.xml --no-skip -popd - -%install -mkdir -p %{buildroot} -make install DESTDIR=%{buildroot} -mkdir -p %{buildroot}/%{_var}/cache/fts3rest/ -mkdir -p %{buildroot}/%{_var}/log/fts3rest/ - - -cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib} -cat > %{buildroot}/%{python_sitelib}/fts3.egg-info < - 3.9.2-1 -- New bugfix release diff --git a/packaging/requirements/python36-jwcrypto-0.7.noarch.rpm b/packaging/requirements/python36-jwcrypto-0.7.noarch.rpm deleted file mode 100644 index 2b58975792fe105309814533191e384b1486d2f8..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-jwcrypto-0.7.noarch.rpm and /dev/null differ diff --git a/packaging/requirements/python36-mako-1.1.2.noarch.rpm b/packaging/requirements/python36-mako-1.1.2.noarch.rpm deleted file mode 100644 index acc006fa9fa59aaa97c0e7ff7f99e81b07517057..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-mako-1.1.2.noarch.rpm and /dev/null differ diff --git a/packaging/requirements/python36-markupsafe-1.1.1.x86_64.rpm b/packaging/requirements/python36-markupsafe-1.1.1.x86_64.rpm deleted file mode 100644 index b68d9ebdc99b6881b5187abae2b84e6a69c33fd0..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-markupsafe-1.1.1.x86_64.rpm and /dev/null differ diff --git a/packaging/requirements/python36-mysqlclient-1.4.6.x86_64.rpm b/packaging/requirements/python36-mysqlclient-1.4.6.x86_64.rpm deleted file mode 100644 index 6388974161ee95601ab1ea501f4438811296a437..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-mysqlclient-1.4.6.x86_64.rpm and /dev/null differ diff --git a/packaging/requirements/python36-oic-1.2.0.noarch.rpm b/packaging/requirements/python36-oic-1.2.0.noarch.rpm deleted file mode 100644 index 96a74d09f56bfc9c03fd3335296eb2dbaffdf4e1..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-oic-1.2.0.noarch.rpm and /dev/null differ diff --git a/packaging/requirements/python36-pycryptodomex-3.9.7.x86_64.rpm b/packaging/requirements/python36-pycryptodomex-3.9.7.x86_64.rpm deleted file mode 100644 index f37b9f4b453b2c3640a5dc00f20954c95d359e8b..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-pycryptodomex-3.9.7.x86_64.rpm and /dev/null differ diff --git a/packaging/requirements/python36-pyjwkest-1.4.3.noarch.rpm b/packaging/requirements/python36-pyjwkest-1.4.3.noarch.rpm deleted file mode 100644 index 8078f008e3fb207eb00f39f16259b4d65329df73..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-pyjwkest-1.4.3.noarch.rpm and /dev/null differ diff --git a/packaging/requirements/python36-pyjwt-1.7.1.noarch.rpm b/packaging/requirements/python36-pyjwt-1.7.1.noarch.rpm deleted file mode 100644 index f5eee325fb95aca2b82e7ac2a4ef533b2a26bd94..0000000000000000000000000000000000000000 Binary files a/packaging/requirements/python36-pyjwt-1.7.1.noarch.rpm and /dev/null differ diff --git a/packaging/todo.txt b/packaging/todo.txt deleted file mode 100644 index 422a704b17d791826f2f9eddce3f0d64fb24e184..0000000000000000000000000000000000000000 --- a/packaging/todo.txt +++ /dev/null @@ -1,7 +0,0 @@ -- Check https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/ -- Program files go in /usr/lib/python3.6/site-packages -- Where does the WSGI file go? /usr/libexec - -- Missing rpms will be generated by downloading the latest package from pip and -doing a setup.py bdist_rpm - diff --git a/src/cli/fts3client.cfg b/src/cli/fts3client.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8ffce888ab761765a01603c43dec29b0c116ce34 --- /dev/null +++ b/src/cli/fts3client.cfg @@ -0,0 +1,4 @@ +[Main] +endpoint: None +ukey: None +ucert: None diff --git a/src/fts3/rest/client/easy/submission.py b/src/fts3/rest/client/easy/submission.py index b029968679899716981266a7b88c72ce2cfd5b6f..fe724e0d35382fffdb5746151dd7812909a653b2 100644 --- a/src/fts3/rest/client/easy/submission.py +++ b/src/fts3/rest/client/easy/submission.py @@ -78,7 +78,10 @@ def new_transfer( Returns: An initialized transfer """ - transfer = dict(sources=[source], destinations=[destination],) + transfer = dict( + sources=[source], + destinations=[destination], + ) if checksum: transfer["checksum"] = checksum if filesize: diff --git a/src/fts3rest/fts3rest/config/middleware.py b/src/fts3rest/fts3rest/config/middleware.py index 910d944aee2c8d5663a2b8d46511c795584edab1..548742c515e187448009e82c9903ae3c321fa84a 100644 --- a/src/fts3rest/fts3rest/config/middleware.py +++ b/src/fts3rest/fts3rest/config/middleware.py @@ -15,6 +15,7 @@ from fts3rest.lib.helpers.connection_validator import ( connection_validator, connection_set_sqlmode, ) +from fts3rest.lib.heartbeat import Heartbeat from fts3rest.lib.middleware.fts3auth.fts3authmiddleware import FTS3AuthMiddleware from fts3rest.lib.middleware.timeout import TimeoutHandler from fts3rest.lib.openidconnect import oidc_manager @@ -118,11 +119,18 @@ def create_app(default_config_file=None, test=False): response = e.get_response() # replace the body with JSON response.data = json.dumps( - {"status": f"{e.code} {e.name}", "message": e.description,} + { + "status": f"{e.code} {e.name}", + "message": e.description, + } ) response.content_type = "application/json" return response + # Heartbeat thread + if not test: + Heartbeat("fts_rest", int(app.config.get("fts3.HeartBeatInterval", 60))).start() + # Start OIDC clients if "fts3.Providers" in app.config and app.config["fts3.Providers"]: oidc_manager.setup(app.config) diff --git a/src/fts3rest/fts3rest/config/routing/__init__.py b/src/fts3rest/fts3rest/config/routing/__init__.py index 58e3e075e1eda2a5d5c623ff189b50f2a837ae64..5b4e66ca1690dca30c66ab4c816ce9063d8ed25c 100644 --- a/src/fts3rest/fts3rest/config/routing/__init__.py +++ b/src/fts3rest/fts3rest/config/routing/__init__.py @@ -15,13 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Routes configuration - -The more specific and detailed routes should be defined first so they -may take precedent over the more generic routes. For more information -refer to the routes manual at http://routes.groovie.org/docs/ -""" - import pkgutil import sys diff --git a/src/fts3rest/fts3rest/lib/heartbeat.py b/src/fts3rest/fts3rest/lib/heartbeat.py new file mode 100644 index 0000000000000000000000000000000000000000..194146fe4ef74ea4a863b7a98228c885bdc33831 --- /dev/null +++ b/src/fts3rest/fts3rest/lib/heartbeat.py @@ -0,0 +1,44 @@ +import logging +import socket +import time +from datetime import datetime +from threading import Thread + +from fts3rest.model import Host +from fts3rest.model.meta import Session + +log = logging.getLogger(__name__) + + +class Heartbeat(Thread): + """ + Keeps running on the background updating the db marking the process as alive + """ + + def __init__(self, tag, interval): + """ + Constructor + """ + Thread.__init__(self) + self.tag = tag + self.interval = interval + self.daemon = True + + def run(self): + """ + Thread logic + """ + host = Host( + hostname=socket.getfqdn(), + service_name=self.tag, + ) + + while self.interval: + host.beat = datetime.utcnow() + try: + Session.merge(host) + Session.commit() + log.debug("Hearbeat") + except Exception as e: + log.warning("Failed to update the heartbeat: %s" % str(e)) + time.sleep(self.interval) diff --git a/src/fts3rest/fts3rest/lib/middleware/error_as_json.py b/src/fts3rest/fts3rest/lib/middleware/error_as_json.py deleted file mode 100644 index 80fe24dd3b715095bf158953609327410d12fc46..0000000000000000000000000000000000000000 --- a/src/fts3rest/fts3rest/lib/middleware/error_as_json.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright Members of the EMI Collaboration, 2013. -# Copyright 2020 CERN -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - - -class ErrorAsJson: - """ - This middleware encodes an error as a json message if json was - requested in the headers. Otherwise, let the error go and someone else catch it - """ - - def __init__(self, wrap_app): - self.app = wrap_app - - def __call__(self, environ, start_response): - accept = environ.get("HTTP_ACCEPT", "application/json") - is_json_accepted = "application/json" in accept - - self._status_msg = None - self._status_code = None - - def override_start_response(status, headers, exc_info=None): - self._status_code = int(status.split()[0]) - if self._status_code >= 400 and is_json_accepted: - headers = [ - h - for h in headers - if h[0].lower() not in ("content-type", "content-length") - ] - headers.append(("Content-Type", "application/json")) - self._status_msg = status - return start_response(status, headers, exc_info) - - response = self.app(environ, override_start_response) - - if self._status_code >= 400 and is_json_accepted: - # todo the problem is this contains html - # check src/fts3rest/fts3rest/lib/JobBuilder.py: raise BadRequest("Invalid value within the request: %s" % str(ex)) - err_msg = "".join(response.decode()) - json_error = {"status": self._status_msg, "message": err_msg} - response = [json.dumps(json_error)] - return response diff --git a/src/fts3rest/fts3rest/lib/middleware/fts3auth/methods/http.py b/src/fts3rest/fts3rest/lib/middleware/fts3auth/methods/http.py index ea3028aebe9dbcc072a96839a1220b6823ca5106..2a6c8dcedba55617792a691f7e94956052f0d39e 100644 --- a/src/fts3rest/fts3rest/lib/middleware/fts3auth/methods/http.py +++ b/src/fts3rest/fts3rest/lib/middleware/fts3auth/methods/http.py @@ -1,4 +1,4 @@ """ -This cannot be migrated becaue m2ext is a 9 year old obsolete package +This cannot be migrated because m2ext is a 9 year old obsolete package Apparently it's used by WebFTS """ diff --git a/src/fts3rest/fts3rest/templates/mako.py b/src/fts3rest/fts3rest/templates/mako.py index 7a8a49551d4fdc9a2c82fd158b100d4bbd713ceb..f16935992aeb0ce7ba94353a6cf15eb1799f2f54 100644 --- a/src/fts3rest/fts3rest/templates/mako.py +++ b/src/fts3rest/fts3rest/templates/mako.py @@ -7,7 +7,7 @@ path = os.path.abspath(os.path.dirname(__file__)) mylookup = TemplateLookup(directories=[path], module_directory=tempfile.mkdtemp()) -# todo: are static files found? + def render_template(template_name, **context): mytemplate = mylookup.get_template(template_name) return mytemplate.render(**context) diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py index eb646bb39959580d0ef698bda23d1a034f6a0178..d058f023d524205017e0e1c2d08fe2d62b257d0e 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_cloud.py @@ -94,7 +94,11 @@ class TestConfigCloud(TestController): self.test_add_s3() self.app.post_json( url="/config/cloud_storage/S3:host", - params={"vo_name": "testvo", "access_key": "1234", "secret_key": "abcd",}, + params={ + "vo_name": "testvo", + "access_key": "1234", + "secret_key": "abcd", + }, status=201, ) user = Session.query(CloudStorageUser).get(("", "S3:host", "testvo")) @@ -111,7 +115,11 @@ class TestConfigCloud(TestController): """ self.app.post_json( url="/config/cloud_storage/S3:host", - params={"vo_name": "dteam", "access_key": "1234", "secret_key": "abcd",}, + params={ + "vo_name": "dteam", + "access_key": "1234", + "secret_key": "abcd", + }, status=404, ) diff --git a/src/fts3rest/fts3rest/tests/functional/test_config_se.py b/src/fts3rest/fts3rest/tests/functional/test_config_se.py index b1dcbbd4aabb6bafe6ad860cbdf4908d63ac76cc..6ab96239e17b80a7c08e85fe4d46b5884e0e34bf 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_config_se.py +++ b/src/fts3rest/fts3rest/tests/functional/test_config_se.py @@ -14,7 +14,10 @@ class TestConfigSe(TestController): Session.commit() self.host_config = { "operations": { - "atlas": {"delete": 22, "staging": 32,}, + "atlas": { + "delete": 22, + "staging": 32, + }, "dteam": {"delete": 10, "staging": 11}, }, "se_info": { @@ -93,7 +96,10 @@ class TestConfigSe(TestController): config = { "test.cern.ch": { "operations": { - "atlas": {"delete": 1, "staging": 2,}, + "atlas": { + "delete": 1, + "staging": 2, + }, "dteam": {"delete": 3, "staging": 4}, }, "se_info": { diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_invalid_submit.py b/src/fts3rest/fts3rest/tests/functional/test_job_invalid_submit.py index cc7930fea3c7ea23892d0f379f5680bdfe438e9e..6a677f6d1c3bb81c85b8ab3054b87e4e511e7852 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_invalid_submit.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_invalid_submit.py @@ -43,7 +43,14 @@ class TestJobInvalidSubmits(TestController): self.setup_gridsite_environment() self.push_delegation() - job = {"files": [{"sources": ["/etc/passwd"], "destinations": ["/srv/pub"],}]} + job = { + "files": [ + { + "sources": ["/etc/passwd"], + "destinations": ["/srv/pub"], + } + ] + } error = self.app.post( url="/jobs", diff --git a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py index d882c34620ab5a3114158a363f0866006736503f..61917faccbc9ac440e0e96f2059f580dffe7963a 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_job_submission.py +++ b/src/fts3rest/fts3rest/tests/functional/test_job_submission.py @@ -916,7 +916,9 @@ class TestJobSubmission(TestController): "destinations": [dest_surl], } ], - "params": {"priority": 5,}, + "params": { + "priority": 5, + }, } job_id = self.app.post( url="/jobs", diff --git a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py index 05625c4a7f8bb76143aa448762bb797229a0c847..ce020a943e5a21892c6befed8897cae80853029a 100644 --- a/src/fts3rest/fts3rest/tests/functional/test_optimizer.py +++ b/src/fts3rest/fts3rest/tests/functional/test_optimizer.py @@ -51,7 +51,10 @@ class TestOptimizer(TestController): self.app.post_json( "/optimizer/current", - params={"destination": "only-dest", "nostreams": 16,}, + params={ + "destination": "only-dest", + "nostreams": 16, + }, status=400, ) self.app.post_json(