Commit 439a0a97 authored by Andrea Manzi's avatar Andrea Manzi
Browse files

fix merge

parents c226cda4 ed7ffdaa
Pipeline #715550 passed with stage
in 1 minute and 25 seconds
......@@ -3,6 +3,19 @@ FTS3-REST
This is the FTS3 RESTful API.
For more detailed information about installation, usage, etc... please, check the [docs](docs/README.md) subdirectory.
## Firewalld: How to set up a firewall using firewalld on centos7
In order to setup the firewalld support for FTS REST you need to install the fts-rest-firewalld package.
When running fts-rest, we can allow this traffic for interfaces in our "public" zone for this session by typing:
sudo firewall-cmd --zone=public --add-service=fts3rest
You can leave out the --zone= if you wish to modify the default zone. We can verify the operation was successful by using the --list-all or --list-services operations:
sudo firewall-cmd --zone=public --list-services
## Vagrant
This repository contains a [Vagrantfile](https://www.vagrantup.com/) to make easier to develop.
Please, check [Vagran't install documentation](https://www.vagrantup.com/docs/installation/) to see how to get it
......
......@@ -4,7 +4,7 @@
%{!?nosetest_path: %global nosetest_path "/tmp"}
Name: fts-rest
Version: 3.8.2
Version: 3.8.3
Release: 1%{?dist}
BuildArch: noarch
Summary: FTS3 Rest Interface
......@@ -29,6 +29,7 @@ BuildRequires: python-nose1.1
%if %{?rhel}%{!?rhel:0} >= 7
BuildRequires: python-nose
%endif
BuildRequires: python-dateutil
BuildRequires: python-pylons
BuildRequires: scipy
......@@ -46,6 +47,7 @@ BuildRequires: python-slimit
%endif
BuildRequires: pandoc
BuildRequires: python-dirq
BuildRequires: MySQL-python
Requires: gridsite%{?_isa} >= 1.7
%if %{?rhel}%{!?rhel:0} == 6
......@@ -65,6 +67,17 @@ Requires: python-requests
%description
This package provides the FTS3 REST interface
%if %{?rhel}%{!?rhel:0} >= 7
%package firewalld
Summary: FTS3 Rest Firewalld
Group: Applications/Internet
Requires: firewalld-filesystem
%description firewalld
FTS3 Rest firewalld.
%endif
%package cloud-storage
Summary: FTS3 Rest Cloud Storage extensions
Group: Applications/Internet
......@@ -128,6 +141,7 @@ Requires: python-sqlalchemy0.8
This package provides an object model of the FTS3
database, using sqlalchemy ORM.
%post
/sbin/service httpd condrestart >/dev/null 2>&1 || :
if [ "$1" -eq "2" ]; then # Upgrade
......@@ -172,6 +186,7 @@ if [ "$fts_api_ver" != "$fts_spec_ver" ]; then
fi
%cmake . -DCMAKE_INSTALL_PREFIX=/ -DPYTHON_SITE_PACKAGES=%{python_sitelib}
make %{?_smp_mflags}
%check
......@@ -189,12 +204,14 @@ popd
%install
mkdir -p %{buildroot}
make install DESTDIR=%{buildroot}
%if %{?rhel}%{!?rhel:0} == 6
rm -rf %{buildroot}/%{_prefix}/lib/firewalld/services/fts3rest.xml
%endif
mkdir -p %{buildroot}/%{_var}/cache/fts3rest/
mkdir -p %{buildroot}/%{_var}/log/fts3rest/
cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
cat > %{buildroot}/%{python_sitelib}/fts3.egg-info <<EOF
Metadata-Version: 1.0
Name: fts3
......@@ -206,7 +223,8 @@ Author-email: fts-devel@cern.ch
License: Apache2
EOF
%files
%files
%dir %{python_sitelib}/fts3rest/
%{python_sitelib}/fts3rest.egg-info/*
......@@ -266,6 +284,11 @@ EOF
%doc docs/install.md
%doc docs/api.md
%if %{?rhel}%{!?rhel:0} >= 7
%files firewalld
%config(noreplace) %{_prefix}/lib/firewalld/services/fts3rest.xml
%endif
%files cloud-storage
%{python_sitelib}/fts3rest/config/routing/cstorage.py*
%{python_sitelib}/fts3rest/controllers/cloudStorage.py*
......@@ -300,7 +323,50 @@ EOF
%{python_sitelib}/fts3.egg-info
%doc LICENSE
%changelog
* Thu Feb 21 2019 Andrea Manzi <amanzi@cern.ch> - 3.8.3-1
- New bugfix release
* Thu Jan 10 2019 Andrea Manzi <amanzi@cern.ch> - 3.8.2-1
- New bugfix release
* Tue Oct 16 2018 Andrea Manzi <amanzi@cern.ch> - 3.8.1-1
- New bugfix release
* Mon Sep 24 2018 Andrea Manzi <amanzi@cern.ch> - 3.8.0-1
- New Minor release
- Initial support for Macaroons/Scitokens
- Multihop Scheduler merged into Standard Scheduling system
- AutoSessionReuse support
* Fri Jul 13 2018 Fedora Release Engineering <releng@fedoraproject.org> - 3.7.1-2
- Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild
* Wed Feb 07 2018 Fedora Release Engineering <releng@fedoraproject.org>
- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild
* Sun Dec 17 2017 Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
- Python 2 binary package renamed to python2-fts
See https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3
* Sat Aug 19 2017 Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl> - 3.6.3-3
- Python 2 binary package renamed to python2-fts-rest
See https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3
* Wed Jul 26 2017 Fedora Release Engineering <releng@fedoraproject.org> - 3.6.3-2
- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild
* Tue Apr 18 2017 Alejandro Alvarez Ayllon <aalvarez@cern.ch> - 3.6.3-1
- Update for new upstream release
* Fri Feb 10 2017 Fedora Release Engineering <releng@fedoraproject.org> - 3.5.4-2
- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild
* Tue Nov 15 2016 Alejandro Alvarez Ayllon <aalvarez@cern.ch> - 3.5.4-1
- Update for new upstream release
* Tue Apr 19 2016 Alejandro Alvarez Ayllon <aalvarez@cern.ch> - 3.4.0-1
- Update for new upstream release
......
......@@ -67,7 +67,7 @@ base_dir = os.path.dirname(__file__)
setup(
name='fts3-rest',
version='3.8.2',
version='3.8.3',
description='FTS3 Python Libraries',
long_description='FTS3 Python Libraries',
author='FTS3 Developers',
......
......@@ -108,7 +108,7 @@ def new_job(transfers=None, deletion=None, verify_checksum=False, reuse=None, ov
bring_online=None, copy_pin_lifetime=None,
retry=-1, retry_delay=0, metadata=None, priority=None, strict_copy=False,
max_time_in_queue=None, timeout=None,
id_generator=JobIdGenerator.standard, sid=None, s3alternate=False):
id_generator=JobIdGenerator.standard, sid=None, s3alternate=False, nostreams=1):
"""
Creates a new dictionary representing a job
......@@ -130,6 +130,7 @@ def new_job(transfers=None, deletion=None, verify_checksum=False, reuse=None, ov
id_generator: Job id generator algorithm
sid: Specific id given by the client
s3alternate: Use S3 alternate url schema
nostreams: Number of streams
Returns:
An initialized dictionary representing a job
......@@ -160,7 +161,8 @@ def new_job(transfers=None, deletion=None, verify_checksum=False, reuse=None, ov
timeout=timeout,
id_generator=id_generator,
sid=sid,
s3alternate=s3alternate
s3alternate=s3alternate,
nostreams=nostreams
)
job = dict(
files=transfers,
......
......@@ -28,7 +28,7 @@ class BadEndpoint(FTS3ClientException):
class Unauthorized(FTS3ClientException):
def __init__(self, reason):
def __init__(self, reason=None):
self.reason = reason
def __str__(self):
......
......@@ -25,3 +25,7 @@ install (FILES fts-rest.logrotate
install (FILES fts3rest.conf
DESTINATION etc/httpd/conf.d
)
# firewalld configuration file
install (FILES fts3rest.xml
DESTINATION usr/lib/firewalld/services/
)
<?xml version="1.0" encoding="utf-8"?>
<service>
<short>fts-rest</short>
<description>The File Transfer Service (FTS) coordinates transfers between SEs.</description>
<port protocol="tcp" port="8446"/>
</service>
......@@ -28,7 +28,7 @@ from fts3rest.lib.base import BaseController, Session
from fts3rest.lib.helpers import jsonify
from fts3rest.lib import api
API_VERSION = dict(major=3, minor=8, patch=2)
API_VERSION = dict(major=3, minor=8, patch=3)
def _get_fts_core_version():
......
......@@ -516,28 +516,30 @@ class JobBuilder(object):
max_size_small_file = int(pylons.config.get('fts3.AutoSessionReuseMaxSmallFileSize', 104857600)) #100MB
max_size_big_file = int(pylons.config.get('fts3.AutoSessionReuseMaxBigFileSize', 1073741824)) #1GB
max_big_files = int(pylons.config.get('fts3.AutoSessionReuseMaxBigFiles', 2))
if (auto_session_reuse == 'true' and self.job['source_se'] and self.job['dest_se'] and (job_type is None) and (len(self.files) > 1)) :
if len(self.files) > max_reuse_files:
self.job['job_type'] == 'N'
log.debug("The number of files "+str(len(self.files))+"is bigger than the auto maximum reuse files "+str(max_reuse_files))
else:
small_files = 0
big_files = 0
min_small_files = len(self.files) - max_big_files
for file in self.files:
log.debug(str(file['user_filesize']))
if file['user_filesize'] <= max_size_small_file and file['user_filesize'] > 0:
small_files +=1
else:
if file['user_filesize'] > max_size_small_file and file['user_filesize'] <= max_size_big_file:
big_files +=1
if small_files > min_small_files and big_files <= max_big_files:
self.job['job_type'] = 'Y'
log.debug("Reuse jobs with "+str(small_files)+" small files up to "+str(len(self.files))+" total files")
# Need to reset their hashed_id so they land on the same machine
shared_hashed_id = _generate_hashed_id()
if auto_session_reuse == 'true' and not self.is_multiple:
if ((self.job['source_se']) and (self.job['dest_se']) and (job_type is None) and (len(self.files) > 1)):
if len(self.files) > max_reuse_files:
self.job['job_type'] == 'N'
log.debug("The number of files "+str(len(self.files))+"is bigger than the auto maximum reuse files "+str(max_reuse_files))
else:
small_files = 0
big_files = 0
min_small_files = len(self.files) - max_big_files
for file in self.files:
file['hashed_id'] = shared_hashed_id
log.debug(str(file['user_filesize']))
if file['user_filesize'] <= max_size_small_file and file['user_filesize'] > 0:
small_files +=1
else:
if file['user_filesize'] > max_size_small_file and file['user_filesize'] <= max_size_big_file:
big_files +=1
if small_files > min_small_files and big_files <= max_big_files:
self.job['job_type'] = 'Y'
log.debug("Reuse jobs with "+str(small_files)+" small files up to "+str(len(self.files))+" total files")
# Need to reset their hashed_id so they land on the same machine
shared_hashed_id = _generate_hashed_id()
for file in self.files:
file['hashed_id'] = shared_hashed_id
if self.job['job_type'] is None:
self.job['job_type'] = 'N'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment