fix merge

README.md

@@ -3,6 +3,19 @@ FTS3-REST
 This is the FTS3 RESTful API.
 For more detailed information about installation, usage, etc... please, check the [docs](docs/README.md) subdirectory.
 
+## Firewalld: How to set up a firewall using firewalld on centos7
+
+In order to setup the firewalld support for FTS REST you need to install the fts-rest-firewalld package.
+
+When running fts-rest, we can allow this traffic for interfaces in our "public" zone for this session by typing:
+
+    sudo firewall-cmd --zone=public --add-service=fts3rest
+
+You can leave out the --zone= if you wish to modify the default zone. We can verify the operation was successful by using the --list-all or --list-services operations:
+
+    sudo firewall-cmd --zone=public --list-services
+
+
 ## Vagrant
 This repository contains a [Vagrantfile](https://www.vagrantup.com/) to make easier to develop.
 Please, check [Vagran't install documentation](https://www.vagrantup.com/docs/installation/) to see how to get it

packaging/rpm/fts-rest.spec

@@ -4,7 +4,7 @@
 %{!?nosetest_path: %global nosetest_path "/tmp"}
 
 Name:           fts-rest
-Version:        3.8.2
+Version:        3.8.3
 Release:        1%{?dist}
 BuildArch:      noarch
 Summary:        FTS3 Rest Interface
@@ -29,6 +29,7 @@ BuildRequires:  python-nose1.1
 %if %{?rhel}%{!?rhel:0} >= 7
 BuildRequires:  python-nose
 %endif
+
 BuildRequires:  python-dateutil
 BuildRequires:  python-pylons
 BuildRequires:  scipy
@@ -46,6 +47,7 @@ BuildRequires:  python-slimit
 %endif
 BuildRequires:  pandoc
 BuildRequires:  python-dirq
+BuildRequires:  MySQL-python
 
 Requires:       gridsite%{?_isa} >= 1.7
 %if %{?rhel}%{!?rhel:0} == 6
@@ -65,6 +67,17 @@ Requires: 	python-requests
 %description
 This package provides the FTS3 REST interface
 
+%if %{?rhel}%{!?rhel:0} >= 7
+%package firewalld
+Summary: FTS3 Rest Firewalld
+Group: Applications/Internet
+
+Requires:  firewalld-filesystem
+
+%description firewalld
+FTS3 Rest firewalld.
+%endif
+
 %package cloud-storage
 Summary:        FTS3 Rest Cloud Storage extensions
 Group:          Applications/Internet
@@ -128,6 +141,7 @@ Requires:       python-sqlalchemy0.8
 This package provides an object model of the FTS3
 database, using sqlalchemy ORM.
 
+
 %post
 /sbin/service httpd condrestart >/dev/null 2>&1 || :
 if [ "$1" -eq "2" ]; then # Upgrade
@@ -172,6 +186,7 @@ if [ "$fts_api_ver" != "$fts_spec_ver" ]; then
 fi
 
 %cmake . -DCMAKE_INSTALL_PREFIX=/ -DPYTHON_SITE_PACKAGES=%{python_sitelib}
+
 make %{?_smp_mflags}
 
 %check
@@ -189,12 +204,14 @@ popd
 %install
 mkdir -p %{buildroot}
 make install DESTDIR=%{buildroot}
-
+%if %{?rhel}%{!?rhel:0} == 6
+rm -rf %{buildroot}/%{_prefix}/lib/firewalld/services/fts3rest.xml
+%endif
 mkdir -p %{buildroot}/%{_var}/cache/fts3rest/
 mkdir -p %{buildroot}/%{_var}/log/fts3rest/
 
-cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
 
+cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
 cat > %{buildroot}/%{python_sitelib}/fts3.egg-info <<EOF
 Metadata-Version: 1.0
 Name: fts3
@@ -206,7 +223,8 @@ Author-email: fts-devel@cern.ch
 License: Apache2
 EOF
 
-%files
+%files 
+
 %dir %{python_sitelib}/fts3rest/
 
 %{python_sitelib}/fts3rest.egg-info/*
@@ -266,6 +284,11 @@ EOF
 %doc docs/install.md
 %doc docs/api.md
 
+%if %{?rhel}%{!?rhel:0} >= 7
+%files firewalld
+%config(noreplace) %{_prefix}/lib/firewalld/services/fts3rest.xml
+%endif
+
 %files cloud-storage
 %{python_sitelib}/fts3rest/config/routing/cstorage.py*
 %{python_sitelib}/fts3rest/controllers/cloudStorage.py*
@@ -300,7 +323,50 @@ EOF
 %{python_sitelib}/fts3.egg-info
 %doc LICENSE
 
+
+
 %changelog
+* Thu Feb 21 2019 Andrea Manzi <amanzi@cern.ch> - 3.8.3-1
+- New bugfix release
+
+* Thu Jan 10 2019 Andrea Manzi <amanzi@cern.ch> - 3.8.2-1
+- New bugfix release
+
+* Tue Oct 16 2018 Andrea Manzi <amanzi@cern.ch> - 3.8.1-1
+- New bugfix release
+
+* Mon Sep 24 2018 Andrea Manzi <amanzi@cern.ch> - 3.8.0-1
+- New Minor release
+- Initial support for Macaroons/Scitokens
+- Multihop Scheduler merged into Standard Scheduling system
+- AutoSessionReuse support
+
+* Fri Jul 13 2018 Fedora Release Engineering <releng@fedoraproject.org> - 3.7.1-2
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild
+
+* Wed Feb 07 2018 Fedora Release Engineering <releng@fedoraproject.org>
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild
+
+* Sun Dec 17 2017 Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
+- Python 2 binary package renamed to python2-fts
+  See https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3
+
+* Sat Aug 19 2017 Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl> - 3.6.3-3
+- Python 2 binary package renamed to python2-fts-rest
+  See https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3
+
+* Wed Jul 26 2017 Fedora Release Engineering <releng@fedoraproject.org> - 3.6.3-2
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild
+
+* Tue Apr 18 2017 Alejandro Alvarez Ayllon <aalvarez@cern.ch> - 3.6.3-1
+- Update for new upstream release
+
+* Fri Feb 10 2017 Fedora Release Engineering <releng@fedoraproject.org> - 3.5.4-2
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild
+
+* Tue Nov 15 2016 Alejandro Alvarez Ayllon <aalvarez@cern.ch> - 3.5.4-1
+- Update for new upstream release
+
 * Tue Apr 19 2016 Alejandro Alvarez Ayllon <aalvarez@cern.ch> - 3.4.0-1
 - Update for new upstream release
 

setup.py

@@ -67,7 +67,7 @@ base_dir = os.path.dirname(__file__)
 
 setup(
     name='fts3-rest',
-    version='3.8.2',
+    version='3.8.3',
     description='FTS3 Python Libraries',
     long_description='FTS3 Python Libraries',
     author='FTS3 Developers',

src/fts3/rest/client/easy/submission.py

@@ -108,7 +108,7 @@ def new_job(transfers=None, deletion=None, verify_checksum=False, reuse=None, ov
             bring_online=None, copy_pin_lifetime=None,
             retry=-1, retry_delay=0, metadata=None, priority=None, strict_copy=False,
             max_time_in_queue=None, timeout=None,
-            id_generator=JobIdGenerator.standard, sid=None, s3alternate=False):
+            id_generator=JobIdGenerator.standard, sid=None, s3alternate=False, nostreams=1):
     """
     Creates a new dictionary representing a job
 
@@ -130,6 +130,7 @@ def new_job(transfers=None, deletion=None, verify_checksum=False, reuse=None, ov
         id_generator:      Job id generator algorithm
         sid:               Specific id given by the client
         s3alternate:       Use S3 alternate url schema
+        nostreams:         Number of streams
 
     Returns:
         An initialized dictionary representing a job
@@ -160,7 +161,8 @@ def new_job(transfers=None, deletion=None, verify_checksum=False, reuse=None, ov
         timeout=timeout,
         id_generator=id_generator,
         sid=sid,
-        s3alternate=s3alternate
+        s3alternate=s3alternate,
+        nostreams=nostreams
     )
     job = dict(
         files=transfers,

src/fts3/rest/client/exceptions.py

@@ -28,7 +28,7 @@ class BadEndpoint(FTS3ClientException):
 
 
 class Unauthorized(FTS3ClientException):
-    def __init__(self, reason):
+    def __init__(self, reason=None):
         self.reason = reason
 
     def __str__(self):

src/fts3rest/CMakeLists.txt

@@ -25,3 +25,7 @@ install (FILES          fts-rest.logrotate
 install (FILES          fts3rest.conf
          DESTINATION    etc/httpd/conf.d
 )
+# firewalld configuration file
+install (FILES          fts3rest.xml
+         DESTINATION    usr/lib/firewalld/services/
+)

src/fts3rest/fts3rest.xml

+<?xml version="1.0" encoding="utf-8"?>
+<service>
+  <short>fts-rest</short>
+  <description>The File Transfer Service (FTS) coordinates transfers between SEs.</description>
+  <port protocol="tcp" port="8446"/>
+</service>

src/fts3rest/fts3rest/controllers/api.py

@@ -28,7 +28,7 @@ from fts3rest.lib.base import BaseController, Session
 from fts3rest.lib.helpers import jsonify
 from fts3rest.lib import api
 
-API_VERSION = dict(major=3, minor=8, patch=2)
+API_VERSION = dict(major=3, minor=8, patch=3)
 
 
 def _get_fts_core_version():

src/fts3rest/fts3rest/lib/JobBuilder.py

@@ -516,28 +516,30 @@ class JobBuilder(object):
         max_size_small_file = int(pylons.config.get('fts3.AutoSessionReuseMaxSmallFileSize', 104857600)) #100MB
         max_size_big_file = int(pylons.config.get('fts3.AutoSessionReuseMaxBigFileSize', 1073741824)) #1GB
         max_big_files = int(pylons.config.get('fts3.AutoSessionReuseMaxBigFiles', 2))
-        if (auto_session_reuse == 'true' and self.job['source_se'] and self.job['dest_se'] and (job_type is None) and (len(self.files) > 1)) :
-            if len(self.files) > max_reuse_files:
-                self.job['job_type'] == 'N'
-                log.debug("The number of files "+str(len(self.files))+"is bigger than the auto maximum reuse files "+str(max_reuse_files))
-            else:
-                small_files = 0
-                big_files = 0
-                min_small_files = len(self.files) - max_big_files
-                for file in self.files:
-                    log.debug(str(file['user_filesize']))
-                    if file['user_filesize'] <= max_size_small_file and file['user_filesize'] > 0:
-                        small_files +=1
-                    else:
-                        if file['user_filesize'] > max_size_small_file and file['user_filesize'] <= max_size_big_file:
-                            big_files +=1
-                if small_files > min_small_files and big_files <= max_big_files:
-                    self.job['job_type'] = 'Y'
-                    log.debug("Reuse jobs with "+str(small_files)+" small files up to "+str(len(self.files))+" total files")
-                    # Need to reset their hashed_id so they land on the same machine
-                    shared_hashed_id = _generate_hashed_id()
+
+        if auto_session_reuse == 'true' and not self.is_multiple:
+            if ((self.job['source_se']) and (self.job['dest_se']) and (job_type is None) and (len(self.files) > 1)):
+                if len(self.files) > max_reuse_files:
+                    self.job['job_type'] == 'N'
+                    log.debug("The number of files "+str(len(self.files))+"is bigger than the auto maximum reuse files "+str(max_reuse_files))
+                else:
+                    small_files = 0
+                    big_files = 0
+                    min_small_files = len(self.files) - max_big_files
                     for file in self.files:
-                        file['hashed_id'] = shared_hashed_id
+                        log.debug(str(file['user_filesize']))
+                        if file['user_filesize'] <= max_size_small_file and file['user_filesize'] > 0:
+                            small_files +=1
+                        else:
+                            if file['user_filesize'] > max_size_small_file and file['user_filesize'] <= max_size_big_file:
+                                big_files +=1
+                    if small_files > min_small_files and big_files <= max_big_files:
+                        self.job['job_type'] = 'Y'
+                        log.debug("Reuse jobs with "+str(small_files)+" small files up to "+str(len(self.files))+" total files")
+                        # Need to reset their hashed_id so they land on the same machine
+                        shared_hashed_id = _generate_hashed_id()
+                        for file in self.files:
+                            file['hashed_id'] = shared_hashed_id
         
         if self.job['job_type'] is None:
             self.job['job_type'] = 'N'