Merge branch 'develop' of https://gitlab.cern.ch/fts/fts-rest into develop

docs/releases/v3.5.md

@@ -5,6 +5,7 @@ FTS-REST 3.5.4 - Fri Oct 14 2016
 
 ## Improvement
 - [[FTS-749]](https://its.cern.ch/jira/browse/FTS-749) - 40x statuses are not service errors, so they should be logged as info
+
 FTS-REST 3.5.3 - Thu Sep 15 2016
 ================================
 

docs/releases/v3.6.md

+FTS-REST 3.6.2 - Fri Feb 10 2017
+================================
+
+## Bug
+- [[FTS-880]](https://its.cern.ch/jira/browse/FTS-880) - Put back 'delegation' version to keep compatibility with older clients
+
+## Improvement
+- [[FTS-881]](https://its.cern.ch/jira/browse/FTS-881) - Add __version__ to fts3 module
+
+FTS-REST 3.6.1 - Thu Feb 09 2017
+================================
+
+## Bug
+- [[FTS-875]](https://its.cern.ch/jira/browse/FTS-875) - Session reuse misfires?
+
+FTS-REST 3.6.0 - Tue Feb 07 2017
+================================
+
+## Bug
+- [[FTS-546]](https://its.cern.ch/jira/browse/FTS-546) - Cancel jobs when job\_finished = NULL
+- [[FTS-568]](https://its.cern.ch/jira/browse/FTS-568) - FTS REST and VOMS RFC proxies
+- [[FTS-734]](https://its.cern.ch/jira/browse/FTS-734) - Fix schema reporting in REST API
+- [[FTS-744]](https://its.cern.ch/jira/browse/FTS-744) - Packaging: /var/log/fts3rest must be owned by fts3.fts3
+- [[FTS-784]](https://its.cern.ch/jira/browse/FTS-784) - Use always UTC for timestamp comparison in http
+- [[FTS-798]](https://its.cern.ch/jira/browse/FTS-798) - Client implementation of the delegation broken due to m2crypto API changes
+- [[FTS-836]](https://its.cern.ch/jira/browse/FTS-836) -  fts-rest-http-authz-signed-cert error when comparing datetimes
+
+
+## New Feature
+- [[FTS-705]](https://its.cern.ch/jira/browse/FTS-705) - FTS can decide to enable or not session reuse depending on the size of the files contained on a job
+
+## Task
+- [[FTS-408]](https://its.cern.ch/jira/browse/FTS-408) - Consult roles for job listing
+- [[FTS-603]](https://its.cern.ch/jira/browse/FTS-603) - Provide FTS via containers
+- [[FTS-792]](https://its.cern.ch/jira/browse/FTS-792) - Drop snapshot API
+
+## Improvement
+- [[FTS-727]](https://its.cern.ch/jira/browse/FTS-727) - Remove workarounds for compatibility with schema 1.\*
+- [[FTS-781]](https://its.cern.ch/jira/browse/FTS-781) - All server config now can be done per VO
+- [[FTS-850]](https://its.cern.ch/jira/browse/FTS-850) - Allow staging jobs for all protocols
+- [[FTS-859]](https://its.cern.ch/jira/browse/FTS-859) - Remove wait timeout from storage blacklisting
+

packaging/Makefile

 NAME=fts-rest
 SPEC=rpm/$(NAME).spec
-VERSION=${shell grep '^Version:' $(SPEC) | awk '{print $$2}' }
+VERSION=${shell grep -m1 '^Version:' $(SPEC) | awk '{print $$2}' }
 # Leave blank. To be overriden by CI tools.
 RELEASE=
 

packaging/rpm/fts-rest.spec

@@ -4,19 +4,19 @@
 %{!?nosetest_path: %global nosetest_path "/tmp"}
 
 Name:           fts-rest
-Version:        3.6.0
+Version:        3.6.2
 Release:        1%{?dist}
 BuildArch:      noarch
 Summary:        FTS3 Rest Interface
 Group:          Applications/Internet
 License:        ASL 2.0
 URL:            http://fts3-service.web.cern.ch/
-# git clone https://gitlab.cern.ch/fts/fts-rest.git --depth 1 -b master fts-rest-3.4.0
-# cd fts-rest-3.4.0
-# git checkout v3.4.0
+# git clone https://gitlab.cern.ch/fts/fts-rest.git --depth 1 -b master fts-rest-3.6.2
+# cd fts-rest-3.6.2
+# git checkout v3.6.2
 # git submodule init && git submodule update
 # cd ..
-# tar vczf fts-rest-3.4.0.tar.gz --exclude-vcs fts-rest-3.4.0
+# tar vczf fts-rest-3.6.2.tar.gz --exclude-vcs fts-rest-3.6.2
 Source0:        %{name}-%{version}.tar.gz
 
 BuildRequires:  gfal2-python
@@ -190,6 +190,17 @@ mkdir -p %{buildroot}/%{_var}/log/fts3rest/
 
 cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
 
+cat > %{buildroot}/%{python_sitelib}/fts3.egg-info <<EOF
+Metadata-Version: 1.0
+Name: fts3
+Version: %{version}
+Summary: FTS3 Python Libraries.
+Home-page: http://fts3-service.web.cern.ch
+Author: FTS Developers
+Author-email: fts-devel@cern.ch
+License: Apache2
+EOF
+
 %files
 %dir %{python_sitelib}/fts3rest/
 
@@ -280,6 +291,7 @@ cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
 
 %files -n python-fts
 %{python_sitelib}/fts3
+%{python_sitelib}/fts3.egg-info
 %doc LICENSE
 
 %changelog

setup.py

@@ -52,8 +52,8 @@ if dist[0] in ('redhat', 'centos'):
 base_dir = os.path.dirname(__file__)
 
 setup(
-    name='fts3-rest',
-    version='3.6.0',
+    name='fts3',
+    version='3.6.2',
     description='FTS3 Python Libraries',
     author='FTS3 Developers',
     author_email='fts-devel@cern.ch',

src/fts3/__init__.py

+import pkg_resources
+
+try:
+    __version__ = pkg_resources.get_distribution("fts3").version
+except pkg_resources.DistributionNotFound:
+    __version__ = '3.x.x'

src/fts3rest/fts3rest/controllers/api.py

@@ -28,7 +28,7 @@ from fts3rest.lib.base import BaseController, Session
 from fts3rest.lib.helpers import jsonify
 from fts3rest.lib import api
 
-API_VERSION = dict(major=3, minor=6, patch=0)
+API_VERSION = dict(major=3, minor=6, patch=2)
 
 
 def _get_fts_core_version():
@@ -68,6 +68,7 @@ class ApiController(BaseController):
             .order_by(SchemaVersion.major.desc(), SchemaVersion.minor.desc(), SchemaVersion.patch.desc())\
             .first()
         return {
+            'delegation': dict(major=1, minor=0, patch=0),
             'api': API_VERSION,
             'core': self.fts_core_version,
             'schema': schema_v,

src/fts3rest/fts3rest/lib/JobBuilder.py

@@ -455,7 +455,6 @@ class JobBuilder(object):
         for file_dict in files_list:
             self._populate_files(file_dict, f_index, shared_hashed_id)
             f_index += 1
-            
 
         if len(self.files) == 0:
             raise HTTPBadRequest('No valid pairs available')
@@ -485,16 +484,15 @@ class JobBuilder(object):
 
         self._set_job_source_and_destination(self.files)
         
-        
-        # If reuse is enabled, source and destination SE must be the same
-        # for all entries
+        # If reuse is enabled, source and destination SE must be the same for all entries
+        # Ignore for multiple replica jobs!
         if job_type == 'Y' and (not self.job['source_se'] or not self.job['dest_se']):
             raise HTTPBadRequest('Reuse jobs can only contain transfers for the same source and destination storage')
         
         if job_type == 'Y' and (self.job['source_se'] and self.job['dest_se']):
             self.job['job_type'] == 'Y'
         
-        if job_type == 'N':
+        if job_type == 'N' and not self.is_multiple:
             self.job['job_type'] = 'N'
         
         if (self.job['source_se'] and self.job['dest_se'] and (job_type is None) and (len(self.files) > 1)) :
@@ -504,9 +502,13 @@ class JobBuilder(object):
                 log.debug(str(file['user_filesize']))
                 if file['user_filesize'] < 104857600:
                     small_files +=1
-            if small_files >= min_small_files:
+            if small_files > min_small_files:
                 self.job['job_type'] = 'Y'
                 log.debug("Reuse jobs with "+str(small_files)+" small files up to "+str(len(self.files))+" total files")
+                # Need to reset their hashed_id so they land on the same machine
+                shared_hashed_id = _generate_hashed_id()
+                for file in self.files:
+                    file['hashed_id'] = shared_hashed_id
         
         if self.job['job_type'] is None:
             self.job['job_type'] = 'N'

src/fts3rest/fts3rest/tests/functional/test_multiple.py

@@ -154,6 +154,74 @@ class TestMultiple(TestController):
         uniq_hashes = set(map(lambda f: f.hashed_id, db_job.files))
         self.assertEqual(len(uniq_hashes), 1)
 
+    def test_submit_with_alternatives3(self):
+        """
+        Same as before, but reuse is set explicitly to False, which should be a no-op
+        (Regression bug)
+        """
+        self.setup_gridsite_environment()
+        self.push_delegation()
+
+        job = {
+            'files': [
+                {
+                    'sources': ['http://source.es/file', 'http://source.fr/file'],
+                    'destinations': ['http://dest.ch/file'],
+                    'selection_strategy': 'orderly',
+                    'checksum': 'adler32:1234',
+                    'filesize': 1024,
+                    'activity': 'something something',
+                    'metadata': {'mykey': 'myvalue'},
+                }
+            ],
+            'params': {'reuse': False}
+        }
+
+        answer = self.app.post(url="/jobs",
+                               content_type='application/json',
+                               params=json.dumps(job),
+                               status=200)
+
+        # Validate job in the database
+        job_id = json.loads(answer.body)['job_id']
+        db_job = Session.query(Job).get(job_id)
+
+        self.assertEqual(db_job.job_type, 'R')
+
+    def test_submit_with_alternatives3(self):
+        """
+        Same as before, but reuse is set explicitly to False, which should be a no-op
+        (Regression bug)
+        """
+        self.setup_gridsite_environment()
+        self.push_delegation()
+
+        job = {
+            'files': [
+                {
+                    'sources': ['http://source.es/file', 'http://source.fr/file'],
+                    'destinations': ['http://dest.ch/file'],
+                    'selection_strategy': 'orderly',
+                    'checksum': 'adler32:1234',
+                    'filesize': 1024,
+                    'activity': 'something something',
+                    'metadata': {'mykey': 'myvalue'},
+                }
+            ],
+            'params': {'reuse': False}
+        }
+
+        answer = self.app.post(url="/jobs",
+                               content_type='application/json',
+                               params=json.dumps(job),
+                               status=200)
+
+        # Validate job in the database
+        job_id = json.loads(answer.body)['job_id']
+        db_job = Session.query(Job).get(job_id)
+
+        self.assertEqual(db_job.job_type, 'R')
+
     def test_submit_multiple_transfers(self):
         """
         Submit one job with multiple independent transfers
@@ -317,6 +385,83 @@ class TestMultiple(TestController):
         for f in files:
             self.assertEqual(hashed, f.hashed_id)
 
+    def test_submit_reuse_auto_small(self):
+        """
+        Submit small files with reuse not set (auto). It should be enabled.
+        """
+        self.setup_gridsite_environment()
+        self.push_delegation()
+
+        job = {
+            'files': [
+                {
+                    'sources': ['http://source.es:8446/file'],
+                    'destinations': ['http://dest.ch:8447/file'],
+                    'filesize': 1024,
+                },
+                {
+                    'sources': ['http://source.es:8446/otherfile'],
+                    'destinations': ['http://dest.ch:8447/otherfile'],
+                    'filesize': 1024,
+                }
+            ],
+            'params': {'overwrite': True, 'reuse': None}
+        }
+
+        job_id = self.app.post(
+            url="/jobs",
+            content_type='application/json',
+            params=json.dumps(job),
+            status=200
+        ).json['job_id']
+
+        job = Session.query(Job).get(job_id)
+        self.assertEqual(job.job_type, 'Y')
+
+        files = Session.query(File).filter(File.job_id == job_id)
+        hashed = files[0].hashed_id
+        for f in files:
+            self.assertEqual(1024, f.user_filesize)
+            self.assertEqual(hashed, f.hashed_id)
+
+    def test_submit_reuse_auto_big(self):
+        """
+        Submit big files with reuse not set (auto). It should be disabled.
+        """
+        self.setup_gridsite_environment()
+        self.push_delegation()
+
+        job = {
+            'files': [
+                {
+                    'sources': ['http://source.es:8446/file'],
+                    'destinations': ['http://dest.ch:8447/file'],
+                    'filesize': 2**30,
+                },
+                {
+                    'sources': ['http://source.es:8446/otherfile'],
+                    'destinations': ['http://dest.ch:8447/otherfile'],
+                    'filesize': 2**30,
+                }
+            ],
+            'params': {'overwrite': True, 'reuse': None}
+        }
+
+        job_id = self.app.post(
+            url="/jobs",
+            content_type='application/json',
+            params=json.dumps(job),
+            status=200
+        ).json['job_id']
+
+        job = Session.query(Job).get(job_id)
+        self.assertEqual(job.job_type, 'N')
+
+        files = Session.query(File).filter(File.job_id == job_id)
+        hashed = files[0].hashed_id
+        for f in files:
+            self.assertEqual(2**30, f.user_filesize)
+
     def test_multihop(self):
         """
         Submit a multihop transfer