Commit bbe08a8f authored by Maria Arsuaga Rios's avatar Maria Arsuaga Rios
Browse files

Merge branch 'develop' of https://gitlab.cern.ch/fts/fts-rest into develop

parents bace8e77 9134c638
......@@ -5,6 +5,7 @@ FTS-REST 3.5.4 - Fri Oct 14 2016
## Improvement
- [[FTS-749]](https://its.cern.ch/jira/browse/FTS-749) - 40x statuses are not service errors, so they should be logged as info
FTS-REST 3.5.3 - Thu Sep 15 2016
================================
......
FTS-REST 3.6.2 - Fri Feb 10 2017
================================
## Bug
- [[FTS-880]](https://its.cern.ch/jira/browse/FTS-880) - Put back 'delegation' version to keep compatibility with older clients
## Improvement
- [[FTS-881]](https://its.cern.ch/jira/browse/FTS-881) - Add __version__ to fts3 module
FTS-REST 3.6.1 - Thu Feb 09 2017
================================
## Bug
- [[FTS-875]](https://its.cern.ch/jira/browse/FTS-875) - Session reuse misfires?
FTS-REST 3.6.0 - Tue Feb 07 2017
================================
## Bug
- [[FTS-546]](https://its.cern.ch/jira/browse/FTS-546) - Cancel jobs when job\_finished = NULL
- [[FTS-568]](https://its.cern.ch/jira/browse/FTS-568) - FTS REST and VOMS RFC proxies
- [[FTS-734]](https://its.cern.ch/jira/browse/FTS-734) - Fix schema reporting in REST API
- [[FTS-744]](https://its.cern.ch/jira/browse/FTS-744) - Packaging: /var/log/fts3rest must be owned by fts3.fts3
- [[FTS-784]](https://its.cern.ch/jira/browse/FTS-784) - Use always UTC for timestamp comparison in http
- [[FTS-798]](https://its.cern.ch/jira/browse/FTS-798) - Client implementation of the delegation broken due to m2crypto API changes
- [[FTS-836]](https://its.cern.ch/jira/browse/FTS-836) - fts-rest-http-authz-signed-cert error when comparing datetimes
## New Feature
- [[FTS-705]](https://its.cern.ch/jira/browse/FTS-705) - FTS can decide to enable or not session reuse depending on the size of the files contained on a job
## Task
- [[FTS-408]](https://its.cern.ch/jira/browse/FTS-408) - Consult roles for job listing
- [[FTS-603]](https://its.cern.ch/jira/browse/FTS-603) - Provide FTS via containers
- [[FTS-792]](https://its.cern.ch/jira/browse/FTS-792) - Drop snapshot API
## Improvement
- [[FTS-727]](https://its.cern.ch/jira/browse/FTS-727) - Remove workarounds for compatibility with schema 1.\*
- [[FTS-781]](https://its.cern.ch/jira/browse/FTS-781) - All server config now can be done per VO
- [[FTS-850]](https://its.cern.ch/jira/browse/FTS-850) - Allow staging jobs for all protocols
- [[FTS-859]](https://its.cern.ch/jira/browse/FTS-859) - Remove wait timeout from storage blacklisting
NAME=fts-rest
SPEC=rpm/$(NAME).spec
VERSION=${shell grep '^Version:' $(SPEC) | awk '{print $$2}' }
VERSION=${shell grep -m1 '^Version:' $(SPEC) | awk '{print $$2}' }
# Leave blank. To be overriden by CI tools.
RELEASE=
......
......@@ -4,19 +4,19 @@
%{!?nosetest_path: %global nosetest_path "/tmp"}
Name: fts-rest
Version: 3.6.0
Version: 3.6.2
Release: 1%{?dist}
BuildArch: noarch
Summary: FTS3 Rest Interface
Group: Applications/Internet
License: ASL 2.0
URL: http://fts3-service.web.cern.ch/
# git clone https://gitlab.cern.ch/fts/fts-rest.git --depth 1 -b master fts-rest-3.4.0
# cd fts-rest-3.4.0
# git checkout v3.4.0
# git clone https://gitlab.cern.ch/fts/fts-rest.git --depth 1 -b master fts-rest-3.6.2
# cd fts-rest-3.6.2
# git checkout v3.6.2
# git submodule init && git submodule update
# cd ..
# tar vczf fts-rest-3.4.0.tar.gz --exclude-vcs fts-rest-3.4.0
# tar vczf fts-rest-3.6.2.tar.gz --exclude-vcs fts-rest-3.6.2
Source0: %{name}-%{version}.tar.gz
BuildRequires: gfal2-python
......@@ -190,6 +190,17 @@ mkdir -p %{buildroot}/%{_var}/log/fts3rest/
cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
cat > %{buildroot}/%{python_sitelib}/fts3.egg-info <<EOF
Metadata-Version: 1.0
Name: fts3
Version: %{version}
Summary: FTS3 Python Libraries.
Home-page: http://fts3-service.web.cern.ch
Author: FTS Developers
Author-email: fts-devel@cern.ch
License: Apache2
EOF
%files
%dir %{python_sitelib}/fts3rest/
......@@ -280,6 +291,7 @@ cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
%files -n python-fts
%{python_sitelib}/fts3
%{python_sitelib}/fts3.egg-info
%doc LICENSE
%changelog
......
......@@ -52,8 +52,8 @@ if dist[0] in ('redhat', 'centos'):
base_dir = os.path.dirname(__file__)
setup(
name='fts3-rest',
version='3.6.0',
name='fts3',
version='3.6.2',
description='FTS3 Python Libraries',
author='FTS3 Developers',
author_email='fts-devel@cern.ch',
......
import pkg_resources
try:
__version__ = pkg_resources.get_distribution("fts3").version
except pkg_resources.DistributionNotFound:
__version__ = '3.x.x'
......@@ -28,7 +28,7 @@ from fts3rest.lib.base import BaseController, Session
from fts3rest.lib.helpers import jsonify
from fts3rest.lib import api
API_VERSION = dict(major=3, minor=6, patch=0)
API_VERSION = dict(major=3, minor=6, patch=2)
def _get_fts_core_version():
......@@ -68,6 +68,7 @@ class ApiController(BaseController):
.order_by(SchemaVersion.major.desc(), SchemaVersion.minor.desc(), SchemaVersion.patch.desc())\
.first()
return {
'delegation': dict(major=1, minor=0, patch=0),
'api': API_VERSION,
'core': self.fts_core_version,
'schema': schema_v,
......
......@@ -455,7 +455,6 @@ class JobBuilder(object):
for file_dict in files_list:
self._populate_files(file_dict, f_index, shared_hashed_id)
f_index += 1
if len(self.files) == 0:
raise HTTPBadRequest('No valid pairs available')
......@@ -485,16 +484,15 @@ class JobBuilder(object):
self._set_job_source_and_destination(self.files)
# If reuse is enabled, source and destination SE must be the same
# for all entries
# If reuse is enabled, source and destination SE must be the same for all entries
# Ignore for multiple replica jobs!
if job_type == 'Y' and (not self.job['source_se'] or not self.job['dest_se']):
raise HTTPBadRequest('Reuse jobs can only contain transfers for the same source and destination storage')
if job_type == 'Y' and (self.job['source_se'] and self.job['dest_se']):
self.job['job_type'] == 'Y'
if job_type == 'N':
if job_type == 'N' and not self.is_multiple:
self.job['job_type'] = 'N'
if (self.job['source_se'] and self.job['dest_se'] and (job_type is None) and (len(self.files) > 1)) :
......@@ -504,9 +502,13 @@ class JobBuilder(object):
log.debug(str(file['user_filesize']))
if file['user_filesize'] < 104857600:
small_files +=1
if small_files >= min_small_files:
if small_files > min_small_files:
self.job['job_type'] = 'Y'
log.debug("Reuse jobs with "+str(small_files)+" small files up to "+str(len(self.files))+" total files")
# Need to reset their hashed_id so they land on the same machine
shared_hashed_id = _generate_hashed_id()
for file in self.files:
file['hashed_id'] = shared_hashed_id
if self.job['job_type'] is None:
self.job['job_type'] = 'N'
......
......@@ -154,6 +154,74 @@ class TestMultiple(TestController):
uniq_hashes = set(map(lambda f: f.hashed_id, db_job.files))
self.assertEqual(len(uniq_hashes), 1)
def test_submit_with_alternatives3(self):
"""
Same as before, but reuse is set explicitly to False, which should be a no-op
(Regression bug)
"""
self.setup_gridsite_environment()
self.push_delegation()
job = {
'files': [
{
'sources': ['http://source.es/file', 'http://source.fr/file'],
'destinations': ['http://dest.ch/file'],
'selection_strategy': 'orderly',
'checksum': 'adler32:1234',
'filesize': 1024,
'activity': 'something something',
'metadata': {'mykey': 'myvalue'},
}
],
'params': {'reuse': False}
}
answer = self.app.post(url="/jobs",
content_type='application/json',
params=json.dumps(job),
status=200)
# Validate job in the database
job_id = json.loads(answer.body)['job_id']
db_job = Session.query(Job).get(job_id)
self.assertEqual(db_job.job_type, 'R')
def test_submit_with_alternatives3(self):
"""
Same as before, but reuse is set explicitly to False, which should be a no-op
(Regression bug)
"""
self.setup_gridsite_environment()
self.push_delegation()
job = {
'files': [
{
'sources': ['http://source.es/file', 'http://source.fr/file'],
'destinations': ['http://dest.ch/file'],
'selection_strategy': 'orderly',
'checksum': 'adler32:1234',
'filesize': 1024,
'activity': 'something something',
'metadata': {'mykey': 'myvalue'},
}
],
'params': {'reuse': False}
}
answer = self.app.post(url="/jobs",
content_type='application/json',
params=json.dumps(job),
status=200)
# Validate job in the database
job_id = json.loads(answer.body)['job_id']
db_job = Session.query(Job).get(job_id)
self.assertEqual(db_job.job_type, 'R')
def test_submit_multiple_transfers(self):
"""
Submit one job with multiple independent transfers
......@@ -317,6 +385,83 @@ class TestMultiple(TestController):
for f in files:
self.assertEqual(hashed, f.hashed_id)
def test_submit_reuse_auto_small(self):
"""
Submit small files with reuse not set (auto). It should be enabled.
"""
self.setup_gridsite_environment()
self.push_delegation()
job = {
'files': [
{
'sources': ['http://source.es:8446/file'],
'destinations': ['http://dest.ch:8447/file'],
'filesize': 1024,
},
{
'sources': ['http://source.es:8446/otherfile'],
'destinations': ['http://dest.ch:8447/otherfile'],
'filesize': 1024,
}
],
'params': {'overwrite': True, 'reuse': None}
}
job_id = self.app.post(
url="/jobs",
content_type='application/json',
params=json.dumps(job),
status=200
).json['job_id']
job = Session.query(Job).get(job_id)
self.assertEqual(job.job_type, 'Y')
files = Session.query(File).filter(File.job_id == job_id)
hashed = files[0].hashed_id
for f in files:
self.assertEqual(1024, f.user_filesize)
self.assertEqual(hashed, f.hashed_id)
def test_submit_reuse_auto_big(self):
"""
Submit big files with reuse not set (auto). It should be disabled.
"""
self.setup_gridsite_environment()
self.push_delegation()
job = {
'files': [
{
'sources': ['http://source.es:8446/file'],
'destinations': ['http://dest.ch:8447/file'],
'filesize': 2**30,
},
{
'sources': ['http://source.es:8446/otherfile'],
'destinations': ['http://dest.ch:8447/otherfile'],
'filesize': 2**30,
}
],
'params': {'overwrite': True, 'reuse': None}
}
job_id = self.app.post(
url="/jobs",
content_type='application/json',
params=json.dumps(job),
status=200
).json['job_id']
job = Session.query(Job).get(job_id)
self.assertEqual(job.job_type, 'N')
files = Session.query(File).filter(File.job_id == job_id)
hashed = files[0].hashed_id
for f in files:
self.assertEqual(2**30, f.user_filesize)
def test_multihop(self):
"""
Submit a multihop transfer
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment