Commit d13c3012 authored by Maria Arsuaga Rios's avatar Maria Arsuaga Rios
Browse files

FTS:705 Adding tests for mix small files and big files

parent 254d0231
......@@ -495,15 +495,18 @@ class JobBuilder(object):
if reuse_flag == 'Y' and (not self.job['source_se'] or not self.job['dest_se']):
raise HTTPBadRequest('Reuse jobs can only contain transfers for the same source and destination storage')
if (self.job['source_se'] and self.job['dest_se'] and reuse_flag is None) :
small_files = 0
if (self.job['source_se'] and self.job['dest_se'] and (reuse_flag is None) and (len(self.files) > 1)) :
small_files = 0
min_small_files = len(self.files) - 2
for file in self.files:
log.debug(str(file['user_filesize']))
if file['user_filesize'] < 104857600:
small_files +=1
if small_files == len(self.files):
if small_files >= min_small_files:
self.job['reuse_job'] = 'Y'
log.debug("Reuse jobs with "+str(small_files)+" small files up to "+str(len(self.files))+" total files")
else:
self.job['reuse_job'] = 'N'
def _populate_deletion(self, deletion_dict):
"""
......
......@@ -48,7 +48,7 @@ class TestJobCancel(TestController):
'destinations': ['root://dest.ch/file%d' % i],
'selection_strategy': 'orderly',
'checksum': 'adler32:1234',
'filesize': 104857600,
'filesize': 1024,
'metadata': {'mykey': 'myvalue'},
})
......@@ -64,6 +64,46 @@ class TestJobCancel(TestController):
).json['job_id']
return str(job_id)
def _submit_none_reuse(self, count=1, big_files=0):
"""
Submit a valid job without specifying reuse
"""
self.setup_gridsite_environment()
self.push_delegation()
files = []
for i in xrange(count):
files.append({
'sources': ['root://source.es/file%d' % i],
'destinations': ['root://dest.ch/file%d' % i],
'selection_strategy': 'orderly',
'checksum': 'adler32:1234',
'filesize': 1024,
'metadata': {'mykey': 'myvalue'},
})
for j in xrange(big_files):
files.append({
'sources': ['root://source.es/file%d' % i],
'destinations': ['root://dest.ch/file%d' % i],
'selection_strategy': 'orderly',
'checksum': 'adler32:1234',
'filesize': 104857600,
'metadata': {'mykey': 'myvalue'},
})
job = {
'files': files,
'params': {'overwrite': True, 'verify_checksum': True}
}
job_id = self.app.put(
url="/jobs",
params=json.dumps(job),
status=200
).json['job_id']
return str(job_id)
def test_cancel(self):
"""
......@@ -79,6 +119,8 @@ class TestJobCancel(TestController):
# Is it in the database?
job = Session.query(Job).get(job_id)
self.assertEqual(job.job_state, 'CANCELED')
self.assertEqual(job.reuse_job, 'N')
self.assertNotEqual(None, job.job_finished)
self.assertNotEqual(None, job.finish_time)
for f in job.files:
......@@ -345,6 +387,49 @@ class TestJobCancel(TestController):
file_ids = ','.join(map(lambda f: str(f['file_id']), files[0:2]))
self.app.delete(url="/jobs/%s/files/%s" % (job_id, file_ids), status=400)
def test_cancel_reuse_small_files(self):
"""
Jobs with small files can not be cancelled file per file
"""
job_id = self._submit_none_reuse(10)
files = self.app.get(url="/jobs/%s/files" % job_id, status=200).json
file_ids = ','.join(map(lambda f: str(f['file_id']), files[0:2]))
self.app.delete(url="/jobs/%s/files/%s" % (job_id, file_ids), status=400)
def test_cancel_reuse_big_files(self):
"""
Jobs with small files and one big file can not be cancelled file per file
"""
job_id = self._submit_none_reuse(10, 1)
files = self.app.get(url="/jobs/%s/files" % job_id, status=200).json
file_ids = ','.join(map(lambda f: str(f['file_id']), files[0:2]))
self.app.delete(url="/jobs/%s/files/%s" % (job_id, file_ids), status=400)
def test_cancel_reuse_small_files_and_big_files(self):
"""
Cancel a job with small files and two big files cannot be reused
"""
job_id = self._submit_none_reuse(100, 2)
job = self.app.delete(url="/jobs/%s" % job_id, status=200).json
self.assertEqual(job['job_id'], job_id)
self.assertEqual(job['job_state'], 'CANCELED')
self.assertEqual(job['reason'], 'Job canceled by the user')
# Is it in the database?
job = Session.query(Job).get(job_id)
self.assertEqual(job.job_state, 'CANCELED')
self.assertEqual(job.reuse_job, 'N')
self.assertNotEqual(None, job.job_finished)
self.assertNotEqual(None, job.finish_time)
for f in job.files:
self.assertEqual(f.file_state, 'CANCELED')
self.assertNotEqual(None, f.job_finished)
self.assertNotEqual(None, f.finish_time)
def _become_root(self):
"""
......
......@@ -49,7 +49,7 @@ class TestJobSubmission(TestController):
self.assertEqual(job.dest_se, 'root://dest.ch')
self.assertEqual(job.overwrite_flag, True)
self.assertEqual(job.verify_checksum, True)
#self.assertEqual(job.reuse_job, 'N')
self.assertEqual(job.reuse_job, 'N')
self.assertEqual(job.priority, 3)
self.assertIsNone(job.max_time_in_queue)
......@@ -114,6 +114,73 @@ class TestJobSubmission(TestController):
self._validate_submitted(Session.query(Job).get(job_id))
return str(job_id)
def test_submit_no_reuse(self):
"""
Submit a valid job no reuse
"""
self.setup_gridsite_environment()
self.push_delegation()
job = {
'files': [{
'sources': ['root://source.es/file'],
'destinations': ['root://dest.ch/file'],
'selection_strategy': 'orderly',
'checksum': 'adler32:1234',
'filesize': 1024,
'metadata': {'mykey': 'myvalue'},
}],
'params': {'overwrite': True, 'verify_checksum': True, 'reuse':False}
}
job_id = self.app.put(
url="/jobs",
content_type='application/json',
params=json.dumps(job),
status=200
).json['job_id']
# Make sure it was committed to the DB
self.assertGreater(job_id, 0)
self._validate_submitted(Session.query(Job).get(job_id))
return str(job_id)
def test_submit_no_reuse_N(self):
"""
Submit a valid job, using 'N' instead of False
"""
self.setup_gridsite_environment()
self.push_delegation()
job = {
'files': [{
'sources': ['root://source.es/file'],
'destinations': ['root://dest.ch/file'],
'selection_strategy': 'orderly',
'checksum': 'adler32:1234',
'filesize': 1024,
'metadata': {'mykey': 'myvalue'},
}],
'params': {'overwrite': True, 'verify_checksum': True, 'reuse':'N'}
}
job_id = self.app.put(
url="/jobs",
content_type='application/json',
params=json.dumps(job),
status=200
).json['job_id']
# Make sure it was committed to the DB
self.assertGreater(job_id, 0)
self._validate_submitted(Session.query(Job).get(job_id))
return str(job_id)
def test_submit_reuse(self):
"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment