Commit a6ff2ada authored by Maria Arsuaga Rios's avatar Maria Arsuaga Rios
Browse files

Merge branch 'develop' of https://gitlab.cern.ch/fts/fts-rest into develop

parents ee6257ad f5abd777
Pipeline #77603 passed with stage
in 23 seconds
......@@ -21,7 +21,7 @@ except:
import logging
from datetime import datetime
from pylons import request
from sqlalchemy import distinct, func
from sqlalchemy import distinct, func, and_
from fts3.model import BannedDN, BannedSE, Job, File, JobActiveStates, FileActiveStates
from fts3rest.lib.api import doc
......@@ -82,29 +82,43 @@ def _cancel_transfers(storage=None, vo_name=None):
Returns the list of affected jobs ids.
"""
affected_job_ids = set()
files = Session.query(File)\
.filter((File.source_se == storage) | (File.dest_se == storage))\
.filter(File.file_state.in_(FileActiveStates + ['NOT_USED']))
files = Session.query(File.file_id).filter(
and_(
(File.source_se == storage) | (File.dest_se == storage),
File.file_state.in_(FileActiveStates + ['NOT_USED'])
)
)
if vo_name and vo_name != '*':
files = files.filter(File.vo_name == vo_name)
now = datetime.utcnow()
try:
for file in files:
affected_job_ids.add(file.job_id)
for row in files:
file_id = row[0]
job_id, file_index = Session.query(File.job_id, File.file_index).filter(File.file_id == file_id).one()
affected_job_ids.add(job_id)
# Cancel the affected file
file.file_state = 'CANCELED'
file.reason = 'Storage banned'
file.finish_time = now
Session.merge(file)
Session.query(File).filter(File.file_id == file_id)\
.update({
'file_state': 'CANCELED', 'reason': 'Storage banned',
'finish_time': now
}, synchronize_session=False)
# If there are alternatives, enable them
Session.query(File).filter(File.job_id == file.job_id)\
.filter(File.file_index == file.file_index)\
.filter(File.file_state == 'NOT_USED').update({'file_state': 'SUBMITTED'})
if Session.bind.dialect.name == 'mysql':
limit = " LIMIT 1"
else:
limit = ''
Session.execute(
"UPDATE t_file SET"
" file_state = 'SUBMITTED' "
"WHERE"
" job_id = :job_id AND file_index = :file_index AND file_state = 'NOT_USED' " + limit,
dict(job_id=job_id, file_index=file_index)
)
# Or next queries will not see the changes!
Session.commit()
Session.expire_all()
except Exception:
Session.rollback()
raise
......@@ -143,7 +157,9 @@ def _cancel_jobs(dn):
Cancel all jobs that belong to dn.
Returns the list of affected jobs ids.
"""
jobs = Session.query(Job.job_id).filter(Job.job_state.in_(JobActiveStates)).filter(Job.user_dn == dn)
jobs = Session.query(Job.job_id).filter(
Job.job_state.in_(JobActiveStates), Job.user_dn == dn, Job.job_finished == None
)
job_ids = map(lambda j: j[0], jobs)
try:
......@@ -167,31 +183,39 @@ def _cancel_jobs(dn):
raise
def _set_to_wait_helper(storage, vo_name, from_state, to_state):
"""
Helper for _set_to_wait
"""
file_ids = Session.query(File.file_id).filter(
and_(
File.file_state == from_state),
(File.source_se == storage) | (File.dest_se == storage)
)
if vo_name and vo_name != '*':
file_ids = file_ids.filter(File.vo_name == vo_name)
file_ids = map(lambda j: j[0], file_ids.all())
job_ids = set()
for file_id in file_ids:
Session.query(File).filter(File.file_id == file_id).update({'file_state': to_state}, synchronize_session=False)
job_ids.add(Session.query(File).get(file_id).job_id)
return job_ids
def _set_to_wait(storage, vo_name):
"""
Updates the transfers that have the given storage either in source or destination,
and belong to the given VO.
Returns the list of affected jobs ids.
"""
job_ids = Session.query(distinct(File.job_id))\
.filter((File.source_se == storage) | (File.dest_se == storage)).filter(File.file_state.in_(FileActiveStates))
if vo_name and vo_name != '*':
job_ids = job_ids.filter(File.vo_name == vo_name)
job_ids = map(lambda j: j[0], job_ids.all())
try:
for job_id in job_ids:
Session.query(File).filter(File.job_id == job_id, File.file_state == 'STAGING')\
.update({'file_state': 'ON_HOLD_STAGING'}, synchronize_session=False)
Session.query(File).filter(File.job_id == job_id, File.file_state == 'SUBMITTED')\
.update({'file_state': 'ON_HOLD'}, synchronize_session=False)
job_ids = _set_to_wait_helper(storage, vo_name, 'SUBMITTED', 'ON_HOLD')
job_ids.update(_set_to_wait_helper(storage, vo_name, 'STAGING', 'ON_HOLD_STAGING'))
Session.commit()
Session.expire_all()
except Exception:
Session.rollback()
raise
return job_ids
def _reenter_queue(storage, vo_name):
......
......@@ -40,8 +40,6 @@ function saveStorage(form)
});
}
/**
* Delete a storage
*/
......@@ -66,9 +64,9 @@ function deleteStorage(storage_name, div)
}
/**
* Save a dropbox user
* Save a user
*/
function saveDropboxUser(storage_name, form)
function saveUser(storage_name, form)
{
var msg = {
user_dn: form.find("input[name='user-dn']").val(),
......@@ -90,29 +88,6 @@ function saveDropboxUser(storage_name, form)
});
}
/**
* Save a S3 user
*/
function saveS3User(storage_name, form)
{
var msg = {
user_dn: form.find("input[name='user-dn']").val(),
vo_name: form.find("input[name='vo-name']").val(),
access-key: form.find("input[name='access-token']").val(),
secret-key: form.find("input[name='access-secret']").val(),
};
console.log(msg);
return $.ajax({
url: "/config/cloud_storage/" + encodeURIComponent(storage_name),
type: "POST",
dataType: "json",
contentType: "application/json",
data: JSON.stringify(msg)
});
}
/**
* Delete a user
*/
......@@ -178,26 +153,12 @@ function refreshCloudStorage()
});
});
// Attach to add a dropbox user
var addDropboxUserFrm = div.find(".frm-add-dropbox-user");
var addDropboxUserBtn = addDropboxUserFrm.find(".btn-add-dropbox-user");
addDropboxUserBtn.click(function(event) {
event.preventDefault();
saveDropboxUser(storage.storage_name, addDropboxUserFrm)
.done(function(data, textStatus, jqXHR) {
refreshCloudStorage();
})
.fail(function(jqXHR) {
errorMessage(jqXHR);
});
});
// Attach to add a s3 user
var addS3UserFrm = div.find(".frm-add-s3-user");
var addS3UserBtn = addS3UserFrm.find(".btn-add-s3-user");
addS3UserBtn.click(function(event) {
// Attach to add a user
var addUserFrm = div.find(".frm-add-user");
var addUserBtn = addUserFrm.find(".btn-add-user");
addUserBtn.click(function(event) {
event.preventDefault();
saveS3User(storage.storage_name, addS3UserFrm)
saveUser(storage.storage_name, addUserFrm)
.done(function(data, textStatus, jqXHR) {
refreshCloudStorage();
})
......@@ -209,29 +170,15 @@ function refreshCloudStorage()
// Attach to remove and modify a user
div.find(".user-entry").each(function() {
var tr = $(this);
var deleteDropboxUserBtn = tr.find(".btn-delete-dropbox-user");
deleteDropboxUserBtn.click(function(event) {
var deleteUserBtn = tr.find(".btn-delete-user");
deleteUserBtn.click(function(event) {
event.preventDefault();
deleteUser(storage.storage_name, tr);
});
var saveDropboxUserBtn = tr.find(".btn-save-dropbox-user");
saveDropboxUserBtn.click(function(event) {
var saveUserBtn = tr.find(".btn-save-user");
saveUserBtn.click(function(event) {
event.preventDefault();
saveDropboxUser(storage.storage_name, tr)
});
});
/ Attach to remove and modify a user
div.find(".user-entry").each(function() {
var tr = $(this);
var deleteS3UserBtn = tr.find(".btn-delete-s3-user");
deleteS3UserBtn.click(function(event) {
event.preventDefault();
deleteUser(storage.storage_name, tr);
});
var saveS3UserBtn = tr.find(".btn-save-s3-user");
saveS3UserBtn.click(function(event) {
event.preventDefault();
saveS3User(storage.storage_name, tr)
saveUser(storage.storage_name, tr)
});
});
......@@ -243,7 +190,6 @@ function refreshCloudStorage()
});
}
/**
* Compile templates embedded into the HTML
*/
......@@ -261,15 +207,14 @@ function setupCloudStorage()
{
compileTemplates();
refreshCloudStorage();
// Bind to form
$("#add-cloud-frm").submit(function(event) {
event.preventDefault();
saveStorage($("#add-cloud-frm"))
.done(function(data, textStatus, jqXHR) {
$("#add-cloud-frm").trigger("reset");
refreshDropboxCloudStorage();
refreshS3CloudStorage();
refreshCloudStorage();
})
.fail(function(jqXHR) {
errorMessage(jqXHR);
......
......@@ -17,6 +17,7 @@
<dd>The name should have the form S3:hostname (i.e. S3:s3.example.com). App key and secret are not used.</dd>
<dt>Grant access to a set of VO and/or users</dt>
<dd>Empty user with a value in VO grant access to all members from that VO.
Access token = access key, access secret = secret key. Leave request fields empty.
</dd>
</dl>
......@@ -77,7 +78,7 @@
<tr class="user-entry">
<td>
<a><i class="glyphicon glyphicon-trash btn-delete-user"></i></a>
<a><i class="glyphicon glyphicon-floppy-disk btn-save-dropbox-user"></i></a>
<a><i class="glyphicon glyphicon-floppy-disk btn-save-user"></i></a>
</td>
<td>
<input type="hidden" name="user-dn" value="{{user_dn}}"/>
......@@ -102,10 +103,10 @@
</tr>
{{/each}}
</tbody>
<tbody class="frm-add-dropbox-user">
<tbody class="frm-add-user">
<tr>
<td>
<a><i class="glyphicon glyphicon-plus btn-add-dropbox-user"></i></a>
<a><i class="glyphicon glyphicon-plus btn-add-user"></i></a>
</td>
<td>
<input name="user-dn" type="text" class="form-control" value=""/>
......@@ -128,62 +129,6 @@
</tr>
</tbody>
</table>
<table class="table">
<thead>
<tr>
<th></th>
<th>User</th>
<th>VO</th>
<th>Access key</th>
<th>Secret key</th>
</tr>
</thead>
<tbody>
{{#each users}}
<tr class="user-entry">
<td>
<a><i class="glyphicon glyphicon-trash btn-delete-user"></i></a>
<a><i class="glyphicon glyphicon-floppy-disk btn-save-s3user"></i></a>
</td>
<td>
<input type="hidden" name="user-dn" value="{{user_dn}}"/>
{{user_dn}}
</td>
<td>
<input type="hidden" name="vo-name" value="{{vo_name}}"/>
{{vo_name}}
</td>
<td>
<input type="text" name="access-key" class="form-control" value="{{access_token}}"/>
</td>
<td>
<input type="text" name="secret-key" class="form-control" value="{{access_token_secret}}"/>
</td>
</tr>
{{/each}}
</tbody>
<tbody class="frm-add-s3-user">
<tr>
<td>
<a><i class="glyphicon glyphicon-plus btn-add-s3-user"></i></a>
</td>
<td>
<input name="user-dn" type="text" class="form-control" value=""/>
</td>
<td>
<input name="vo-name" type="text" class="form-control" value=""/>
</td>
<td>
<input name="access-key" type="text" class="form-control" value=""/>
</td>
<td>
<input name="secret-key" type="text" class="form-control" value=""/>
</td>
</tr>
</tbody>
</table>
<div class="panel-footer">
<button class="btn btn-success btn-save">Save</button>
<button class="btn btn-danger btn-delete">Delete</button>
......
......@@ -312,9 +312,9 @@ class TestBanning(TestController):
status=200
).json
self.assertEqual(2, len(waiting_ids))
self.assertEqual(1, len(waiting_ids))
self.assertIn(jobs[0], waiting_ids)
self.assertIn(jobs[1], waiting_ids)
self.assertNotIn(jobs[1], waiting_ids)
self.assertNotIn(jobs[2], waiting_ids)
for job_id in jobs[0:2]:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment