Commit 997ebcb3 authored by Maria Arsuaga Rios's avatar Maria Arsuaga Rios
Browse files

FTS-977: Update fts-rest config for se and global

parent 71bb387b
Pipeline #156607 passed with stage
in 2 minutes and 55 seconds
......@@ -43,20 +43,13 @@ class LinkConfig(Base):
source = Column(String(150), primary_key=True)
destination = Column(String(150), primary_key=True)
state = Column(Flag(negative='off', positive='on'))
symbolicname = Column(String(255), unique=True)
nostreams = Column(Integer)
min_active = Column(Integer)
max_active = Column(Integer)
optimizer_mode = Column (Integer)
tcp_buffer_size = Column(Integer)
urlcopy_tx_to = Column(Integer)
auto_tuning = Column(Flag(negative='off', positive='on'))
# share_config =\
# relation('ShareConfig', backref='link_config',
# primaryjoin='ShareConfig.source == LinkConfig.source and '
# 'ShareConfig.destination == LinkConfig.destination',
# foreign_keys=(source, destination),
# uselist=False,
# lazy=False)
nostreams = Column(Integer)
def __str__(self):
return "%s => %s" % (self.source, self.destination)
......@@ -65,42 +58,17 @@ class LinkConfig(Base):
class Se(Base):
__tablename__ = 't_se'
se_id_info = Column(Integer)
name = Column(String(150), primary_key=True)
endpoint = Column(String(1024))
se_type = Column(String(30))
site = Column(String(100))
state = Column(String(30))
version = Column(String(30))
host = Column(String(100))
se_transfer_type = Column(String(30))
se_transfer_protocol = Column(String(30))
se_control_protocol = Column(String(30))
gocdb_id = Column(String(100))
source_on =\
relation('LinkConfig', backref='source_se',
primaryjoin=and_(LinkConfig.source == name,
LinkConfig.destination != '*'),
foreign_keys=name, uselist=True, lazy=True)
destination_on =\
relation('LinkConfig', backref='destination_se',
primaryjoin=and_(LinkConfig.destination == name,
LinkConfig.source != '*'),
foreign_keys=name, uselist=True, lazy=True)
standalone_source =\
relation('LinkConfig', backref=None,
primaryjoin=and_(LinkConfig.source == name,
LinkConfig.destination == '*'),
foreign_keys=name, lazy=True)
standalone_destination =\
relation('LinkConfig', backref=None,
primaryjoin=and_(LinkConfig.destination == name,
LinkConfig.source == '*'),
foreign_keys=name, lazy=True)
storage = Column(String(150), primary_key=True)
site = Column(String(45))
metadata = Column(String(255))
ipv6 = Column(Flag(negative='off', positive='on'), default='off')
udt = Column(Flag(negative='off', positive='on'))
debug_level = Column(Integer)
inbound_max_active = Column(Integer)
inbound_max_throughput = Column(Float)
outbound_max_active = Column(Integer)
outbound_max_throughput = Column(Float)
def __str__(self):
return self.name
......@@ -122,22 +90,6 @@ class ShareConfig(Base):
return "%s: %s => %s" % (self.vo, self.source, self.destination)
class FileShareConfig(Base):
__tablename__ = 't_file_share_config'
file_id = Column(Integer, primary_key=True)
source = Column(String(150), primary_key=True)
destination = Column(String(150), primary_key=True)
vo = Column(String(100), nullable=False)
__table_args__ = (
ForeignKeyConstraint(['source', 'destination', 'vo'],
[ShareConfig.source, ShareConfig.destination, ShareConfig.vo]),
ForeignKeyConstraint(['file_id'],
[File.file_id])
)
class Group(Base):
__tablename__ = 't_group_members'
......@@ -163,6 +115,7 @@ class DebugConfig(Base):
source_se = Column(String(150), primary_key=True)
dest_se = Column(String(150), primary_key=True)
debug = Column(String(3), default='off')
debug_level = Column(Integer, default=1)
def __str__(self):
......@@ -187,12 +140,13 @@ class ServerConfig(Base):
sec_per_mb = Column(Integer, default=0)
vo_name = Column(String(100), primary_key=True)
show_user_dn = Column(Flag(negative='off', positive='on'), default='off')
max_per_se = Column(Integer, default=0)
max_per_link = Column(Integer, default=0)
class OptimizerConfig(Base):
__tablename__ = 't_optimize_mode'
auto_number = Column(Integer)
source_se = Column(String(150))
mode = Column(Integer, default=1, primary_key=True, name='mode_opt')
......
......@@ -26,7 +26,6 @@ class OptimizerEvolution(Base):
datetime = Column(DateTime, primary_key=True)
source_se = Column(String(150), primary_key=True)
dest_se = Column(String(150), primary_key=True)
throughput_limit = Column(Float)
active = Column(Integer)
throughput = Column(Float)
success = Column(Float)
......@@ -34,6 +33,9 @@ class OptimizerEvolution(Base):
diff = Column(Integer)
actual_active = Column(Integer)
queue_size = Column(Integer)
ema = Column(Double)
filesize_avg = Column(Double)
filesize_stddeve = Column(Double)
class OptimizerActive(Base):
......@@ -41,7 +43,7 @@ class OptimizerActive(Base):
source_se = Column(String(150), primary_key=True)
dest_se = Column(String(150), primary_key=True)
active = Column(Integer, default=2)
active = Column(Integer, default=5)
datetime = Column(DateTime, default=None)
ema = Column(Float, default=0)
fixed = Column(Flag(negative='off', positive='on'), default='off')
......@@ -73,5 +75,6 @@ class Optimize(Base):
dest_se = Column(String(150), nullable=True)
active = Column(Integer)
throughput = Column(Float)
datetime = Column(DateTime)
udt = Column(Flag(negative='off', positive='on'))
ipv6 = Column(Flag(negative='off', positive='on'), default='off')
......@@ -60,8 +60,6 @@ class GlobalConfigController(BaseController):
result['*'] = r
else:
result[r.vo_name] = dict(retry=r.retry)
opt = Session.query(OptimizerConfig).first()
setattr(result['*'], 'optimizer_mode', opt.mode if opt else None)
return result
@doc.response(400, 'Invalid values passed in the request')
......@@ -79,16 +77,6 @@ class GlobalConfigController(BaseController):
if not db_cfg:
db_cfg = ServerConfig(vo_name=vo_name)
if vo_name == '*' and 'optimizer_mode' in cfg:
try:
opt_mode = int(cfg.pop('optimizer_mode'))
if opt_mode > 0 and opt_mode < 4:
Session.query(OptimizerConfig).delete()
opt = OptimizerConfig(mode=opt_mode)
Session.merge(opt)
except:
raise HTTPBadRequest('Invalid optimizer_mode value')
for key, value in cfg.iteritems():
value = validate_type(ServerConfig, key, value)
setattr(db_cfg, key, value)
......
......@@ -56,27 +56,15 @@ class SeConfigurationController(BaseController):
try:
for storage, cfg in input_dict.iteritems():
# As source
as_source_new = cfg.get('as_source', None)
if as_source_new:
as_source = Session.query(Optimize).filter(Optimize.source_se == storage).first()
if not as_source:
as_source = Optimize(source_se=storage)
for key, value in as_source_new.iteritems():
value = validate_type(Optimize, key, value)
setattr(as_source, key, value)
audit_configuration('set-se-config', 'Set config as source %s: %s' % (storage, json.dumps(cfg)))
Session.merge(as_source)
# As destination
as_dest_new = cfg.get('as_destination', None)
if as_dest_new:
as_dest = Session.query(Optimize).filter(Optimize.dest_se == storage).first()
if not as_dest:
as_dest = Optimize(dest_se=storage)
for key, value in as_dest_new.iteritems():
value = validate_type(Optimize, key, value)
setattr(as_dest, key, value)
audit_configuration('set-se-config', 'Set config as destination %s: %s' % (storage, json.dumps(cfg)))
Session.merge(as_dest)
se_info_new = cfg.get('se_info', None)
if se_info_new:
se_info = Session.query(Se).filter(Se.storage == storage).first()
for key, value in se_info_new.iteritems():
value = validate_type(Se, key, value)
setattr(se_info, key, value)
audit_configuration('set-se-config', 'Set config %s: %s' % (storage, json.dumps(cfg)))
Session.merge(se_info)
# Operation limits
operations = cfg.get('operations', None)
if operations:
......@@ -112,24 +100,24 @@ class SeConfigurationController(BaseController):
Get the configurations status for a given SE
"""
se = request.params.get('se', None)
from_optimize = Session.query(Optimize)
from_se = Session.query(Se)
from_ops = Session.query(OperationConfig)
if se:
from_optimize = from_optimize.filter((Optimize.source_se == se) | (Optimize.dest_se == se))
from_se = from_optimize.filter(Se.storage == se)
from_ops = from_ops.filter(OperationConfig.host == se)
# Merge both
response = dict()
for opt in from_optimize:
se = opt.source_se if opt.source_se else opt.dest_se
for opt in from_se:
se = opt.storage
config = response.get(se, dict())
link_config = dict()
for attr in ['active', 'throughput', 'udt', 'ipv6']:
for attr in ['inbound_min_actives', 'inbound_max_actives', 'inbound_min_throughput','inbound_max_throughput', 'udt', 'ipv6']:
link_config[attr] = getattr(opt, attr)
if opt.source_se:
config['as_source'] = link_config
else:
config['as_destination'] = link_config
for attr in ['outbound_min_actives', 'outbound_max_actives', 'out_min_throughput','out_max_throughput', 'udt', 'ipv6']:
link_config[attr] = getattr(opt, attr)
config['as_source'] = link_config
response[se] = config
for op in from_ops:
......@@ -155,7 +143,7 @@ class SeConfigurationController(BaseController):
raise HTTPBadRequest('Missing storage (se)')
try:
Session.query(Optimize).filter((Optimize.source_se == se) | (Optimize.dest_se == se)).delete()
Session.query(Se).filter(Se.storage == se).delete()
Session.query(OperationConfig).filter(OperationConfig.host == se).delete()
Session.commit()
except:
......
......@@ -66,13 +66,15 @@ function getSubForm(form, klass)
{
var subform = form.find("." + klass);
var subset = {};
subset.vo = (subform.find("select[name='vo']").val());
subset.active = parseInt(subform.find("input[name='active']").val());
subset.metadata = subform.find("input[name='metadata']").val()
subset.inbound_max_active = parseInt(subform.find("input[name='inbound_max_active']").val());
subset.outbound_max_active = parseInt(subform.find("input[name='outbound_max_active']").val());
subset.inbound_max_active = parseFloat(subform.find("input[name='inbound_max_throughput']").val());
subset.outbound_max_active = parseFloat(subform.find("input[name='outbound_max_throughput']").val());
subset.ipv6 = (subform.find("select[name='ipv6']").val() == "true");
subset.throughput = parseFloat(subform.find("input[name='throughput']").val());
subset.udt = (subform.find("select[name='udt']").val() == "true");
subset.debug_level = parseInt(subform.find("input[name='debug_level']").val());
return subset;
}
......@@ -112,8 +114,7 @@ function handleSeSave(form)
var setup = {};
var se = form.find("input[name='se']").val();
setup.as_source = getSubForm(form, 'as-source');
setup.as_destination = getSubForm(form, 'as-destination');
setup.se-info = getSubForm(form, 'se-info');
setup.operations = getOperations(form, 'ops-list');
......@@ -264,7 +265,4 @@ function setupSe()
$("#se-add-field-se").autocomplete({
source: "/autocomplete/storage"
});
$("#se-add-field-vo").autocomplete({
source: "/autocomplete/vo"
});
}
......@@ -61,22 +61,6 @@
<span class="input-group-addon"> seconds</span>
</span>
</p>
<p class="form-group">
<label for="max_per_link">Max. transfers per link</label>
<span class="input-group">
<input type="number" id="max_per_link" name="max_per_link" class="form-control"
value="${data['*']['max_per_link']}"/>
<span class="input-group-addon"> transfers</span>
</span>
</p>
<p class="form-group">
<label for="max_per_se">Max. transfers per storage</label>
<span class="input-group">
<input type="number" id="max_per_se" name="max_per_se" class="form-control"
value="${data['*']['max_per_se']}"/>
<span class="input-group-addon"> transfers</span>
</span>
</p>
<p class="form-group">
<label for="max_time_queue">Max. time in the queue</label>
<span class="input-group">
......@@ -87,23 +71,6 @@
</p>
</div>
<div class="col-md-6">
<p class="form-group">
<label for="optimizer_mode">Optimizer mode</label>
<span class="input-group">
<select id="optimizer_mode" name="optimizer_mode" class="form-control">
% if not data['*']['optimizer_mode']:
<option>-- Default --</option>
% endif
% for value, label in enumerate([('Conservative', 'text-primary'), ('Normal', 'text-success'), ('Aggressive', 'text-danger')], 1):
% if value == data['*']['optimizer_mode']:
<option value="${value}" class="${label[1]}" selected="selected">${label[0]}</option>
% else:
<option value="${value}" class="${label[1]}">${label[0]}</option>
% endif
% endfor
</select>
</span>
</p>
<p class="form-group">
<label for="sec_per_mb">
Add seconds per MiB
......
......@@ -86,8 +86,6 @@ class TestConfigGlobal(TestController):
global_timeout=55,
sec_per_mb=1,
show_user_dn=True,
max_per_se=10,
max_per_link=15,
vo_name='atlas'
),
status=200
......@@ -112,33 +110,11 @@ class TestConfigGlobal(TestController):
global_timeout=55,
sec_per_mb=1,
show_user_dn=True,
max_per_se=10,
max_per_link=15,
vo_name='atlas'
),
status=400
)
def test_set_optimizer_mode(self):
"""
Set the optimizer mode
"""
self.app.post_json(url="/config/global",
params={'optimizer_mode': 2},
status=200
)
opt = Session.query(OptimizerConfig).first()
self.assertEqual(2, opt.mode)
def test_set_wrong_optimizer_mode(self):
"""
Wrong value for optimizer_mode
"""
self.app.post_json(url="/config/global",
params={'optimizer_mode': 'dfsdf'},
status=400
)
def test_delete_vo_global_config(self):
"""
Delete the global configuration for the given VO
......@@ -148,4 +124,6 @@ class TestConfigGlobal(TestController):
self.app.delete(url="/config/global", status=400)
self.app.delete(url="/config/global?vo_name=dteam", status=204)
vo_name = Session.query(OptimizerConfig).get("dteam")
self.assertIsNone(vo_name)
\ No newline at end of file
self.assertIsNone(vo_name)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment