Skip to content

fts-monitoring
fts-monitoring
Commit ec2a1954 by Alejandro Alvarez Ayllon
Options

FTS-894: Adapt views to new config

parent f82bd137
Showing with 117 additions and 683 deletions
src/apps/ftsmon/templates/entry.html
......@@ -65,7 +65,6 @@
</a>
<ul class="dropdown-menu">
<li><a href="#optimizer" data-toggle="collapse" data-target=".nav-collapse" apply-global-filter>Active</a></li>
<li><a href="#optimizer/streams" data-toggle="collapse" data-target=".nav-collapse" apply-global-filter>Streams</a></li>
</ul>
</li>
<li><a href="#errors" data-toggle="collapse" data-target=".nav-collapse" apply-global-filter>Error reasons</a></li>
......@@ -84,11 +83,10 @@
Configuration<b class="caret"></b>
</a>
<ul class="dropdown-menu">
<li><a href="#config/status" data-toggle="collapse" data-target=".nav-collapse">Status</a></li>
<li><a href="#config/server" data-toggle="collapse" data-target=".nav-collapse">Server</a></li>
<li><a href="#config/storages" data-toggle="collapse" data-target=".nav-collapse">Storages</a></li>
<li><a href="#config/links" data-toggle="collapse" data-target=".nav-collapse">Links</a></li>
<li><a href="#config/activities" data-toggle="collapse" data-target=".nav-collapse" apply-global-filter>Activities</a></li>
<li><a href="#config/limits" data-toggle="collapse" data-target=".nav-collapse">Limits</a></li>
<li><a href="#config/range" data-toggle="collapse" data-target=".nav-collapse">Range configuration</a></li>
<li><a href="#config/audit" data-toggle="collapse" data-target=".nav-collapse">Audit</a></li>
<li><a href="#config/gfal2" data-toggle="collapse" data-target=".nav-collapse">GFAL2</a></li>
</ul>
......
src/apps/ftsmon/urls.py
......@@ -35,12 +35,10 @@ urlpatterns = patterns('ftsmon.views',
url(r'^transfers$', 'jobs.get_transfer_list'),
url(r'^config/audit$', 'config.get_audit'),
url(r'^config/audit$', 'config.get_audit'),
url(r'^config/storages$', 'config.get_se_config'),
url(r'^config/links$', 'config.get_link_config'),
url(r'^config/server$', 'config.get_server_config'),
url(r'^config/debug$', 'config.get_debug_config'),
url(r'^config/limits$', 'config.get_limit_config'),
url(r'^config/range', 'config.get_ranges'),
url(r'^config/gfal2$', 'config.get_gfal2_config'),
url(r'^config/activities$', 'config.get_activities'),
url(r'^config/activities/(?P<vo>(.+))$', 'config.get_actives_per_activity'),
......@@ -51,7 +49,6 @@ urlpatterns = patterns('ftsmon.views',
url(r'^optimizer/$', 'optimizer.get_optimizer_pairs'),
url(r'^optimizer/detailed$', 'optimizer.get_optimizer_details'),
url(r'^optimizer/streams', 'optimizer.get_optimizer_streams'),
url(r'^errors/$', 'errors.get_errors'),
url(r'^errors/list$', 'errors.get_errors_for_pair'),
......
src/apps/ftsmon/views/atlas.py
......@@ -28,37 +28,6 @@ from overview import OverviewExtended
from util import get_order_by, paged
def _get_pair_limits(limits, source, destination):
pair_limits = {'source': dict(), 'destination': dict()}
for l in limits:
if l[0] == source:
if l[2]:
pair_limits['source']['bandwidth'] = l[2]
elif l[3]:
pair_limits['source']['active'] = l[3]
elif l[1] == destination:
if l[2]:
pair_limits['destination']['bandwidth'] = l[2]
elif l[3]:
pair_limits['destination']['active'] = l[3]
if len(pair_limits['source']) == 0:
del pair_limits['source']
if len(pair_limits['destination']) == 0:
del pair_limits['destination']
return pair_limits
def _get_pair_udt(udt_pairs, source, destination):
for u in udt_pairs:
if udt_pairs[0] == source and udt_pairs[1] == destination:
return True
elif udt_pairs[0] == source and not udt_pairs[1]:
return True
elif not udt_pairs[0] and udt_pairs[1] == destination:
return True
return False
@require_certificate
@cache_page(60)
@jsonify
......@@ -123,16 +92,6 @@ def get_overview(http_request):
triplets[triplet_key] = triplet
# Limitations
limit_query = "SELECT source_se, dest_se, throughput, active FROM t_optimize WHERE throughput IS NOT NULL or active IS NOT NULL"
cursor.execute(limit_query)
limits = cursor.fetchall()
# UDT
udt_query = "SELECT source_se, dest_se FROM t_optimize WHERE udt = 'on'"
cursor.execute(udt_query)
udt_pairs = cursor.fetchall()
# Transform into a list
objs = []
for (triplet, obj) in triplets.iteritems():
......@@ -149,10 +108,6 @@ def get_overview(http_request):
obj['rate'] = (finished * 100.0) / total
else:
obj['rate'] = None
# Append limit, if any
obj['limits'] =_get_pair_limits(limits, triplet[0], triplet[1])
# Mark UDT-enabled
obj['udt'] = _get_pair_udt(udt_pairs, triplet[0], triplet[1])
objs.append(obj)
# Ordering
......
src/apps/ftsmon/views/config.py
......@@ -22,8 +22,7 @@ import os
from django.db import connection
from django.db.models import Q, Count
from ftsweb.models import ConfigAudit
from ftsweb.models import LinkConfig, ShareConfig
from ftsweb.models import DebugConfig, Optimize, OptimizeActive
from ftsweb.models import LinkConfig, ShareConfig, Storage
from ftsweb.models import ActivityShare, File, OperationLimit
from authn import require_certificate
from jsonify import jsonify, jsonify_paged
......@@ -49,8 +48,7 @@ def get_server_config(http_request):
config = dict()
cursor = connection.cursor()
cursor.execute("""
SELECT retry, max_time_queue, global_timeout, sec_per_mb, vo_name,
max_per_se, max_per_link, global_tcp_stream
SELECT retry, max_time_queue, global_timeout, sec_per_mb, vo_name
FROM t_server_config
""")
server_config = cursor.fetchall()
......@@ -61,20 +59,18 @@ def get_server_config(http_request):
max_time_queue=entry[1],
global_timeout=entry[2],
sec_per_mb=entry[3],
vo_name=entry[4],
max_per_se=entry[5],
max_per_link=entry[6],
tcp_streams=entry[7]
vo_name=entry[4]
)
config['per_vo'].append(c)
cursor.execute("SELECT mode_opt FROM t_optimize_mode")
modes = cursor.fetchall()
if len(modes) > 0:
config['optimizer_mode'] = modes[0][0]
return config
@require_certificate
@jsonify_paged
def get_se_config(http_request):
return Storage.objects.all()
# Wrap a list of link config, and push the
# vo shares on demand (lazy!)
class AppendShares:
......@@ -87,7 +83,7 @@ class AppendShares:
def __getitem__(self, i):
for link in self.rs[i]:
shares = ShareConfig.objects.filter(source=link.source, destination=link.destination).all()
shares = ShareConfig.objects.filter(source=link.source_se, destination=link.dest_se).all()
link.shares = {}
for share in shares:
link.shares[share.vo] = share.active
......@@ -100,65 +96,12 @@ def get_link_config(http_request):
links = LinkConfig.objects
if http_request.GET.get('source_se'):
links = links.filter(source=http_request.GET['source_se'])
links = links.filter(source_se=http_request.GET['source_se'])
if http_request.GET.get('dest_se'):
links = links.filter(destination=http_request.GET['dest_se'])
links = links.filter(dest_se=http_request.GET['dest_se'])
return AppendShares(links.all())
@require_certificate
@jsonify_paged
def get_debug_config(http_request):
return DebugConfig.objects.order_by('source_se', 'dest_se')
@require_certificate
@jsonify
def get_limit_config(http_request):
max_cfg = Optimize.objects.filter(Q(active__isnull=False) | Q(bandwidth__isnull=False))
(order_by, order_desc) = get_order_by(http_request)
if order_by == 'bandwidth':
max_cfg = max_cfg.order_by(ordered_field('bandwidth', order_desc))
elif order_by == 'active':
max_cfg = max_cfg.order_by(ordered_field('active', order_desc))
elif order_by == 'source_se':
max_cfg = max_cfg.order_by(ordered_field('source_se', order_desc))
elif order_by == 'dest_se':
max_cfg = max_cfg.order_by(ordered_field('dest_se', order_desc))
else:
max_cfg = max_cfg.order_by('-active')
return {
'transfers': max_cfg,
'operations': OperationLimit.objects.all()
}
@require_certificate
@jsonify_paged
def get_ranges(http_request):
return OptimizeActive.objects.filter(fixed='on').all()
@require_certificate
@jsonify
def get_gfal2_config(http_request):
try:
config_files = os.listdir('/etc/gfal2.d')
except:
config_files = list()
config_files = filter(lambda c: c.endswith('.conf'), config_files)
config = dict()
for cfg in config_files:
cfg_path = os.path.join('/etc/gfal2.d', cfg)
config[cfg_path] = open(cfg_path).read()
return config
@require_certificate
@jsonify
def get_activities(http_request):
......
src/apps/ftsmon/views/optimizer.py
......@@ -22,7 +22,7 @@ from django.db import connection
from django.db.models import Sum, Q
from django.http import Http404
from ftsweb.models import OptimizerEvolution, OptimizerStreams, OptimizeActive
from ftsweb.models import OptimizerEvolution, Optimizer
from authn import require_certificate
from jsonify import jsonify, jsonify_paged
from util import paged
......@@ -48,61 +48,7 @@ def get_optimizer_pairs(http_request):
from_optimizer = from_optimizer.filter(datetime__gte=not_before)
from_optimizer = from_optimizer.distinct()
# Append configured fixed active for convenience
from_fixed = OptimizeActive.objects
if http_request.GET.get('source_se', None):
from_fixed = from_fixed.filter(source_se=http_request.GET['source_se'])
if http_request.GET.get('dest_se', None):
from_fixed = from_fixed.filter(dest_se=http_request.GET['dest_se'])
from_fixed = from_fixed.filter(fixed='on')
from_fixed = from_fixed.values('source_se', 'dest_se', 'fixed').distinct()
return list(from_optimizer) + list(from_fixed)
class OptimizerAppendLimits(object):
"""
Query for the limits
"""
def __init__(self, source_se, dest_se, evolution):
self.source_se = source_se
self.dest_se = dest_se
self.evolution = evolution
def __len__(self):
return len(self.evolution)
def __getitem__(self, index):
entries = self.evolution[index]
if isinstance(entries, list):
for e in entries:
e['bandwidth_limits'] = {
'source': self._get_source_limit(),
'destination': self._get_destination_limit()
}
return entries
def _get_source_limit(self):
cursor = connection.cursor()
cursor.execute("SELECT throughput FROM t_optimize WHERE source_se = %s AND throughput IS NOT NULL",
[self.source_se])
result = cursor.fetchall()
if len(result) < 1:
return None
else:
return result[0][0]
def _get_destination_limit(self):
cursor = connection.cursor()
cursor.execute("SELECT throughput FROM t_optimize WHERE dest_se = %s AND throughput IS NOT NULL",
[self.dest_se])
result = cursor.fetchall()
if len(result) < 1:
return None
else:
return result[0][0]
return from_optimizer
@require_certificate
......@@ -129,46 +75,6 @@ def get_optimizer_details(http_request):
)
optimizer = optimizer.order_by('-datetime')
fixed = OptimizeActive.objects.filter(fixed='on', source_se=source_se, dest_se=dest_se)
if len(fixed):
fixed = fixed[0]
else:
fixed = None
return {
'evolution': paged(OptimizerAppendLimits(source_se, dest_se, optimizer), http_request),
'fixed': fixed
'evolution': paged(optimizer, http_request),
}
def is_stream_optimizer_disabled():
"""
Check if the stream optimizer is disabled
"""
cursor = connection.cursor()
cursor.execute("SELECT mode_opt FROM t_optimize_mode")
modes = cursor.fetchall()
return len(modes) > 0 and modes[0][0] <= 1
@require_certificate
@jsonify
def get_optimizer_streams(http_request):
if is_stream_optimizer_disabled():
return None
try:
time_window = timedelta(hours=int(http_request.GET['time_window']))
except:
time_window = timedelta(hours=1)
streams = OptimizerStreams.objects
if http_request.GET.get('source_se', None):
streams = streams.filter(source_se=http_request.GET['source_se'])
if http_request.GET.get('dest_se', None):
streams = streams.filter(dest_se=http_request.GET['dest_se'])
streams = streams.filter(datetime__gte=datetime.utcnow() - time_window)
streams = streams.order_by('-datetime')
return paged(streams, http_request)
src/apps/ftsmon/views/overview.py
......@@ -20,47 +20,15 @@
import types
from datetime import datetime, timedelta
from django.db import connection
from django.db.models import Count
from django.views.decorators.cache import cache_page
from authn import require_certificate
from ftsweb.models import File, OptimizeActive, OptimizerEvolution
from ftsweb.models import OptimizerEvolution
from jobs import setup_filters
from jsonify import jsonify
from util import get_order_by, paged
def _get_pair_limits(limits, source, destination):
pair_limits = {'source': dict(), 'destination': dict()}
for l in limits:
if l[0] == source:
if l[2]:
pair_limits['source']['bandwidth'] = l[2]
elif l[3]:
pair_limits['source']['active'] = l[3]
elif l[1] == destination:
if l[2]:
pair_limits['destination']['bandwidth'] = l[2]
elif l[3]:
pair_limits['destination']['active'] = l[3]
if len(pair_limits['source']) == 0:
del pair_limits['source']
if len(pair_limits['destination']) == 0:
del pair_limits['destination']
return pair_limits
def _get_pair_udt(udt_pairs, source, destination):
for u in udt_pairs:
if udt_pairs[0] == source and udt_pairs[1] == destination:
return True
elif udt_pairs[0] == source and not udt_pairs[1]:
return True
elif not udt_pairs[0] and udt_pairs[1] == destination:
return True
return False
def _seconds(td):
return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10 ** 6
......@@ -81,16 +49,6 @@ class OverviewExtended(object):
def __len__(self):
return len(self.objects)
def _get_fixed(self, source, destination):
"""
Return true if the number of actives is fixed for this pair
"""
oa = OptimizeActive.objects.filter(source_se=source, dest_se=destination).values('fixed').all()
if len(oa):
oa = oa[0]
return oa['fixed'] is not None and oa['fixed'].lower == 'on'
return False
def _get_throughput(self, source, destination, vo):
"""
Calculate throughput (in MB) over this pair + vo over the last minute
......@@ -112,7 +70,6 @@ class OverviewExtended(object):
if isinstance(indexes, types.SliceType):
return_list = self.objects[indexes]
for item in return_list:
item['active_fixed'] = self._get_fixed(item['source_se'], item['dest_se'])
item['transferred'], item['current'] = self._get_throughput(item['source_se'], item['dest_se'], item['vo_name'])
return return_list
else:
......@@ -180,16 +137,6 @@ def get_overview(http_request):
triplets[triplet_key] = triplet
# Limitations
limit_query = "SELECT source_se, dest_se, throughput, active FROM t_optimize WHERE throughput IS NOT NULL or active IS NOT NULL"
cursor.execute(limit_query)
limits = cursor.fetchall()
# UDT
udt_query = "SELECT source_se, dest_se FROM t_optimize WHERE udt = 'on'"
cursor.execute(udt_query)
udt_pairs = cursor.fetchall()
# Transform into a list
objs = []
for (triplet, obj) in triplets.iteritems():
......@@ -205,11 +152,6 @@ def get_overview(http_request):
obj['rate'] = (finished * 100.0) / total
else:
obj['rate'] = None
# Append limit, if any
obj['limits'] =_get_pair_limits(limits, triplet[0], triplet[1])
# Mark UDT-enabled
obj['udt'] = _get_pair_udt(udt_pairs, triplet[0], triplet[1])
objs.append(obj)
# Ordering
......
src/libs/ftsweb/models.py
......@@ -206,14 +206,13 @@ class ConfigAudit(models.Model):
class LinkConfig(models.Model):
source = models.CharField(max_length = 255, primary_key = True)
destination = models.CharField(max_length = 255)
state = models.CharField(max_length = 30)
symbolicName = models.CharField(max_length = 255)
nostreams = models.IntegerField()
source_se = models.CharField(max_length = 255, primary_key = True)
dest_se = models.CharField(max_length = 255)
min_active = models.IntegerField()
max_active = models.IntegerField()
optimizer_mode = models.IntegerField()
tcp_buffer_size = models.IntegerField()
urlcopy_tx_to = models.IntegerField()
auto_tuning = models.CharField(max_length = 3)
nostreams = models.IntegerField()
def __eq__(self, b):
return isinstance(b, self.__class__) and \
......@@ -224,6 +223,22 @@ class LinkConfig(models.Model):
db_table = 't_link_config'
class Storage(models.Model):
storage = models.CharField(max_length=150, primary_key=True)
site = models.CharField(max_length=45)
metadata = models.TextField()
ipv6 = models.BooleanField()
udt = models.BooleanField()
debug_level = models.IntegerField()
inbound_max_active = models.IntegerField()
inbound_max_throughput = models.FloatField()
outbound_max_active = models.IntegerField()
outbound_max_throughput = models.FloatField()
class Meta:
db_table = 't_se'
class ShareConfig(models.Model):
source = models.CharField(max_length = 255, primary_key = True)
destination = models.CharField(max_length = 255)
......@@ -240,31 +255,13 @@ class ShareConfig(models.Model):
db_table = 't_share_config'
class Optimize(models.Model):
source_se = models.CharField(max_length = 255, primary_key = True)
dest_se = models.CharField(max_length = 255, primary_key = True)
active = models.IntegerField()
bandwidth = models.FloatField(db_column = 'throughput')
def __eq__(self, other):
if type(self) != type(other):
return False
return self.source_se == other.source_se and \
self.dest_se == other.dest_se and \
self.active == other.active
class Meta:
db_table = 't_optimize'
class OptimizeActive(models.Model):
source_se = models.CharField(max_length = 255)
class Optimizer(models.Model):
source_se = models.CharField(max_length = 255, primary_key=True)
dest_se = models.CharField(max_length = 255)
active = models.IntegerField(primary_key = True)
fixed = models.CharField(max_length=3)
max_active = models.IntegerField()
min_active = models.IntegerField()
datetime = models.DateTimeField()
ema = models.FloatField()
active = models.IntegerField()
nostreams = models.IntegerField()
def __eq__(self, other):
if type(self) != type(other):
......@@ -274,7 +271,7 @@ class OptimizeActive(models.Model):
self.dest_se == other.dest_se
class Meta:
db_table = 't_optimize_active'
db_table = 't_optimizer'
class OptimizerEvolution(models.Model):
......@@ -296,24 +293,6 @@ class OptimizerEvolution(models.Model):
db_table = 't_optimizer_evolution'
class OptimizerStreams(models.Model):
source_se = models.CharField(max_length = 255)
dest_se = models.CharField(max_length = 255)
nostreams = models.IntegerField()
datetime = models.DateTimeField(primary_key = True)
def __eq__(self, other):
if type(self) != type(other):
return False
return self.source_se == other.source_se and \
self.dest_se == other.dest_se and \
self.nostreams == other.nostreams
class Meta:
db_table = 't_optimize_streams'
class Host(models.Model):
hostname = models.CharField(primary_key = True, max_length = 64)
beat = models.DateTimeField()
......@@ -324,19 +303,6 @@ class Host(models.Model):
db_table = 't_hosts'
class DebugConfig(models.Model):
source_se = models.CharField(primary_key = True)
dest_se = models.CharField()
debug_level = models.IntegerField()
def __eq__(self, other):
# Consider all entries different
return False
class Meta:
db_table = 't_debug'
class ActivityShare(models.Model):
vo = models.CharField(primary_key=True)
activity_share = models.TextField()
......
src/media/html/config/limits.html deleted 100644 → 0
<h2>Limit configuration</h2>
<h3>Limits for transfers</h3>
<table class="table">
<thead>
<tr>
<th>
<span data-order-by="source_se">Source</span>
</th>
<th>
<span data-order-by="dest_se">Destination</span>
</th>
<th>
<span data-order-by="active">Max. Active</span>
</th>
<th>
<span data-order-by="bandwidth">Max. Bandwidth</span>
</th>
</tr>
</thead>
<tbody data-ng-repeat="limit in limits.transfers" data-ng-class-odd="'odd'">
<tr>
<td>{{ limit.source_se }}</td>
<td>{{ limit.dest_se }}</td>
<td>{{ limit.active }}</td>
<td><span data-optional-number="{{limit.bandwidth}}" data-digits="2" data-suffix="MB/s"></span></td>
</tr>
</tbody>
</table>
<h3>Limits for operations</h3>
<table class="table">
<thead>
<tr>
<th>
VO
</th>
<th>
Host
</th>
<th>
Operation
</th>
<th>
Concurrent operations
</th>
</tr>
</thead>
<tbody data-ng-repeat="limit in limits.operations" data-ng-class-odd="'odd'">
<tr>
<td>{{ limit.vo }}</td>
<td>{{ limit.host }}</td>
<td>{{ limit.operation }}</td>
<td>{{ limit.concurrent_ops }}</td>
</tr>
</tbody>
</table>
src/media/html/config/links.html
......@@ -8,38 +8,34 @@
<table class="table">
<thead>
<tr>
<th>Name</th>
<th>State</th>
<th>Source</th>
<th>Destination</th>
<th>Auto tuning</th>
<th>TCP streams</th>
<th>Optimizer mode</th>
<th>TCP buffersize</th>
<th>Number of streams</th>
<th>Min. Active</th>
<th>Max. Active</th>
</tr>
</thead>
<tbody data-ng-repeat="link in links.items" data-ng-class-odd="'odd'">
<tr>
<td>
<i class="icon-plus" data-ng-click="config.show = !config.show"></i>
{{ link.symbolicName }}
{{ link.source_se }}
</td>
<td>{{ link.state }}</td>
<td>{{ link.source }}</td>
<td>{{ link.destination }}</td>
<td>{{ link.auto_tuning }}</td>
<td>{{ link.dest_se }}</td>
<td>{{ link.optimizer_mode }}</td>
<td>{{ link.tcp_buffer_size }}</td>
<td>{{ link.nostreams }}</td>
<td>{{ link.min_active }}</td>
<td>{{ link.max_active }}</td>
</tr>
<tr style="background: #CCC" data-ng-show="config.show">
<td colspan="5">
<td colspan="7">
Shares:
<ul>
<li>TCP Buffer Size: {{ link.tcp_buffer_size }}</li>
<li>Number of streams: {{ link.nostreams }}</li>
<li>Timeout: {{ link.urlcopy_tx_to }}</li>
<li>Shares:
<ul>
<li data-ng-repeat="(vo, active) in link.shares">
{{ vo }}: {{ active }}
</li>
</ul>
<li data-ng-repeat="(vo, active) in link.shares">
{{ vo }}: {{ active }}
</li>
</ul>
</td>
......
src/media/html/config/range.html deleted 100644 → 0
<h2>Range configuration</h2>
<pagination rotate="false"
page="range.page" total-items="range.count" items-per-page="range.pageSize"
max-size="15" class="pagination" boundary-links="true"
on-select-page="pageChanged(page)"></pagination>