diff --git a/conf/httpd.conf.d/ftsmon_opedID_connect.conf b/conf/httpd.conf.d/ftsmon_opedID_connect.conf new file mode 100644 index 0000000000000000000000000000000000000000..0aa556e148fe0a7a8390b575f9d619d98871b1dc --- /dev/null +++ b/conf/httpd.conf.d/ftsmon_opedID_connect.conf @@ -0,0 +1,112 @@ +<IfModule !ssl_module> + LoadModule ssl_module modules/mod_ssl.so +</IfModule> +<IfModule !wsgi_module> + LoadModule wsgi_module modules/mod_wsgi.so +</IfModule> +<IfModule !version_module> + LoadModule version_module modules/mod_version.so +</IfModule> + +<IfModule !auth_openidc_module> + LoadModule auth_openidc_module modules/mod_auth_openidc.so +</IfModule> + + +# Monitoring in port 8449 +Listen 8449 +<VirtualHost *:8449> + # SSL configuration + SSLEngine On + SSLProtocol all -SSLv2 -SSLv3 + SSLCipherSuite RC4-SHA:AES128-SHA:HIGH:!aNULL:!MD5:!RC4 + SSLHonorCipherOrder on + + # Certificates + SSLCertificateFile /etc/grid-security/hostcert.pem + SSLCertificateKeyFile /etc/grid-security/hostkey.pem + SSLCACertificatePath /etc/grid-security/certificates + + # Client certificate by default is optional + # The application will take care of more fine-grained authorization + # If you want, you can add require in order to force a client certificate + SSLVerifyClient optional + SSLVerifyDepth 10 + SSLOptions +StdEnvVars + + # Disable the session files of libgridsite + GridSiteGridHTTP off + GridSiteAutoPasscode off + + # Mind that by default FTS3 Monitoring will require just a valid certificate + # for every path, except for the server overview (so no robot certificate is required + # by default for the Service Level feedback) + # That's why optional is the default + # If you want to do white-listing, have a look at this document + # http://httpd.apache.org/docs/2.0/ssl/ssl_howto.html#certauthenticate + + # Django application + WSGIScriptAlias /fts3 /usr/share/fts3web/fts3web.wsgi + AllowEncodedSlashes On + + # Run in a separate process + WSGIDaemonProcess fts3wmon processes=2 threads=10 maximum-requests=500 inactivity-timeout=60 display-name=fts3wmon + WSGIProcessGroup fts3wmon + + <Location /fts3> + <IfVersion >= 2.4> + Require all granted + </IfVersion> + <IfVersion < 2.4> + Order allow,deny + Allow from all + </IfVersion> + </Location> + + # Redirect to the monitoring webapp from the root + RewriteEngine On + RewriteRule ^/$ /fts3/ftsmon [R] + RewriteRule ^/ftsmon/ /fts3/ftsmon [R] + + # Static content + Alias /fts3/media /usr/share/fts3web/media + <Location /media> + <IfVersion >= 2.4> + Require all granted + </IfVersion> + <IfVersion < 2.4> + Order allow,deny + Allow from all + </IfVersion> + + SetOutputFilter DEFLATE + ExpiresActive On + ExpiresDefault "access plus 1 month" + </Location> + + # FTS3 transfer logs + OIDCResponseType "code" + OIDCScope "openid email profile" + OIDCProviderMetadataURL https://iam.extreme-datacloud.eu/.well-known/openid-configuration + OIDCClientID <new_ID> + OIDCClientSecret <new_secret> + OIDCProviderTokenEndpointAuth client_secret_basic + OIDCCryptoPassphrase cylon + OIDCRedirectURI https://hla.cern.ch:8449/var/log/fts3/transfers + + Alias /var/log/fts3/transfers /var/log/fts3/transfers + <Location /var/log/fts3/transfers> + AuthType openid-connect + Require valid-user +# <IfVersion >= 2.4> +# Require all granted +# </IfVersion> +# <IfVersion < 2.4> +# Order allow,deny +# Allow from all +# </IfVersion> + SetOutputFilter DEFLATE + ForceType text/plain + Options +Indexes + </Location> +</VirtualHost> diff --git a/src/apps/ftsmon/templates/entry.html b/src/apps/ftsmon/templates/entry.html index bdb574a57e86c7cfc3e6fac4e7de92f4622577bc..a0b6e231f2d98f2abfef75de9b9541b2e70505f3 100644 --- a/src/apps/ftsmon/templates/entry.html +++ b/src/apps/ftsmon/templates/entry.html @@ -47,6 +47,7 @@ </a> <ul class="dropdown-menu"> <li><a href="#overview/activities" data-toggle="collapse" data-target=".nav-collapse" apply-global-filter>Activity Shares</a></li> + <li><a href="#overview/deletion" data-toggle="collapse" data-target=".nav-collapse" apply-global-filter>Deletion jobs</a></li> </ul> </li> <li class="dropdown"> @@ -121,9 +122,11 @@ <script>var SITE_ALIAS="{% getSetting 'ALIAS' %}";</script> <script src="{{STATIC_URL}}js/resources.js"></script> <script src="{{STATIC_URL}}js/jobs.js"></script> + <script src="{{STATIC_URL}}js/jobs_del.js"></script> <script src="{{STATIC_URL}}js/transfers.js"></script> <script src="{{STATIC_URL}}js/overview.js"></script> <script src="{{STATIC_URL}}js/activities.js"></script> + <script src="{{STATIC_URL}}js/deletion.js"></script> <script src="{{STATIC_URL}}js/optimizer.js"></script> <script src="{{STATIC_URL}}js/errors.js"></script> <script src="{{STATIC_URL}}js/config.js"></script> diff --git a/src/apps/ftsmon/urls.py b/src/apps/ftsmon/urls.py index 4f99e96d38ab955e8a87dea921801cfc97eabfaf..62feea622068d1ac4a746735970455987c34c8fd 100644 --- a/src/apps/ftsmon/urls.py +++ b/src/apps/ftsmon/urls.py @@ -25,14 +25,19 @@ except: urlpatterns = patterns('ftsmon.views', url(r'^$', 'index.index'), + url(r'^$', 'jobs_del.jobs_del'), url(r'^overview$', 'overview.get_overview'), url(r'^overview/activities$', 'activities.get_overview'), - + url(r'^overview/deletion$', 'deletion.get_deletion'), + url(r'^jobs/?$', 'jobs.get_job_list'), + url(r'^jobs_del/?$', 'jobs_del.get_job_list'), url(r'^jobs/(?P<job_id>[a-fA-F0-9\-]+)$', 'jobs.get_job_details'), + url(r'^jobs_del/(?P<job_id>[a-fA-F0-9\-]+)$', 'jobs_del.get_job_details'), url(r'^jobs/(?P<job_id>[a-fA-F0-9\-]+)/files$', 'jobs.get_job_transfers'), - + url(r'^jobs_del/(?P<job_id>[a-fA-F0-9\-]+)/files$', 'jobs_del.get_job_transfers'), + url(r'^transfers$', 'jobs.get_transfer_list'), url(r'^config/audit$', 'config.get_audit'), diff --git a/src/apps/ftsmon/views/config.py b/src/apps/ftsmon/views/config.py index f8e7c121db2037127acdace70a1d95131b3042ed..af2666a9ddf80c9cf1b8bac4167132a9872c843f 100644 --- a/src/apps/ftsmon/views/config.py +++ b/src/apps/ftsmon/views/config.py @@ -90,7 +90,6 @@ class AppendShares: link.shares[share.vo] = share.active yield link - @jsonify_paged def get_link_config(http_request): links = LinkConfig.objects diff --git a/src/apps/ftsmon/views/deletion.py b/src/apps/ftsmon/views/deletion.py new file mode 100644 index 0000000000000000000000000000000000000000..d54fdf4e65e98cd7b677da412d6828b99546d396 --- /dev/null +++ b/src/apps/ftsmon/views/deletion.py @@ -0,0 +1,136 @@ +# Copyright notice: +# +# Copyright (C) CERN 2013-2015 +# +# Copyright (C) Members of the EMI Collaboration, 2010-2013. +# See www.eu-emi.eu for details on the copyright holders +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta +from django.db import connection +from django.views.decorators.cache import cache_page + +from authn import require_certificate +from jobs_del import setup_filters +from jsonify import jsonify +from overview import OverviewExtendedDel +from util import get_order_by, paged + +@require_certificate +@jsonify +def get_deletion(http_request): + filters = setup_filters(http_request) + if filters['time_window']: + not_before = datetime.utcnow() - timedelta(hours=filters['time_window']) + else: + not_before = datetime.utcnow() - timedelta(hours=1) + + cursor = connection.cursor() + + # Get all pairs first + pairs_filter = "" + se_params = [] + if filters['source_se']: + pairs_filter += " AND source_se = %s " + se_params.append(filters['source_se']) + if filters['vo']: + pairs_filter += " AND vo_name = %s " + se_params.append(filters['vo']) + + + # Result + triplets = dict() + + # Non terminal + query = """ + SELECT COUNT(file_state) as count, file_state, source_se, vo_name + FROM t_dm + WHERE file_state in ('SUBMITTED', 'ACTIVE') %s + GROUP BY file_state, source_se, vo_name order by NULL + """ % pairs_filter + cursor.execute(query, se_params) + for row in cursor.fetchall(): + triplet_key = (row[2], row[3]) + triplet = triplets.get(triplet_key, dict()) + triplet[row[1].lower()] = row[0] + triplets[triplet_key] = triplet + + # Terminal + query = """ + SELECT COUNT(file_state) as count, file_state, source_se, vo_name + FROM t_dm + WHERE file_state in ('FINISHED', 'FAILED', 'CANCELED') %s + AND finish_time > %%s + GROUP BY file_state, source_se, vo_name order by NULL + """ % pairs_filter + cursor.execute(query, se_params + [not_before.strftime('%Y-%m-%d %H:%M:%S')]) + for row in cursor.fetchall(): + triplet_key = (row[2], row[3]) # source_se, vo_name + triplet = triplets.get(triplet_key, dict()) # source_se:______, vo_name:_____ + triplet[row[1].lower()] = row[0] + + triplets[triplet_key] = triplet + + + + # Transform into a list + objs = [] + for (triplet, obj) in triplets.iteritems(): + obj['source_se'] = triplet[0] + obj['vo_name'] = triplet[1] + failed = obj.get('failed', 0) + finished = obj.get('finished', 0) + total = failed + finished + if total > 0: + obj['rate'] = (finished * 100.0) / total + else: + obj['rate'] = None + objs.append(obj) + # Ordering + (order_by, order_desc) = get_order_by(http_request) + + if order_by == 'active': + sorting_method = lambda o: (o.get('active', 0), o.get('submitted', 0)) + elif order_by == 'finished': + sorting_method = lambda o: (o.get('finished', 0), o.get('failed', 0)) + elif order_by == 'failed': + sorting_method = lambda o: (o.get('failed', 0), o.get('finished', 0)) + elif order_by == 'canceled': + sorting_method = lambda o: (o.get('canceled', 0), o.get('finished', 0)) + elif order_by == 'rate': + sorting_method = lambda o: (o.get('rate', 0), o.get('finished', 0)) + else: + sorting_method = lambda o: (o.get('submitted', 0), o.get('active', 0)) + + # Generate summary - sum of all values + summary = { + 'submitted': sum(map(lambda o: o.get('submitted', 0), objs), 0), + 'active': sum(map(lambda o: o.get('active', 0), objs), 0), + 'finished': sum(map(lambda o: o.get('finished', 0), objs), 0), + 'failed': sum(map(lambda o: o.get('failed', 0), objs), 0), + 'canceled': sum(map(lambda o: o.get('canceled', 0), objs), 0), + } + if summary['finished'] > 0 or summary['failed'] > 0: + summary['rate'] = (float(summary['finished']) / (summary['finished'] + summary['failed'])) * 100 + + # Return + # tables - list of job + return { + 'overview': paged( + OverviewExtendedDel(not_before, sorted(objs, key=sorting_method, reverse=order_desc), cursor=cursor), + http_request + ), + # sum of values (Generate summary) + 'summary': summary + } diff --git a/src/apps/ftsmon/views/jobs.py b/src/apps/ftsmon/views/jobs.py index a96e25dc7a74244bcd57e786bd31b2a192bf8b82..8fe755d509e0f03412adcb1cf1df9ce5297b730c 100644 --- a/src/apps/ftsmon/views/jobs.py +++ b/src/apps/ftsmon/views/jobs.py @@ -133,7 +133,7 @@ class JobListDecorator(object): return self.job_ids[index] else: return self._decorated(index) - + def __iter__(self): class _Iter(object): def __init__(self, container): @@ -247,7 +247,6 @@ class RetriesFetcher(object): }, retries.all()) yield f - class LogLinker(object): """ Change the log so it is the actual link @@ -265,7 +264,6 @@ class LogLinker(object): f.log_file = log_link(f.transfer_host, f.log_file) yield f - @jsonify def get_job_transfers(http_request, job_id): files = File.objects.filter(job=job_id) diff --git a/src/apps/ftsmon/views/jobs_del.py b/src/apps/ftsmon/views/jobs_del.py new file mode 100644 index 0000000000000000000000000000000000000000..2bc8227647ab5dd71f95025a40c975798de99bb2 --- /dev/null +++ b/src/apps/ftsmon/views/jobs_del.py @@ -0,0 +1,347 @@ +# Copyright notice: +# +# Copyright (C) CERN 2013-2015 +# +# Copyright (C) Members of the EMI Collaboration, 2010-2013. +# See www.eu-emi.eu for details on the copyright holders +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.db import connection +from django.db.models import Q, Count +from django.http import Http404 +from datetime import datetime, timedelta + +from jsonify import jsonify, jsonify_paged +from ftsweb.models import Job, File, RetryError, DmFile +from util import get_order_by, ordered_field, paged, log_link + + +def setup_filters(http_request): + # Default values + filters = { + 'state': None, + 'time_window': 1, + 'vo': None, + 'source_se': None, + 'source_surl': None, + 'metadata': None, + 'hostname': None, + 'reason': None, + 'with_file': None, + 'dest_se': None, + } + + for key in filters.keys(): + try: + if key in http_request.GET: + if key == 'time_window': + filters[key] = int(http_request.GET[key]) + elif key == 'state' or key == 'with_file': + if http_request.GET[key]: + filters[key] = http_request.GET[key].split(',') + else: + filters[key] = http_request.GET[key] + except: + pass + + return filters + + +class JobListDecorator(object): + """ + Wraps the list of jobs and appends additional information, as + file count per state + This way we only do it for the number that is being actually sent + """ + + def __init__(self, job_ids): + self.job_ids = job_ids + self.cursor = connection.cursor() + + def __len__(self): + return len(self.job_ids) + + def get_job(self, job_id): + job = {'job_id': job_id} + self.cursor.execute( + "SELECT submit_time, job_state, vo_name, source_se, dest_se, job_finished " + "FROM t_job WHERE job_id = %s", [job_id]) + job_desc = self.cursor.fetchall()[0] + job['submit_time'] = job_desc[0] + job['job_state'] = job_desc[1] + job['vo_name'] = job_desc[2] + job['source_se'] = job_desc[3] + job['dest_se'] = job_desc[4] + job['job_finished'] = job_desc[5] + self.cursor.execute( + """SELECT file_state, COUNT(file_id) + FROM t_dm WHERE job_id = %s GROUP BY file_state ORDER BY NULL + """, [job_id]) + result = self.cursor.fetchall() + count = dict() + total = 0 + for r in result: + count[r[0]] = r[1] + total += r[1] + job['files'] = count + job['count'] = total + return job + + def _decorated(self, index): + for job_id in self.job_ids[index]: + yield self.get_job(job_id) + + def __getitem__(self, index): + if not isinstance(index, slice): + index = slice(index, index, 1) + + step = index.step if index.step else 1 + nelems = (index.stop - index.start if index.start else 0) / step + if nelems > 100: + return self.job_ids[index] + else: + return self._decorated(index) + + def __iter__(self): + class _Iter(object): + def __init__(self, container): + self.container = container + self.job_id_iter = iter(container.job_ids) + + def next(self): + job_id = self.job_id_iter.next() + return self.container.get_job(job_id) + + return _Iter(self) + +@jsonify_paged +def get_job_list(http_request): + """ + This view is a little bit trickier than the others. + We get the list of job ids from t_job or t_dm depending if + filtering by state of with_file is used. + Luckily, the rest of fields are available in both tables + """ + filters = setup_filters(http_request) + # for None destination + if filters['with_file']: + job_ids = DmFile.objects.values('job_id').distinct().filter(file_state__in=filters['with_file']).filter(dest_se__isnull=True) + elif filters['state']: + job_ids = Job.objects.values('job_id').filter(job_state__in=filters['state']).order_by('-submit_time').filter(dest_se__isnull=True) + else: + job_ids = Job.objects.values('job_id').order_by('-submit_time').filter(dest_se__isnull=True) + + if filters['time_window']: + not_before = datetime.utcnow() - timedelta(hours=filters['time_window']) + if filters['with_file']: + job_ids = job_ids.filter(Q(finish_time__gte=not_before) | Q(finish_time=None)).filter(dest_se__isnull=True) + else: + job_ids = job_ids.filter(Q(job_finished__gte=not_before) | Q(job_finished=None)).filter(dest_se__isnull=True) + + if filters['vo']: + job_ids = job_ids.filter(vo_name=filters['vo']).filter(dest_se__isnull=True) + if filters['source_se']: + job_ids = job_ids.filter(source_se=filters['source_se']).filter(dest_se__isnull=True) + + job_list = JobListDecorator(map(lambda j: j['job_id'], job_ids)) + return job_list + +@jsonify +def get_job_details(http_request, job_id): + reason = http_request.GET.get('reason', None) + try: + file_id = http_request.GET.get('file', None) + if file_id is not None: + file_id = int(file_id) + except: + file_id = None + + try: + job = Job.objects.get(job_id=job_id) + count_dm = DmFile.objects.filter(job=job_id) + except Job.DoesNotExist: + raise Http404 + + if reason: + count_dm = count_dm.filter(reason=reason) + if file_id: + count_dm = count_dm.filter(file_id=file_id) + count_dm = count_dm.values('file_state').annotate(count=Count('file_state')) + + # Set job duration + if job.job_finished: + job.__dict__['duration'] = job.job_finished - job.submit_time + else: + job.__dict__['duration'] = datetime.utcnow() - job.submit_time + + # Count as dictionary + state_count = {} + for st in count_dm: + state_count[st['file_state']] = st['count'] + + return {'job': job, 'states': state_count} + + +class RetriesFetcher(object): + """ + Fetches, on demand and if necessary, the retry error messages + """ + + def __init__(self, files): + self.files = files + + def __len__(self): + return len(self.files) + + def __getitem__(self, i): + for f in self.files[i]: + retries = RetryError.objects.filter(file_id=f.file_id) + f.retries = map(lambda r: { + 'reason': r.reason, + 'datetime': r.datetime, + 'attempt': r.attempt + }, retries.all()) + yield f + +class LogLinker(object): + """ + Change the log so it is the actual link + """ + + def __init__(self, files): + self.files = files + + def __len__(self): + return len(self.files) + + def __getitem__(self, i): + for f in self.files[i]: + if hasattr(f, 'log_file') and f.log_file: + f.log_file = log_link(f.transfer_host, f.log_file) + yield f + +@jsonify +def get_job_transfers(http_request, job_id): + files = File.objects.filter(job=job_id) + + if not files: + files = DmFile.objects.filter(job=job_id) + if not files: + raise Http404 + + # Ordering + (order_by, order_desc) = get_order_by(http_request) + if order_by == 'id': + files = files.order_by(ordered_field('file_id', order_desc)) + elif order_by == 'size': + files = files.order_by(ordered_field('filesize', order_desc)) + elif order_by == 'finish_time': + files = files.order_by(ordered_field('finish_time', order_desc)) + + # Pre-fetch + files = list(files) + + # Job submission time + submission_time = Job.objects.get(job_id=job_id).submit_time + + # Build up stats + now = datetime.utcnow() + first_start_time = min(map(lambda f: f.get_start_time() if f.get_start_time() else now, files)) + if files[0].finish_time: + running_time = files[0].finish_time - first_start_time + else: + running_time = now - first_start_time + running_time = (running_time.seconds + running_time.days * 24 * 3600) + + total_size = sum(map(lambda f: f.filesize if f.filesize else 0, files)) + transferred = sum(map(lambda f: f.transferred if f.transferred else 0, files)) + + stats = { + 'total_size': total_size, + 'total_done': transferred, + 'first_start': first_start_time + } + + if first_start_time: + stats['queued_first'] = first_start_time - submission_time + else: + stats['queued_first'] = now - submission_time + + if running_time: + stats['time_transfering'] = running_time + + # Now we got the stats, apply filters + if http_request.GET.get('state', None): + files = filter(lambda f: f.file_state in http_request.GET['state'].split(','), files) + if http_request.GET.get('reason', None): + files = filter(lambda f: f.reason == http_request.GET['reason'], files) + if http_request.GET.get('file', None): + try: + file_id = int(http_request.GET['file']) + files = filter(lambda f: f.file_id == file_id, files) + except: + pass + + return { + 'files': paged(RetriesFetcher(LogLinker(files)), http_request), + 'stats': stats + } + + +#@jsonify_paged +#def get_transfer_list(http_request): +# filters = setup_filters(http_request) + +# transfers = File.objects +# if filters['state']: +# transfers = transfers.filter(file_state__in=filters['state']) +# else: +# transfers = transfers.exclude(file_state='NOT_USED') +# if filters['source_se']: +# transfers = transfers.filter(source_se=filters['source_se']) +# if filters['dest_se']: +# transfers = transfers.filter(dest_se=filters['dest_se']) +# if filters['source_surl']: +# transfers = transfers.filter(source_surl=filters['source_surl']) +# if filters['vo']: +# transfers = transfers.filter(vo_name=filters['vo']) +# if filters['time_window']: +# not_before = datetime.utcnow() - timedelta(hours=filters['time_window']) +# if _contains_active_state(filters['state']): +# transfers = transfers.filter(Q(finish_time__isnull=True) | (Q(finish_time__gte=not_before))) +# else: +# transfers = transfers.filter(Q(finish_time__gte=not_before)) +# if filters['hostname']: +# transfers = transfers.filter(transfer_host=filters['hostname']) +# if filters['reason']: +# transfers = transfers.filter(reason=filters['reason']) + +# transfers = transfers.values( +# 'file_id', 'file_state', 'job_id', +# 'source_se', 'dest_se', 'start_time', 'finish_time', +# 'user_filesize', 'filesize' +# ) + + # Ordering +# (order_by, order_desc) = get_order_by(http_request) +# if order_by == 'id': +# transfers = transfers.order_by(ordered_field('file_id', order_desc)) +# elif order_by == 'start_time': +# transfers = transfers.order_by(ordered_field('start_time', order_desc)) +# elif order_by == 'finish_time': +# transfers = transfers.order_by(ordered_field('finish_time', order_desc)) +# else: +# transfers = transfers.order_by('-finish_time') + +# return transfers diff --git a/src/apps/ftsmon/views/overview.py b/src/apps/ftsmon/views/overview.py index 0df46ae06727fb512f65ae6dffc03d4eeb00e765..9157404473abb7a1a038c5503036a4a6fdf49554 100644 --- a/src/apps/ftsmon/views/overview.py +++ b/src/apps/ftsmon/views/overview.py @@ -74,6 +74,21 @@ class OverviewExtended(object): else: return self.objects[indexes] +class OverviewExtendedDel(object): + """ + The return of overview for only deletion jobs + """ + def __init__(self, not_before, objects, cursor): + self.objects = objects + self.not_before = not_before + self.cursor = cursor + + def __len__(self): + return len(self.objects) + + def __getitem__(self, indexes): + if isinstance(indexes, types.SliceType): + return self.objects[indexes] @cache_page(60) @jsonify diff --git a/src/libs/ftsweb/models.py b/src/libs/ftsweb/models.py index 118a98be4e45b70efa0d86bdd7adc0a34ca2f85e..fe0d57c6825df8a61e717fab332afc6477c7c8e8 100644 --- a/src/libs/ftsweb/models.py +++ b/src/libs/ftsweb/models.py @@ -168,7 +168,6 @@ class DmFile(models.Model): class Meta: db_table = 't_dm' - class RetryError(models.Model): attempt = models.IntegerField() datetime = models.DateTimeField() diff --git a/src/media/html/global_filter.html b/src/media/html/global_filter.html index ce6892338c51f7aa4c53f243fd22d091ee054ee1..5c9bc68a17815ad18ce633518624b561d282f2fa 100644 --- a/src/media/html/global_filter.html +++ b/src/media/html/global_filter.html @@ -14,8 +14,9 @@ <input id="global-source" type="text" data-ng-model="globalFilter.source_se" placeholder="Source storage" data-typeahead="source for source in unique.sources | safeFilter:$viewValue" data-typeahead-on-select="applyGlobalFilter()"/> - <i class="icon-arrow-right"></i> - <input id="global-destination" type="text" data-ng-model="globalFilter.dest_se" placeholder="Destination storage" + <i class="icon-arrow-right" data-ng-show="!attrs.hideIconright"></i> + <input data-ng-show="!attrs.hidePairdest" + id="global-destination" type="text" data-ng-model="globalFilter.dest_se" placeholder="Destination storage" typeahead="dest for dest in unique.destinations | safeFilter:$viewValue" data-typeahead-on-select="applyGlobalFilter()"/> </div> @@ -30,6 +31,7 @@ <option value="4">4 hours</option> <option value="5">5 hours</option> <option value="6">6 hours</option> + <option value="24">24 hours</option> </select> </div> </div> diff --git a/src/media/html/jobs/index.html b/src/media/html/jobs/index.html index ab4d42f2ce7ac6fc4349aa4e932b94daf269b4c8..ce88a5f9ef8269cc491f96f76c382bdd5d0fbf43 100644 --- a/src/media/html/jobs/index.html +++ b/src/media/html/jobs/index.html @@ -40,7 +40,7 @@ <td>{{job.space_token}}</td> </tr> <tr style="background: #CCC" data-ng-show="job.show"> - <td colspan="8"> + <td colspan="9"> <dl> <dt>File count</dt> <dd> diff --git a/src/media/html/jobs/view.html b/src/media/html/jobs/view.html index 5813ffe4838069c71b673179aa69c5c1301c0924..6b3ff42a11b03342b9a8f5e5a5108cc8a093851b 100644 --- a/src/media/html/jobs/view.html +++ b/src/media/html/jobs/view.html @@ -142,7 +142,7 @@ <td class="status {{file.file_state}}"> {{file.file_state}} <span data-ng-show="file.source_surl == file.dest_surl" title="Staging only">(B)</span> - <span data-ng-show="file." + <span data-ng-show="file."></span> </td> <td>{{file.filesize|filesize}}</td> <td>{{file.throughput|number:2}} MB/s</td> diff --git a/src/media/html/jobs_del/filter_del.html b/src/media/html/jobs_del/filter_del.html new file mode 100644 index 0000000000000000000000000000000000000000..e4526a0cc47e8b229247c905d7bcb128576e7fca --- /dev/null +++ b/src/media/html/jobs_del/filter_del.html @@ -0,0 +1,27 @@ +<div id="filterDialog" class="modal" role="dialog" style="display: none"> + <form autocomplete="off"> + <div class="modal-header"> + <h3>Filters</h3> + </div> + <div class="modal-body"> + <label>State:</label> + <p class="btn-group"> + <button type="button" class="btn btn-info btn-small" data-ng-model="filter.state.submitted" data-btn-checkbox>SUBMITTED</button> + <button type="button" class="btn btn-primary btn-small" data-ng-model="filter.state.active" data-btn-checkbox>ACTIVE</button> + <button type="button" class="btn btn-primary btn-small" data-ng-model="filter.state.delete" data-btn-checkbox>DELETE</button> + <button type="button" class="btn btn-success btn-small" data-ng-model="filter.state.finished" data-btn-checkbox>FINISHED</button> + <br/> + <button type="button" class="btn btn-danger btn-small" data-ng-model="filter.state.canceled" data-btn-checkbox>CANCELED</button> + <button type="button" class="btn btn-danger btn-small" data-ng-model="filter.state.failed" data-btn-checkbox>FAILED</button> + <button type="button" class="btn btn-warning btn-small" data-ng-model="filter.state.finisheddirty" data-btn-checkbox>FINISHED DIRTY</button> + </p> + + <label for="timewindow">Time window:</label> + <input type="number" placeholder="time window in hours" id="timewindow" data-ng-model="filter.time_window"/> + </div> + <div class="modal-footer"> + <a class="btn btn-success" data-ng-click="applyFilters()">Apply</a> + <a class="btn btn-danger" data-ng-click="filtersModal = false">Cancel</a> + </div> + </form> +</div> diff --git a/src/media/html/jobs_del/jobs_del.html b/src/media/html/jobs_del/jobs_del.html new file mode 100644 index 0000000000000000000000000000000000000000..2126225407edb1e75dfdbfd942b74e88a5d6702f --- /dev/null +++ b/src/media/html/jobs_del/jobs_del.html @@ -0,0 +1,86 @@ +<global-filter data-hide-iconright="true" data-hide-pairdest="true" on-more="showFilterDialog()"></global-filter> <!--hidden menu--> + +<h3> + Showing {{jobs_del.startIndex}} to {{jobs_del.endIndex}} out of {{jobs_del.count}} +</h3> + +<pagination rotate="false" + page="jobs_del.page" total-items="jobs_del.count" items-per-page="jobs_del.pageSize" + max-size="15" class="pagination" boundary-links="true" + on-select-page="pageChanged(page)"></pagination> + +<table class="table"> + <thead> + <tr> + <th style="width: 12%">Job id</th> + <th style="width: 12%">Submit time</th> + <th style="width: 8%">Job state</th> + <th style="width: 4%">VO</th> + <th style="width: 15%">Source SE</th> + <th style="width: 4%">Files</th> + </tr> + </thead> + <tbody data-ng-repeat="job_del in jobs_del.items" data-ng-class-odd="'odd'" data-ng-class="classFromMetadata(job_del)"> + <tr> + <td> + <i class="icon-warning-sign" data-ng-show="job_del.diagnosis" data-tooltip="{{job_del.diagnosis}}"></i> + <a href=#/job_del/{{job_del.job_id}}>{{job_del.job_id}}</a></td> + <td>{{job_del.submit_time}}</td> + <td class="status {{job_del.job_state}}"> + <span class="active" data-ng-click="job_del.show=!job_del.show" title="Details">{{job_del.job_state}}</span> + </td> + <td>{{job_del.vo_name}}</td> + <td class="hscroll">{{job_del.source_se}}</td> + <td>{{job_del.count}}</td> + </tr> + <tr style="background: #CCC" data-ng-show="job_del.show"> + <td colspan="6"> + <dl> + <dt>File count</dt> + <dd> + <span data-ng-repeat="(state, count) in job_del.files">{{count}} {{state}} </span> + </dd> + <dt>Job finished time<dt> + <dd>{{job_del.job_finished}}</dd> + </dl> + </td> + </tr> + </tbody> +</table> + +<pagination rotate="false" + page="jobs_del.page" total-items="jobs_del.count" items-per-page="jobs_del.pageSize" + max-size="15" class="pagination" boundary-links="true" + on-select-page="pageChanged(page)"></pagination> + +<!-- Modals --> +<div id="filterDialog" class="modal" role="dialog" style="display: none"> + <form data-ng-submit="applyFilters()" autocomplete="off"> + <div class="modal-header"> + <h3>Filters</h3> + </div> + <div class="modal-body"> + <label>State:</label> + <p class="btn-group"> + <button type="button" class="btn btn-info btn-small" data-ng-model="filter.state.submitted" data-btn-checkbox>SUBMITTED</button> + <button type="button" class="btn btn-primary btn-small" data-ng-model="filter.state.active" data-btn-checkbox>ACTIVE</button> + <button type="button" class="btn btn-primary btn-small" data-ng-model="filter.state.delete" data-btn-checkbox>DELETE</button> + <button type="button" class="btn btn-success btn-small" data-ng-model="filter.state.finished" data-btn-checkbox>FINISHED</button> + <br/> + <button type="button" class="btn btn-danger btn-small" data-ng-model="filter.state.canceled" data-btn-checkbox>CANCELED</button> + <button type="button" class="btn btn-danger btn-small" data-ng-model="filter.state.failed" data-btn-checkbox>FAILED</button> + <button type="button" class="btn btn-warning btn-small" data-ng-model="filter.state.finisheddirty" data-btn-checkbox>FINISHED DIRTY</button> + </p> + + <p> + <label for="timewindow">Time window:</label> + <input type="number" placeholder="time window in hours" id="timewindow" data-ng-model="filter.time_window"/> + </p> + </div> + <div class="modal-footer"> + <a class="btn btn-success" data-ng-click="applyFilters()">Apply</a> + <a class="btn btn-danger" data-ng-click="cancelFilters()">Cancel</a> + </div> + </form> + +</div> diff --git a/src/media/html/jobs_del/view_del.html b/src/media/html/jobs_del/view_del.html new file mode 100644 index 0000000000000000000000000000000000000000..422449831d0b885824033e49d9ee34a50c5980c5 --- /dev/null +++ b/src/media/html/jobs_del/view_del.html @@ -0,0 +1,164 @@ +<h3>Deletion '{{job_del.job.job_id}}' {{job_del.job.job_state}}</h3> + +<div class="alert" data-ng-show="filter.reason"> + <strong>Filtered by error message</strong> + {{filter.reason}} + <button type="button" class="close" title="Remove filter" data-ng-click="resetReasonFilter()">×</button> +</div> +<p> + <i class="icon-group"></i> VO: <span class="vo">{{job_del.job.vo_name}}</span><br/> +</p> +<div class="row-fluid"> + <div class="span6"> + <p> + <i class="icon-user"></i> Delegation ID: {{job_del.job.cred_id}}<br/> + <i class="icon-time"></i> Submitted time: {{job_del.job.submit_time}}<br/> + <i class="icon-time"></i> Job finished: {{job_del.job.job_finished}}<br/> + </p> + </div> + <div class="span6"> + <p> + <i class="icon-hdd"></i> Received by {{job_del.job.submit_host}}<br/> + <i class="icon-remove"></i> Overwrite flag: {{job_del.job.overwrite_flag}}<br/> + <i class="icon-refresh"></i> Job type: {{job_del.job.job_type}}<br/> + <i class="icon-ban-circle"></i> Cancel flag: {{job_del.job.cancel_job}}<br/> + </p> + </div> +</div> +<p> + <i class="icon-pencil"></i> Metadata: + <pre>{{job_del.job.job_metadata}}</pre> +</p> + +<table class="table"> + <thead> + <tr> + <th>Total size</th> + <th>Done</th> + <th>Submission time</th> + <th>Start time</th> + <th title="Since start time">Running time</th> + </tr> + </thead> + <tbody> + <tr> + <td>{{files.stats.total_size|filesize}}</td> + <td>{{files.stats.total_done|filesize}}</td> + <td>{{job_del.job.submit_time}}</td> + <td>{{files.stats.first_start}} (<span style="color:red">+{{files.stats.queued_first}}s</span>) + <td><span data-optional-number="{{files.stats.time_transfering}}" data-decimals="0" data-suffix="s"></span></td> + </tr> + <tr> + <td colspan="5" class="progress"> + <progressbar max="files.stats.total_size" value="files.stats.total_done"></progressbar> + </td> + </tr> + </tbody> +</table> + +<h4>Showing {{files.files.startIndex}} to {{files.files.endIndex}} out of {{files.files.count}}</h4> +<p class="btn-group" data-ng-click="filterByState()"> + <button type="button" class="btn btn-info btn-small" + data-ng-model="filter.state.submitted" data-btn-checkbox + data-ng-disabled="!job_del.states.SUBMITTED">{{job_del.states.SUBMITTED}} SUBMITTED</button> + <button type="button" class="btn btn-delete btn-small" + data-ng-model="filter.state.delete" data-btn-checkbox + data-ng-disabled="!job_del.states.DELETE">{{job_del.states.DELETE}} DELETE</button> + <button type="button" class="btn btn-primary btn-small" + data-ng-model="filter.state.active" data-btn-checkbox + data-ng-disabled="!job_del.states.ACTIVE">{{job_del.states.ACTIVE}} ACTIVE</button> + <button type="button" class="btn btn-danger btn-small" + data-ng-model="filter.state.canceled" data-btn-checkbox + data-ng-disabled="!job_del.states.CANCELED">{{job_del.states.CANCELED}} CANCELED</button> + <button type="button" class="btn btn-danger btn-small" + data-ng-model="filter.state.failed" data-btn-checkbox + data-ng-disabled="!job_del.states.FAILED">{{job_del.states.FAILED}} FAILED</button> + <button type="button" class="btn btn-success btn-small" + data-ng-model="filter.state.finished" data-btn-checkbox + data-ng-disabled="!job_del.states.FINISHED">{{job_del.states.FINISHED}} FINISHED</button> + <button type="button" class="btn btn-small" + data-ng-model="filter.state.not_used" data-btn-checkbox + data-ng-disabled="!job_del.states.NOT_USED">{{job_del.states.NOT_USED}} NOT_USED</button> +</p> + +<pagination rotate="false" + page="files.files.page" total-items="files.files.count" items-per-page="files.files.pageSize" + max-size="15" class="pagination" boundary-links="true" + on-select-page="pageChanged(page)"></pagination> + +<table class="table table-small"> + <thead> + <tr> + <th style="width: 8%"> + <span data-order-by="id">File ID</span> + </th> + <th style="width: 6%">File State</th> + <th style="width: 7%"> + <span data-order-by="size">File Size</span> + </th> + <th style="width: 10%"> + <span data-order-by="start_time">Start Time</span> + </th> + <th style="width: 10%"> + <span data-order-by="finish_time">Finish Time</span> + </th> + <th style="width: 6%"></th> + </tr> + </thead> + <tbody data-ng-repeat="file in files.files.items"> + <tr class="transfer_header"> + <td> + <i class="icon-plus" data-ng-click="file.show = !file.show"></i> + {{file.file_id}} + <i class="icon-refresh" data-ng-show="file.retry"></i> + </td> + <td class="status {{file.file_state}}"> + {{file.file_state}} + <span data-ng-show="file."></span> + </td> + <td>{{file.filesize|filesize}}</td> + <td>{{file.start_time}}</td> + <td>{{file.finish_time}}</td> + <td> + <span data-ng-if="file.log_file"> + <a href="{{file.log_file}}"> + <i class="glyphicon icon-file"></i> Log + </a> + </span> + </td> + </tr> + <tr class="nested"> + <td colspan="5" class="hscroll"> + <i class="glyphicon icon-home" title="Source"></i> {{file.source_surl}} + </td> + </tr> + <tr style="background: #CCC" data-ng-show="file.show"> + <td colspan="6"> + <ul> + <li>Transfer host: {{file.transfer_host}}</li> + <li>PID: {{file.pid}}</li> + <li>Hash: {{file.hashed_id|hex}}</li> + <li>Attempts: {{file.retry}}</li> + <li>Duration: {{file.tx_duration}} seconds</li> + <li>Configuration: {{file.symbolicname}}</li> + <li>Finished time: {{file.finish_time}}</li> + <li>Error reason: {{file.reason}}</li> + <li>Metadata: + <pre>{{file.file_metadata}}</pre> + </li> + <li data-ng-show="file.retry">Retries: + <dl data-ng-repeat="retry in file.retries"> + <dt>{{retry.datetime}}</dt> + <dd>{{retry.reason}}</dd> + </dl> + </li> + </ul> + </td> + </tr> + </tbody> +</table> + +<pagination rotate="false" + page="files.files.page" total-items="files.files.count" items-per-page="files.files.pageSize" + max-size="15" class="pagination" boundary-links="true" + on-select-page="pageChanged(page)"></pagination> diff --git a/src/media/html/overview/deletion.html b/src/media/html/overview/deletion.html new file mode 100644 index 0000000000000000000000000000000000000000..d04b83dc676b740deb60dc7e394eeec4f41d05a7 --- /dev/null +++ b/src/media/html/overview/deletion.html @@ -0,0 +1,134 @@ +<global-filter></global-filter> + +<h2>Overview Deletion Jobs</h2> +<h3> + Showing {{overview.overview.startIndex}} to {{overview.overview.endIndex}} + out of {{overview.overview.count}} from the last {{globalFilter.time_window || '1' }} + <span data-ng-show="globalFilter.time_window == 1 || !globalFilter.time_window">hour</span> + <span data-ng-show="globalFilter.time_window > 1">hours</span> +</h3> + +<pagination rotate="false" + page="overview.overview.page" total-items="overview.overview.count" items-per-page="overview.overview.pageSize" + max-size="15" class="pagination" boundary-links="true" + on-select-page="pageChanged(page)"></pagination> + +<table class="table table-small"> + <thead> + <tr> + <th> + <i class="icon-th-list" data-ng-click="filterBy({source_se: ''})"></i> + Source + </th> + <th style="width: 3.5%"> + <i class="icon-th-list" data-ng-click="filterBy({vo: ''})"></i> + VO + </th> + <th style="width: 6%" class="numeric"> + <span data-order-by="submitted">Submitted</span> + </th> + <th style="width: 4%" class="numeric"> + <span data-order-by="active">Active</span> + </th> + <th style="width: 5%" class="numeric"> + <span data-order-by="finished">Finished</span> + </th> + <th style="width: 3.5%" class="numeric"> + <span data-order-by="failed">Failed</span> + </th> + <th style="width: 3.5%" class="numeric"> + <span data-order-by="canceled">Cancel</span> + </th> + + <th style="width: 7%" class="numeric"> + <span data-order-by="rate">Rate</span> + </th> + </tr> + </thead> + <tbody data-ng-repeat="o in overview.overview.items" data-ng-class-odd="'odd'" class="overview" data-ng-class="pairState(o)"> + <tr> + <td> + <i class="icon-plus" data-ng-click="o.show = !o.show"></i> + <span class="filter-on-click" data-ng-click="filterBy({source_se: o.source_se})" title="Filter source SE"> + {{ o.source_se }} + </span> + </td> + <td> + <span class="filter-on-click" data-ng-click="filterBy({vo: o.vo_name})" title="Filter VO"> + {{ o.vo_name }} + </span> + </td> + <td class="numeric"> + <a href="#/jobs_del?source_se={{o.source_se|escape}}&dest_se={{o.dest_se|escape}}&vo={{o.vo_name|escape}}&with_file=SUBMITTED&time_window={{globalFilter.time_window}}"> + <span data-optional-number="{{o.submitted}}"></span> + </a> + </td> + <td class="numeric"> + <span data-ng-show="o.active_fixed"> + <i class="icon-magnet" title="Number of active fixed"></i> + </span> + <a href="#/jobs_del?source_se={{o.source_se|escape}}&dest_se={{o.dest_se|escape}}&vo={{o.vo_name|escape}}&with_file=ACTIVE&time_window={{globalFilter.time_window}}"> + <span data-optional-number="{{o.active}}" data-decimals="0"></span> + </a> + </td> + <td class="numeric"> + <a href="#/jobs_del?source_se={{o.source_se|escape}}&dest_se={{o.dest_se|escape}}&vo={{o.vo_name|escape}}&with_file=FINISHED&time_window={{globalFilter.time_window}}"> + <span data-optional-number="{{o.finished}}" data-decimals="0"></span> + </a> + </td> + <td class="numeric"> + <a href="#/jobs_del?source_se={{o.source_se|escape}}&dest_se={{o.dest_se|escape}}&vo={{o.vo_name|escape}}&with_file=FAILED&time_window={{globalFilter.time_window}}"> + <span data-optional-number="{{o.failed}}" data-decimals="0"></span> + </a> + </td> + <td class="numeric"> + <a href="#/jobs_del?source_se={{o.source_se|escape}}&dest_se={{o.dest_se|escape}}&vo={{o.vo_name|escape}}&with_file=CANCELED&time_window={{globalFilter.time_window}}"> + <span data-optional-number="{{o.canceled}}" data-decimals="0"></span> + </a> + </td> + <td class="numeric"> + <span data-optional-number="{{o.rate}}" data-suffix="%" data-decimals="2" data-display-zero="true"></span> + </td> + </tr> + <tr> + <td colspan="8" style="background: #CCC" data-ng-show="o.show"> + <dl> + <dt>Most frequent error</dt> + <dd>{{o.most_frequent_error}}</dd> + </dl> + </td> + </tr> + </tbody> + <tfoot> + <tr> + <td> </td> + <td> </td> + <td class="numeric">{{overview.summary.submitted}}</td> + <td class="numeric">{{overview.summary.active}}</td> + <td class="numeric">{{overview.summary.finished}}</td> + <td class="numeric">{{overview.summary.failed}}</td> + <td class="numeric">{{overview.summary.canceled}}</td> + <td class="numeric"> + <span data-optional-number="{{overview.summary.rate}}" data-suffix="%" data-decimals="2"></span> + </td> + </tr> + </tfoot> +</table> + +<pagination rotate="false" + page="overview.overview.page" total-items="overview.overview.count" items-per-page="overview.overview.pageSize" + max-size="15" class="pagination" boundary-links="true" + on-select-page="pageChanged(page)"></pagination> + +<div> + <p> + <span class="label label-important">Bad shape</span> + There are submitted but no active, less than 3 active with more than 3 submitted, or a failure rate >= 20%<br/> + <span class="label label-warning">Underused</span> + Less than three actives, but no submitted waiting.<br/> + <span class="label label-success">Good shape</span> + Success rate >= 90%, or more than three actives with a failure rate < 20%.<br/> + <span class="label">Nothing special</span> + No active, no submitted, success rate between 80% and 90%. + </p> +</div> diff --git a/src/media/html/statistics/overview.html b/src/media/html/statistics/overview.html index 50fb52b363d1e6e59e658c1e3917fb8d26b0d96b..ba477d951f264249c48dee5d09890e1b8f021c5d 100644 --- a/src/media/html/statistics/overview.html +++ b/src/media/html/statistics/overview.html @@ -1,4 +1,4 @@ -<global-filter data-hide-vo="true" data-hide-pair="true"></global-filter> +<global-filter data-hide-vo="true" data-hide-pair="true" data-hide-pairdest="true"></global-filter> <h2> Statistics - @@ -95,4 +95,4 @@ <div class="span6"> <canvas id="lastHourPlot"></canvas> </div> -</div> \ No newline at end of file +</div> diff --git a/src/media/html/statistics/servers.html b/src/media/html/statistics/servers.html index 79fd9c0ba3df57514878cb37bacf9afc87b41a38..09a21bce008417b9d1d80303decb57afbe90bca2 100644 --- a/src/media/html/statistics/servers.html +++ b/src/media/html/statistics/servers.html @@ -1,4 +1,4 @@ -<global-filter data-hide-vo="true" data-hide-pair="true"></global-filter> +<global-filter data-hide-vo="true" data-hide-pair="true" data-hide-pairdest="true"></global-filter> <h2>Statistics - Servers</h2> <h3> diff --git a/src/media/html/statistics/vos.html b/src/media/html/statistics/vos.html index f51715768c47d34a50fb44b7e881eea779b57bca..77a790be4665aa62cfeaeae792db43f7a696992b 100644 --- a/src/media/html/statistics/vos.html +++ b/src/media/html/statistics/vos.html @@ -1,4 +1,4 @@ -<global-filter data-hide-vo="true" data-hide-pair="true"></global-filter> +<global-filter data-hide-vo="true" data-hide-pair="true" data-hide-pairdest="true"></global-filter> <h2>Statistics - VO</h2> <h3> diff --git a/src/media/html/transfers.html b/src/media/html/transfers.html index 346f0b74c591ad076c45b1b1fe3b5a37e71c2266..a3ea90656aade4ef10c49e97ad7e112a93e4377d 100644 --- a/src/media/html/transfers.html +++ b/src/media/html/transfers.html @@ -13,7 +13,7 @@ <table class="table" style="font-size: 90%"> <thead> <tr> - <th style="width: 5%"> + <th style="width: 7%"> <span order-by="id">File Id</span> </th> <th style="width: 10%">File state</th> @@ -21,10 +21,10 @@ <th style="width: 18%">Source</th> <th style="width: 18%">Destination</th> <th style="width: 11%">Activity</th> - <th style="width: 10%"> + <th style="width: 12%"> <span order-by="start_time">Start time</span> </th> - <th style="width: 10%"> + <th style="width: 12%"> <span order-by="finish_time" title="Finish time">Finish time</span> </th> </tr> diff --git a/src/media/js/deletion.js b/src/media/js/deletion.js new file mode 100644 index 0000000000000000000000000000000000000000..fe857447e07ac40f9b833f7221d6c4ba30afc882 --- /dev/null +++ b/src/media/js/deletion.js @@ -0,0 +1,117 @@ + +function pairState(pair) +{ + var klasses; + + // No active with submitted is bad, and so it is + // less than three active and more than three submitted + if ((!pair.active && pair.submitted) || (pair.active < 2 && pair.submitted >= 2)) + klasses = 'bad-state'; + // Very high rate of failures, that's pretty bad + else if ((!pair.finished && pair.failed) || (pair.finished / pair.failed <= 0.8)) + klasses = 'bad-state'; + // Less than three actives is so-so + else if (pair.active < 2) + klasses = 'underused'; + // More than three active, that's good enough + else if (pair.active >= 2) + klasses = 'good-state'; + // High rate of success, that's good + else if ((pair.finished && !pair.failed) || (pair.finished / pair.failed >= 0.9)) + klasses = 'good-state'; + // Meh + else + klasses = ''; + + // If any active, always give that + if (pair.active) + klasses += ' active'; + + return klasses; +} + +function mergeAttrs(a, b) +{ + for (var attr in b) { + if (typeof(b[attr]) == 'string') + a[attr] = b[attr]; + else + a[attr] = ''; + } + return a; +} + +function getLimitDescription(limit) +{ + if (!limit) + return ''; + + var descr = 'Limited at '; + if (limit.bandwidth) + descr += limit.bandwidth + 'MB/s'; + if (limit.bandwidth && limit.active) + descr += ' and '; + if (limit.active) + descr += limit.active + ' actives'; + return descr; +} + + +function OverviewDeletionCtrl($rootScope, $location, $scope, overview, OverviewDeletion) +{ + $scope.overview = overview; + $scope.monit_url = SITE_MONIT; + $scope.alias = SITE_ALIAS; + // On page change, reload + $scope.pageChanged = function(newPage) { + $location.search('page', newPage); + }; + + // Method to choose a style for a pair + $scope.pairState = pairState; + + // Render a human-readable representation of the limits + $scope.getLimitDescription = getLimitDescription; + + // Filter + $scope.filterBy = function(filter) { + $location.search($.extend({}, $location.$$search, filter)); + } + + // Set timer to trigger autorefresh + $scope.autoRefresh = setInterval(function() { + var filter = $location.$$search; + filter.page = $scope.overview.page; + loading($rootScope); + OverviewDeletion.query(filter, function(updatedOverview) { + for(var i = 0; i < updatedOverview.overview.items.length; i++) { + updatedOverview.overview.items[i].show = $scope.overview.overview.items[i].show; + } + $scope.overview = updatedOverview; + stopLoading($rootScope); + }); + }, REFRESH_INTERVAL); + $scope.$on('$destroy', function() { + clearInterval($scope.autoRefresh); + }); +} + + +OverviewDeletionCtrl.resolve = { + overview: function($rootScope, $location, $q, OverviewDeletion) { + loading($rootScope); + + var deferred = $q.defer(); + + var page = $location.$$search.page; + if (!page || page < 1) + page = 1; + + OverviewDeletion.query($location.$$search, + genericSuccessMethod(deferred, $rootScope), + genericFailureMethod(deferred, $rootScope, $location)); + + return deferred.promise; + } +} + diff --git a/src/media/js/ftsmon.js b/src/media/js/ftsmon.js index 338d7f6132f9c9f6cf46a6196ec6ea0f53c79b81..73dd2a2d8fbc99ec66d9f24b017d3f1c5e782af1 100644 --- a/src/media/js/ftsmon.js +++ b/src/media/js/ftsmon.js @@ -6,15 +6,23 @@ config(function($routeProvider) { resolve: OverviewCtrl.resolve}). when('/jobs', {templateUrl: STATIC_ROOT + 'html/jobs/index.html', controller: JobListCtrl, - resolve: JobListCtrl.resolve}). + resolve: JobListCtrl.resolve}). when('/job/:jobId', {templateUrl: STATIC_ROOT + 'html/jobs/view.html', controller: JobViewCtrl, resolve: JobViewCtrl.resolve}). +//______________________________________________________________________________________ + when('/jobs_del', {templateUrl: STATIC_ROOT + 'html/jobs_del/jobs_del.html', + controller: JobListDelCtrl, + resolve: JobListDelCtrl.resolve}). + when('/job_del/:jobId', {templateUrl: STATIC_ROOT + 'html/jobs_del/view_del.html', + controller: JobDelViewCtrl, + resolve: JobDelViewCtrl.resolve}). +//_______________________________________________________________________________________ when('/transfers', {templateUrl: STATIC_ROOT + 'html/transfers.html', controller: TransfersCtrl, resolve: TransfersCtrl.resolve}). - + when('/optimizer/', {templateUrl: STATIC_ROOT + 'html/optimizer/optimizer.html', controller: OptimizerCtrl, resolve: OptimizerCtrl.resolve}). @@ -68,6 +76,11 @@ config(function($routeProvider) { when('/overview/activities', {templateUrl: STATIC_ROOT + 'html/overview/activities.html', controller: OverviewActivitiesCtrl, resolve: OverviewActivitiesCtrl.resolve}). +//_______________________________________________________________________________________________________________ + when('/overview/deletion', {templateUrl: STATIC_ROOT + 'html/overview/deletion.html', + controller: OverviewDeletionCtrl, + resolve: OverviewDeletionCtrl.resolve}). +//_______________________________________________________________________________________________________________ when('/500', {templateUrl: STATIC_ROOT + 'html/500.html'}). @@ -190,6 +203,19 @@ config(function($routeProvider) { $location.path('/job/' + $rootScope.jobId).search({}); } }) + + +//_______________________________________________________________ +.run(function($rootScope, $location) { + $rootScope.searchJob_del = function() { + $location.path('/job_del/' + $rootScope.jobId).search({}); + } +}) + +//_______________________________________________________________ + + + .filter('safeFilter', function($filter) { return function(list, expr) { if (typeof(list) == 'undefined') diff --git a/src/media/js/jobs_del.js b/src/media/js/jobs_del.js new file mode 100644 index 0000000000000000000000000000000000000000..2f7da63490cf47b705c471dcbd1e75e5808f312f --- /dev/null +++ b/src/media/js/jobs_del.js @@ -0,0 +1,210 @@ + +function searchJob_del(jobList, jobId) +{ + for (j in jobList) { + if (jobList[j].job_id == jobId) + return jobList[j]; + } + return {show: false}; +} + +function JobListDelCtrl($rootScope, $location, $scope, jobs_del, Job_del) +{ + // Jobs + $scope.jobs_del = jobs_del; + + // On page change, reload + $scope.pageChanged = function(newPage) { + $location.search('page', newPage); + }; + + // Set timer to trigger autorefresh + $scope.autoRefresh = setInterval(function() { + loading($rootScope); + var filter = $location.$$search; + filter.page = $scope.jobs_del.page; + Job_del.query(filter, function(updatedJobs) { + for (j in updatedJobs.items) { + var job_del = updatedJobs.items[j]; + job_del.show = searchJob_del($scope.jobs_del.items, job_del.job_id).show; + } + $scope.jobs_del = updatedJobs; + stopLoading($rootScope); + }, + genericFailureMethod(null, $rootScope, $location)); + }, REFRESH_INTERVAL); + $scope.$on('$destroy', function() { + clearInterval($scope.autoRefresh); + }); + + // Set up filters + $scope.filter = { + vo: validString($location.$$search.vo), + source_se: validString($location.$$search.source_se), + time_window: withDefault($location.$$search.time_window, 1), + state: statesFromString($location.$$search.state), + } + + $scope.showFilterDialog = function() { + document.getElementById('filterDialog').style.display = 'block'; + } + + $scope.cancelFilters = function() { + document.getElementById('filterDialog').style.display = 'none'; + } + + $scope.applyFilters = function() { + document.getElementById('filterDialog').style.display = 'none'; + $location.search({ + page: 1, + time_window: $scope.filter.time_window, + state: joinStates($scope.filter.state), + }); + } + + // Method to set class depending on the metadata value + $scope.classFromMetadata = function(job_del) { + var metadata = job_del.job_metadata; + if (metadata) { + metadata = eval('(' + metadata + ')'); + if (metadata && typeof(metadata) == 'object' && 'label' in metadata) + return 'label-' + metadata.label; + } + return ''; + } +} + + +JobListDelCtrl.resolve = { + jobs_del: function($rootScope, $location, $q, Job_del) { + loading($rootScope); + + var deferred = $q.defer(); + + var page = $location.$$search.page; + if (!page || page < 1) + page = 1; + + Job_del.query($location.$$search, + genericSuccessMethod(deferred, $rootScope), + genericFailureMethod(deferred, $rootScope, $location)); + + return deferred.promise; + } +} + +/** Job_del view + */ +function JobDelViewCtrl($rootScope, $location, $scope, job_del, files, Job_del, Files_del) +{ + var page = $location.$$search.page; + if (!page) + page = 1; + + $scope.itemPerPage = 50; + + $scope.job_del = job_del; + $scope.files = files; +//__________________________________________________________________________________________ + // $scope.getRemainingTime = function(file) { + // if (file.file_state == 'ACTIVE') { + // if (file.throughput && file.filesize) { + // var bytes_per_sec = file.throughput * (1024 * 1024); + // var remaining_bytes = file.filesize - file.transferred; + // var remaining_time = remaining_bytes / bytes_per_sec; + // return (Math.round(remaining_time*100)/100).toString() + ' s'; + // } + // else { + // return '?'; + // } + // } + // else { + // return '-'; + // } + //} +//____________________________________________________________________________________________ + // On page change + $scope.pageChanged = function(newPage) { + $location.search('page', newPage); + } + + // Filtering + $scope.filter = { + state: statesFromString($location.$$search.state), + reason: validString($location.$$search.reason), + file: validString($location.$$search.file), + } + $scope.filterByState = function() { + $location.search('state', joinStates($scope.filter.state)); + } + + $scope.resetReasonFilter = function() { + $location.search({state: $location.$$search.state}); + } + + // Reloading + $scope.autoRefresh = setInterval(function() { + loading($rootScope); + var filter = $location.$$search; + filter.jobId = $scope.job_del.job.job_id; + Job_del.query(filter, function(updatedJob) { + $scope.job_del = updatedJob; + }) + // Do this in two steps so we can copy the show attribute + Files_del.query(filter, function (updatedFiles) { + for(var i = 0; i < updatedFiles.files.items.length; i++) { + updatedFiles.files.items[i].show = $scope.files.files.items[i].show; + } + $scope.files = updatedFiles; + stopLoading($rootScope); + }, + genericFailureMethod(null, $rootScope, $location)); + }, REFRESH_INTERVAL); + $scope.$on('$destroy', function() { + clearInterval($scope.autoRefresh); + }); +} + + +JobDelViewCtrl.resolve = { + job_del: function ($rootScope, $location, $route, $q, Job_del) { + loading($rootScope); + + var deferred = $q.defer(); + + var filter = { + jobId: $route.current.params.jobId + }; + if ($route.current.params.file) + filter.file = $route.current.params.file; + if ($route.current.params.reason) + filter.reason = $route.current.params.reason; + + Job_del.query(filter, + genericSuccessMethod(deferred, $rootScope), + genericFailureMethod(deferred, $rootScope)); + + return deferred.promise; + }, + + files: function ($rootScope, $location, $route, $q, Files_del) { + loading($rootScope); + + var deferred = $q.defer(); + + var filter = $location.$$search; + filter.jobId = $route.current.params.jobId + filter.jobId = $route.current.params.jobId + Files_del.query(filter, + function (data) { + genericSuccessMethod(deferred, $rootScope)(data); + // If file filter is set, by default show the details + if ($location.$$search.file) { + data.files.items[0].show = true; + } + }, + genericFailureMethod(deferred, $rootScope, $location)); + + return deferred.promise; + } +} diff --git a/src/media/js/resources.js b/src/media/js/resources.js index aac04edee69d2ed945e8ed049a64544ea561ae32..989ed0fd756c33d6103a3790cc6c696d0d1b4a2b 100644 --- a/src/media/js/resources.js +++ b/src/media/js/resources.js @@ -11,6 +11,21 @@ angular.module('ftsmon.resources', ['ngResource']) isArray: false}, }) }) +//__________________________________________________________ +.factory('Job_del', function($resource) { + return $resource('jobs_del/:jobId', {}, { + query: {method: 'GET', + isArray: false}, + }) +}) + +.factory('Files_del', function($resource) { + return $resource('jobs_del/:jobId/files', {}, { + query: {method: 'GET', + isArray: false}, + }) +}) +//__________________________________________________________ .factory('Transfers', function($resource) { return $resource('transfers', {}, { query: {method: 'GET', isArray: false} @@ -26,6 +41,13 @@ angular.module('ftsmon.resources', ['ngResource']) query: {method: 'GET', isArray: false}, }) }) + +.factory('OverviewDeletion', function($resource) { + return $resource('overview/deletion', {}, { + query: {method: 'GET', isArray: false}, + }) +}) + .factory('Optimizer', function($resource) { return $resource('optimizer', {}, { query: {method: 'GET', isArray: false}