FTS-183: Web interface for configuration

.gitignore

@@ -5,3 +5,5 @@
 *.rpm
 *.pyc
 *.pyo
+*.sublime-workspace
+

dist/fts-rest.spec

@@ -25,9 +25,9 @@ BuildRequires:  python-pylons
 BuildRequires:  scipy
 BuildRequires:  m2crypto
 BuildRequires:  python-m2ext
-BuildRequires:  python-coverage
 BuildRequires:  python-sqlalchemy
 BuildRequires:  python-requests
+BuildRequires:  python-slimit
 BuildRequires:  pandoc
 
 Requires:       gridsite%{?_isa} >= 1.7
@@ -165,6 +165,7 @@ cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
 
 %{python_sitelib}/fts3rest/controllers/api.py*
 %{python_sitelib}/fts3rest/controllers/archive.py*
+%{python_sitelib}/fts3rest/controllers/autocomplete.py*
 %{python_sitelib}/fts3rest/controllers/banning.py*
 %{python_sitelib}/fts3rest/controllers/config.py*
 %{python_sitelib}/fts3rest/controllers/datamanagement.py*
@@ -191,6 +192,7 @@ cp --preserve=timestamps -r src/fts3 %{buildroot}/%{python_sitelib}
 
 %{python_sitelib}/fts3rest/public/
 %{python_sitelib}/fts3rest/templates/delegation.html
+%{python_sitelib}/fts3rest/templates/config/
 
 %{_libexecdir}/fts3
 %config(noreplace) %{_sysconfdir}/fts3/fts3rest.ini

docs/CMakeLists.txt

@@ -20,6 +20,9 @@ foreach(md ${md_files})
     add_dependencies(man_pages "${man_name}.1")
 endforeach()
 
+if (NOT SHARE_INSTALL_PREFIX)
+    set (SHARE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/share")
+endif (NOT SHARE_INSTALL_PREFIX)
 
 install(DIRECTORY
             "${MAN_INPUT_DIR}"

rest.sublime-project

+{
+	"folders":
+	[
+		{
+			"path": "/home/aalvarez/AFS/Source/fts-rest"
+		}
+	]
+}

src/fts3/model/__init__.py

@@ -26,6 +26,7 @@ from file import *
 from job import *
 from oauth2 import *
 from optimizer import *
+from server import *
 from version import *
 
 

src/fts3/model/base.py

@@ -21,7 +21,14 @@ import json
 import types
 
 
-Base = declarative_base()
+class BaseAsDict(object):
+    def __getitem__(self, item):
+        if hasattr(self, item):
+            return getattr(self, item)
+        else:
+            raise KeyError()
+
+Base = declarative_base(cls=BaseAsDict)
 
 
 class Json(TypeDecorator):

src/fts3/model/config.py

@@ -49,13 +49,13 @@ class LinkConfig(Base):
     urlcopy_tx_to     = Column(Integer)
     auto_tuning       = Column(Flag(negative='off', positive='on'))
 
-    share_config =\
-        relation('ShareConfig', backref='link_config',
-                 primaryjoin='ShareConfig.source == LinkConfig.source and '
-                             'ShareConfig.destination == LinkConfig.destination',
-                 foreign_keys=(source, destination),
-                 uselist=False,
-                 lazy=False)
+#    share_config =\
+#        relation('ShareConfig', backref='link_config',
+#                 primaryjoin='ShareConfig.source == LinkConfig.source and '
+#                             'ShareConfig.destination == LinkConfig.destination',
+#                 foreign_keys=(source, destination),
+#                 uselist=False,
+#                 lazy=False)
 
     def __str__(self):
         return "%s => %s" % (self.source, self.destination)
@@ -111,7 +111,7 @@ class ShareConfig(Base):
     source      = Column(String, primary_key=True)
     destination = Column(String, primary_key=True)
     vo          = Column(String, primary_key=True)
-    active      = Column(Integer)
+    share       = Column(Integer, name='active')
 
     __table_args__ = (ForeignKeyConstraint(['source', 'destination'],
                                            [LinkConfig.source,

src/fts3/model/server.py

+#   Copyright notice:
+#   Copyright CERN, 2014.
+#
+#   See www.eu-emi.eu for details on the copyright holders
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from sqlalchemy import Column, DateTime, Integer, String
+
+from base import Base
+
+
+class Host(Base):
+    __tablename__ = 't_hosts'
+
+    hostname     = Column(String(64), primary_key=True)
+    service_name = Column(String(64), primary_key=True)
+    beat         = Column(DateTime)
+    drain        = Column(Integer)

src/fts3rest/fts3rest/CMakeLists.txt

@@ -32,3 +32,12 @@ install (DIRECTORY templates
 install (DIRECTORY public
          DESTINATION ${PYTHON_SITE_PACKAGES}/fts3rest
 )
+
+# Minify Javascript
+find_program (SLIMIT slimit)
+if (SLIMIT)
+    message (STATUS "Found slimit: ${SLIMIT}")
+    install(SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/MinifyJS.cmake"
+        CODE "MINIFY(\"${SLIMIT}\" \"\$ENV{DESTDIR}/${PYTHON_SITE_PACKAGES}/fts3rest/public/js\")"
+    )
+endif (SLIMIT)

src/fts3rest/fts3rest/MinifyJS.cmake

+function (Minify SLIMIT JSROOT)
+
+    message (STATUS "Trying to minimize files under ${JSROOT}")
+
+    file (GLOB_RECURSE JSFILES
+        "${JSROOT}/*.js"
+    )
+
+    foreach (JS ${JSFILES})
+        message ("Minimizing ${JS}")
+        execute_process (
+            COMMAND "${SLIMIT}" -mt "${JS}"
+            OUTPUT_VARIABLE MINJS
+        )
+        file(WRITE "${JS}" "${MINJS}")
+    endforeach (JS)
+
+endfunction (Minify)

src/fts3rest/fts3rest/config/routing/base.py

@@ -84,6 +84,14 @@ def do_connect(config, map):
     map.connect('/api-docs/{resource}', controller='api', action='resource_doc',
                 conditions=dict(method=['GET']))
 
+    # Config entry point
+    map.connect('/config', controller='config', action='index',
+                conditions=dict(method=['GET']))
+
+    # Set/unset draining mode
+    map.connect('/config/drain', controller='config', action='set_drain',
+                conditions=dict(method=['POST']))
+
     # Configuration audit
     map.connect('/config/audit', controller='config', action='audit',
                 conditions=dict(method=['GET']))
@@ -101,12 +109,8 @@ def do_connect(config, map):
                 conditions=dict(method=['POST']))
     map.connect('/config/global', controller='config', action='get_global_config',
                 conditions=dict(method=['GET']))
-
-    # Optimizer mode
-    map.connect('/config/optimizer_mode', controller='config', action='set_optimizer_mode',
-                conditions=dict(method='POST'))
-    map.connect('/config/optimizer_mode', controller='config', action='get_optimizer_mode',
-                conditions=dict(method='GET'))
+    map.connect('/config/global', controller='config', action='delete_vo_global_config',
+                conditions=dict(method=['DELETE']))
 
     # Groups and group members
     map.connect('/config/groups', controller='config', action='add_to_group',
@@ -128,6 +132,14 @@ def do_connect(config, map):
     map.connect('/config/links/{sym_name}', controller='config', action='delete_link_config',
                 conditions=dict(method=['DELETE']))
 
+    # Shares
+    map.connect('/config/shares', controller='config', action='set_share',
+                conditions=dict(method=['POST']))
+    map.connect('/config/shares', controller='config', action='get_shares',
+                conditions=dict(method=['GET']))
+    map.connect('/config/shares', controller='config', action='delete_share',
+                conditions=dict(method=['DELETE']))
+
     # Fixed number of actives
     map.connect('/config/fixed', controller='config', action='fix_active',
                 conditions=dict(method=['POST']))
@@ -139,6 +151,8 @@ def do_connect(config, map):
                 conditions=dict(method=['POST']))
     map.connect('/config/se', controller='config', action='get_se_config',
                 conditions=dict(method=['GET']))
+    map.connect('/config/se', controller='config', action='delete_se_config',
+                conditions=dict(method=['DELETE']))
 
     # Grant special permissions to given DNs
     map.connect('/config/authorize', controller='config', action='add_authz',
@@ -186,3 +200,17 @@ def do_connect(config, map):
                 conditions=dict(method=['DELETE']))
     map.connect('/ban/dn', controller='banning', action='list_banned_dn',
                 conditions=dict(method=['GET']))
+
+    # Autocomplete
+    map.connect('/autocomplete/dn', controller='autocomplete', action='autocomplete_dn',
+                conditions=dict(method=['GET']))
+    map.connect('/autocomplete/source', controller='autocomplete', action='autocomplete_source',
+                conditions=dict(method=['GET']))
+    map.connect('/autocomplete/destination', controller='autocomplete', action='autocomplete_destination',
+                conditions=dict(method=['GET']))
+    map.connect('/autocomplete/storage', controller='autocomplete', action='autocomplete_storage',
+                conditions=dict(method=['GET']))
+    map.connect('/autocomplete/vo', controller='autocomplete', action='autocomplete_vo',
+                conditions=dict(method=['GET']))
+    map.connect('/autocomplete/groupname', controller='autocomplete', action='autocomplete_groupname',
+                conditions=dict(method=['GET']))

src/fts3rest/fts3rest/controllers/CSInterface.py

@@ -29,7 +29,6 @@ from CSdropbox import DropboxConnector
 class CSInterface(object):
 
     def __init__(self, user_dn, service):
-
        # try:
             # Dynamic load of the class required for this External Storage
             #module = __import__("CS" + service.strip().lower())

src/fts3rest/fts3rest/controllers/autocomplete.py

+#   Copyright notice:
+#   Copyright CERN, 2015.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from pylons import request
+
+from fts3.model import Credential, OptimizerActive, Job, Group
+from fts3rest.lib.api import doc
+from fts3rest.lib.base import BaseController, Session
+from fts3rest.lib.helpers import jsonify
+from fts3rest.lib.middleware.fts3auth import authorize
+from fts3rest.lib.middleware.fts3auth.constants import *
+
+
+class AutocompleteController(BaseController):
+    """
+    Autocomplete API
+    """
+
+    @doc.query_arg('term', 'Beginning of the DN')
+    @authorize(CONFIG)
+    @jsonify
+    def autocomplete_dn(self):
+        """
+    	Autocomplete for users' dn
+    	"""
+        term = request.params.get('term', '/DC=cern.ch')
+        matches = Session.query(Credential.dn).filter(Credential.dn.startswith(term)).distinct().all()
+        return map(lambda r: r[0], matches)
+
+    @doc.query_arg('term', 'Beginning of the source storage')
+    @authorize(CONFIG)
+    @jsonify
+    def autocomplete_source(self):
+        """
+        Autocomplete source SE
+        """
+        term = request.params.get('term', 'srm://')
+        matches = Session.query(OptimizerActive.source_se)\
+            .filter(OptimizerActive.source_se.startswith(term)).distinct().all()
+        return map(lambda r: r[0], matches)
+
+    @doc.query_arg('term', 'Beginning of the destination storage')
+    @authorize(CONFIG)
+    @jsonify
+    def autocomplete_destination(self):
+        """
+        Autocomplete destination SE
+        """
+        term = request.params.get('term', 'srm://')
+        matches = Session.query(OptimizerActive.dest_se)\
+            .filter(OptimizerActive.dest_se.startswith(term)).distinct().all()
+        return map(lambda r: r[0], matches)
+
+    @doc.query_arg('term', 'Beginning of the destination storage')
+    @authorize(CONFIG)
+    @jsonify
+    def autocomplete_storage(self):
+        """
+        Autocomplete a storage, regardless of it being source or destination
+        """
+        term = request.params.get('term', 'srm://')
+        src_matches = Session.query(OptimizerActive.source_se)\
+            .filter(OptimizerActive.source_se.startswith(term)).distinct().all()
+        dest_matches = Session.query(OptimizerActive.dest_se)\
+            .filter(OptimizerActive.dest_se.startswith(term)).distinct().all()
+
+        srcs = map(lambda r: r[0], src_matches)
+        dsts = map(lambda r: r[0], dest_matches)
+
+        return set(srcs).union(set(dsts))
+
+    @doc.query_arg('term', 'Beginning of the VO')
+    @authorize(CONFIG)
+    @jsonify
+    def autocomplete_vo(self):
+        """
+        Autocomplete VO
+        """
+        term = request.params.get('term', 'srm://')
+        matches = Session.query(Job.vo_name)\
+            .filter(Job.vo_name.startswith(term)).distinct().all()
+        return map(lambda r: r[0], matches)
+
+    @doc.query_arg('term', 'Beginning of the group name')
+    @authorize(CONFIG)
+    @jsonify
+    def autocomplete_groupname(self):
+        """
+        Autocomplete group names
+        """
+        term = request.params.get('term', 'group')
+        matches = Session.query(Group.groupname)\
+            .filter(Group.groupname.startswith(term)).distinct().all()
+        return map(lambda r: r[0], matches)

src/fts3rest/fts3rest/lib/helpers/__init__.py

 #   Copyright notice:
 #   Copyright  Members of the EMI Collaboration, 2013.
-# 
+#
 #   See www.eu-emi.eu for details on the copyright holders
-# 
+#
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
-# 
+#
 #       http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 #   Unless required by applicable law or agreed to in writing, software
 #   distributed under the License is distributed on an "AS IS" BASIS,
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+from accept import *
 from jsonify import *

src/fts3rest/fts3rest/lib/helpers/accept.py

+#   Copyright notice:
+#   Copyright CERN, 2015.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import pylons
+
+from decorator import decorator
+from fts3rest.lib.http_exceptions import *
+from fts3rest.lib.helpers.jsonify import to_json
+from pylons.templating import render_mako as render
+from pylons.controllers.util import redirect
+
+
+def accept(html_template=None, html_redirect=None, json=True):
+    """
+    Depending on the Accept headers returns a different representation of the data
+    returned by the decorated method
+    """
+    assert((html_template and not html_redirect) or (not html_template and html_redirect))
+
+    offers = ['text/html']
+    if json:
+        offers.append('application/json')
+
+    @decorator
+    def accept_inner(f, *args, **kwargs):
+        try:
+            best_match = pylons.request.accept.best_match(offers, default_match='application/json')
+        except:
+            best_match = 'application/json'
+
+        if not best_match:
+            raise HTTPNotAcceptable('Available: %s' % ', '.join(offers))
+
+        data = f(*args, **kwargs)
+        if best_match == 'text/html':
+            if html_template:
+                return render(html_template, extra_vars={
+                    'data': data, 'config': pylons.config, 'user': pylons.request.environ['fts3.User.Credentials']
+                })
+            else:
+                return redirect(html_redirect, code=HTTPSeeOther.code)
+        else:
+            pylons.response.headers['Content-Type'] = 'application/json'
+            return to_json(data)
+
+    return accept_inner

src/fts3rest/fts3rest/public/css/style.css

+.monospace {
+	font-family: monospace;
+}
+
+
+.table>tbody>tr>td {
+	vertical-align: middle;
+}
+
+
+.table-striped-tbody>tbody:nth-child(odd) {
+	background-color: #DDD;
+}
+
+
+.table-striped-tbody>tbody:hover {
+	background-color: #CCC;	
+}
+
+
+.panel-collapse>.panel-body {
+    display: none;
+}

src/fts3rest/fts3rest/public/js/config/authz.js

+/*
+ *  Copyright 2015 CERN
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+**/
+
+/**
+ * Queries from the server the list of authorized dns and
+ * refresh the display
+ */
+function refreshAuthzList()
+{
+    var tbody = $("#authz-list");
+
+    $.ajax({
+        url: "/config/authorize?"
+    })
+    .done(function (data) {
+        tbody.empty();
+        $.each(data, function(i, user) {
+            var tr = $("<tr></tr>");
+
+            var deleteBtn = $("<button class='btn btn-link'></button>")
+                .append("<i class='glyphicon glyphicon-trash'></i>");
+
+            deleteBtn.click(function() {
+                tr.css("background", "#d9534f");
+                $.ajax({
+                    url: "/config/authorize?dn=" + encodeURIComponent(user.dn) + "&operation=" + encodeURIComponent(user.operation),
+                    type: "DELETE"
+                })
+                .done(function(data, textStatus, jqXHR) {
+                    tr.fadeOut(300, function() {tr.remove();})
+                })
+                .fail(function(jqXHR) {
+                    errorMessage(jqXHR);
+                    tr.css("background", "#ffffff");
+                });
+            });
+
+            tbody.append(
+                tr.append($("<td></td>").append(deleteBtn))
+                  .append($("<td></td>").append($("<span class='monospace'></span>").text(user.dn)))
+                  .append($("<td></td>").text(user.operation))
+            );
+        });
+    })
+    .fail(function(jqXHR) {
+        errorMessage(jqXHR);
+    });
+}
+
+
+/**
+ * Initialize the authz view
+ */
+function setupAuthz()
+{
+    // Load list
+    refreshAuthzList();
+
+    // Attach to the form
+    $("#authz-add-frm").submit(function(event) {
+        $.ajax({
+            url: "/config/authorize?",
+            type: "POST",
+            dataType: "json",
+            data: $(this).serialize()
+        })
+        .done(function(data, textStatus, jqXHR) {
+            refreshAuthzList();
+            $("#authz-add-frm").trigger("reset");
+        })
+        .fail(function(jqXHR) {
+            errorMessage(jqXHR);
+        })
+        .always(function() {
+            $("#auth-add-frm-submit > i").attr("class", "glyphicon glyphicon-plus");
+        });
+
+        $("#auth-add-frm-submit > i").attr("class", "glyphicon glyphicon-refresh");
+
+        event.preventDefault();
+    });
+
+    // Autocomplete
+    $("#authz-add-field-dn").autocomplete({
+        source: "/autocomplete/dn"
+    });
+}

src/fts3rest/fts3rest/public/js/config/common.js

+/*
+ *  Copyright 2015 CERN
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+**/
+
+
+/**
+ * Display an error
+ */
+function errorMessage(jqXHR)
+{
+    var msg;
+    if (jqXHR.responseJSON && jqXHR.responseJSON.message)
+        msg = jqXHR.responseJSON.message;
+    else
+        msg = "The server didn't return an error message: " + jqXHR.textStatus;
+