FTS-117: OAuth2 provider

dist/fts-rest.spec

@@ -21,6 +21,7 @@ BuildRequires:  scipy
 BuildRequires:  m2crypto
 BuildRequires:  python-coverage
 BuildRequires:  python-sqlalchemy
+BuildRequires:  python-requests
 BuildRequires:  pandoc
 
 Requires:     gridsite%{?_isa} >= 1.7

docs/api.md

@@ -2,6 +2,104 @@ API
 ===
 This document has been generated automatically
 
+### /cs
+### /oauth2
+#### GET /oauth2/token
+Get an access token
+
+#### POST /oauth2/token
+Get an access token
+
+#### GET /oauth2/apps
+Returns the list of registered apps
+
+##### Returns
+Array of [OAuth2Application](#oauth2application)
+
+#### GET /oauth2/register
+Registration form
+
+#### POST /oauth2/register
+Register a new third party application
+
+##### Returns
+client_id
+
+##### Responses
+
+|Code|Description                                                     |
+|----|----------------------------------------------------------------|
+|403 |Tried to update an application that does not belong to the user |
+|400 |Bad request                                                     |
+|303 |Application registered, follow redirection (when html requested)|
+|201 |Application registered                                          |
+
+#### GET /oauth2/authorize
+Perform OAuth2 authorization step
+
+#### POST /oauth2/authorize
+Triggered by user action. Confirm, or reject, access.
+
+#### GET /oauth2/apps/{client_id}
+Return information about a given app
+
+##### Returns
+[OAuth2Application](#oauth2application)
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
+##### Responses
+
+|Code|Description                                |
+|----|-------------------------------------------|
+|404 |Application not found                      |
+|403 |The application does not belong to the user|
+
+#### POST /oauth2/apps/{client_id}
+Update an application
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
+##### Responses
+
+|Code|Description                                |
+|----|-------------------------------------------|
+|404 |Application not found                      |
+|403 |The application does not belong to the user|
+
+#### DELETE /oauth2/apps/{client_id}
+Delete an application from the database
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
+##### Responses
+
+|Code|Description                                |
+|----|-------------------------------------------|
+|404 |Application not found                      |
+|403 |The application does not belong to the user|
+
+#### GET /oauth2/revoke/{client_id}
+Current user revokes all tokens for a given application
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
 ### /whoami
 #### GET /whoami
 Returns the active credentials of the user
@@ -150,6 +248,103 @@ Stat a remote file
 |403 |Permission denied                                    |
 |400 |Protocol not supported OR the SURL is not a directory|
 
+### OAuth2.0 controller
+#### GET /oauth2/token
+Get an access token
+
+#### POST /oauth2/token
+Get an access token
+
+#### GET /oauth2/apps
+Returns the list of registered apps
+
+##### Returns
+Array of [OAuth2Application](#oauth2application)
+
+#### GET /oauth2/register
+Registration form
+
+#### POST /oauth2/register
+Register a new third party application
+
+##### Returns
+client_id
+
+##### Responses
+
+|Code|Description                                                     |
+|----|----------------------------------------------------------------|
+|403 |Tried to update an application that does not belong to the user |
+|400 |Bad request                                                     |
+|303 |Application registered, follow redirection (when html requested)|
+|201 |Application registered                                          |
+
+#### GET /oauth2/authorize
+Perform OAuth2 authorization step
+
+#### POST /oauth2/authorize
+Triggered by user action. Confirm, or reject, access.
+
+#### GET /oauth2/apps/{client_id}
+Return information about a given app
+
+##### Returns
+[OAuth2Application](#oauth2application)
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
+##### Responses
+
+|Code|Description                                |
+|----|-------------------------------------------|
+|404 |Application not found                      |
+|403 |The application does not belong to the user|
+
+#### POST /oauth2/apps/{client_id}
+Update an application
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
+##### Responses
+
+|Code|Description                                |
+|----|-------------------------------------------|
+|404 |Application not found                      |
+|403 |The application does not belong to the user|
+
+#### DELETE /oauth2/apps/{client_id}
+Delete an application from the database
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
+##### Responses
+
+|Code|Description                                |
+|----|-------------------------------------------|
+|404 |Application not found                      |
+|403 |The application does not belong to the user|
+
+#### GET /oauth2/revoke/{client_id}
+Current user revokes all tokens for a given application
+
+##### Path arguments
+
+|Name     |Type  |
+|---------|------|
+|client_id|string|
+
 ### Operations on archived jobs and transfers
 #### GET /archive/{job_id}/{field}
 Get a specific field from the job identified by id
@@ -282,6 +477,9 @@ Submission description (SubmitSchema)
 |403 |The user doesn't have enough permissions to submit|
 |400 |The submission request could not be understood    |
 
+#### OPTIONS /jobs
+Answer the OPTIONS method over /jobs
+
 #### GET /jobs/{job_id}/files
 Get the files within a job
 
@@ -342,6 +540,15 @@ Returns the canceled job with its current status. CANCELED if it was canceled,<b
 |413 |The user doesn't have enough privileges|
 |404 |The job doesn't exist                  |
 
+#### OPTIONS /jobs/{job_id}
+Answers the OPTIONS method over /jobs/job-id
+
+##### Path arguments
+
+|Name  |Type  |
+|------|------|
+|job_id|string|
+
 #### GET /jobs/{job_id}/{field}
 Get a specific field from the job identified by id
 
@@ -570,6 +777,15 @@ Models
 |agent_dn            |string  |
 |reason_class        |string  |
 
+### FileRetryLog
+
+|Field   |Type    |
+|--------|--------|
+|reason  |string  |
+|attempt |integer |
+|file_id |integer |
+|datetime|dateTime|
+
 ### OptimizerEvolution
 
 |Field     |Type    |
@@ -584,14 +800,17 @@ Models
 |active    |integer |
 |source_se |string  |
 
-### FileRetryLog
-
-|Field   |Type    |
-|--------|--------|
-|reason  |string  |
-|attempt |integer |
-|file_id |integer |
-|datetime|dateTime|
+### OAuth2Application
+
+|Field        |Type  |
+|-------------|------|
+|website      |string|
+|name         |string|
+|redirect_to  |string|
+|client_id    |string|
+|owner        |string|
+|client_secret|string|
+|description  |string|
 
 ### ArchivedJob
 

docs/generate-api-md.py

@@ -2,15 +2,15 @@
 
 #   Copyright notice:
 #   Copyright  Members of the EMI Collaboration, 2013.
-# 
+#
 #   See www.eu-emi.eu for details on the copyright holders
-# 
+#
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
-# 
+#
 #       http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 #   Unless required by applicable law or agreed to in writing, software
 #   distributed under the License is distributed on an "AS IS" BASIS,
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -31,7 +31,7 @@ def _add_to_syspath(base_dir, components):
 _base_dir = os.path.dirname(__file__)
 _add_to_syspath(_base_dir, ('..', 'src'))
 _add_to_syspath(_base_dir, ('..', 'src', 'fts3rest'))
-
+_add_to_syspath(_base_dir, ('..', 'src', 'fts3rest', 'fts3rest', 'controllers'))
 
 from fts3rest.lib import api
 
@@ -117,59 +117,60 @@ def write_markdown(resources, apis, models, md):
         if not description:
             description = path
         md.h3(description)
-        for call in apis[path]:
-            path = call['path']
-            for operation in call['operations']:
-                md.h4("%s %s" % (operation['method'], path))
-                md.paragraph(operation['summary'])
-
-                type = operation.get('type', None)
-                if type == 'array':
-                    item_type = operation['items']['$ref']
-                    if item_type in available_models:
-                        type = 'Array of ' + md.href_to_header(item_type)
-                    else:
-                        type = 'Array of ' + item_type
-                elif type and type in available_models:
-                        type = md.href_to_header(type)
-
-                if type:
-                    md.h5('Returns')
-                    md.paragraph(type)
-
-                if operation['notes']:
-                    md.h5('Notes')
-                    md.paragraph(operation['notes'])
-
-                parameters = operation['parameters']
-                responses = operation['responseMessages']
-
-                query_args = filter(lambda a: a['paramType'] == 'query', parameters)
-                path_args = filter(lambda a: a['paramType'] == 'path', parameters)
-                body_args = filter(lambda a: a['paramType'] == 'body', parameters)
-
-                if path_args:
-                    md.h5('Path arguments')
-                    md.table(
-                        ('Name', 'Type'),
-                        map(lambda a: (a['name'], a['type']), path_args)
-                    )
-                if query_args:
-                    md.h5('Query arguments')
-                    md.table(
-                        ('Name', 'Type', 'Required', 'Description'),
-                        map(lambda a: (a['name'], a['type'], a['required'], a['description']), query_args)
-                    )
-                if body_args:
-                    md.h5('Expected request body')
-                    md.paragraph("%s (%s)" % (body_args[0]['description'], body_args[0]['type']))
-
-                if responses:
-                    md.h5('Responses')
-                    md.table(
-                        ('Code', 'Description'),
-                        map(lambda r: (r['code'], r['message']), responses)
-                    )
+        if path in apis:
+            for call in apis[path]:
+                path = call['path']
+                for operation in call['operations']:
+                    md.h4("%s %s" % (operation['method'], path))
+                    md.paragraph(operation['summary'])
+
+                    type = operation.get('type', None)
+                    if type == 'array':
+                        item_type = operation['items']['$ref']
+                        if item_type in available_models:
+                            type = 'Array of ' + md.href_to_header(item_type)
+                        else:
+                            type = 'Array of ' + item_type
+                    elif type and type in available_models:
+                            type = md.href_to_header(type)
+
+                    if type:
+                        md.h5('Returns')
+                        md.paragraph(type)
+
+                    if operation['notes']:
+                        md.h5('Notes')
+                        md.paragraph(operation['notes'])
+
+                    parameters = operation['parameters']
+                    responses = operation['responseMessages']
+
+                    query_args = filter(lambda a: a['paramType'] == 'query', parameters)
+                    path_args = filter(lambda a: a['paramType'] == 'path', parameters)
+                    body_args = filter(lambda a: a['paramType'] == 'body', parameters)
+
+                    if path_args:
+                        md.h5('Path arguments')
+                        md.table(
+                            ('Name', 'Type'),
+                            map(lambda a: (a['name'], a['type']), path_args)
+                        )
+                    if query_args:
+                        md.h5('Query arguments')
+                        md.table(
+                            ('Name', 'Type', 'Required', 'Description'),
+                            map(lambda a: (a['name'], a['type'], a['required'], a['description']), query_args)
+                        )
+                    if body_args:
+                        md.h5('Expected request body')
+                        md.paragraph("%s (%s)" % (body_args[0]['description'], body_args[0]['type']))
+
+                    if responses:
+                        md.h5('Responses')
+                        md.table(
+                            ('Code', 'Description'),
+                            map(lambda r: (r['code'], r['message']), responses)
+                        )
     md.h2('Models')
     printed_models = []
     for model, model_desc in model_dict.iteritems():

src/fts3/model/__init__.py

@@ -23,6 +23,7 @@ from config import *
 from credentials import *
 from file import *
 from job import *
+from oauth2 import *
 from optimizer import *
 from version import *
 

src/fts3/model/oauth2.py

+#   Copyright notice:
+#   Copyright CERN, 2014.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from base import Base
+from sqlalchemy import Column, String, DateTime, ForeignKey
+
+
+class OAuth2Application(Base):
+    __tablename__ = 't_oauth2_apps'
+
+    client_id     = Column(String(64), nullable=False, primary_key=True)
+    client_secret = Column(String(128), nullable=False)
+    owner         = Column(String(1024), nullable=False)
+
+    name          = Column(String(128), nullable=False, unique=True)
+    description   = Column(String(512))
+    website       = Column(String(1024))
+    redirect_to   = Column(String(4096))
+
+
+class OAuth2Code(Base):
+    __tablename__ = 't_oauth2_codes'
+
+    client_id = Column(String(64), nullable=False)
+    code      = Column(String(128), nullable=False, primary_key=True)
+    scope     = Column(String(512), nullable=True)
+    dlg_id    = Column(String(100), nullable=False)
+
+
+class OAuth2Token(Base):
+    __tablename__ = 't_oauth2_tokens'
+
+    client_id     = Column(String(64), ForeignKey(OAuth2Application.client_id), nullable=False, primary_key=True)
+    dlg_id        = Column(String(100), nullable=False)
+    scope         = Column(String(512), nullable=True)
+    access_token  = Column(String(128), nullable=False)
+    token_type    = Column(String(64), nullable=False)
+    expires       = Column(DateTime, nullable=False)
+    refresh_token = Column(String(128), nullable=False, primary_key=True)

src/fts3rest/fts3rest.conf

@@ -45,6 +45,9 @@ Listen 8446
   # Send everything to the FTS3 REST interface
   WSGIScriptAlias / /usr/libexec/fts3/fts3rest.wsgi
 
+  # For OAuth2 support, the Authentication header needs to be passed
+  WSGIPassAuthorization On
+
   # We need to avoid the underlying libraries (i.e. Globus) from modifying the status
   # of the server process (httpd), so isolate the application
   WSGIDaemonProcess fts3rest processes=2 threads=15

src/fts3rest/fts3rest.ini

@@ -13,7 +13,7 @@ port = 5000
 [app:main]
 use = egg:fts3rest
 full_stack = true
-static_files = false
+static_files = true
 
 cache_dir = /var/cache/fts3rest/data
 beaker.session.key = fts3rest

src/fts3rest/fts3rest/CMakeLists.txt

@@ -24,3 +24,11 @@ install (DIRECTORY   model
          DESTINATION ${PYTHON_SITE_PACKAGES}/fts3rest
          PATTERN     "*.pyc" EXCLUDE
 )
+
+install (DIRECTORY templates
+         DESTINATION ${PYTHON_SITE_PACKAGES}/fts3rest
+)
+
+install (DIRECTORY public
+         DESTINATION ${PYTHON_SITE_PACKAGES}/fts3rest
+)

src/fts3rest/fts3rest/config/environment.py

@@ -18,6 +18,7 @@
 """Pylons environment configuration"""
 import os
 
+from mako.lookup import TemplateLookup
 from pylons import config
 from sqlalchemy import engine_from_config
 
@@ -63,6 +64,11 @@ def load_environment(global_conf, app_conf):
     engine = engine_from_config(config, 'sqlalchemy.', pool_recycle = 7200)
     init_model(engine)
 
+    # Mako templating
+    config['pylons.app_globals'].mako_lookup = TemplateLookup(
+        directories=paths['templates'],
+    )
+
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)
     return config

src/fts3rest/fts3rest/config/routing.py

@@ -129,5 +129,19 @@ def make_map(config):
                 conditions=dict(method=['GET']))
     map.connect('/cs/file_urllink/{service}/{path}', controller='cloudStorage', action='getCSFileLink',
                 conditions=dict(method=['GET']))
-    
+
+    # OAuth 2.0
+    map.redirect('/oauth2', '/oauth2/apps')
+    map.connect('/oauth2/apps', controller='oauth2', action='get_my_apps', conditions=dict(method=['GET']))
+    map.connect('/oauth2/register', controller='oauth2', action='register_form', conditions=dict(method=['GET']))
+    map.connect('/oauth2/register', controller='oauth2', action='register', conditions=dict(method=['POST']))
+    map.connect('/oauth2/apps/{client_id}', controller='oauth2', action='get_app', conditions=dict(method=['GET']))
+    map.connect('/oauth2/apps/{client_id}', controller='oauth2', action='update_app', conditions=dict(method=['POST']))
+    map.connect('/oauth2/apps/{client_id}', controller='oauth2', action='delete_app', conditions=dict(method=['DELETE']))
+
+    map.connect('/oauth2/authorize', controller='oauth2', action='authorize', conditions=dict(method=['GET']))
+    map.connect('/oauth2/authorize', controller='oauth2', action='confirm', conditions=dict(method=['POST']))
+    map.connect('/oauth2/token', controller='oauth2', action='get_token', conditions=dict(method=['GET', 'POST']))
+    map.connect('/oauth2/revoke/{client_id}', controller='oauth2', action='revoke_token', conditions=dict(method=['GET']))
+
     return map

src/fts3rest/fts3rest/controllers/delegation.py

@@ -15,19 +15,23 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+import json
+import logging
+import types
+
 from datetime import datetime
 from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPNotFound
+from M2Crypto import X509, RSA, EVP, BIO
+from pylons import request
+from pylons.templating import render_mako as render
+
 from fts3.model import CredentialCache, Credential
 from fts3rest.lib.api import doc
 from fts3rest.lib.base import BaseController, Session
 from fts3rest.lib.helpers import jsonify
 from fts3rest.lib.helpers import voms
 from fts3rest.lib.http_exceptions import HTTPMethodFailure
-from M2Crypto import X509, RSA, EVP, BIO
-from pylons import request
-import json
-import logging
-import types
+
 
 log = logging.getLogger(__name__)
 
@@ -217,14 +221,14 @@ class DelegationController(BaseController):
         credential_cache = Session.query(CredentialCache)\
             .get((user.delegation_id, user.user_dn))
 
-        if credential_cache is None:
+        if credential_cache is None or credential_cache.cert_request is None:
             (x509_request, private_key) = _generate_proxy_request()
             credential_cache = CredentialCache(dlg_id=user.delegation_id, dn=user.user_dn,
                                                cert_request=x509_request.as_pem(),
                                                priv_key=private_key.as_pem(cipher=None),
                                                voms_attrs=' '.join(user.voms_cred))
             try:
-                Session.add(credential_cache)
+                Session.merge(credential_cache)
                 Session.commit()
             except Exception:
                 Session.rollback()
@@ -235,7 +239,7 @@ class DelegationController(BaseController):
 
         start_response('200 Ok', [('X-Delegation-ID', credential_cache.dlg_id),
                                   ('Content-Type', 'text/plain')])
-        return credential_cache.cert_request
+        return [credential_cache.cert_request]
 
     @doc.input('Signed certificate', 'PEM encoded certificate')
     @doc.response(403, 'The requested delegation ID does not belong to the user')
@@ -337,3 +341,10 @@ class DelegationController(BaseController):
 
         start_response('203 Non-Authoritative Information', [('Content-Type', 'text/plain')])
         return [str(new_termination_time)]
+
+    def delegation_page(self):
+        """
+        Render an HTML form to delegate the credentials
+        """
+        user = request.environ['fts3.User.Credentials']