Commit acedc9e0 authored by Pablo Panero's avatar Pablo Panero
Browse files

Merge branch 'dev' into 'master'

Release v0.5.3

See merge request webservices/cern-search/cern-search-rest-api!45
parents c93bb525 f224f770
Changes
=======
Version 0.5.3 (released 2018-11-13)
Features:
- Add an endpoint to perform ``Update By Query`` actions over single documents.
- Add ``document_v1.0.0`` schema for EDMS instance.
- Add ``Health`` blueprint with three possible endpoints (uWSGI, Elasticsearch and database).
- Make ``access`` parameter optional in requests.
- Make optional the use of CERN e-groups for permissions.
Fixes:
- Document creation should just check if the user is authenticated in the first iteration. Permissions over the schema are checked on the second iteration.
\ No newline at end of file
......@@ -3,6 +3,7 @@
from __future__ import absolute_import, print_function
import ast
import copy
import os
from invenio_oauthclient.contrib import cern
......@@ -70,6 +71,7 @@ INDEXER_DEFAULT_INDEX = os.getenv('CERN_SEARCH_DEFAULT_INDEX', 'cernsearch-test-
# =====================
SEARCH_MAPPINGS = [os.getenv('CERN_SEARCH_INSTANCE', 'cernsearch-test')]
SEARCH_USE_EGROUPS = ast.literal_eval(os.getenv('CERN_SEARCH_USE_EGROUPS', 'True'))
# Records REST configuration
# ===========================
......@@ -112,6 +114,8 @@ RECORDS_REST_ENDPOINTS = dict(
# ===
RATELIMIT_DEFAULT = os.getenv('CERN_SEARCH_INSTANCE_RATELIMIT', '5000/hour')
APP_HEALTH_BLUEPRINT_ENABLED = True
APP_HEALTH_BLUEPRINT = 'cern_search_rest_api.modules.cernsearch.views:build_health_blueprint'
# Flask Security
# ==============
......
{
"title": "Custom record schema for collection v0.0.1",
"id": "http://localhost:5000/schemas/cernsearch-test/api_ubq_v0.0.1.json",
"$schema": "http://localhost:5000/schemas/cernsearch-test/api_ubq_v0.0.1.json",
"type": "object",
"properties": {
"_access": {
"type": "object",
"properties": {
"owner":{
"type": "array",
"items": {
"type": "string"
}
},
"read":{
"type": "array",
"items": {
"type": "string"
}
},
"update":{
"type": "array",
"items": {
"type": "string"
}
},
"delete":{
"type": "array",
"items": {
"type": "string"
}
}
}
},
"links":{
"type": "array",
"items": {
"type": "object"
}
},
"$schema": {
"type": "string"
}
}
}
\ No newline at end of file
{
"settings": {
"index.percolator.map_unmapped_fields_as_string": true,
"index.mapping.total_fields.limit": 3000
},
"mappings": {
"api_ubq_v0.0.1": {
"numeric_detection": true,
"_meta": {
"_owner": "CernSearch-Administrators@cern.ch"
},
"_all": {
"analyzer": "english"
},
"properties": {
"_access": {
"type": "nested",
"properties": {
"owner":{
"type": "keyword"
},
"read": {
"type": "keyword"
},
"update": {
"type": "keyword"
},
"delete": {
"type": "keyword"
}
}
},
"links":{
"type":"nested"
},
"custom_pid": {
"type": "string",
"index": "not_analyzed"
},
"$schema": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
\ No newline at end of file
{
"title": "EDMS Object Type Document schema v1.0.0",
"id": "http://0.0.0.0:5000/schemas/cernsearch-edms/document_v1.0.0.json",
"$schema": "http://0.0.0.0:5000/schemas/cernsearch-edms/document_v1.0.0.json",
"type": "object",
"properties": {
"_access": {
"type": "object",
"properties": {
"owner":{
"type": "array",
"items": {
"type": "string"
}
},
"read":{
"type": "array",
"items": {
"type": "string"
}
},
"update":{
"type": "array",
"items": {
"type": "string"
}
},
"delete":{
"type": "array",
"items": {
"type": "string"
}
}
}
},
"object_type": {
"type": "string"
},
"cid": {
"type": "integer"
},
"id": {
"type": "integer"
},
"cern_id": {
"type": "string"
},
"version": {
"type": "string"
},
"latest_version": {
"type": "boolean"
},
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"creation_date": {
"type": "string"
},
"modification_date": {
"type": "string"
},
"context": {
"type": "string"
},
"document_type": {
"type": "string"
},
"release_procedure": {
"type": "string"
},
"status": {
"type": "string"
},
"obsolete": {
"type": "boolean"
},
"author": {
"type": "string"
},
"keywords": {
"type": "string"
},
"equipment_code": {
"type": "string"
},
"attributes": {
"type": "array",
"items": {
"type": "string"
}
},
"properties": {
"type": "array",
"items": {
"type": "object"
}
},
"parents": {
"type": "array",
"items": {
"type": "object"
}
},
"custom_pid": {
"type": "string"
},
"$schema": {
"type": "string"
}
}
}
\ No newline at end of file
{
"settings": {
"index.percolator.map_unmapped_fields_as_string": true,
"index.mapping.total_fields.limit": 100
},
"mappings": {
"document_v1.0.0": {
"numeric_detection": true,
"_meta": {
"_owner": "CernSearch-Administrators@cern.ch,en-dep-ace-edm@cern.ch"
},
"properties": {
"_access": {
"type": "nested",
"properties": {
"owner":{
"type": "keyword"
},
"read": {
"type": "keyword"
},
"update": {
"type": "keyword"
},
"delete": {
"type": "keyword"
}
}
},
"object_type": {
"type": "keyword"
},
"cid": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"cern_id": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"latest_version": {
"type": "keyword"
},
"title": {
"type": "text",
"fields": {
"english": {
"type": "text",
"analyzer": "english"
},
"french": {
"type": "text",
"analyzer": "french"
}
}
},
"description": {
"type": "text",
"fields": {
"english": {
"type": "text",
"analyzer": "english"
},
"french": {
"type": "text",
"analyzer": "french"
}
}
},
"creation_date": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss"
},
"modification_date": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss"
},
"context": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"document_type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"release_procedure": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword"
},
"obsolete": {
"type": "keyword"
},
"author": {
"type": "text"
},
"keywords": {
"type": "text"
},
"equipment_code": {
"type": "text"
},
"attributes": {
"type": "keyword"
},
"properties": {
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "text"
}
}
},
"parents": {
"type": "nested",
"properties": {
"type": {
"type": "keyword"
},
"id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
},
"custom_pid": {
"type": "keyword"
},
"$schema": {
"type": "keyword"
}
}
}
}
}
\ No newline at end of file
# -*- coding: utf-8 -*-
from cern_search_rest_api.modules.cernsearch.views import build_blueprint, build_health_blueprint
class CERNSearch(object):
"""CERN Search extension."""
def __init__(self, app=None):
"""Extension initialization."""
if app:
self.init_app(app)
def init_app(self, app):
"""Flask application initialization."""
self.init_config(app)
blueprint = build_blueprint(app)
app.register_blueprint(build_health_blueprint())
app.register_blueprint(blueprint)
app.extensions["cern-search"] = self
def init_config(self, app):
"""Initialize configuration."""
# Set up API endpoints for records.
for k in dir(app.config):
if k.startswith('CERN_SEARCH'):
app.config.setdefault(k, getattr(app.config, k))
\ No newline at end of file
......@@ -5,7 +5,8 @@ from flask_security import current_user
from flask import request, current_app
from invenio_search import current_search_client
from cern_search_rest_api.modules.cernsearch.utils import get_user_provides, cern_search_record_to_index
from cern_search_rest_api.modules.cernsearch.utils import get_user_provides, cern_search_record_to_index, \
get_index_from_request
"""Access control for CERN Search."""
......@@ -85,6 +86,11 @@ class RecordPermission(object):
def has_owner_permission(user, record=None):
"""Check if user is authenticated and has create access"""
log_action(user, 'CREATE/OWNER')
# First authentication phase, decorator level
if not record:
return user.is_authenticated
# Second authentication phase, record level
if user.is_authenticated:
# Allow based in the '_access' key
user_provides = get_user_provides()
......@@ -104,14 +110,6 @@ def has_owner_permission(user, record=None):
return False
def get_index_from_request(record=None):
if record is not None and record.get('$schema', '') is not None:
return cern_search_record_to_index(record)
current_app.logger.debug('get_index_from_schema() No record or no $schema in it, using defaults')
return (current_app.config['INDEXER_DEFAULT_INDEX'],
current_app.config['INDEXER_DEFAULT_DOC_TYPE'])
def has_list_permission(user, record=None):
"""Check if user is authenticated and has create access"""
if user:
......
......@@ -3,7 +3,9 @@
from elasticsearch_dsl import Q
from invenio_search import RecordsSearch
from invenio_search.api import DefaultFilter
from flask import request
from flask import request, current_app
from cern_search_rest_api.modules.cernsearch.utils import get_user_provides
"""
The Filter emulates the following query:
......@@ -61,11 +63,17 @@ def cern_search_filter():
def get_egroups():
egroups = request.args.get('access', None)
try:
return ['{0}@cern.ch'.format(egroup) for egroup in egroups.split(',')]
except AttributeError:
return None
# If access rights are sent or is a search query
if egroups or (request.path == '/records/' and request.method == 'GET'):
try:
if current_app.config['SEARCH_USE_EGROUPS']:
return ['{0}@cern.ch'.format(egroup) for egroup in egroups.split(',')]
else:
return egroups.split(',')
except AttributeError:
return None
# Else use user's token ACLs
return get_user_provides()
class RecordCERNSearch(RecordsSearch):
......
......@@ -14,6 +14,17 @@ def get_user_provides():
return [need.value for need in g.identity.provides]
def get_index_from_request(record=None):
if record is not None and record.get('$schema', '') is not None:
return cern_search_record_to_index(record)
current_app.logger.debug('get_index_from_schema(): Record {record} - $schema {schema}. Using defaults'.format(
record=record,
schema='No record' if record is None else record.get('$schema')
))
return (current_app.config['INDEXER_DEFAULT_INDEX'],
current_app.config['INDEXER_DEFAULT_DOC_TYPE'])
def cern_search_record_to_index(record):
"""Get index/doc_type given a record.
It tries to extract from `record['$schema']` the index and doc_type,
......
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Custom UPDATE REST API for CERN Search to support _update_by_query.
Limitation: The query fails when the _version value (version_id in invenio-records) is 0 (<1).
"""
from __future__ import absolute_import, print_function
import logging
from copy import deepcopy
from functools import partial
from elasticsearch_dsl.query import QueryString
from flask_sqlalchemy import SQLAlchemy
from invenio_db import db
from invenio_records_rest import current_records_rest
from sqlalchemy import MetaData, util
from invenio_records_rest.errors import UnsupportedMediaRESTError, InvalidDataRESTError
from invenio_records_rest.utils import obj_or_import_string
from invenio_records_rest.views import create_error_handlers as records_rest_error_handlers
from flask import Blueprint, Response, json, url_for, request, make_response, current_app
from invenio_records_rest.views import need_record_permission, pass_record
from invenio_rest import ContentNegotiatedMethodView
from invenio_search import current_search_client
from cern_search_rest_api.modules.cernsearch.search import RecordCERNSearch
from cern_search_rest_api.modules.cernsearch.utils import get_index_from_request
def create_error_handlers(blueprint):
"""Create error handlers on blueprint."""
records_rest_error_handlers(blueprint)
def build_url_action_for_pid(pid, action):