Commit 6492b1a8 authored by Noor Afshan Fathima's avatar Noor Afshan Fathima Committed by Carina Antunes
Browse files

[#106] Add asr initial client config

parent e33ce586
......@@ -11,5 +11,4 @@ AUTHORS.md
CONTRIBUTING.md
LICENSE
.logging.yaml
docker
with_env.sh
CERN_SEARCH_INSTANCE=asrservice
INVENIO_INDEXER_DEFAULT_DOC_TYPE=vtt_v1.0.0
INVENIO_INDEXER_DEFAULT_INDEX=asrservice-vtt_v1.0.0
......@@ -9,9 +9,7 @@ variables:
RESOURCE: cern-search-rest-api
### OpenShift namespace and server values
NAMESPACE: cern-search-master
NAMESPACE_DEV: test-cern-search-master
OPENSHIFT_SERVER: https://openshift.cern.ch
OPENSHIFT_SERVER_DEV: https://openshift-dev.cern.ch
OPENSHIFT_SERVER: https://api.paas.okd.cern.ch
### By default, there are 6 stages that we may use:
### Feel free to adapt this to your specific case.
......@@ -145,23 +143,8 @@ tag_image_merge_request_prod:
variables:
CI_REGISTRY_TAG: $CI_COMMIT_REF_SLUG
TAG: $CI_COMMIT_REF_SLUG
TOKEN: ${SERVICE_ACCOUNT_TOKEN_PROD}
TOKEN: ${SA_TOKEN}
### Create tag in OpenShift DEV.
tag_image_merge_request_dev:
extends: .tag_image
only:
refs:
- merge_requests
environment:
name: branch/$CI_COMMIT_REF_SLUG
when: manual
variables:
CI_REGISTRY_TAG: $CI_COMMIT_REF_SLUG
TAG: $CI_COMMIT_REF_SLUG
TOKEN: ${SERVICE_ACCOUNT_TOKEN_DEV}
NAMESPACE: ${NAMESPACE_DEV}
OPENSHIFT_SERVER: ${OPENSHIFT_SERVER_DEV}
# Import image into OpenShift.
import_image_merge_request_prod:
......@@ -174,22 +157,8 @@ import_image_merge_request_prod:
when: manual
variables:
TAG: $CI_COMMIT_REF_SLUG
TOKEN: ${SERVICE_ACCOUNT_TOKEN_PROD}
TOKEN: ${SA_TOKEN}
# Import image into OpenShift DEV.
import_image_merge_request_dev:
extends: .import_image
only:
refs:
- merge_requests
environment:
name: branch/$CI_COMMIT_REF_SLUG
when: manual
variables:
TAG: $CI_COMMIT_REF_SLUG
TOKEN: ${SERVICE_ACCOUNT_TOKEN_DEV}
NAMESPACE: ${NAMESPACE_DEV}
OPENSHIFT_SERVER: ${OPENSHIFT_SERVER_DEV}
######################################################
##################### Dev branch #####################
......@@ -207,20 +176,8 @@ import_image_dev_branch_prod:
name: staging
variables:
TAG: latest
TOKEN: ${SERVICE_ACCOUNT_TOKEN_PROD}
TOKEN: ${SA_TOKEN}
### Import image into OpenShift DEV
import_image_dev_branch_dev:
extends: .import_image
only:
- dev
environment:
name: staging
variables:
TAG: latest
TOKEN: ${SERVICE_ACCOUNT_TOKEN_DEV}
NAMESPACE: ${NAMESPACE_DEV}
OPENSHIFT_SERVER: ${OPENSHIFT_SERVER_DEV}
######################################################
####################### Tags ########################
......@@ -239,19 +196,8 @@ tag_image_prod:
variables:
CI_REGISTRY_TAG: ${CI_COMMIT_TAG}
TAG: ${CI_COMMIT_TAG}
TOKEN: ${SERVICE_ACCOUNT_TOKEN_PROD}
TOKEN: ${SA_TOKEN}
### Create tag in OpenShift DEV.
tag_image_dev:
extends: .tag_image
environment:
name: production
variables:
CI_REGISTRY_TAG: ${CI_COMMIT_TAG}
TAG: ${CI_COMMIT_TAG}
TOKEN: ${SERVICE_ACCOUNT_TOKEN_DEV}
NAMESPACE: ${NAMESPACE_DEV}
OPENSHIFT_SERVER: ${OPENSHIFT_SERVER_DEV}
### Import image into OpenShift.
import_tagged_image_prod:
......@@ -262,20 +208,7 @@ import_tagged_image_prod:
- tags
variables:
TAG: ${CI_COMMIT_TAG}
TOKEN: ${SERVICE_ACCOUNT_TOKEN_PROD}
### Import image into OpenShift DEV.
import_tagged_image_dev:
extends: .import_image
environment:
name: production
only:
- tags
variables:
TAG: ${CI_COMMIT_TAG}
TOKEN: ${SERVICE_ACCOUNT_TOKEN_DEV}
NAMESPACE: ${NAMESPACE_DEV}
OPENSHIFT_SERVER: ${OPENSHIFT_SERVER_DEV}
TOKEN: ${SA_TOKEN}
### Deploy stable into OpenShift.
deploy_production:
......@@ -287,18 +220,4 @@ deploy_production:
variables:
CI_REGISTRY_TAG: ${CI_COMMIT_TAG}
TAG: stable
TOKEN: ${SERVICE_ACCOUNT_TOKEN_PROD}
### Deploy stable into OpenShift DEV.
deploy_dev:
extends: .tag_image
stage: deploy
environment:
name: production
when: manual
variables:
CI_REGISTRY_TAG: ${CI_COMMIT_TAG}
TAG: stable
TOKEN: ${SERVICE_ACCOUNT_TOKEN_DEV}
NAMESPACE: ${NAMESPACE_DEV}
OPENSHIFT_SERVER: ${OPENSHIFT_SERVER_DEV}
TOKEN: ${SA_TOKEN}
[settings]
line_length=120
known_third_party = celery,click,elasticsearch,elasticsearch_dsl,flask,flask_login,flask_security,invenio_accounts,invenio_app,invenio_db,invenio_files_processor,invenio_files_rest,invenio_indexer,invenio_oauth2server,invenio_oauthclient,invenio_pidstore,invenio_records,invenio_records_files,invenio_records_rest,invenio_rest,invenio_search,kombu,marshmallow,pytest,setuptools,six,sqlalchemy,sqlalchemy_continuum,werkzeug
known_third_party = celery,click,elasticsearch,elasticsearch_dsl,flask,flask_login,flask_security,invenio_accounts,invenio_app,invenio_config,invenio_db,invenio_files_processor,invenio_files_rest,invenio_indexer,invenio_oauth2server,invenio_oauthclient,invenio_pidstore,invenio_records,invenio_records_files,invenio_records_rest,invenio_rest,invenio_search,kombu,marshmallow,pytest,setuptools,six,sqlalchemy,sqlalchemy_continuum,werkzeug
multi_line_output = 3
include_trailing_comma = True
force_grid_wrap = 0
......
......@@ -96,7 +96,7 @@ generate-certificates:
pytest:
docker-compose -f $(DOCKER_FILE) exec -T $(SERVICE_NAME) /bin/bash -c \
"pytest tests -vv;"
"pytest --collect-only && pytest tests -vv;"
.PHONY: pytest
ci-test: build-env pytest
......
......@@ -101,7 +101,9 @@ JSONSCHEMAS_REGISTER_ENDPOINTS_UI = True
# Search configuration
# =====================
SEARCH_MAPPINGS = [os.getenv("CERN_SEARCH_INSTANCE", "test")]
INSTANCE = os.getenv("CERN_SEARCH_INSTANCE", "test")
SEARCH_MAPPINGS = [INSTANCE]
SEARCH_USE_EGROUPS = ast.literal_eval(os.getenv("CERN_SEARCH_USE_EGROUPS", "True"))
SEARCH_DOC_PIPELINES = ast.literal_eval(os.getenv("CERN_SEARCH_DOC_PIPELINES", "{}"))
......@@ -149,7 +151,7 @@ RECORDS_REST_ENDPOINTS = dict(
"application/json-patch+json": lambda: request.get_json(force=True),
},
search_class="cern_search_rest_api.modules.cernsearch.search.RecordCERNSearch",
search_index=os.getenv("CERN_SEARCH_INSTANCE", "test"),
search_index=INSTANCE,
search_serializers={
"application/json": ("cern_search_rest_api.modules.cernsearch.serializers" ":json_v1_search"),
},
......
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of CERN Search.
# Copyright (C) 2018-2021 CERN.
#
# Citadel Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Required init file to allow autoload os schemas."""
{
"title": "ASR vtt line schema v1.0.0",
"id": "http://0.0.0.0:5000/schemas/asr/vtt_line_v1.0.0.json",
"$schema": "http://0.0.0.0:5000/schemas/vtt_line_v1.0.0.json",
"type": "object",
"properties": {
"_access": {
"type": "object",
"properties": {
"owner": {
"type": "array",
"items": {
"type": "string"
}
},
"read": {
"type": "array",
"items": {
"type": "string"
}
},
"update": {
"type": "array",
"items": {
"type": "string"
}
},
"delete": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"_data": {
"type": "object",
"mtext": {
"type": "string"
}
},
"path": {
"type": "string"
},
"uri": {
"type": "string"
},
"url": {
"type": "string"
},
"language": {
"type": "string"
},
"begin": {
"type": "string"
},
"end": {
"type": "string"
},
"_updated": {
"type": "string",
"description": "Internal last updated datetime of the record"
},
"_created": {
"type": "string",
"description": "Internal creation datetime of the record"
},
"control_number": {
"type": "string"
},
"$schema": {
"type": "string"
}
}
}
{
"title": "ASR vtt schema v1.0.0",
"id": "http://0.0.0.0:5000/schemas/asr/vtt_v1.0.0.json",
"$schema": "http://0.0.0.0:5000/schemas/vtt_v1.0.0.json",
"type": "object",
"properties": {
"_access": {
"type": "object",
"properties": {
"owner": {
"type": "array",
"items": {
"type": "string"
}
},
"read": {
"type": "array",
"items": {
"type": "string"
}
},
"update": {
"type": "array",
"items": {
"type": "string"
}
},
"delete": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"_data": {
"type": "object",
"message": {
"type": "string"
}
},
"path": {
"type": "string"
},
"uri": {
"type": "string"
},
"url": {
"type": "string"
},
"language": {
"type": "string"
},
"begin": {
"type": "string"
},
"end": {
"type": "string"
},
"_updated": {
"type": "string",
"description": "Internal last updated datetime of the record"
},
"_created": {
"type": "string",
"description": "Internal creation datetime of the record"
},
"control_number": {
"type": "string"
},
"$schema": {
"type": "string"
}
}
}
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of CERN Search.
# Copyright (C) 2018-2021 CERN.
#
# Citadel Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Required init file to allow autoload os schemas."""
{
"settings": {
"index.percolator.map_unmapped_fields_as_text": true,
"index.mapping.total_fields.limit": 500
},
"mappings": {
"properties": {
"_access": {
"type": "object",
"properties": {
"owner": {
"type": "keyword"
},
"read": {
"type": "keyword"
},
"update": {
"type": "keyword"
},
"delete": {
"type": "keyword"
}
}
},
"_data": {
"type": "object",
"properties": {
"mtext": {
"type": "text"
}
}
},
"path": {
"type": "keyword"
},
"uri": {
"type": "keyword"
},
"url": {
"type": "keyword"
},
"language": {
"type": "keyword"
},
"begin": {
"type": "keyword"
},
"end": {
"type": "keyword"
},
"_updated": {
"type": "date"
},
"_created": {
"type": "date"
},
"control_number": {
"type": "keyword"
},
"$schema": {
"enabled": false
}
}
}
}
{
"settings": {
"index.percolator.map_unmapped_fields_as_text": true,
"index.mapping.total_fields.limit": 500
},
"mappings": {
"properties": {
"_access": {
"type": "object",
"properties": {
"owner": {
"type": "keyword"
},
"read": {
"type": "keyword"
},
"update": {
"type": "keyword"
},
"delete": {
"type": "keyword"
}
}
},
"_data": {
"type": "object",
"properties": {
"message": {
"type": "text"
}
}
},
"path": {
"type": "keyword"
},
"uri": {
"type": "keyword"
},
"url": {
"type": "keyword"
},
"language": {
"type": "keyword"
},
"begin": {
"type": "keyword"
},
"end": {
"type": "keyword"
},
"_updated": {
"type": "date"
},
"_created": {
"type": "date"
},
"control_number": {
"type": "keyword"
},
"$schema": {
"enabled": false
}
}
}
}
......@@ -134,7 +134,7 @@ services:
retries: 3
flower:
image: mher/flower
image: mher/flower:0.9.7
env_file:
- .env
command: [
......
......@@ -56,6 +56,7 @@ setup(
"webservices = cern_search_rest_api.modules.cernsearch.mappings.webservices",
"edms = cern_search_rest_api.modules.cernsearch.mappings.edms",
"archives = cern_search_rest_api.modules.cernsearch.mappings.archives",
"asrservice = cern_search_rest_api.modules.cernsearch.mappings.asrservice",
],
"invenio_jsonschemas.schemas": [
"test = cern_search_rest_api.modules.cernsearch.jsonschemas.test",
......@@ -63,6 +64,7 @@ setup(
"webservices = cern_search_rest_api.modules.cernsearch.jsonschemas.webservices",
"edms = cern_search_rest_api.modules.cernsearch.jsonschemas.edms",
"archives = cern_search_rest_api.modules.cernsearch.jsonschemas.archives",
"asrservice = cern_search_rest_api.modules.cernsearch.jsonschemas.asrservice",
],
"invenio_base.apps": ["cern-search = cern_search_rest_api.modules.cernsearch.ext:CERNSearch"],
"invenio_base.api_apps": ["cern-search = cern_search_rest_api.modules.cernsearch.ext:CERNSearch"],
......
......@@ -15,6 +15,8 @@ fixtures are available.
from __future__ import absolute_import, print_function
import os
import pytest
from invenio_app.factory import create_api
......@@ -23,3 +25,15 @@ from invenio_app.factory import create_api
def create_app():
"""Application factory fixture."""
return create_api
@pytest.fixture(scope="module")
def app_config(app_config):
"""Application configuration fixture."""
# Use .env DB in integration Tests
if "INVENIO_SQLALCHEMY_DATABASE_URI" in os.environ:
app_config["SQLALCHEMY_DATABASE_URI"] = os.environ["INVENIO_SQLALCHEMY_DATABASE_URI"]
if "SQLALCHEMY_ENGINE_OPTIONS" in os.environ:
app_config["SQLALCHEMY_ENGINE_OPTIONS"] = os.environ["SQLALCHEMY_ENGINE_OPTIONS"]
return app_config
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of CERN Search.
# Copyright (C) 2018-2021 CERN.
#
# Citadel Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Instance initialisation tests."""
from copy import deepcopy
from http import HTTPStatus
import pytest
from flask import current_app
from invenio_search import current_search_client
from invenio_search.utils import build_alias_name
from cern_search_rest_api.config import RECORDS_REST_ENDPOINTS
def _compute_config(instance, default_doc):
rest_endpoints = deepcopy(RECORDS_REST_ENDPOINTS)
rest_endpoints["docid"]["search_index"] = instance
return {
"INSTANCE": instance,
# search
"SEARCH_MAPPINGS": instance,
# indexer
"INDEXER_DEFAULT_DOC_TYPE": default_doc,
"INDEXER_DEFAULT_INDEX": f"{instance}-{default_doc}",
# records rest
"RECORDS_REST_ENDPOINTS": rest_endpoints,
}
init_instance_params = [
{
**_compute_config("indico", "events_v1.0.0"),
"PROCESS_FILE_META": "collection",
},
{
**_compute_config("asrservice", "vtt_v1.0.0"),
},
{
**_compute_config("edms", "document_v5.0.0"),
},
{**_compute_config("archives", "archive_v3.0.0"), "SEARCH_INDEX_PREFIX": "cernsearch-egroups"},