Skip to content

GitLab

Commit 8dabc57e authored by Carina Antunes's avatar Carina Antunes Committed by Carina Antunes
Browse files

[SEARCH-67] Migrate to ES v7

parent d5a1d3d0
Hide whitespace changes
Inline Side-by-side
.env-edms 0 → 100644
View file @ 8dabc57e
CERN_SEARCH_INSTANCE=edms
INVENIO_INDEXER_DEFAULT_DOC_TYPE=document_v5.0.0
INVENIO_INDEXER_DEFAULT_INDEX=edms-document_v5.0.0
.env-egroupsarchives 0 → 100644
View file @ 8dabc57e
CERN_SEARCH_INSTANCE=egroupsarchives
INVENIO_INDEXER_DEFAULT_DOC_TYPE=archive_v1.0.0
INVENIO_INDEXER_DEFAULT_INDEX=egroupsarchives-archive_v1.0.0
.env-indico 0 → 100644
View file @ 8dabc57e
CERN_SEARCH_INSTANCE=indico
INVENIO_INDEXER_DEFAULT_DOC_TYPE=events_v1.0.0
INVENIO_INDEXER_DEFAULT_INDEX=indico-events_v1.0.0
.env-webservices 0 → 100644
View file @ 8dabc57e
CERN_SEARCH_INSTANCE=webservices
INVENIO_INDEXER_DEFAULT_DOC_TYPE=generic_website_v2.0.0
INVENIO_INDEXER_DEFAULT_INDEX=webservices-generic_website_v2.0.0
Dockerfile
View file @ 8dabc57e
......@@ -7,7 +7,7 @@
# under the terms of the MIT License; see LICENSE file for more details.
# Use CentOS7:
FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:c1bddb6006266dfa83319a44c3ca7d219e117792
FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:dafbe485df036940ba6ceb2e5a216db33e816872
ARG build_env
# CERN Search installation
......
Pipfile
View file @ 8dabc57e
......@@ -19,15 +19,15 @@ invenio-app = "<1.3.0,>=1.2.0"
invenio-base = "<1.3.0,>=1.2.0"
invenio-config = "<1.1.0,>=1.0.2"
invenio-db = {version = ">=1.0.0,<1.1.0",extras = ["postgresql", "versioning"]}
invenio-indexer = {extras = ["elasticsearch6"],version = ">=1.1.0,<1.2.0"}
invenio-indexer = {extras = ["elasticsearch7"],version = "<1.2.0,>=1.1.1"}
invenio-jsonschemas = ">=1.0.0,<1.1.0"
invenio-logging = ">=1.0.0,<1.1.0"
invenio-records-rest = {extras = ["elasticsearch6"],version = "<1.7.0,>=1.6.0"}
invenio-records-rest = {extras = ["elasticsearch7"],version = "<1.7.0,>=1.6.0"}
invenio-records = {extras = ["postgresql"],version = ">=1.0.0,<1.3.0"}
invenio-rest = {extras = ["cors"],version = "<1.2.0,>=1.1.0"}
invenio-oauthclient = ">=1.0.0,<1.1.0"
invenio-oauth2server = ">=1.0.0,<1.1.0"
invenio-search = {extras = ["elasticsearch6"],version = "<1.3.0,>=1.2.0"}
invenio-search = {extras = ["elasticsearch7"],version = ">=1.2.0,<1.3.0"}
invenio-theme = ">=1.1.0,<1.2.0"
ldap = "<1.1.0,>=1.0.2"
npm = ">=0.1.1"
......
Pipfile.lock
View file @ 8dabc57e
{
"_meta": {
"hash": {
"sha256": "061e60378237ec14c4df7d171c8606c13f3949164a2898ac854ef84ff87f7be5"
"sha256": "44f92ed52a7e7cfbf6f6bc2dd505d6fae2d726253296409e21ee8b66ea302dfc"
},
"pipfile-spec": 6,
"requires": {
......@@ -73,10 +73,10 @@
},
"bleach": {
"hashes": [
"sha256:53165a6596e7899c4338d847315fec508110a53bd6fd15c127c2e0d0860264e3",
"sha256:f8dfd8a7e26443e986c4e44df31870da8e906ea61096af06ba5d5cc2d519842a"
"sha256:cc8da25076a1fe56c3ac63671e2194458e0c4d9c7becfd52ca251650d517903c",
"sha256:e78e426105ac07026ba098f04de8abe9b6e3e98b5befbf89b51a5ef0a4292b03"
],
"version": "==3.1.3"
"version": "==3.1.4"
},
"blinker": {
"hashes": [
......@@ -188,17 +188,17 @@
},
"elasticsearch": {
"hashes": [
"sha256:73de69f2d1cfe64811a8f269389f768b15a769f881855aa1157414a1b276cc7e",
"sha256:a0be2392d696e6b9547c0e3145352347aa67b1f9b61558f2a66f0ba8c4d12995"
"sha256:d228b2d37ac0865f7631335268172dbdaa426adec1da3ed006dddf05134f89c8",
"sha256:f4bb05cfe55cf369bdcb4d86d0129d39d66a91fd9517b13cd4e4231fbfcf5c81"
],
"version": "==6.8.0"
"version": "==7.6.0"
},
"elasticsearch-dsl": {
"hashes": [
"sha256:5114a38a88e93a4663782eae07a1e8084ba333c49887335c83de8b8043bc72b2",
"sha256:d6d974cd2289543a3350690494a43fe9996485b8dc6f1d8758cb56bee01244bd"
"sha256:3e3a5dbec143b8142c3db3710610306fc6294ba0d7a14403694b2dbc2d3ed641",
"sha256:3f860e0304d703f63b458fea3782f09a823ab07da7ee84ae4bff1aa63e22aedb"
],
"version": "==6.1.0"
"version": "==7.1.0"
},
"eventlet": {
"hashes": [
......@@ -458,15 +458,16 @@
},
"importlib-metadata": {
"hashes": [
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
"sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f",
"sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"
],
"markers": "python_version < '3.8'",
"version": "==1.5.0"
"version": "==1.6.0"
},
"infinity": {
"hashes": [
"sha256:dc4aa138d7e366fc00d2e741e32c78a0fecd16b74f8daeb3f7408b459668005c"
"sha256:dc4aa138d7e366fc00d2e741e32c78a0fecd16b74f8daeb3f7408b459668005c",
"sha256:f95385def01c7723bdd0d01150f5b6ecba00cd1364721b1f8336658eda3a86ba"
],
"version": "==1.4"
},
......@@ -582,7 +583,7 @@
},
"invenio-indexer": {
"extras": [
"elasticsearch6"
"elasticsearch7"
],
"hashes": [
"sha256:a8b4052604bba21ae1e4ccf5249ad60a8967a8ccdeebec710dcbeea33322f0c5"
......@@ -657,7 +658,7 @@
},
"invenio-records-rest": {
"extras": [
"elasticsearch6"
"elasticsearch7"
],
"hashes": [
"sha256:3b88511d14182ef385123f04e8f3680600a4035dd80586ff46aeab2edf8f57d7",
......@@ -679,7 +680,7 @@
},
"invenio-search": {
"extras": [
"elasticsearch6"
"elasticsearch7"
],
"hashes": [
"sha256:5956be4f6f024f84d4732307356b618ff826336e437e9d1c7ec99edfc1b7d734",
......@@ -696,13 +697,6 @@
"index": "pypi",
"version": "==1.1.4"
},
"ipaddress": {
"hashes": [
"sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc",
"sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2"
],
"version": "==1.0.23"
},
"ipython": {
"hashes": [
"sha256:ca478e52ae1f88da0102360e57e528b92f3ae4316aabac80a2cd7f7ab2efb48a",
......@@ -967,10 +961,10 @@
},
"prompt-toolkit": {
"hashes": [
"sha256:859e1b205b6cf6a51fa57fa34202e45365cf58f8338f0ee9f4e84a4165b37d5b",
"sha256:ebe6b1b08c888b84c50d7f93dee21a09af39860144ff6130aadbd61ae8d29783"
"sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8",
"sha256:df7e9e63aea609b1da3a65641ceaf5bc7d05e0a04de5bd45d05dbeffbabf9e04"
],
"version": "==3.0.4"
"version": "==3.0.5"
},
"psycopg2-binary": {
"hashes": [
......@@ -1060,9 +1054,9 @@
},
"pyrsistent": {
"hashes": [
"sha256:cdc7b5e3ed77bed61270a47d35434a30617b9becdf2478af76ad2c6ade307280"
"sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3"
],
"version": "==0.15.7"
"version": "==0.16.0"
},
"python-dateutil": {
"hashes": [
......@@ -1396,11 +1390,11 @@
},
"importlib-metadata": {
"hashes": [
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
"sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f",
"sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"
],
"markers": "python_version < '3.8'",
"version": "==1.5.0"
"version": "==1.6.0"
},
"isort": {
"hashes": [
......@@ -1556,19 +1550,19 @@
},
"pytest-invenio": {
"hashes": [
"sha256:8f2625312714a61a0ab18b96efa2002253acfa38d1577270d3d120b7b7acb078",
"sha256:9a7e12c21a7cca0bccadc88e03016283fd7aeac5d58c2ade30aeefc9eb686f3b"
"sha256:4418d2ac90eca43f14e803402a5b27e346b00fdc67d6ec0b29f5027214e59202",
"sha256:934446235e4813ed6ee33d589140e50ec70e9bfa7bca1dc5a2b60e46f8e477b4"
],
"index": "pypi",
"version": "==1.2.1"
"version": "==1.3.0"
},
"pytest-mock": {
"hashes": [
"sha256:b35eb281e93aafed138db25c8772b95d3756108b601947f89af503f8c629413f",
"sha256:cb67402d87d5f53c579263d37971a164743dc33c159dfb4fb4a86f37c5552307"
"sha256:98e02534f170e4f37d7e1abdfc5973fd4207aa609582291717f643764e71c925",
"sha256:a4494016753a30231f8519bfd160242a0f3c8fb82ca36e7b6f82a7fb602ac6b8"
],
"index": "pypi",
"version": "==2.0.0"
"version": "==3.0.0"
},
"python-dotenv": {
"hashes": [
......
cern_search_rest_api/config.py
View file @ 8dabc57e
......@@ -133,7 +133,7 @@ RECORDS_REST_ENDPOINTS = dict(
)
RECORDS_REST_FACETS = {
'cernsearchqa-webservices': {
'webservices': {
'aggs': {
'collection': {
'terms': {'field': 'collection'}
......@@ -154,7 +154,7 @@ RECORDS_REST_FACETS = {
}
RECORDS_REST_SORT_OPTIONS = {
'cernsearchqa-webservices': {
'webservices': {
'bestmatch': {
'fields': ['-_score'],
'title': 'Best match',
......@@ -166,7 +166,7 @@ RECORDS_REST_SORT_OPTIONS = {
'default_order': 'asc',
}
},
'cernsearchqa-edms': {
'edms': {
'bestmatch': {
'fields': ['-_score'],
'title': 'Best match',
......
cern_search_rest_api/modules/cernsearch/mappings/__init__.py
View file @ 8dabc57e
......@@ -6,3 +6,4 @@
#
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Init file."""
cern_search_rest_api/modules/cernsearch/mappings/edms/__init__.py
View file @ 8dabc57e
......@@ -6,3 +6,4 @@
#
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Init file."""
cern_search_rest_api/modules/cernsearch/mappings/edms/v6/edms/file_v6.0.0.json
View file @ 8dabc57e
......@@ -32,7 +32,8 @@
"type": "object",
"properties": {
"_content": {
"type": "text"
"type": "text",
"boost": 20
}
}
},
......
cern_search_rest_api/modules/cernsearch/mappings/edms/v7/__init__.py 0 → 100644
View file @ 8dabc57e
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of CERN Search.
# Copyright (C) 2018-2019 CERN.
#
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Init file."""
cern_search_rest_api/modules/cernsearch/mappings/edms/v7/edms/document_v5.0.0.json 0 → 100644
View file @ 8dabc57e
{
"settings": {
"index.percolator.map_unmapped_fields_as_text": true,
"index.mapping.total_fields.limit": 500,
"index.query.default_field" : ["_data.*"]
},
"mappings": {
"dynamic": "strict",
"numeric_detection": true,
"properties": {
"_access": {
"type": "object",
"properties": {
"owner": {
"type": "keyword"
},
"read": {
"type": "keyword"
},
"update": {
"type": "keyword"
},
"delete": {
"type": "keyword"
}
}
},
"_data": {
"type": "object",
"properties": {
"document_id": {
"type": "keyword",
"boost": 20
},
"document_id_version": {
"type": "keyword",
"boost": 20
},
"cern_id": {
"type": "keyword",
"boost": 20
},
"cern_id_version": {
"type": "keyword",
"boost": 20
},
"external_reference": {
"type": "text",
"boost": 8,
"fields": {
"exact_match": {
"type": "keyword",
"boost": 8
}
}
},
"version": {
"type": "keyword",
"boost": 1
},
"title": {
"type": "text",
"boost": 8,
"fields": {
"english": {
"type": "text",
"boost": 8,
"analyzer": "english"
},
"french": {
"type": "text",
"boost": 8,
"analyzer": "french"
},
"exact_match": {
"type": "keyword",
"boost": 8
}
}
},
"description": {
"type": "text",
"boost": 4,
"fields": {
"english": {
"type": "text",
"boost": 4,
"analyzer": "english"
},
"french": {
"type": "text",
"boost": 4,
"analyzer": "french"
}
}
},
"author": {
"type": "object",
"properties": {
"full_name": {
"type": "text",
"boost": 2
},
"email": {
"type": "text",
"boost": 2
}
}
},
"keywords": {
"type": "text",
"boost": 6,
"fields": {
"english": {
"type": "text",
"analyzer": "english",
"boost": 6
},
"french": {
"type": "text",
"analyzer": "french",
"boost": 6
},
"exact_match": {
"type": "keyword",
"boost": 6
}
}
},
"equipment_code": {
"type": "keyword",
"boost": 6
}
}
},
"cid": {
"type": "keyword"
},
"owner": {
"type": "keyword"
},
"group": {
"type": "keyword"
},
"object_type": {
"type": "keyword"
},
"parents": {
"type": "object",
"properties": {
"parent_type": {
"type": "keyword"
},
"parent_id": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword"
},
"latest_version": {
"type": "keyword"
},
"obsolete": {
"type": "keyword"
},
"context": {
"type": "object",
"properties": {
"name": {
"type": "keyword"
},
"context_id": {
"type": "keyword"
}
}
},
"document_type": {
"type": "keyword"
},
"release_procedure": {
"type": "keyword"
},
"object_properties": {
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "text",
"fields": {
"english": {
"type": "text",
"analyzer": "english"
},
"french": {
"type": "text",
"analyzer": "french"
},
"exact_match": {
"type": "keyword"
}
}
}
}
},
"attributes": {
"type": "keyword"
},
"creation_date": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss"
},
"modification_date": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss"
},
"_updated": {
"type": "date"
},
"_created": {
"type": "date"
},
"control_number": {
"type": "keyword"
},
"$schema": {
"enabled": false
}
}
}
}
cern_search_rest_api/modules/cernsearch/mappings/edms/v7/edms/file_v6.0.0.json 0 → 100644
View file @ 8dabc57e
{
"settings": {
"index.percolator.map_unmapped_fields_as_text": true,
"index.mapping.total_fields.limit": 500,
"index.query.default_field" : ["_data.*"]
},
"mappings": {
"numeric_detection": true,
"properties": {
"_access": {
"type": "object",
"properties": {
"owner": {
"type": "keyword"
},
"read": {
"type": "keyword"
},
"update": {
"type": "keyword"
},
"delete": {
"type": "keyword"
}
}
},
"_data": {
"type": "object",
"properties": {
"_attachment": {
"dynamic": true,
"type": "object",
"properties": {
"_content": {
"type": "text",
"boost": 20
}
}
},
"file_name": {
"type": "text",
"fields": {
"english": {
"type": "text",
"analyzer": "english",
"boost": 20
},
"french": {
"type": "text",
"analyzer": "french",
"boost": 20
}
}
},
"document_id": {
"type": "keyword",
"boost": 20
},
"document_id_version": {
"type": "keyword",
"boost": 20
},
"cern_id": {
"type": "keyword",
"boost": 20
},
"cern_id_version": {
"type": "keyword",
"boost": 20
},
"external_reference": {
"type": "text",
"boost": 8,
"fields": {
"exact_match": {
"type": "keyword",
"boost": 8
}
}
},