Commit a0cec983 authored by Carina Antunes's avatar Carina Antunes
Browse files

Switch from pipenv to poetry

parent 200d3049
...@@ -62,8 +62,8 @@ build_base_image: ...@@ -62,8 +62,8 @@ build_base_image:
- merge_requests - merge_requests
changes: changes:
- Dockerfile-base - Dockerfile-base
- Pipfile - pyproject.toml
- Pipfile.lock - poetry.lock
when: manual when: manual
variables: variables:
DOCKER_FILE: Dockerfile-base DOCKER_FILE: Dockerfile-base
...@@ -77,8 +77,8 @@ validate-base-image-updated: ...@@ -77,8 +77,8 @@ validate-base-image-updated:
- merge_requests - merge_requests
changes: changes:
- Dockerfile-base - Dockerfile-base
- Pipfile - pyproject.toml
- Pipfile.lock - poetry.lock
script: script:
- git diff --name-only origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME | grep -q -E "(^|[^-])\bDockerfile\b([^-]|$)" - git diff --name-only origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME | grep -q -E "(^|[^-])\bDockerfile\b([^-]|$)"
......
CELERY_LOG_LEVEL=INFO CELERY_LOG_LEVEL=error
CERN_SEARCH_INSTANCE=test CERN_SEARCH_INSTANCE=test
CERN_SEARCH_REMOTE_APP_RESOURCE=localhost CERN_SEARCH_REMOTE_APP_RESOURCE=localhost
CERN_SEARCH_SERVER_NAME=localhost
CERN_SEARCH_USE_EGROUPS='False' CERN_SEARCH_USE_EGROUPS='False'
CERN_SEARCH_FILES_PROCESSOR_QUEUE=files_processor CERN_SEARCH_FILES_PROCESSOR_QUEUE=files_processor
CERN_SEARCH_FILES_PROCESSOR_QUEUE_DLX=files_processor_dlx CERN_SEARCH_FILES_PROCESSOR_QUEUE_DLX=files_processor_dlx
CERN_SEARCH_FILES_PROCESSOR_EXCHANGE=default CERN_SEARCH_FILES_PROCESSOR_EXCHANGE=default
CERN_SEARCH_FILES_PROCESSOR_EXCHANGE_DLX=dlx CERN_SEARCH_FILES_PROCESSOR_EXCHANGE_DLX=dlx
CERN_SEARCH_INSTANCE_IMMUTABLE='False'
CERN_SEARCH_FILE_INDEXER='True'
CONTAINER_NAME=cern-search-rest-api CONTAINER_NAME=cern-search-rest-api
WORKER_APP=invenio_app.celery
DEFAULT_RECORDS_FILES_LOCATION=/usr/share/cern-search-api/files
ENV=dev ENV=dev
FLASK_DEBUG='True'
FLASK_SKIP_DOTENV=1 FLASK_SKIP_DOTENV=1
FLASK_DEBUG='True'
INVENIO_DEBUG=0
FLOWER_PASS=password FLOWER_PASS=password
INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://localhost:6379/1 INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://localhost:6379/1
INVENIO_ADMIN_ACCESS_GROUPS=CernSearch-Administrators@cern.ch INVENIO_ADMIN_ACCESS_GROUPS=search-admin
INVENIO_ADMIN_USER=test@example.com INVENIO_ADMIN_USER=search-admin
INVENIO_ADMIN_VIEW_ACCESS_GROUPS=CernSearch-Administrators@cern.ch INVENIO_ADMIN_VIEW_ACCESS_GROUPS=search-admin
INVENIO_APP_ALLOWED_HOSTS=['localhost','127.0.0.1'] INVENIO_APP_ALLOWED_HOSTS=['localhost','127.0.0.1']
INVENIO_BROKER_URL=amqp://guest:password@localhost:5672 INVENIO_BROKER_URL=amqp://guest:password@localhost:5672
INVENIO_CACHE_REDIS_HOST=localhost INVENIO_CACHE_REDIS_HOST=localhost
...@@ -31,8 +37,10 @@ INVENIO_CERN_APP_CREDENTIALS_CONSUMER_KEY=xxx ...@@ -31,8 +37,10 @@ INVENIO_CERN_APP_CREDENTIALS_CONSUMER_KEY=xxx
INVENIO_CELERY_BROKER_URL=amqp://guest:password@localhost:5672 INVENIO_CELERY_BROKER_URL=amqp://guest:password@localhost:5672
INVENIO_CELERY_RESULT_BACKEND=redis://localhost:6379/2 INVENIO_CELERY_RESULT_BACKEND=redis://localhost:6379/2
INVENIO_COLLECT_STORAGE=flask_collect.storage.file INVENIO_COLLECT_STORAGE=flask_collect.storage.file
INVENIO_INDEXER_BULK_REQUEST_TIMEOUT=10
INVENIO_INDEXER_DEFAULT_DOC_TYPE=doc_v0.0.2 INVENIO_INDEXER_DEFAULT_DOC_TYPE=doc_v0.0.2
INVENIO_INDEXER_DEFAULT_INDEX=test-doc_v0.0.2 INVENIO_INDEXER_DEFAULT_INDEX=test-doc_v0.0.2
INVENIO_JSONSCHEMAS_HOST=localhost
INVENIO_LOGGING_CONSOLE='True' INVENIO_LOGGING_CONSOLE='True'
INVENIO_LOGGING_CONSOLE_LEVEL=DEBUG INVENIO_LOGGING_CONSOLE_LEVEL=DEBUG
INVENIO_RATELIMIT_STORAGE_URL='redis://localhost:6379/3' INVENIO_RATELIMIT_STORAGE_URL='redis://localhost:6379/3'
......
...@@ -7,17 +7,17 @@ ...@@ -7,17 +7,17 @@
# under the terms of the MIT License; see LICENSE file for more details. # under the terms of the MIT License; see LICENSE file for more details.
# Use CentOS7: # Use CentOS7:
FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:4cc14deb49f42c505062110461b96c12cda9b377 FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:3b6147ba87cd87992329c2dfaf4ba5f5d42ecf27
ARG build_env ARG build_env
# CERN Search installation # CERN Search installation
WORKDIR /${WORKING_DIR}/src WORKDIR /${WORKING_DIR}/src
ADD . /${WORKING_DIR}/src ADD . /${WORKING_DIR}/src
RUN pip freeze RUN python -V && pip -V
# If env is development, install development dependencies # If env is development, install development dependencies
RUN if [ "$build_env" != "prod" ]; then pipenv install --system --ignore-pipfile --deploy --dev; fi RUN if [ "$build_env" != "prod" ]; then poetry install --no-root; fi
RUN pip freeze RUN pip freeze
......
...@@ -20,8 +20,10 @@ RUN yum update -y && \ ...@@ -20,8 +20,10 @@ RUN yum update -y && \
# CERN Search installation # CERN Search installation
WORKDIR /${WORKING_DIR}/src WORKDIR /${WORKING_DIR}/src
ADD ./Pipfile /${WORKING_DIR}/src/Pipfile ADD ./poetry.lock /${WORKING_DIR}/src
ADD ./Pipfile.lock /${WORKING_DIR}/src/Pipfile.lock ADD ./pyproject.toml /${WORKING_DIR}/src
# Install dependencies globally # Install dependencies globally
RUN pipenv install --system --deploy RUN pip install poetry && \
poetry config virtualenvs.create false && \
poetry install --no-dev --no-root
...@@ -109,23 +109,23 @@ lint: ...@@ -109,23 +109,23 @@ lint:
.PHONY: lint .PHONY: lint
################### Local development helpful directives #################### ################### Local development helpful directives ####################
################### (pipenv + docker) #################### ################### (poetry + docker) ####################
# #
# Usage: # Usage:
# make logs # displays log outputs from running services # make logs # displays log outputs from running services
# make build-local-env # build pipenv, create and start containers # make build-local-env # build poetry env, create and start containers
# make check-requirements-local # check requirements # make check-requirements-local # check requirements
# make local-env # build virtual environment, load fixtures and starts api # make local-env # build virtual environment, load fixtures and starts api
# make populate-instance-local # create database, tables and indices # make populate-instance-local # create database, tables and indices
# make serve-api-local # start serving api # make serve-api-local # start serving api
# make shell-local-env # start bash inside pipenv # make shell-local-env # start bash inside poetry
# make destroy-local-env # stop and remove containers, networks, images, and volume and pipenv # make destroy-local-env # stop and remove containers, networks, images, and volume and poetry env
# make reload-local-env # restart containers, networks, images, and volume and pipenv # make reload-local-env # restart containers, networks, images, and volume and poetry env
# make load-fixtures-local # loads fixtures # make load-fixtures-local # loads fixtures
# make local-test # runs tests # make local-test # runs tests
# make local-lint # runs linting tools # make local-lint # runs linting tools
PIPENV_DOTENV := .pipenv.env POETRY_DOTENV := .poetry.env
PYTHON_VERSION_FILE := .python-version PYTHON_VERSION_FILE := .python-version
PYTHON_VERSION := $(cat $(PYTHON_VERSION_FILE) | xargs) PYTHON_VERSION := $(cat $(PYTHON_VERSION_FILE) | xargs)
PIPENV_DOCKER_FILE := docker-compose.yml PIPENV_DOCKER_FILE := docker-compose.yml
...@@ -140,32 +140,31 @@ check-requirements-local: ...@@ -140,32 +140,31 @@ check-requirements-local:
build-local-env: check-requirements-local build-local-env: check-requirements-local
docker-compose -f $(PIPENV_DOCKER_FILE) up -d --build --remove-orphans docker-compose -f $(PIPENV_DOCKER_FILE) up -d --build --remove-orphans
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/bootstrap sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/pipenv/bootstrap
.PHONY: build-local-env .PHONY: build-local-env
populate-instance-local: populate-instance-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/populate-instance.sh sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/pipenv/populate-instance.sh
.PHONY: populate-instance-local .PHONY: populate-instance-local
load-fixtures-local: load-fixtures-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/create-test-user.sh sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/create-test-user.sh
.PHONY: load-fixtures-local .PHONY: load-fixtures-local
serve-api-local: serve-api-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/server sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/pipenv/server
.PHONY: serve-api-local .PHONY: serve-api-local
local-env: build-local-env populate-instance-local serve-api-local local-env: build-local-env populate-instance-local serve-api-local
.PHONY: local-env .PHONY: local-env
shell-local-env: shell-local-env:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv shell sh with_env.sh $(POETRY_DOTENV) poetry shell
.PHONY: shell-local-env .PHONY: shell-local-env
destroy-local-env: destroy-local-env:
docker-compose -f $(PIPENV_DOCKER_FILE) down --volumes docker-compose -f $(PIPENV_DOCKER_FILE) down --volumes
docker-compose -f $(PIPENV_DOCKER_FILE) rm -f docker-compose -f $(PIPENV_DOCKER_FILE) rm -f
pipenv --rm
.PHONY: destroy-local-env .PHONY: destroy-local-env
reload-local-env: destroy-local-env local-env reload-local-env: destroy-local-env local-env
...@@ -173,12 +172,12 @@ reload-local-env: destroy-local-env local-env ...@@ -173,12 +172,12 @@ reload-local-env: destroy-local-env local-env
local-test: local-test:
@echo running tests...; @echo running tests...;
pipenv run pytest tests -v; sh with_env.sh $(POETRY_DOTENV) poetry run pytest tests -v;
.PHONY: local-test .PHONY: local-test
local-lint: local-lint:
@echo running isort...; @echo running isort...;
pipenv run isort -rc -c -df; sh with_env.sh $(POETRY_DOTENV) poetry run isort -rc -c -df .;
@echo running flake8...; @echo running flake8...;
pipenv run flake8 --max-complexity 10 --ignore E501,D401 sh with_env.sh $(POETRY_DOTENV) poetry run flake8 --max-complexity 10 --ignore E501,D401
.PHONY: local-lint .PHONY: local-lint
This diff is collapsed.
...@@ -16,7 +16,7 @@ documents and search among them when needed! ...@@ -16,7 +16,7 @@ documents and search among them when needed!
Read more on the makefile. Read more on the makefile.
- **Docker + Pipenv: Read more on the makefile** - **Docker + Poetry: Read more on the makefile**
1. Run `make local-env MODE=test` 1. Run `make local-env MODE=test`
2. Follow [instructions](#tls---how-to-install-certificate) to install certificate. 2. Follow [instructions](#tls---how-to-install-certificate) to install certificate.
......
...@@ -12,7 +12,7 @@ from celery import shared_task ...@@ -12,7 +12,7 @@ from celery import shared_task
from celery.exceptions import MaxRetriesExceededError, Reject from celery.exceptions import MaxRetriesExceededError, Reject
from flask import current_app from flask import current_app
from invenio_files_processor.errors import InvalidProcessor from invenio_files_processor.errors import InvalidProcessor
from invenio_files_processor.processors.tika import TikaProcessor from invenio_files_processor.processors.tika.unpack import UnpackProcessor
from invenio_files_processor.proxies import current_processors from invenio_files_processor.proxies import current_processors
from invenio_files_rest.models import ObjectVersion from invenio_files_rest.models import ObjectVersion
...@@ -30,7 +30,7 @@ def process_file_async(self, bucket_id, key_id): ...@@ -30,7 +30,7 @@ def process_file_async(self, bucket_id, key_id):
current_app.logger.debug(f"Processing file {bucket_id}:{key_id}") current_app.logger.debug(f"Processing file {bucket_id}:{key_id}")
obj = ObjectVersion.get(bucket_id, key_id) # type: ObjectVersion obj = ObjectVersion.get(bucket_id, key_id) # type: ObjectVersion
processor = current_processors.get_processor(name=TikaProcessor.id()) # type: TikaProcessor processor = current_processors.get_processor(name=UnpackProcessor.id) # type: UnpackProcessor
processor.process(obj) processor.process(obj)
current_app.logger.debug(f"Processed file {bucket_id}:{key_id}") current_app.logger.debug(f"Processed file {bucket_id}:{key_id}")
......
# To be used with pipenv locally # To be used with poetry locally
# #
# Requires: # Requires:
# - pipenv # - poetry
# - python@3.6.0 # - python@3.6.0
version: "2.3" version: "2.3"
......
This diff is collapsed.
[[source]] [tool.poetry]
name = "pypi" name = "cern-search-rest-api"
url = "https://pypi.org/simple" version = "1.4.0-beta"
verify_ssl = true description = "CERN Search as a Service"
authors = ["Citadel Search <cernsearch.support@cern.ch>"]
license = "MIT"
[dev-packages] [tool.poetry.dependencies]
flake8 = ">=3.7.9" Flask = "^1.1.2"
flake8-docstrings = ">=1.5.0"
isort = "==4.3.21"
pytest-dotenv = "*"
pytest-invenio = ">=1.2.0"
pytest-mock = ">=1.6.0"
[packages]
Flask = "*"
eventlet = "*"
gevent = ">=1.4.0"
invenio-access = ">=1.4.1,<1.5.0" invenio-access = ">=1.4.1,<1.5.0"
invenio-accounts = ">=1.3.0,<1.4.0" invenio-accounts = ">=1.3.0,<1.4.0"
invenio-admin = ">=1.2.1,<1.3.0" invenio-admin = ">=1.2.1,<1.3.0"
...@@ -23,34 +15,33 @@ invenio-assets = ">=1.1.3,<1.2.0" ...@@ -23,34 +15,33 @@ invenio-assets = ">=1.1.3,<1.2.0"
invenio-base = ">=1.2.3,<1.3.0" invenio-base = ">=1.2.3,<1.3.0"
invenio-celery = ">=1.2.0,<1.3.0" invenio-celery = ">=1.2.0,<1.3.0"
invenio-config = ">=1.0.3,<1.1.0" invenio-config = ">=1.0.3,<1.1.0"
invenio-db = {version = ">=1.0.5,<1.1.0",extras = ["postgresql", "versioning"]} invenio-db = {version = ">=1.0.5,<1.1.0", extras = ["postgresql", "versioning"]}
invenio-files-processor = {extras = ["tika"],git = "https://github.com/carantunes/invenio-files-processor.git",ref = "1.0.2-alpha"} invenio-files-processor = {extras = ["tika"], version = "^0.1.0"}
invenio-files-rest = ">=1.2.0,<1.3.0" invenio-files-rest = ">=1.2.0,<1.3.0"
invenio-indexer = ">=1.1.1,<1.2.0"
invenio-i18n = "<1.3.0,>=1.2.0" invenio-i18n = "<1.3.0,>=1.2.0"
invenio-indexer = ">=1.1.1,<1.2.0"
invenio-jsonschemas = ">=1.1.0,<1.2.0" invenio-jsonschemas = ">=1.1.0,<1.2.0"
invenio-logging = {extras = ["sentry-sdk"],version = ">=1.3.0,<1.4.0"} invenio-logging = {version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk"]}
invenio-oauth2server = ">=1.2.0,<1.3.0" invenio-oauth2server = ">=1.2.0,<1.3.0"
invenio-oauthclient = "<1.4.0,>=1.3.5" invenio-oauthclient = "<1.4.0,>=1.3.5"
invenio-records = {extras = ["postgresql"],version = ">=1.3.1,<1.4.0"} invenio-records = {version = ">=1.3.1,<1.4.0", extras = ["postgresql"]}
invenio-records-files = ">=1.2.1,<1.3.0" invenio-records-files = ">=1.2.1,<1.3.0"
invenio-records-rest = ">=1.7.1,<1.8.0" invenio-records-rest = ">=1.7.1,<1.8.0"
invenio-rest = ">=1.2.1,<1.3.0" invenio-rest = ">=1.2.1,<1.3.0"
invenio-search = {extras = ["elasticsearch7"],version = ">=1.3.1,<1.4.0"} invenio-search = {version = ">=1.3.1,<1.4.0", extras = ["elasticsearch7"]}
invenio-theme = ">=1.1.4,<1.2.0" invenio-theme = ">=1.1.4,<1.2.0"
ldap = "<1.1.0,>=1.0.2" python = "^3.6"
npm = ">=0.1.1" uWSGI = "^2.0.19"
pip = "*" uwsgi-tools = "^1.1.1"
python-ldap = "<3.2.0,>=3.1.0"
raven = ">=6.9.0,<6.10.0"
redis = ">=2.10.0,<3.0.0"
uWSGI = ">=2.0.16"
uwsgi-tools = ">=1.1.1,<1.2.0"
wtforms = "<2.3.0"
tika = "==1.24"
[requires] [tool.poetry.dev-dependencies]
python_version = "3.6" flake8 = "^3.8.4"
flake8-docstrings = "^1.5.0"
isort = "^5.6.4"
pytest-dotenv = "^0.5.2"
pytest-invenio = "^1.4.0"
pytest-mock = "^3.3.1"
[pipenv] [build-system]
allow_prereleases = true requires = ["poetry-core>=1.0.0", "setuptools"]
build-backend = "poetry.core.masonry.api"
...@@ -13,7 +13,7 @@ readonly SYS_PREFIX=$(python -c "import sys; print(sys.prefix)") ...@@ -13,7 +13,7 @@ readonly SYS_PREFIX=$(python -c "import sys; print(sys.prefix)")
readonly INVENIO_INSTANCE_PATH="$SYS_PREFIX/var/instance" readonly INVENIO_INSTANCE_PATH="$SYS_PREFIX/var/instance"
# Installs all packages specified in Pipfile (to avoid OS incompabilities) # Installs all packages specified in Pipfile (to avoid OS incompabilities)
pipenv install --dev --skip-lock poetry install --no-root
# Install application code and entrypoints from 'setup.py' # Install application code and entrypoints from 'setup.py'
pip install -e $SCRIPT_PATH/../.. pip install -e $SCRIPT_PATH/../..
......
...@@ -6,10 +6,10 @@ else ...@@ -6,10 +6,10 @@ else
exit 1; exit 1;
fi fi
# check pipenv installed # check poetry installed
if [[ "$(pipenv --version)" =~ "pipenv, version" ]]; then if [[ "$(poetry --version)" =~ "poetry, version" ]]; then
echo pipenv is installed echo poetry is installed
else else
echo pipenv is not installed. Aborting. echo poetry is not installed. Aborting.
exit 1; exit 1;
fi fi
...@@ -14,7 +14,7 @@ readonly TLS_DIR="$SCRIPT_PATH/../../nginx/tls" ...@@ -14,7 +14,7 @@ readonly TLS_DIR="$SCRIPT_PATH/../../nginx/tls"
export FLASK_ENV=development export FLASK_ENV=development
# Start Worker # Start Worker
pipenv run celery worker -A invenio_app.celery -l DEBUG --autoscale=10,1 & pid_celery=$! poetry run celery worker -A invenio_app.celery -l DEBUG --autoscale=10,1 & pid_celery=$!
# Start Server # Start Server
invenio run \ invenio run \
......
...@@ -13,4 +13,4 @@ test = pytest ...@@ -13,4 +13,4 @@ test = pytest
env_files = env_files =
# uncomment according to the environment being used # uncomment according to the environment being used
.env .env
#.pipenv.env #.poetry.env
...@@ -10,8 +10,9 @@ ...@@ -10,8 +10,9 @@
from unittest.mock import patch from unittest.mock import patch
from celery.exceptions import MaxRetriesExceededError, Reject, Retry from celery.exceptions import MaxRetriesExceededError, Reject, Retry
from invenio_files_processor.processors.tika.unpack import UnpackProcessor
from cern_search_rest_api.modules.cernsearch.tasks import process_file_async from cern_search_rest_api.modules.cernsearch.tasks import process_file_async
from invenio_files_processor.processors.tika import TikaProcessor
from pytest import raises from pytest import raises
...@@ -34,7 +35,7 @@ class TestProcessFileAsync: ...@@ -34,7 +35,7 @@ class TestProcessFileAsync:
process_file_async('00000000-0000-0000-0000-000000000000', 'test.pdf') process_file_async('00000000-0000-0000-0000-000000000000', 'test.pdf')
object_version_get_mock.assert_called_once_with('00000000-0000-0000-0000-000000000000', 'test.pdf') object_version_get_mock.assert_called_once_with('00000000-0000-0000-0000-000000000000', 'test.pdf')
get_processor_mock.assert_called_once_with(name=TikaProcessor.id()) get_processor_mock.assert_called_once_with(name=UnpackProcessor.id)
get_processor_mock.return_value.process.assert_called_once_with(object_version) get_processor_mock.return_value.process.assert_called_once_with(object_version)
@patch('cern_search_rest_api.modules.cernsearch.tasks.ObjectVersion.get', side_effect=Exception()) @patch('cern_search_rest_api.modules.cernsearch.tasks.ObjectVersion.get', side_effect=Exception())
......
#!/usr/bin/env bash
# -*- coding: utf-8 -*-
#
# This file is part of CERN Search.
# Copyright (C) 2018-2019 CERN.
#
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
env_file="$1"
cmd="${@:2}"
bash -c "source $env_file && $cmd"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment