Commit a0cec983 authored by Carina Antunes's avatar Carina Antunes
Browse files

Switch from pipenv to poetry

parent 200d3049
......@@ -62,8 +62,8 @@ build_base_image:
- merge_requests
changes:
- Dockerfile-base
- Pipfile
- Pipfile.lock
- pyproject.toml
- poetry.lock
when: manual
variables:
DOCKER_FILE: Dockerfile-base
......@@ -77,8 +77,8 @@ validate-base-image-updated:
- merge_requests
changes:
- Dockerfile-base
- Pipfile
- Pipfile.lock
- pyproject.toml
- poetry.lock
script:
- git diff --name-only origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME | grep -q -E "(^|[^-])\bDockerfile\b([^-]|$)"
......
CELERY_LOG_LEVEL=INFO
CELERY_LOG_LEVEL=error
CERN_SEARCH_INSTANCE=test
CERN_SEARCH_REMOTE_APP_RESOURCE=localhost
CERN_SEARCH_SERVER_NAME=localhost
CERN_SEARCH_USE_EGROUPS='False'
CERN_SEARCH_FILES_PROCESSOR_QUEUE=files_processor
CERN_SEARCH_FILES_PROCESSOR_QUEUE_DLX=files_processor_dlx
CERN_SEARCH_FILES_PROCESSOR_EXCHANGE=default
CERN_SEARCH_FILES_PROCESSOR_EXCHANGE_DLX=dlx
CERN_SEARCH_INSTANCE_IMMUTABLE='False'
CERN_SEARCH_FILE_INDEXER='True'
CONTAINER_NAME=cern-search-rest-api
WORKER_APP=invenio_app.celery
DEFAULT_RECORDS_FILES_LOCATION=/usr/share/cern-search-api/files
ENV=dev
FLASK_DEBUG='True'
FLASK_SKIP_DOTENV=1
FLASK_DEBUG='True'
INVENIO_DEBUG=0
FLOWER_PASS=password
INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://localhost:6379/1
INVENIO_ADMIN_ACCESS_GROUPS=CernSearch-Administrators@cern.ch
INVENIO_ADMIN_USER=test@example.com
INVENIO_ADMIN_VIEW_ACCESS_GROUPS=CernSearch-Administrators@cern.ch
INVENIO_ADMIN_ACCESS_GROUPS=search-admin
INVENIO_ADMIN_USER=search-admin
INVENIO_ADMIN_VIEW_ACCESS_GROUPS=search-admin
INVENIO_APP_ALLOWED_HOSTS=['localhost','127.0.0.1']
INVENIO_BROKER_URL=amqp://guest:password@localhost:5672
INVENIO_CACHE_REDIS_HOST=localhost
......@@ -31,8 +37,10 @@ INVENIO_CERN_APP_CREDENTIALS_CONSUMER_KEY=xxx
INVENIO_CELERY_BROKER_URL=amqp://guest:password@localhost:5672
INVENIO_CELERY_RESULT_BACKEND=redis://localhost:6379/2
INVENIO_COLLECT_STORAGE=flask_collect.storage.file
INVENIO_INDEXER_BULK_REQUEST_TIMEOUT=10
INVENIO_INDEXER_DEFAULT_DOC_TYPE=doc_v0.0.2
INVENIO_INDEXER_DEFAULT_INDEX=test-doc_v0.0.2
INVENIO_JSONSCHEMAS_HOST=localhost
INVENIO_LOGGING_CONSOLE='True'
INVENIO_LOGGING_CONSOLE_LEVEL=DEBUG
INVENIO_RATELIMIT_STORAGE_URL='redis://localhost:6379/3'
......
......@@ -7,17 +7,17 @@
# under the terms of the MIT License; see LICENSE file for more details.
# Use CentOS7:
FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:4cc14deb49f42c505062110461b96c12cda9b377
FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:3b6147ba87cd87992329c2dfaf4ba5f5d42ecf27
ARG build_env
# CERN Search installation
WORKDIR /${WORKING_DIR}/src
ADD . /${WORKING_DIR}/src
RUN pip freeze
RUN python -V && pip -V
# If env is development, install development dependencies
RUN if [ "$build_env" != "prod" ]; then pipenv install --system --ignore-pipfile --deploy --dev; fi
RUN if [ "$build_env" != "prod" ]; then poetry install --no-root; fi
RUN pip freeze
......
......@@ -20,8 +20,10 @@ RUN yum update -y && \
# CERN Search installation
WORKDIR /${WORKING_DIR}/src
ADD ./Pipfile /${WORKING_DIR}/src/Pipfile
ADD ./Pipfile.lock /${WORKING_DIR}/src/Pipfile.lock
ADD ./poetry.lock /${WORKING_DIR}/src
ADD ./pyproject.toml /${WORKING_DIR}/src
# Install dependencies globally
RUN pipenv install --system --deploy
RUN pip install poetry && \
poetry config virtualenvs.create false && \
poetry install --no-dev --no-root
......@@ -109,23 +109,23 @@ lint:
.PHONY: lint
################### Local development helpful directives ####################
################### (pipenv + docker) ####################
################### (poetry + docker) ####################
#
# Usage:
# make logs # displays log outputs from running services
# make build-local-env # build pipenv, create and start containers
# make build-local-env # build poetry env, create and start containers
# make check-requirements-local # check requirements
# make local-env # build virtual environment, load fixtures and starts api
# make populate-instance-local # create database, tables and indices
# make serve-api-local # start serving api
# make shell-local-env # start bash inside pipenv
# make destroy-local-env # stop and remove containers, networks, images, and volume and pipenv
# make reload-local-env # restart containers, networks, images, and volume and pipenv
# make shell-local-env # start bash inside poetry
# make destroy-local-env # stop and remove containers, networks, images, and volume and poetry env
# make reload-local-env # restart containers, networks, images, and volume and poetry env
# make load-fixtures-local # loads fixtures
# make local-test # runs tests
# make local-lint # runs linting tools
PIPENV_DOTENV := .pipenv.env
POETRY_DOTENV := .poetry.env
PYTHON_VERSION_FILE := .python-version
PYTHON_VERSION := $(cat $(PYTHON_VERSION_FILE) | xargs)
PIPENV_DOCKER_FILE := docker-compose.yml
......@@ -140,32 +140,31 @@ check-requirements-local:
build-local-env: check-requirements-local
docker-compose -f $(PIPENV_DOCKER_FILE) up -d --build --remove-orphans
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/bootstrap
sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/pipenv/bootstrap
.PHONY: build-local-env
populate-instance-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/populate-instance.sh
sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/pipenv/populate-instance.sh
.PHONY: populate-instance-local
load-fixtures-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/create-test-user.sh
sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/create-test-user.sh
.PHONY: load-fixtures-local
serve-api-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/server
sh with_env.sh $(POETRY_DOTENV) poetry run sh scripts/pipenv/server
.PHONY: serve-api-local
local-env: build-local-env populate-instance-local serve-api-local
.PHONY: local-env
shell-local-env:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv shell
sh with_env.sh $(POETRY_DOTENV) poetry shell
.PHONY: shell-local-env
destroy-local-env:
docker-compose -f $(PIPENV_DOCKER_FILE) down --volumes
docker-compose -f $(PIPENV_DOCKER_FILE) rm -f
pipenv --rm
.PHONY: destroy-local-env
reload-local-env: destroy-local-env local-env
......@@ -173,12 +172,12 @@ reload-local-env: destroy-local-env local-env
local-test:
@echo running tests...;
pipenv run pytest tests -v;
sh with_env.sh $(POETRY_DOTENV) poetry run pytest tests -v;
.PHONY: local-test
local-lint:
@echo running isort...;
pipenv run isort -rc -c -df;
sh with_env.sh $(POETRY_DOTENV) poetry run isort -rc -c -df .;
@echo running flake8...;
pipenv run flake8 --max-complexity 10 --ignore E501,D401
sh with_env.sh $(POETRY_DOTENV) poetry run flake8 --max-complexity 10 --ignore E501,D401
.PHONY: local-lint
This diff is collapsed.
......@@ -16,7 +16,7 @@ documents and search among them when needed!
Read more on the makefile.
- **Docker + Pipenv: Read more on the makefile**
- **Docker + Poetry: Read more on the makefile**
1. Run `make local-env MODE=test`
2. Follow [instructions](#tls---how-to-install-certificate) to install certificate.
......
......@@ -12,7 +12,7 @@ from celery import shared_task
from celery.exceptions import MaxRetriesExceededError, Reject
from flask import current_app
from invenio_files_processor.errors import InvalidProcessor
from invenio_files_processor.processors.tika import TikaProcessor
from invenio_files_processor.processors.tika.unpack import UnpackProcessor
from invenio_files_processor.proxies import current_processors
from invenio_files_rest.models import ObjectVersion
......@@ -30,7 +30,7 @@ def process_file_async(self, bucket_id, key_id):
current_app.logger.debug(f"Processing file {bucket_id}:{key_id}")
obj = ObjectVersion.get(bucket_id, key_id) # type: ObjectVersion
processor = current_processors.get_processor(name=TikaProcessor.id()) # type: TikaProcessor
processor = current_processors.get_processor(name=UnpackProcessor.id) # type: UnpackProcessor
processor.process(obj)
current_app.logger.debug(f"Processed file {bucket_id}:{key_id}")
......
# To be used with pipenv locally
# To be used with poetry locally
#
# Requires:
# - pipenv
# - poetry
# - python@3.6.0
version: "2.3"
......
This diff is collapsed.
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[tool.poetry]
name = "cern-search-rest-api"
version = "1.4.0-beta"
description = "CERN Search as a Service"
authors = ["Citadel Search <cernsearch.support@cern.ch>"]
license = "MIT"
[dev-packages]
flake8 = ">=3.7.9"
flake8-docstrings = ">=1.5.0"
isort = "==4.3.21"
pytest-dotenv = "*"
pytest-invenio = ">=1.2.0"
pytest-mock = ">=1.6.0"
[packages]
Flask = "*"
eventlet = "*"
gevent = ">=1.4.0"
[tool.poetry.dependencies]
Flask = "^1.1.2"
invenio-access = ">=1.4.1,<1.5.0"
invenio-accounts = ">=1.3.0,<1.4.0"
invenio-admin = ">=1.2.1,<1.3.0"
......@@ -23,34 +15,33 @@ invenio-assets = ">=1.1.3,<1.2.0"
invenio-base = ">=1.2.3,<1.3.0"
invenio-celery = ">=1.2.0,<1.3.0"
invenio-config = ">=1.0.3,<1.1.0"
invenio-db = {version = ">=1.0.5,<1.1.0",extras = ["postgresql", "versioning"]}
invenio-files-processor = {extras = ["tika"],git = "https://github.com/carantunes/invenio-files-processor.git",ref = "1.0.2-alpha"}
invenio-db = {version = ">=1.0.5,<1.1.0", extras = ["postgresql", "versioning"]}
invenio-files-processor = {extras = ["tika"], version = "^0.1.0"}
invenio-files-rest = ">=1.2.0,<1.3.0"
invenio-indexer = ">=1.1.1,<1.2.0"
invenio-i18n = "<1.3.0,>=1.2.0"
invenio-indexer = ">=1.1.1,<1.2.0"
invenio-jsonschemas = ">=1.1.0,<1.2.0"
invenio-logging = {extras = ["sentry-sdk"],version = ">=1.3.0,<1.4.0"}
invenio-logging = {version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk"]}
invenio-oauth2server = ">=1.2.0,<1.3.0"
invenio-oauthclient = "<1.4.0,>=1.3.5"
invenio-records = {extras = ["postgresql"],version = ">=1.3.1,<1.4.0"}
invenio-records = {version = ">=1.3.1,<1.4.0", extras = ["postgresql"]}
invenio-records-files = ">=1.2.1,<1.3.0"
invenio-records-rest = ">=1.7.1,<1.8.0"
invenio-rest = ">=1.2.1,<1.3.0"
invenio-search = {extras = ["elasticsearch7"],version = ">=1.3.1,<1.4.0"}
invenio-search = {version = ">=1.3.1,<1.4.0", extras = ["elasticsearch7"]}
invenio-theme = ">=1.1.4,<1.2.0"
ldap = "<1.1.0,>=1.0.2"
npm = ">=0.1.1"
pip = "*"
python-ldap = "<3.2.0,>=3.1.0"
raven = ">=6.9.0,<6.10.0"
redis = ">=2.10.0,<3.0.0"
uWSGI = ">=2.0.16"
uwsgi-tools = ">=1.1.1,<1.2.0"
wtforms = "<2.3.0"
tika = "==1.24"
python = "^3.6"
uWSGI = "^2.0.19"
uwsgi-tools = "^1.1.1"
[requires]
python_version = "3.6"
[tool.poetry.dev-dependencies]
flake8 = "^3.8.4"
flake8-docstrings = "^1.5.0"
isort = "^5.6.4"
pytest-dotenv = "^0.5.2"
pytest-invenio = "^1.4.0"
pytest-mock = "^3.3.1"
[pipenv]
allow_prereleases = true
[build-system]
requires = ["poetry-core>=1.0.0", "setuptools"]
build-backend = "poetry.core.masonry.api"
......@@ -13,7 +13,7 @@ readonly SYS_PREFIX=$(python -c "import sys; print(sys.prefix)")
readonly INVENIO_INSTANCE_PATH="$SYS_PREFIX/var/instance"
# Installs all packages specified in Pipfile (to avoid OS incompabilities)
pipenv install --dev --skip-lock
poetry install --no-root
# Install application code and entrypoints from 'setup.py'
pip install -e $SCRIPT_PATH/../..
......
......@@ -6,10 +6,10 @@ else
exit 1;
fi
# check pipenv installed
if [[ "$(pipenv --version)" =~ "pipenv, version" ]]; then
echo pipenv is installed
# check poetry installed
if [[ "$(poetry --version)" =~ "poetry, version" ]]; then
echo poetry is installed
else
echo pipenv is not installed. Aborting.
echo poetry is not installed. Aborting.
exit 1;
fi
......@@ -14,7 +14,7 @@ readonly TLS_DIR="$SCRIPT_PATH/../../nginx/tls"
export FLASK_ENV=development
# Start Worker
pipenv run celery worker -A invenio_app.celery -l DEBUG --autoscale=10,1 & pid_celery=$!
poetry run celery worker -A invenio_app.celery -l DEBUG --autoscale=10,1 & pid_celery=$!
# Start Server
invenio run \
......
......@@ -13,4 +13,4 @@ test = pytest
env_files =
# uncomment according to the environment being used
.env
#.pipenv.env
#.poetry.env
......@@ -10,8 +10,9 @@
from unittest.mock import patch
from celery.exceptions import MaxRetriesExceededError, Reject, Retry
from invenio_files_processor.processors.tika.unpack import UnpackProcessor
from cern_search_rest_api.modules.cernsearch.tasks import process_file_async
from invenio_files_processor.processors.tika import TikaProcessor
from pytest import raises
......@@ -34,7 +35,7 @@ class TestProcessFileAsync:
process_file_async('00000000-0000-0000-0000-000000000000', 'test.pdf')
object_version_get_mock.assert_called_once_with('00000000-0000-0000-0000-000000000000', 'test.pdf')
get_processor_mock.assert_called_once_with(name=TikaProcessor.id())
get_processor_mock.assert_called_once_with(name=UnpackProcessor.id)
get_processor_mock.return_value.process.assert_called_once_with(object_version)
@patch('cern_search_rest_api.modules.cernsearch.tasks.ObjectVersion.get', side_effect=Exception())
......
#!/usr/bin/env bash
# -*- coding: utf-8 -*-
#
# This file is part of CERN Search.
# Copyright (C) 2018-2019 CERN.
#
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
env_file="$1"
cmd="${@:2}"
bash -c "source $env_file && $cmd"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment