Commit d16ca131 authored by Carina Antunes's avatar Carina Antunes
Browse files

Merge branch '53/create-docker-compose' into 'dev'

[RFC][53] Create docker-compose environments

See merge request webservices/cern-search/cern-search-rest-api!51
parents 0cc5f976 53de0ae1
CERN_SEARCH_INSTANCE=cernsearch-test
CERN_SEARCH_INSTANCE_RATELIMIT=1000/hour
CERN_SEARCH_REMOTE_APP_RESOURCE=localhost
CERN_SEARCH_USE_EGROUPS='False'
CONTAINER_NAME=cern-search-rest-api
ENV=dev
FLASK_SKIP_DOTENV=1
INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://redis:6379/1
INVENIO_ADMIN_ACCESS_GROUPS=b
INVENIO_ADMIN_USER=test@example.com
INVENIO_ADMIN_VIEW_ACCESS_GROUPS=a
INVENIO_APP_ALLOWED_HOSTS=['localhost', 'nginx']
INVENIO_CACHE_REDIS_HOST=cache
INVENIO_CACHE_REDIS_URL=redis://redis:6379/0
INVENIO_CERN_APP_CREDENTIALS={'consumer_key':'bah'}
INVENIO_CERN_APP_CREDENTIALS_CONSUMER_KEY=xxx
INVENIO_COLLECT_STORAGE=flask_collect.storage.file
INVENIO_INDEXER_DEFAULT_DOC_TYPE=test-doc_v0.0.2
INVENIO_INDEXER_DEFAULT_INDEX=cernsearch-test-test-doc_v0.0.2
INVENIO_LOGGING_CONSOLE='False'
INVENIO_LOGGING_CONSOLE_LEVEL=WARNING
INVENIO_RATELIMIT_STORAGE_URL='redis://cache:6379/3'
INVENIO_SEARCH_ELASTIC_HOSTS=elasticsearch
INVENIO_SEARCH_INDEX_PREFIX=cernsearch-
INVENIO_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://uservice:itsjust1234@postgres/uservice
INVENIO_THEME_FRONTPAGE_TITLE='CERN Search DEV'
INVENIO_THEME_LOGO=/images/cernsearchicon.png
INVENIO_THEME_LOGO_ADMIN=/images/cernsearchicon.png
INVENIO_THEME_SITENAME='CERN Search DEV'
POSTGRESQL_DATABASE=uservice
POSTGRESQL_PASSWORD=itsjust1234
POSTGRESQL_USER=uservice
......@@ -16,6 +16,7 @@ secrets/
*.key
*.crt
*.csr
# Dump files
......
......@@ -59,7 +59,7 @@ build_dev_version:
- docker-image-build
script: 'echo "Building Dev/QA Docker image..."'
variables:
BUILD_ARG: build_devel=1
BUILD_ARG: build_env=dev
########## PROD ##########
build_prod_version:
......@@ -70,6 +70,8 @@ build_prod_version:
tags:
- docker-image-build
script: 'echo "Building Prod Docker image..."'
variables:
BUILD_ARG: build_env=prod
########## ALL ##########
### When building tags, use the git tag as the docker tag of the image
......
CERN_SEARCH_INSTANCE=cernsearch-test
CERN_SEARCH_INSTANCE_RATELIMIT=1000/hour
CERN_SEARCH_REMOTE_APP_RESOURCE=localhost
CERN_SEARCH_USE_EGROUPS='False'
CONTAINER_NAME=cern-search-rest-api
ENV=dev
FLASK_SKIP_DOTENV=1
INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://localhost:6379/1
INVENIO_ADMIN_ACCESS_GROUPS=b
INVENIO_ADMIN_USER=test@example.com
INVENIO_ADMIN_VIEW_ACCESS_GROUPS=a
INVENIO_APP_ALLOWED_HOSTS=['localhost']
INVENIO_CACHE_REDIS_HOST=localhost
INVENIO_CACHE_REDIS_URL=redis://localhost:6379/0
INVENIO_CERN_APP_CREDENTIALS={'consumer_key':'bah'}
INVENIO_CERN_APP_CREDENTIALS_CONSUMER_KEY=xxx
INVENIO_COLLECT_STORAGE=flask_collect.storage.file
INVENIO_INDEXER_DEFAULT_DOC_TYPE=test-doc_v0.0.1
INVENIO_INDEXER_DEFAULT_INDEX=cernsearch-test-test-doc_v0.0.1
INVENIO_LOGGING_CONSOLE='False'
INVENIO_LOGGING_CONSOLE_LEVEL=WARNING
INVENIO_RATELIMIT_STORAGE_URL='redis://localhost:6379/3'
INVENIO_SEARCH_ELASTIC_HOSTS=localhost
INVENIO_SEARCH_INDEX_PREFIX=cernsearch-
INVENIO_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://uservice:itsjust1234@localhost/uservice
INVENIO_THEME_FRONTPAGE_TITLE='CERN Search DEV'
INVENIO_THEME_LOGO=/images/cernsearchicon.png
INVENIO_THEME_LOGO_ADMIN:=/images/cernsearchicon.png
INVENIO_THEME_SITENAME='CERN Search DEV'
POSTGRESQL_DATABASE=uservice
POSTGRESQL_PASSWORD=itsjust1234
POSTGRESQL_USER=uservice
......@@ -7,16 +7,15 @@
# under the terms of the MIT License; see LICENSE file for more details.
# Use CentOS7:
FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:f813c09e1f92c96f7eb71b2d43c076388be57680
ARG build_devel
ENV DEVEL=$build_devel
FROM gitlab-registry.cern.ch/webservices/cern-search/cern-search-rest-api/cern-search-rest-api-base:7ad59c5ed72e1672454acf3bed61e45f8df132d6
ARG build_env
# CERN Search installation
WORKDIR /${WORKING_DIR}/src
ADD . /${WORKING_DIR}/src
# If env is development, install development dependencies
RUN if [ -n "${DEVEL-}" ]; then pip install -r requirements-devel.txt; fi
RUN if [ "$build_env" != "prod" ]; then pipenv install --system --ignore-pipfile --deploy --dev; fi
# Install CSaS
RUN pip install -e .
......
################### Docker development helpful directives ####################
#
# Usage:
# make logs # displays log outputs from running services
# make build-env # build environment, create and start containers
# make env # build environment, load fixtures and enters shell
# make destroy-env # stop and remove containers, networks, images, and volume
# make reload-env # restart containers, networks, images, and volume
# make shell-env # start bash inside service
# make root-shell-env # start bash inside service as root
# make load-fixtures # loads fixtures
# make populate-instance # create database, tables and indeces
# make generate-certificates # generate nginx certificates
# make test # run test
SERVICE_NAME := cern-search-api
DOCKER_FILE := docker-compose.full.yml
build-env:
docker-compose -f $(DOCKER_FILE) up -d --build --remove-orphans
.PHONY: env
logs:
docker-compose -f $(DOCKER_FILE) logs -f
.PHONY: logs
populate-instance:
docker-compose -f $(DOCKER_FILE) exec $(SERVICE_NAME) /bin/bash -c \
"sh /opt/invenio/src/scripts/populate-instance.sh"
.PHONY: load-fixtures
load-fixtures:
docker-compose -f $(DOCKER_FILE) exec $(SERVICE_NAME) /bin/bash -c \
"sh /opt/invenio/src/scripts/create-test-user.sh"
.PHONY: load-fixtures
destroy-env:
docker-compose -f $(DOCKER_FILE) down --volumes
docker-compose -f $(DOCKER_FILE) rm -f
.PHONY: destroy-env
reload-env: destroy-env env
.PHONY: reload-env
shell-env:
docker-compose -f $(DOCKER_FILE) exec $(SERVICE_NAME) /bin/bash
.PHONY: shell-env
root-shell-env:
docker-compose -f $(DOCKER_FILE) exec -u root $(SERVICE_NAME) /bin/bash
.PHONY: root-shell-env
env: generate-certificates build-env populate-instance load-fixtures shell-env
.PHONY: env
generate-certificates:
sh scripts/gen-cert.sh
.PHONY: generate-certificates
test:
make pytest
.PHONY: test
################### Local development helpful directives ####################
################### (pipenv + docker) ####################
#
# Usage:
# make logs # displays log outputs from running services
# make build-local-env # build pipenv, create and start containers
# make check-requirements-local # check requirements
# make local-env # build virtual environment, load fixtures and starts api
# make populate-instance-local # create database, tables and indices
# make serve-api-local # start serving api
# make shell-local-env # start bash inside pipenv
# make destroy-local-env # stop and remove containers, networks, images, and volume and pipenv
# make reload-local-env # restart containers, networks, images, and volume and pipenv
# make load-fixtures-local # loads fixtures
PIPENV_DOTENV := .pipenv.env
PYTHON_VERSION_FILE := .python-version
PYTHON_VERSION := $(shell cat $(PYTHON_VERSION_FILE) | xargs)
PIPENV_DOCKER_FILE := docker-compose.yml
check-requirements-local:
PYTHON_VERSION=$(PYTHON_VERSION) sh scripts/pipenv/requirements.sh
.PHONY: check-requirements-local
build-local-env: check-requirements-local
docker-compose -f $(PIPENV_DOCKER_FILE) up -d --build --remove-orphans
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/bootstrap
.PHONY: build-local-env
populate-instance-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/populate-instance.sh
.PHONY: populate-instance-local
load-fixtures-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/create-test-user.sh
.PHONY: load-fixtures-local
serve-api-local:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv run sh scripts/pipenv/server
.PHONY: serve-api-local
local-env: build-local-env populate-instance-local serve-api-local
.PHONY: local-env
shell-local-env:
PIPENV_DOTENV_LOCATION=$(PIPENV_DOTENV) pipenv shell
.PHONY: shell-local-env
destroy-local-env:
docker-compose -f $(PIPENV_DOCKER_FILE) down --volumes
docker-compose -f $(PIPENV_DOCKER_FILE) rm -f
pipenv --rm
.PHONY: destroy-local-env
reload-local-env: destroy-local-env local-env
.PHONY: reload-local-env
......@@ -4,6 +4,10 @@ url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
pytest = "*"
flake8 = "*"
flake8-docstrings = "*"
isort = "==4.3.21"
[packages]
invenio-access = ">=1.0.0,<1.1.0"
......@@ -33,4 +37,4 @@ Flask = "*"
uWSGI = ">=2.0.16"
[requires]
python_version = "3.4"
python_version = "3.6"
This diff is collapsed.
version: "3.7"
services:
cern-search-api:
build:
context: .
args:
build_env: ${ENV}
container_name: ${CONTAINER_NAME}
ports:
- "5000:5000"
depends_on:
- elasticsearch
- redis
- postgres
networks:
- default
env_file:
- .env
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4
ports:
- "9200:9200"
- "9300:9300"
environment:
discovery.type: single-node
networks:
- default
volumes:
- es-data:/usr/share/elasticsearch/data
kibana:
image: docker.elastic.co/kibana/kibana-oss:6.2.4
depends_on:
- elasticsearch
ports:
- 5601:5601
networks:
- default
redis:
image: redis:5.0.6
networks:
- default
postgres:
image: centos/postgresql-95-centos7
environment:
- POSTGRESQL_USER
- POSTGRESQL_PASSWORD
- POSTGRESQL_DATABASE
volumes:
- pgsql-data:/var/lib/pgsql/data
networks:
- default
ports:
- 5432:5432
nginx:
image: nginx:1.17.4-alpine
ports:
- "8080:8080"
depends_on:
- cern-search-api
volumes:
- ./nginx/tls:/etc/nginx/tls
- nginx-cache:/var/cache/nginx
- nginx-run:/var/run
- ./nginx/nginx.conf:/etc/nginx/nginx.conf
networks:
default:
volumes:
es-data:
pgsql-data:
nginx-run:
nginx-cache:
nginx-tls:
# To be used with pipenv locally
#
# Requires:
# - pipenv
# - python@3.6.0
version: "3.7"
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4
ports:
- "9200:9200"
- "9300:9300"
environment:
discovery.type: single-node
networks:
- default
volumes:
- es-data:/usr/share/elasticsearch/data
kibana:
image: docker.elastic.co/kibana/kibana-oss:6.2.4
depends_on:
- elasticsearch
ports:
- 5601:5601
networks:
- default
redis:
image: redis:5.0.6
networks:
- default
postgres:
image: centos/postgresql-95-centos7
environment:
- POSTGRESQL_USER
- POSTGRESQL_PASSWORD
- POSTGRESQL_DATABASE
volumes:
- pgsql-data:/var/lib/pgsql/data
networks:
- default
ports:
- 5432:5432
networks:
default:
volumes:
es-data:
pgsql-data:
#!/usr/bin/env bash
# -*- coding: utf-8 -*-
#
# This file is part of CERN Search.
......@@ -5,3 +6,7 @@
#
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
invenio users create test@example.com --password test1234 --active
export API_TOKEN=$(invenio tokens create -n test -u test@example.com)
echo "API TOKEN: $API_TOKEN"
......@@ -7,9 +7,11 @@
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
openssl genrsa -des3 -passout pass:x -out nginx.pass.key 2048
openssl rsa -passin pass:x -in nginx.pass.key -out nginx.key
rm nginx.pass.key
openssl req -new -key nginx.key -out nginx.csr \
-subj "/C=CH/ST=Geneve/L=Geneve/O=CERN/OU=IT Department/CN=Search as a Service"
openssl x509 -req -days 365 -in nginx.csr -signkey nginx.key -out nginx.crt
readonly SCRIPT_PATH=$(dirname $0)
readonly TLS_DIR="$SCRIPT_PATH/../nginx/tls"
readonly KEY="tls.key"
readonly CRT="tls.crt"
openssl req -x509 -nodes -newkey rsa:4096 \
-subj '/C=CH/ST=Geneve/L=Geneve/O=CERN/OU=IT Department/CN=Search as a Service' \
-keyout "$TLS_DIR/$KEY" -out "$TLS_DIR/$CRT"
......@@ -7,7 +7,8 @@
# CERN Search is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
location=$(pip show invenio-oauthclient | grep Location | awk '{print $2}')
readonly LOCATION=$(pip show invenio-oauthclient | grep Location | awk '{print $2}')
readonly SCRIPT_PATH=$(dirname $0)
rm -f ${location}/invenio_oauthclient/contrib/cern.py
cp /${WORKING_DIR}/src/scripts/patch/cern.py ${location}/invenio_oauthclient/contrib/cern.py
rm -f ${LOCATION}/invenio_oauthclient/contrib/cern.py
cp ${SCRIPT_PATH}/cern.py ${LOCATION}/invenio_oauthclient/contrib/cern.py
#!/usr/bin/env bash
# -*- coding: utf-8 -*-
#
# Copyright (C) 2019 CERN.
#
# CERN Search is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.
set -e
readonly SCRIPT_PATH=$(dirname "$0")
readonly SYS_PREFIX=$(python -c "import sys; print(sys.prefix)")
readonly INVENIO_INSTANCE_PATH="$SYS_PREFIX/var/instance"
# Installs all packages specified in Pipfile.lock
pipenv sync --dev
# Install application code and entrypoints from 'setup.py'
pip install -e $SCRIPT_PATH/../..
# Patch auth
sh $SCRIPT_PATH/../patch/oauth_patch.sh
# Build assets
invenio collect -v
invenio webpack buildall
# Move static files to instance folder
cp ${SCRIPT_PATH}/../../static/images/cernsearchicon.png ${INVENIO_INSTANCE_PATH}/static/images/
# check python version
if [[ "$(python --version)" =~ "Python ${PYTHON_VERSION}" ]]; then
echo Python ${PYTHON_VERSION} is installed
else
echo Python ${PYTHON_VERSION} is not installed. Aborting.
exit 1;
fi
# check pipenv installed
if [[ "$(pipenv --version)" =~ "pipenv, version" ]]; then
echo pipenv is installed
else
echo pipenv is not installed. Aborting.
exit 1;
fi
#!/usr/bin/env bash
# -*- coding: utf-8 -*-
#
# Copyright (C) 2019 CERN.
#
# CERN Search is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.
set -e
readonly SCRIPT_PATH=$(dirname "$0")
readonly TLS_DIR="$SCRIPT_PATH/../../nginx/tls"
export FLASK_ENV=development
# Start Server
invenio run \
--cert "$TLS_DIR/tls.crt" \
--key "$TLS_DIR/tls.key" & pid_server=$!
trap 'kill $pid_server &>/dev/null' EXIT
wait $pid_server
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment