Skip to content
Snippets Groups Projects
Commit a4aa83db authored by Konstantinos Samaras-Tsakiris's avatar Konstantinos Samaras-Tsakiris
Browse files

Move drupal-runtime image building here

parent 27c5fa8a
Branches
Tags
No related merge requests found
Showing
with 700 additions and 0 deletions
# This Dockerfile builds the *sitebuilder-base* image that deploys the CERN Drupal distribution
# and serves as a basis for all Drupal websites.
# This image will also be used to server PHP container in the deployment
ARG PHP_VERSION
FROM php:${PHP_VERSION}
# COMPOSER_VERSION and DRUSH_VERSION are passed by Gitlab as an argument through the version files in drupalVersions directory
ARG COMPOSER_VERSION
ARG DRUSH_VERSION
LABEL io.openshift.s2i.scripts-url="image:///usr/libexec/s2i" \
io.s2i.scripts-url="image:///usr/libexec/s2i" \
io.k8s.description="Drupal Site Builder s2i build & php-fpm infra" \
io.k8s.display-name="Drupal Site Builder + php-fpm" \
io.openshift.tags="builder,drupal,php,php-fpm" \
maintainer="Drupal Admins <drupal-admins@cern.ch>"
# from https://www.drupal.org/docs/8/system-requirements/drupal-8-php-requirements
# from https://github.com/docker-library/docs/blob/master/php/README.md#supported-tags-and-respective-dockerfile-links
# install some utils
RUN apk --update add \
# Some composer packages need git
git \
patch \
curl \
gettext \
zip \
unzip \
mysql-client \
jq \
tzdata \
rsync
# Configured timezone.
ENV TZ=Europe/Zurich
RUN touch /usr/share/zoneinfo/$TZ \
&& cp /usr/share/zoneinfo/$TZ /etc/localtime \
&& echo $TZ > /etc/timezone && \
apk del tzdata \
&& rm -rf /var/cache/apk/*
# PHP FPM
RUN set -eux; \
\
apk add --no-cache --virtual .build-deps autoconf g++ make \
coreutils \
freetype-dev \
libjpeg-turbo-dev \
libpng-dev \
libzip-dev \
mysql-client \
; \
\
rm -rf /tmp/pear \
; \
\
docker-php-ext-configure gd \
--with-freetype-dir=/usr/include \
--with-jpeg-dir=/usr/include \
--with-png-dir=/usr/include \
; \
\
docker-php-ext-install -j "$(nproc)" \
gd \
opcache \
pdo_mysql \
zip \
; \
\
runDeps="$( \
scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
| tr ',' '\n' \
| sort -u \
| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
)"; \
apk add --virtual .drupal-phpexts-rundeps $runDeps; \
apk del .build-deps
# RUN mkdir -p /tmp/drush
# WORKDIR /tmp/drush
RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer --version=${COMPOSER_VERSION}
# RUN composer clearcache; composer require drush/drush ${DRUSH_VERSION}; composer install;
# ENV PATH=$PATH:/tmp/drush/vendor/bin
COPY ./s2i/bin/ ./hooks/ /usr/libexec/s2i/
COPY ./php-fpm/fix-permissions /fix-permissions
RUN chmod -R +x /usr/libexec/s2i/; \
chmod +x /fix-permissions
# DRUPAL
# Path configuration
# Set up drupal site folder and drupal operations folder
RUN mkdir -p /app; \
mkdir -p /operations
ENV DRUPAL_APP_DIR /app
ENV DRUPAL_OPERATIONS_DIR /operations
# The following folders are copied from the CI environment during image build
COPY cern-drupal-distribution ${DRUPAL_APP_DIR}
# Add scripts for Drupal operations
COPY drupal-operations-scripts ${DRUPAL_OPERATIONS_DIR}
RUN chmod +x ${DRUPAL_OPERATIONS_DIR}/*
WORKDIR ${DRUPAL_APP_DIR}
# Create necessary folders from composer
RUN rm -rf .git; \
mkdir ${DRUPAL_APP_DIR}/.composer; \
/fix-permissions ${DRUPAL_APP_DIR}/.composer
ENV COMPOSER_HOME=${DRUPAL_APP_DIR} COMPOSER_CACHE_DIR=${DRUPAL_APP_DIR}/.composer
# Do not run Composer as root/super user! See https://getcomposer.org/root for details
# Set up drupal minimum stack
ENV COMPOSER_MEMORY_LIMIT=-1
RUN composer install --optimize-autoloader -v
ENV PATH=$PATH:${DRUPAL_APP_DIR}/vendor/bin
# Clean-up composer installation
# Rename composer.json to composer.admins.json so that when user injects its composer,
# there is no conflict between then.
RUN cp ${DRUPAL_APP_DIR}/composer.json ${DRUPAL_APP_DIR}/composer.admins.json
# Put symlinks for profiles/themes/modules to a path where old Drupal sites expect to find them
RUN ln -s ${DRUPAL_APP_DIR}/web/profiles/contrib/cern-install-profiles/cern ${DRUPAL_APP_DIR}/web/profiles/cern && \
ln -s ${DRUPAL_APP_DIR}/web/profiles/contrib/cern-install-profiles/easystart ${DRUPAL_APP_DIR}/web/profiles/easystart && \
ln -s ${DRUPAL_APP_DIR}/web/themes/custom/cern-theme ${DRUPAL_APP_DIR}/web/themes/custom/cernclean &&\
ln -s ${DRUPAL_APP_DIR}/web/themes/custom/cern-base-theme ${DRUPAL_APP_DIR}/web/themes/custom/cernbase
# Add extra configurations
# At this point, composer has created the required settings.php through:
# post-update-cmd: DrupalProject\composer\ScriptHandler::createRequiredFiles
# Overwrite settings.php with ours.
# - settings.php
ADD ./configuration/sitebuilder/settings.php ${DRUPAL_APP_DIR}/web/sites/default/settings.php
# Remove ${DRUPAL_APP_DIR}/web/sites/default/{files, private, modules, themes}, preparing it to be symbolic link;
RUN rm -rf ${DRUPAL_APP_DIR}/web/sites/default/files; \
rm -rf ${DRUPAL_APP_DIR}/web/sites/default/private; \
rm -rf ${DRUPAL_APP_DIR}/web/sites/default/modules; \
rm -rf ${DRUPAL_APP_DIR}/web/sites/default/themes
# Explicity create the site configuration dir as configured in settings-d8.php#L17 file
RUN mkdir -p config/sync; \
/fix-permissions ${DRUPAL_APP_DIR}
# Add extra configurations
# At this point, composer has created the required settings.php through post-update-cmd: DrupalProject\composer\ScriptHandler::createRequiredFiles
# Overwrite settings.php with ours.
# - php-fpm
ADD ./php-fpm/config/php-fpm/ /usr/local/etc/php-fpm.d/
# - opcache
ADD ./php-fpm/config/opcache/ /usr/local/etc/php/conf.d/
ADD ./php-fpm/run-php-fpm.sh /
RUN chmod +x /run-php-fpm.sh
ENV DRUPAL_SHARED_VOLUME /drupal-data
RUN ln -s ${DRUPAL_SHARED_VOLUME}/files ${DRUPAL_APP_DIR}/web/sites/default/files && \
ln -s ${DRUPAL_SHARED_VOLUME}/private ${DRUPAL_APP_DIR}/web/sites/default/private && \
ln -s ${DRUPAL_SHARED_VOLUME}/modules ${DRUPAL_APP_DIR}/web/sites/default/modules && \
ln -s ${DRUPAL_SHARED_VOLUME}/themes ${DRUPAL_APP_DIR}/web/sites/default/themes
# The directory sites/default is not protected from modifications and poses a security risk.
# Change the directory's permissions to be non-writable is needed.
RUN chmod -R 555 ${DRUPAL_APP_DIR}/web/sites/default
RUN chmod 444 ${DRUPAL_APP_DIR}/web/sites/default/settings.php
CMD ["/usr/libexec/s2i/usage"]
# Drupal Runtime
This repository defines the runtime environment of the Drupal infrastructure,
orchestrated by the [Drupal Site Operator](https://gitlab.cern.ch/drupal/paas/drupalsite-operator/).
It includes:
- [drupalVersions](drupalVersions): list of different drupal version files, which also defines configuration for different versions
- [runtime configuration](configuration): configmaps for the server software stack (nginx/php)
- Dockerfiles and source-to-image (s2i) recipes to build the images that eventually run on the infrastructure
The images in this repo are the sources of BuildConfigs that generate the actual runtime images for every Drupal site
based also on extra configuration injected by the users.
## Supporting a new `drupalVersion`
To let DrupalSites use a new value for `drupalVersion`, create a new file in [`drupalVersions/${drupalVersions}`](drupalVersions)
folder.
Example format:
```
variables:
drupalDistroRefspec: '9.1.x'
phpVersion: '7.3.23-fpm-alpine3.12'
nginxVersion: '1.18.0'
composerVersion: '2.0.0'
drushVersion: '10.3.6'
```
Notes:
- the name of the file itself will be used as the value for `drupalVersion`
- the indentation is important, since this will be injected
- all the files under `drupalVersions` are what generates the "registry" of all the website configurations supported by the Drupal infrastructure.
## CI & Registry
Drupal runtime CI builds 2 images for every commit in this registry, for a given "Drupal version" from the files in [drupalVersions](drupalVersions).
- [nginx](./nginx/Dockerfile)
- Based on upstream Nginx and configures nginx for openshift
- [site-builder-base](./Dockerfile)
- Based on the `php` upstream image, installs required php dependencies, copies the s2i scripts and also builds the basic drupal project cloned from [cern-drupal-distribution](https://gitlab.cern.ch/drupal/paas/cern-drupal-distribution)
The [generate-pipeline.sh](generate-pipeline.sh) file creates stages on the fly from each of the version files and upon creation of the stages, the two image build stages namely `Build-nginx` and `Build-sitebuilder-base` run in parallel
![Pipeline](./pipeline.png)
### <a id="tags"></a> Image Tags
commit location | tag
--- | ---
master | `<drupalVersion-file-name>`
other branch/ MR | `<drupalVersion-file-name>-dev-<commit-short-sha>`
Every time a commit is pushed, CI generates 1 tag for each of the 2 images
**for each file in `drupalVersions` directory**.
CI not running on master also appends the commit sha to the generated tags.
This creates a lot of tags, but insulates testing from production.
The image tags created and the registry to which it is pushed, will be echoed at the end of a successful pipeline
#### Creating new tags
Simply push a commit to this repo.
To create "production" tags (ie. `<drupalVersion>` instead of `<drupalVersion>-dev-<commit-sha>`, merge to master.
### Build arguments and CI variables
The following table lists the build arguments and variables used in the Dockerfiles and the CI.
| Variable | Description |
| ----------- | ----------- |
| COMPOSER_VERSION | Controls the composer version to be installed |
| NGINX_VERSION | Controls the nginx image version to be pulled |
| PHP_VERSION | Controls the php image version to be pulled |
| DRUSH_VERSION | Controls the drush version to be used |
| DRUPAL_APP_DIR | The directory that hosts the drupal code |
| DRUPAL_SHARED_VOLUME | Mount path of persistent volume |
| SITE_BUILDER_DIR | This directory points to the location, where the drupal code will be available to be copied from in the Openshift buildConfig |
| CI_COMMIT_REF_NAME | Gitlab tag/ branch name |
## Configuration
### PHP
Global default configuration in `php-fpm/config` is baked into the sitebuilder-base image
QOS-specific config in each of the `configuration/qos*/php-fpm.conf` directories is mounted as configmap
### Nginx
Global default configuration in `nginx/config` is baked into the nginx image
QOS-specific config in each of the `configuration/qos*/nginx.conf` directories is mounted as configmap
## Architecture of a Drupal project
This diagram shows an exploded view of the parts of a Drupal site and the images they need.
These images are built by this repo.
![Architecture Diagram](https://gitlab.cern.ch/paas-tools/okd4-install/-/raw/master/docs/cluster-flavours/drupal/drupal-design.svg)
Ref: https://gitlab.cern.ch/paas-tools/okd4-install/-/tree/master/docs/cluster-flavours/drupal
#!/bin/sh
# Change working directory to the drupal code
cd /app
# Check if Drupal site is installed
drush status bootstrap | grep -q Successful
# Make sure we get a successful response
if [ "$?" -ne "0" ]; then
echo "Drupal is not installed" >&2
exit 1
fi
exit 0
#!/bin/sh
# Change working directory to the drupal code
cd /app
# Check updb status
drush updatedb-status --no-entity-updates --format=json 2>/dev/null | jq '. | length'
#!/bin/sh
# Change working directory to the drupal code
cd /app
# Clean cache
drush cr -q
#!/bin/sh
usage() {
echo "Usage: $0 --filename <filename.sql>" 1>&2;
exit 1;
}
# Options
ARGS=$(getopt -o 'f:' --long 'filename:' -- "$@") || exit 1
eval "set -- $ARGS"
while true; do
case "$1" in
(-f|--filename)
export FILENAME="$2"; shift 2;;
(--) shift; break;;
(*) usage;;
esac
done
[[ -z "$FILENAME" ]] && usage
# Change working directory to the drupal code
cd /app
# Clone data from source to destination
rsync -rv /drupal-data-source/ /drupal-data
/operations/database-restore.sh -f $FILENAME
/operations/clear-cache.sh
#!/bin/sh
usage() {
echo "Usage: $0 --filename <filename.sql>" 1>&2;
exit 1;
}
# Options
ARGS=$(getopt -o 'f:' --long 'filename:' -- "$@") || exit 1
eval "set -- $ARGS"
while true; do
case "$1" in
(-f|--filename)
export FILENAME="$2"; shift 2;;
(--) shift; break;;
(*) usage;;
esac
done
[[ -z "$FILENAME" ]] && usage
# Change working directory to the drupal code
cd /app
# Database backup
echo "Backing up database to" $FILENAME
drush sql-dump > /drupal-data/$FILENAME
#!/bin/sh
usage() {
echo "Usage: $0 --filename <filename.sql>" 1>&2;
exit 1;
}
# Options
ARGS=$(getopt -o 'f:' --long 'filename:' -- "$@") || exit 1
eval "set -- $ARGS"
while true; do
case "$1" in
(-f|--filename)
export FILENAME="$2"; shift 2;;
(--) shift; break;;
(*) usage;;
esac
done
[[ -z "$FILENAME" ]] && usage
# Change working directory to the drupal code
cd /app
# Database drop
echo "Dropping database"
drush sql-drop -y
# Database restore
echo "Restoring database from" $FILENAME
`drush sql-connect` < /drupal-data/$FILENAME
#!/bin/sh
# Change working directory to the drupal code
cd /app
# Disable maintenance mode
echo "Disabling maintenance mode"
drush state:set system.maintenance_mode 0 --input-format=integer
drush cache:rebuild -q
#!/bin/sh
# Change working directory to the drupal code
cd /app
# Enable maintenance mode
echo "Enabling maintenance mode"
drush state:set system.maintenance_mode 1 --input-format=integer
drush cache:rebuild -q
#!/bin/sh
# Change working directory to the drupal code
cd /app
# Run update db
echo "Running drush updatedb"
/operations/enable-maintenance-mode.sh && drush updatedb -y -q
/operations/disable-maintenance-mode.sh
#!/bin/sh
set -exu
# Change working directory to the drupal code
cd /app
# Drop tables
drush sql:drop -y
# Install Drupal site
echo "Installing Drupal site"
drush site-install cern -y --config-dir=../config/sync --account-name=admin
drush cr
if [ "$?" -ne "0" ]; then
drush cr
fi
ARG NGINX_VERSION
FROM nginx:${NGINX_VERSION}
LABEL io.k8s.description="Drupal managed infra nginx" \
io.k8s.display-name="Drupal 8 Managed Infra nginx" \
io.openshift.tags="managed,drupal,nginx" \
maintainer="Drupal Admins <drupal-admins@cern.ch>"
ENV DRUPAL_SHARED_VOLUME /drupal-data
# Nginx default configuration
ADD config/default.conf /etc/nginx/conf.d/default.conf
ADD config/webdav.conf /etc/nginx/conf.d/webdav.conf
ADD config/nginx.conf /etc/nginx/nginx.conf
### n.b.: https://www.redpill-linpro.com/sysadvent/2017/12/10/jekyll-openshift.html
RUN \
mkdir -p /etc/nginx/ /var/cache/nginx /var/lib/nginx /var/log/nginx /var/tmp/nginx/ && \
chgrp -R 0 /etc/nginx/ /var/cache/nginx /var/lib/nginx /var/log/nginx /var/tmp/nginx/ && \
chmod -R g=u /etc/nginx/ /var/cache/nginx /var/lib/nginx /var/log/nginx /var/tmp/nginx/ && \
sed -i -e '/^user/d' /etc/nginx/nginx.conf
ADD run-nginx.sh /
RUN chmod +x /run-nginx.sh
ENTRYPOINT ["/run-nginx.sh"]
upstream php {
server unix:/var/run/drupal.sock;
}
server {
#listen 8080 ssl;
listen 8080;
#ssl_certificate /etc/ssl/certs/ca-certificates.crt;
#ssl_certificate_key /etc/ssl/certs/ca-cert-COMODO_Certification_Authority.pem;
#ssl_session_cache shared:SSL:20m;
#ssl_session_timeout 4h;
root /var/run/app/web;
# Disables specifying the port in absolute redirects
port_in_redirect off;
location = /favicon.ico {
log_not_found off;
access_log off;
}
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
# Status page of PHP. Commenting to avoid access from outside the cluster
location = /_site/_php-fpm-status {
include fastcgi_params;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_pass php;
allow all;
}
# Very rarely should these ever be accessed outside of your lan
location ~* \.(txt|log)$ {
allow 192.168.0.0/16;
deny all;
}
# https://drupal.stackexchange.com/questions/192151/cannot-install-any-theme
rewrite ^/core/authorize.php/core/authorize.php(.*)$ /core/authorize.php$1;
location ~ \..*/.*\.php$ {
return 403;
}
location ~ ^/sites/.*/private/ {
return 403;
}
# Block access to scripts in site files directory
location ~ ^/sites/[^/]+/files/.*\.php$ {
deny all;
}
# Allow "Well-Known URIs" as per RFC 5785
location ~* ^/.well-known/ {
allow all;
}
# Block access to "hidden" files and directories whose names begin with a
# period. This includes directories used by version control systems such
# as Subversion or Git to store control files.
location ~ (^|/)\. {
return 403;
}
location / {
# try_files $uri @rewrite; # For Drupal <= 6
try_files $uri /index.php?$query_string; # For Drupal >= 7
}
location @rewrite {
rewrite ^/(.*)$ /index.php?q=$1;
}
# Don't allow direct access to PHP files in the vendor directory.
location ~ /vendor/.*\.php$ {
deny all;
return 404;
}
# In Drupal 8, we must also match new paths where the '.php' appears in
# the middle, such as update.php/selection. The rule we use is strict,
# and only allows this pattern with the update.php front controller.
# This allows legacy path aliases in the form of
# blog/index.php/legacy-path to continue to route to Drupal nodes. If
# you do not have any paths like that, then you might prefer to use a
# laxer rule, such as:
# location ~ \.php(/|$) {
# The laxer rule will continue to work if Drupal uses this new URL
# pattern with front controllers other than update.php in a future
# release.
location ~ '\.php$|^/update.php' {
fastcgi_split_path_info ^(.+?\.php)(|/.*)$;
# Security note: If you're running a version of PHP older than the
# latest 5.3, you should have "cgi.fix_pathinfo = 0;" in php.ini.
# See http://serverfault.com/q/627903/94922 for details.
include fastcgi_params;
# Block httpoxy attacks. See https://httpoxy.org/ .
fastcgi_param HTTP_PROXY "";
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_param QUERY_STRING $query_string;
fastcgi_intercept_errors on;
# PHP 5 socket location.
#fastcgi_pass unix:/var/run/php5-fpm.sock;
# PHP 7 socket location.
fastcgi_pass php;
}
# Fighting with Styles? This little gem is amazing.
# location ~ ^/sites/.*/files/imagecache/ { # For Drupal <= 6
location ~ ^/sites/.*/files/styles/ { # For Drupal >= 7
try_files $uri @rewrite;
}
# Handle private files through Drupal. Private file's path can come
# with a language prefix.
location ~ ^(/[a-z\-]+)?/system/files/ { # For Drupal >= 7
try_files $uri /index.php?$query_string;
}
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg)$ {
try_files $uri @rewrite;
expires max;
log_not_found off;
}
location ^~ /simplesaml {
alias /app/vendor/simplesamlphp/simplesamlphp/www;
location ~ ^(?<prefix>/simplesaml)(?<phpfile>.+?\.php)(?<pathinfo>/.*)?$ {
include fastcgi_params;
fastcgi_pass php;
fastcgi_index index.php;
fastcgi_split_path_info ^(.+?\.php)(/.+)$;
fastcgi_param SCRIPT_FILENAME $document_root$phpfile;
fastcgi_param PATH_INFO $pathinfo if_not_empty;
}
}
# We want to make the redirection https://sitename.web.cern.ch/sitename/sites/something to https://sitename.web.cern.ch/sites/default/.
# This location matches any url containing /sites/sitename/.../file and redirects to /sites/default/.../file.
# It omits redirection for urls containing /sites/default.
# To develop this section the following links were used:
# - http://nginx.org/en/docs/http/ngx_http_rewrite_module.html#rewrite
# - https://www.nginx.com/blog/creating-nginx-rewrite-rules/
location ~ "^/sites/((?!default)).*$" {
rewrite ^(/sites)/[^\/]+/(.*)$ $1/default/$2 permanent;
}
}
pid /var/run/nginx.pid;
error_log /dev/stderr debug;
include /etc/nginx/custom.conf;
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /dev/stdout;
sendfile on;
include /etc/nginx/conf.d/*.conf;
#https://serverfault.com/questions/587386/an-upstream-response-is-buffered-to-a-temporary-file
proxy_max_temp_file_size 0;
}
server {
listen 8081;
root /var/run/app/web/sites/default;
auth_basic "Authorization required";
# The file containing authorized users
auth_basic_user_file /etc/nginx/webdav/password;
# dav allowed method
dav_methods PUT DELETE MKCOL COPY MOVE;
dav_access user:rw group:rw all:r;
# MAX size of uploaded file, 0 mean unlimited
client_max_body_size 0;
# Allow autocreate folder if necessary
create_full_put_path on;
autoindex on;
location = /favicon.ico {
log_not_found off;
access_log off;
}
}
#!/bin/sh
set -ex
# Run Nginx
# For debugging change nginx to nginx-debug
exec nginx -g "daemon off;"
# set recommended PHP.ini settings
# see https://secure.php.net/manual/en/opcache.installation.php
opcache.memory_consumption=128
opcache.interned_strings_buffer=8
opcache.max_accelerated_files=4000
opcache.revalidate_freq=60
opcache.fast_shutdown=1
opcache.enable_cli=1
\ No newline at end of file
[global]
pid = /var/run/php-fpm.pid
[www]
listen = 9000
pm = dynamic
pm.max_children = 5
pm.start_servers = 2
pm.min_spare_servers = 1
pm.max_spare_servers = 3
catch_workers_output = yes
php_admin_value[error_log] = /var/log/fpm-php.www.log
php_admin_flag[log_errors] = on
access.log = /proc/self/fd/2
#!/bin/sh
# Fix permissions on the given directory to allow group read/write of
# regular files and execute of directories.
find -L "$1" -exec chgrp 0 {} \;
find -L "$1" -exec chmod g+rw {} \;
find -L "$1" -type d -exec chmod g+x {} +
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment