From d6394c91599b4f305ccad2b5e584ce1c29754c79 Mon Sep 17 00:00:00 2001
From: Brice Copy <brice.copy@cern.ch>
Date: Mon, 13 May 2024 14:33:16 +0200
Subject: [PATCH] Update NXCALS imports
---
.gitlab-ci.yml | 14 ++++-----
Dockerfile | 66 ++++++++++++++++++++++++++++---------------
README.md | 4 +--
entrypoint.sh | 4 +--
influxdb.centos7.repo | 6 ----
influxdb.rhel9.repo | 6 ++++
pom.xml | 3 +-
7 files changed, 60 insertions(+), 43 deletions(-)
delete mode 100644 influxdb.centos7.repo
create mode 100644 influxdb.rhel9.repo
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4e3845a..f3c67c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,12 +9,10 @@ build:
- wget https://cern.ch/maven/settings-ci.xml
# Use the help plugin to extract metadata for the Kaniko builder
- mvn $MAVEN_CLI_OPTS help:evaluate -Dexpression=project.version -Doutput=.POM_VERSION
- - mvn $MAVEN_CLI_OPTS help:evaluate -Dexpression=spark.nxcals.url -Doutput=.SPARK_NXCALS_URL
- mvn $MAVEN_CLI_OPTS help:evaluate -Dexpression=docker.from -Doutput=.FROM
artifacts:
paths:
- .POM_VERSION
- - .SPARK_NXCALS_URL
- .FROM
# CERN Gitlab now uses Kaniko for unprivileged Docker builds.
@@ -26,9 +24,8 @@ build:
entrypoint: [""]
before_script:
- POM_VERSION=`cat .POM_VERSION`
- - SPARK_NXCALS_URL=`cat .SPARK_NXCALS_URL`
- FROM=`cat .FROM`
- - echo Building Docker Image v $POM_VERSION - $SPARK_NXCALS_URL - $FROM
+ - echo Building Docker Image v $POM_VERSION - $FROM
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
@@ -36,13 +33,16 @@ push latest:
<<: *kanikoSetup
stage: test
script:
- - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --build-arg=SPARK_NXCALS_URL=${SPARK_NXCALS_URL} --build-arg=FROM=${FROM} --destination $CI_REGISTRY_IMAGE
+ - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --build-arg=FROM=${FROM} --destination $CI_REGISTRY_IMAGE
+ except:
+ - master
+ - tags
push tagged version:
<<: *kanikoSetup
stage: deploy
script:
- - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --build-arg=SPARK_NXCALS_URL=${SPARK_NXCALS_URL} --build-arg=FROM=${FROM} --destination $CI_REGISTRY_IMAGE:$POM_VERSION
+ - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --build-arg=FROM=${FROM} --destination $CI_REGISTRY_IMAGE:$POM_VERSION
only:
- tags
@@ -50,7 +50,7 @@ push stable version:
<<: *kanikoSetup
stage: deploy
script:
- - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --build-arg=SPARK_NXCALS_URL=${SPARK_NXCALS_URL} --build-arg=FROM=${FROM} --destination $CI_REGISTRY_IMAGE:stable
+ - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --build-arg=FROM=${FROM} --destination $CI_REGISTRY_IMAGE:stable
only:
- master
diff --git a/Dockerfile b/Dockerfile
index 8c28468..70c102f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,42 +1,62 @@
-ARG FROM=gitlab-registry.cern.ch/industrial-controls/sw-infra/cc7-openjdk:jre-8
+ARG FROM=gitlab-registry.cern.ch/acc-co/devops/python/distribution-images/acc-py_el9_openjdk11_gui_ci:pro
FROM $FROM
-ARG SPARK_NXCALS_URL=http://photons-resources.cern.ch/downloads/nxcals_pro/spark/spark-nxcals.zip
+COPY influxdb.rhel9.repo /etc/yum.repos.d/influxdb.repo
-COPY influxdb.centos7.repo /etc/yum.repos.d/influxdb.repo
+RUN dnf makecache && \
+ dnf install -y krb5-workstation krb5-libs git unzip bsdtar tar rsync CERN-CA-certs which influxdb && \
+ dnf clean all && \
+ curl -o /etc/krb5.conf https://linux.web.cern.ch/docs/krb5.conf
-RUN yum update -y && yum install -y python3 python3-cython python3-pip python3-virtualenv git unzip bsdtar rsync CERN-CA-certs which influxdb && \
- yum clean all
+# RUN yum update -y && yum install -y python3 python3-pip python3-virtualenv git unzip bsdtar rsync CERN-CA-certs which influxdb && \
+# yum clean all
-ENV SPARK_HOME=/opt/nxcals-spark
-RUN curl -s -k -L -o /tmp/nxcals-spark.zip $SPARK_NXCALS_URL && \
- mkdir -p $SPARK_HOME && \
- mkdir -p $SPARK_HOME/work && \
- chmod 777 $SPARK_HOME/work && \
- cd $SPARK_HOME && bsdtar --strip-components=1 -xvf /tmp/nxcals-spark.zip && \
- rm /tmp/nxcals-spark.zip
+# nxcals-bundle/conf/build_packed_venv.sh
+
+# dnf install -y git unzip bsdtar rsync CERN-CA-certs which influxdb
+# dnf install -y krb5-workstation krb5-libs
+## curl -o /etc/krb5.conf https://linux.web.cern.ch/docs/krb5.conf
+
+
+ENV SPARK_INSTALL=/opt/nxcals-spark
+ENV NXCALS_PACK_ALL_PACKAGES="true"
+
+RUN mkdir -p $SPARK_INSTALL && \
+ mkdir -p $SPARK_INSTALL/work && \
+ chmod 777 $SPARK_INSTALL/work
RUN adduser etlworker -m
-RUN mkdir -p $SPARK_HOME/nxcals-python3-env && \
- chown -R etlworker $SPARK_HOME
+RUN cd $SPARK_INSTALL && \
+ python -m venv . && \
+ chown -R etlworker $SPARK_INSTALL
USER etlworker
-RUN cd $SPARK_HOME && \
- yes | source $SPARK_HOME/source-me.sh && \
- source $SPARK_HOME/nxcals-python3-env/bin/activate && \
- pip3 list && pip3 install pyspark[sql]==2.3.2 pandas scipy influxdb pyarrow==1.0.0
+RUN cd $SPARK_INSTALL && \
+ source $SPARK_INSTALL/bin/activate && \
+ pip list && \
+ pip install nxcals pandas scipy pyarrow
+
+RUN chmod 755 ${SPARK_INSTALL}/nxcals-bundle/conf/spark-env.sh
+RUN export SPARK_HOME="/opt/nxcals-spark/nxcals-bundle" \
+ && export SPARK_CONF_DIR="${SPARK_INSTALL}/nxcals-bundle/conf" \
+ && export NXCALS_WORKSPACE_TEMP_DIR="$SPARK_INSTALL/work" \
+ && cd $SPARK_INSTALL \
+ && ${SPARK_INSTALL}/nxcals-bundle/conf/spark-env.sh
+
+# pip3 install --index-url https://acc-py-repo.cern.ch/repository/vr-py-releases/simple nxcals
+
+RUN source $SPARK_INSTALL/bin/activate && \
+ pip install git+https://gitlab.cern.ch/industrial-controls/services/dash/dash-etl.git@master
-RUN source $SPARK_HOME/nxcals-python3-env/bin/activate && \
- pip3 install git+https://gitlab.cern.ch/industrial-controls/services/dash/dash-etl.git@master
VOLUME /auth
-VOLUME $SPARK_HOME/work
+VOLUME $SPARK_INSTALL/work
-WORKDIR $SPARK_HOME
+WORKDIR $SPARK_INSTALL
USER root
COPY entrypoint.sh /
@@ -46,4 +66,4 @@ USER etlworker
ENTRYPOINT ["/entrypoint.sh"]
-CMD ["bin/spark-submit","./work/script.py"]
+CMD ["spark-submit","./work/script.py"]
diff --git a/README.md b/README.md
index 6c8d424..3cd24e0 100644
--- a/README.md
+++ b/README.md
@@ -57,8 +57,6 @@ Once back on the develop branch, update the version and git push
# How to build manually
- docker build --build-arg FROM="cern/cc8-base" \
- --build-arg SPARK_NXCALS_URL="http://photons-resources.cern.ch/downloads/nxcals_testbed/spark/spark-nxcals.zip" \
- -t etlworker .
+ docker build --build-arg FROM="cern/cc8-base" -t etlworker .
diff --git a/entrypoint.sh b/entrypoint.sh
index 5a3b5e8..7dc8f31 100644
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -1,7 +1,7 @@
#!/bin/bash
-source $SPARK_HOME/nxcals-python3-env/bin/activate
+source $SPARK_INSTALL/bin/activate
-export PYTHONPATH=$SPARK_HOME/nxcals-python3-env/lib/python3.6/site-packages:$PYTHONPATH
+export PYTHONPATH=$SPARK_INSTALL/lib/python3.7/site-packages:$PYTHONPATH
if [ -z "$KPRINCIPAL" ]
then
diff --git a/influxdb.centos7.repo b/influxdb.centos7.repo
deleted file mode 100644
index e57289e..0000000
--- a/influxdb.centos7.repo
+++ /dev/null
@@ -1,6 +0,0 @@
-[influxdb]
-name = InfluxDB Repository - RHEL 7
-baseurl = https://repos.influxdata.com/rhel/7/x86_64/stable
-enabled = 1
-gpgcheck = 1
-gpgkey = https://repos.influxdata.com/influxdb.key
diff --git a/influxdb.rhel9.repo b/influxdb.rhel9.repo
new file mode 100644
index 0000000..360744d
--- /dev/null
+++ b/influxdb.rhel9.repo
@@ -0,0 +1,6 @@
+[influxdb]
+name = InfluxDB Repository - RHEL 9
+baseurl = https://repos.influxdata.com/rhel/9/x86_64/stable
+enabled = 1
+gpgcheck = 1
+gpgkey = https://repos.influxdata.com/influxdata-archive_compat.key
diff --git a/pom.xml b/pom.xml
index 0415e64..ca7ab63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -14,8 +14,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <spark.nxcals.url>http://photons-resources.cern.ch/downloads/nxcals_pro/spark/spark-nxcals.zip</spark.nxcals.url>
- <docker.from>gitlab-registry.cern.ch/industrial-controls/sw-infra/cc7-openjdk:jre-8</docker.from>
+ <docker.from>gitlab-registry.cern.ch/acc-co/devops/python/distribution-images/acc-py_el9_openjdk11_gui_ci:pro</docker.from>
</properties>
<build>
--
GitLab