diff --git a/.ci/base-pipeline.yml b/.ci/base-pipeline.yml
deleted file mode 100644
index db94341dda79395a796066d30e771123c661a8aa..0000000000000000000000000000000000000000
--- a/.ci/base-pipeline.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-include:
- - project: acc-logging-team/gitlab-templates
- file: gitlab-ci-gradle.yml
- ref: cd08931bd66170aabda5335266cabc11d3241f2f
-
-build_project:
- stage: build
- extends:
- - .gradle_build
- after_script:
- - find . | grep -i venv$ | xargs rm -r # remove all venvs
- - rm -r python/type-stubs/build/*ClasspathJars
- variables:
- BUILD_ADDITIONAL_OPTS: "--stacktrace -x distZip --parallel"
-
-unit_tests:
- stage: test
- extends:
- - .gradle_unit_tests
- needs:
- - job: build_project
- artifacts: true
- variables:
- ACTIVE_PROFILE: "gitlab-ci"
- GITLAB_MR_USER_NAME: "$GITLAB_USER_LOGIN"
- TEST_ADDITIONAL_OPTS: "-p $MODULE_NAME --parallel"
- parallel:
- matrix:
- - MODULE_NAME: [ "backport-api", "cern-extraction-api", "common", "compaction", "db", "docs", "extraction-api", "extraction-api-thin", "extraction-starter", "ingestion-api", "kafka-etl", "metadata-api", "monitoring-grok", "monitoring-producer", "monitoring-reader", "python", "service", "service-db", "spark-server" ]
-
-sonar:
- stage: qa
- extends:
- - .gradle_sonar
- needs:
- - job: unit_tests
- artifacts: true
\ No newline at end of file
diff --git a/.ci/deploy.yml b/.ci/deploy.yml
index f2992afbec691494afd422cde5bf42c70d593b6e..58c730958b2be67d10a89de5786f2d620181f8a4 100644
--- a/.ci/deploy.yml
+++ b/.ci/deploy.yml
@@ -1,10 +1,4 @@
include:
- - project: acc-logging-team/gitlab-templates
- file: gitlab-ci-common.yml
- ref: cd08931bd66170aabda5335266cabc11d3241f2f
- - project: acc-logging-team/gitlab-templates
- file: gitlab-ci-logging.yml
- ref: cd08931bd66170aabda5335266cabc11d3241f2f
- local: .ci/templates.yml
workflow:
@@ -15,7 +9,7 @@ workflow:
deploy-infrastructure:
stage: deploy
extends:
- - .local_nxcals_ansible
+ - .local_ansible
variables:
PLAYBOOKS: "certificates-install-with-sudo.yml kafka-install-with-sudo.yml prepare-service-with-sudo.yml monit-install.yml iptables-install.yml"
@@ -23,7 +17,7 @@ deploy-services:
stage: deploy
when: on_success
extends:
- - .local_nxcals_ansible
+ - .local_ansible
resource_group: "$RESOURCE_GROUP"
parallel:
matrix:
@@ -49,7 +43,7 @@ deploy-services:
deploy-artifacts:
stage: deploy
extends:
- - .local_nxcals_ansible
+ - .local_ansible
when: on_success
parallel:
matrix:
@@ -62,7 +56,7 @@ deploy-artifacts:
deploy-wiki:
stage: deploy
extends:
- - .local_nxcals_ansible
+ - .local_ansible
when: on_success
needs:
- deploy-artifacts
diff --git a/.ci/templates.yml b/.ci/templates.yml
index 4c1cdc63acd2d8ae9e0d1378166a3f461844ab46..0043f961d928198c012e9776b254ddf5def50fcc 100644
--- a/.ci/templates.yml
+++ b/.ci/templates.yml
@@ -1,33 +1,12 @@
include:
- project: acc-logging-team/gitlab-templates
file: gitlab-ci-common.yml
- ref: cd08931bd66170aabda5335266cabc11d3241f2f
+ ref: 0.1.31
- project: acc-logging-team/gitlab-templates
file: gitlab-ci-logging.yml
- ref: cd08931bd66170aabda5335266cabc11d3241f2f
-
-.local_nxcals_python_publish:
- extends:
- - .base_job
- script:
- - name_of_variable_to_check=VERSION
- - !reference [ .scripts, check_variable ]
- - !reference [ .logging_scripts, activate_kerberos ]
- - |
- cd /tmp && mkdir release && cd release
- for NAME in nxcals-extraction-api-python3 nxcals-spark-session-builder nxcals nxcals-py4j-utils nxcals-type-stubs
- do
- FILENAME="${NAME//-/_}"-"$VERSION"-py3-none-any.whl
- curl --output "$FILENAME" http://photons-resources.cern.ch/downloads/nxcals_pro/spark/"$FILENAME"
- acc-py devrelease "$FILENAME"
- done
-
- FILENAME="pytimber"-"4.${VERSION}"-py3-none-any.whl
- curl --output "$FILENAME" http://photons-resources.cern.ch/downloads/nxcals_pro/spark/"$FILENAME"
- acc-py devrelease "$FILENAME"
+ ref: 0.1.31
.local_integration_tests:
- stage: integration-tests
extends:
- .logging_integration_tests
variables:
@@ -41,15 +20,22 @@ include:
matrix:
- TEST_VARIANT: [ "-base", "-thin", "-python", "-op", "-auth" ]
-.local_nxcals_ansible:
+.local_ansible:
extends:
- .logging_base_ansible_deploy_job
script:
- PYTIMBER_MAJOR_VERSION=$(awk -F "=" '/pytimberMajorVersion/ {print $2}' gradle.properties)
- - PLAYBOOK_ADDITIONAL_OPTS="-u acclog -e pytimber_major_version=$PYTIMBER_MAJOR_VERSION $PLAYBOOK_ADDITIONAL_OPTS"
+ - PLAYBOOK_ADDITIONAL_OPTS="-e pytimber_major_version=$PYTIMBER_MAJOR_VERSION $PLAYBOOK_ADDITIONAL_OPTS"
- !reference [ .logging_base_ansible_deploy_job, script ]
variables:
ARTIFACT_NAME: nxcals-ansible
interruptible: false
tags:
- - TN
\ No newline at end of file
+ - TN
+
+.local_unit_tests:
+ extends:
+ - .gradle_unit_tests
+ parallel:
+ matrix:
+ - MODULE_NAME: [ "backport-api", "cern-extraction-api", "common", "compaction", "db", "docs", "extraction-api", "extraction-api-thin", "extraction-starter", "ingestion-api", "kafka-etl", "metadata-api", "monitoring-grok", "monitoring-producer", "monitoring-reader", "python", "service", "service-db", "spark-server" ]
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4dad6651b426d424945ca32affd3cb83929e378b..0c6b5d00af48389b044da2c5bbd26d7b6a0d0a27 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,16 +1,18 @@
include:
- local: '.ci/templates.yml'
- - local: '.ci/base-pipeline.yml'
+ - project: acc-logging-team/gitlab-templates
+ file: workflows/base.yml
+ ref: 0.1.31
- project: acc-logging-team/gitlab-templates
file: workflows/test.yml
- ref: cd08931bd66170aabda5335266cabc11d3241f2f
+ ref: 0.1.31
- project: acc-logging-team/gitlab-templates
file: workflows/staging.yml
- ref: cd08931bd66170aabda5335266cabc11d3241f2f
+ ref: 0.1.31
variables:
ANSIBLE_FILES: ansible
- BASE_IMAGE: gitlab-registry.cern.ch/acc-devops/gitlab-runner/acc-py-java-ansible-jf-hadoop:0.5.3
+ BASE_IMAGE: gitlab-registry.cern.ch/acc-co/devops/common/gitlab-runner/acc-py-java-ansible-jf-hadoop:0.5.5
HTTPS_PROXY: "http://photons-resources:8888"
HTTP_PROXY: "http://photons-resources:8888"
MAIN_BRANCH: "develop"
@@ -40,6 +42,9 @@ workflow:
ARTIFACT_GROUP_ID: "cern.nxcals"
CAS_REPOSITORY: ds-release-local
CAS_ALIAS_NAME: PRO
+ PYTHON_PACKAGE_SUFFIX: "py3-none-any.whl"
+ ACCPY_ARTIFACTS: "nxcals-extraction-api-python3-$VERSION-$PYTHON_PACKAGE_SUFFIX nxcals-spark-session-builder-$VERSION-$PYTHON_PACKAGE_SUFFIX nxcals nxcals-py4j-utils-$VERSION-$PYTHON_PACKAGE_SUFFIX nxcals-type-stubs-$VERSION-$PYTHON_PACKAGE_SUFFIX pytimber-4.$VERSION-$PYTHON_PACKAGE_SUFFIX"
+ ACCPY_ARTIFACT_DOWNLOAD_PATH: http://photons-resources.cern.ch/downloads/nxcals_pro/spark
- if: $CI_COMMIT_BRANCH == $MAIN_BRANCH && $STAGING_PIPELINE != 'true'
variables:
PIPELINE_NAME: "[Test] Test pipeline for $CI_COMMIT_SHORT_SHA"
@@ -60,4 +65,8 @@ stages:
- integration-tests
- release
- deploy-release
- - publish-release
\ No newline at end of file
+ - publish-release
+
+after_script:
+ - find . | grep -i venv$ | xargs rm -r # remove all venvs
+ - rm -r python/type-stubs/build/*ClasspathJars
\ No newline at end of file
diff --git a/integration-tests.sh b/integration-tests.sh
index 8095ad4c9242bca39dc55d2457f5663c557392f8..18c27879afbaa1b7d9cc3e09a2effa7a3dc23a89 100755
--- a/integration-tests.sh
+++ b/integration-tests.sh
@@ -65,7 +65,7 @@ while [[ "$#" -gt 0 ]]; do
shift
done
-if [ -z "${JENKINS_ENV:-}" ]; then
+if [ -z "${CI_ENV:-}" ]; then
API_STABILITY=false
INVENTORY=inventory/dev-"$USER"
SERVICE_URL=https://nxcals-"$USER"-1.cern.ch:19093,https://nxcals-"$USER"-2.cern.ch:19093
@@ -77,8 +77,8 @@ if [ -z "${JENKINS_ENV:-}" ]; then
RBAC_ENV=test
else
HOME=/tmp/"$USER"/nxcals_integration_tests
- INVENTORY=inventory/"$JENKINS_ENV"
- NAMESPACE=nxcals_"$JENKINS_ENV"
+ INVENTORY=inventory/"$CI_ENV"
+ NAMESPACE=nxcals_"$CI_ENV"
fi
@@ -112,7 +112,7 @@ if [ "$INSTALL" -eq 1 ] && [ "$PYTHON_TESTS" -eq 1 ]; then
fi
# needed, because otherwise ansible complains and won't use ansible.cfg from world-writable dir
-if [ "$JENKINS_ENV" ]; then
+if [ "$CI_ENV" ]; then
chmod 744 ansible
fi
diff --git a/nxcals_staging.jenkinsfile b/nxcals_staging.jenkinsfile
index d13cfa8f31ec28beb9f8a7b6bd31fbf002614c41..6a550ff5db7bb1bb2b1a7e7bf27ceb26451ddaff 100644
--- a/nxcals_staging.jenkinsfile
+++ b/nxcals_staging.jenkinsfile
@@ -142,7 +142,7 @@ pipeline {
env.SERVICE_URL = 'https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
env.KAFKA_SERVERS_URL = 'https://nxcals-staging3.cern.ch:9092,https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092'
env.SPARK_SERVERS_URL = 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
- env.JENKINS_ENV = 'staging'
+ env.CI_ENV = 'staging'
env.HOME = '/opt/nxcals/'
env.API_STABILITY = false
env.RBAC_ENV = 'test'
@@ -180,7 +180,7 @@ pipeline {
env.SERVICE_URL = 'https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
env.KAFKA_SERVERS_URL = 'https://nxcals-staging3.cern.ch:9092,https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092'
env.SPARK_SERVERS_URL = 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
- env.JENKINS_ENV = 'staging'
+ env.CI_ENV = 'staging'
//should be removed later, user set in the integration-tests.sh script
env.USER = 'acclog'
env.HOME = '/opt/nxcals/'
@@ -254,7 +254,7 @@ pipeline {
env.SERVICE_URL = 'https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
env.KAFKA_SERVERS_URL = 'https://nxcals-staging3.cern.ch:9092,https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092'
env.SPARK_SERVERS_URL = 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
- env.JENKINS_ENV = 'staging'
+ env.CI_ENV = 'staging'
//should be removed later, user set in the integration-tests.sh script
env.USER = 'acclog'
env.HOME = '/opt/nxcals/'
diff --git a/nxcals_test.jenkinsfile b/nxcals_test.jenkinsfile
index 4008fa7d65e6833d85a8a5dd84ad566621fc5720..a46f77c7573edb9676d40563252021c866fab95c 100644
--- a/nxcals_test.jenkinsfile
+++ b/nxcals_test.jenkinsfile
@@ -111,7 +111,7 @@ pipeline {
env.SERVICE_URL = 'https://nxcals-test4.cern.ch:19093,https://nxcals-test5.cern.ch:19093,https://nxcals-test6.cern.ch:19093'
env.KAFKA_SERVERS_URL = 'https://nxcals-test4.cern.ch:9092,https://nxcals-test5.cern.ch:9092,https://nxcals-test6.cern.ch:9092'
env.SPARK_SERVERS_URL = 'nxcals-test5.cern.ch:15000,nxcals-test6.cern.ch:15000'
- env.JENKINS_ENV = 'test'
+ env.CI_ENV = 'test'
env.HOME = '/opt/nxcals/'
env.API_STABILITY = false
env.RBAC_ENV = 'test'
@@ -151,7 +151,7 @@ pipeline {
env.SERVICE_URL = 'https://nxcals-test4.cern.ch:19093,https://nxcals-test5.cern.ch:19093,https://nxcals-test6.cern.ch:19093'
env.KAFKA_SERVERS_URL = 'https://nxcals-test4.cern.ch:9092,https://nxcals-test5.cern.ch:9092,https://nxcals-test6.cern.ch:9092'
env.SPARK_SERVERS_URL = 'nxcals-test5.cern.ch:15000,nxcals-test6.cern.ch:15000'
- env.JENKINS_ENV = 'test'
+ env.CI_ENV = 'test'
env.USER = 'acclog'
env.HOME = '/opt/nxcals/'
env.API_STABILITY = true
diff --git a/performance-tests.jenkinsfile b/performance-tests.jenkinsfile
index e28f9edeb27a7913bca1eef745cc48d626fd670c..feffccf82c05d0bb46122749af4fd5f88b3f9b12 100644
--- a/performance-tests.jenkinsfile
+++ b/performance-tests.jenkinsfile
@@ -91,7 +91,7 @@ pipeline {
steps {
script {
env.GRADLE = tool 'gradle'
- env.JENKINS_ENV = 'perf-test'
+ env.CI_ENV = 'perf-test'
env.HOME = '/opt/nxcals/'
}
withCredentials([file(credentialsId: 'NXCALS-ANSIBLE-VAULT-PASS', variable: 'VAULT_PASS')]) {
diff --git a/performance-tests.sh b/performance-tests.sh
index 50857da0ddf1bb95da115e46acfe0d72cbb45591..8847edf41ef9e7a9a6170aac3bcd5428be307846 100755
--- a/performance-tests.sh
+++ b/performance-tests.sh
@@ -47,7 +47,7 @@ HOME=/tmp/"$USER"/nxcals_performance_tests
NAMESPACE=nxcals_perf_test
RBAC_ENV=test
-if [ -z "${JENKINS_ENV:-}" ]; then
+if [ -z "${CI_ENV:-}" ]; then
VAULT_PASS=.pass
fi