NXCALS-7429 Extract accpy publish to templates

.ci/base-pipeline.yml

-include:
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-gradle.yml
-    ref: cd08931bd66170aabda5335266cabc11d3241f2f
-
-build_project:
-  stage: build
-  extends:
-    - .gradle_build
-  after_script:
-    - find . | grep -i venv$ | xargs rm -r # remove all venvs
-    - rm -r python/type-stubs/build/*ClasspathJars
-  variables:
-    BUILD_ADDITIONAL_OPTS: "--stacktrace -x distZip --parallel"
-
-unit_tests:
-  stage: test
-  extends:
-    - .gradle_unit_tests
-  needs:
-    - job: build_project
-      artifacts: true
-  variables:
-    ACTIVE_PROFILE: "gitlab-ci"
-    GITLAB_MR_USER_NAME: "$GITLAB_USER_LOGIN"
-    TEST_ADDITIONAL_OPTS: "-p $MODULE_NAME --parallel"
-  parallel:
-    matrix:
-      - MODULE_NAME: [ "backport-api", "cern-extraction-api", "common", "compaction", "db", "docs", "extraction-api", "extraction-api-thin", "extraction-starter", "ingestion-api", "kafka-etl", "metadata-api", "monitoring-grok", "monitoring-producer", "monitoring-reader", "python", "service", "service-db", "spark-server" ]
-
-sonar:
-  stage: qa
-  extends:
-    - .gradle_sonar
-  needs:
-    - job: unit_tests
-      artifacts: true
\ No newline at end of file

.ci/deploy.yml

 include:
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-common.yml
-    ref: cd08931bd66170aabda5335266cabc11d3241f2f
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-logging.yml
-    ref: cd08931bd66170aabda5335266cabc11d3241f2f
   - local: .ci/templates.yml
 
 workflow:
@@ -15,7 +9,7 @@ workflow:
 deploy-infrastructure:
   stage: deploy
   extends:
-    - .local_nxcals_ansible
+    - .local_ansible
   variables:
     PLAYBOOKS: "certificates-install-with-sudo.yml kafka-install-with-sudo.yml prepare-service-with-sudo.yml monit-install.yml iptables-install.yml"
 
@@ -23,7 +17,7 @@ deploy-services:
   stage: deploy
   when: on_success
   extends:
-    - .local_nxcals_ansible
+    - .local_ansible
   resource_group: "$RESOURCE_GROUP"
   parallel:
     matrix:
@@ -49,7 +43,7 @@ deploy-services:
 deploy-artifacts:
   stage: deploy
   extends:
-    - .local_nxcals_ansible
+    - .local_ansible
   when: on_success
   parallel:
     matrix:
@@ -62,7 +56,7 @@ deploy-artifacts:
 deploy-wiki:
   stage: deploy
   extends:
-    - .local_nxcals_ansible
+    - .local_ansible
   when: on_success
   needs:
     - deploy-artifacts

.ci/templates.yml

 include:
   - project: acc-logging-team/gitlab-templates
     file: gitlab-ci-common.yml
-    ref: cd08931bd66170aabda5335266cabc11d3241f2f
+    ref: 0.1.31
   - project: acc-logging-team/gitlab-templates
     file: gitlab-ci-logging.yml
-    ref: cd08931bd66170aabda5335266cabc11d3241f2f
-
-.local_nxcals_python_publish:
-  extends:
-    - .base_job
-  script:
-    - name_of_variable_to_check=VERSION
-    - !reference [ .scripts, check_variable ]
-    - !reference [ .logging_scripts, activate_kerberos ]
-    - |
-      cd /tmp && mkdir release && cd release
-      for NAME in nxcals-extraction-api-python3 nxcals-spark-session-builder nxcals nxcals-py4j-utils nxcals-type-stubs 
-      do
-        FILENAME="${NAME//-/_}"-"$VERSION"-py3-none-any.whl
-        curl --output "$FILENAME" http://photons-resources.cern.ch/downloads/nxcals_pro/spark/"$FILENAME"
-        acc-py devrelease "$FILENAME"
-      done
-      
-      FILENAME="pytimber"-"4.${VERSION}"-py3-none-any.whl
-      curl --output "$FILENAME" http://photons-resources.cern.ch/downloads/nxcals_pro/spark/"$FILENAME"
-      acc-py devrelease "$FILENAME"
+    ref: 0.1.31
 
 .local_integration_tests:
-  stage: integration-tests
   extends:
     - .logging_integration_tests
   variables:
@@ -41,15 +20,22 @@ include:
     matrix:
       - TEST_VARIANT: [ "-base", "-thin", "-python", "-op", "-auth" ]
 
-.local_nxcals_ansible:
+.local_ansible:
   extends:
     - .logging_base_ansible_deploy_job
   script:
     - PYTIMBER_MAJOR_VERSION=$(awk -F "=" '/pytimberMajorVersion/ {print $2}' gradle.properties)
-    - PLAYBOOK_ADDITIONAL_OPTS="-u acclog -e pytimber_major_version=$PYTIMBER_MAJOR_VERSION $PLAYBOOK_ADDITIONAL_OPTS"
+    - PLAYBOOK_ADDITIONAL_OPTS="-e pytimber_major_version=$PYTIMBER_MAJOR_VERSION $PLAYBOOK_ADDITIONAL_OPTS"
     - !reference [ .logging_base_ansible_deploy_job, script ]
   variables:
     ARTIFACT_NAME: nxcals-ansible
   interruptible: false
   tags:
-    - TN
+    - TN
\ No newline at end of file
+
+.local_unit_tests:
+  extends:
+    - .gradle_unit_tests
+  parallel:
+    matrix:
+      - MODULE_NAME: [ "backport-api", "cern-extraction-api", "common", "compaction", "db", "docs", "extraction-api", "extraction-api-thin", "extraction-starter", "ingestion-api", "kafka-etl", "metadata-api", "monitoring-grok", "monitoring-producer", "monitoring-reader", "python", "service", "service-db", "spark-server" ]

.gitlab-ci.yml

 include:
   - local: '.ci/templates.yml'
-  - local: '.ci/base-pipeline.yml'
+  - project: acc-logging-team/gitlab-templates
+    file: workflows/base.yml
+    ref: 0.1.31
   - project: acc-logging-team/gitlab-templates
     file: workflows/test.yml
-    ref: cd08931bd66170aabda5335266cabc11d3241f2f
+    ref: 0.1.31
   - project: acc-logging-team/gitlab-templates
     file: workflows/staging.yml
-    ref: cd08931bd66170aabda5335266cabc11d3241f2f
+    ref: 0.1.31
 
 variables:
   ANSIBLE_FILES: ansible
-  BASE_IMAGE: gitlab-registry.cern.ch/acc-devops/gitlab-runner/acc-py-java-ansible-jf-hadoop:0.5.3
+  BASE_IMAGE: gitlab-registry.cern.ch/acc-co/devops/common/gitlab-runner/acc-py-java-ansible-jf-hadoop:0.5.5
   HTTPS_PROXY: "http://photons-resources:8888"
   HTTP_PROXY: "http://photons-resources:8888"
   MAIN_BRANCH: "develop"
@@ -40,6 +42,9 @@ workflow:
         ARTIFACT_GROUP_ID: "cern.nxcals"
         CAS_REPOSITORY: ds-release-local
         CAS_ALIAS_NAME: PRO
+        PYTHON_PACKAGE_SUFFIX: "py3-none-any.whl"
+        ACCPY_ARTIFACTS: "nxcals-extraction-api-python3-$VERSION-$PYTHON_PACKAGE_SUFFIX nxcals-spark-session-builder-$VERSION-$PYTHON_PACKAGE_SUFFIX nxcals nxcals-py4j-utils-$VERSION-$PYTHON_PACKAGE_SUFFIX nxcals-type-stubs-$VERSION-$PYTHON_PACKAGE_SUFFIX pytimber-4.$VERSION-$PYTHON_PACKAGE_SUFFIX"
+        ACCPY_ARTIFACT_DOWNLOAD_PATH: http://photons-resources.cern.ch/downloads/nxcals_pro/spark
     - if: $CI_COMMIT_BRANCH == $MAIN_BRANCH && $STAGING_PIPELINE != 'true'
       variables:
         PIPELINE_NAME: "[Test] Test pipeline for $CI_COMMIT_SHORT_SHA"
@@ -60,4 +65,8 @@ stages:
   - integration-tests
   - release
   - deploy-release
-  - publish-release
+  - publish-release
\ No newline at end of file
+
+after_script:
+  - find . | grep -i venv$ | xargs rm -r # remove all venvs
+  - rm -r python/type-stubs/build/*ClasspathJars
\ No newline at end of file

integration-tests.sh

@@ -65,7 +65,7 @@ while [[ "$#" -gt 0 ]]; do
 	shift
 done
 
-if [ -z "${JENKINS_ENV:-}" ]; then
+if [ -z "${CI_ENV:-}" ]; then
 	API_STABILITY=false
 	INVENTORY=inventory/dev-"$USER"
 	SERVICE_URL=https://nxcals-"$USER"-1.cern.ch:19093,https://nxcals-"$USER"-2.cern.ch:19093
@@ -77,8 +77,8 @@ if [ -z "${JENKINS_ENV:-}" ]; then
 	RBAC_ENV=test
 else
 	HOME=/tmp/"$USER"/nxcals_integration_tests
-	INVENTORY=inventory/"$JENKINS_ENV"
-	NAMESPACE=nxcals_"$JENKINS_ENV"
+	INVENTORY=inventory/"$CI_ENV"
+	NAMESPACE=nxcals_"$CI_ENV"
 fi
 
 
@@ -112,7 +112,7 @@ if [ "$INSTALL" -eq 1  ] && [ "$PYTHON_TESTS" -eq 1 ]; then
 fi
 
 # needed, because otherwise ansible complains and won't use ansible.cfg from world-writable dir
-if [ "$JENKINS_ENV" ]; then
+if [ "$CI_ENV" ]; then
   chmod 744 ansible
 fi
 

nxcals_staging.jenkinsfile

@@ -142,7 +142,7 @@ pipeline {
                     env.SERVICE_URL = 'https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
                     env.KAFKA_SERVERS_URL = 'https://nxcals-staging3.cern.ch:9092,https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092'
                     env.SPARK_SERVERS_URL = 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
-                    env.JENKINS_ENV = 'staging'
+                    env.CI_ENV = 'staging'
                     env.HOME = '/opt/nxcals/'
                     env.API_STABILITY = false
                     env.RBAC_ENV = 'test'
@@ -180,7 +180,7 @@ pipeline {
                     env.SERVICE_URL = 'https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
                     env.KAFKA_SERVERS_URL = 'https://nxcals-staging3.cern.ch:9092,https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092'
                     env.SPARK_SERVERS_URL = 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
-                    env.JENKINS_ENV = 'staging'
+                    env.CI_ENV = 'staging'
                     //should be removed later, user set in the integration-tests.sh script
                     env.USER = 'acclog'
                     env.HOME = '/opt/nxcals/'
@@ -254,7 +254,7 @@ pipeline {
                     env.SERVICE_URL = 'https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
                     env.KAFKA_SERVERS_URL = 'https://nxcals-staging3.cern.ch:9092,https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092'
                     env.SPARK_SERVERS_URL = 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
-                    env.JENKINS_ENV = 'staging'
+                    env.CI_ENV = 'staging'
                     //should be removed later, user set in the integration-tests.sh script
                     env.USER = 'acclog'
                     env.HOME = '/opt/nxcals/'

nxcals_test.jenkinsfile

@@ -111,7 +111,7 @@ pipeline {
                                     env.SERVICE_URL = 'https://nxcals-test4.cern.ch:19093,https://nxcals-test5.cern.ch:19093,https://nxcals-test6.cern.ch:19093'
                                     env.KAFKA_SERVERS_URL = 'https://nxcals-test4.cern.ch:9092,https://nxcals-test5.cern.ch:9092,https://nxcals-test6.cern.ch:9092'
                                     env.SPARK_SERVERS_URL = 'nxcals-test5.cern.ch:15000,nxcals-test6.cern.ch:15000'
-                                    env.JENKINS_ENV = 'test'
+                                    env.CI_ENV = 'test'
                                     env.HOME = '/opt/nxcals/'
                                     env.API_STABILITY = false
                                     env.RBAC_ENV = 'test'
@@ -151,7 +151,7 @@ pipeline {
                                     env.SERVICE_URL = 'https://nxcals-test4.cern.ch:19093,https://nxcals-test5.cern.ch:19093,https://nxcals-test6.cern.ch:19093'
                                     env.KAFKA_SERVERS_URL = 'https://nxcals-test4.cern.ch:9092,https://nxcals-test5.cern.ch:9092,https://nxcals-test6.cern.ch:9092'
                                     env.SPARK_SERVERS_URL = 'nxcals-test5.cern.ch:15000,nxcals-test6.cern.ch:15000'
-                                    env.JENKINS_ENV = 'test'
+                                    env.CI_ENV = 'test'
                                     env.USER = 'acclog'
                                     env.HOME = '/opt/nxcals/'
                                     env.API_STABILITY = true

performance-tests.jenkinsfile

@@ -91,7 +91,7 @@ pipeline {
             steps {
                 script {
                     env.GRADLE = tool 'gradle'
-                    env.JENKINS_ENV = 'perf-test'
+                    env.CI_ENV = 'perf-test'
                     env.HOME = '/opt/nxcals/'
                 }
                 withCredentials([file(credentialsId: 'NXCALS-ANSIBLE-VAULT-PASS', variable: 'VAULT_PASS')]) {

performance-tests.sh

@@ -47,7 +47,7 @@ HOME=/tmp/"$USER"/nxcals_performance_tests
 NAMESPACE=nxcals_perf_test
 RBAC_ENV=test
 
-if [ -z "${JENKINS_ENV:-}" ]; then	
+if [ -z "${CI_ENV:-}" ]; then
 	VAULT_PASS=.pass
 fi