.ci/deploy-pro.yml

-include:
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-common.yml
-    ref: 0.1.19
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-templates-gradle.yml
-    ref: 0.1.19
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-nxcals-templates.yml
-    ref: 0.1.19
-  - local: '.ci/templates.yml'
-
-
-stages:
-  - deploy
-
-
-workflow:
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-    - if: $CI_COMMIT_BRANCH == $MAIN_BRANCH
-
-
-deploy-infrastructure:
-  stage: deploy
-  extends:
-    - .nxcals_infrastructure_deploy:pro
-
-
-deploy-services:
-  stage: deploy
-  when: on_success
-  extends:
-    - .nxcals_services_deploy:pro
-  needs:
-    - deploy-infrastructure
-
-
-deploy-artifacts:
-  stage: deploy
-  when: on_success
-  extends:
-    - .nxcals_artifacts_deploy:pro

.ci/deploy-snapshot.yml

-include:
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-common.yml
-    ref: 0.1.19
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-templates-gradle.yml
-    ref: 0.1.19
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-nxcals-templates.yml
-    ref: 0.1.19
-  - local: '.ci/templates.yml'
-
-
-stages:
-  - deploy
-
-
-workflow:
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-    - if: $CI_COMMIT_BRANCH == $MAIN_BRANCH
-
-
-deploy-infrastructure:
-  stage: deploy
-  extends:
-    - .nxcals_infrastructure_deploy
-
-
-deploy-services:
-  stage: deploy
-  when: on_success
-  extends:
-    - .nxcals_services_deploy
-  needs:
-    - deploy-infrastructure
-
-
-deploy-artifacts:
-  stage: deploy
-  when: on_success
-  extends:
-    - .nxcals_artifacts_deploy

.ci/deploy.yml

+include:
+  - local: .ci/templates.yml
+
+workflow:
+  name: "Deploy $VERSION to $ANSIBLE_INVENTORY"
+  rules:
+    - if: $PERFORMANCE_TESTS == "true"
+      variables:
+        PLAYBOOKS: nxcals-full-perf-install-with-sudo.yml
+    - if: $PERFORMANCE_TESTS != "true"
+      variables:
+        PLAYBOOKS: nxcals-full-install-with-sudo.yml
+
+deploy:
+  stage: deploy
+  extends:
+    - .local_ansible

.ci/examples.yml

+include:
+  - project: acc-co/devops/common/gitlab-templates
+    file: gitlab-ci-common.yml
+    ref: 0.1.38
+  - project: acc-co/devops/common/gitlab-templates
+    file: gitlab-ci-logging.yml
+    ref: 0.1.38
+
+run-examples:
+  stage: publish-release
+  rules:
+    - if: $STAGING_PIPELINE == "true" && $PRO_RELEASE == 'true' && $CAS == 'true' #on_pro_release
+  needs:
+    - set_version:pro
+    - cas_set_alias:pro
+  variables:
+    NXCALS_VERSION: "$VERSION"
+    CI_PIPELINE_TRIGGERED: true
+    BASE_IMAGE: gitlab-registry.cern.ch/acc-co/devops/common/gitlab-runner/acc-py-java-ansible-cbng:0.5.5
+  trigger:
+    project: acc-logging-team/nxcals-examples
+    branch: master
+    strategy: depend
+    forward:
+      pipeline_variables: true
\ No newline at end of file

.ci/templates.yml

-.nxcals_deploy:
+include:
+  - project: acc-co/devops/common/gitlab-templates
+    file: gitlab-ci-common.yml
+    ref: 0.1.38
+  - project: acc-co/devops/common/gitlab-templates
+    file: gitlab-ci-logging.yml
+    ref: 0.1.38
+
+.local_integration_tests:
   extends:
-    - .nxcals_base_ansible_deploy_job
-  script:
-    - PYTIMBER_MAJOR_VERSION=$(awk -F "=" '/pytimberMajorVersion/ {print $2}' gradle.properties)
-    - PLAYBOOK_ADDITIONAL_OPTS="-u acclog -e pytimber_major_version=$PYTIMBER_MAJOR_VERSION"
-    - !reference [ .nxcals_base_ansible_deploy_job, script ]
+    - .logging_integration_tests
   variables:
-    ARTIFACT_NAME: nxcals-ansible
-    PLAYBOOKS: nxcals-full-install-with-sudo.yml
-  interruptible: false
+    INTEGRATION_TESTS_OPTIONS: "$TEST_VARIANT"
+    RBAC_ENV: 'test'
+    USER: "root"
+    VAULT_PASS: ".pass"
   tags:
     - TN
-
-
-.nxcals_infrastructure_deploy:
-  extends:
-    - .nxcals_deploy
-  variables:
-    PLAYBOOKS: "certificates-install-with-sudo.yml kafka-install-with-sudo.yml prepare-service-with-sudo.yml monit-install.yml iptables-install.yml"
-
-
-.nxcals_infrastructure_deploy:pro:
-  extends:
-    - .nxcals_infrastructure_deploy
-  before_script:
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-
-
-.nxcals_services_deploy:
-  extends:
-    - .nxcals_deploy
-  resource_group: "$RESOURCE_GROUP"
   parallel:
     matrix:
-      - PLAYBOOKS: [ "zookeeper-install.yml",
-                     "kafka-install.yml",
-                     "service-install.yml",
-                     "kafka-etl-hbase-install.yml",
-                     "kafka-etl-hdfs-install.yml",
-                     "kafka-etl-hdfs-retransmission-install.yml",
-                     "compaction-install.yml",
-                     "spark-server-install.yml",
-                     "monitoring-producer-install.yml",
-                     "monitoring-reader-install.yml",
-                     "monitoring-grok-install.yml",
-                     "grafana-install.yml",
-                     "elk-logstash-filebeat-install.yml",
-                     "alertmanager-install.yml",
-                     "prometheus-node-exporter-install.yml",
-                     "prometheus-install.yml" ]
-
+      - TEST_VARIANT: [ "-base", "-thin", "-python", "-op", "-auth" ]
 
-.nxcals_services_deploy:pro:
+.local_ansible:
   extends:
-    - .nxcals_services_deploy
-  before_script:
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-
+    - .logging_base_ansible_deploy_job
+  script:
+    - PYTIMBER_MAJOR_VERSION=$(awk -F "=" '/pytimberMajorVersion/ {print $2}' gradle.properties)
+    - PLAYBOOK_ADDITIONAL_OPTS="-e pytimber_major_version=$PYTIMBER_MAJOR_VERSION $PLAYBOOK_ADDITIONAL_OPTS"
+    - !reference [ .logging_base_ansible_deploy_job, script ]
+  variables:
+    ARTIFACT_NAME: nxcals-ansible
+  interruptible: false
+  tags:
+    - TN
 
-.nxcals_artifacts_deploy:
+.local_unit_tests:
   extends:
-    - .nxcals_deploy
+    - .gradle_unit_tests
   parallel:
     matrix:
-      - PLAYBOOKS: [ "python-wheels-upload.yml",
-                     "swan-publish.yml",
-                     "wikis-deployment-install.yml",
-                     "docs-deployment-install.yml",
-                     "spark-common-install.yml" ]
-
-
-.nxcals_artifacts_deploy:pro:
-  extends:
-    - .nxcals_artifacts_deploy
-  before_script:
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-
-
-.nxcals_python_publish:
-  extends:
-    - .base_job
-  script:
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-    - !reference [ .scripts, set_version_variable_from_gradle_properties ]
-    - !reference [ .nxcals_scripts, activate_kerberos ]
-    - |
-      cd /tmp && mkdir release && cd release
-      for NAME in nxcals-extraction-api-python3 nxcals-spark-session-builder nxcals nxcals-py4j-utils nxcals-type-stubs 
-      do
-        FILENAME="${NAME//-/_}"-"$VERSION"-py3-none-any.whl
-        curl --output "$FILENAME" http://photons-resources.cern.ch/downloads/nxcals_pro/spark/"$FILENAME"
-        acc-py devrelease "$FILENAME"
-      done
-      
-      FILENAME="pytimber"-"4.${VERSION}"-py3-none-any.whl
-      curl --output "$FILENAME" http://photons-resources.cern.ch/downloads/nxcals_pro/spark/"$FILENAME"
-      acc-py devrelease "$FILENAME"
+      - MODULE_NAME: [ "backport-api", "cern-extraction-api", "common", "compaction", "db", "docs", "extraction-api", "extraction-api-thin", "extraction-starter", "ingestion-api", "kafka-etl", "metadata-api", "monitoring-grok", "monitoring-producer", "monitoring-reader", "python", "service", "service-db", "spark-server" ]

.gitlab-ci.yml

 include:
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-common.yml
-    ref: 0.1.19
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-templates-gradle.yml
-    ref: 0.1.19
-  - project: acc-logging-team/gitlab-templates
-    file: gitlab-ci-nxcals-templates.yml
-    ref: 0.1.19
   - local: '.ci/templates.yml'
+  - local: '.ci/examples.yml'
+  - project: acc-co/devops/common/gitlab-templates
+    file: workflows/base.yml
+    ref: 0.1.38
+  - project: acc-co/devops/common/gitlab-templates
+    file: workflows/test.yml
+    ref: 0.1.38
+  - project: acc-co/devops/common/gitlab-templates
+    file: workflows/staging.yml
+    ref: 0.1.38
 
 variables:
   ANSIBLE_FILES: ansible
-  BASE_IMAGE: registry.cern.ch/acc-logging-team/gitlab-runner:0.5.1.4
+  BASE_IMAGE: gitlab-registry.cern.ch/acc-co/devops/common/gitlab-runner/acc-py-java-ansible-jf-hadoop:0.5.5
   HTTPS_PROXY: "http://photons-resources:8888"
   HTTP_PROXY: "http://photons-resources:8888"
   MAIN_BRANCH: "develop"
   NO_PROXY: ".cern.ch,localhost,127.0.0.1"
   SONAR_HOST_URL: http://nxcals-jenkins-master.cern.ch:9000
+  PIPELINE_NAME: "$CI_COMMIT_NAME"
+  PUBLISH_IMAGES: true
+
+workflow:
+  name: "$PIPELINE_NAME"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      variables:
+        PIPELINE_NAME: "[Merge request] pipeline for $CI_COMMIT_SHORT_SHA - $CI_COMMIT_MESSAGE"
+    - if: $CI_COMMIT_BRANCH == $MAIN_BRANCH && $STAGING_PIPELINE == 'true'
+      variables:
+        PIPELINE_NAME: "[Staging] Staging pipeline for $CI_COMMIT_SHORT_SHA"
+        KAFKA_SERVERS_URL: 'https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092,https://nxcals-staging6.cern.ch:9092'
+        SERVICE_URL: 'https://nxcals-staging4.cern.ch:19093,https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
+        SPARK_SERVERS_URL: 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
+        STAGING_ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-staging-local/cern/nxcals/nxcals-ansible'
+        PRO_ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-release-local/cern/nxcals/nxcals-ansible'
+        REGRESSION_TESTS: true
+        ACCPY: true
+        CAS: true
+        MATTERMOST_RELEASE_URL: "$MATTERMOST_NXCALS_RELEASE_URL"
+        MATTERMOST_MESSAGE: "A new version of NXCALS PRO - $$VERSION and PyTimber - 4.$$VERSION has been released. Release notes: [$$VERSION](https://gitlab.cern.ch/acc-logging-team/nxcals/-/releases/$$VERSION)"
+        CAS_MODULES: "nxcals-hadoop-dev-config nxcals-hadoop-pro-config nxcals-common nxcals-metadata-api nxcals-cern-metadata-api nxcals-extraction-api nxcals-cern-extraction-api nxcals-ingestion-api nxcals-extraction-starter nxcals-backport-api nxcals-extraction-api-thin"
+        ARTIFACT_GROUP_ID: "cern.nxcals"
+        CAS_REPOSITORY: ds-release-local
+        CAS_ALIAS_NAME: PRO
+        PYTHON_PACKAGE_SUFFIX: "py3-none-any.whl"
+        ACCPY_ARTIFACTS: "nxcals_extraction_api_python3-$$VERSION-$PYTHON_PACKAGE_SUFFIX nxcals_spark_session_builder-$$VERSION-$$PYTHON_PACKAGE_SUFFIX nxcals_py4j_utils-$$VERSION-$$PYTHON_PACKAGE_SUFFIX nxcals_type_stubs-$$VERSION-$$PYTHON_PACKAGE_SUFFIX nxcals-$$VERSION-$$PYTHON_PACKAGE_SUFFIX pytimber-4.$$VERSION-$PYTHON_PACKAGE_SUFFIX"
+        ACCPY_ARTIFACT_DOWNLOAD_PATH: http://photons-resources.cern.ch/downloads/nxcals_pro/spark
+    - if: $CI_COMMIT_BRANCH == $MAIN_BRANCH && $STAGING_PIPELINE != 'true'
+      variables:
+        PIPELINE_NAME: "[Test] Test pipeline for $CI_COMMIT_SHORT_SHA"
+        TEST_PIPELINE: "true"
+        KAFKA_SERVERS_URL: 'https://nxcals-test4.cern.ch:9092,https://nxcals-test5.cern.ch:9092,https://nxcals-test6.cern.ch:9092'
+        SERVICE_URL: 'https://nxcals-test4.cern.ch:19093,https://nxcals-test5.cern.ch:19093,https://nxcals-test6.cern.ch:19093'
+        SPARK_SERVERS_URL: 'nxcals-test5.cern.ch:15000,nxcals-test6.cern.ch:15000'
+        TEST_ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-development-local/cern/nxcals/nxcals-ansible'
+        PERFORMANCE_TESTS: true
+    - when: never
 
 stages:
   - build
@@ -30,301 +70,6 @@ stages:
   - deploy-release
   - publish-release
 
-build_project:
-  stage: build
-  extends:
-    - .base_build_project_job
-  rules:
-    - !reference [ .run_rules, 'on:merge_request' ]
-    - !reference [ .run_rules, 'on:main_branch' ]
-  after_script:
-    - find . | grep venv$ | xargs rm -r # remove all venvs
-    - find . | grep Venv$ | xargs rm -r # remove all venvs
-    - rm -r python/type-stubs/build/*ClasspathJars
-  variables:
-    BUILD_ADDITIONAL_OPTS: "--stacktrace -x distZip --parallel"
-
-unit_tests:
-  stage: test
-  extends:
-    - .base_unit_tests_job
-  rules:
-    - !reference [ .run_rules, 'on:merge_request' ]
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: build_project
-      artifacts: true
-  variables:
-    ACTIVE_PROFILE: "gitlab-ci"
-    GITLAB_MR_USER_NAME: "$GITLAB_USER_LOGIN"
-    TEST_ADDITIONAL_OPTS: "-p $MODULE_NAME --parallel"
-  parallel:
-    matrix:
-      - MODULE_NAME: [ "backport-api", "cern-extraction-api", "common", "compaction", "db", "docs", "extraction-api", "extraction-api-thin", "extraction-starter", "ingestion-api", "kafka-etl", "metadata-api", "monitoring-grok", "monitoring-producer", "monitoring-reader", "python", "service", "service-db", "spark-server" ]
-
-sonar:
-  stage: qa
-  extends:
-    - .base_sonar_job
-  rules:
-    - !reference [ .run_rules, 'on:merge_request' ]
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: unit_tests
-      artifacts: true
-
-publish_to_artifactory:test:
-  stage: publish
-  extends:
-    - .base_artifactory_publish:dev
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: build_project
-      artifacts: true
-    - job: sonar
-  variables:
-    BUILD_NAME: nxcals-test
-  artifacts:
-
-deploy:test:
-  stage: deploy
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  trigger:
-    include:
-      - local: .ci/deploy-snapshot.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs:
-    - publish_to_artifactory:test
-  variables:
-    ANSIBLE_INVENTORY: 'test'
-    ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-development-local/cern/nxcals/nxcals-ansible'
-    RESOURCE_GROUP: 'deploy-test'
-
-integration_tests:test:
-  stage: integration-tests
-  extends:
-    - .nxcals_integration_tests
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - deploy:test
-  variables:
-    INTEGRATION_TESTS_OPTIONS: "$TEST_VARIANT"
-    JENKINS_ENV: "test"
-    KAFKA_SERVERS_URL: 'https://nxcals-test4.cern.ch:9092,https://nxcals-test5.cern.ch:9092,https://nxcals-test6.cern.ch:9092'
-    RBAC_ENV: 'test'
-    SERVICE_URL: 'https://nxcals-test4.cern.ch:19093,https://nxcals-test5.cern.ch:19093,https://nxcals-test6.cern.ch:19093'
-    SPARK_SERVERS_URL: 'nxcals-test5.cern.ch:15000,nxcals-test6.cern.ch:15000'
-    USER: "root"
-    VAULT_PASS: ".pass"
-  resource_group: integration-tests-test
-  tags:
-    - TN
-  parallel:
-    matrix:
-      - TEST_VARIANT: [ "-base", "-thin", "-python", "-op", "-auth" ]
-
-deploy:perf-test:
-  stage: deploy
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  trigger:
-    include:
-      - local: .ci/deploy-snapshot.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs:
-    - publish_to_artifactory:test
-  variables:
-    ANSIBLE_INVENTORY: 'perf-test'
-    ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-development-local/cern/nxcals/nxcals-ansible'
-    RESOURCE_GROUP: 'deploy-perf-test'
-
-performance_tests:test:
-  stage: integration-tests
-  extends:
-    - .base_job
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - deploy:perf-test
-  script:
-    - echo "$ANSIBLE_VAULT" > ${ANSIBLE_FILES}/.pass
-    - ./performance-tests.sh
-  variables:
-    USER: "root"
-  tags:
-    - TN
-  timeout: 5h
-  allow_failure: true
-
-publish_to_artifactory:staging:
-  stage: publish
-  extends:
-    - .base_artifactory_publish:pro
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: sonar
-  variables:
-    BUILD_NAME: nxcals-staging
-  artifacts:
-
-deploy:staging:
-  stage: deploy
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  trigger:
-    include:
-      - local: .ci/deploy-pro.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs:
-    - publish_to_artifactory:staging
-  variables:
-    ANSIBLE_INVENTORY: 'staging'
-    ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-staging-local/cern/nxcals/nxcals-ansible'
-    RESOURCE_GROUP: 'deploy-staging'
-
-integration_tests:staging:
-  stage: integration-tests
-  extends:
-    - .nxcals_integration_tests
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - deploy:staging
-  before_script:
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-  variables:
-    INTEGRATION_TESTS_OPTIONS: "$TEST_VARIANT"
-    JENKINS_ENV: "staging"
-    KAFKA_SERVERS_URL: 'https://nxcals-staging4.cern.ch:9092,https://nxcals-staging5.cern.ch:9092,https://nxcals-staging6.cern.ch:9092'
-    RBAC_ENV: 'test'
-    SERVICE_URL: 'https://nxcals-staging4.cern.ch:19093,https://nxcals-staging5.cern.ch:19093,https://nxcals-staging6.cern.ch:19093'
-    SPARK_SERVERS_URL: 'nxcals-staging5.cern.ch:15000,nxcals-staging6.cern.ch:15000'
-    USER: "root"
-    VAULT_PASS: ".pass"
-  resource_group: integration-tests-staging
-  tags:
-    - TN
-  parallel:
-    matrix:
-      - TEST_VARIANT: [ "-base", "-thin", "-python", "-op", "-auth" ]
-
-publish_to_artifactory:pro:
-  stage: release
-  extends:
-    - .nxcals_release:pro
-  rules:
-    - !reference [ .run_rules, 'manual:on:main_branch' ]
-  needs:
-    - job: build_project
-      artifacts: true
-    - job: integration_tests:staging
-    - job: integration_tests:test
-    - job: performance_tests:test
-  variables:
-    BUILD_NAME: nxcals-staging
-  artifacts:
-
-deploy:pro:
-  stage: deploy-release
-  rules:
-    - !reference [ .run_rules, 'manual:on:main_branch' ]
-  trigger:
-    include:
-      - local: .ci/deploy-pro.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs:
-    - publish_to_artifactory:pro
-  variables:
-    ANSIBLE_INVENTORY: 'pro'
-    ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-release-local/cern/nxcals/nxcals-ansible'
-    RESOURCE_GROUP: 'deploy-pro'
-
-deploy:testbed:
-  stage: deploy-release
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  trigger:
-    include:
-      - local: .ci/deploy-pro.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs:
-    - publish_to_artifactory:pro
-  variables:
-    ANSIBLE_INVENTORY: 'testbed'
-    ANSIBLE_ZIP_LINK: 'https://artifactory.cern.ch/artifactory/ds-release-local/cern/nxcals/nxcals-ansible'
-    RESOURCE_GROUP: 'deploy-testbed'
-
-acc_py_publish:pro:
-  stage: publish-release
-  extends:
-    - .nxcals_python_publish
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: publish_to_artifactory:pro
-    - job: deploy:pro
-
-cas_publish:pro:
-  stage: publish-release
-  extends:
-    - .base_cas_register_artifacts
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: publish_to_artifactory:pro
-      artifacts: false
-  variables:
-    ARTIFACTS: "nxcals-hadoop-dev-config nxcals-hadoop-pro-config nxcals-common nxcals-metadata-api nxcals-cern-metadata-api nxcals-extraction-api nxcals-cern-extraction-api nxcals-ingestion-api nxcals-extraction-starter nxcals-backport-api nxcals-extraction-api-thin"
-    GROUP_ID: "cern.nxcals"
-    REPOSITORY: "ds-release-local"
-  before_script:
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-
-cas_set_alias:pro:
-  stage: publish-release
-  extends:
-    - .base_cas_alias_artifacts
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: cas_publish:pro
-    - job: deploy:pro
-  variables:
-    ARTIFACTS: "nxcals-hadoop-dev-config nxcals-hadoop-pro-config nxcals-common nxcals-metadata-api nxcals-cern-metadata-api nxcals-extraction-api nxcals-cern-extraction-api nxcals-ingestion-api nxcals-extraction-starter nxcals-backport-api nxcals-extraction-api-thin"
-    GROUP_ID: "cern.nxcals"
-    REPOSITORY: "ds-release-local"
-    ALIAS: "PRO"
-  before_script:
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-
-mattermost_notification:
-  stage: publish-release
-  extends: .base_job
-  rules:
-    - !reference [ .run_rules, 'on:main_branch' ]
-  needs:
-    - job: cas_set_alias:pro
-  script:
-    - name_of_variable_to_check=MATTERMOST_NXCALS_RELEASE_URL
-    - !reference [ .scripts, check_variable ]
-
-    - !reference [ .scripts, remove_snapshot_from_version_number ]
-    - !reference [ .scripts, set_version_variable_from_gradle_properties ]
-    - |
-      MESSAGE="A new version of NXCALS PRO - ${VERSION} and PyTimber - 4.${VERSION} has been released. For more details: https://confluence.cern.ch/display/NXCALS/${VERSION}"
-    - "curl -i -X POST -H 'Content-Type: application/json' -d '{ \"text\":
-       \"'\"${MESSAGE}\"'\"}' $MATTERMOST_NXCALS_RELEASE_URL"
+after_script:
+  - find . | grep -i venv$ | xargs rm -r # remove all venvs
+  - rm -r python/type-stubs/build/*ClasspathJars
\ No newline at end of file

ansible/build.gradle

@@ -3,7 +3,7 @@
  */
 
 apply plugin: 'distribution'
-apply from: '../zipDistribution.gradle'
+apply from: "$gradleUtilsRepository/zipDistribution.gradle/raw?ref=$gradleUtilsVersion"
 
 distributions {
     main {
@@ -68,7 +68,7 @@ task extractCommons(type: Copy, dependsOn: cleanupCommons) {
 
 
 dependencies {
-    ansibleCommon group: 'cern.nxcals.ansible', name: 'ansible-common', version: ansibleCommonVersion, ext: 'zip'
+    ansibleCommon group: 'cern.accdevops.ansible', name: 'ansible-common', version: ansibleCommonVersion, ext: 'zip'
 }
 
 clean.dependsOn cleanupCommons

ansible/gitlab-runner-install.yml

+# After provisioning node, it must be registered. To do so, you need to start process on gitlab
+# Then register node with docker: gitlab-runner register --url https://gitlab.cern.ch
+# Final step is fine-tuning configuration in /etc/gitlab-runner/config.toml
+# Required options:
+# [[runners]]
+# environment = ["SPARK_DRIVER_HOST=nxcals-gitlab-runner-1.cern.ch"]
+# [runners.docker]
+# ulimit = "host"
+# network_mode = "host"
+# disable_cache = true
+# And configure docker socket by:
+# systemctl --user status podman.socket | grep Listen
+# ln -s /run/user/0/podman/podman.sock /var/run/docker.sock
+# Final step is adding node to TN (if needed)
+# More details: https://confluence.cern.ch/pages/viewpage.action?pageId=414047432
+
+- hosts: gitlab-runner
+  strategy: free
+  vars:
+    http_proxy: '{{proxy}}'
+    https_proxy: '{{proxy}}'
+  vars_files:
+    - '{{inventory_dir}}/group_vars/vault-encrypted'
+    - '{{inventory_dir}}/group_vars/all.yml'
+  roles:
+    - { role: openstack, openstack_yum_extra_packages: ["git", "iptables-services", "curl", "podman"], user: "acclog", user_pub_key: "./files/acclog-id_rsa.pub", user_priv_key: "acclog-id_rsa-encrypted", become: true }
+    - { role: sudoers, users: '{{nxcals_developers}}', become: true }
+    - { role: gitlab-runner, become: true }

ansible/group_vars/all/all.yml

@@ -16,7 +16,7 @@ ingestion_allowed_external_hosts: ['cs-ccr-nxdev1.cern.ch', 'cs-ccr-nxdev2.cern.
 jenkins_hosts: ['nxcals-jenkins-master.cern.ch',
                 'nxcals-jenkins-slave1.cern.ch','nxcals-jenkins-slave2.cern.ch', 'nxcals-jenkins-slave3.cern.ch',
                 'nxcals-jenkins-slave-1.cern.ch', 'nxcals-jenkins-slave-2.cern.ch', 'nxcals-jenkins-slave-3.cern.ch',
-                'nxcals-gitlab-runner-1.cern.ch']
+                'nxcals-gitlab-runner-1.cern.ch', 'nxcals-gitlab-runner-2.cern.ch']
 
 nxcals_ds_cmw_hosts: []
 #The maximum record size. For future use, currently it is not referenced anywhere as we have some
@@ -341,7 +341,7 @@ service_compaction_max_partition_size: 1573741824
 service_compaction_tolerance_hours: 26
 service_compaction_migration_tolerance_hours: 1
 service_compaction_min_file_modification_hours: 2
-service_compaction_staging_partition_splits: "FIVE_MINUTE_SPLIT"
+service_compaction_staging_partition_splits: "ONE_HOUR_SPLIT"
 service_kerberos_keytab: '{{install_dir}}/.service.keytab'
 service_kerberos_realm: 'CERN.CH'
 service_kerberos_type: 'HTTPS'
@@ -961,5 +961,3 @@ ignore_eos_errors: false
 #Certificate
 #################################################################################
 valid_days_threshold_for_cert_renewal: 180
-
-rhel9: true

ansible/inventory/openstack/group_vars/all.yml

 #Sonar
 #################################################################################
-sonar_version: "7.9.1"
+sonar_version: "9.5.0.56709"
 sonar_db_url: "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=cerndbu-s.cern.ch)(PORT=10121))(LOAD_BALANCE=on)(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=cerndbu_s.cern.ch)(FAILOVER_MODE=(TYPE=SELECT)(METHOD=BASIC))))"
 sonar_db_user: nxcals_sonar
 sonar_db_user_password: '{{vault_sonar_db_user_password}}'
+nxcals_namespace: "openstack"

ansible/inventory/openstack/hosts.yml

@@ -5,6 +5,8 @@ nxcals-migration[1:10].cern.ch
 nxcals-jenkins-master.cern.ch
 nxcals-jenkins-slave[1:2].cern.ch
 photons-resources.cern.ch
+nxcals-gitlab-runner-[1:2].cern.ch
+nxcals-sonarqube.cern.ch
 
 [websources]
 photons-resources.cern.ch
@@ -19,4 +21,7 @@ nxcals-jenkins-slave-test.cern.ch
 nxcals-migration[1:10].cern.ch
 
 [sonar]
-nxcals-jenkins-master.cern.ch
+nxcals-sonarqube.cern.ch
+
+[gitlab-runner]
+nxcals-gitlab-runner-[1:2].cern.ch
\ No newline at end of file

ansible/openstack-install-with-sudo.yml

@@ -4,7 +4,7 @@
     - '{{inventory_dir}}/group_vars/vault-encrypted'
     - '{{inventory_dir}}/group_vars/all.yml'
   roles:
-    - { role: openstack, rhel9: true, openstack_yum_extra_packages: ["git", "iptables-services"], user: "acclog", user_pub_key: "./files/acclog-id_rsa.pub", user_priv_key: "acclog-id_rsa-encrypted", become: true }
+    - { role: openstack, openstack_yum_extra_packages: ["git", "iptables-services"], user: "acclog", user_pub_key: "./files/acclog-id_rsa.pub", user_priv_key: "acclog-id_rsa-encrypted", become: true }
     - { role: sudoers, users: '{{nxcals_developers}}', become: true }
     - { role: dnsmasq, become: true }
     - { role: jdk-copy, jdk_target_dir: /usr/java, become: true }

ansible/roles/gitlab-runner/tasks/install-gitlab-runner.yml

+- name: add gitlab yum repo
+  shell: curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh" | sudo bash
+  become: true
+
+- name: install gitlab-runner
+  become: true
+  yum:
+    name: gitlab-runner
+    state: present
+    update_cache: true
\ No newline at end of file

ansible/roles/gitlab-runner/tasks/main.yml

+- import_tasks: install-gitlab-runner.yml

ansible/roles/sonar/tasks/download.yml

+- name: check download dir {{download_dir}}
+  file:
+    path: '{{download_dir}}'
+    state: directory
+
 - name: download sonar
   import_role:
     name: common-download
   vars:
-    download_url: "https://sonarsource.bintray.com/CommercialDistribution/sonarqube-developer/sonarqube-developer-{{sonar_version}}.zip"
+    download_url: "https://binaries.sonarsource.com/CommercialDistribution/sonarqube-developer/sonarqube-developer-{{sonar_version}}.zip"
+    download_path: "{{download_dir}}/sonarqube-developer-{{sonar_version}}.zip"
+    download_resource: 'sonarqube-developer-{{sonar_version}}.zip'
 

backport-api/build.gradle

-apply from: '../jacoco.gradle'
+apply from: "$gradleUtilsRepository/jacoco.gradle/raw?ref=$gradleUtilsVersion"
 
 dependencies {
     api project(':nxcals-extraction-starter')

cern-extraction-api/build.gradle

-apply from: '../jacoco.gradle'
+apply from: "$gradleUtilsRepository/jacoco.gradle/raw?ref=$gradleUtilsVersion"
 dependencies {
     api project(':nxcals-extraction-api')
 

cern-metadata-api/build.gradle

-apply from: '../jacoco.gradle'
+apply from: "$gradleUtilsRepository/jacoco.gradle/raw?ref=$gradleUtilsVersion"
 dependencies {
     api project(':nxcals-metadata-api')
     api group: 'com.google.code.gson', name: 'gson', version: gsonVersion

cern-metadata-api/src/test/java/cern/nxcals/api/custom/extraction/metadata/SnapshotServiceImplTest.java

@@ -167,7 +167,6 @@ class SnapshotServiceImplTest {
     }
 
 
-
     @Test
     void getVariables() {
         //given

common/build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'com.google.protobuf' version "$gradleProtobufPluginVersion"
 }
 
-apply from: '../jacoco.gradle'
+apply from: "$gradleUtilsRepository/jacoco.gradle/raw?ref=$gradleUtilsVersion"
 
 configurations.configureEach {
     exclude group: "org.springframework.boot", module: "spring-boot-starter-logging"
@@ -54,20 +54,20 @@ dependencies {
     compileOnly group: 'io.micrometer', name: 'micrometer-core', version: micrometerVersion
     compileOnly group: 'io.micrometer', name: 'micrometer-registry-jmx', version: micrometerVersion
 
-    compileOnly (group: 'org.apache.spark', name: 'spark-core_2.12', version: sparkVersion) {
+    compileOnly(group: 'org.apache.spark', name: 'spark-core_2.12', version: sparkVersion) {
         exclude group: 'org.apache.avro', module: 'avro'
         exclude group: 'org.apache.avro', module: 'avro-mapred'
         exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j2-impl'
         exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
         exclude group: 'org.apache.spark', module: 'spark-common-utils_2.12'
     }
-    compileOnly (group: 'org.apache.spark', name: 'spark-yarn_2.12', version: sparkVersion) {
+    compileOnly(group: 'org.apache.spark', name: 'spark-yarn_2.12', version: sparkVersion) {
         exclude group: 'org.apache.avro', module: 'avro'
         exclude group: 'org.apache.avro', module: 'avro-mapred'
         exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j2-impl'
         exclude group: 'org.apache.spark', module: 'spark-core_2.12'
     }
-    compileOnly (group: 'org.apache.spark', name: 'spark-sql_2.12', version: sparkVersion) {
+    compileOnly(group: 'org.apache.spark', name: 'spark-sql_2.12', version: sparkVersion) {
         exclude group: 'com.fasterxml.jackson.datatype', module: 'jackson-datatype-jsr310'
         exclude group: 'org.apache.avro', module: 'avro'
         exclude group: 'org.apache.avro', module: 'avro-mapred'
@@ -76,14 +76,14 @@ dependencies {
         exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
         exclude group: 'org.apache.spark', module: 'spark-common-utils_2.12'
     }
-    compileOnly (group: 'org.apache.spark', name: 'spark-hive_2.12', version: sparkVersion) {
+    compileOnly(group: 'org.apache.spark', name: 'spark-hive_2.12', version: sparkVersion) {
         exclude group: 'org.apache.avro', module: 'avro'
         exclude group: 'org.apache.avro', module: 'avro-mapred'
         exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j2-impl'
         exclude group: 'org.apache.spark', module: 'spark-core_2.12'
     }
 
-    compileOnly (group: "org.apache.spark", name: "spark-common-utils_2.12", version: sparkVersion) {
+    compileOnly(group: "org.apache.spark", name: "spark-common-utils_2.12", version: sparkVersion) {
         exclude group: 'com.fasterxml.jackson.module', module: 'jackson-module-scala_2.12'
         exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
         exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j2-impl'