From d8bd5d1cfc81f18f6fa74545e85472a1a5e1d5d1 Mon Sep 17 00:00:00 2001 From: Alex Iribarren <Alex.Iribarren@cern.ch> Date: Mon, 24 Sep 2018 18:04:31 +0200 Subject: [PATCH 1/4] Get reposync job name from environment variable --- reposync_oneshot.nomad | 2 +- reposync_scheduler.nomad | 2 +- scheduler/schedule.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/reposync_oneshot.nomad b/reposync_oneshot.nomad index 51d9bc6..bb68e3f 100644 --- a/reposync_oneshot.nomad +++ b/reposync_oneshot.nomad @@ -34,7 +34,7 @@ job "${JOB_PREFIX}reposync_oneshot" { env { NOMAD_ADDR = "$NOMAD_ADDR" - VAULT_ADDR = "$VAULT_ADDR" + REPOSYNC_JOB = "${JOB_PREFIX}reposync" } resources { diff --git a/reposync_scheduler.nomad b/reposync_scheduler.nomad index 2e87815..a2e866d 100644 --- a/reposync_scheduler.nomad +++ b/reposync_scheduler.nomad @@ -37,7 +37,7 @@ job "${JOB_PREFIX}reposync_scheduler" { env { NOMAD_ADDR = "$NOMAD_ADDR" - VAULT_ADDR = "$VAULT_ADDR" + REPOSYNC_JOB = "${JOB_PREFIX}reposync" } resources { diff --git a/scheduler/schedule.py b/scheduler/schedule.py index 14e2d59..b9120ce 100755 --- a/scheduler/schedule.py +++ b/scheduler/schedule.py @@ -13,8 +13,8 @@ import itertools CONFIG = '/root/repos.yaml' REPOPATH = '/etc/yum.repos.d/' -JOBNAME = 'reposync' -STATUS_CHECK_SECS = 5 +JOBNAME = os.getenv('REPOSYNC_JOB', 'reposync') +STATUS_CHECK_SECS = os.getenv('STATUS_CHECKS', 5) REPOID = os.getenv('NOMAD_META_REPOID', None) def log(**kwargs): -- GitLab From 7605193603fbdc0031778a441b2468cbd1cc20a9 Mon Sep 17 00:00:00 2001 From: Alex Iribarren <Alex.Iribarren@cern.ch> Date: Wed, 26 Sep 2018 14:23:15 +0200 Subject: [PATCH 2/4] Fix force_pull options and simplify CI --- .gitlab-ci.yml | 56 +++------------------------------------- reposync.nomad | 3 ++- reposync_oneshot.nomad | 3 ++- reposync_scheduler.nomad | 3 ++- 4 files changed, 10 insertions(+), 55 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2152d0a..237cd7c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,9 +35,7 @@ build_reposync: script: "echo" # unused but this line is required by GitLab CI variables: CONTEXT_DIR: reposync - TO: $CI_REGISTRY_IMAGE/do_reposync:latest # latest is the image used in production - only: - - master + TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_COMMIT_REF_NAME build_scheduler: stage: build @@ -46,53 +44,7 @@ build_scheduler: script: "echo" # unused but this line is required by GitLab CI variables: CONTEXT_DIR: scheduler - TO: $CI_REGISTRY_IMAGE/reposync_scheduler:latest # latest is the image used in production - only: - - master - -build_reposync_branch: - stage: build - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: reposync - TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_BUILD_REF_NAME # put name of branch in image - except: - - master - -build_scheduler_branch: - stage: build - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: scheduler - TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_BUILD_REF_NAME # put name of branch in image - except: - - master - -tag_reposync: - stage: tag - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: reposync - TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_COMMIT_TAG - only: - - tags - -tag_scheduler: - stage: tag - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: scheduler - TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_COMMIT_TAG - only: - - tags + TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_COMMIT_REF_NAME .nomad_deps_template: &nomad_deps @@ -100,9 +52,9 @@ tag_scheduler: curl https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip -o /tmp/nomad.zip && unzip /tmp/vault.zip -d /usr/local/bin/ && unzip /tmp/nomad.zip -d /usr/local/bin/ && - export IMAGE_VER="${CI_COMMIT_TAG:-latest}" && vault read nomad/creds/submitjobs -format=json > token.json && - export `jq -r '"NOMAD_TOKEN=" + .data.secret_id' token.json` + export `jq -r '"NOMAD_TOKEN=" + .data.secret_id' token.json` && + if [ -z ${CI_COMMIT_TAG+x} ]; then export FORCE_PULL="true"; else export FORCE_PULL="false"; fi deploy_test: stage: deploy_test diff --git a/reposync.nomad b/reposync.nomad index 959a581..96a287c 100644 --- a/reposync.nomad +++ b/reposync.nomad @@ -12,7 +12,8 @@ job "${JOB_PREFIX}reposync" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/do_reposync:${IMAGE_VER}" + image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/do_reposync:${CI_COMMIT_REF_NAME}" + force_pull = "${FORCE_PULL}" logging { type = "gelf" config { diff --git a/reposync_oneshot.nomad b/reposync_oneshot.nomad index bb68e3f..5da4bc0 100644 --- a/reposync_oneshot.nomad +++ b/reposync_oneshot.nomad @@ -11,7 +11,8 @@ job "${JOB_PREFIX}reposync_oneshot" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${IMAGE_VER}" + image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}" + force_pull = "${FORCE_PULL}" logging { type = "gelf" config { diff --git a/reposync_scheduler.nomad b/reposync_scheduler.nomad index a2e866d..a1158ff 100644 --- a/reposync_scheduler.nomad +++ b/reposync_scheduler.nomad @@ -14,7 +14,8 @@ job "${JOB_PREFIX}reposync_scheduler" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${IMAGE_VER}" + image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}" + force_pull = "${FORCE_PULL}" logging { type = "gelf" config { -- GitLab From 20981fe901d56cd59ca0d72dece1cf6a9bb6dc19 Mon Sep 17 00:00:00 2001 From: Alex Iribarren <Alex.Iribarren@cern.ch> Date: Wed, 26 Sep 2018 14:25:17 +0200 Subject: [PATCH 3/4] Fix types --- reposync.nomad | 2 +- reposync_oneshot.nomad | 2 +- reposync_scheduler.nomad | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/reposync.nomad b/reposync.nomad index 96a287c..162d4ab 100644 --- a/reposync.nomad +++ b/reposync.nomad @@ -13,7 +13,7 @@ job "${JOB_PREFIX}reposync" { config { image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/do_reposync:${CI_COMMIT_REF_NAME}" - force_pull = "${FORCE_PULL}" + force_pull = ${FORCE_PULL} logging { type = "gelf" config { diff --git a/reposync_oneshot.nomad b/reposync_oneshot.nomad index 5da4bc0..940035c 100644 --- a/reposync_oneshot.nomad +++ b/reposync_oneshot.nomad @@ -12,7 +12,7 @@ job "${JOB_PREFIX}reposync_oneshot" { config { image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}" - force_pull = "${FORCE_PULL}" + force_pull = ${FORCE_PULL} logging { type = "gelf" config { diff --git a/reposync_scheduler.nomad b/reposync_scheduler.nomad index a1158ff..7968097 100644 --- a/reposync_scheduler.nomad +++ b/reposync_scheduler.nomad @@ -15,7 +15,7 @@ job "${JOB_PREFIX}reposync_scheduler" { config { image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}" - force_pull = "${FORCE_PULL}" + force_pull = ${FORCE_PULL} logging { type = "gelf" config { -- GitLab From 8900e52161015b0760dd891e29115702f0b183c8 Mon Sep 17 00:00:00 2001 From: Alex Iribarren <Alex.Iribarren@cern.ch> Date: Wed, 26 Sep 2018 14:39:20 +0200 Subject: [PATCH 4/4] Delete testing jobs when we move to production --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 237cd7c..7275579 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ stages: - tag - deploy_test - deploy + - cleanup prepare_dirs: stage: prepare @@ -72,3 +73,15 @@ deploy: script: - *nomad_deps - for j in *.nomad; do echo -e "\n${j}:"; nomad job run <(envsubst < $j); done + +stop_testing: + stage: cleanup + dependencies: + - deploy + script: + - *nomad_deps + - export JOB_PREFIX="${DEV_JOB_PREFIX}" + - for j in *.nomad; do echo -e "\n${j}:"; JOB=`grep task <(envsubst < $j) | sed 's/.*"\([^"]*\)".*/\1/'`; nomad job stop -purge ${JOB}; done + only: + - tags + - master -- GitLab