diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2152d0aa148f2005d07f86efc180887f6165ee10..727557970bfb93428542aa09e9e11d763e742435 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ stages: - tag - deploy_test - deploy + - cleanup prepare_dirs: stage: prepare @@ -35,9 +36,7 @@ build_reposync: script: "echo" # unused but this line is required by GitLab CI variables: CONTEXT_DIR: reposync - TO: $CI_REGISTRY_IMAGE/do_reposync:latest # latest is the image used in production - only: - - master + TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_COMMIT_REF_NAME build_scheduler: stage: build @@ -46,53 +45,7 @@ build_scheduler: script: "echo" # unused but this line is required by GitLab CI variables: CONTEXT_DIR: scheduler - TO: $CI_REGISTRY_IMAGE/reposync_scheduler:latest # latest is the image used in production - only: - - master - -build_reposync_branch: - stage: build - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: reposync - TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_BUILD_REF_NAME # put name of branch in image - except: - - master - -build_scheduler_branch: - stage: build - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: scheduler - TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_BUILD_REF_NAME # put name of branch in image - except: - - master - -tag_reposync: - stage: tag - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: reposync - TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_COMMIT_TAG - only: - - tags - -tag_scheduler: - stage: tag - tags: - - docker-image-build - script: "echo" # unused but this line is required by GitLab CI - variables: - CONTEXT_DIR: scheduler - TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_COMMIT_TAG - only: - - tags + TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_COMMIT_REF_NAME .nomad_deps_template: &nomad_deps @@ -100,9 +53,9 @@ tag_scheduler: curl https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip -o /tmp/nomad.zip && unzip /tmp/vault.zip -d /usr/local/bin/ && unzip /tmp/nomad.zip -d /usr/local/bin/ && - export IMAGE_VER="${CI_COMMIT_TAG:-latest}" && vault read nomad/creds/submitjobs -format=json > token.json && - export `jq -r '"NOMAD_TOKEN=" + .data.secret_id' token.json` + export `jq -r '"NOMAD_TOKEN=" + .data.secret_id' token.json` && + if [ -z ${CI_COMMIT_TAG+x} ]; then export FORCE_PULL="true"; else export FORCE_PULL="false"; fi deploy_test: stage: deploy_test @@ -120,3 +73,15 @@ deploy: script: - *nomad_deps - for j in *.nomad; do echo -e "\n${j}:"; nomad job run <(envsubst < $j); done + +stop_testing: + stage: cleanup + dependencies: + - deploy + script: + - *nomad_deps + - export JOB_PREFIX="${DEV_JOB_PREFIX}" + - for j in *.nomad; do echo -e "\n${j}:"; JOB=`grep task <(envsubst < $j) | sed 's/.*"\([^"]*\)".*/\1/'`; nomad job stop -purge ${JOB}; done + only: + - tags + - master diff --git a/reposync.nomad b/reposync.nomad index 959a5818cca13971c7d0f95f162e57e94438fe6d..162d4abccd3d72dfb53495f530c372e07873c4ed 100644 --- a/reposync.nomad +++ b/reposync.nomad @@ -12,7 +12,8 @@ job "${JOB_PREFIX}reposync" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/do_reposync:${IMAGE_VER}" + image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/do_reposync:${CI_COMMIT_REF_NAME}" + force_pull = ${FORCE_PULL} logging { type = "gelf" config { diff --git a/reposync_oneshot.nomad b/reposync_oneshot.nomad index 51d9bc64abbe92619f2b6271c1687f943f127fe2..940035c9fec4f82835c4e0fc1cc6b470be9b118c 100644 --- a/reposync_oneshot.nomad +++ b/reposync_oneshot.nomad @@ -11,7 +11,8 @@ job "${JOB_PREFIX}reposync_oneshot" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${IMAGE_VER}" + image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}" + force_pull = ${FORCE_PULL} logging { type = "gelf" config { @@ -34,7 +35,7 @@ job "${JOB_PREFIX}reposync_oneshot" { env { NOMAD_ADDR = "$NOMAD_ADDR" - VAULT_ADDR = "$VAULT_ADDR" + REPOSYNC_JOB = "${JOB_PREFIX}reposync" } resources { diff --git a/reposync_scheduler.nomad b/reposync_scheduler.nomad index 2e87815828afcb3f8cd441ebd2f965029478148f..79680970ebf0b7ba65c0ddd427dff739a4f24dad 100644 --- a/reposync_scheduler.nomad +++ b/reposync_scheduler.nomad @@ -14,7 +14,8 @@ job "${JOB_PREFIX}reposync_scheduler" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${IMAGE_VER}" + image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}" + force_pull = ${FORCE_PULL} logging { type = "gelf" config { @@ -37,7 +38,7 @@ job "${JOB_PREFIX}reposync_scheduler" { env { NOMAD_ADDR = "$NOMAD_ADDR" - VAULT_ADDR = "$VAULT_ADDR" + REPOSYNC_JOB = "${JOB_PREFIX}reposync" } resources { diff --git a/scheduler/schedule.py b/scheduler/schedule.py index 14e2d59f7aa15ff57dfe1bf66cd5533f01e03bc5..b9120ceda02521b11302398129511dcbcbc2f5d0 100755 --- a/scheduler/schedule.py +++ b/scheduler/schedule.py @@ -13,8 +13,8 @@ import itertools CONFIG = '/root/repos.yaml' REPOPATH = '/etc/yum.repos.d/' -JOBNAME = 'reposync' -STATUS_CHECK_SECS = 5 +JOBNAME = os.getenv('REPOSYNC_JOB', 'reposync') +STATUS_CHECK_SECS = os.getenv('STATUS_CHECKS', 5) REPOID = os.getenv('NOMAD_META_REPOID', None) def log(**kwargs):