Merge branch 'dev' into 'master'

Better dev workflow

See merge request linuxsupport/containers/reposync_test!2

.gitlab-ci.yml

@@ -16,6 +16,7 @@ stages:
   - tag
   - deploy_test
   - deploy
+  - cleanup
 
 prepare_dirs:
   stage: prepare
@@ -35,9 +36,7 @@ build_reposync:
   script: "echo" # unused but this line is required by GitLab CI
   variables:
     CONTEXT_DIR: reposync
-    TO: $CI_REGISTRY_IMAGE/do_reposync:latest # latest is the image used in production
-  only:
-    - master
+    TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_COMMIT_REF_NAME
 
 build_scheduler:
   stage: build
@@ -46,53 +45,7 @@ build_scheduler:
   script: "echo" # unused but this line is required by GitLab CI
   variables:
     CONTEXT_DIR: scheduler
-    TO: $CI_REGISTRY_IMAGE/reposync_scheduler:latest # latest is the image used in production
-  only:
-    - master
-
-build_reposync_branch:
-  stage: build
-  tags:
-    - docker-image-build
-  script: "echo" # unused but this line is required by GitLab CI
-  variables:
-    CONTEXT_DIR: reposync
-    TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_BUILD_REF_NAME # put name of branch in image
-  except:
-    - master
-
-build_scheduler_branch:
-  stage: build
-  tags:
-    - docker-image-build
-  script: "echo" # unused but this line is required by GitLab CI
-  variables:
-    CONTEXT_DIR: scheduler
-    TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_BUILD_REF_NAME # put name of branch in image
-  except:
-    - master
-
-tag_reposync:
-  stage: tag
-  tags:
-    - docker-image-build
-  script: "echo" # unused but this line is required by GitLab CI
-  variables:
-    CONTEXT_DIR: reposync
-    TO: $CI_REGISTRY_IMAGE/do_reposync:$CI_COMMIT_TAG
-  only:
-    - tags
-
-tag_scheduler:
-  stage: tag
-  tags:
-    - docker-image-build
-  script: "echo" # unused but this line is required by GitLab CI
-  variables:
-    CONTEXT_DIR: scheduler
-    TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_COMMIT_TAG
-  only:
-    - tags
+    TO: $CI_REGISTRY_IMAGE/reposync_scheduler:$CI_COMMIT_REF_NAME
 
 
 .nomad_deps_template: &nomad_deps
@@ -100,9 +53,9 @@ tag_scheduler:
   curl https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip -o /tmp/nomad.zip &&
   unzip /tmp/vault.zip -d /usr/local/bin/ &&
   unzip /tmp/nomad.zip -d /usr/local/bin/ &&
-  export IMAGE_VER="${CI_COMMIT_TAG:-latest}" &&
   vault read nomad/creds/submitjobs -format=json > token.json &&
-  export `jq -r '"NOMAD_TOKEN=" + .data.secret_id' token.json`
+  export `jq -r '"NOMAD_TOKEN=" + .data.secret_id' token.json` &&
+  if [ -z ${CI_COMMIT_TAG+x} ]; then export FORCE_PULL="true"; else export FORCE_PULL="false"; fi
 
 deploy_test:
   stage: deploy_test
@@ -120,3 +73,15 @@ deploy:
   script:
     - *nomad_deps
     - for j in *.nomad; do echo -e "\n${j}:"; nomad job run <(envsubst < $j); done
+
+stop_testing:
+  stage: cleanup
+  dependencies:
+    - deploy
+  script:
+    - *nomad_deps
+    - export JOB_PREFIX="${DEV_JOB_PREFIX}"
+    - for j in *.nomad; do echo -e "\n${j}:"; JOB=`grep task <(envsubst < $j) | sed 's/.*"\([^"]*\)".*/\1/'`; nomad job stop -purge ${JOB}; done
+  only:
+    - tags
+    - master

reposync.nomad

@@ -12,7 +12,8 @@ job "${JOB_PREFIX}reposync" {
     driver = "docker"
 
     config {
-      image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/do_reposync:${IMAGE_VER}"
+      image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/do_reposync:${CI_COMMIT_REF_NAME}"
+      force_pull = ${FORCE_PULL}
       logging {
         type = "gelf"
         config {

reposync_oneshot.nomad

@@ -11,7 +11,8 @@ job "${JOB_PREFIX}reposync_oneshot" {
     driver = "docker"
 
     config {
-      image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${IMAGE_VER}"
+      image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}"
+      force_pull = ${FORCE_PULL}
       logging {
         type = "gelf"
         config {
@@ -34,7 +35,7 @@ job "${JOB_PREFIX}reposync_oneshot" {
 
     env {
       NOMAD_ADDR = "$NOMAD_ADDR"
-      VAULT_ADDR = "$VAULT_ADDR"
+      REPOSYNC_JOB = "${JOB_PREFIX}reposync"
     }
 
     resources {

reposync_scheduler.nomad

@@ -14,7 +14,8 @@ job "${JOB_PREFIX}reposync_scheduler" {
     driver = "docker"
 
     config {
-      image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${IMAGE_VER}"
+      image = "https://gitlab-registry.cern.ch/linuxsupport/containers/reposync_test/reposync_scheduler:${CI_COMMIT_REF_NAME}"
+      force_pull = ${FORCE_PULL}
       logging {
         type = "gelf"
         config {
@@ -37,7 +38,7 @@ job "${JOB_PREFIX}reposync_scheduler" {
 
     env {
       NOMAD_ADDR = "$NOMAD_ADDR"
-      VAULT_ADDR = "$VAULT_ADDR"
+      REPOSYNC_JOB = "${JOB_PREFIX}reposync"
     }
 
     resources {

scheduler/schedule.py

@@ -13,8 +13,8 @@ import itertools
 
 CONFIG = '/root/repos.yaml'
 REPOPATH = '/etc/yum.repos.d/'
-JOBNAME = 'reposync'
-STATUS_CHECK_SECS = 5
+JOBNAME = os.getenv('REPOSYNC_JOB', 'reposync')
+STATUS_CHECK_SECS = os.getenv('STATUS_CHECKS', 5)
 REPOID = os.getenv('NOMAD_META_REPOID', None)
 
 def log(**kwargs):