Rework pick 2

897468be · Domenico Giordano · 6480a82b · 897468be · 897468be · 897468be
Commit 897468be authored 4 years ago by Domenico Giordano
--- a/cms/cms-patatrack-ci.yml
+++ b/cms/cms-patatrack-ci.yml
 ---
 stages:
-    - build_0
-    - build_1
-    - build_2
-    - snapshot
-    - build_standalone
+    - build_base_image
+    - build_data_image
+    - build_interim_image
+    - snapshot_cvmfs
+    - build_standalone_image
    - test
 #- publish
 #- announce
@@ -61,48 +61,48 @@ stages:
 # docker in docker image: to trigger other docker runs
 ###########################################################

-job_build_image_step0: 
-    stage: build_0
+job_build_base_image: 
+    stage: build_base_image
    before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.base
        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
-        - export IMAGE_NAME=cms/cms-patatrack-nvidia-0
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-base
        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
    <<: *template_build_image
    only:
        changes:
-          - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+          - cms/patatrack/ci-scripts/Dockerfile.nvidia.base


-job_build_image_step1: 
-    stage: build_1
+job_build_data_image: 
+    stage: build_data_image
    before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.data
        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
-        - export IMAGE_NAME=cms/cms-patatrack-nvidia-1
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-data
        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
    <<: *template_build_image
    only:
        changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.base
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.data

-job_build_image_step2: 
-    stage: build_2
+job_build_interim_image: 
+    stage: build_interim_image
    before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.full
        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
-        - export IMAGE_NAME=cms/cms-patatrack-nvidia-2
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-interim
        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
    <<: *template_build_image
    only:
        changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.*
            - cms/patatrack/cms-patatrack/*
            - cms/patatrack/cms-patatrack/utility_scripts/*

 job_snapshot_cvmfs: 
-    stage: snapshot
+    stage: snapshot_cvmfs
    tags:
        - hep-workload-gpu-docker-builder
    image:
@@ -121,7 +121,7 @@ job_snapshot_cvmfs:
            - $CI_COMMIT_BRANCH =~ /^qa.*$/
            - $CI_COMMIT_TAG =~ /^v.*$/
        changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.*
            - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
            - cms/patatrack/cms-patatrack/*
            - cms/patatrack/cms-patatrack/utility_scripts/*
@@ -135,16 +135,16 @@ job_snapshot_cvmfs:
        when: always

 job_build_standalone_image: 
-    stage: build_standalone
+    stage: build_standalone_image
    before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.full
        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
        - export IMAGE_NAME=cms/cms-patatrack-nvidia-bmk
        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
    <<: *template_build_image
    only:
        changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.*
            - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
            - cms/patatrack/cms-patatrack/*
            - cms/patatrack/cms-patatrack/utility_scripts/*

--- a/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+++ b/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
-# FIXME: need to build in gitlab this base image. Was done by hand
-FROM nvidia/cuda:10.1-devel-centos7 
+# This dockerfile generates a base image for 
+# later importing bmk code and cvmfs.
+# This image is not frequently updated.
+FROM nvidia/cuda:10.1-base-centos7

 RUN yum install -y \
    which \

--- a/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+++ b/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
-FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-0:qa
+FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-base:qa


 # Prepare a data directory for downloading large files that should normally be cacheable (BMK-159)

--- a/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+++ b/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
-FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-1:qa
+FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-data:qa

 COPY . /stage/
 RUN ls -la /stage/*
-RUN /stage/ci-scripts/build_2.sh
+RUN /stage/ci-scripts/build_final.sh

 ENTRYPOINT ["/bmk/./cms-patatrack/cms-patatrack-bmk.sh"]
--- a/cms/patatrack/ci-scripts/build_2.sh
+++ b/cms/patatrack/ci-scripts/build_2.sh
@@ -22,7 +22,7 @@ chmod -R 555 /cvmfs
 # This MUST happen before linking the data dir
 # otherwise will take a lot of time to tar
 date
-tar -cf /tmp/bmk_checksum.tar /bmk && md5sum /tmp/bmk_checksum.tar | cut -f1 -d" " >/tmp/bmk_checksum && rm /tmp/bmk_checksum.tar #FIXME
+tar -cf /tmp/bmk_checksum.tar /bmk --exclude='data' && md5sum /tmp/bmk_checksum.tar | cut -f1 -d" " >/tmp/bmk_checksum && rm /tmp/bmk_checksum.tar #FIXME

 # The data dir has already a checksum in /tmp/bmkdata_checksum
 # generated in nvidia.Dockerfile.1

--- a/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+++ b/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
@@ -15,8 +15,8 @@ function _script() {
    # check cvmfs is running
    docker exec cvmfs_${CI_JOB_ID} cvmfs_config probe
    # Here comes the dry run of the CMS Patatrack container. Arguments are for the time being defaults/hardcoded FIXME
-    docker pull gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-2:qa
-    docker run --name patatrack_container --gpus '"device=0"' -v ${CIENV_CVMFSVOLUME}:/cvmfs gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-2:qa -e 100 -t 8 -c 1
+    docker pull ${INTERIM_IMAGE}
+    docker run --name patatrack_container --gpus '"device=0"' -v ${CIENV_CVMFSVOLUME}:/cvmfs ${INTERIM_IMAGE} -e 100 -t 8 -c 1
    # run shrinkwrapper
    docker exec cvmfs_${CI_JOB_ID} /root/shrinkwrap.sh -t /tmp/traces -e ${CVMFS_EXPORT_DIR}
    # get the generated traces from the container
@@ -45,4 +45,5 @@ function _after_script() {
 export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
 export CVMFS_EXPORT_DIR=${CI_PROJECT_DIR}/cms/patatrack
 export CIENV_CVMFSREPO=cms.cern.ch
-export CVMFS_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/cvmfs-image:${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
\ No newline at end of file
+export CVMFS_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/cvmfs-image:qa
+export INTERIM_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-interim:${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
\ No newline at end of file
--- a/cms/patatrack/cms-patatrack/cms-reco-bmk.sh
+++ b/cms/patatrack/cms-patatrack/cms-reco-bmk.sh
-#!/bin/bash
-
-#set -x # enable debug printouts
-
-#set -e # immediate exit on error
-
-# Function doOne must be defined in each benchmark
-# Input argument $1: process index (between 1 and $NCOPIES)
-# Return value: please return 0 if this workload copy was successful, 1 otherwise
-# The following variables are guaranteed to be defined and exported: NCOPIES, NTHREADS, NEVENTS_THREAD, BMKDIR, DEBUG
-# The function is started in process-specific working directory <basewdir>/proc_$1:
-# please store here the individual log files for each of the NCOPIES processes
-function doOne(){
-  if [ "$1" == "" ] || [ "$2" != "" ]; then echo "[doOne] ERROR! Invalid arguments '$@' to doOne"; return 1; fi
-  echo "[doOne ($1)] $(date) starting in $(pwd)"
-  # Extra CMS-RECO-specific setup
-  export CMSSW_RELEASE=CMSSW_10_2_9
-  export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch
-  source $VO_CMS_SW_DIR/cmsset_default.sh
-  export SCRAM_ARCH=slc6_amd64_gcc700
-  [[ ! -e ${CMSSW_RELEASE} ]] && scram project CMSSW ${CMSSW_RELEASE}
-  pushd ${CMSSW_RELEASE}; eval `scramv1 runtime -sh`; popd
-  # Configure WL copy
-  ln -s ${BMKDIR}/data/GlobalTag.db ./GlobalTag.db
-  ln -s ${BMKDIR}/data/*.root .
-  CMSSW_CONF=step3_RAW2DIGI_L1Reco_RECO_EI_PAT_DQM.py
-  JOB_EVENTS=$(( NEVENTS_THREAD * NTHREADS )) # bash shell arithmetic, may use var instead of $var
-  cp ${BMKDIR}/${CMSSW_CONF}_template ./${CMSSW_CONF}
-  sed -e "s@_NEVENTS_@${JOB_EVENTS}@g" -e "s@_NTHREADS_@$NTHREADS@g" -i ./${CMSSW_CONF}
-  # Execute WL copy
-  LOG=out_$1.log
-  cmsRun ./${CMSSW_CONF} >>$LOG 2>&1 3>&1
-  status=${?}
-  echo "[doOne ($1)] $(date) completed (status=$status)"
-  # Return 0 if this workload copy was successful, 1 otherwise
-  return $status
-}
-
-# Optional function validateInputArguments may be defined in each benchmark
-# If it exists, it is expected to set NCOPIES, NTHREADS, NEVENTS_THREAD
-# (based on previous defaults and on user inputs USER_NCOPIES, USER_NTHREADS, USER_NEVENTS_THREADS)
-# Input arguments: none
-# Return value: please return 0 if input arguments are valid, 1 otherwise
-# The following variables are guaranteed to be defined: NCOPIES, NTHREADS, NEVENTS_THREAD
-# (benchmark defaults) and USER_NCOPIES, USER_NTHREADS, USER_NEVENTS_THREADS (user inputs)
-function validateInputArguments(){
-  if [ "$1" != "" ]; then echo "[validateInputArguments] ERROR! Invalid arguments '$@' to validateInputArguments"; return 1; fi
-  echo "[validateInputArguments] validate input arguments"
-  # Number of copies and number of threads per copy
-  if [ "$USER_NTHREADS" != "" ] && [ "$USER_NCOPIES" != "" ]; then
-    NCOPIES=$USER_NCOPIES
-    NTHREADS=$USER_NTHREADS
-  elif [ "$USER_NTHREADS" != "" ]; then
-    NTHREADS=$USER_NTHREADS
-    NCOPIES=$((`nproc`/$NTHREADS))
-  elif [ "$USER_NCOPIES" != "" ]; then
-    NCOPIES=$USER_NCOPIES
-    NTHREADS=$((`nproc`/$NCOPIES))
-  fi
-  # Number of events per thread
-  if [ "$USER_NEVENTS_THREAD" != "" ]; then NEVENTS_THREAD=$USER_NEVENTS_THREAD; fi
-  # Return 0 if input arguments are valid, 1 otherwise
-  # Report any issues to parseResults via s_msg
-  export s_msg="ok"
-  tot_load=$(($NCOPIES*$NTHREADS))
-  if [ $tot_load -gt `nproc` ]; then
-    s_msg="[ERROR] NCOPIES*NTHREADS=$NCOPIES*$NTHREADS=$tot_load > number of available cores (`nproc`)"
-    return 1
-  elif [ $tot_load -eq 0 ]; then
-    s_msg="[ERROR] NCOPIES*NTHREADS=$NCOPIES*$NTHREADS=$tot_load. Please fix it"
-    return 1
-  elif [ $tot_load -ne `nproc` ];
-    then s_msg="[WARNING] NCOPIES*NTHREADS ($NCOPIES*$NTHREADS=$tot_load) != `nproc` (number of available cores nproc)"
-    echo $s_msg
-  fi
-  return 0
-}
-
-# Default values for NCOPIES, NTHREADS, NEVENTS_THREAD must be set in each benchmark
-NTHREADS=4
-NCOPIES=$(( `nproc` / $NTHREADS ))
-NEVENTS_THREAD=100
-if [ "$NCOPIES" -lt 1 ]; then # when $NTHREADS > nproc
-  NCOPIES=1
-  NTHREADS=`nproc`
-fi
-
-# Source the common benchmark driver
-if [ -f $(dirname $0)/bmk-driver.sh ]; then
-  . $(dirname $0)/bmk-driver.sh
-else
-  . $(dirname $0)/../../../common/bmk-driver.sh
-fi