Commit 183c7f92 authored by Domenico Giordano's avatar Domenico Giordano
Browse files

improving ci_scripts

parent 25e03f2f
......@@ -216,10 +216,11 @@ spark_etl: &template_test_spark_etl
rules:
- if: '$CI_COMMIT_BRANCH =~ /^qa.*$/ || $CI_COMMIT_TAG =~ /^v.*$/'
script:
- ${CI_PROJECT_DIR}/tests/spark_etl/ci_test_script.sh
- . ${CI_PROJECT_DIR}/tests/spark_etl/ci_test_script.sh
- start_docker_compose
after_script:
- cd $CI_PROJECT_DIR/tests/spark_etl
- docker-compose down --rmi all --remove-orphans --volumes
- . ${CI_PROJECT_DIR}/tests/spark_etl/ci_test_script.sh
- stop_docker_compose
artifacts:
paths:
- $CI_PROJECT_DIR/tests/spark_etl/*
......@@ -265,12 +266,11 @@ qa_adcern:
rules:
- if: '$CI_COMMIT_BRANCH =~ /^qa.*$/ || $CI_COMMIT_TAG =~ /^v.*$/'
script:
- ${CI_PROJECT_DIR}/tests/adcern/integration/ci_test_script.sh
- . ${CI_PROJECT_DIR}/tests/adcern/integration/ci_test_script.sh
- start_docker_compose
after_script:
- cd $CI_PROJECT_DIR/tests/adcern/integration/
- docker-compose down --remove-orphans --volumes # First remove containers
- docker container prune -f
- docker-compose down --rmi all --remove-orphans --volumes # Then remove images
- . $CI_PROJECT_DIR/tests/adcern/integration/ci_run_script.sh
- stop_docker_compose
artifacts:
paths:
- $CI_PROJECT_DIR/tests/adcern/integration/*
......
......@@ -15,7 +15,7 @@
# CI_PROJECT_DIR=`pwd | sed -e 's@/tests/.*@@'`
# docker run --rm -e CI_USER=$CI_USER -e CI_USER_PASSWD=$CI_USER_PASSWD -e CI_PROJECT_DIR=${CI_PROJECT_DIR}
# -v /tmp:/tmp -v /builds:/builds -v `pwd`:/work -v /var/run/docker.sock:/var/run/docker.sock
# gitlab-registry.cern.ch/cloud-infrastructure/data-analytics/compose:v1.0 /work/tests/adcern/integration/ci_test_script.sh
# gitlab-registry.cern.ch/cloud-infrastructure/data-analytics/compose:v1.0 bash -c '. /work/ci_test_script.sh; start_docker_compose'
#
# Consider to open the Spark connection ports in iptables
#
......@@ -23,6 +23,28 @@
# sudo iptables -I DOCKER-USER -p tcp -m multiport --dports 5001:6000 -m comment --comment "00200 firewall for hadoop jobs" -j ACCEPT
#
function start_docker_compose() {
mkdir -p $KRB5DIR
export KRB5CCNAME=$KRB5DIR/krb5cc_docker; kdestroy -c $KRB5CCNAME ; echo $CI_USER_PASSWD | kinit -c $KRB5CCNAME $CI_USER@CERN.CH; klist -c $KRB5CCNAME
ls -l $KRB5DIR
ls /eos/project-i/it-cloud-data-analytics/
echo -e "\n------- DUMP docker-compose.yml ----\n"
docker-compose -f docker-compose.yml config
echo -e "\n------- END docker-compose.yml ----\n"
docker-compose -f docker-compose.yml down --remove-orphans --volumes
docker-compose pull
docker-compose -f docker-compose.yml -p adcern up --remove-orphans --renew-anon-volumes --abort-on-container-exit --exit-code-from srv_test_adcern
#docker-compose logs -f 2>&1 >> compose.log &
}
function stop_docker_compose(){
cd $WORK_DIR
docker-compose down --remove-orphans --volumes # First remove containers
docker container prune -f
docker-compose down --rmi all --remove-orphans --volumes # Then remove images
}
WORK_DIR=$(readlink -f $(dirname $0))
echo WORK_DIR $WORK_DIR
......@@ -34,19 +56,3 @@ export CVMFSDIR=/builds/cvmfs-${COMMIT_TAG}
export KRB5DIR=/builds/krb5-${COMMIT_TAG}
export IMAGE_TAG=${COMMIT_TAG}
# Preparing Kerberos ticket for same user as in sparknotebook
groupadd -g 50000 docker; useradd -u 50000 -g 50000 airflow;
mkdir -p $KRB5DIR
export KRB5CCNAME=$KRB5DIR/krb5cc_docker; kdestroy -c $KRB5CCNAME ; echo $CI_USER_PASSWD | kinit -c $KRB5CCNAME $CI_USER@CERN.CH; klist -c $KRB5CCNAME
ls -l $KRB5DIR
ls /eos/project-i/it-cloud-data-analytics/
echo -e "\n------- DUMP docker-compose.yml ----\n"
docker-compose -f docker-compose.yml config
echo -e "\n------- END docker-compose.yml ----\n"
docker-compose -f docker-compose.yml down --remove-orphans --volumes
docker-compose pull
docker-compose -f docker-compose.yml -p adcern up --remove-orphans --renew-anon-volumes --abort-on-container-exit --exit-code-from srv_test_adcern
#docker-compose logs -f 2>&1 >> compose.log &
......@@ -31,6 +31,7 @@ function run_reader() {
function stop_docker_compose(){
cd $WORK_DIR
docker-compose down --rmi all --remove-orphans --volumes
docker container prune -f
cat compose.log
}
......
......@@ -13,7 +13,7 @@
# In order to run the same script manually, assuming only docker available, run
#
# CI_PROJECT_DIR=`pwd | sed -e 's@/tests/spark_etl@@'`
# docker run --rm -e CI_USER=$CI_USER -e CI_USER_PASSWD=$CI_USER_PASSWD -e CI_PROJECT_DIR=${CI_PROJECT_DIR} -v /tmp:/tmp -v /builds:/builds -v `pwd`:/work -v /var/run/docker.sock:/var/run/docker.sock gitlab-registry.cern.ch/cloud-infrastructure/data-analytics/compose:qa /work/tests/spark_etl/ci_test_script.sh
# docker run --rm -e CI_USER=$CI_USER -e CI_USER_PASSWD=$CI_USER_PASSWD -e CI_PROJECT_DIR=${CI_PROJECT_DIR} -v /tmp:/tmp -v /builds:/builds -v `pwd`:/work -v /var/run/docker.sock:/var/run/docker.sock gitlab-registry.cern.ch/cloud-infrastructure/data-analytics/compose:qa bash -c '. /work/ci_test_script.sh; start_docker_compose'
#
# Consider to open the Spark connection ports in iptables
#
......@@ -21,6 +21,27 @@
# sudo iptables -I DOCKER-USER -p tcp -m multiport --dports 5001:6000 -m comment --comment "00200 firewall for hadoop jobs" -j ACCEPT
#
function start_docker_compose() {
mkdir -p $KRB5DIR
export KRB5CCNAME=$KRB5DIR/krb5cc_docker; kdestroy -c $KRB5CCNAME ; echo $CI_USER_PASSWD | kinit -c $KRB5CCNAME $CI_USER@CERN.CH; klist -c $KRB5CCNAME
ls -l $KRB5DIR
echo -e "\n------- DUMP docker-compose.yml ----\n"
docker-compose -f docker-compose.yml config
echo -e "\n------- END docker-compose.yml ----\n"
docker-compose -f docker-compose.yml down --remove-orphans --volumes
docker-compose pull
docker-compose -f docker-compose.yml -p spark_etl up --remove-orphans --renew-anon-volumes --abort-on-container-exit --exit-code-from srv_spark_etl
#docker-compose logs -f 2>&1 >> compose.log &
}
function stop_docker_compose(){
cd $WORK_DIR
docker-compose down --remove-orphans --volumes # First remove containers
docker container prune -f
docker-compose down --rmi all --remove-orphans --volumes # Then remove images
}
WORK_DIR=$(readlink -f $(dirname $0))
echo WORK_DIR $WORK_DIR
......@@ -31,15 +52,5 @@ export CVMFSDIR=/builds/cvmfs-${COMMIT_TAG}
export KRB5DIR=/builds/krb5-${COMMIT_TAG}
export IMAGE_TAG=${COMMIT_TAG}
mkdir -p $KRB5DIR
export KRB5CCNAME=$KRB5DIR/krb5cc_docker; kdestroy -c $KRB5CCNAME ; echo $CI_USER_PASSWD | kinit -c $KRB5CCNAME $CI_USER@CERN.CH; klist -c $KRB5CCNAME
ls -l $KRB5DIR
echo -e "\n------- DUMP docker-compose.yml ----\n"
docker-compose -f docker-compose.yml config
echo -e "\n------- END docker-compose.yml ----\n"
docker-compose -f docker-compose.yml down --remove-orphans --volumes
docker-compose pull
docker-compose -f docker-compose.yml -p spark_etl up --remove-orphans --renew-anon-volumes --abort-on-container-exit --exit-code-from srv_spark_etl
#docker-compose logs -f 2>&1 >> compose.log &
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment