ci_test_script.sh 2.27 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#!/bin/bash -e

# This script is used by the gitlab CI of this repository to run a 
# test of adcern lib functionalities
# Requirements:
#     - spark_etl libs (distributed in docker image)
#     - Kerberos authentication (passed via CI)
#     - cvmfs spark libs (exposed via cvmfs)

# In order to access the needed libraries from cvmfs, 
# a cvmfs service is started with docker-compose 
# 
# In order to run the same script manually, assuming only docker available, run
#
# CI_PROJECT_DIR=`pwd | sed -e 's@/tests/.*@@'`
# docker run --rm -e CI_USER=$CI_USER -e CI_USER_PASSWD=$CI_USER_PASSWD -e CI_PROJECT_DIR=${CI_PROJECT_DIR} 
#         -v /tmp:/tmp -v /builds:/builds -v `pwd`:/work -v /var/run/docker.sock:/var/run/docker.sock 
#          gitlab-registry.cern.ch/cloud-infrastructure/data-analytics/compose:v1.0 /work/tests/adcern/integration/ci_test_script.sh
#
# Consider to open the Spark connection ports in iptables
#
# sudo iptables -I INPUT -p tcp -m multiport --dports 5001:6000 -m comment --comment "00200 firewall for hadoop jobs" -j ACCEPT
# sudo iptables -I DOCKER-USER -p tcp -m multiport --dports 5001:6000 -m comment --comment "00200 firewall for hadoop jobs" -j ACCEPT
#

WORK_DIR=$(readlink -f $(dirname $0))
echo WORK_DIR $WORK_DIR

cd $WORK_DIR
export CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA:-noCI}
export CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-noCI}
export COMMIT_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
export CVMFSDIR=/builds/cvmfs-${COMMIT_TAG}
export KRB5DIR=/builds/krb5-${COMMIT_TAG}
export IMAGE_TAG=${COMMIT_TAG}

Domenico Giordano's avatar
Domenico Giordano committed
37
38
39
# Preparing Kerberos ticket for same user as in sparknotebook
groupadd -g 50000 docker; useradd -u 50000 -g 50000 airflow;

40
41
42
mkdir -p $KRB5DIR
export KRB5CCNAME=$KRB5DIR/krb5cc_docker; kdestroy -c $KRB5CCNAME ; echo $CI_USER_PASSWD | kinit -c $KRB5CCNAME $CI_USER@CERN.CH; klist -c $KRB5CCNAME
ls -l $KRB5DIR
Domenico Giordano's avatar
Domenico Giordano committed
43
ls /eos/project-i/it-cloud-data-analytics/
44

Domenico Giordano's avatar
Domenico Giordano committed
45
echo -e "\n------- DUMP docker-compose.yml ----\n"
Domenico Giordano's avatar
Domenico Giordano committed
46
docker-compose -f docker-compose.yml config
Domenico Giordano's avatar
Domenico Giordano committed
47
echo -e "\n------- END docker-compose.yml ----\n"
48
49
50
51
52
docker-compose -f docker-compose.yml down --remove-orphans --volumes
docker-compose pull
docker-compose -f docker-compose.yml -p adcern up --remove-orphans --renew-anon-volumes --abort-on-container-exit --exit-code-from srv_test_adcern
#docker-compose logs -f 2>&1 >> compose.log &