install_AD.sh 7.89 KB
Newer Older
Domenico Giordano's avatar
Domenico Giordano committed
1
2
#!/bin/bash -e

Domenico Giordano's avatar
Domenico Giordano committed
3
4
5
6
7
8
9
10
11
12
13
14
15
# Script to install pkgs and configure the Anomaly Detection System
# To be run as root

function uninstall(){
    # remove the system
    rm -R /opt/ad_system/

    # remove soft link to control room
    rm /opt/control_ad_system
}

function install_centos7() {
    # INSTALL WGET
Domenico Giordano's avatar
Domenico Giordano committed
16
    yum -y install wget git python3-pip\
17
         kstart screen emacs
Domenico Giordano's avatar
Domenico Giordano committed
18
19
20
21
22
23

    # INSTALL DOCKER
    yum-config-manager \
        --add-repo \
        https://download.docker.com/linux/centos/docker-ce.repo
    yum -y install docker-ce docker-ce-cli containerd.io
24
 
Domenico Giordano's avatar
Domenico Giordano committed
25
26
27
28
    install_eos
}

function install_eos(){
29
    # INSTALL EOS
Domenico Giordano's avatar
Domenico Giordano committed
30
31
    # https://cern.service-now.com/service-portal?id=kb_article&n=KB0003846
    # For CERN CentOS 7 and CentOS8 desktops, please be sure that the EPEL repository is enabled and then use (as root):
32

Domenico Giordano's avatar
Domenico Giordano committed
33
34
35
    echo -e "install EOS: this can require few minutes"
    locmap --enable eosclient
    locmap --configure eosclient
Domenico Giordano's avatar
Domenico Giordano committed
36

Domenico Giordano's avatar
Domenico Giordano committed
37
38
39
40
41
}

function install_common(){
    # INSTALL DOCKER COMPOSE
    # https://docs.docker.com/compose/install/
Domenico Giordano's avatar
Domenico Giordano committed
42
    curl -L "https://github.com/docker/compose/releases/download/1.28.6/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
Domenico Giordano's avatar
Domenico Giordano committed
43
44
45
    chmod +x /usr/local/bin/docker-compose
    [ ! -e /usr/bin/docker-compose ] && ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
    docker-compose --version
46
47
48
49
50
51
}

function install_ad(){

    # STOP DOCKER
    systemctl stop docker
Domenico Giordano's avatar
Domenico Giordano committed
52

Domenico Giordano's avatar
Domenico Giordano committed
53
    groupmod -g 50000 docker
Domenico Giordano's avatar
Domenico Giordano committed
54
55

    AD_USER=airflow
56
    # Add user airflow
Domenico Giordano's avatar
Domenico Giordano committed
57
    useradd -g docker -u 50000 ${AD_USER}
58
59
60

    # CHANGE SYSTEM GID / UID of DOCKER and AIRFLOW to match the
    # internal docker airflow configuration
Domenico Giordano's avatar
Domenico Giordano committed
61
    #groupmod -g 1001 airflow  # test
62
63
64
65
    # update the permission on the main docker file launcher
    #chmod 660 /var/run/docker.sock
    #chgrp docker /var/run/docker.sock
    # add user airflow in the group docker
Domenico Giordano's avatar
Domenico Giordano committed
66
    #usermod -aG docker ${AD_USER}
67
68
69
70
71

    # START DOCKER
    systemctl start docker

    BASE_AD_DIR=/opt/ad_system
Domenico Giordano's avatar
Domenico Giordano committed
72
73
    # PREPARE MAIN FOLDER
    # main folder with files of the system
74
    [ ! -e ${BASE_AD_DIR} ] && mkdir -p ${BASE_AD_DIR}
Domenico Giordano's avatar
Domenico Giordano committed
75
76
77
78

    # go to the main folder
    # download the repository file FROM QA
    # and create a folder
79
    cd ${BASE_AD_DIR}
Domenico Giordano's avatar
Domenico Giordano committed
80
81
82
83
84
85
86
87
    INSTALL_AD_DIR=${BASE_AD_DIR}/data-analytics-$branch
    if [[ -e ${INSTALL_AD_DIR} ]];then
        echo "Folder ${INSTALL_AD_DIR} already exists."   
        INSTALL_AD_DIR=${INSTALL_AD_DIR}_$RANDOM
        echo "Using folder ${INSTALL_AD_DIR}"
    fi
    #wget https://gitlab.cern.ch/cloud-infrastructure/data-analytics/-/archive/$branch/data-analytics-$branch.tar.gz -O - | tar -xz
    git clone -b $branch --single-branch https://gitlab.cern.ch/cloud-infrastructure/data-analytics.git ${INSTALL_AD_DIR}
Domenico Giordano's avatar
Domenico Giordano committed
88
89
    [[ "$?" -ne 0 ]] && echo "git clone did not succeed. EXIT" && exit 1

Domenico Giordano's avatar
Domenico Giordano committed
90
    chown -R ${AD_USER} ${BASE_AD_DIR}
Domenico Giordano's avatar
Domenico Giordano committed
91
    # create the symlink to give a standard directory for the dags
92
    CONTROL_AD_DIR=${BASE_AD_DIR}/control_ad_system
Domenico Giordano's avatar
Domenico Giordano committed
93
    [[ -L "${CONTROL_AD_DIR}" && -d "${CONTROL_AD_DIR}" ]] && rm ${CONTROL_AD_DIR}
94
    sudo -u ${AD_USER} ln -s ${INSTALL_AD_DIR}/deploy_AD/airflow-compose ${CONTROL_AD_DIR}
95
    echo " "
Domenico Giordano's avatar
Domenico Giordano committed
96
97
98
}

function set_firewall(){
99
    # Set firewall rules to close some ports
Domenico Giordano's avatar
Domenico Giordano committed
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
    # and open the port to communicate with Spark Cluster

    if [ `iptables-save | grep DOCKER-USER | grep -c '\-\-dport 5003 -j DROP'` -eq 0 ];
    then
        echo "defining firewall rule to close port 5003 used by filebrowser and give only localhost access"
        iptables -I DOCKER-USER ! -s 127.0.0.1/32 -i eth0 -p tcp -m tcp --dport 5003 -j DROP
        /sbin/service iptables save
    fi

    if [ `iptables-save | grep DOCKER-USER | grep -c '\-\-dport 6555 -j DROP'` -eq 0 ];
    then
        echo "defining firewall rule to close port 6555 used by airflow and give only localhost access"
        iptables -I DOCKER-USER ! -s 127.0.0.1/32 -i eth0 -p tcp -m tcp --dport 6555 -j DROP
        /sbin/service iptables save
    fi

    if [ `iptables-save | grep DOCKER-USER | grep -c '\-\-dport 8080 -j DROP'` -eq 0 ];
    then
        echo "defining firewall rule to close port 8080 used by airflow Web UI and give only localhost access"
        iptables -I DOCKER-USER ! -s 127.0.0.1/32 -i eth0 -p tcp -m tcp --dport 8080 -j DROP
        /sbin/service iptables save
    fi

    if [ `iptables-save | grep DOCKER-USER | grep -c '\-\-dport 8793 -j DROP'` -eq 0 ];
    then
        echo "defining firewall rule to close port 8793 used by airflow and give only localhost access"
        iptables -I DOCKER-USER ! -s 127.0.0.1/32 -i eth0 -p tcp -m tcp --dport 8793 -j DROP
        /sbin/service iptables save
    fi

    if [ `iptables-save | grep DOCKER-USER | grep -c '\-\-dport 24224 -j DROP'` -eq 0 ];
    then
        echo "defining firewall rule to close port 24224 used by fluentd and give only localhost access"
        iptables -I DOCKER-USER ! -s 127.0.0.1/32 -i eth0 -p tcp -m tcp --dport 24224 -j DROP
        /sbin/service iptables save
    fi

    if [ `iptables-save | grep INPUT | grep -c '\-p tcp \-m multiport \-\-dports 5001:6000 \-m comment \-\-comment "00200 firewall for hadoop jobs" \-j ACCEPT'` -eq 0 ];
    then
        echo "defining firewall rule to open the Spark ports"
        iptables -I INPUT -p tcp -m multiport --dports 5001:6000 -m comment --comment "00200 firewall for hadoop jobs" -j ACCEPT
        /sbin/service iptables save
    fi

    if [ `iptables-save | grep DOCKER-USER | grep -c '\-p tcp \-m multiport \-\-dports 5001:6000 \-m comment \-\-comment "00200 firewall for hadoop jobs" \-j ACCEPT'` -eq 0 ];
    then
        echo "defining firewall rule to make the Spark ports accessible for docker"
        iptables -I DOCKER-USER -p tcp -m multiport --dports 5001:6000 -m comment --comment "00200 firewall for hadoop jobs" -j ACCEPT
        /sbin/service iptables save
    fi

    iptables -L DOCKER-USER

}

function install_all(){
    set -x #to display commands to be executed

    if [ `hostnamectl | grep Oper | grep -c -w -i 'CentOS Linux 7'` -gt 0 ];
    then
        install_centos7
    else
        echo "ERROR! This script is only supported on CentOS 7"
        exit 1
    fi

    install_common
    set_firewall
Domenico Giordano's avatar
Domenico Giordano committed
168
    install_ad
Domenico Giordano's avatar
Domenico Giordano committed
169

Domenico Giordano's avatar
Domenico Giordano committed
170
171
172
173
174
175
176
177
178
179
    set +x
    echo -e "\nInstallation finished
    1. To run Airflow change user to airflow user
            su - airflow
    
    2. Make sure that a Kerberos cache ticket is available, 
       for an user having access to the EOS storage area used in the dags
       Enable the Kerberos ticket running

            source ${CONTROL_AD_DIR}/secret.sh
Domenico Giordano's avatar
Domenico Giordano committed
180
            export KRB5CCNAME
Domenico Giordano's avatar
Domenico Giordano committed
181
            kinit -c \${KRB5CCNAME} srvdaana@CERN.CH  #srvdaana can be changed with another user 
Domenico Giordano's avatar
Domenico Giordano committed
182
183
184
185

    2.1 In order to renew the Kerberos ticket follow this procedure
        
            source ${CONTROL_AD_DIR}/secret.sh
Domenico Giordano's avatar
Domenico Giordano committed
186
            export KRB5CCNAME
187
            cd dir_where_ACCOUNT.kt_is (*)
Domenico Giordano's avatar
Domenico Giordano committed
188
            kinit -k -t ACCOUNT.kt -c \$KRB5CCNAME srvdaana@CERN.CH
189
            export KINIT_PROG=eosfusebind
Domenico Giordano's avatar
Domenico Giordano committed
190
            k5start -f \`pwd\`/ACCOUNT.kt -L -K 30 -k \${KRB5CCNAME} -b -U -t
Domenico Giordano's avatar
Domenico Giordano committed
191
            ps -f --pid \`pgrep k5start\`
192
            # eosfusebind needs to be renewed (using cron to workaround this)
Domenico Giordano's avatar
Domenico Giordano committed
193
            line=\"* * * * * (date; export KRB5CCNAME=$KRB5CCNAME ;klist; eosfusebind; ls /eos/) &> /tmp/cron_eosfusebind.txt\"
Domenico Giordano's avatar
Domenico Giordano committed
194
            (crontab -l; echo "$line" ) | crontab -
195
            crontab -l
Domenico Giordano's avatar
Domenico Giordano committed
196
        
197
        (*) where the keytab ACCOUNT.kt is generated on lxplus via
Domenico Giordano's avatar
Domenico Giordano committed
198
199
200
201
202
203
204
205
206
207
208
209
210

            cern-get-keytab --keytab private/ACCOUNT.kt --user --login ACCOUNT

        as documented in https://cern.service-now.com/kb_view.do?sysparm_article=KB0003405

    3. Before starting the docker-compose of Anomaly Detection System
       You may want to change dummy passwords in ${CONTROL_AD_DIR}/secret.sh
       Then run:
       
            ${CONTROL_AD_DIR}/start_ad_system.sh 
  
    4. 'screen' and 'emacs' are available  
    "
Domenico Giordano's avatar
Domenico Giordano committed
211
212
    cd $HOME
}
Domenico Giordano's avatar
Domenico Giordano committed
213
214
215
216
217
218
219
220
221
222
223
224


if [[ $0 != $BASH_SOURCE ]];
then
    echo "Script is being sourced"
else
    echo "Script is being run"
    install_all
fi