Commit cc2424c2 authored by dejangolubovic's avatar dejangolubovic
Browse files

Initial commit

parents
Pipeline #2380624 passed with stage
in 3 minutes and 22 seconds
stages:
- build
- deploy
build:docker:tensorflow-notebook-gpu:
stage: build
only:
- branches
except:
- /^cern/.*
- master
- qa
tags:
- docker-image-build
script: "echo" # unused but this line is required by GitLab CI
variables:
TO: gitlab-registry.cern.ch/ai-ml/kubeflow_images/tensorflow-notebook-gpu:2.4.1-$CI_COMMIT_SHA # where to push resulting image. Default, the GitLab registry
CONTEXT_DIR: "./"
DOCKER_FILE: "docker/Dockerfile"
deploy:docker:tensorflow-notebook-gpu:
stage: deploy
only:
- /^v.*/
except:
- branches
tags:
- docker-image-build
script: "echo" # unused but this line is required by GitLab CI
variables:
TO: gitlab-registry.cern.ch/ai-ml/kubeflow_images/tensorflow-notebook-gpu:2.4.1-$CI_COMMIT_REF_NAME # where to push resulting image. Default, the GitLab registry
CONTEXT_DIR: "./"
DOCKER_FILE: "docker/Dockerfile"
# kubeflow_images
Dockerfiles and additional CERN specific resources to build images for notebooks, pipelines and Katib jobs.
\ No newline at end of file
ARG IMAGE_TYPE="gpu"
ARG IMAGE_TAG="1.0.0"
ARG TENSORFLOW_VERSION="1.15.2"
FROM gcr.io/kubeflow-images-public/tensorflow-${TENSORFLOW_VERSION}-notebook-${IMAGE_TYPE}:${IMAGE_TAG}
USER root
# Install basic dependencies
RUN apt-get -qq update && \
apt-get -yqq install libpam-krb5 krb5-user && \
apt-get -yqq clean && \
apt-get install -y --no-install-recommends \
ca-certificates bash-completion tar less \
python-pip python-setuptools build-essential python-dev \
python3-pip python3-wheel && \
rm -rf /var/lib/apt/lists/*
ENV SHELL /bin/bash
# Clone Kale and add bashrc
WORKDIR /
RUN git clone -b cern/v0.6.1 https://gitlab.cern.ch/ai-ml/kale-tmp.git && \
cp /kale-tmp/docker/jupyterlab/bashrc /etc/bash.bashrc && \
echo "set background=dark" >> /etc/vim/vimrc.local
# Install other pip requirements
COPY docker/requirements.txt /requirements.txt
RUN pip3 install --upgrade pip && pip3 install -r /requirements.txt
# Install jupyterlab
RUN pip3 install --upgrade pip && \
pip3 install --upgrade "enum34==1.1.8" && \
pip3 install --upgrade "jupyterlab>=2.0.0,<3.0.0"
# Setup kerberos
COPY docker/krb5.conf /etc/krb5.conf
# Install Kale
WORKDIR /kale-tmp/backend
RUN pip3 install -U .
# Install labextension
WORKDIR /kale-tmp/labextension
RUN jlpm install && \
jlpm run build && \
jupyter labextension install . && \
jupyter lab build --dev-build=False
# Clone KFP
WORKDIR /
RUN git clone -b cern/v1.4.1 https://gitlab.cern.ch/ai-ml/pipelines.git
# Install kfp-server-api
WORKDIR /pipelines/backend/api/python_http_client
RUN pip3 install -U . --upgrade
# Install kfp
WORKDIR /pipelines/sdk/python
RUN pip3 install -U . --upgrade
# Install golang
WORKDIR /
RUN wget -c https://golang.org/dl/go1.15.8.linux-amd64.tar.gz
RUN tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz
ENV PATH $PATH:/usr/local/go/bin
RUN rm go1.15.8.linux-amd64.tar.gz
# Install ormb
WORKDIR /
RUN git clone https://github.com/kleveross/ormb
WORKDIR /ormb
RUN go mod tidy
RUN make build-local
# Init namespace and general setup
ENV NB_USER jovyan
ENV HOME /home/$NB_USER
RUN chown -R ${NB_USER}:users $HOME
RUN echo "jovyan ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers.d/jovyan
WORKDIR /home/jovyan
ADD init_scripts/init_namespace.py /
RUN chmod +x /init_namespace.py
ADD init_scripts/start.sh /
RUN chmod +x /start.sh
USER jovyan
CMD ["sh", "-c", "/start.sh"]
; AD : This Kerberos configuration is for CERN's Active Directory realm.
;
; /etc/krb5.conf
[libdefaults]
default_realm = CERN.CH
ticket_lifetime = 25h
renew_lifetime = 120h
forwardable = true
proxiable = true
default_tkt_enctypes = arcfour-hmac-md5 aes256-cts aes128-cts des3-cbc-sha1 des-cbc-md5 des-cbc-crc
chpw_prompt = true
rdns = false
[appdefaults]
pam = {
external = true
krb4_convert = false
krb4_convert_524 = false
krb4_use_as_req = false
}
[domain_realm]
.cern.ch = CERN.CH
.fnal.gov = FNAL.GOV
.hep.man.ac.uk = HEP.MAN.AC.UK
.in2p3.fr = IN2P3.FR
# No default domain for KFKI.HU specified.
[realms]
# Start of puppet output for CERN.CH
CERN.CH = {
default_domain = cern.ch
kpasswd_server = cerndc.cern.ch
admin_server = cerndc.cern.ch
kdc = cerndc.cern.ch
v4_name_convert = {
host = {
rcmd = host
}
}
}
# Start of puppet output for FNAL.GOV
FNAL.GOV = {
default_domain = fnal.gov
admin_server = krb-fnal-admin.fnal.gov
kdc = krb-fnal-fcc3.fnal.gov:88
kdc = krb-fnal-2.fnal.gov:88
kdc = krb-fnal-3.fnal.gov:88
kdc = krb-fnal-1.fnal.gov:88
kdc = krb-fnal-4.fnal.gov:88
kdc = krb-fnal-enstore.fnal.gov:88
kdc = krb-fnal-fg2.fnal.gov:88
kdc = krb-fnal-cms188.fnal.gov:88
kdc = krb-fnal-cms204.fnal.gov:88
kdc = krb-fnal-d0online.fnal.gov:88
}
# Start of puppet output for HEP.MAN.AC.UK
HEP.MAN.AC.UK = {
default_domain = hep.man.ac.uk
kpasswd_server = afs4.hep.man.ac.uk
admin_server = afs4.hep.man.ac.uk
kdc = afs1.hep.man.ac.uk
kdc = afs2.hep.man.ac.uk
kdc = afs3.hep.man.ac.uk
kdc = afs4.hep.man.ac.uk
}
# Start of puppet output for IN2P3.FR
IN2P3.FR = {
default_domain = in2p3.fr
kpasswd_server = kerberos-admin.in2p3.fr
admin_server = kerberos-admin.in2p3.fr
kdc = kerberos-1.in2p3.fr
kdc = kerberos-2.in2p3.fr
kdc = kerberos-3.in2p3.fr
}
# Start of puppet output for KFKI.HU
KFKI.HU = {
admin_server = kerberos.kfki.hu
kdc = kerberos.kfki.hu
}
tensorflow==2.4.1
scikit-learn
pandas
boto3
minio
kubeflow-metadata
#msrestazure
#kubeflow-fairing
import kfp
client = kfp.Client()
with open(kfp.Client.NAMESPACE_PATH, 'r') as f:
current_namespace = f.read()
client.set_user_namespace(current_namespace)
#!/bin/bash
#echo '. /start_jupyter.sh &'
#. /start_jupyter.sh &
#echo 'sleep'
#sleep 1m
#python3 /init_namespace.py
echo 'Starting Jupyter'
jupyter lab --notebook-dir=/home/jovyan --ip=0.0.0.0 --no-browser --allow-root --port=8888 --LabApp.token='' --LabApp.password='' --LabApp.allow_origin='*' --LabApp.base_url=${NB_PREFIX} &
jupyter_pid=$!
echo 'Sleeping 1m'
sleep 1m
echo 'Init namespace'
python3 /init_namespace.py
echo 'Waiting'
wait $jupyter_pid
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment