Commit d72b2f3a authored by Fabio Luchetti's avatar Fabio Luchetti
Browse files

Apply 'added_geotags_and_proxy.diff' proposed by Comtrade (but some minor edits)

parent 10818234
Pipeline #823332 skipped with stage
......@@ -18,6 +18,11 @@ ADD eos-docker/xrd.cf.* eos-docker/krb5.conf /etc/
ADD eos-docker/fuse.conf /etc/eos/fuse.conf
ADD eos-docker/fstfmd.dict /var/eos/md/
# Add configuration files for forwarding proxy server
ADD eos-docker/xrootd.conf /etc/tmpfiles.d/
ADD eos-docker/xrootd-fwd-proxy.cfg /etc/xrootd/
ADD eos-docker/xrootd@fwd-proxy.service /etc/systemd/system/
RUN mkdir /var/tmp/eosxd-cache/ /var/tmp/eosxd-journal/
RUN useradd eos-user
......@@ -31,7 +36,7 @@ RUN yum -y --nogpg install \
gdb gcc-c++ cmake3 libacl-devel perl-Test-Harness \
rpm-build bzip2 automake autoconf libtool sudo vim \
centos-release-scl-rh at
# Install new EOS from created repo - the ADD command will reset the docker cache,
# and any commands after that point will be uncached.
ENV EOSREPODIR="/repo/eos"
......@@ -51,4 +56,7 @@ RUN yes | xrdsssadmin -k eos-test del /etc/eos.keytab; \
yes | xrdsssadmin -u daemon -g daemon -k eos-test+ -n 1234567890123456789 add /etc/eos.keytab; \
chown daemon:daemon /etc/eos.keytab
# Change owner of /var/spool/xrootd directory to daemon
RUN chown daemon:daemon /var/spool/xrootd
ENTRYPOINT ["/bin/bash"]
#!/usr/bin/env bash
if [[ $# -eq 0 ]]; then
echo "Usage: $0 <id> [-u <uuid>] [-d <mountpoint>] [-s <space>] [-c <configstatus>]"
exit 1
fi
usage()
{
echo "Usage:"
echo "$(basename $0) <fsid> [-u <uuid>] [-d <mountpoint>] [-s <space>] [-c <configstatus>] [-g geotag]"
echo
echo "-h show usage and exit"
echo
}
[[ $1 = -h ]] && usage && exit 0
id=$1
shift
if [[ -z $id ]]; then
echo -e "Filesystem ID (fsid) must be specified.\n"
usage
exit 1
fi
UUID=fst${id}
DATADIR=/home/data/eos${id}
SPACE=default
CONFIG=rw
GEOTAG=""
FSTHOSTNAME=$(hostname -f)
while getopts 'u:d:s:c:' flag; do
while getopts 'u:d:s:c:g:' flag; do
case "${flag}" in
u) UUID="${OPTARG}" ;;
d) DATADIR="${OPTARG}" ;;
s) SPACE="${OPTARG}" ;;
c) CONFIG="${OPTARG}" ;;
*) echo "Unexpected option ${flag}" ;;
g) GEOTAG="${OPTARG}" ;;
*) usage
exit 1;;
esac
done
FSTHOSTNAME=$(hostname -f)
# If specified, set new geotag instead of default one for FST server
[[ -n $GEOTAG ]] && sed -i "s/EOS_GEOTAG=.*/EOS_GEOTAG=$GEOTAG/" /etc/sysconfig/eos
source /etc/sysconfig/eos
export EOS_MGM_URL=root://eos-mgm-test.eoscluster.cern.ch//
......
......@@ -2,11 +2,7 @@
n_fst=7
if [ $# -ne 0 ]
then
n_fst=$1
fi
[[ $# -ne 0 ]] && n_fst=$1
# Enable default space with quota disabled
eos -b space set default on
......@@ -16,10 +12,10 @@ echo "Wait for FSTs to become online ..."
for i in `seq 1 30`; do
if [ `eos fs ls | grep online | wc -l` -eq $n_fst ]; then
echo "All FSTs are online"
break
echo "All FSTs are online"
break
else
sleep 1
sleep 1
fi
done
......@@ -29,13 +25,14 @@ if [ `eos fs ls | grep online | wc -l` -ne $n_fst ]; then
exit 1;
fi
# Boot filesystems
eos -b fs boot \*
eos -b config save -f default
echo "Wait for FSTs to boot ..."
for i in `seq 1 60`; do
if [ `eos fs ls | grep booted | grep docker::test | wc -l` -eq $n_fst ]; then
if [ `eos fs ls | grep booted | wc -l` -eq $n_fst ]; then
echo "All FSTs are booted"
break
else
......@@ -43,7 +40,7 @@ for i in `seq 1 60`; do
fi
done
if [ `eos fs ls | grep booted | grep docker::test | wc -l` -ne $n_fst ]; then
if [ `eos fs ls | grep booted | wc -l` -ne $n_fst ]; then
echo "Some of the FSTs are not booted ... aborting!"
eos fs ls
exit 1;
......
#!/usr/bin/env bash
# Start XRootD proxy service
/usr/bin/xrootd -n proxy -c /etc/xrootd/xrootd-fwd-proxy.cfg -l /var/log/eos/xrdlog.proxy -b -Rdaemon
#!/usr/bin/env bash
n_fst=7
n_client=1
PIDS=""
usage()
{
echo "Usage:"
echo "$(basename $0) [-n <number of FSTs>] [-c <number of clients>]"
echo
echo "-n provide number of FST servers (default is 7)"
echo "-c provide number of client servers (default is 1)"
echo
echo "-h show usage and exit"
echo
}
while getopts 'n:c:' flag; do
# Read provided arguments
while getopts 'n:c:h' flag; do
case "${flag}" in
n) n_fst=${OPTARG} ;;
c) n_client=${OPTARG} ;;
*) echo "Unexpected option ${flag}" ;;
h) usage
exit 0;;
*) usage
exit 1;;
esac
done
# Stop the FSTs in parallel
PIDS=""
echo -e "\n*** Removing containers"
# Removing the FST containers in parallel
for (( i=1; i<=$n_fst; i++))
do
FSTHOSTNAME=eos-fst${i}-test
echo $FSTHOSTNAME
echo "Removing container: "$FSTHOSTNAME
docker rm -f $FSTHOSTNAME &
PIDS="${PIDS} $!"
done
# Stop the Clients in parallel
# Removing the client containers in parallel
for (( i=1; i<=$n_client; i++))
do
CLIENTHOSTNAME=eos-client${i}-test
echo $CLIENTHOSTNAME
echo "Removing container: "$CLIENTHOSTNAME
docker rm -f $CLIENTHOSTNAME &
PIDS="${PIDS} $!"
done
for CONT in eos-client-test eos-mgm-test eos-mq-test eos-krb-test eos-qdb-test; do
docker rm -f ${CONT} &
# Removing all other containers from EOS cluster
for CONT in eos-mgm-test eos-mq-test eos-krb-test eos-qdb-test eos-proxy-test; do
echo "Removing container: "$CONT
docker rm -f ${CONT} &
PIDS="${PIDS} $!"
done
echo "Waiting for all containers to be removed ..."
echo -e "\n\n*** Waiting for all containers to be removed ..."
for PID in ${PIDS}; do
wait ${PID}
done
docker network inspect eoscluster.cern.ch
docker network disconnect -f eoscluster.cern.ch eos-fst1-test
docker network disconnect -f eoscluster.cern.ch eos-fst2-test
docker network disconnect -f eoscluster.cern.ch eos-fst3-test
docker network disconnect -f eoscluster.cern.ch eos-fst4-test
docker network disconnect -f eoscluster.cern.ch eos-fst5-test
docker network disconnect -f eoscluster.cern.ch eos-fst6-test
docker network disconnect -f eoscluster.cern.ch eos-fst7-test
docker network disconnect -f eoscluster.cern.ch eos-mgm-test
docker network disconnect -f eoscluster.cern.ch eos-client-test
docker network disconnect -f eoscluster.cern.ch eos-krb-test
docker network disconnect -f eoscluster.cern.ch eos-mq-test
docker network disconnect -f eoscluster.cern.ch eos-qdb-test
docker network disconnect -f eoscluster.cern.ch eos-client1-test
docker network inspect eoscluster.cern.ch
echo -e "\n\n*** Removing eoscluster.cern.ch network"
docker network rm eoscluster.cern.ch
exit 0
......@@ -6,59 +6,120 @@ deb_cli_img=""
n_fst=7
with_qdb=0
n_client=1
while getopts 'i:n:c:u:q' flag; do
with_proxy=0
geotags=()
regular_EOS_MGM_URL="EOS_MGM_URL=root://eos-mgm-test.eoscluster.cern.ch:1094"
proxy_EOS_MGM_URL="EOS_MGM_URL=root://eos-proxy-test.eoscluster.cern.ch:1094//root://eos-mgm-test.eoscluster.cern.ch:1094"
usage()
{
echo "Usage:"
echo "$(basename $0) -i <name of the docker image> [-n <number of FSTs>] [-c <number of clients>] [-u <debian client image>] [-g <geotag1> <geotag2> ...] [-q] [-p]"
echo
echo "-i specify docker image to be used for container creation"
echo "-n specify desired number of FST servers (default is 7)"
echo "-c specify desired number of client servers (default is 1)"
echo "-u specify debian docker image to be used for client containers creation"
echo "-g specify geotags for FST servers (default is docker-test)"
echo "-q create container for QuarkDB server and use QuarkDB instead of In-memory Namespace"
echo "-p create container for proxy server and use it as cluster access point for EOS clients"
echo
echo "-h show usage and exit"
echo
}
# Read provided arguments
while getopts 'i:n:c:u:g:qph' flag; do
case "${flag}" in
i) image="${OPTARG}" ;;
n) n_fst="${OPTARG}" ;;
c) n_client="${OPTARG}" ;;
u) deb_cli_img="${OPTARG}" ;;
u) deb_cli_img="${OPTARG}" ;;
g) if [[ ${OPTARG} != -* ]]; then
geotags="${OPTARG}"
until [[ $(eval "echo \${$OPTIND}") =~ ^-.* ]] || [ -z $(eval "echo \${$OPTIND}") ]; do
geotags+=($(eval "echo \${$OPTIND}"))
OPTIND=$((OPTIND + 1))
done
else
echo "Geotags starting with dash (-) cannot be used."
exit 1
fi ;;
q) with_qdb=1 ;;
*) echo "Unexpected option ${flag}" ;;
p) with_proxy=1 ;;
h) usage
exit 0;;
*) usage
exit 1;;
esac
done
if [[ $image == "" ]]; then
echo "Usage: -i <name of the docker image>"
echo "Docker image to be used for container creation must be specified using -i argument."
exit 1
fi
if [[ $deb_cli_img == "" ]]; then
echo "Usage: Default client configuration on Arch Systems"
fi
echo $deb_cli_img
echo $image
if [[ ${#geotags[@]} != 0 ]] && [[ $n_fst != ${#geotags[@]} ]]; then
echo "Number of geotags (${#geotags[@]}) is not the same as number of FST nodes (${n_fst})"
exit 1
fi
# Creation of the network for EOS cluster
echo -e "\n\n*** Creation of the network for EOS cluster"
docker network create eoscluster.cern.ch || true
# Kerberos server creation and setup
echo -e "\n\n*** Kerberos server creation and setup"
docker run -dit -h eos-krb-test.eoscluster.cern.ch --name eos-krb-test --net=eoscluster.cern.ch --net-alias=eos-krb-test $image
docker exec -i eos-krb-test /kdc.sh
# MQ server creation and setup
echo -e "\n\n*** MQ server creation and setup"
docker run -dit -h eos-mq-test.eoscluster.cern.ch --name eos-mq-test --net=eoscluster.cern.ch --net-alias=eos-mq-test $image
docker exec -i eos-mq-test /eos_mq_setup.sh
# MGM server creation
echo -e "\n\n*** MGM server creation"
docker run --privileged -dit -h eos-mgm-test.eoscluster.cern.ch --name eos-mgm-test --net=eoscluster.cern.ch --net-alias=eos-mgm-test $image
if [[ $with_qdb == 1 ]]; then
# When running in QuarkDB mode we need to change the namespace library loaded by the MGM
echo -e "\n\n*** QuarkDB server creation and setup"
# Namespace library which will be loaded by the MGM should be changed to enable QuarkDB mode
docker exec -i eos-mgm-test sed -i 's/libEosNsInMemory.so/libEosNsQuarkdb.so/g' /etc/xrd.cf.mgm
# Start QuarkDB container
# QuarkDB server creation and setup
docker run --privileged -dit -h eos-qdb-test.eoscluster.cern.ch --name eos-qdb-test --net=eoscluster.cern.ch --net-alias=eos-qdb-test $image
docker exec -i eos-qdb-test /eos_qdb_setup.sh
fi
if [[ $with_proxy == 1 ]]; then
echo -e "\n\n*** Proxy server creation and setup"
# Proxy server creation
docker run --privileged -dit -h eos-proxy-test.eoscluster.cern.ch --name eos-proxy-test --net=eoscluster.cern.ch --net-alias=eos-proxy-test $image
# Start XRootD proxy service
docker exec -i eos-proxy-test /eos_proxy_setup.sh
fi
# Applying Kerberos keytab to EOS cluster
echo -e "\n\n*** Applying Kerberos keytab on EOS cluster"
TMP_EOS_KEYTAB=mktemp
docker cp eos-krb-test:/root/eos.keytab $TMP_EOS_KEYTAB
docker cp $TMP_EOS_KEYTAB eos-mgm-test:/etc/eos.krb5.keytab
rm -f $TMP_EOS_KEYTAB
# MGM server setup
echo -e "\n\n*** MGM server setup"
docker exec -i eos-mgm-test /eos_mgm_setup.sh
# Start and configure the FSTs in parallel
# FST servers parallel creation
echo -e "\n\n*** FST servers parallel creation"
FAILURE=0
PIDS=""
for (( i=1; i<=$n_fst; i++ )); do
FSTHOSTNAME=eos-fst${i}-test
docker run --privileged -dit -h $FSTHOSTNAME.eoscluster.cern.ch --name $FSTHOSTNAME --net=eoscluster.cern.ch --net-alias=$FSTHOSTNAME $image &
PIDS="${PIDS} $!"
sleep 0.1
FSTHOSTNAME=eos-fst${i}-test
docker run --privileged -dit -h $FSTHOSTNAME.eoscluster.cern.ch --name $FSTHOSTNAME --net=eoscluster.cern.ch --net-alias=$FSTHOSTNAME $image &
PIDS="${PIDS} $!"
sleep 0.1
done
for PID in ${PIDS}; do
......@@ -66,15 +127,17 @@ for PID in ${PIDS}; do
done
if [ "${FAILURE}" == "1" ]; then
echo "Failed to start one of the FSTs";
echo "Failed to start one of the FSTs"
exit 1
fi
# FST servers parallel setup
echo -e "\n\n*** FST servers parallel setup"
PIDS=""
for (( i=1; i<=$n_fst; i++ )); do
FSTHOSTNAME=eos-fst${i}-test
docker exec -i $FSTHOSTNAME /eos_fst_setup.sh $i &
docker exec -i $FSTHOSTNAME /eos_fst_setup.sh $i ${geotags[i-1]:+"-g${geotags[i-1]}"} &
PIDS="${PIDS} $!"
sleep 0.1
done
......@@ -88,38 +151,28 @@ if [ "${FAILURE}" == "1" ]; then
exit 1
fi
# Enabling default space with quota disabled and booting filesystems
echo -e "\n\n*** Enabling default space with quota disabled and booting filesystems"
docker exec -i eos-mgm-test /eos_mgm_fs_setup.sh $n_fst
# Client configuration
if [ $n_client -eq 1 ]; then
# For retro-compatibility
if [[ $deb_cli_img == "" ]]; then
CLIENTHOSTNAME=eos-client-test
docker run --privileged --pid=host -dit -h ${CLIENTHOSTNAME}.eoscluster.cern.ch --name ${CLIENTHOSTNAME} --net=eoscluster.cern.ch --net-alias=${CLIENTHOSTNAME} $image
docker exec -i eos-krb-test cat /root/admin1.keytab | docker exec -i ${CLIENTHOSTNAME} bash -c "cat > /root/admin1.keytab"
docker exec -i ${CLIENTHOSTNAME} kinit -kt /root/admin1.keytab admin1@TEST.EOS
docker exec -i ${CLIENTHOSTNAME} kvno host/eos-mgm-test.eoscluster.cern.ch
# Client servers creation and setup
echo -e "\n\n*** Client servers creation and setup"
for (( i=1; i<=$n_client; i++ )); do
CLIENTHOSTNAME=eos-client${i}-test
docker run --privileged --pid=host -dit -h ${CLIENTHOSTNAME}.eoscluster.cern.ch --name ${CLIENTHOSTNAME} --net=eoscluster.cern.ch --net-alias=${CLIENTHOSTNAME} ${deb_cli_img:-$image}
# Kerberos client configuration
docker exec -i eos-krb-test cat /root/admin1.keytab | docker exec -i ${CLIENTHOSTNAME} bash -c "cat > /root/admin1.keytab"
docker exec -i ${CLIENTHOSTNAME} kinit -kt /root/admin1.keytab admin1@TEST.EOS
docker exec -i ${CLIENTHOSTNAME} kvno host/eos-mgm-test.eoscluster.cern.ch
if [[ $with_proxy == 1 ]];
then
# Set created proxy server as cluster access point for EOS client
docker exec -i ${CLIENTHOSTNAME} bash -c "echo 'export '"$proxy_EOS_MGM_URL" >> /root/.bashrc; source /root/.bashrc"
else
CLIENTHOSTNAME=eos-client-test
docker run --privileged --pid=host -dit -h ${CLIENTHOSTNAME}.eoscluster.cern.ch --name ${CLIENTHOSTNAME} --net=eoscluster.cern.ch --net-alias=${CLIENTHOSTNAME} $deb_cli_img
docker exec -i eos-krb-test cat /root/admin1.keytab | docker exec -i ${CLIENTHOSTNAME} bash -c "cat > /root/admin1.keytab"
docker exec -i ${CLIENTHOSTNAME} kinit -kt /root/admin1.keytab admin1@TEST.EOS
docker exec -i ${CLIENTHOSTNAME} kvno host/eos-mgm-test.eoscluster.cern.ch
# Set MGM server as cluster access point for EOS client
docker exec -i ${CLIENTHOSTNAME} bash -c "echo 'export '"$regular_EOS_MGM_URL" >> /root/.bashrc; source /root/.bashrc"
fi
else
for (( i=1; i<=$n_client; i++ )); do
if [[ $deb_cli_img == "" ]]; then
CLIENTHOSTNAME=eos-client${i}-test
docker run --privileged --pid=host -dit -h ${CLIENTHOSTNAME}.eoscluster.cern.ch --name ${CLIENTHOSTNAME} --net=eoscluster.cern.ch --net-alias=${CLIENTHOSTNAME} $image
docker exec -i eos-krb-test cat /root/admin1.keytab | docker exec -i ${CLIENTHOSTNAME} bash -c "cat > /root/admin1.keytab"
docker exec -i ${CLIENTHOSTNAME} kinit -kt /root/admin1.keytab admin1@TEST.EOS
docker exec -i ${CLIENTHOSTNAME} kvno host/eos-mgm-test.eoscluster.cern.ch
else
CLIENTHOSTNAME=eos-client${i}-test
docker run --privileged --pid=host -dit -h ${CLIENTHOSTNAME}.eoscluster.cern.ch --name ${CLIENTHOSTNAME} --net=eoscluster.cern.ch --net-alias=${CLIENTHOSTNAME} $deb_cli_img
docker exec -i eos-krb-test cat /root/admin1.keytab | docker exec -i ${CLIENTHOSTNAME} bash -c "cat > /root/admin1.keytab"
docker exec -i ${CLIENTHOSTNAME} kinit -kt /root/admin1.keytab admin1@TEST.EOS
docker exec -i ${CLIENTHOSTNAME} kvno host/eos-mgm-test.eoscluster.cern.ch
fi
done
fi
done
# Trace all possible trace levels
xrd.trace all
# All URLs with xroot and root protocols will be forwarded
all.export /xroot:/
all.export /root:/
# Load the shared library implementing a special pss (storage system) component
# which is the plug-in that changes a regular xrootd server into a proxy server
ofs.osslib /usr/lib64/libXrdPss.so
# The pss.origin directive below specifies that the endpoint will be supplied
# by the client - forwarding mode proxy
pss.origin =
d /var/run/xrootd - daemon daemon -
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment