From cbcf39095502b7d389d0f1afe320a334292280f9 Mon Sep 17 00:00:00 2001
From: yatin <yatin.karel@nectechnologies.in>
Date: Mon, 5 Dec 2016 21:36:47 +0530
Subject: [PATCH] [cern] [k8s_fedora_atomic] Enable TLS in Etcd cluster
Cherry-pick: https://review.openstack.org/#/c/407374/
With this patch following are done:-
- Configure Etcd with TLS support
Configure Following to commuicate with TLS enabled Etcd:-
- Flannel
Etcd also listens at http://127.0.0.1:2379, so on master nodes
etcdctl and kube apiserver can communicate without using
certificates.
if TLS_DISABLED="True" then TLS is not enabled for etcd.
Change-Id: I2147b67c4e346a4415e1f76c19ac68e94cb0a0fa
Partially-Implements: blueprint secure-etcd-cluster-coe
Conflicts:
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh
---
.../kubernetes/fragments/configure-etcd.sh | 27 ++++++++++++++---
.../fragments/configure-kubernetes-master.sh | 6 +---
.../fragments/configure-kubernetes-minion.sh | 30 ++++++++++++++++---
.../kubernetes/fragments/make-cert-client.sh | 10 +++++--
.../kubernetes/fragments/make-cert.sh | 10 +++++--
.../fragments/network-config-service.sh | 27 +++++++++++++++--
6 files changed, 90 insertions(+), 20 deletions(-)
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh
index 74563c8ab..223bd8b01 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh
@@ -8,18 +8,37 @@ if [ -z "$KUBE_NODE_IP" ]; then
fi
myip="${KUBE_NODE_IP}"
+cert_dir="/srv/kubernetes"
+protocol="https"
+
+if [ "$TLS_DISABLED" = "True" ]; then
+ protocol="http"
+fi
cat > /etc/etcd/etcd.conf <<EOF
ETCD_NAME="$myip"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
-ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
-ETCD_LISTEN_PEER_URLS="http://$myip:2380"
+ETCD_LISTEN_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
+ETCD_LISTEN_PEER_URLS="$protocol://$myip:2380"
-ETCD_ADVERTISE_CLIENT_URLS="http://$myip:2379"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://$myip:2380"
+ETCD_ADVERTISE_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="$protocol://$myip:2380"
ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
EOF
+if [ "$TLS_DISABLED" = "False" ]; then
+
+cat >> /etc/etcd/etcd.conf <<EOF
+ETCD_CA_FILE=$cert_dir/ca.crt
+ETCD_CERT_FILE=$cert_dir/server.crt
+ETCD_KEY_FILE=$cert_dir/server.key
+ETCD_PEER_CA_FILE=$cert_dir/ca.crt
+ETCD_PEER_CERT_FILE=$cert_dir/server.crt
+ETCD_PEER_KEY_FILE=$cert_dir/server.key
+EOF
+
+fi
+
if [ -n "$HTTP_PROXY" ]; then
echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
fi
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
index 0d6308b58..d481ff492 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
@@ -10,11 +10,7 @@ if [ -z "$KUBE_NODE_IP" ]; then
fi
sed -i '
- /^ETCD_LISTEN_CLIENT_URLS=/ s/=.*/="http:\/\/0.0.0.0:2379"/
-' /etc/etcd/etcd.conf
-
-sed -i '
- /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
+ /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
' /etc/kubernetes/config
KUBE_API_ARGS="--runtime_config=api/all=true"
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
index eea4de84c..2ce5a8afb 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
@@ -9,9 +9,30 @@ if [ -z "$KUBE_NODE_IP" ]; then
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
fi
+CERT_DIR=/srv/kubernetes
+PROTOCOL=https
+FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \
+-etcd-certfile $CERT_DIR/client.crt \
+-etcd-keyfile $CERT_DIR/client.key"
+ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \
+--cert $CERT_DIR/client.crt --key $CERT_DIR/client.key"
ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}
KUBE_PROTOCOL="https"
KUBE_CONFIG=""
+FLANNELD_CONFIG=/etc/sysconfig/flanneld
+
+if [ "$TLS_DISABLED" = "True" ]; then
+ PROTOCOL=http
+ FLANNEL_OPTIONS=""
+ ETCD_CURL_OPTIONS=""
+fi
+
+sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG
+
+cat >> $FLANNELD_CONFIG <<EOF
+FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
+EOF
+
if [ "$TLS_DISABLED" = "True" ]; then
KUBE_PROTOCOL="http"
else
@@ -52,12 +73,13 @@ sed -i '
if [ "$NETWORK_DRIVER" = "flannel" ]; then
sed -i '
- /^FLANNEL_ETCD=/ s|=.*|="http://'"$ETCD_SERVER_IP"':2379"|
- ' /etc/sysconfig/flanneld
+ /^FLANNEL_ETCD=/ s|=.*|="'"$PROTOCOL"'://'"$ETCD_SERVER_IP"':2379"|
+ ' $FLANNELD_CONFIG
# Make sure etcd has a flannel configuration
- . /etc/sysconfig/flanneld
- until curl -sf "$FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config?quorum=false&recursive=false&sorted=false"
+ . $FLANNELD_CONFIG
+ until curl -sf $ETCD_CURL_OPTIONS \
+ "$FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config?quorum=false&recursive=false&sorted=false"
do
echo "Waiting for flannel configuration in etcd..."
sleep 5
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh b/magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh
index d19d8c274..27f025d30 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh
@@ -107,8 +107,14 @@ curl -X POST \
-d "$csr_req" \
$MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CLIENT_CERT}
-chmod 500 "${cert_dir}"
-chown -R kube:kube "${cert_dir}"
+# Common certs and key are created for both etcd and kubernetes services.
+# Both etcd and kube user should have permission to access the certs and key.
+groupadd kube_etcd
+usermod -a -G kube_etcd etcd
+usermod -a -G kube_etcd kube
+chmod 550 "${cert_dir}"
+chown -R kube:kube_etcd "${cert_dir}"
+chmod 440 $CLIENT_KEY
sed -i '
s|CA_CERT|'"$CA_CERT"'|
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh
index 1612184aa..2c618e7c9 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh
@@ -124,5 +124,11 @@ curl -X POST \
-d "$csr_req" \
$MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT}
-chmod 500 "${cert_dir}"
-chown -R kube:kube "${cert_dir}"
+# Common certs and key are created for both etcd and kubernetes services.
+# Both etcd and kube user should have permission to access the certs and key.
+groupadd kube_etcd
+usermod -a -G kube_etcd etcd
+usermod -a -G kube_etcd kube
+chmod 550 "${cert_dir}"
+chown -R kube:kube_etcd "${cert_dir}"
+chmod 440 $SERVER_KEY
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh
index 00d0c8f9d..1cac6b580 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh
@@ -5,8 +5,28 @@
if [ "$NETWORK_DRIVER" != "flannel" ]; then
exit 0
fi
+CERT_DIR=/srv/kubernetes
+PROTOCOL=https
+FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \
+-etcd-certfile $CERT_DIR/server.crt \
+-etcd-keyfile $CERT_DIR/server.key"
+ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \
+--cert $CERT_DIR/server.crt --key $CERT_DIR/server.key"
+FLANNELD_CONFIG=/etc/sysconfig/flanneld
-. /etc/sysconfig/flanneld
+if [ "$TLS_DISABLED" = "True" ]; then
+ PROTOCOL=http
+ FLANNEL_OPTIONS=""
+ ETCD_CURL_OPTIONS=""
+fi
+
+sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG
+
+cat >> $FLANNELD_CONFIG <<EOF
+FLANNEL_OPTIONS="$FLANNEL_OPTIONS"
+EOF
+
+. $FLANNELD_CONFIG
FLANNEL_CONFIG_BIN=/usr/local/bin/flannel-config
FLANNEL_CONFIG_SERVICE=/etc/systemd/system/flannel-config.service
@@ -27,8 +47,9 @@ if ! [ "$FLANNEL_ETCD" ] && [ "$FLANNEL_ETCD_KEY" ]; then
fi
echo "creating flanneld config in etcd"
-while ! curl -sf -L $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
- -X PUT --data-urlencode value@${FLANNEL_JSON}; do
+while ! curl -sf -L $ETCD_CURL_OPTIONS \
+ $FLANNEL_ETCD/v2/keys${FLANNEL_ETCD_KEY}/config \
+ -X PUT --data-urlencode value@${FLANNEL_JSON}; do
echo "waiting for etcd"
sleep 1
done
--
GitLab