From cbcf39095502b7d389d0f1afe320a334292280f9 Mon Sep 17 00:00:00 2001 From: yatin Date: Mon, 5 Dec 2016 21:36:47 +0530 Subject: [PATCH] [cern] [k8s_fedora_atomic] Enable TLS in Etcd cluster Cherry-pick: https://review.openstack.org/#/c/407374/ With this patch following are done:- - Configure Etcd with TLS support Configure Following to commuicate with TLS enabled Etcd:- - Flannel Etcd also listens at http://127.0.0.1:2379, so on master nodes etcdctl and kube apiserver can communicate without using certificates. if TLS_DISABLED="True" then TLS is not enabled for etcd. Change-Id: I2147b67c4e346a4415e1f76c19ac68e94cb0a0fa Partially-Implements: blueprint secure-etcd-cluster-coe Conflicts: magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh --- .../kubernetes/fragments/configure-etcd.sh | 27 ++++++++++++++--- .../fragments/configure-kubernetes-master.sh | 6 +--- .../fragments/configure-kubernetes-minion.sh | 30 ++++++++++++++++--- .../kubernetes/fragments/make-cert-client.sh | 10 +++++-- .../kubernetes/fragments/make-cert.sh | 10 +++++-- .../fragments/network-config-service.sh | 27 +++++++++++++++-- 6 files changed, 90 insertions(+), 20 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh index 74563c8ab..223bd8b01 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh @@ -8,18 +8,37 @@ if [ -z "$KUBE_NODE_IP" ]; then fi myip="${KUBE_NODE_IP}" +cert_dir="/srv/kubernetes" +protocol="https" + +if [ "$TLS_DISABLED" = "True" ]; then + protocol="http" +fi cat > /etc/etcd/etcd.conf <> /etc/etcd/etcd.conf <> /etc/etcd/etcd.conf fi diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 0d6308b58..d481ff492 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -10,11 +10,7 @@ if [ -z "$KUBE_NODE_IP" ]; then fi sed -i ' - /^ETCD_LISTEN_CLIENT_URLS=/ s/=.*/="http:\/\/0.0.0.0:2379"/ -' /etc/etcd/etcd.conf - -sed -i ' - /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/ + /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/ ' /etc/kubernetes/config KUBE_API_ARGS="--runtime_config=api/all=true" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index eea4de84c..2ce5a8afb 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -9,9 +9,30 @@ if [ -z "$KUBE_NODE_IP" ]; then KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) fi +CERT_DIR=/srv/kubernetes +PROTOCOL=https +FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \ +-etcd-certfile $CERT_DIR/client.crt \ +-etcd-keyfile $CERT_DIR/client.key" +ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \ +--cert $CERT_DIR/client.crt --key $CERT_DIR/client.key" ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP} KUBE_PROTOCOL="https" KUBE_CONFIG="" +FLANNELD_CONFIG=/etc/sysconfig/flanneld + +if [ "$TLS_DISABLED" = "True" ]; then + PROTOCOL=http + FLANNEL_OPTIONS="" + ETCD_CURL_OPTIONS="" +fi + +sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG + +cat >> $FLANNELD_CONFIG < ${CLIENT_CERT} -chmod 500 "${cert_dir}" -chown -R kube:kube "${cert_dir}" +# Common certs and key are created for both etcd and kubernetes services. +# Both etcd and kube user should have permission to access the certs and key. +groupadd kube_etcd +usermod -a -G kube_etcd etcd +usermod -a -G kube_etcd kube +chmod 550 "${cert_dir}" +chown -R kube:kube_etcd "${cert_dir}" +chmod 440 $CLIENT_KEY sed -i ' s|CA_CERT|'"$CA_CERT"'| diff --git a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh index 1612184aa..2c618e7c9 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh @@ -124,5 +124,11 @@ curl -X POST \ -d "$csr_req" \ $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT} -chmod 500 "${cert_dir}" -chown -R kube:kube "${cert_dir}" +# Common certs and key are created for both etcd and kubernetes services. +# Both etcd and kube user should have permission to access the certs and key. +groupadd kube_etcd +usermod -a -G kube_etcd etcd +usermod -a -G kube_etcd kube +chmod 550 "${cert_dir}" +chown -R kube:kube_etcd "${cert_dir}" +chmod 440 $SERVER_KEY diff --git a/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh index 00d0c8f9d..1cac6b580 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/network-config-service.sh @@ -5,8 +5,28 @@ if [ "$NETWORK_DRIVER" != "flannel" ]; then exit 0 fi +CERT_DIR=/srv/kubernetes +PROTOCOL=https +FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \ +-etcd-certfile $CERT_DIR/server.crt \ +-etcd-keyfile $CERT_DIR/server.key" +ETCD_CURL_OPTIONS="--cacert $CERT_DIR/ca.crt \ +--cert $CERT_DIR/server.crt --key $CERT_DIR/server.key" +FLANNELD_CONFIG=/etc/sysconfig/flanneld -. /etc/sysconfig/flanneld +if [ "$TLS_DISABLED" = "True" ]; then + PROTOCOL=http + FLANNEL_OPTIONS="" + ETCD_CURL_OPTIONS="" +fi + +sed -i '/FLANNEL_OPTIONS/'d $FLANNELD_CONFIG + +cat >> $FLANNELD_CONFIG <