From 303e67775443823f7e27d6205aab2b8bda870126 Mon Sep 17 00:00:00 2001 From: Diogo Guerra Date: Tue, 23 Aug 2022 14:51:17 +0200 Subject: [PATCH 1/4] [cern] Add kube-bench disable profilling CherryPick: https://review.opendev.org/c/openstack/magnum/+/854390 Story: 2010248 Task: 46083 Change-Id: I0241afbb3ce0b5690fbe7821afc71be2c9b5ee31 --- .../kubernetes/fragments/configure-kubernetes-master.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 15bcd6904..b9ac7824c 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -350,6 +350,7 @@ KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_acc KUBE_API_ARGS="$KUBE_API_ARGS --service-account-signing-key-file=${CERT_DIR}/service_account_private.key" KUBE_API_ARGS="$KUBE_API_ARGS --service-account-issuer=https://kubernetes.default.svc.cluster.local" KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key" +KUBE_API_ARGS="$KUBE_API_ARGS --profiling=false" # Allow for metrics-server/aggregator communication KUBE_API_ARGS="${KUBE_API_ARGS} \ --proxy-client-cert-file=${CERT_DIR}/server.crt \ @@ -475,6 +476,7 @@ KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true --kubeconfig=/etc/kubernetes/a KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}" KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true" KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${CLUSTER_CIDR} ${NODE_CIDR_MASK_SIZE_IPV6}" +KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --profiling=false" KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS" if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt" @@ -510,6 +512,7 @@ if [[ "${supported_min_version}" = "${min_version}" ]]; then else sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"#' /etc/kubernetes/scheduler fi +sed -i '/^KUBE_SCHEDULER_ARGS=/ s#="\(.*\)"#="\1 --profiling=false#' /etc/kubernetes/scheduler $ssh_cmd mkdir -p /etc/kubernetes/manifests KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}" -- GitLab From d6f950f70280c2ddb7d8e890a37db40ac778fc56 Mon Sep 17 00:00:00 2001 From: Diogo Guerra Date: Mon, 8 Aug 2022 17:53:13 +0200 Subject: [PATCH 2/4] [cern] Ensure kube-apiserver TLS connection to etcd server CherryPick: https://review.opendev.org/c/openstack/magnum/+/854394 Improve kube-bench remediations 1.2.26 and 1.2.29 Story: 2010248 Task: 46091 Change-Id: I8349265d0fc23513a96328669debe5b554db4df6 --- .../kubernetes/fragments/configure-kubernetes-master.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index b9ac7824c..14c50e444 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -346,6 +346,7 @@ KUBE_API_ADDRESS="--bind-address=0.0.0.0 --secure-port=$KUBE_API_PORT" KUBE_API_ARGS="$KUBE_API_ARGS --authorization-mode=Node,RBAC --tls-cert-file=$CERT_DIR/server.crt" KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key" KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt" +KUBE_API_ARGS="$KUBE_API_ARGS --etcd-cafile=$CERT_DIR/ca.crt --etcd-certfile=$CERT_DIR/server.crt --etcd-keyfile=$CERT_DIR/server.key" KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_account.key" KUBE_API_ARGS="$KUBE_API_ARGS --service-account-signing-key-file=${CERT_DIR}/service_account_private.key" KUBE_API_ARGS="$KUBE_API_ARGS --service-account-issuer=https://kubernetes.default.svc.cluster.local" -- GitLab From ed7b8a3c5953a00391478ed016b53b5f67cd4e52 Mon Sep 17 00:00:00 2001 From: Diogo Guerra Date: Thu, 11 Aug 2022 16:49:18 +0200 Subject: [PATCH 3/4] [cern] Fix kube-bench 1.2.32 and 4.2.13 CherryPick: https://review.opendev.org/c/openstack/magnum/+/854396 Story: 2010248 Task: 46086 Change-Id: Idd8b02df0e9191408a0757901d7e8785e952175b --- .../kubernetes/fragments/configure-kubernetes-master.sh | 3 +++ .../kubernetes/fragments/configure-kubernetes-minion.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 14c50e444..d93425d86 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -362,6 +362,8 @@ KUBE_API_ARGS="${KUBE_API_ARGS} \ --requestheader-group-headers=X-Remote-Group \ --requestheader-username-headers=X-Remote-User" +KUBE_API_ARGS="$KUBE_API_ARGS --tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384" + KUBE_ADMISSION_CONTROL="" if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_ADMISSION_CONTROL="--admission-control=NodeRestriction,${ADMISSION_CONTROL_LIST}" @@ -548,6 +550,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=node-role.kubernetes.io/master=:NoSchedule" KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/role=${NODEGROUP_ROLE}" KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/nodegroup=${NODEGROUP_NAME}" +KUBELET_ARGS="${KUBELET_ARGS} --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256" KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml cat << EOF >> ${KUBELET_KUBECONFIG} diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index e846898bb..0142eb8bf 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -266,6 +266,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=$ KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins" KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/role=${NODEGROUP_ROLE}" KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/nodegroup=${NODEGROUP_NAME}" +KUBELET_ARGS="${KUBELET_ARGS} --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256" KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}" if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then -- GitLab From 5ca33ee91356628a9978ff6c23bae0028a6a0de1 Mon Sep 17 00:00:00 2001 From: Diogo Guerra Date: Tue, 9 Aug 2022 14:08:36 +0200 Subject: [PATCH 4/4] [cern] Fix kube-bench 1.2.1, 1.2.23, 1.3.1. CherryPick: https://review.opendev.org/c/openstack/magnum/+/854395 Story: 2010248 Task: 46085 Task: 46087 Task: 46088 Change-Id: Ic197f83866dfa636fcb62c9875691d94bb42f964 --- .../kubernetes/fragments/configure-kubernetes-master.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index d93425d86..545588d51 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -361,7 +361,8 @@ KUBE_API_ARGS="${KUBE_API_ARGS} \ --requestheader-extra-headers-prefix=X-Remote-Extra- \ --requestheader-group-headers=X-Remote-Group \ --requestheader-username-headers=X-Remote-User" - +KUBE_API_ARGS="$KUBE_API_ARGS --anonymous-auth=false" +KUBE_API_ARGS="$KUBE_API_ARGS --request-timeout=10s" KUBE_API_ARGS="$KUBE_API_ARGS --tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384" KUBE_ADMISSION_CONTROL="" @@ -480,6 +481,7 @@ KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLU KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true" KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${CLUSTER_CIDR} ${NODE_CIDR_MASK_SIZE_IPV6}" KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --profiling=false" +KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --terminated-pod-gc-threshold=200" KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS" if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt" -- GitLab