From 9bfaf421dbe7efd891ea141ac2141d508ef18cd7 Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Wed, 10 Nov 2021 09:13:58 +0100 Subject: [PATCH 1/4] [cern] Add dual stack support --- .../kubernetes/fragments/calico-service.sh | 17 ++++++++++-- .../fragments/configure-kubernetes-master.sh | 27 ++++++++++++++----- .../fragments/configure-kubernetes-minion.sh | 16 +++++++++-- .../kubernetes/fragments/core-dns-service.sh | 14 ++++++++-- .../kubernetes/fragments/make-cert.sh | 1 + .../fragments/write-heat-params-master.sh | 2 ++ .../kubernetes/fragments/write-heat-params.sh | 1 + .../drivers/heat/k8s_fedora_template_def.py | 1 + .../templates/kubecluster.yaml | 13 +++++++++ .../templates/kubemaster.yaml | 11 ++++++++ .../templates/kubeminion.yaml | 5 ++++ 11 files changed, 96 insertions(+), 12 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 936cd89b3..dd238a031 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -6,6 +6,17 @@ set +x . /etc/sysconfig/heat-params set -x +kube_min_version=$(echo "${KUBE_TAG}" | cut -d'.' -f2) +if [ "$kube_min_version" -ge "21" ]; then + ASSIGN_IP="\"assign_ipv4\": \"true\",\n \"assign_ipv6\": \"true\"" + FELIX_IPV6SUPPORT="true" + IP6_EXTRAS="- name: CALICO_IPV6POOL_CIDR\n value: '2001:4860::0/108'\n - name: IP6\n value: autodetect\n" +else + ASSIGN_IP="\"assign_ipv4\": \"true\"" + FELIX_IPV6SUPPORT="false" + IP6_EXTRAS="" +fi + if [ "$NETWORK_DRIVER" = "calico" ]; then _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/calico/} @@ -280,7 +291,8 @@ data: "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { - "type": "calico-ipam" + "type": "calico-ipam", + $(echo -e "${ASSIGN_IP}") }, "policy": { "type": "k8s" @@ -470,7 +482,8 @@ spec: value: "ACCEPT" # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT - value: "false" + value: "${FELIX_IPV6SUPPORT}" + $(echo -e "${IP6_EXTRAS}") # Set Felix logging to "info" - name: FELIX_LOGSEVERITYSCREEN value: "info" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index e5d8f4f5a..00968dcc4 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -25,8 +25,20 @@ $ssh_cmd rm -rf /opt/cni/* $ssh_cmd mkdir -p /opt/cni/bin $ssh_cmd mkdir -p /etc/cni/net.d/ +kube_min_version=$(echo "${KUBE_TAG}" | cut -d'.' -f2) +if [ "$kube_min_version" -ge "21" ]; then + CLUSTER_CIDR="${PODS_NETWORK_CIDR},${PODS_NETWORK6_CIDR}" + PORTAL_CIDR="${PORTAL_NETWORK_CIDR},${PORTAL_NETWORK6_CIDR}" + NODE_CIDR_MASK_SIZE_IPV6="--node-cidr-mask-size-ipv6=108" +else + CLUSTER_CIDR="${PODS_NETWORK_CIDR}" + PORTAL_CIDR="${PORTAL_NETWORK_CIDR}" + NODE_CIDR_MASK_SIZE_IPV6="" +fi + if [ "$NETWORK_DRIVER" = "calico" ]; then echo "net.ipv4.conf.all.rp_filter = 1" >> /etc/sysctl.conf + echo "net.ipv6.conf.all.forwarding = 1" >> /etc/sysctl.conf $ssh_cmd sysctl -p if [ "`systemctl status NetworkManager.service | grep -o "Active: active"`" = "Active: active" ]; then CALICO_NM=/etc/NetworkManager/conf.d/calico.conf @@ -45,6 +57,9 @@ elif [ "$NETWORK_DRIVER" = "flannel" ]; then echo "vxlan" > /etc/modules-load.d/vxlan.conf fi +if [ "$kube_min_version" -ge "21" ]; then + $ssh_cmd 'ip a | grep 2001; while [ $? -ne 0 ]; do systemctl restart NetworkManager; sleep 5; ip a | grep 2001; done' +fi KUBE_MASTER_URI="https://127.0.0.1:$KUBE_API_PORT" mkdir -p /srv/magnum/kubernetes/ @@ -291,7 +306,7 @@ CERT_DIR=/etc/kubernetes/certs # kube-proxy config PROXY_KUBECONFIG=/etc/kubernetes/proxy-kubeconfig.yaml -KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME}" +KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr="${CLUSTER_CIDR}" --hostname-override=${INSTANCE_NAME}" cat > /etc/kubernetes/proxy << EOF KUBE_PROXY_ARGS="${KUBE_PROXY_ARGS} ${KUBEPROXY_OPTIONS}" EOF @@ -387,7 +402,7 @@ fi if [ "$(echo $KUBE_CSI_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then if [[ ! $KUBEAPI_OPTIONS == *"--feature-gates="* ]]; then - KUBE_API_ARGS="$KUBE_API_ARGS --feature-gates=KubeletPluginsWatcher=true,CSINodeInfo=true,CSIDriverRegistry=true" + KUBE_API_ARGS="$KUBE_API_ARGS --feature-gates=KubeletPluginsWatcher=true,CSINodeInfo=true,CSIDriverRegistry=true,IPv6DualStack=true,EndpointSlice=true" fi if [[ ! $KUBEAPI_OPTIONS == *"--runtime-config="* ]]; then KUBE_API_ARGS="$KUBE_API_ARGS --runtime-config=storage.k8s.io/v1alpha1=true" @@ -421,7 +436,7 @@ fi sed -i ' /^KUBE_API_ADDRESS=/ s/=.*/="'"${KUBE_API_ADDRESS}"'"/ - /^KUBE_SERVICE_ADDRESSES=/ s|=.*|="--service-cluster-ip-range='"$PORTAL_NETWORK_CIDR"'"| + /^KUBE_SERVICE_ADDRESSES=/ s|=.*|="--service-cluster-ip-range='"${PORTAL_CIDR}"'"| /^KUBE_API_ARGS=/ s|=.*|="'"${KUBE_API_ARGS}"'"| /^KUBE_ETCD_SERVERS=/ s/=.*/="--etcd-servers=http:\/\/127.0.0.1:2379"/ /^KUBE_ADMISSION_CONTROL=/ s/=.*/="'"${KUBE_ADMISSION_CONTROL}"'"/ @@ -459,7 +474,7 @@ export KUBECONFIG=${ADMIN_KUBECONFIG} KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf" KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}" KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true" -KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}" +KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${CLUSTER_CIDR} ${NODE_CIDR_MASK_SIZE_IPV6}" KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS" if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt" @@ -479,7 +494,7 @@ fi if [ "$(echo $KUBE_CSI_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then if [[ ! $KUBECONTROLLER_OPTIONS == *"--feature-gates="* ]]; then - KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --feature-gates=KubeletPluginsWatcher=true,CSINodeInfo=true,CSIDriverRegistry=true" + KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --feature-gates=KubeletPluginsWatcher=true,CSINodeInfo=true,CSIDriverRegistry=true,IPv6DualStack=true,EndpointSlice=true" fi fi @@ -509,7 +524,7 @@ fi if [ "$(echo $KUBE_CSI_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then if [[ ! $KUBELET_OPTIONS == *"--feature-gates="* ]]; then - KUBELET_ARGS="$KUBELET_ARGS --feature-gates=KubeletPluginsWatcher=true,CSINodeInfo=true,CSIDriverRegistry=true" + KUBELET_ARGS="$KUBELET_ARGS --feature-gates=KubeletPluginsWatcher=true,CSINodeInfo=true,CSIDriverRegistry=true,IPv6DualStack=true,EndpointSlice=true" fi fi diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 0d62e1e9a..b1238e637 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -27,8 +27,16 @@ $ssh_cmd mkdir -p /opt/cni/bin $ssh_cmd mkdir -p /etc/cni/net.d/ _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}' +kube_min_version=$(echo "${KUBE_TAG}" | cut -d'.' -f2) +if [ "$kube_min_version" -ge "21" ]; then + CLUSTER_CIDR="${PODS_NETWORK_CIDR},${PODS_NETWORK6_CIDR}" +else + CLUSTER_CIDR="${PODS_NETWORK_CIDR}" +fi + if [ "$NETWORK_DRIVER" = "calico" ]; then echo "net.ipv4.conf.all.rp_filter = 1" >> /etc/sysctl.conf + echo "net.ipv6.conf.all.forwarding = 1" >> /etc/sysctl.conf $ssh_cmd sysctl -p if [ "$($ssh_cmd systemctl status NetworkManager.service | grep -o "Active: active")" = "Active: active" ]; then CALICO_NM=/etc/NetworkManager/conf.d/calico.conf @@ -47,6 +55,10 @@ elif [ "$NETWORK_DRIVER" = "flannel" ]; then echo "vxlan" > /etc/modules-load.d/vxlan.conf fi +if [ "$kube_min_version" -ge "21" ]; then + $ssh_cmd 'ip a | grep 2001; while [ $? -ne 0 ]; do systemctl restart NetworkManager; sleep 5; ip a | grep 2001; done' +fi + mkdir -p /srv/magnum/kubernetes/ cat > /etc/kubernetes/config < /etc/kubernetes/proxy << EOF KUBE_PROXY_ARGS="${KUBE_PROXY_ARGS} ${KUBEPROXY_OPTIONS}" EOF diff --git a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh index 808c316c6..84c582508 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh @@ -5,8 +5,17 @@ printf "Starting to run ${step}\n" _dns_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/coredns/} _autoscaler_prefix=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/} - CORE_DNS=/srv/magnum/kubernetes/manifests/kube-coredns.yaml + +kube_min_version=$(echo "${KUBE_TAG}" | cut -d'.' -f2) +if [ "$kube_min_version" -ge "21" ]; then + DNS_LIST="${DNS_CLUSTER_DOMAIN} ${PORTAL_NETWORK_CIDR} ${PORTAL_NETWORK6_CIDR} ${PODS_NETWORK_CIDR} ${PODS_NETWORK6_CIDR}" + IP_FAMILY_POLICY="ipFamilyPolicy: RequireDualStack" +else + DNS_LIST="${DNS_CLUSTER_DOMAIN} ${PORTAL_NETWORK_CIDR} ${PODS_NETWORK_CIDR}" + IP_FAMILY_POLICY="" +fi + [ -f ${CORE_DNS} ] || { echo "Writing File: $CORE_DNS" mkdir -p $(dirname ${CORE_DNS}) @@ -71,7 +80,7 @@ data: errors log stdout health - kubernetes ${DNS_CLUSTER_DOMAIN} ${PORTAL_NETWORK_CIDR} ${PODS_NETWORK_CIDR} { + kubernetes ${DNS_LIST} { pods verified fallthrough in-addr.arpa ip6.arpa } @@ -192,6 +201,7 @@ metadata: kubernetes.io/cluster-service: "true" kubernetes.io/name: "CoreDNS" spec: + ${IP_FAMILY_POLICY} selector: k8s-app: kube-dns clusterIP: ${DNS_SERVICE_IP} diff --git a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh index 2089f0833..223ae0dc5 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh @@ -67,6 +67,7 @@ fi sans="${sans},IP:127.0.0.1" KUBE_SERVICE_IP=$(echo $PORTAL_NETWORK_CIDR | awk 'BEGIN{FS="[./]"; OFS="."}{print $1,$2,$3,$4 + 1}') +# TODO: need ip6 here? sans="${sans},IP:${KUBE_SERVICE_IP}" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh index e05eabbab..d2d6eb515 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh @@ -29,7 +29,9 @@ FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN" FLANNEL_BACKEND="$FLANNEL_BACKEND" PODS_NETWORK_CIDR="$PODS_NETWORK_CIDR" +PODS_NETWORK6_CIDR="$PODS_NETWORK6_CIDR" PORTAL_NETWORK_CIDR="$PORTAL_NETWORK_CIDR" +PORTAL_NETWORK6_CIDR="$PORTAL_NETWORK6_CIDR" ADMISSION_CONTROL_LIST="$ADMISSION_CONTROL_LIST" ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL" USERNAME="$USERNAME" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh index 94ada8bd0..93b1e2e29 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh @@ -44,6 +44,7 @@ WAIT_CURL="$WAIT_CURL" KUBE_TAG="$KUBE_TAG" FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" PODS_NETWORK_CIDR="$PODS_NETWORK_CIDR" +PODS_NETWORK6_CIDR="$PODS_NETWORK6_CIDR" KUBE_VERSION="$KUBE_VERSION" TRUSTEE_USER_ID="$TRUSTEE_USER_ID" TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index dcdfe978d..c85e91be8 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -58,6 +58,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): if cluster_template.network_driver == 'calico': extra_params["pods_network_cidr"] = \ cluster.labels.get('calico_ipv4pool', '192.168.0.0/16') + extra_params["pods_network6_cidr"] = '2001:4860::0/108' # check cloud provider and cinder options. If cinder is selected, # the cloud provider needs to be enabled. diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index e37d675a2..5a9e39c32 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -182,6 +182,12 @@ parameters: address range used by kubernetes for service portals default: 10.254.0.0/16 + portal_network6_cidr: + type: string + description: > + address range used by kubernetes for service portals + default: "fd5e:d3bb:de2e::/108" + network_driver: type: string description: network driver to use for instantiating container networks @@ -593,6 +599,10 @@ parameters: type: string description: Configure the IP pool/range from which pod IPs will be chosen + pods_network6_cidr: + type: string + description: Configure the IP pool/range from which pod IPs will be chosen + ingress_controller: type: string description: > @@ -1271,6 +1281,7 @@ resources: system_pods_initial_delay: {get_param: system_pods_initial_delay} system_pods_timeout: {get_param: system_pods_timeout} portal_network_cidr: {get_param: portal_network_cidr} + portal_network6_cidr: {get_param: portal_network6_cidr} admission_control_list: {get_param: admission_control_list} discovery_url: {get_param: discovery_url} cluster_uuid: {get_param: cluster_uuid} @@ -1323,6 +1334,7 @@ resources: calico_ipv4pool: {get_param: calico_ipv4pool} calico_ipv4pool_ipip: {get_param: calico_ipv4pool_ipip} pods_network_cidr: {get_param: pods_network_cidr} + pods_network6_cidr: {get_param: pods_network6_cidr} ingress_controller: {get_param: ingress_controller} ingress_controller_role: {get_param: ingress_controller_role} octavia_ingress_controller_tag: {get_param: octavia_ingress_controller_tag} @@ -1593,6 +1605,7 @@ resources: nodes_server_group_id: {get_resource: worker_nodes_server_group} availability_zone: {get_param: availability_zone} pods_network_cidr: {get_param: pods_network_cidr} + pods_network6_cidr: {get_param: pods_network6_cidr} kubelet_options: {get_param: kubelet_options} kubeproxy_options: {get_param: kubeproxy_options} octavia_enabled: {get_param: octavia_enabled} diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index 130e138af..f593adcda 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -56,6 +56,11 @@ parameters: description: > address range used by kubernetes for service portals + portal_network6_cidr: + type: string + description: > + address range used by kubernetes for service portals + kube_allow_priv: type: string description: > @@ -393,6 +398,10 @@ parameters: type: string description: Configure the IP pool/range from which pod IPs will be chosen + pods_network6_cidr: + type: string + description: Configure the IP pool/range from which pod IPs will be chosen + ingress_controller: type: string description: > @@ -936,7 +945,9 @@ resources: "$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay} "$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout} "$PODS_NETWORK_CIDR": {get_param: pods_network_cidr} + "$PODS_NETWORK6_CIDR": {get_param: pods_network6_cidr} "$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr} + "$PORTAL_NETWORK6_CIDR": {get_param: portal_network6_cidr} "$ADMISSION_CONTROL_LIST": {get_param: admission_control_list} "$ETCD_DISCOVERY_URL": {get_param: discovery_url} "$AUTH_URL": {get_param: auth_url} diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml index b15a0ca53..dae78b075 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml @@ -274,6 +274,10 @@ parameters: type: string description: Configure the IP pool/range from which pod IPs will be chosen + pods_network6_cidr: + type: string + description: Configure the IP pool/range from which pod IPs will be chosen + kubelet_options: type: string description: > @@ -466,6 +470,7 @@ resources: $KUBE_TAG: {get_param: kube_tag} $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr} $PODS_NETWORK_CIDR: {get_param: pods_network_cidr} + $PODS_NETWORK6_CIDR: {get_param: pods_network6_cidr} $KUBE_VERSION: {get_param: kube_version} $TRUSTEE_USER_ID: {get_param: trustee_user_id} $TRUSTEE_PASSWORD: {get_param: trustee_password} -- GitLab From fd296ed195a13d09a3e310b7db873afae092b4ce Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Wed, 24 Nov 2021 12:55:39 +0100 Subject: [PATCH 2/4] [cern] fix snapshot params in kubecluster --- .../k8s_fedora_coreos_v1/templates/kubecluster.yaml | 8 ++++---- .../k8s_fedora_coreos_v1/templates/kubemaster.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index 5a9e39c32..33af4b643 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -1122,25 +1122,25 @@ parameters: description: base64 encoded helm values for the cern meta chart default: "" - snapshot_controller_enabled: {get_param: snapshot_controller_enabled} + snapshot_controller_enabled: type: boolean description: > Indicates whether snapshot controller should be started. default: false - snapshot_controller_version: {get_param: snapshot_controller_version} + snapshot_controller_version: type: string description: > Indicates the version of snapshot controller to use. default: "v0.2.0" - snapshot_validation_webhook_enabled: {get_param: snapshot_validation_webhook_enabled} + snapshot_validation_webhook_enabled: type: boolean description: > Indicates whether snapshot validation webhook should be started. default: false - snapshot_validation_webhook_version: {get_param: snapshot_validation_webhook_version} + snapshot_validation_webhook_version: type: string description: > Indicates the version of snapshot validation webhook to use. diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index f593adcda..4667dc576 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -834,25 +834,25 @@ parameters: description: The cern username of the user that creates the cluster default: '' - snapshot_controller_enabled: {get_param: snapshot_controller_enabled} + snapshot_controller_enabled: type: boolean description: > Indicates whether snapshot controller should be started. default: false - snapshot_controller_version: {get_param: snapshot_controller_version} + snapshot_controller_version: type: string description: > Indicates the version of snapshot controller to use. default: "v0.2.0" - snapshot_validation_webhook_enabled: {get_param: snapshot_validation_webhook_enabled} + snapshot_validation_webhook_enabled: type: boolean description: > Indicates whether snapshot validation webhook should be started. default: false - snapshot_validation_webhook_version: {get_param: snapshot_validation_webhook_version} + snapshot_validation_webhook_version: type: string description: > Indicates the version of snapshot validation webhook to use. -- GitLab From 648d0156a1ce7d2e2f2c91e6ddcbc2f52e7095eb Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Wed, 24 Nov 2021 14:56:06 +0100 Subject: [PATCH 3/4] [cern] v1 for authorization resources, csidriver cvmfs Move out of auth resources v1beta1, add csidriver definition to cvmfs script since we moved cephfs csi to helm. Restart network manager until we get an ipv6. --- .../kubernetes/fragments/configure-kubernetes-master.sh | 2 +- .../common/templates/kubernetes/fragments/cvmfs-csi-1x.sh | 4 ++-- .../templates/kubernetes/fragments/enable-keystone-auth.sh | 4 ++-- .../kubernetes/fragments/kube-apiserver-to-kubelet-role.sh | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 00968dcc4..5021c595a 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -50,7 +50,7 @@ if [ "$NETWORK_DRIVER" = "calico" ]; then unmanaged-devices=interface-name:cali*;interface-name:tunl* EOF } - systemctl restart NetworkManager + $ssh_cmd systemctl restart NetworkManager fi elif [ "$NETWORK_DRIVER" = "flannel" ]; then $ssh_cmd modprobe vxlan diff --git a/magnum/drivers/common/templates/kubernetes/fragments/cvmfs-csi-1x.sh b/magnum/drivers/common/templates/kubernetes/fragments/cvmfs-csi-1x.sh index 4e94358c0..f2d989a56 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/cvmfs-csi-1x.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/cvmfs-csi-1x.sh @@ -14,7 +14,7 @@ metadata: name: cvmfs-csi-nodeplugin namespace: kube-system --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodSecurityPolicy metadata: name: cvmfs.privileged @@ -242,7 +242,7 @@ roleRef: name: cvmfs-external-provisioner-cfg apiGroup: rbac.authorization.k8s.io --- -apiVersion: storage.k8s.io/v1beta1 +apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi-cvmfsplugin diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh index 2dd345ea4..8c3c2384e 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh @@ -21,7 +21,7 @@ metadata: name: k8s-keystone-auth namespace: kube-system --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: @@ -39,7 +39,7 @@ rules: - list - watch --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh index 562c515d7..0b2c55a3a 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh @@ -12,7 +12,7 @@ do done cat < Date: Thu, 25 Nov 2021 13:46:12 +0100 Subject: [PATCH 4/4] [cern] move metrics-server to umbrella chart --- magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh | 2 ++ .../drivers/common/templates/kubernetes/helm/metrics-server.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh b/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh index fbc158625..4be926b7d 100644 --- a/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh +++ b/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh @@ -123,6 +123,8 @@ ${NVIDIA_GPU_VALUES} tag: ${TRAEFIK_INGRESS_CONTROLLER_TAG} nodeSelector: role: ${INGRESS_CONTROLLER_ROLE} + metrics-server: + enabled: ${METRICS_SERVER_ENABLED} EOF fi diff --git a/magnum/drivers/common/templates/kubernetes/helm/metrics-server.sh b/magnum/drivers/common/templates/kubernetes/helm/metrics-server.sh index 38d9043a6..e78e10e41 100755 --- a/magnum/drivers/common/templates/kubernetes/helm/metrics-server.sh +++ b/magnum/drivers/common/templates/kubernetes/helm/metrics-server.sh @@ -10,7 +10,8 @@ printf "Starting to run ${step}\n" ############################################################################### CHART_NAME="metrics-server" -if [ "$(echo ${METRICS_SERVER_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ]; then +if [ "$(echo ${METRICS_SERVER_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ] && \ + [[ ( $(echo ${CERN_CHART_VERSION} | cut -d. -f2) -lt 9 ) ]]; then HELM_MODULE_CONFIG_FILE="/srv/magnum/kubernetes/helm/${CHART_NAME}.yaml" [ -f ${HELM_MODULE_CONFIG_FILE} ] || { -- GitLab