From 7f6fc7389ea58fe6dbff187db69422f9de16b4fe Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Mon, 12 Jul 2021 14:19:31 +0200 Subject: [PATCH] [cern] Add cern_chart_values label cern_chart_values is a bsae64 --wrap=0 encoded string that stores the helm values of the cern metachart. https://gitlab.cern.ch/helm/releases/cern-magnum/-/blob/f60ddc94c7264d3968b07f62048a17d8cbbfdeea/values.yaml If the values exist labels like monitoring_enabled, logging_producer will have no effect since they should be defined in the values file. The values will be decoded, indented and writen in the ConfigMap of the install-cern-chart job. If cern_chart_values is "" (the default value), the cern_chart will be installed with the values already in helm/cern-chart.sh. The values can contain bash variables in yaml which will be replaced for backwwards compatbility. eg in: nvidia-gpu: enabled: ${NVIDIA_GPU_ENABLED} ${NVIDIA_GPU_ENABLED} will be replaced by the value of the nvidia_gpu_enabled label. Change-Id: I93319d2a42fc52fe3d9966e04008eac34eb7e927 Signed-off-by: Spyros Trigazis --- dockerfiles/heat-container-agent/Dockerfile | 1 + .../fragments/write-heat-params-master.sh | 1 + .../templates/kubernetes/helm/cern-chart.sh | 69 ++++++++++++------- .../drivers/heat/k8s_fedora_template_def.py | 2 +- .../templates/kubecluster.yaml | 6 ++ .../templates/kubemaster.yaml | 5 ++ 6 files changed, 59 insertions(+), 25 deletions(-) diff --git a/dockerfiles/heat-container-agent/Dockerfile b/dockerfiles/heat-container-agent/Dockerfile index 599a0e03b..b48696f63 100644 --- a/dockerfiles/heat-container-agent/Dockerfile +++ b/dockerfiles/heat-container-agent/Dockerfile @@ -15,6 +15,7 @@ RUN dnf -y --setopt=tsflags=nodocs --nogpgcheck install \ bash \ findutils \ gcc \ + gettext \ kubernetes-client \ libffi-devel \ openssh-clients \ diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh index 083554e30..4f6e5ae07 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh @@ -156,6 +156,7 @@ NVIDIA_GPU_ENABLED="$NVIDIA_GPU_ENABLED" NVIDIA_GPU_TAG="$NVIDIA_GPU_TAG" CERN_CHART_ENABLED="$CERN_CHART_ENABLED" CERN_CHART_VERSION="$CERN_CHART_VERSION" +CERN_CHART_VALUES="$CERN_CHART_VALUES" OIDC_ENABLED="$OIDC_ENABLED" OIDC_CLIENT_ID="$OIDC_CLIENT_ID" OIDC_ISSUER_URL="$OIDC_ISSUER_URL" diff --git a/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh b/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh index da88d4e4f..0f958aca3 100644 --- a/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh +++ b/magnum/drivers/common/templates/kubernetes/helm/cern-chart.sh @@ -15,6 +15,8 @@ if [ "$(echo ${MONITORING_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ] && else CERN_CENTRAL_MONITORING="false" fi +# CERN_CENTRAL_MONITORING is not a magnum value, need to write it in heat-params +echo "CERN_CENTRAL_MONITORING=\"${CERN_CENTRAL_MONITORING}\"" >> /etc/sysconfig/heat-params if [ "$(echo ${LOGGING_INSTALLER} | tr '[:upper:]' '[:lower:]')" = "helm" ] && \ [ ! -z "${LOGGING_PRODUCER}" ]; then @@ -22,6 +24,8 @@ if [ "$(echo ${LOGGING_INSTALLER} | tr '[:upper:]' '[:lower:]')" = "helm" ] && \ else CERN_CENTRAL_LOGGING="false" fi +# CERN_CENTRAL_LOGGING is not a magnum value, need to write it in heat-params +echo "CERN_CENTRAL_LOGGING=\"${CERN_CENTRAL_LOGGING}\"" >> /etc/sysconfig/heat-params NVIDIA_GPU_VALUES="" if [ "$(echo ${NVIDIA_GPU_TAG})" != "" ]; then @@ -38,6 +42,46 @@ fi ### Configuration ############################################################################### CHART_NAME="cern-magnum" +_cern_chart_values_file="/srv/magnum/kubernetes/cern_chart_values.yaml" +if [ "$(echo ${CERN_CHART_VALUES})" != "" ]; then + # decode base64 and indent by 4 spaces to write the ConfigMap below + echo ${CERN_CHART_VALUES} | base64 -d | sed 's/^/ /g'> ${_cern_chart_values_file} + # backwards compatible with labels happens later with envsubst + # envsubst does not work with shell variables, it works only with environment variables. + while read line + do + heat_param_name=$(echo $line | cut -d"=" -f1) + heat_param_value=$(echo $line | cut -d"=" -f2 | sed 's/"//g') + sed -i "s#\${${heat_param_name}}#${heat_param_value}#g" ${_cern_chart_values_file} + done < /etc/sysconfig/heat-params +else + cat < ${_cern_chart_values_file} + eosxd: + enabled: ${EOS_ENABLED} + nvidia-gpu: + enabled: ${NVIDIA_GPU_ENABLED} +${NVIDIA_GPU_VALUES} + fluentd: + enabled: ${CERN_CENTRAL_LOGGING} + output: + producer: ${LOGGING_PRODUCER} + endpoint: ${LOGGING_HTTP_DESTINATION} + includeInternal: ${LOGGING_INCLUDE_INTERNAL} + containerRuntime: ${CONTAINER_RUNTIME} + landb-sync: + enabled: ${LANDB_SYNC_ENABLED} + prometheus-cern: + enabled: ${CERN_CENTRAL_MONITORING} + ceph-csi-cephfs: + enabled: ${CEPHFS_CSI_ENABLED} + openstack-cinder-csi: + enabled: ${CINDER_CSI_ENABLED} + openstack-manila-csi: + enabled: ${MANILA_CSI_ENABLED} + base: + enabled: ${CERN_ENABLED} +EOF +fi if [ "$(echo ${CERN_CHART_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ]; then HELM_MODULE_CONFIG_FILE="/srv/magnum/kubernetes/helm/${CHART_NAME}.yaml" @@ -71,30 +115,7 @@ data: fi install-${CHART_NAME}-values.yaml: | - eosxd: - enabled: ${EOS_ENABLED} - nvidia-gpu: - enabled: ${NVIDIA_GPU_ENABLED} -${NVIDIA_GPU_VALUES} - fluentd: - enabled: ${CERN_CENTRAL_LOGGING} - output: - producer: ${LOGGING_PRODUCER} - endpoint: ${LOGGING_HTTP_DESTINATION} - includeInternal: ${LOGGING_INCLUDE_INTERNAL} - containerRuntime: ${CONTAINER_RUNTIME} - landb-sync: - enabled: ${LANDB_SYNC_ENABLED} - prometheus-cern: - enabled: ${CERN_CENTRAL_MONITORING} - ceph-csi-cephfs: - enabled: ${CEPHFS_CSI_ENABLED} - openstack-cinder-csi: - enabled: ${CINDER_CSI_ENABLED} - openstack-manila-csi: - enabled: ${MANILA_CSI_ENABLED} - base: - enabled: ${CERN_ENABLED} +$(cat ${_cern_chart_values_file}) --- diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index 04de825f5..bd88b78da 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -131,7 +131,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): 'nvidia_gpu_tag', 'cern_chart_enabled', 'cern_chart_version', 'oidc_issuer_url', 'oidc_username_claim', 'oidc_groups_claim', 'oidc_username_prefix', 'oidc_groups_prefix', 'oidc_client_id', - 'oidc_enabled', 'ignition_version'] + 'oidc_enabled', 'ignition_version', 'cern_chart_values'] labels = self._get_relevant_labels(cluster, kwargs) diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index 575d121d1..80d9d9b14 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -1101,6 +1101,11 @@ parameters: description: The ignition version for the FCOS server bootstrap default: 3.0.0 + cern_chart_values: + type: string + description: base64 encoded helm values for the cern meta chart + default: "" + resources: network: @@ -1377,6 +1382,7 @@ resources: oidc_groups_prefix: {get_param: oidc_groups_prefix} oidc_username: {get_param: oidc_username} ignition_version: {get_param: ignition_version} + cern_chart_values: {get_param: cern_chart_values} kube_cluster_config: condition: create_cluster_resources diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index d5afbf25a..587678ec1 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -823,6 +823,10 @@ parameters: type: string description: The ignition version for the FCOS server bootstrap + cern_chart_values: + type: string + description: base64 encoded helm values for the cern meta chart + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -1026,6 +1030,7 @@ resources: "$NVIDIA_GPU_TAG": {get_param: nvidia_gpu_tag} "$CERN_CHART_ENABLED": {get_param: cern_chart_enabled} "$CERN_CHART_VERSION": {get_param: cern_chart_version} + "$CERN_CHART_VALUES": {get_param: cern_chart_values} "$OIDC_ENABLED": {get_param: oidc_enabled} "$OIDC_CLIENT_ID": {get_param: oidc_client_id} "$OIDC_ISSUER_URL": {get_param: oidc_issuer_url} -- GitLab