sps: First commit

sps/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

sps/Chart.yaml

+apiVersion: v2
+#
+name: sps
+type: application
+version: 0.0.1
+appVersion: 4.8.78
+#
+description: An XRootD Standard Proxy Service chart
+icon: https://xrootd.slac.stanford.edu/doc/dev51/pss_config_files/image001.gif
+dependencies:
+- name: utils
+  version: 0.1.4
+  repository: "https://registry.cern.ch/chartrepo/eos"

sps/README.md

+# SPS chart
+
+### Helm Chart for the deployment of the XRootD Standard Proxy Service.
+
+-----
+
+
+### What for
+This chart provides the ability to run an XRootD Standard Proxy Service to expose an EOS outside the k8s cluster.
+- Pros: It removes the need for exposing the MGM and all the FTS to the outside of the k8s virtual networking, avoiding multihoming issues and requirement for port / hostNetwork availability on the hosts where EOS pods are deployed.
+- Cons: It is a single point of failure and a bottleneck in terms of throughput. For deployments with specific requirements for throughput and availability, please consider deploying a [proxy cluster](https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581499).
+
+For more details on XRooD Proxy, check the [upstream documentation](https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581497).
+
+
+### Notes on Ingress
+The Standard Proxy Service runs on L4 TCP (xrootd protocol), while the ingress is a L7 (http) facility.
+To expose SPS to the outside world on TCP, it is required to patch the ingress controller, which is a cluster-wide configuration change.
+Patching the ingress controller is not implemented in this Helm chart, but rather required manual configuration.
+
+Below, we provide the recipe to configure (by patching) the nginx ingress controller,
+which supports exposing an internal service on an external port via `--tcp-services-configmap`
+([upstream documentation](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md)).
+
+The example considers to patch the nginx ingress controller:
+- living in namespace 'ingress-nginx'
+- to expose a service 'sps' in namespace 'default' on port '1094' protocol TCP
+- on external port '1094'
+
+1. Patch the ingress-controller configMap:
+```sh
+namespace="default"
+service_name="sps"
+host_port="1094"
+container_port="1094"
+
+target="$namespace/$service_name:$container_port"
+json_patch=$(jq --null-input --compact-output \
+  --arg host_port "$host_port" \
+  --arg target "$target" \
+  '{"data":{($host_port):($target)}}'
+  )
+
+kubectl -n ingress-nginx patch configmap tcp-services --patch=$json_patch
+```
+
+2. Patch the ingress-controller deployment
+```sh
+host_port="1094"
+container_port="1094"
+
+json_patch=$(jq --null-input --compact-output \
+  --arg hp "$host_port" \
+  --arg cp "$container_port" \
+  '{"spec":{"template":{"spec":{"containers":[{"name":"controller","ports":[{"containerPort":($cp | tonumber),"hostPort":($hp | tonumber)}]}]}}}}' \
+  )
+
+kubectl -n ingress-nginx patch deployment ingress-nginx-controller --patch=$json_patch
+```
+

sps/templates/_helpers.tpl

+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "sps.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "sps.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "sps.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "sps.labels" -}}
+helm.sh/chart: {{ include "sps.chart" . }}
+{{ include "sps.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "sps.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "sps.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "sps.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "sps.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Namespace definition
+*/}}
+{{- define "sps.namespace" -}}
+{{- $namespace := default "default" .Values.namespace -}}
+{{- if .Values.global -}}
+    {{ dig "namespace" $namespace .Values.global }}
+{{- else -}}
+    {{ $namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+SPS network ports definition
+  - xrootd sps port (defaults to 1095)
+
+All the ports can be set according to (example for xrootd sps):
+  - Global value '.Values.global.ports.xrootd_sps' (has the highest priority)
+  - Local value '.Values.ports.xrootd_sps' (has lower priority)
+  - Default value (shown above for each port)
+*/}}
+{{- define "sps.service.port.xrootd_sps" -}}
+{{- $spsDefault := "1094" -}}
+{{- $spsLocal := "" -}}
+{{- $spsGlobal := "" -}}
+{{- if .Values.ports -}}
+  {{ $spsLocal = dig "xrootd_sps" "" .Values.ports -}}
+{{- end }}
+{{- if .Values.global -}}
+  {{ $spsGlobal = dig "ports" "xrootd_sps" "" .Values.global -}}
+{{- end }}
+{{- coalesce $spsGlobal $spsLocal $spsDefault }}
+{{- end }}
+
+{{/*
+MGM's FQDN definition.
+  Define the value for environment variable EOS_MGM_URL.
+  - It is mainly used by the initContainer to make sure the MGM is online before starting the proxy.
+  - It is also set in the main proxy container for convenience while debugging
+
+Returns:
+  - The FQDN provided by utils.mgm_fqdn, if eosMgmUrlAuto is set;
+  - The value manually defined in eosMgmUrl otherwise.
+
+** This is copy-pasted from the fusex chart **
+
+*/}}
+{{- define "sps.eos_mgm_url" -}}
+{{- if .Values.checkMgmOnline.eosMgmUrlAuto -}}
+{{- printf "%s" ( include "utils.mgm_fqdn" . ) -}}
+{{- else }}
+{{- printf "%s" .Values.checkMgmOnline.eosMgmUrl -}}
+{{- end }}
+{{- end }}
+
+
+{{/*
+Liveness Probe definition
+*/}}
+{{- define "sps.livenessProbe" -}}
+{{- $livenessEnabled := "true" -}}
+{{- if .Values.probes -}}
+  {{- $livenessEnabled = dig "liveness" $livenessEnabled .Values.probes -}}
+{{- end }}
+{{- if .Values.global -}}
+  {{- $livenessEnabled = dig "probes" "sps_liveness" $livenessEnabled .Values.global }}
+{{- end }}
+{{- if $livenessEnabled -}}
+livenessProbe:
+  tcpSocket:
+    port: 1094
+  initialDelaySeconds: 5
+  periodSeconds: 10
+  successThreshold: 1
+  failureThreshold: 3
+{{- end }}
+{{- end }}
+
+{{/*
+Name of the secret storing the SSS keytab for sps
+Returns:
+  - "<release_fullname>-sps.sss-keytab" when .Values.sps.keytab.value or .file are passed
+  - the name of the secret passed as .Values.sps.keytab.secret
+  - "<release_fullname>-sps.sss-keytab" by default.
+If the secret does not exist, the pod will hang due to the missing mount.
+
+** This is copy-pasted from the fusex chart **
+
+*/}}
+{{- define "sps.sssKeytabSecretName" -}}
+{{- if or .Values.sps.keytab.value .Values.sps.keytab.file -}}
+{{- printf "%s%s" (include "sps.fullname" .) "-sps.sss-keytab" }}
+{{- else if .Values.sps.keytab.secret }}
+{{- printf "%s" .Values.sps.keytab.secret }}
+{{- else }}
+{{- printf "%s%s" (include "sps.fullname" .) "-sps.sss-keytab" }}
+{{- end }}
+{{- end }}
+

sps/templates/configmap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "sps.fullname" . }}-cfgmap-xrd-cf-sps
+  labels:
+    {{- include "sps.labels" . | nindent 4 }}
+data:
+  xrd.cf.sps: |
+    ###########################################################
+    # See documentation at
+    # https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581497
+    #
+
+    # We need to load the proxy plugin for this to actually work
+    ofs.osslib libXrdPss.so
+
+    # Specify that we are a direct mode proxy fronting the EOS cluster at the specified URL (originHost:originPort)
+    pss.origin {{ .Values.sps.originHost }}:{{ default 1094 .Values.sps.originPort }}
+    
+    # The export allows access to any path via proxy as the origin host will enforce its own exports
+    all.export {{ default "/" .Values.sps.exportPath }}
+    
+    # To support checksum calculation you must also indicate that the checksum manager is a proxy as well
+    {{- if .Values.sps.enableChecksum }}
+    ofs.ckslib * libXrdPss.so
+    {{- else }}
+    #   checksum disabled.
+    #   Enable at .Values.sps.enableChecksum if needed
+    {{- end }}
+    
+    # The sec.protocol directive can restrict access to authenticated clients.
+    sec.protocol sss -c /etc/eos.keytab -s /etc/eos.keytab
+    
+    ## # The xrd.allow directive can restrict access by domain when you use the asterisk notation
+    ## xrd.allow host {{ default "*" .Values.sps.allowedHosts }}
+
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "sps.fullname" . }}-cfgmap-sps-waitformgm
+  labels:
+    {{- include "sps.labels" . | nindent 4 }}
+data:
+  sps_waitformgm.sh: |
+    #!/bin/bash
+    set -x
+    alias eos='eos -r 0 0'
+    shopt -s expand_aliases
+
+    init_probe() {
+      local cmd=$@
+      local max_wait=180
+      local sleep=5
+      start_time=$(date +%s)
+      rc=-1
+      while [ $rc -ne 0 ];
+      do
+        timeout --preserve-status $sleep $cmd >/dev/null 2>&1
+        rc=$?
+
+        # Bail out after max_wait
+        tot_wait=$(($(date +%s)-start_time))
+        echo "        $tot_wait seconds... (timeout at $max_wait)"
+        if [ $tot_wait -ge $max_wait ]; then
+          echo "ERROR: cmd \`$cmd\` failed after $tot_wait secs. Giving up."
+          exit 1
+        fi
+        sleep $sleep
+      done
+    }
+
+    # Wait for the MGM to be online before registering the node and the filesystem
+    echo "INFO: Checking the MGM is online..."
+    echo "INFO: EOS_MGM_URL="$EOS_MGM_URL
+    init_probe eos ns
+    echo "INFO: MGM is online."

sps/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "sps.fullname" . }}
+  labels:
+    {{- include "sps.labels" . | nindent 4 }}
+spec:
+  replicas: 1   # One replica for now. More repliacas, would require clustering
+                # https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581499
+  selector:
+    matchLabels:
+      {{- include "sps.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "sps.selectorLabels" . | nindent 8 }}
+        {{- toYaml .Values.customLabels | nindent 8 }}
+    spec:
+      hostNetwork: {{ default false .Values.hostNetwork }}
+      dnsPolicy: {{ default "ClusterFirst" .Values.dnsPolicy }}
+      {{- if .Values.podAssignment.enableNodeSelector }}
+      nodeSelector:
+        {{- toYaml .Values.customLabels | nindent 8 }}
+      {{- end }}
+      initContainers:
+        # Required until https://github.com/kubernetes/kubernetes/issues/81089 is merged
+        - name: eos-sps-init0-sss-keytab-ownership
+          image: {{ include "utils.image" . }}
+          imagePullPolicy: {{ include "utils.imagePullPolicy" . }}
+          command: ["/bin/bash", "-c"]
+          args: ["cp /root/sss_keytab/input/eos.keytab /root/sss_keytab/output/eos.keytab; chown daemon:daemon /root/sss_keytab/output/eos.keytab; chmod 400 /root/sss_keytab/output/eos.keytab"]
+          volumeMounts:
+            - name: eos-sss-keytab
+              mountPath: /root/sss_keytab/input/eos.keytab
+              subPath: eos.keytab
+            - name: eos-sss-keytab-fixedownership
+              mountPath: /root/sss_keytab/output
+        {{- if .Values.checkMgmOnline.enabled }}
+        - name: eos-sps-init1-waitformgm
+          image: {{ include "utils.image" . }}
+          imagePullPolicy: {{ include "utils.imagePullPolicy" . }}
+          env:
+            - name: EOS_MGM_URL
+              value: root://{{ include "sps.eos_mgm_url" . }}
+          command: ["/bin/bash", "/root/sps_waitformgm.sh"]
+          volumeMounts:
+            - name: sps-cfgmap-sps-waitformgm
+              mountPath: /root/sps_waitformgm.sh
+              subPath: sps_waitformgm.sh
+            - name: eos-sss-keytab-fixedownership
+              mountPath: /etc/eos.keytab
+              subPath: eos.keytab
+        {{- end }}
+      containers:
+        - name: eos-sps
+          image: {{ include "utils.image" . }}
+          imagePullPolicy: {{ include "utils.imagePullPolicy" . }}
+          command: ["/bin/sh", "-c"]
+          args: ["/opt/eos/xrootd/bin/xrootd -n sps -c /etc/xrd.cf.sps -l /var/log/eos/xrdlog.sps -Rdaemon"]
+          env:
+            - name: EOS_MGM_URL
+              value: root://{{ include "sps.eos_mgm_url" . }}
+            {{- with .Values.extraEnv }}
+            {{- include "utils.extraEnv" . | nindent 12 }}
+            {{- end }}
+          securityContext:
+            privileged: false
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+              - SYS_PTRACE
+          {{- include "sps.livenessProbe" . | nindent 10 }}
+          volumeMounts:
+            - name: sps-cfgmap-xrd-cf-sps
+              mountPath: /etc/xrd.cf.sps
+              subPath: xrd.cf.sps
+            - name: eos-sss-keytab-fixedownership
+              mountPath: /etc/eos.keytab
+              subPath: eos.keytab
+            - name: sps-logs
+              mountPath: /var/log/eos
+      volumes:
+        {{- if .Values.checkMgmOnline.enabled }}
+        - name: sps-cfgmap-sps-waitformgm
+          configMap:
+            name: {{ include "sps.fullname" . }}-cfgmap-sps-waitformgm
+            defaultMode: 0755
+        {{- end }}
+        - name: sps-cfgmap-xrd-cf-sps
+          configMap:
+            name: {{ include "sps.fullname" . }}-cfgmap-xrd-cf-sps
+            defaultMode: 0755
+        - name: eos-sss-keytab
+          secret:
+            secretName: {{ include "sps.sssKeytabSecretName" . }}
+            defaultMode: 0400
+        - name: eos-sss-keytab-fixedownership
+          emptyDir: {}
+        - name: sps-logs
+          emptyDir: {}

sps/templates/service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "sps.fullname" . }}
+  labels:
+    {{- include "sps.labels" . | nindent 4 }}
+spec:
+  clusterIP: None
+  ports:
+    - port: {{ include "sps.service.port.xrootd_sps" . }}
+      targetPort: 1094
+      protocol: TCP
+      name: xrootd-sps
+  selector:
+    {{- include "sps.selectorLabels" . | nindent 4 }}

sps/templates/sss_keytab.yaml

+#
+# SSS keytab configmap
+#
+#   Example of keytab:
+#     '0 u:daemon g:daemon n:eos-test+ N:6927582626958016513 c:1612953522 e:0 f:0 k:4d6faa5829d44b32a19c74e2915d94dd86125bfe7dfffb7c2badcb000f9a8327'
+#
+
+#  Reads file at the location given by '.Values.sps.keytab.file' and creates a secret out of it.
+#  If the source file does not exist (or it is empty), no secrets will be created.
+{{- if .Values.sps.keytab.file }}
+{{- $keytab := .Files.Get .Values.sps.keytab.file }}
+{{- if $keytab }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "sps.sssKeytabSecretName" . }}
+  labels:
+    {{- include "sps.labels" . | nindent 4 }}
+data:
+  eos.keytab: |-
+    {{ $keytab | b64enc }}
+{{- end }}
+#  Uses the value variable as keytab source and creates a secret out of it.
+#  If value is not defined (or it is empty), no secrets will be created.
+{{- else if .Values.sps.keytab.value }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "sps.sssKeytabSecretName" . }}
+  labels:
+    {{- include "sps.labels" . | nindent 4 }}
+data:
+  eos.keytab: |-
+    {{ .Values.sps.keytab.value | b64enc }}
+{{- end }}

sps/values.yaml

+global:
+  # Set this to the domain name of your cluster if it does not use the kubernetes default.
+  clusterDomain: "cluster.local"
+
+image:
+  repository: gitlab-registry.cern.ch/dss/eos/eos-all
+  tag: 4.8.78
+  pullPolicy: Always
+
+#
+# Assign sps pods to a node with a specific label
+#   and distribute them on different nodes to avoid single points of failure.
+#
+podAssignment:
+  # If true, requires a node labeled as per customLabels.
+  enableNodeSelector: false
+
+#
+# Custom labels to identify eos sps pods.
+#
+# They are used by node selection, if enabled (see above).
+# Label nodes accordingly to avoid scheduling problems.
+#
+customLabels:
+  service: eos
+  component: eos-sps
+
+#
+# Pod networking
+#   - hostNetwork: Share host network namespace with pod
+#       Docs: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces
+#   - dnsPolicy: Sets the policy for DNS
+#       --> Change to 'ClusterFirstWithHostNet' when 'hostNetwotk: true'
+#       Docs: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy
+#
+hostNetwork: false
+dnsPolicy: ClusterFirst
+
+
+#
+# Check for the MGM to be online before starting the proxy
+#
+#   Parameters:
+#   - enabled: If set to true, an initContainer running `eos ns` will execute before starting the mount
+#   - eosMgmUrlAuto: If set to true, use the FQDN provided by utils.mgm_fqdn.
+#                      This is helpful only when deploying the proxy as dependency of a full eos deployment,
+#                      e.g., via the server chart or ScienceBox. Otherwise, it will not be possible to
+#                      infer the FQDN of the mgm automatically. Use eosMgmUrl to set it manually instead.
+#   - eosMgmUrl: Set the FQDN of the MGM manually. In this case, eosMgmUrlAuto should be set to false.
+#                  Example: "eos-mgm.default.svc.cluster.local" will result in the environment variable
+#                             EOS_MGM_URL="root://eos-mgm-0.eos-mgm.default.svc.cluster.local".
+#
+checkMgmOnline:
+  enabled: false
+  eosMgmUrlAuto: false
+  eosMgmUrl:
+
+#
+# Additional Extra environment varialbes
+#   Several formats of defining variables are supported, including:
+#     ```
+#     extraEnv:
+#       ENV_VAR1: var1
+#       ENV_VAR2: var2
+#     ```
+#   or
+#     ```
+#     extraEnv:
+#     - name: ENV_VAR1
+#       value: "var1"
+#     - name: ENV_VAR2
+#       value: "var2"
+#     ```
+extraEnv: {}
+
+#
+# Enable or disable health probes for sps.
+#   Docs: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+#
+# Liveness Probe:
+#   Checks every 10 seconds whether it is possible to open a TCP socket against port 1094.
+#   The sps container will be restarted after 3 failures.
+#
+# Default: All probes enabled.
+#   This can be overridden with:
+#   - .Values.probes.liveness below
+#   - Global .Values.global.probes.sps_liveness in a parent chart.
+#       Global takes precedence over local values.
+#
+probes:
+  liveness: true
+
+#
+# Service port declaration for sps.
+# These are the ports exposed by the Kubernetes service.
+#
+# Defaults:
+#   - xrootd_sps: 1094
+#
+# Values can be overridden with:
+# - .Values.ports.{xrtood_sps} below
+# - Global .Values.global.ports.<service_name> in a parent chart.
+#     Global takes precedence over local values.
+#
+ports:
+  xrootd_sps:
+
+#
+# Ingress configuration
+#   See README.md
+#
+
+#
+# Configuration for the Standard Proxy Service
+#
+sps:
+  # Hostname and port of the cluster for which this sps is a fronting direct mode proxy.
+  originHost: myeoscluster.mydomain
+  originPort: 1094
+
+  # Define the paths that are publically available.
+  #   This directive is specific to the proxy server (i.e., the proxy will only allow access to the specified paths).
+  #   Data servers can provide read-write access to particular paths while the proxy server can only provide read-only access to some or all such paths.
+  exportPath: "/"
+
+  # Limit hosts that can use the proxy server
+  allowedHosts: "*"
+
+  # Enable checksumming on the proxy server
+  enableChecksum: false
+
+  # The keytab to connect to the MGM via SSS
+  #   Options:
+  #     - secret: Use an existing secret (containing the eos keytab) by providing its name
+  #     - value: Provide the full keytab as a string