Commit df6c3000 authored by Enrico Bocchi's avatar Enrico Bocchi
Browse files

sps: First commit

parent a7994446
Pipeline #4048284 passed with stage
in 9 seconds
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v2
#
name: sps
type: application
version: 0.0.1
appVersion: 4.8.78
#
description: An XRootD Standard Proxy Service chart
icon: https://xrootd.slac.stanford.edu/doc/dev51/pss_config_files/image001.gif
dependencies:
- name: utils
version: 0.1.4
repository: "https://registry.cern.ch/chartrepo/eos"
# SPS chart
### Helm Chart for the deployment of the XRootD Standard Proxy Service.
-----
### What for
This chart provides the ability to run an XRootD Standard Proxy Service to expose an EOS outside the k8s cluster.
- Pros: It removes the need for exposing the MGM and all the FTS to the outside of the k8s virtual networking, avoiding multihoming issues and requirement for port / hostNetwork availability on the hosts where EOS pods are deployed.
- Cons: It is a single point of failure and a bottleneck in terms of throughput. For deployments with specific requirements for throughput and availability, please consider deploying a [proxy cluster](https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581499).
For more details on XRooD Proxy, check the [upstream documentation](https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581497).
### Notes on Ingress
The Standard Proxy Service runs on L4 TCP (xrootd protocol), while the ingress is a L7 (http) facility.
To expose SPS to the outside world on TCP, it is required to patch the ingress controller, which is a cluster-wide configuration change.
Patching the ingress controller is not implemented in this Helm chart, but rather required manual configuration.
Below, we provide the recipe to configure (by patching) the nginx ingress controller,
which supports exposing an internal service on an external port via `--tcp-services-configmap`
([upstream documentation](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md)).
The example considers to patch the nginx ingress controller:
- living in namespace 'ingress-nginx'
- to expose a service 'sps' in namespace 'default' on port '1094' protocol TCP
- on external port '1094'
1. Patch the ingress-controller configMap:
```sh
namespace="default"
service_name="sps"
host_port="1094"
container_port="1094"
target="$namespace/$service_name:$container_port"
json_patch=$(jq --null-input --compact-output \
--arg host_port "$host_port" \
--arg target "$target" \
'{"data":{($host_port):($target)}}'
)
kubectl -n ingress-nginx patch configmap tcp-services --patch=$json_patch
```
2. Patch the ingress-controller deployment
```sh
host_port="1094"
container_port="1094"
json_patch=$(jq --null-input --compact-output \
--arg hp "$host_port" \
--arg cp "$container_port" \
'{"spec":{"template":{"spec":{"containers":[{"name":"controller","ports":[{"containerPort":($cp | tonumber),"hostPort":($hp | tonumber)}]}]}}}}' \
)
kubectl -n ingress-nginx patch deployment ingress-nginx-controller --patch=$json_patch
```
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "sps.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "sps.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "sps.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Common labels
*/}}
{{- define "sps.labels" -}}
helm.sh/chart: {{ include "sps.chart" . }}
{{ include "sps.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}
{{/*
Selector labels
*/}}
{{- define "sps.selectorLabels" -}}
app.kubernetes.io/name: {{ include "sps.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "sps.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "sps.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{/*
Namespace definition
*/}}
{{- define "sps.namespace" -}}
{{- $namespace := default "default" .Values.namespace -}}
{{- if .Values.global -}}
{{ dig "namespace" $namespace .Values.global }}
{{- else -}}
{{ $namespace }}
{{- end }}
{{- end }}
{{/*
SPS network ports definition
- xrootd sps port (defaults to 1095)
All the ports can be set according to (example for xrootd sps):
- Global value '.Values.global.ports.xrootd_sps' (has the highest priority)
- Local value '.Values.ports.xrootd_sps' (has lower priority)
- Default value (shown above for each port)
*/}}
{{- define "sps.service.port.xrootd_sps" -}}
{{- $spsDefault := "1094" -}}
{{- $spsLocal := "" -}}
{{- $spsGlobal := "" -}}
{{- if .Values.ports -}}
{{ $spsLocal = dig "xrootd_sps" "" .Values.ports -}}
{{- end }}
{{- if .Values.global -}}
{{ $spsGlobal = dig "ports" "xrootd_sps" "" .Values.global -}}
{{- end }}
{{- coalesce $spsGlobal $spsLocal $spsDefault }}
{{- end }}
{{/*
MGM's FQDN definition.
Define the value for environment variable EOS_MGM_URL.
- It is mainly used by the initContainer to make sure the MGM is online before starting the proxy.
- It is also set in the main proxy container for convenience while debugging
Returns:
- The FQDN provided by utils.mgm_fqdn, if eosMgmUrlAuto is set;
- The value manually defined in eosMgmUrl otherwise.
** This is copy-pasted from the fusex chart **
*/}}
{{- define "sps.eos_mgm_url" -}}
{{- if .Values.checkMgmOnline.eosMgmUrlAuto -}}
{{- printf "%s" ( include "utils.mgm_fqdn" . ) -}}
{{- else }}
{{- printf "%s" .Values.checkMgmOnline.eosMgmUrl -}}
{{- end }}
{{- end }}
{{/*
Liveness Probe definition
*/}}
{{- define "sps.livenessProbe" -}}
{{- $livenessEnabled := "true" -}}
{{- if .Values.probes -}}
{{- $livenessEnabled = dig "liveness" $livenessEnabled .Values.probes -}}
{{- end }}
{{- if .Values.global -}}
{{- $livenessEnabled = dig "probes" "sps_liveness" $livenessEnabled .Values.global }}
{{- end }}
{{- if $livenessEnabled -}}
livenessProbe:
tcpSocket:
port: 1094
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
{{- end }}
{{- end }}
{{/*
Name of the secret storing the SSS keytab for sps
Returns:
- "<release_fullname>-sps.sss-keytab" when .Values.sps.keytab.value or .file are passed
- the name of the secret passed as .Values.sps.keytab.secret
- "<release_fullname>-sps.sss-keytab" by default.
If the secret does not exist, the pod will hang due to the missing mount.
** This is copy-pasted from the fusex chart **
*/}}
{{- define "sps.sssKeytabSecretName" -}}
{{- if or .Values.sps.keytab.value .Values.sps.keytab.file -}}
{{- printf "%s%s" (include "sps.fullname" .) "-sps.sss-keytab" }}
{{- else if .Values.sps.keytab.secret }}
{{- printf "%s" .Values.sps.keytab.secret }}
{{- else }}
{{- printf "%s%s" (include "sps.fullname" .) "-sps.sss-keytab" }}
{{- end }}
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "sps.fullname" . }}-cfgmap-xrd-cf-sps
labels:
{{- include "sps.labels" . | nindent 4 }}
data:
xrd.cf.sps: |
###########################################################
# See documentation at
# https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581497
#
# We need to load the proxy plugin for this to actually work
ofs.osslib libXrdPss.so
# Specify that we are a direct mode proxy fronting the EOS cluster at the specified URL (originHost:originPort)
pss.origin {{ .Values.sps.originHost }}:{{ default 1094 .Values.sps.originPort }}
# The export allows access to any path via proxy as the origin host will enforce its own exports
all.export {{ default "/" .Values.sps.exportPath }}
# To support checksum calculation you must also indicate that the checksum manager is a proxy as well
{{- if .Values.sps.enableChecksum }}
ofs.ckslib * libXrdPss.so
{{- else }}
# checksum disabled.
# Enable at .Values.sps.enableChecksum if needed
{{- end }}
# The sec.protocol directive can restrict access to authenticated clients.
sec.protocol sss -c /etc/eos.keytab -s /etc/eos.keytab
## # The xrd.allow directive can restrict access by domain when you use the asterisk notation
## xrd.allow host {{ default "*" .Values.sps.allowedHosts }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "sps.fullname" . }}-cfgmap-sps-waitformgm
labels:
{{- include "sps.labels" . | nindent 4 }}
data:
sps_waitformgm.sh: |
#!/bin/bash
set -x
alias eos='eos -r 0 0'
shopt -s expand_aliases
init_probe() {
local cmd=$@
local max_wait=180
local sleep=5
start_time=$(date +%s)
rc=-1
while [ $rc -ne 0 ];
do
timeout --preserve-status $sleep $cmd >/dev/null 2>&1
rc=$?
# Bail out after max_wait
tot_wait=$(($(date +%s)-start_time))
echo " $tot_wait seconds... (timeout at $max_wait)"
if [ $tot_wait -ge $max_wait ]; then
echo "ERROR: cmd \`$cmd\` failed after $tot_wait secs. Giving up."
exit 1
fi
sleep $sleep
done
}
# Wait for the MGM to be online before registering the node and the filesystem
echo "INFO: Checking the MGM is online..."
echo "INFO: EOS_MGM_URL="$EOS_MGM_URL
init_probe eos ns
echo "INFO: MGM is online."
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "sps.fullname" . }}
labels:
{{- include "sps.labels" . | nindent 4 }}
spec:
replicas: 1 # One replica for now. More repliacas, would require clustering
# https://xrootd.slac.stanford.edu/doc/dev51/pss_config.htm#_Toc50581499
selector:
matchLabels:
{{- include "sps.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
{{- include "sps.selectorLabels" . | nindent 8 }}
{{- toYaml .Values.customLabels | nindent 8 }}
spec:
hostNetwork: {{ default false .Values.hostNetwork }}
dnsPolicy: {{ default "ClusterFirst" .Values.dnsPolicy }}
{{- if .Values.podAssignment.enableNodeSelector }}
nodeSelector:
{{- toYaml .Values.customLabels | nindent 8 }}
{{- end }}
initContainers:
# Required until https://github.com/kubernetes/kubernetes/issues/81089 is merged
- name: eos-sps-init0-sss-keytab-ownership
image: {{ include "utils.image" . }}
imagePullPolicy: {{ include "utils.imagePullPolicy" . }}
command: ["/bin/bash", "-c"]
args: ["cp /root/sss_keytab/input/eos.keytab /root/sss_keytab/output/eos.keytab; chown daemon:daemon /root/sss_keytab/output/eos.keytab; chmod 400 /root/sss_keytab/output/eos.keytab"]
volumeMounts:
- name: eos-sss-keytab
mountPath: /root/sss_keytab/input/eos.keytab
subPath: eos.keytab
- name: eos-sss-keytab-fixedownership
mountPath: /root/sss_keytab/output
{{- if .Values.checkMgmOnline.enabled }}
- name: eos-sps-init1-waitformgm
image: {{ include "utils.image" . }}
imagePullPolicy: {{ include "utils.imagePullPolicy" . }}
env:
- name: EOS_MGM_URL
value: root://{{ include "sps.eos_mgm_url" . }}
command: ["/bin/bash", "/root/sps_waitformgm.sh"]
volumeMounts:
- name: sps-cfgmap-sps-waitformgm
mountPath: /root/sps_waitformgm.sh
subPath: sps_waitformgm.sh
- name: eos-sss-keytab-fixedownership
mountPath: /etc/eos.keytab
subPath: eos.keytab
{{- end }}
containers:
- name: eos-sps
image: {{ include "utils.image" . }}
imagePullPolicy: {{ include "utils.imagePullPolicy" . }}
command: ["/bin/sh", "-c"]
args: ["/opt/eos/xrootd/bin/xrootd -n sps -c /etc/xrd.cf.sps -l /var/log/eos/xrdlog.sps -Rdaemon"]
env:
- name: EOS_MGM_URL
value: root://{{ include "sps.eos_mgm_url" . }}
{{- with .Values.extraEnv }}
{{- include "utils.extraEnv" . | nindent 12 }}
{{- end }}
securityContext:
privileged: false
allowPrivilegeEscalation: false
capabilities:
add:
- SYS_PTRACE
{{- include "sps.livenessProbe" . | nindent 10 }}
volumeMounts:
- name: sps-cfgmap-xrd-cf-sps
mountPath: /etc/xrd.cf.sps
subPath: xrd.cf.sps
- name: eos-sss-keytab-fixedownership
mountPath: /etc/eos.keytab
subPath: eos.keytab
- name: sps-logs
mountPath: /var/log/eos
volumes:
{{- if .Values.checkMgmOnline.enabled }}
- name: sps-cfgmap-sps-waitformgm
configMap:
name: {{ include "sps.fullname" . }}-cfgmap-sps-waitformgm
defaultMode: 0755
{{- end }}
- name: sps-cfgmap-xrd-cf-sps
configMap:
name: {{ include "sps.fullname" . }}-cfgmap-xrd-cf-sps
defaultMode: 0755
- name: eos-sss-keytab
secret:
secretName: {{ include "sps.sssKeytabSecretName" . }}
defaultMode: 0400
- name: eos-sss-keytab-fixedownership
emptyDir: {}
- name: sps-logs
emptyDir: {}
apiVersion: v1
kind: Service
metadata:
name: {{ include "sps.fullname" . }}
labels:
{{- include "sps.labels" . | nindent 4 }}
spec:
clusterIP: None
ports:
- port: {{ include "sps.service.port.xrootd_sps" . }}
targetPort: 1094
protocol: TCP
name: xrootd-sps
selector:
{{- include "sps.selectorLabels" . | nindent 4 }}
#
# SSS keytab configmap
#
# Example of keytab:
# '0 u:daemon g:daemon n:eos-test+ N:6927582626958016513 c:1612953522 e:0 f:0 k:4d6faa5829d44b32a19c74e2915d94dd86125bfe7dfffb7c2badcb000f9a8327'
#
# Reads file at the location given by '.Values.sps.keytab.file' and creates a secret out of it.
# If the source file does not exist (or it is empty), no secrets will be created.
{{- if .Values.sps.keytab.file }}
{{- $keytab := .Files.Get .Values.sps.keytab.file }}
{{- if $keytab }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "sps.sssKeytabSecretName" . }}
labels:
{{- include "sps.labels" . | nindent 4 }}
data:
eos.keytab: |-
{{ $keytab | b64enc }}
{{- end }}
# Uses the value variable as keytab source and creates a secret out of it.
# If value is not defined (or it is empty), no secrets will be created.
{{- else if .Values.sps.keytab.value }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "sps.sssKeytabSecretName" . }}
labels:
{{- include "sps.labels" . | nindent 4 }}
data:
eos.keytab: |-
{{ .Values.sps.keytab.value | b64enc }}
{{- end }}
global:
# Set this to the domain name of your cluster if it does not use the kubernetes default.
clusterDomain: "cluster.local"
image:
repository: gitlab-registry.cern.ch/dss/eos/eos-all
tag: 4.8.78
pullPolicy: Always
#
# Assign sps pods to a node with a specific label
# and distribute them on different nodes to avoid single points of failure.
#
podAssignment:
# If true, requires a node labeled as per customLabels.
enableNodeSelector: false
#
# Custom labels to identify eos sps pods.
#
# They are used by node selection, if enabled (see above).
# Label nodes accordingly to avoid scheduling problems.
#
customLabels:
service: eos
component: eos-sps
#
# Pod networking
# - hostNetwork: Share host network namespace with pod
# Docs: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces
# - dnsPolicy: Sets the policy for DNS
# --> Change to 'ClusterFirstWithHostNet' when 'hostNetwotk: true'
# Docs: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy
#
hostNetwork: false
dnsPolicy: ClusterFirst
#
# Check for the MGM to be online before starting the proxy
#
# Parameters:
# - enabled: If set to true, an initContainer running `eos ns` will execute before starting the mount
# - eosMgmUrlAuto: If set to true, use the FQDN provided by utils.mgm_fqdn.
# This is helpful only when deploying the proxy as dependency of a full eos deployment,
# e.g., via the server chart or ScienceBox. Otherwise, it will not be possible to
# infer the FQDN of the mgm automatically. Use eosMgmUrl to set it manually instead.
# - eosMgmUrl: Set the FQDN of the MGM manually. In this case, eosMgmUrlAuto should be set to false.
# Example: "eos-mgm.default.svc.cluster.local" will result in the environment variable
# EOS_MGM_URL="root://eos-mgm-0.eos-mgm.default.svc.cluster.local".
#
checkMgmOnline:
enabled: false
eosMgmUrlAuto: false
eosMgmUrl:
#
# Additional Extra environment varialbes
# Several formats of defining variables are supported, including:
# ```
# extraEnv:
# ENV_VAR1: var1
# ENV_VAR2: var2
# ```
# or
# ```
# extraEnv:
# - name: ENV_VAR1
# value: "var1"
# - name: ENV_VAR2
# value: "var2"
# ```
extraEnv: {}
#
# Enable or disable health probes for sps.
# Docs: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
#
# Liveness Probe:
# Checks every 10 seconds whether it is possible to open a TCP socket against port 1094.
# The sps container will be restarted after 3 failures.
#
# Default: All probes enabled.
# This can be overridden with:
# - .Values.probes.liveness below
# - Global .Values.global.probes.sps_liveness in a parent chart.
# Global takes precedence over local values.
#
probes:
liveness: true
#
# Service port declaration for sps.
# These are the ports exposed by the Kubernetes service.
#
# Defaults:
# - xrootd_sps: 1094
#
# Values can be overridden with:
# - .Values.ports.{xrtood_sps} below
# - Global .Values.global.ports.<service_name> in a parent chart.
# Global takes precedence over local values.
#
ports:
xrootd_sps:
#
# Ingress configuration
# See README.md
#
#
# Configuration for the Standard Proxy Service
#
sps:
# Hostname and port of the cluster for which this sps is a fronting direct mode proxy.
originHost: myeoscluster.mydomain
originPort: 1094
# Define the paths that are publically available.
# This directive is specific to the proxy server (i.e., the proxy will only allow access to the specified paths).
# Data servers can provide read-write access to particular paths while the proxy server can only provide read-only access to some or all such paths.
exportPath: "/"
# Limit hosts that can use the proxy server
allowedHosts: "*"
# Enable checksumming on the proxy server
enableChecksum: false
# The keytab to connect to the MGM via SSS
# Options:
# - secret: Use an existing secret (containing the eos keytab) by providing its name
# - value: Provide the full keytab as a string