diff --git a/bootstrap/thanos/Chart.yaml b/bootstrap/thanos/Chart.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d6311d4e56fa450926ffc5067fbeab15f745d00a
--- /dev/null
+++ b/bootstrap/thanos/Chart.yaml
@@ -0,0 +1,10 @@
+apiVersion: v2
+name: thanos
+description: A Helm chart for Thanos
+type: application
+version: 0.1.0
+appVersion: "1.16.0"
+dependencies:
+- name: thanos
+ repository: https://charts.bitnami.com/bitnami
+ version: 12.4.3
diff --git a/bootstrap/thanos/charts/thanos-12.4.3.tgz b/bootstrap/thanos/charts/thanos-12.4.3.tgz
new file mode 100644
index 0000000000000000000000000000000000000000..60de7077f37e089f6de1a7791589be72e5f03e36
Binary files /dev/null and b/bootstrap/thanos/charts/thanos-12.4.3.tgz differ
diff --git a/bootstrap/thanos/templates/thanos-service-discovery-cm.yaml b/bootstrap/thanos/templates/thanos-service-discovery-cm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..874762ea956fc04b54ec20399b2cebf902c8a1c6
--- /dev/null
+++ b/bootstrap/thanos/templates/thanos-service-discovery-cm.yaml
@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: thanos-service-discovery
+ namespace: {{ .Release.Namespace }}
+data:
+ service-discovery-script.sh: |
+ #!/bin/bash
+
+ # list argocd clusters by listing secrets with argocd.argoproj.io/secret-type=cluster label
+ CLUSTERS=$(kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster --no-headers -o custom-columns=":metadata.name")
+ # for every cluster save the cluster_url to file
+ echo "" > clusters
+ for cluster in $CLUSTERS;
+ do kubectl get secret -n argocd $cluster -o jsonpath="{.data['server']}" | echo $(base64 -d) >> clusters;
+ done
+
+ # for every cluster_url create a kubeconfig, connect to cluster, and check if it has and ingress with label thanos-discovery="1"
+ # save ingress host to file
+ for cluster_url in $(cat clusters); do
+ argocd -n argocd admin cluster kubeconfig $cluster_url config;
+ kubectl --kubeconfig ./config get ingress -n kube-system -l thanos-discovery="1" --no-headers -o custom-columns=":spec.rules[].host" >> ingress_hosts;
+ done
+
+ # convert file with ingress hosts to yaml as defined in thanos docs https://thanos.io/tip/thanos/service-discovery.md/#file-service-discovery
+ # Example of valid file: - targets: [host1, host2, host3, ]
+ echo -n '- targets: [' > servicediscovery.yml
+
+ # get the in-cluster thanos service (the one with label role=thanos-discovery), and add it to the servicediscovery.yml file
+ SERVICE=$(kubectl get service -n kube-system -l role=thanos-discovery --no-headers -o custom-columns=":metadata.name")
+ PORT=10901
+
+ if [ -z "$SERVICE" ]
+ then
+ echo "no thanos discovery service was found"
+ else
+ echo -n "$SERVICE.kube-system.svc.cluster.local:$PORT, " >> servicediscovery.yml
+ fi
+
+ for host in $(cat ingress_hosts); do echo -n "$host:80, " >> servicediscovery.yml; done;
+ echo ']' >> servicediscovery.yml
+
+ # clean up
+ rm config clusters ingress_hosts
+
+ # update cm with new service discovery file, wait thanos query to pick it up
+ kubectl create configmap thanos-query-targets -n thanos --from-file servicediscovery.yml -o yaml --dry-run=client | kubectl apply -f -
diff --git a/bootstrap/thanos/templates/thanos-service-discovery-cronjon.yaml b/bootstrap/thanos/templates/thanos-service-discovery-cronjon.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e417f5743223cc76abd6989cc271aedb5db47156
--- /dev/null
+++ b/bootstrap/thanos/templates/thanos-service-discovery-cronjon.yaml
@@ -0,0 +1,127 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: thanos-update-query-endpoints
+ namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: thanos-update-query-endpoints
+ namespace: {{ .Release.Namespace }}
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ verbs:
+ - create
+ - get
+ - list
+ - watch
+ - update
+ - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: thanos-update-query-endpoints
+ namespace: {{ .Release.Namespace }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: thanos-update-query-endpoints
+subjects:
+- namespace: {{ .Release.Namespace }}
+ kind: ServiceAccount
+ name: thanos-update-query-endpoints
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: argocd-get-clusters
+ namespace: {{ .Values.serviceDiscovery.argocdNamespace }}
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - secrets
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: argocd-get-clusters
+ namespace: {{ .Values.serviceDiscovery.argocdNamespace }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: argocd-get-clusters
+subjects:
+- namespace: {{ .Release.Namespace }}
+ kind: ServiceAccount
+ name: thanos-update-query-endpoints
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: thanos-discovery-get-service
+ namespace: kube-system
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - services
+ verbs:
+ - get
+ - list
+ - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: thanos-discovery-get-service
+ namespace: kube-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: thanos-discovery-get-service
+subjects:
+- namespace: {{ .Release.Namespace }}
+ kind: ServiceAccount
+ name: thanos-update-query-endpoints
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: update-thanos-query-endpoints
+ namespace: {{ .Release.Namespace }}
+spec:
+ schedule: "{{ .Values.serviceDiscovery.cronJobSchedule }}"
+ jobTemplate:
+ spec:
+ template:
+ spec:
+ containers:
+ - name: update
+ image: registry.cern.ch/kubernetes/ops:0.3.0
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/sh
+ - -c
+ - /home/service-discovery-script.sh
+ volumeMounts:
+ - name: thanos-service-discovery
+ mountPath: /home/service-discovery-script.sh
+ subPath: service-discovery-script.sh
+ volumes:
+ - name: thanos-service-discovery
+ configMap:
+ defaultMode: 0777
+ name: thanos-service-discovery
+ restartPolicy: Never
+ serviceAccountName: thanos-update-query-endpoints
diff --git a/bootstrap/thanos/values.yaml b/bootstrap/thanos/values.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..53b848738000b076c3c0aa6b975d3fd9bf613ba0
--- /dev/null
+++ b/bootstrap/thanos/values.yaml
@@ -0,0 +1,34 @@
+thanos:
+ objstoreConfig: |-
+ type: s3
+ config:
+ bucket: argocd-101
+ endpoint: s3.cern.ch
+ access_key: "<path:kv/data/kubernetes/argocd-101/s3#access_key>"
+ secret_key: "<path:kv/data/kubernetes/argocd-101/s3#secret_key>"
+
+ query:
+ enabled: true
+ existingSDConfigmap: "thanos-query-targets"
+
+ storegateway:
+ enabled: true
+ persistence:
+ storageClass: "geneva-cephfs-testing"
+ size: 5Gi
+ containerSecurityContext:
+ runAsUser: 1001
+ runAsGroup: 1001
+
+ compactor:
+ enabled: true
+ persistence:
+ storageClass: "geneva-cephfs-testing"
+ size: 5Gi
+ containerSecurityContext:
+ runAsUser: 1001
+ runAsGroup: 1001
+
+serviceDiscovery:
+ argocdNamespace: "argocd"
+ cronJobSchedule: "*/10 * * * *"
diff --git a/main.yaml b/main.yaml
index c3b31ebd9c50e5663f15052fa29896e4986a9ba0..b7f92af0d0c3ffbf48bce38867876023fd4cbf73 100644
--- a/main.yaml
+++ b/main.yaml
@@ -28,6 +28,8 @@ spec:
- path: bootstrap/*
- path: bootstrap/vault
exclude: true
+ - path: bootstrap/thanos
+ exclude: true
template:
metadata:
name: '{{path.basename}}'
@@ -73,4 +75,25 @@ spec:
automated:
prune: true
allowEmpty: true
-
+---
+# I have to think how to deploy bootstrap/* to argocd namespace, but bootstrap/thanos to thanos namespace
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+ name: thanos
+ namespace: argocd
+spec:
+ destination:
+ namespace: thanos
+ server: https://kubernetes.default.svc
+ project: bootstrap
+ source:
+ path: bootstrap/thanos
+ repoURL: https://gitlab.cern.ch/kubernetes/automation/101/argocd.git
+ targetRevision: master
+ syncPolicy:
+ automated:
+ prune: true
+ allowEmpty: true
+ syncOptions:
+ - CreateNamespace=true