Reshuffle apps and srv templates

infra/addons/kube-prometheus-stack/Chart.yaml → addon/thanos/Chart.yaml

 apiVersion: v2
-name: kube-prometheus-stack
+name: mon
 description: A Helm chart for Kubernetes
 type: application
 version: 0.1.0

infra/addons/kube-prometheus-stack/templates/ingress-prometheus.yaml → addon/thanos/templates/ingress-prometheus.yaml

-{{- if not (eq .Values.clusterName "in-cluster") }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -19,4 +18,3 @@ spec:
               number: 9090
         path: /
         pathType: Prefix
-{{- end }}

infra/addons/kube-prometheus-stack/templates/ingress-thanos.yaml → addon/thanos/templates/ingress-thanos.yaml

-{{- if not (eq .Values.clusterName "in-cluster") }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -20,4 +19,3 @@ spec:
               number: 10901
         path: /
         pathType: Prefix
-{{- end }}

services/kube-prometheus-stack-addons.yaml → app/addon-thanos.yaml

-apiVersion: argoproj.io/v1alpha1
-kind: AppProject
-metadata:
-  name: addons
-  namespace: argocd
-spec:
-  destinations:
-    - namespace: '*'
-      server: '*'
-  sourceRepos:
-    - '*'
-  clusterResourceWhitelist:
-  - group: '*'
-    kind: '*'
-
 ---
-
 apiVersion: argoproj.io/v1alpha1
 kind: ApplicationSet
 metadata:
-  name: kube-prometheus-stack
+  name: addon-thanos
   namespace: argocd
 spec:
   generators:
   - clusters:
       selector:
         matchLabels:
-          kube-prometheus-stack: "true"
+          mon: "true"
   template:
     metadata:
-      name: 'kube-prometheus-stack-{{name}}'
+      name: '{{name}}-addon-thanos'
+      labels:
+        area: "monitoring"
+        type: "addon"
     spec:
-      project: addons
+      project: default
       source:
         repoURL: https://gitlab.cern.ch/kubernetes/automation/101/argocd.git
         targetRevision: master
-        path: infra/addons/kube-prometheus-stack/
+        path: addon/thanos
         plugin:
           env:
+          # this is required to pass the cluster name to the target cluster.
+          # used to set the ingress name (dns) for thanos and prometheus, for example.
           - name: HELM_ARGS
             value: --set clusterName={{name}}
       destination:
-        name: '{{name}}'
-        namespace: 'kube-system'
+        server: '{{server}}'
+        namespace: kube-system
       syncPolicy:
         automated:
+          allowEmpty: true
           prune: true
           selfHeal: true
         syncOptions:
           - CreateNamespace=true
+            #- ServerSideApply=true # needed due to: https://www.arthurkoziel.com/fixing-argocd-crd-too-long-error/

app/srv-argocd.yaml

+#---
+#apiVersion: argoproj.io/v1alpha1
+#kind: ApplicationSet
+#metadata:
+#  name: argocd
+#  namespace: argocd
+#spec:
+#  generators:
+#  - clusters:
+#      selector:
+#        matchLabels:
+#          central: "true"
+#  template:
+#    metadata:
+#      name: '{{name}}-argocd'
+#    spec:
+#      project: default
+#      source:
+#        repoURL: https://gitlab.cern.ch/kubernetes/automation/101/argocd.git
+#        targetRevision: master
+#        path: srv/argocd
+#        plugin: {}
+#      destination:
+#        server: '{{server}}'
+#        namespace: argocd
+#      syncPolicy:
+#        automated:
+#          allowEmpty: true
+#          prune: true
+#          selfHeal: true
+#        syncOptions:
+#          - CreateNamespace=true

app/srv-ml-app.yaml

+---
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: ml-app
+  namespace: argocd
+spec:
+  generators:
+  - clusters:
+      selector:
+        matchLabels:
+          ml: "true"
+  template:
+    metadata:
+      name: '{{name}}-ml-app'
+      labels:
+        area: "ml"
+        type: "service"
+    spec:
+      project: default
+      source:
+        repoURL: https://gitlab.cern.ch/rbritoda/ml-app.git
+        targetRevision: master
+        path: '.'
+        plugin: {}
+      destination:
+        server: '{{server}}'
+        namespace: ml
+      syncPolicy:
+        automated:
+          allowEmpty: true
+          prune: true
+          selfHeal: true
+        syncOptions:
+          - CreateNamespace=true

services/ml-serving.yaml → app/srv-thanos.yaml

 ---
 apiVersion: argoproj.io/v1alpha1
-kind: AppProject
-metadata:
-  name: ml
-  namespace: argocd
-spec:
-  destinations:
-    - namespace: '*'
-      server: '*'
-  sourceRepos:
-    - '*'
-  clusterResourceWhitelist:
-  - group: '*'
-    kind: '*'
----
-apiVersion: argoproj.io/v1alpha1
 kind: ApplicationSet
 metadata:
-  name: ml-serving
+  name: srv-thanos
   namespace: argocd
 spec:
   generators:
   - clusters:
       selector:
         matchLabels:
-          ml: "true"
+          central: "true"
   template:
     metadata:
-      name: 'ml-serving-{{name}}'
+      name: '{{name}}-thanos'
+      labels:
+        area: "monitoring"
+        type: "service"
     spec:
-      project: "ml"
+      project: default
       source:
         repoURL: https://gitlab.cern.ch/kubernetes/automation/101/argocd.git
         targetRevision: master
-        path: 'apps/ml-serving'
+        path: srv/thanos
+        plugin: {}
       destination:
         server: '{{server}}'
-        namespace: '{{name}}'
+        namespace: thanos
       syncPolicy:
         automated:
-          prune: true
           allowEmpty: true
+          prune: true
+          selfHeal: true
         syncOptions:
           - CreateNamespace=true

app/srv-workflows.yaml

+---
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: workflows
+  namespace: argocd
+spec:
+  generators:
+  - clusters:
+      selector:
+        matchLabels:
+          central: "true"
+  template:
+    metadata:
+      name: '{{name}}-workflows'
+      labels:
+        area: "infra"
+        type: "service"
+    spec:
+      project: default
+      source:
+        repoURL: https://gitlab.cern.ch/kubernetes/automation/101/argocd.git
+        targetRevision: master
+        path: srv/workflows
+        plugin: {}
+      destination:
+        server: '{{server}}'
+        namespace: workflows
+      syncPolicy:
+        automated:
+          allowEmpty: true
+          prune: true
+          selfHeal: true
+        syncOptions:
+          - CreateNamespace=true
+
+

apps/ml-serving/.helmignore

-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

apps/ml-serving/Chart.yaml

-apiVersion: v2
-name: ml-serving
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"

apps/ml-serving/templates/deployment.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: ml-serving
-  labels:
-    app: ml-serving
-spec:
-  replicas: 5
-  selector:
-    matchLabels:
-      app: ml-serving
-  template:
-    metadata:
-      labels:
-        app: ml-serving
-    spec:
-      containers:
-        - name: ml-serving
-          image: registry.cern.ch/docker.io/library/nginx
-          ports:
-            - name: http
-              containerPort: 80
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /
-              port: http
-          readinessProbe:
-            httpGet:
-              path: /
-              port: http

apps/ml-serving/templates/service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: ml-serving
-  labels:
-    app: ml-serving
-spec:
-  ports:
-    - port: 80
-      targetPort: http
-      protocol: TCP
-      name: http
-  selector:
-    app: ml-serving

apps/ml-serving/values.yaml

-# Default values for ml-serving.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-replicaCount: 1
-
-image:
-  repository: nginx
-  pullPolicy: IfNotPresent
-  # Overrides the image tag whose default is the chart appVersion.
-  tag: ""
-
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: true
-  # Annotations to add to the service account
-  annotations: {}
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name: ""
-
-podAnnotations: {}
-
-podSecurityContext: {}
-  # fsGroup: 2000
-
-securityContext: {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
-
-service:
-  type: ClusterIP
-  port: 80
-
-ingress:
-  enabled: false
-  className: ""
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: chart-example.local
-      paths:
-        - path: /
-          pathType: ImplementationSpecific
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
-
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
-
-autoscaling:
-  enabled: false
-  minReplicas: 1
-  maxReplicas: 100
-  targetCPUUtilizationPercentage: 80
-  # targetMemoryUtilizationPercentage: 80
-
-nodeSelector: {}
-
-tolerations: []
-
-affinity: {}

infra/clusters/templates/cluster-rolebinding.yaml → clusters/templates/cluster-rolebinding.yaml

@@ -11,4 +11,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: default
-  namespace: clusters
+  namespace: workflows

clusters/templates/secret-creds.yaml

+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: openstack-creds
+type: Opaque
+stringData:
+  OS_AUTH_URL: "https://keystone.cern.ch/v3"
+  OS_IDENTITY_API_VERSION: "3"
+  OS_PASSWORD: "<path:kv/data/kubernetes/argocd-101/os#password>"
+  OS_PROJECT_DOMAIN_ID: "default"
+  OS_PROJECT_NAME: 'Personal rbritoda'
+  OS_REGION_NAME: ""
+  OS_USER_DOMAIN_ID: "default"
+  OS_USERNAME: "rbritoda"

infra/clusters/templates/workflows-cluster.yaml → clusters/templates/workflows-cluster.yaml

@@ -249,7 +249,7 @@ spec:
           fi
 
           if [ "$nc" = "{{`{{inputs.parameters.node-count}}`}}" ]; then
-            echo "cluster {{`{{inputs.parameters.name}}`}} size matches. expected {{`{{inputs.parameters.node-count}}`}} vs ${nc}. existing."
+            echo "cluster {{`{{inputs.parameters.name}}`}} size matches. expected {{`{{inputs.parameters.node-count}}`}} vs ${nc}. exiting."
             exit 0
           fi
 
@@ -371,7 +371,7 @@ spec:
           fi
 
           if [ "$nc" = "{{`{{inputs.parameters.node-count}}`}}" ]; then
-            echo "nodegroup {{`{{inputs.parameters.name}}`}} in cluster {{`{{inputs.parameters.cluster}}`}} size matches. expected {{`{{inputs.parameters.node-count}}`}} vs ${nc}. existing."
+            echo "nodegroup {{`{{inputs.parameters.name}}`}} in cluster {{`{{inputs.parameters.cluster}}`}} size matches. expected {{`{{inputs.parameters.node-count}}`}} vs ${nc}. exiting."
             exit 0
           fi
 
@@ -431,6 +431,7 @@ spec:
           set -x
           openstack coe cluster config --dir /tmp --force {{`{{inputs.parameters.name}}`}}
 
+          # argo register
           kubectl -n argocd get secret argocd-secret -o jsonpath='{.data.admin\.password}' | base64 -d-
 
           echo "y" | argocd login --username admin --password "$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d)" $(kubectl -n argocd get service argocd-server -o jsonpath='{.spec.clusterIP}')
@@ -438,6 +439,18 @@ spec:
           argocd cluster get {{`{{inputs.parameters.name}}`}}
           argocd cluster add --name {{`{{inputs.parameters.name}}`}} --kubeconfig /tmp/config {{`{{inputs.parameters.labels}}`}} --upsert default
 
+          # thanos
+          export KUBECONFIG=/tmp/config
+          node=$(kubectl get no -o jsonpath='{.items[1].metadata.name}')
+          kubectl label node $node role=ingress
+          openstack server set --property landb-alias="{{`{{inputs.parameters.name}}`}}-thanos-sidecar,{{`{{inputs.parameters.name}}`}}-prometheus" ${node}
+
+      outputs:
+        parameters:
+          - name: config
+            valueFrom:
+              path: /tmp/config
+
     - name: cluster-config
       inputs:
         parameters: