[MONIT-4091] Add fluent-bit agent to send K8s events to OTEL

README.md

@@ -23,6 +23,22 @@ See [getting started](docs/getting_started.md).
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | crds.enabled | bool | `true` | whether to install Prometheus operator's CRDs |
+| events.enabled | bool | `false` | indicates if components to capture Kubernetes events should be enabled or not. |
+| events.fluentbit.enabled | bool | `false` | indicates if fluentbit Kubernetes events component should be installed or not |
+| events.fluentbit.extraVolumeMounts | list | `[]` |  |
+| events.fluentbit.extraVolumes | list | `[]` |  |
+| events.fluentbit.filters | string | partly autogenerated -- see values.yaml | fluentbit filters as a yaml list in a multiline string |
+| events.fluentbit.image.imagePullPolicy | string | `""` | image pull policy for Fluent Bit |
+| events.fluentbit.image.repository | string | `""` | repository to use for Fluent Bit |
+| events.fluentbit.image.tag | string | `""` | tag to use for Fluent Bit |
+| events.fluentbit.inputs | string | partly autogenerated -- see values.yaml | fluentbit inputs as a yaml list in a multiline string |
+| events.fluentbit.luaScripts | object | `{}` | extra Lua scripts for user-provided transformations |
+| events.fluentbit.outputs | string | partly autogenerated -- see values.yaml | fluentbit outputs as a yaml list in a multiline string |
+| events.fluentbit.resources.limits.cpu | string | `"20m"` |  |
+| events.fluentbit.resources.limits.memory | string | `"25Mi"` |  |
+| events.fluentbit.resources.requests.cpu | string | `"5m"` |  |
+| events.fluentbit.resources.requests.memory | string | `"15Mi"` |  |
+| events.fluentbit.service | string | partly autogenerated -- see values.yaml | fluentbit service configuration options in a multiline string |
 | fluentbit.image.imagePullPolicy | string | `"IfNotPresent"` | image pull policy applied to all Fluent Bit instances |
 | fluentbit.image.repository | string | `"registry.cern.ch/monit/cern-it-monitoring-fluent-bit"` | image repository applied to all Fluent Bit instances |
 | fluentbit.image.tag | string | `"3.2.6"` | image tag applied to all Fluent Bit instances |

docs/logs.md

@@ -141,3 +141,19 @@ Be cautious when enabling Fluentbit for large clusters (100+ nodes). The **Kuber
 - Adjust Fluentbit resource requests and limits.
 - Tune the API request rate by modifying the filters.
 - Consider other optimization techniques, such as excluding specific log sources.
+
+## 5. Kubernetes events
+
+This chart also allows sending Kubernetes events as log entries to
+MONIT, however this feature is disabled by default. To enable it,
+please add the following keys to your custom values:
+
+```yaml
+events:
+  enabled: true
+  fluentbit:
+    enabled: true
+```
+
+Events are shipped with `monit_type` set to `kubernetes_events` for
+easy filtering.

templates/fluentbit-events/clusterrole.yaml

+{{- if and .Values.events.enabled .Values.events.fluentbit.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: it-monit-events-collector-fluentbit
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - events
+      - namespaces
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+{{- end -}}

templates/fluentbit-events/clusterrolebinding.yaml

+{{- if and .Values.events.enabled .Values.events.fluentbit.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: it-monit-events-collector-fluentbit
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: it-monit-events-collector-fluentbit
+subjects:
+  - kind: ServiceAccount
+    name: it-monit-events-collector-fluentbit
+    namespace: {{ .Release.Namespace }}
+{{- end -}}

templates/fluentbit-events/configmap-luascripts.yaml

+{{- if and .Values.events.enabled .Values.events.fluentbit.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: it-monit-events-collector-fluentbit-scripts
+data:
+  date_to_ts.lua: |
+    function add_time_from_creation_timestamp(tag, timestamp, record)
+      record['time'] = record['metadata']['creationTimestamp']
+      return 2, timestamp, record
+    end
+  {{- if .Values.events.fluentbit.luaScripts -}}
+  {{ range $key, $value := .Values.events.fluentbit.luaScripts }}
+  {{ $key }}: {{ $value | quote }}
+  {{ end }}
+  {{ end }}
+{{- end -}}

templates/fluentbit-events/configmap.yaml

+{{- if and .Values.events.enabled .Values.events.fluentbit.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: it-monit-events-collector-fluentbit
+  namespace: {{ .Release.Namespace }}
+data:
+  fluent-bit.conf: |
+    {{- (tpl .Values.events.fluentbit.service $)  | nindent 4 }}
+    {{- (tpl .Values.events.fluentbit.inputs $)  | nindent 4 }}
+    {{- (tpl .Values.events.fluentbit.filters $)  | nindent 4 }}
+    {{- (tpl .Values.events.fluentbit.outputs $)  | nindent 4 }}
+{{- end -}}

templates/fluentbit-events/deployment.yaml

+{{- if and .Values.events.enabled .Values.events.fluentbit.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: it-monit-events-collector-fluentbit
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+      matchLabels:
+        name: it-monit-events-collector-fluentbit
+  replicas: 1
+  template:
+    metadata:
+      name: it-monit-events-collector-fluentbit
+      namespace: {{ .Release.Namespace }}
+      labels:
+        name: it-monit-events-collector-fluentbit
+      annotations:
+        checksum/cm: {{ include (print $.Template.BasePath "/fluentbit-events/configmap.yaml") . | sha256sum }}
+        checksum/scripts: {{ include (print $.Template.BasePath "/fluentbit-events/configmap-luascripts.yaml") . | sha256sum }}
+    spec:
+      serviceAccountName: it-monit-events-collector-fluentbit
+      containers:
+        - name: it-monit-events-collector-fluentbit
+          image: {{ default .Values.fluentbit.image.repository .Values.events.fluentbit.image.repository }}:{{ default .Values.fluentbit.image.tag .Values.events.fluentbit.image.tag }}
+          imagePullPolicy: {{ default .Values.fluentbit.image.imagePullPolicy .Values.events.fluentbit.image.imagePullPolicy }}
+          command: [ "/fluent-bit/bin/fluent-bit" ]
+          args:
+            - --workdir=/fluent-bit/etc
+            - --config=/fluent-bit/etc/conf/fluent-bit.conf
+          resources:
+            requests:
+              cpu: {{ .Values.events.fluentbit.resources.requests.cpu }}
+              memory: {{ .Values.events.fluentbit.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.events.fluentbit.resources.limits.cpu }}
+              memory: {{ .Values.events.fluentbit.resources.limits.memory }}
+          volumeMounts:
+            - name: config
+              mountPath: /fluent-bit/etc/conf
+            - name: scripts
+              mountPath: /fluent-bit/etc/scripts
+          {{- if .Values.events.fluentbit.extraVolumeMounts }}
+            {{- toYaml .Values.events.fluentbit.extraVolumeMounts | nindent 12 }}
+          {{- end }}
+      volumes:
+        - name: config
+          configMap:
+            name: it-monit-events-collector-fluentbit
+        - name: scripts
+          configMap:
+            name: it-monit-events-collector-fluentbit-scripts
+      {{- if .Values.events.fluentbit.extraVolumes }}
+        {{- toYaml .Values.events.fluentbit.extraVolumes | nindent 8 }}
+      {{- end }}
+{{- end -}}

templates/fluentbit-events/serviceaccount.yaml

+{{- if and .Values.events.enabled .Values.events.fluentbit.enabled -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: it-monit-events-collector-fluentbit
+  namespace: {{ .Release.Namespace }}
+{{- end -}}

templates/fluentbit-logs/clusterrole.yaml

@@ -12,7 +12,6 @@ rules:
       - pods
       - nodes
       - nodes/proxy
-      - events
     verbs:
       - get
       - list

tests/fluentbit-events/deployment.yaml

+suite: test fluentbit-events deployment
+templates:
+  - fluentbit-events/deployment.yaml
+  - fluentbit-events/configmap.yaml
+  - fluentbit-events/configmap-luascripts.yaml
+tests:
+  - it: should not be deployed by default
+    template: fluentbit-events/deployment.yaml
+    asserts:
+      - containsDocument:
+          kind: Deployment
+          apiVersion: "apps/v1"
+          name: it-monit-events-collector-fluentbit
+        not: true
+  - it: should be deployed if events.enabled and events.fluentbit.enabled is true
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+    asserts:
+      - containsDocument:
+          kind: Deployment
+          apiVersion: "apps/v1"
+          name: it-monit-events-collector-fluentbit
+  - it: should not be deployed if events.enabled is false
+    template: fluentbit-events/deployment.yaml
+    set:
+      events.enabled: false
+    asserts:
+      - containsDocument:
+          kind: Deployment
+          apiVersion: "apps/v1"
+          name: it-monit-events-collector-fluentbit
+        not: true
+  - it: should not be deployed if events.enabled is true and events.fluentbit.enabled is false
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: false
+    asserts:
+      - containsDocument:
+          kind: Deployment
+          apiVersion: "apps/v1"
+          name: it-monit-events-collector-fluentbit
+        not: true
+  - it: should mount configuration and scripts
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: config
+            mountPath: /fluent-bit/etc/conf
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: scripts
+            mountPath: /fluent-bit/etc/scripts
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: config
+            configMap:
+              name: it-monit-events-collector-fluentbit
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: scripts
+            configMap:
+              name: it-monit-events-collector-fluentbit-scripts
+  - it: should allow configuring memory requests and limits
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+      events.fluentbit.resources.requests.memory: "44Mi"
+      events.fluentbit.resources.limits.memory: "88Mi"
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: "44Mi"
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: "88Mi"
+  - it: should configure a service account
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: it-monit-events-collector-fluentbit
+  - it: should configure annotations for all input configuration
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+    asserts:
+      - exists:
+          path: spec.template.metadata.annotations.checksum/cm
+      - exists:
+          path: spec.template.metadata.annotations.checksum/scripts
+  - it: should deploy extra volumes and volume mounts if configured
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+      events.fluentbit.extraVolumeMounts:
+        - name: test
+          mountPath: /test
+      events.fluentbit.extraVolumes:
+        - name: test
+          configMap:
+            name: test-configmap
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: test
+            mountPath: /test
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: test
+            configMap:
+              name: test-configmap
+  - it: should deploy by default with an image served by registry.cern.ch/monit and semver
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+    asserts:
+      - matchRegex:
+          path: spec.template.spec.containers[0].image
+          pattern: ^registry\.cern\.ch/monit/cern-it-monitoring-fluent-bit:\d+\.\d+\.\d+$
+      - equal:
+          path: spec.template.spec.containers[0].imagePullPolicy
+          value: IfNotPresent
+  - it: the image repository and tag can be overridden at component level
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+      events.fluentbit.image.repository: registry.cern/test
+      events.fluentbit.image.tag: 1
+      events.fluentbit.image.imagePullPolicy: Always
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: registry.cern/test:1
+      - equal:
+          path: spec.template.spec.containers[0].imagePullPolicy
+          value: Always
+  - it: the image repository, tag and pull policy can be overridden at global level
+    template: fluentbit-events/deployment.yaml
+    set:
+      tenant.name: test
+      tenant.password: test
+      kubernetes.clusterName: test
+      events.enabled: true
+      events.fluentbit.enabled: true
+      fluentbit.image.repository: registry.cern/toast
+      fluentbit.image.tag: 2
+      fluentbit.image.imagePullPolicy: Always
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: registry.cern/toast:2
+      - equal:
+          path: spec.template.spec.containers[0].imagePullPolicy
+          value: Always

values.yaml

@@ -485,3 +485,108 @@ logs:
     #     return 2, timestamp, record
     #   end
     #  my_other_lua_script.lua: ...
+
+events:
+  # -- indicates if components to capture Kubernetes events should be enabled or not.
+  enabled: false
+
+  fluentbit:
+    # -- indicates if fluentbit Kubernetes events component should be installed or not
+    enabled: false
+    ## Fluent Bit for events image
+    image:
+      # -- repository to use for Fluent Bit
+      repository: ""
+      # -- tag to use for Fluent Bit
+      tag: ""
+      # -- image pull policy for Fluent Bit
+      imagePullPolicy: ""
+    resources:
+      requests:
+        cpu: "5m"
+        memory: "15Mi"
+      limits:
+        cpu: "20m"
+        memory: "25Mi"
+
+    # -- fluentbit service configuration options in a multiline string
+    # @default -- partly autogenerated -- see values.yaml
+    service: |
+      [SERVICE]
+          Daemon Off
+          Flush 1
+          Log_Level WARN
+          HTTP_Server On
+          HTTP_Listen 0.0.0.0
+          HTTP_Port 2020
+          Health_Check On
+
+    # -- fluentbit inputs as a yaml list in a multiline string
+    # @default -- partly autogenerated -- see values.yaml
+    inputs: |
+      [INPUT]
+          Name kubernetes_events
+          Tag monit.k8s.events
+
+    # -- fluentbit filters as a yaml list in a multiline string
+    # @default -- partly autogenerated -- see values.yaml
+    filters: |
+      [FILTER]
+          Name lua
+          Match *
+          Script /fluent-bit/etc/scripts/date_to_ts.lua
+          Call add_time_from_creation_timestamp
+
+      [FILTER]
+          Name nest
+          Match *
+          Operation lift
+          Nested_under metadata
+
+      [FILTER]
+          Name nest
+          Match *
+          Operation lift
+          Nested_under involvedObject
+          Add_prefix involvedObject_
+
+      [FILTER]
+          Name nest
+          Match *
+          Operation lift
+          Nested_under source
+          Add_prefix source_
+
+      [FILTER]
+          Name modify
+          Match *
+          Add kubernetes_cluster {{ required "kubernetes.clusterName is missing" (.Values.kubernetes).clusterName }}
+          Add monit_type kubernetes_events
+          Remove managedFields
+
+    # -- fluentbit outputs as a yaml list in a multiline string
+    # @default -- partly autogenerated -- see values.yaml
+    outputs: |
+      [OUTPUT]
+          name opentelemetry
+          match *
+          host {{ .Values.otlp.endpoint }}
+          port {{ .Values.otlp.port }}
+          metrics_uri /v1/metrics
+          logs_uri /v1/logs
+          traces_uri /v1/traces
+          tls on
+          tls.verify off
+          http_user {{ required "Tenant name is required" (.Values.tenant).name }}
+          http_passwd {{ required "Tenant password is required" (.Values.tenant).password }}
+          header tag monit
+          header log_type kubernetes_events
+          header User-Agent {{ .Chart.Name }}/{{ .Chart.Version }}
+          logs_body_key imaginary_non_existing_field
+
+    ## -- extra volumes meant to be used in the fluentbits, can be used to scrape metrics from pvcs
+    extraVolumes: []
+    ## -- extra volumes to mount in the fluentbits, can be used to scrape metrics from pvcs
+    extraVolumeMounts: []
+    # -- extra Lua scripts for user-provided transformations
+    luaScripts: {}