From 4cad3bde52e0ab8d46f880b75728e53f2635ddc8 Mon Sep 17 00:00:00 2001
From: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
Date: Tue, 21 Jan 2025 13:59:37 +0100
Subject: [PATCH 1/6] [MONIT-4106] fluentbit/logs: add support for user
provided lua scripts
While doing trying to mimic the OpenShift logging setup with the current
Helm chart we noticed that they were using the lua scripts input from
the fluent bit official Helm chart. In our case this was not possible
and had to be done by adding secrets one by one.
We agreed to mimic this behaviour and had our own helm chart have the
option to add lua scripts to the fluent bit via helm chart values.
This commit modifies the configmap used to create the lua script that
we required for the timestampt in such a way that now it creates this
first lua script but also creates any other provided by the users via
the values of the helm chart.
Reported-at: https://its.cern.ch/jira/browse/MONIT-4106
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@gmail.com>
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
---
.../{scripts.yaml => configmap-luascripts.yaml} | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
rename templates/fluentbit-logs/{scripts.yaml => configmap-luascripts.yaml} (74%)
diff --git a/templates/fluentbit-logs/scripts.yaml b/templates/fluentbit-logs/configmap-luascripts.yaml
similarity index 74%
rename from templates/fluentbit-logs/scripts.yaml
rename to templates/fluentbit-logs/configmap-luascripts.yaml
index 954535c..916c74d 100644
--- a/templates/fluentbit-logs/scripts.yaml
+++ b/templates/fluentbit-logs/configmap-luascripts.yaml
@@ -12,4 +12,9 @@ data:
record['timestamp'] = exp_sec * 1000
return 2, timestamp, record
end
-{{- end -}}
+ {{- if .Values.logs.fluentbit.luaScripts -}}
+ {{ range $key, $value := .Values.logs.fluentbit.luaScripts }}
+ {{ $key }}: {{ $value | quote }}
+ {{ end }}
+ {{ end }}
+{{- end -}}
\ No newline at end of file
--
GitLab
From 9891591dee8a42bf3aae9e074437b6bffe04a097 Mon Sep 17 00:00:00 2001
From: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
Date: Tue, 21 Jan 2025 14:10:51 +0100
Subject: [PATCH 2/6] [MONIT-4106] values/logs/fluentbit: add default
luaScripts value
In commit 8c054c8 we added support for lua scripts provided via user
values.
This commit adds a default value and documents this in the values file.
Reported-at: https://its.cern.ch/jira/browse/MONIT-4106
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@gmail.com>
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
---
values.yaml | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/values.yaml b/values.yaml
index 15c1580..ae64715 100644
--- a/values.yaml
+++ b/values.yaml
@@ -158,6 +158,17 @@ metrics:
uri: /api/prom/push
threaded: false
+ # These scripts are available in the fluentbit /fluent-bit/etc/scripts path.
+ # Include your lua scripts in the following format:
+ # luaScripts:
+ # my_lua_script.lua: |
+ # function my_function(tag, timestamp, record)
+ # // Do something...
+ # return 2, timestamp, record
+ # end
+ # my_other_lua_script.lua: ...
+ luaScripts: {}
+
# -- max size for in-disk storage for fluent-bit
diskMaxCache: "5G"
--
GitLab
From 2e1fa1f6cd6b70c8245f8ae23bf7a5b911c3f256 Mon Sep 17 00:00:00 2001
From: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
Date: Tue, 21 Jan 2025 14:38:09 +0100
Subject: [PATCH 3/6] [MONIT-4106] fluentbit/metrics: add support for user
provided lua scripts
While doing trying to mimic the OpenShift logging setup with the current
Helm chart we noticed that they were using the lua scripts input from
the fluent bit official Helm chart. In our case this was not possible
and had to be done by adding secrets one by one.
We agreed to mimic this behaviour and had our own helm chart have the
option to add lua scripts to the fluent bit via helm chart values.
This commit modifies the configmap used to create the lua script that
we required for the timestampt in such a way that now it creates this
first lua script but also creates any other provided by the users via
the values of the helm chart.
Reported-at: https://its.cern.ch/jira/browse/MONIT-4106
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@gmail.com>
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
---
.../fluentbit-metrics/configmap-luascripts.yaml | 12 ++++++++++++
templates/fluentbit-metrics/statefulset.yaml | 9 +++++++++
2 files changed, 21 insertions(+)
create mode 100644 templates/fluentbit-metrics/configmap-luascripts.yaml
diff --git a/templates/fluentbit-metrics/configmap-luascripts.yaml b/templates/fluentbit-metrics/configmap-luascripts.yaml
new file mode 100644
index 0000000..ea6f6a8
--- /dev/null
+++ b/templates/fluentbit-metrics/configmap-luascripts.yaml
@@ -0,0 +1,12 @@
+{{- if and .Values.metrics.enabled .Values.metrics.fluentbit.enabled .Values.metrics.fluentbit.luaScripts -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: it-monit-metrics-collector-fluentbit-scripts
+data:
+ {{- if .Values.metrics.fluentbit.luaScripts -}}
+ {{ range $key, $value := .Values.metrics.fluentbit.luaScripts }}
+ {{ $key }}: {{ $value | quote }}
+ {{ end }}
+ {{ end }}
+{{- end -}}
\ No newline at end of file
diff --git a/templates/fluentbit-metrics/statefulset.yaml b/templates/fluentbit-metrics/statefulset.yaml
index 32eff77..1a757ee 100644
--- a/templates/fluentbit-metrics/statefulset.yaml
+++ b/templates/fluentbit-metrics/statefulset.yaml
@@ -33,6 +33,10 @@ spec:
volumeMounts:
- name: config
mountPath: /fluent-bit/etc/conf
+ {{- if .Values.metrics.fluentbit.luaScripts -}}
+ - name: scripts
+ mountPath: /fluent-bit/etc/scripts
+ {{- end }}
- name: fluentbit
mountPath: /flb-storage/
{{- if .Values.metrics.fluentbit.extraVolumeMounts }}
@@ -46,6 +50,11 @@ spec:
- name: config
configMap:
name: it-monit-metrics-collector-fluentbit
+ {{- if .Values.metrics.fluentbit.luaScripts -}}
+ - name: scripts
+ configMap:
+ name: it-monit-metrics-collector-fluentbit-scripts
+ {{- end }}
- name: fluentbit
emptyDir:
sizeLimit: {{ .Values.metrics.fluentbit.diskMaxCache }}
--
GitLab
From 3491d5194d3af7cc7473c4df5775013d3b5b0f59 Mon Sep 17 00:00:00 2001
From: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
Date: Tue, 21 Jan 2025 14:49:17 +0100
Subject: [PATCH 4/6] [MONIT-4106] values/metrics/fluentbit: add default
luaScripts value
In commit c6c3db2 we added support for lua scripts provided via user
values.
This commit adds a default value and documents this in the values file.
Reported-at: https://its.cern.ch/jira/browse/MONIT-4106
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@gmail.com>
Signed-off-by: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
---
values.yaml | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/values.yaml b/values.yaml
index ae64715..960930d 100644
--- a/values.yaml
+++ b/values.yaml
@@ -423,3 +423,14 @@ logs:
extraVolumes: []
## -- extra volumes to mount in the fluentbits, can be used to scrape metrics from pvcs
extraVolumeMounts: []
+
+ # These scripts are available in the fluentbit /fluent-bit/etc/scripts path.
+ # Include your lua scripts in the following format:
+ # luaScripts:
+ # my_lua_script.lua: |
+ # function my_function(tag, timestamp, record)
+ # // Do something...
+ # return 2, timestamp, record
+ # end
+ # my_other_lua_script.lua: ...
+ luaScripts: {}
--
GitLab
From 7ab67bc48bb642417296d3ad4b54cfb18119fe88 Mon Sep 17 00:00:00 2001
From: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
Date: Tue, 21 Jan 2025 15:10:52 +0100
Subject: [PATCH 5/6] [MONIT-4106] docs/logs/fluentbit: add values provided lua
scripts section
---
docs/logs.md | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/docs/logs.md b/docs/logs.md
index f11945c..b791898 100644
--- a/docs/logs.md
+++ b/docs/logs.md
@@ -101,6 +101,21 @@ outputs: |
http_passwd {{ .Values.tenant.password }}
```
+### Fluentbit Custom Lua Scripts
+In Fluent Bit we can use Lua scripts to create processors that transform the records. These scripts are available in the Fluent Bit container under the path `/fluent-bit/etc/scripts`. In order to create new Lua scripts just add them in the values `logs.fluentbit.luaScripts` key.
+
+For example, the following configuration will create two files in `/fluent-bit/etc/scripts`. One named `my_lua_script.lua` and another one named `my_lua_script2.lua`.
+```yaml
+luaScripts:
+ my_lua_script.lua: |
+ function my_function(tag, timestamp, record)
+ // Do something...
+ return 2, timestamp, record
+ end
+ my_other_lua_script.lua: |
+ ...
+```
+
## 3. Customizing Fluentbit for Additional Log Sources
You can extend the logging configuration by extending previous configurations with custom Fluentbit **inputs**, **filters**, or **outputs**. This is especially useful if you want to gather logs from additional sources or apply different processing to specific logs.
--
GitLab
From a640841d5cc82e87b93b24286b4339ad8036b825 Mon Sep 17 00:00:00 2001
From: Guillermo Facundo Colunga <guillermo.facundo.colunga@cern.ch>
Date: Mon, 27 Jan 2025 15:40:36 +0100
Subject: [PATCH 6/6] docs/metrics: fix conflics from monit4106 on 5b88e74
---
docs/metrics.md | 24 +++++++++++++++++++++++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/docs/metrics.md b/docs/metrics.md
index 938a833..38c55c0 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -121,8 +121,30 @@ metrics:
regex: ".*_unwanted_metric"
```
+## 4. Pushing Metrics to Central Monitoring Infrastructure
+By default the Helm chart uses a dedicated Fluent Bit to upload the metrics to the Central Monitoring Infrastructure via OpenTelemetry protocol.
-## 4 . Further Customizations
+This Fluent Bit already has a default configuration for the inputs, filters and outputs. Please, check the `values.yaml` to see the default configuration.
+
+Besides these basic configuration options, others are also offered.
+
+### Fluentbit Custom Lua Scripts
+In Fluent Bit we can use Lua scripts to create processors that transform the records. These scripts are available in the Fluent Bit container under the path `/fluent-bit/etc/scripts`. In order to create new Lua scripts just add them in the values `logs.fluentbit.luaScripts` key.
+
+For example, the following configuration will create two files in `/fluent-bit/etc/scripts`. One named `my_lua_script.lua` and another one named `my_lua_script2.lua`.
+```yaml
+luaScripts:
+ my_lua_script.lua: |
+ function my_function(tag, timestamp, record)
+ // Do something...
+ return 2, timestamp, record
+ end
+ my_other_lua_script.lua: |
+ ...
+```
+
+
+## 5. Further Customizations
You can further customize metrics collection with the following settings:
--
GitLab