feat(fluent-bit): yaml configuration support

Signed-off-by: ABWassim <[email protected]>
fluent · Nov 7, 2024 · c8feab9 · c8feab9
1 parent 5b3161a
commit c8feab9
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 6 deletions.
diff --git a/charts/fluent-bit/templates/_pod.tpl b/charts/fluent-bit/templates/_pod.tpl
@@ -64,6 +64,11 @@ containers:
     {{- if .Values.hotReload.enabled }}
       - --enable-hot-reload
     {{- end }}
+    {{- if .Values.useYamlConfig }}
+      - --config=/fluent-bit/etc/conf/fluent-bit.yaml
+    {{- else }}
+      - --config=/fluent-bit/etc/conf/fluent-bit.conf
+    {{- end }}
   {{- end}}
     ports:
       - name: http

diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml
@@ -7,13 +7,28 @@ metadata:
   labels:
     {{- include "fluent-bit.labels" . | nindent 4 }}
 data:
+  {{ if .Values.useYamlConfig -}}
+  fluent-bit.yaml: |
+    service:
+      {{- tpl (toYaml .Values.yamlConfig.service) . | nindent 6 }}
+    pipeline:
+      inputs:
+        {{- tpl (toYaml .Values.yamlConfig.pipeline.inputs) . | nindent 8 }}
+      filters:
+        {{- tpl (toYaml .Values.yamlConfig.pipeline.filters) . | nindent 8 }}
+      outputs:
+        {{- tpl (toYaml .Values.yamlConfig.pipeline.outputs) . | nindent 8 }}
+    parsers:
+      {{- tpl (toYaml .Values.yamlConfig.parsers) . | nindent 6 }}
+  {{ else -}}
   custom_parsers.conf: |
-    {{- (tpl .Values.config.customParsers $) | nindent 4 }}
+    {{- (tpl .Values.config.customParsers .) | nindent 4 }}
   fluent-bit.conf: |
-    {{- (tpl .Values.config.service $)  | nindent 4 }}
-    {{- (tpl .Values.config.inputs $)  | nindent 4 }}
-    {{- (tpl .Values.config.filters $)  | nindent 4 }}
-    {{- (tpl .Values.config.outputs $)  | nindent 4 }}
+    {{- (tpl .Values.config.service .)  | nindent 4 }}
+    {{- (tpl .Values.config.inputs .)  | nindent 4 }}
+    {{- (tpl .Values.config.filters .)  | nindent 4 }}
+    {{- (tpl .Values.config.outputs .)  | nindent 4 }}    
+  {{ end -}}
   {{- range $key, $val := .Values.config.upstream }}
   {{ $key }}: |
     {{- (tpl $val $) | nindent 4 }}

diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml
@@ -364,6 +364,56 @@ networkPolicy:
 
 luaScripts: {}
 
+useYamlConfig: true
+
+yamlConfig:
+  service:
+    daemon: "Off"
+    flush: '{{ .Values.flush }}'
+    log_level: '{{ .Values.logLevel }}'
+    parsers_file: "/fluent-bit/etc/parsers.conf"
+    http_server: "On"
+    http_listen: "0.0.0.0"
+    http_port: '{{ .Values.metricsPort }}'
+    Health_Check: "On"
+  pipeline:
+    inputs:
+      - name: tail
+        path: "/var/log/containers/*.log"
+        multiline.parser: "docker, cri"
+        Tag: "kube.*"
+        Mem_Buf_Limit: "5MB"
+        Skip_Long_Lines: "On"
+      - name: systemd
+        Tag: host.*
+        Systemd_Filter: "_SYSTEMD_UNIT=kubelet.service"
+        Read_From_Tail: "On"
+    filters:
+      - name: kubernetes
+        match: kube.*
+        Merge_Log: "On"
+        Keep_Log: "Off"
+        K8S-Logging.Parser: "On"
+        K8S-Logging.Exclude: "On"
+    outputs:
+      - name: es
+        match: kube.*
+        Host: elasticsearch-master
+        Logstash_Format: "On"
+        Retry_Limit: "False"
+      - name: es
+        match: host.*
+        Host: elasticsearch-master
+        Logstash_Format: "On"
+        Logstash_Prefix: "node"
+        Retry_Limit: "False"
+  parsers:
+    name: docker_no_time
+    format: json
+    time_keep: "false"
+    time_key: time
+    time_format: "%Y-%m-%dT%H:%M:%S.%L"
+
 ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file
 config:
   service: |
@@ -483,7 +533,6 @@ command:
 
 args:
   - --workdir=/fluent-bit/etc
-  - --config=/fluent-bit/etc/conf/fluent-bit.conf
 
 # This supports either a structured array or a templatable string
 initContainers: []