feat(fluent-bit): yaml configuration support

Signed-off-by: ABWassim <[email protected]>
fluent · Nov 7, 2024 · 77576dc · 77576dc
1 parent 5b3161a
commit 77576dc
Show file tree

Hide file tree

Showing 4 changed files with 320 additions and 6 deletions.
diff --git a/charts/fluent-bit/templates/_pod.tpl b/charts/fluent-bit/templates/_pod.tpl
@@ -64,6 +64,11 @@ containers:
     {{- if .Values.hotReload.enabled }}
       - --enable-hot-reload
     {{- end }}
+    {{- if .Values.useYamlConfig }}
+      - --config=/fluent-bit/etc/conf/fluent-bit.yaml
+    {{- else }}
+      - --config=/fluent-bit/etc/conf/fluent-bit.conf
+    {{- end }}
   {{- end}}
     ports:
       - name: http

diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml
@@ -7,13 +7,28 @@ metadata:
   labels:
     {{- include "fluent-bit.labels" . | nindent 4 }}
 data:
+  {{ if .Values.useYamlConfig -}}
+  fluent-bit.yaml: |
+    service:
+      {{- tpl (toYaml .Values.yamlConfig.service) . | nindent 6 }}
+    pipeline:
+      inputs:
+        {{- tpl (toYaml .Values.yamlConfig.pipeline.inputs) . | nindent 8 }}
+      filters:
+        {{- tpl (toYaml .Values.yamlConfig.pipeline.filters) . | nindent 8 }}
+      outputs:
+        {{- tpl (toYaml .Values.yamlConfig.pipeline.outputs) . | nindent 8 }}
+    parsers:
+      {{- tpl (toYaml .Values.yamlConfig.parsers) . | nindent 6 }}
+  {{ else -}}
   custom_parsers.conf: |
-    {{- (tpl .Values.config.customParsers $) | nindent 4 }}
+    {{- (tpl .Values.config.customParsers .) | nindent 4 }}
   fluent-bit.conf: |
-    {{- (tpl .Values.config.service $)  | nindent 4 }}
-    {{- (tpl .Values.config.inputs $)  | nindent 4 }}
-    {{- (tpl .Values.config.filters $)  | nindent 4 }}
-    {{- (tpl .Values.config.outputs $)  | nindent 4 }}
+    {{- (tpl .Values.config.service .)  | nindent 4 }}
+    {{- (tpl .Values.config.inputs .)  | nindent 4 }}
+    {{- (tpl .Values.config.filters .)  | nindent 4 }}
+    {{- (tpl .Values.config.outputs .)  | nindent 4 }}    
+  {{ end -}}
   {{- range $key, $val := .Values.config.upstream }}
   {{ $key }}: |
     {{- (tpl $val $) | nindent 4 }}

diff --git a/charts/fluent-bit/test.yaml b/charts/fluent-bit/test.yaml
@@ -0,0 +1,245 @@
+---
+# Source: fluent-bit/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: release-name-fluent-bit
+  namespace: default
+  labels:
+    helm.sh/chart: fluent-bit-0.47.10
+    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/version: "3.1.9"
+    app.kubernetes.io/managed-by: Helm
+---
+# Source: fluent-bit/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: release-name-fluent-bit
+  namespace: default
+  labels:
+    helm.sh/chart: fluent-bit-0.47.10
+    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/version: "3.1.9"
+    app.kubernetes.io/managed-by: Helm
+data:
+  custom_parsers.conf: |
+    [PARSER]
+        Name docker_no_time
+        Format json
+        Time_Keep Off
+        Time_Key time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+    
+  fluent-bit.conf: |
+    [SERVICE]
+        Daemon Off
+        Flush 1
+        Log_Level info
+        Parsers_File /fluent-bit/etc/parsers.conf
+        Parsers_File /fluent-bit/etc/conf/custom_parsers.conf
+        HTTP_Server On
+        HTTP_Listen 0.0.0.0
+        HTTP_Port 2020
+        Health_Check On
+    
+    [INPUT]
+        Name tail
+        Path /var/log/containers/*.log
+        multiline.parser docker, cri
+        Tag kube.*
+        Mem_Buf_Limit 5MB
+        Skip_Long_Lines On
+    
+    [INPUT]
+        Name systemd
+        Tag host.*
+        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+        Read_From_Tail On
+    
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
+        K8S-Logging.Exclude On
+    
+    [OUTPUT]
+        Name es
+        Match kube.*
+        Host elasticsearch-master
+        Logstash_Format On
+        Retry_Limit False
+    
+    [OUTPUT]
+        Name es
+        Match host.*
+        Host elasticsearch-master
+        Logstash_Format On
+        Logstash_Prefix node
+        Retry_Limit False
+---
+# Source: fluent-bit/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: release-name-fluent-bit
+  labels:
+    helm.sh/chart: fluent-bit-0.47.10
+    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/version: "3.1.9"
+    app.kubernetes.io/managed-by: Helm
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Source: fluent-bit/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: release-name-fluent-bit
+  labels:
+    helm.sh/chart: fluent-bit-0.47.10
+    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/version: "3.1.9"
+    app.kubernetes.io/managed-by: Helm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: release-name-fluent-bit
+subjects:
+  - kind: ServiceAccount
+    name: release-name-fluent-bit
+    namespace: default
+---
+# Source: fluent-bit/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: release-name-fluent-bit
+  namespace: default
+  labels:
+    helm.sh/chart: fluent-bit-0.47.10
+    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/version: "3.1.9"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  type: ClusterIP
+  ports:
+    - port: 2020
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/instance: release-name
+---
+# Source: fluent-bit/templates/daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: release-name-fluent-bit
+  namespace: default
+  labels:
+    helm.sh/chart: fluent-bit-0.47.10
+    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/version: "3.1.9"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: fluent-bit
+      app.kubernetes.io/instance: release-name
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: fluent-bit
+        app.kubernetes.io/instance: release-name
+      annotations:
+        checksum/config: 8003667e2604cff6c423f47c794c63b1746863824bfb98867804dd2ffa183423
+    spec:
+      serviceAccountName: release-name-fluent-bit
+      hostNetwork: false
+      dnsPolicy: ClusterFirst
+      containers:
+        - name: fluent-bit
+          image: "cr.fluentbit.io/fluent/fluent-bit:3.1.9"
+          imagePullPolicy: IfNotPresent
+          command:
+            - /fluent-bit/bin/fluent-bit
+          args:
+            - --workdir=/fluent-bit/etc
+            - --config=/fluent-bit/etc/conf/fluent-bit.conf
+          ports:
+            - name: http
+              containerPort: 2020
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /api/v1/health
+              port: http
+          volumeMounts:
+            - name: config
+              mountPath: /fluent-bit/etc/conf
+            - mountPath: /var/log
+              name: varlog
+            - mountPath: /var/lib/docker/containers
+              name: varlibdockercontainers
+              readOnly: true
+            - mountPath: /etc/machine-id
+              name: etcmachineid
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: release-name-fluent-bit
+        - hostPath:
+            path: /var/log
+          name: varlog
+        - hostPath:
+            path: /var/lib/docker/containers
+          name: varlibdockercontainers
+        - hostPath:
+            path: /etc/machine-id
+            type: File
+          name: etcmachineid
+---
+# Source: fluent-bit/templates/tests/test-connection.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "release-name-fluent-bit-test-connection"
+  namespace: default
+  labels:
+    helm.sh/chart: fluent-bit-0.47.10
+    app.kubernetes.io/version: "3.1.9"
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    helm.sh/hook: test
+    helm.sh/hook-delete-policy: hook-succeeded
+spec:
+  containers:
+    - name: wget
+      image: "busybox:latest"
+      imagePullPolicy: Always
+      command: ["sh"]
+      args: ["-c", "wget -O- release-name-fluent-bit:2020"]
+  restartPolicy: Never
diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml
@@ -364,6 +364,56 @@ networkPolicy:
 
 luaScripts: {}
 
+useYamlConfig: false
+
+yamlConfig:
+  service:
+    daemon: "Off"
+    flush: '{{ .Values.flush }}'
+    log_level: '{{ .Values.logLevel }}'
+    parsers_file: "/fluent-bit/etc/parsers.conf"
+    http_server: "On"
+    http_listen: "0.0.0.0"
+    http_port: '{{ .Values.metricsPort }}'
+    Health_Check: "On"
+  pipeline:
+    inputs:
+      - name: tail
+        path: "/var/log/containers/*.log"
+        multiline.parser: "docker, cri"
+        Tag: "kube.*"
+        Mem_Buf_Limit: "5MB"
+        Skip_Long_Lines: "On"
+      - name: systemd
+        Tag: host.*
+        Systemd_Filter: "_SYSTEMD_UNIT=kubelet.service"
+        Read_From_Tail: "On"
+    filters:
+      - name: kubernetes
+        match: kube.*
+        Merge_Log: "On"
+        Keep_Log: "Off"
+        K8S-Logging.Parser: "On"
+        K8S-Logging.Exclude: "On"
+    outputs:
+      - name: es
+        match: kube.*
+        Host: elasticsearch-master
+        Logstash_Format: "On"
+        Retry_Limit: "False"
+      - name: es
+        match: host.*
+        Host: elasticsearch-master
+        Logstash_Format: "On"
+        Logstash_Prefix: "node"
+        Retry_Limit: "False"
+  parsers:
+    name: docker_no_time
+    format: json
+    time_keep: "false"
+    time_key: time
+    time_format: "%Y-%m-%dT%H:%M:%S.%L"
+
 ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file
 config:
   service: |
@@ -483,7 +533,6 @@ command:
 
 args:
   - --workdir=/fluent-bit/etc
-  - --config=/fluent-bit/etc/conf/fluent-bit.conf
 
 # This supports either a structured array or a templatable string
 initContainers: []