fluent-bit.yaml

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluent-bit
  labels:
    app: fluent-bit
rules:
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluent-bit
  labels:
    app: fluent-bit
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluent-bit
subjects:
  - kind: ServiceAccount
    name: fluent-bit
    namespace: fluent-bit

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluent-bit
  namespace: kube-system
  labels:
    app: fluent-bit

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit
  namespace: kube-system
  labels:
    app: fluent-bit
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush        5
        Daemon       Off
        Log_Level    info
        Health_Check On
        HTTP_Server  On
        HTTP_Listen  0.0.0.0
        HTTP_Port    2020
        Parsers_File parsers.conf
        Parsers_File parsers_custom.conf

    [INPUT]
        Name              tail
        Path              /var/log/containers/*.log
        Multiline.Parser  docker, cri
        Tag               kube.*
        Refresh_Interval  5
        Buffer_Chunk_Size 256k
        Buffer_Max_Size   256k
        Mem_Buf_Limit     128MB
        Skip_Long_Lines   Off
        DB                /tail-db/tail-containers-state.db
        DB.Sync           Normal

    [FILTER]
        Name                kubernetes
        Match               kube.*
        Buffer_Size         256k
        Kube_Tag_Prefix     kube.var.log.containers.
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Merge_Log           On
        Merge_Log_Key       content
        K8S-Logging.Parser  On
        K8S-Logging.Exclude On

    [FILTER]
        Name  modify
        Match *
        Add   cluster kind

    [OUTPUT]
        Name                  clickhouse
        Match                 *
        Address               clickhouse-clickhouse.kube-system.svc.cluster.local:9000
        Database              logs
        Username              admin
        Password              admin
        Async_Insert          true
        Wait_For_Async_Insert true
        Batch_Size            1000
        Flush_Interval        10s
        Force_Number_Fields   content.duration,content.upstream_service_time
        Log_Format            json
        Log_Level             DEBUG

  parsers.conf: |
    [PARSER]
        Name        logfmt
        Format      logfmt

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: kube-system
  labels:
    app: fluent-bit
spec:
  selector:
    matchLabels:
      app: fluent-bit
  template:
    metadata:
      labels:
        app: fluent-bit
    spec:
      serviceAccountName: fluent-bit
      serviceAccount: fluent-bit
      containers:
        - image: localhost:5001/klogs:latest
          imagePullPolicy: Always
          name: fluent-bit
          ports:
            - containerPort: 2020
              name: http-metrics
              protocol: TCP
          volumeMounts:
            - mountPath: /var/log
              name: varlog
            - mountPath: /var/lib/docker/containers
              name: varlibdockercontainers
              readOnly: true
            - mountPath: /fluent-bit/etc/fluent-bit.conf
              name: config
              subPath: fluent-bit.conf
            - mountPath: /fluent-bit/etc/parsers_custom.conf
              name: config
              subPath: parsers.conf
            - mountPath: /tail-db
              name: tail-db
          livenessProbe:
            httpGet:
              path: /
              port: http-metrics
          readinessProbe:
            httpGet:
              path: /api/v1/health
              port: http-metrics
          resources:
            limits:
              cpu: 100m
              memory: 256Mi
            requests:
              cpu: 10m
              memory: 64Mi
      dnsPolicy: ClusterFirst
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
        - effect: NoExecute
          operator: Exists
      volumes:
        - hostPath:
            path: /var/log
          name: varlog
        - hostPath:
            path: /var/lib/docker/containers
          name: varlibdockercontainers
        - hostPath:
            path: /var/lib/fluent-bit
            type: DirectoryOrCreate
          name: tail-db
        - configMap:
            defaultMode: 420
            name: fluent-bit
          name: config

---
apiVersion: v1
kind: Service
metadata:
  name: fluent-bit
  namespace: kube-system
  labels:
    app: fluent-bit
spec:
  ports:
    - name: http-metrics
      port: 2020
      protocol: TCP
      targetPort: http-metrics
  selector:
    app: fluent-bit
  type: ClusterIP