apiVersion: v1
kind: Namespace
metadata:
  labels:
    app: filebeat
  name: filebeat
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat-serviceaccount
  namespace: filebeat
  labels:
    k8s-app: filebeat
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: filebeat-clusterrole
  labels:
    k8s-app: filebeat
rules:
- apiGroups: [""]
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: filebeat-clusterrolebinding
  labels:
    k8s-app: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat-serviceaccount
  namespace: filebeat
roleRef:
  kind: ClusterRole
  name: filebeat-clusterrole
  apiGroup: rbac.authorization.k8s.io
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: filebeat-main
  namespace: filebeat
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |
    filebeat:
      registry:
        path: /var/lib/filebeat/registry
      config:
        inputs:
          enabled: true
          path: /conf.d/*.yml
          reload.enabled: true
          reload.period: 10s

    processors:
    - drop_fields:
        fields: ["ecs", "host", "agent", "input", "log", "stream", "container"]
        ignore_missing: true

    output:
      kafka:
        enabled: true
        version: 0.10.0.0
        bulk_max_size: 20480
        timeout: 5s
        hosts: '${KAFKA_ADDR}'
        max_message_bytes: 10485760 # 10M
        required_acks: 1
        compression: gzip
        worker: 3
        partition.round_robin:
          reachable_only: true 
        metadata:
          retry.max: 3
          retry.backoff: 250ms
          refresh_frequency: 10s
        topic: fullLog_noncore
        topics:
        - topic: '%{[fields.items]}'
          when.has_fields: ['fields.items']
        - topic: fullLog_core
          when.regexp:
            message: "^{.*"
      elasticsearch:
        enabled: false

    logging:
      level: info
      metrics.enabled: false

    queue:
      mem:
        events: 256
        flush.min_events: 128

    http:
      enabled: true
      host: 0.0.0.0
      port: 5066
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: filebeat-input
  namespace: filebeat
  labels:
    k8s-app: filebeat
data:
  container-logs.yml: |
    - type: container
      enabled: true
      stream: all # both stdout and stderr
      harvester_buffer_size: 16384
      max_bytes: 10485760 # 10M
      #ignore_older: 2h # ignore_older must be greater than close_inactive
      close_inactive: 10m # default 5m
      scan_frequency: 10s # default 10s
      tail_files: false
      paths:
      - "/var/log/containers/*.log"
      fields:
        cluster: ${FILEBEAT_ENV:filebeat_env_not_set}
        host: ${NODENAME:nodename_not_set}
        from: container-logs
      multiline:
        pattern: '^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}\]|\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\,\d{3}|\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - .+ \[\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4}\]|^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - \[\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\] - .+ \[\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4}\]|^{"tags":"nginx-http"|^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[([a-zA-Z]+)\]|^\[\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\] \[\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4}\]|^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}'
        negate: true
        match: after
      processors:
      - add_kubernetes_metadata:
          in_cluster: true
          default_indexers.enabled: false
          default_matchers.enabled: false
          indexers:
          - container:
          matchers:
          - logs_path:
              logs_path: '/var/log/containers'
              resource_type: 'container'
          include_labels:
          - app
          - k8s-app
          - tier
      - drop_event:
          when:
            equals:
              kubernetes.labels.k8s-app: 'filebeat'
      - copy_fields:
          fields:
          - from: kubernetes.labels.tier
            to: fields.items
          - from: kubernetes.pod.name
            to: fields.podname
          - from: kubernetes.labels.app
            to: fields.project
          - from: kubernetes.labels.k8s-app
            to: fields.project
          - from: kubernetes.labels.name
            to: fields.project
          - from: kubernetes.container.name
            to: fields.container
          fail_on_error: false
          ignore_missing: true
      - drop_fields:
          fields: ["kubernetes"]
          ignore_missing: true
     # - drop_event:
     #     when:
     #       not:
     #         has_fields: ['fields.project']
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-envvars
  namespace: filebeat
  labels:
    k8s-app: filebeat
data:
  FILEBEAT_ENV: "d1-prod"
  KAFKA_ADDR: "my-cluster-kafka-bootstrap.kafka:9092"
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: filebeat
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      terminationGracePeriodSeconds: 60
      serviceAccountName: filebeat-serviceaccount
      tolerations:
      - operator: Exists
        effect: NoSchedule
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: filebeat_disabled
                operator: DoesNotExist
      containers:
      - name: filebeat
        #image: registry.cn-hangzhou.aliyuncs.com/zhengyu1992/filebeat:m5-7.9.0
        image: harbor.uenpay.com/base/filebeat:m5-7.9.0
        imagePullPolicy: IfNotPresent
        securityContext:
          privileged: true  
        ports:
        - containerPort: 9090
        env:
        - name: NODENAME
          valueFrom:  
            fieldRef:  
              fieldPath: spec.nodeName
        envFrom:
        - configMapRef:
            name: filebeat-envvars
        volumeMounts: 
        - name: filebeat-main
          mountPath: /filebeat.yml
          subPath: filebeat.yml
          readOnly: true
        - name: filebeat-input
          mountPath: /conf.d
        - name: varlibfilebeat
          mountPath: /var/lib/filebeat
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: datadockercontainers
          mountPath: /data/docker/containers
          readOnly: true
      volumes: 
      - name: filebeat-main
        configMap:
          name: filebeat-main
      - name: filebeat-input
        configMap:
          name: filebeat-input
      - name: varlibfilebeat
        hostPath: 
          path: /var/lib/filebeat
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: datadockercontainers
        hostPath:
          path: /data/docker/containers
---
apiVersion: v1
kind: Service
metadata:
  name: filebeat
  namespace: filebeat
  labels:
    k8s-app: filebeat
spec:
  ports:
  - port: 9090
    targetPort: 9090
  selector:
      k8s-app: filebeat