fluentd-firehose.yaml

deployment-sample.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-sweet-api
spec:
  template:
    metadata:
      annotations:
        fluentd_firehose_delivery_stream_name: "my-sweet-firehose-stream-name"

fluentd-firehose.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd-firehose
  namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd-firehose-role
rules:
  - apiGroups: [""]
    resources:
      - namespaces
      - pods
      - pods/logs
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluentd-firehose-role-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluentd-firehose-role
subjects:
  - kind: ServiceAccount
    name: fluentd-firehose
    namespace: default
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-firehose-config
  namespace: default
  labels:
    k8s-app: fluentd-firehose
data:
  fluent.conf: |
    @include containers.conf

    <match fluent.**>
      @type null
    </match>
  containers.conf: |
    <source>
      @type tail
      @id in_tail_container_logs
      @label @containers
      path /var/log/containers/*.log
      exclude_path ["/var/log/containers/cloudwatch-agent*", "/var/log/containers/fluentd*"]
      pos_file /var/log/fluentd-firehose-containers.log.pos
      tag *
      read_from_head true
      <parse>
        @type json
        time_format %Y-%m-%dT%H:%M:%S.%NZ
      </parse>
    </source>

    <label @containers>
      <filter **>
        @type kubernetes_metadata
        @id filter_kube_metadata
        annotation_match ["fluentd_firehose_delivery_stream_name"]
      </filter>

      # Collect only logs of pods having annotations.fluentd_firehose_delivery_stream_name
      <filter **>
        @type grep
        <regexp>
          key $.kubernetes.annotations.fluentd_firehose_delivery_stream_name
          pattern /.+/
        </regexp>
      </filter>

      <filter **>
        @type concat
        key log
        multiline_start_regexp /^\S/
        separator ""
        flush_interval 5
        timeout_label @NORMAL
      </filter>

      <match **>
        @type relabel
        @label @NORMAL
      </match>
    </label>

    <label @NORMAL>
      # <filter **>
      #   @type stdout
      # </filter>

      <match **>
        @type kinesis_firehose
        @id out_kinesis_firehose_containers
        region "#{ENV.fetch('REGION')}"
        delivery_stream_name ${$.kubernetes.annotations.fluentd_firehose_delivery_stream_name}
        <buffer $.kubernetes.annotations.fluentd_firehose_delivery_stream_name>
          flush_interval 5
          chunk_limit_size 2m
          queued_chunks_limit_size 32
          retry_forever true
        </buffer>
      </match>
    </label>
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd-firehose
  namespace: default
spec:
  selector:
    matchLabels:
      k8s-app: fluentd-firehose
  template:
    metadata:
      labels:
        k8s-app: fluentd-firehose
      annotations:
        configHash: 8915de4cf9c3551a8dc74c0137a3e83569d28c71044b0359c2578d2e0461825
    spec:
      serviceAccountName: fluentd-firehose
      terminationGracePeriodSeconds: 30
      # Because the image's entrypoint requires to write on /fluentd/etc but we mount configmap there which is read-only,
      # this initContainers workaround or other is needed.
      # See https://github.com/fluent/fluentd-kubernetes-daemonset/issues/90
      initContainers:
        - name: copy-fluentd-firehose-config
          image: busybox
          command: ["sh", "-c", "cp /config-volume/..data/* /fluentd/etc"]
          volumeMounts:
            - name: config-volume
              mountPath: /config-volume
            - name: fluentdconf
              mountPath: /fluentd/etc
        - name: update-log-driver
          image: busybox
          command: ["sh", "-c", ""]
      containers:
        - name: fluentd-firehose
          image: fluent/fluentd-kubernetes-daemonset:v1.11.5-debian-kinesis-1.0
          env:
            - name: REGION
              value: ap-northeast-1
          resources:
            limits:
              memory: 400Mi
            requests:
              cpu: 100m
              memory: 200Mi
          volumeMounts:
            - name: config-volume
              mountPath: /config-volume
            - name: fluentdconf
              mountPath: /fluentd/etc
            - name: varlog
              mountPath: /var/log
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: runlogjournal
              mountPath: /run/log/journal
              readOnly: true
            - name: dmesg
              mountPath: /var/log/dmesg
              readOnly: true
      volumes:
        - name: config-volume
          configMap:
            name: fluentd-firehose-config
        - name: fluentdconf
          emptyDir: {}
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: runlogjournal
          hostPath:
            path: /run/log/journal
        - name: dmesg
          hostPath:
            path: /var/log/dmesg

https://qiita.com/sonots/items/c1b891786779d54d6c68