Kubernetes Audit log collect with PLG Stack

steps.md

Start Kubernetes Cluster

$ minikube start --profile audit

Create audit policy

$ minikube ssh --profile audit
$ curl -sLO https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/audit/audit-policy.yaml
$ sudo mkdir -p /var/lib/k8s_audit
$ sudo mv audit-policy.yaml /var/lib/k8s_audit

Configure kube-apiserver

$ minikube ssh --profile audit
$ sudo vi /etc/kubernetes/manifests/kube-apiserver.yaml
# volumeMounts
...
volumeMounts:
  - mountPath: /var/lib/k8s_audit/audit-policy.yaml
    name: audit
    readOnly: true
  - mountPath: /var/log/audit/audit.log
    name: audit-log
    readOnly: false

# volumes
...
- name: audit
  hostPath:
    path: /var/lib/k8s_audit/audit-policy.yaml
    type: File

- name: audit-log
  hostPath:
    path: /var/log/audit/audit.log
    type: FileOrCreate

Setup PLG Stack

promtail:
  enabled: true
  extraVolumeMounts:
  - name: audit
    mountPath: /var/log/audit
    readOnly: true
  extraVolumes:
  - name: audit
    hostPath:
      path: /var/log/audit
      type: DirectoryOrCreate
loki:
  persistence:
    enabled: false

$ helm install loki-stack loki/loki-stack \
                               --create-namespace \
                               --namespace loki-stack \
                               -f overrides.yaml

helm install loki-grafana grafana/grafana \
                              --set persistence.enabled=false \
                              --namespace=loki-stack

Configure Promtail scrap configs

- job_name: system
   pipeline_stages:
   static_configs:
   - targets:
      - localhost
     labels:
      job: varlogs
      host: yourhost
      __path__: /var/log/audit/*.log