Nginx + Fluentd, log to prometheus metrics

compose.yml

services:
  nginx:
    image: nginx:1.25.4-alpine-slim
    restart: always
    container_name: "nginx"
    ports:
      - "127.0.0.1:8080:80"
    volumes:
      - ./config/nginx/default.conf:/etc/nginx/conf.d/default.conf
      - ./config/nginx/nginx.conf:/etc/nginx/nginx.conf
      - ./config/nginx/.htpasswd:/etc/nginx/.htpasswd
      - ./host.access.log:/var/log/nginx/host.access.log

  fluentd:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: fluentd
    volumes:
      - ./host.access.log:/var/log/nginx/access.log:ro
      - ./config/fluent/fluent.conf:/fluentd/etc/fluent.conf
    ports:
      - "127.0.0.1:24231:24231"
    logging:
      options:
        max-size: "512m"

  nginx_exporter:
    image: nginx/nginx-prometheus-exporter:1.1.0
    container_name: nginx_exporter
    command:
      - "--nginx.scrape-uri=http://nginx:80/basic_status"
      - "--web.config.file=/web.config.yml"
    volumes:
      - ./config/nginx_exporter/config.yml:/web.config.yml:ro
    ports:
      - "127.0.0.1:9113:9113"
    logging:
      options:
        max-size: "512m"

# http://localhost:9786/metrics

default.conf

limit_conn_zone $server_name zone=servconn:5m;
limit_req_zone $binary_remote_addr zone=onerps:5m rate=150r/s;
# proxy_cache_path /data/nginx/cache keys_zone=mycache:10m;

server {
    listen       80;
    listen  [::]:80;
    server_name  localhost;

    proxy_connect_timeout  120;
    proxy_send_timeout  120;
    proxy_read_timeout  120;

    #access_log  /var/log/nginx/host.access.log  main;

    location / {
        access_log  /var/log/nginx/host.access.log custom;
        limit_rate 2M;
        limit_conn servconn 1024;
        limit_req zone=onerps burst=75;

        location ~* \.(js|css|woff|gif|png|svg|ico)$ {
            # proxy_cache mycache;
            # add_header X-Cache $upstream_cache_status;
            expires 60s;
        }

        root   /usr/share/nginx/html;
        index  index.html index.htm;
    }

    #error_page  404              /404.html;

    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

    location = /basic_status {
        stub_status;
        satisfy all;
        auth_basic  "Login";
        auth_basic_user_file    /etc/nginx/.htpasswd;
    }

}

Dockerfile

FROM fluent/fluentd:v1.16-1

USER root

RUN apk add --no-cache --update --virtual .build-deps \
        sudo build-base ruby-dev \
 && sudo gem install fluent-plugin-prometheus \
 && sudo gem sources --clear-all \
 && apk del .build-deps \
 && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

# COPY fluent.conf /fluentd/etc/
# COPY entrypoint.sh /bin/

USER fluent

fluent.conf

<source>
    @type prometheus_tail_monitor
</source>

<source>
  @type prometheus
</source>

<source>
    @type tail
    <parse>
    @type regexp
    expression /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] \"(?<method>\w+)(?:\s+(?<path>[^\"]*?)(?:\s+\S*)?)?\" (?<status_code>[^ ]*) (?<size>[^ ]*)(?:\s"(?<referer>[^\"]*)") "(?<agent>[^\"]*)" (?<urt>[^ ]*)$/
        time_format %d/%b/%Y:%H:%M:%S %z
        keep_time_key true
        types size:integer,reqtime:float,uct:float,uht:float,urt:float
    </parse>
    tag nginx
    path /var/log/nginx/access.log
    pos_file /tmp/fluent_nginx.pos
</source>
<filter nginx>
    @type prometheus

  <metric>
    name nginx_size_bytes_total
    type counter
    desc nginx bytes sent
    key size
  </metric>

  <metric>
    name nginx_request_status_code_total
    type counter
    desc nginx request status code
    <labels>
      method ${method}
      path ${path}
      status_code ${status_code}
    </labels>
  </metric>

  <metric>
    name nginx_http_request_duration_seconds
    type histogram
    desc Histogram of the total time spent on receiving the response from the upstream server.
    key urt
    <labels>
      method ${method}
      path ${path}
      status_code ${status_code}
    </labels>
  </metric>

</filter>

nginx.conf

user  nginx;
worker_processes  2;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;


events {
    worker_connections  512;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format custom '$remote_addr - $remote_user [$time_local] '
                      '"$request" $status $body_bytes_sent '
                      '"$http_referer" "$http_user_agent" '
                      '$upstream_response_time';

    access_log  /var/log/nginx/access.log  custom;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}