nginx_pipeline.yaml

id: nginx
metric_id: nginx
facets:
  -
    name: Status Code
    source: log
    path: http.status_code
    groups:
      - Web Access
  -
    name: Method
    source: log
    path: http.method
    groups:
      - Web Access
  -
    name: Client IP
    source: log
    path: network.client.ip
    groups:
      - Web Access
  -
    name: Referer
    source: log
    path: http.referer
    groups:
      - Web Access
  -
    name: URL Path
    source: log
    path: http.url_details.path
    groups:
      - Web Access
  -
    name: Browser
    source: log
    path: http.useragent_details.browser.family
    groups:
      - Web Access
  -
    name: Device
    source: log
    path: http.useragent_details.device.family
    groups:
      - Web Access
  -
    name: OS
    source: log
    path: http.useragent_details.os.family
    groups:
      - Web Access
pipeline:
  type: pipeline
  name: Nginx
  enabled: true
  filter:
    query: 'source:nginx'
  processors:
    -
      type: grok-parser
      name: Parsing Nginx logs
      enabled: true
      source: message
      samples:
        - '127.0.0.1 - frank [13/Jul/2016:10:55:36 +0000] "GET /apache_pb.gif HTTP/1.0" 200 2326'
        - '172.17.0.1 - - [06/Jan/2017:16:16:37 +0000] "GET /datadoghq/company?test=var1%20Pl HTTP/1.1" 200 612 "http://www.perdu.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" "-"'
        - '2017/09/26 14:36:50 [error] 8409#8409: *317058 "/usr/share/nginx/html/sql/sql-admin/index.html" is not found (2: No such file or directory), client: 217.92.148.44, server: localhost, request: "HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1", host: "174.138.82.103"'
      grok:
        supportRules: |
          _auth %{notSpace:http.auth:nullIf("-")}
          _bytes_written %{integer:network.bytes_written}
          _client_ip %{ipOrHost:network.client.ip}
          _version HTTP\/%{regex("\\d+\\.\\d+"):http.version}
          _url %{notSpace:http.url}
          _ident %{notSpace:http.ident:nullIf("-")}
          _user_agent %{regex("[^\\\"]*"):http.useragent}
          _referer %{notSpace:http.referer}
          _status_code %{integer:http.status_code}
          _method %{word:http.method}
          _date_access %{date("dd/MMM/yyyy:HH:mm:ss Z"):date_access}
          _x_forwarded_for %{regex("[^\\\"]*"):http._x_forwarded_for:nullIf("-")}
        matchRules: |
          access.common %{_client_ip} %{_ident} %{_auth} \[%{_date_access}\] "(?>%{_method} |)%{_url}(?> %{_version}|)" %{_status_code} (?>%{_bytes_written}|-)
          access.combined %{access.common} (%{number:duration:scale(1000000000)} )?"%{_referer}" "%{_user_agent}"( "%{_x_forwarded_for}")?.*
          error.format %{date("yyyy/MM/dd HH:mm:ss"):date_access} \[%{word:level}\] %{data:error.message}(, %{data::keyvalue(": ",",")})?
    -
      type: attribute-remapper
      name: 'Remap client to client ip'
      enabled: true
      sources:
        - client
      target: 'network.client.ip'
      preserveSource: false
      overrideOnConflict: false
    -
      type: grok-parser
      name: Parsing Nginx Error log requests
      enabled: true
      source: request
      grok:
        supportRules: |
          _method %{word:http.method}
          _url %{notSpace:http.url}
          _version HTTP\/%{regex("\\d+\\.\\d+"):http.version}
        matchRules: |
          request_parsing (?>%{_method} |)%{_url}(?> %{_version}|)
    -
      type: url-parser
      name: ''
      enabled: true
      sources:
        - http.url
      target: http.url_details
    -
      type: user-agent-parser
      name: ''
      enabled: true
      sources:
        - http.useragent
      target: http.useragent_details
      encoded: false
    -
      type: date-remapper
      name: Define Date_access as the official timestamp of the log
      enabled: true
      sources:
        - date_access
    -
      type: category-processor
      name: Categorise status code
      enabled: true
      categories:
        - filter:
            query: '@http.status_code:[200 TO 299]'
          name: OK
        - filter:
            query: '@http.status_code:[300 TO 399]'
          name: notice
        - filter:
            query: '@http.status_code:[400 TO 499]'
          name: warning
        - filter:
            query: '@http.status_code:[500 TO 599]'
          name: error
      target: http.status_category
    -
      type: status-remapper
      name: Set the log status based on the status code value
      enabled: true
      sources:
        - http.status_category
        - level