Grok pattern and Setup for Datadog log parsing of AWS CloudFront Access logs

cfn-template-snippet.yaml

  CloudFrontLogDestination:
    Type: AWS::Logs::DeliveryDestination
    Properties:
      Name: !Sub "CloudFrontS3Destination-${MyCloudFrontDistribution}"
      DestinationResourceArn: !Sub "arn:${AWS::Partition}:s3:::<insert your bucket name or ref here>"
      OutputFormat: "plain"  # has to be plain as json can't be handled by DD due to the brackets in the field naming

  CloudFrontLogSource:
    Type: AWS::Logs::DeliverySource
    Properties:
      Name: !Sub "CloudFrontDistributionSource-${MyCloudFrontDistribution}"
      LogType: "ACCESS_LOGS"
      ResourceArn: !Sub "arn:aws:cloudfront::${AWS::AccountId}:distribution/${MyCloudFrontDistribution}"

  CloudFrontLogDelivery:
    Type: AWS::Logs::Delivery
    Properties:
      DeliverySourceName: !Ref CloudFrontLogSource
      DeliveryDestinationArn: !GetAtt CloudFrontLogDestination.Arn
      RecordFields: # full list is at https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/standard-logs-reference.html#BasicDistributionFileFormat
        - "DistributionId"
        - "date"
        - "time"
        - "x-edge-location"
        - "sc-bytes"
        - "c-ip"
        - "cs-method"
        - "cs(Host)"
        - "cs-uri-stem"
        - "sc-status"
        - "cs(Referer)"
        - "cs(User-Agent)"
        - "cs-uri-query"
        - "cs(Cookie)"
        - "x-edge-result-type"
        - "x-edge-request-id"
        - "x-host-header"
        - "cs-protocol"
        - "cs-bytes"
        - "time-taken"
        - "x-forwarded-for"
        - "ssl-protocol"
        - "ssl-cipher"
        - "x-edge-response-result-type"
        - "cs-protocol-version"
        - "fle-status"
        - "fle-encrypted-fields"
        - "c-port"
        - "time-to-first-byte"
        - "x-edge-detailed-result-type"
        - "sc-content-type"
        - "sc-content-len"
        - "sc-range-start"
        - "sc-range-end"
        - "timestamp(ms)"
        - "origin-fbl"
        - "origin-lbl"
        - "asn"
        - "c-country"
        - "cache-behavior-path-pattern"
        # - timestamp can't be included otherwise DD trow away the log 
      S3SuffixPath: "{DistributionId}/{yyyy}/{MM}/{dd}/"

grok

http %{notSpace:DistributionId}\t%{date("yyyy-MM-dd"):date}\t%{date("HH:mm:ss"):time}\t%{notSpace:cloudfront.edge-location}\t%{integer:sc-bytes}\t%{ipv4:network.client.ip}\t%{word:http.method}\t%{notSpace:http.url_details.host}\t%{notSpace:http.url_details.path}\t%{integer:http.status_code}\t%{data:http.referer}\t%{data:http.useragent}\t%{data:http.url_details.queryString}\t%{data:cloudfront.cookie}\t%{notSpace:cloudfront.edge-result-type}\t%{notSpace:cloudfront.x-edge-request-id}\t%{data:http.ident}\t%{notSpace:http.url_details.scheme}\t%{number:cs_bytes}\t%{number:http.time_taken}\t%{notSpace:x-forwarded-for}\t%{notSpace:http.ssl.protocol}\t%{notSpace:http.ssl.cipher}\t%{notSpace:x-edge-response-result-type}\tHTTP/%{notSpace:http.version}\t%{data:fle-status}\t%{data:cloudfront.encrypted-fields}\t%{integer:network.client.port}\t%{number:http.time_to_first_byte}\t%{notSpace:x-edge-detailed-result-type}\t%{data:http.resource.content_type}\t%{integer:sc-content-len}\t%{data:sc-range-start}\t%{data:sc-range-end}\t%{integer:date_access}\t%{data:origin-fbl}\t%{data:origin-lbl}\t%{integer:network.client.asn}\t%{notSpace:network.client.country}\t%{notSpace:cloudfront.cache-behavior-path-pattern}
logheader DistributionId date time x-edge-location sc-bytes c-ip cs-method cs\(Host\) cs-uri-stem sc-status cs\(Referer\) cs\(User-Agent\) cs-uri-query cs\(Cookie\) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end timestamp\(ms\) origin-fbl origin-lbl asn c-country cache-behavior-path-pattern
websocket %{notSpace:DistributionId}\t%{date("yyyy-MM-dd"):date}\t%{date("HH:mm:ss"):time}\t%{notSpace:cloudfront.edge-location}\t%{integer:sc-bytes}\t%{ipv4:network.client.ip}\t%{word:http.method}\t%{notSpace:http.url_details.host}\t%{notSpace:http.url_details.path}\t%{integer:http.status_code}\t%{data:http.referer}\t%{data:http.useragent}\t%{data:http.url_details.queryString}\t%{data:cloudfront.cookie}\t%{notSpace:cloudfront.edge-result-type}\t%{notSpace:cloudfront.x-edge-request-id}\t%{data:http.ident}\t%{notSpace:http.url_details.scheme}\t%{data:cs_bytes}\t%{data:http.time_taken}\t%{notSpace:x-forwarded-for}\t%{notSpace:http.ssl.protocol}\t%{notSpace:http.ssl.cipher}\t%{notSpace:x-edge-response-result-type}\tHTTP/%{notSpace:http.version}\t%{data:fle-status}\t%{data:cloudfront.encrypted-fields}\t%{data:network.client.port}\t%{data:http.time_to_first_byte}\t%{notSpace:x-edge-detailed-result-type}\t%{data:http.resource.content_type}\t%{data:sc-content-len}\t%{data:sc-range-start}\t%{data:sc-range-end}\t%{integer:date_access}\t%{data:origin-fbl}\t%{data:origin-lbl}\t%{integer:network.client.asn}\t%{notSpace:network.client.country}\t%{notSpace:cloudfront.cache-behavior-path-pattern}