Last active
July 22, 2019 14:44
-
-
Save PhantomRay/e015086a307fcb01cbd4 to your computer and use it in GitHub Desktop.
goaccess log/time/date format for AWS S3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# AWS S3 Log Format | |
# config file | |
date-format %d/%b/%Y | |
time-format %T | |
log-format %^ %^ [%d:%t %^] %h %^ %^ %^ %^ "%^ %r %^" %s %^ %b %^ %^ %^ "%^" "%u" %^ | |
# example log | |
# | |
1d82a3126606ef56450b5ea405d549ce4e5a5c29ccd240012c6032c9d7dc454f random [09/Aug/2015:06:20:46 +0000] 1.129.96.209 - 494F7E82C4F16813 REST.HEAD.OBJECT files/24/2015-08/e86c1ab2-563b-40f8-aa61-3eecefe30867_f.png "HEAD /app/files/24/2015-08/e86c1ab2-563b-40f8-aa61-3eecefe30867_f.png HTTP/1.1" 200 - - 8610 19 - "-" "someapp/3.0.0 (Windows_NT; x64 6.1.7601; Intel R Core TM i3-4160 CPU @ 3.60GHz) nodejs/v0.10.29 device/12345" - |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env sh | |
# report for IIS | |
# MUST have execute access to goiisformat.sh | |
# goaccess -f cec1.log --log-format "$(cat cec1.log | ./goiisformat.sh)" --date-format '%Y-%m-%d' --time-format '%H:%M:%S' | |
while read line; do | |
if [[ $line == \#Fields:* ]]; then | |
line=${line/\#Fields: /} | |
line=${line/date/%d} | |
line=${line/time/%t} | |
line=${line/s-sitename/%^} | |
line=${line/s-computername/%^} | |
line=${line/s-ip/%^} | |
line=${line/cs-method/%m} | |
line=${line/cs-uri-stem/%U} | |
line=${line/cs-uri-query/%^} | |
line=${line/s-port/%^} | |
line=${line/cs-username/%^} | |
line=${line/c-ip/%h} | |
line=${line/cs-version/%H} | |
line=${line/cs(User-Agent)/%u} | |
line=${line/cs(Cookie)/%^} | |
line=${line/cs(Referer)/%R} | |
line=${line/cs-host/%^} | |
line=${line/sc-status/%s} | |
line=${line/sc-substatus/%^} | |
line=${line/sc-win32-status/%^} | |
line=${line/sc-bytes/%b} | |
line=${line/cs-bytes/%^} | |
line=${line/time-taken/%L} | |
echo $line | |
exit; | |
fi | |
done | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// sync s3 log to local | |
aws s3 sync s3:///engagis-logging/ezeimpress . --exclude "*" --include="2016-04*" | |
// generate report based on file patten | |
find /Users/ray/Desktop/s3log -name "2016-04*" | xargs more | goaccess -p ~/.goaccess | |
// generate html | |
find /Users/ray/Desktop/s3log -name "2016-04*" | xargs more | goaccess -p ~/.goaccess > output.html |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment