Log extract

log extract

awk '{
    ip = $2
    url = gensub(/.*"(https?:\/\/[^"]+)".*/, "\\1", "g", $0)
    status = $7
    user_agent = gensub(/.*"[^"]*"\s+"([^"]+)".*/, "\\1", "g", $0)

    # Print fields separated by pipes for easy re-parsing
    print ip "|" url "|" status "|" user_agent
}' esg.access.log-20241016

awk '{
    # Extract IP (third field)
    ip = $3;

    # Extract URL (first quoted string)
    match($0, /"https?:\/\/[^"]+"/);
    url = substr($0, RSTART+1, RLENGTH-2);

    # Extract Status Code (number following URL)
    match($0, /"https?:\/\/[^"]+" [0-9]+/);
    status = substr($0, RSTART + RLENGTH - 3, 3);

    # Extract User Agent (last quoted string)
    match($0, /"[^"]+"$/);
    user_agent = substr($0, RSTART+1, RLENGTH-2);

    # Print fields separated by pipes for easy re-parsing
    print ip "|" url "|" status "|" user_agent;
}' esg.access.log-20241016