Attempt at manipulating Powershell data for GELF Graylog input, in reply to https://www.reddit.com/r/devops/comments/4f9e7x/scraping_apache_weblogs_and_shipping_them_to/

graylog-snippet.ps1

# This isn't tested much as I don't have the original data, but I've tested varios snippets, and they seem to work

# $GraylogUrl = "http://example.tld:12201/gelf"

$GraylogUdpPort = 12201
$GraylogUdpHost = "example.tld"
$UDPclient = new-Object System.Net.Sockets.UdpClient
$UDPclient.Connect($GraylogUdpHost, $GraylogUdpPort)
$Enc = [system.Text.Encoding]::UTF8

$dataColumn = $data.Columns | Select-Object -ExpandProperty ColumnName
$data | ForEach-Object {
    $Row = $_
    New-Object psobject -Property @{
        version = "1.1"
        host = "ApplianceThingy"
        short_message = ($dataColumn | ForEach-Object { $Row.$_.ToString() }) -join "`t"
        timestamp = [Math]::Floor([decimal](Get-Date(Get-Date -Date $Row.Timestamp).ToUniversalTime()-uformat "%s"))
        _Hash = $Row.Hash
        _Clientaddress = $Row.Clientaddress
        _Username = $Row.Username
        _Out = $Row.Out
        _In = $Row.In
        _Proxymode = $Row.Proxymode
        _Dnstime = $Row.Dnstime
        _Peertime = $Row.Peertime
        _Requesttime = $Row.Requesttime
        _Method = $Row.Method
        _Status = $Row.Status
        _Protocol = $Row.Protocol
        _Cachestatus = $Row.Cachestatus
        _URL = $Row.URL
    }
} | ForEach-Object {
    # This will be really, really slow because launching Invoke-RestMethod for every record, but it's a proof-of-concept
    # Invoke-RestMethod -Method Post -Uri $GraylogUrl -Body ($_ | ConvertTo-Json -Compress) -ContentType "application/json"

    # Since it's not re-creating an object maybe it will be faster
    $packet = $Enc.GetBytes( ( $_ | ConvertTo-Json -Compress ) )
    $UDPclient.Send($packet, $packet.Length) | Out-Null
}

local-test.ps1

# This is tested against a Graylog2 server and works!

# $GraylogUrl = "http://192.168.1.73:12201/gelf"

$GraylogUdpPort = 12201
$GraylogUdpHost = "192.168.1.73"
$UDPclient = new-Object System.Net.Sockets.UdpClient
$UDPclient.Connect($GraylogUdpHost, $GraylogUdpPort)
$Enc = [system.Text.Encoding]::UTF8


New-Object psobject -Property @{
    version = "1.1"
    host = $env:COMPUTERNAME
    short_message = "Graylog Powershell testing"
    timestamp = [Math]::Floor([decimal](Get-Date(Get-Date).ToUniversalTime()-uformat "%s"))
} | ForEach-Object {
    # Invoke-RestMethod -Method Post -Uri $GraylogUrl -Body ($_ | ConvertTo-Json -Compress) -ContentType "application/json"
    $packet = $Enc.GetBytes( ( $_ | ConvertTo-Json -Compress ) )
    $UDPclient.Send($packet, $packet.Length)
}

ngnix-clf-test.ps1

$LogFile = "C:\temp\deleteme\access.log.1"

$GraylogUdpPort = 12201
$GraylogUdpHost = "192.168.1.73"
$UDPclient = new-Object System.Net.Sockets.UdpClient
$UDPclient.Connect($GraylogUdpHost, $GraylogUdpPort)
$Enc = [system.Text.Encoding]::UTF8

Get-Content $LogFile |
    # Select-Object -First 4 |
    ForEach-Object {

        $_ -match '([^ ]*) ([^ ]*) ([^ ]*) \[([^]]*)\] "([^"]*)" ([^ ]*) ([^ ]*) "([^"]*)" "([^"]*)" "([^"]*)" ([^ ]*) ([^ ]*) ([^ ]*)' | Out-Null
        if ($Matches) {
            $Time = [DateTime]::ParseExact($Matches[4], "dd/MMM/yyyy:HH:mm:ss zzz", [System.Globalization.CultureInfo]::InvariantCulture)
            #$Matches
            New-Object psobject -Property @{
                version = "1.1"
                host = "192.168.1.111"
                short_message = $Matches[0]
                timestamp = [Math]::Floor([decimal](Get-Date($Time).ToUniversalTime()-uformat "%s"))
                _vhost = $Matches[10]
                _ipaddress = $Matches[1]
                # _ignore = $Matches[2]
                # _user = $Matches[3]
                _date = $Matches[4]
                _request = $Matches[5]
                _status = $Matches[6]
                _size = $Matches[7]
                _referer = $Matches[8]
                _agent = $Matches[9]
                _phpsessid = $Matches[11]
                _jsessionid = $Matches[12]
                _aspsessionid = $Matches[13]
            }
        }
    }  | ForEach-Object {
        $packet = $Enc.GetBytes( ( $_ | ConvertTo-Json -Compress ) )
        $UDPclient.Send($packet, $packet.Length) | Out-Null
    }

# Time taken against a file with 46,711 log entries
#
# Measure-Command { .\ngnix-clf-test.ps1 }
# 
# 
# Days              : 0
# Hours             : 0
# Minutes           : 1
# Seconds           : 5
# Milliseconds      : 378
# Ticks             : 653787858
# TotalDays         : 0.000756698909722222
# TotalHours        : 0.0181607738333333
# TotalMinutes      : 1.08964643
# TotalSeconds      : 65.3787858
# TotalMilliseconds : 65378.7858