Configuration files for NXLog on Windows ---> NXLog on Ubuntu --> LogStash

! Windows NXLog | Linux NXLog | LogStash

Covers configuration for NXLog installed on Windows, transfering to NXLog installed on Linux to LogStash.

NXLog is used for IIS logs. (http://nxlog-ce.sourceforge.net/)

Snare is used for Windows Event Logs. (http://www.intersectalliance.com/projects/SnareWindows/index.html#Download)

Logstash.ini

input {
  # Accept the Gelf Input Stream - Windows Event Logs
  gelf {
    debug => true
    type => "gelf-udp"
    port => 12201
    tags => "winlog"
  }

  # Accept the TCP Stream for the IIS Logs. Has to be json_lines as thats how NXLog streams in
  tcp {
    type => "iis"
    port => 5151
    codec => "json_lines"
    #Disable timeouts as logstash may drop events when timing out
    data_timeout => -1
  }
}
filter {
        # Pickup the date from Windows Event Logs
        date {
                type => "gelf-udp"
                match => [ "id6", "EEE MMM dd HH:mm:ss YYYY" ]
        }
        
        # Pickup the date from Windows Event Logs
        date {
                type => "iis"
                match => [ "EventTime ", "yyyy-MM-dd HH:mm:ss" ]
        }
        
        # Lookup the IP's from IIS to grab their geo details
       geoip {
                type => "iis"
                add_tag => [ "geoip" ]
                source => "c-ip"
        }

        # Run all the mutates on the gelf variables
        if [type] == "gelf-udp" {
          mutate {
            rename => [ "id2", "z_type" ]
            rename => [ "id4", "@win.sourcename" ]
            rename => [ "id7", "@win.eventid" ]
            rename => [ "id8", "@win.eventsourcename" ]
            rename => [ "id9", "@win.accountname" ]
            rename => [ "id10", "@win.sidtype" ]
            rename => [ "id11", "@win.eventtype" ]
            rename => [ "id12", "@hostname" ]
            rename => [ "id13", "@win.category" ]
            rename => [ "id15", "@message" ]
            rename => [ "EventReceivedTime", "z_eventreceived" ]
            rename => [ "MessageSourceAddress", "z_messagesourceaddress" ]
            rename => [ "SourceModuleName", "z_sourcemodulename" ]
            rename => [ "SourceModuleType", "z_sourcemoduletype" ]
            remove_field => [ "message", "full_message", "short_message", "host", "id1", "id3", "id5", "id15", "id16", "id6", "@version", "level", "version" ]
            add_tag => "remote_syslog"
            }
        }
}

output {
  # Output to the console for debugging purposes
  stdout {
    debug => true
    codec => "json"
    }

  # Send output to ElasticSearch
  elasticsearch {
    host => "127.0.0.1"
    cluster => "logstash3"
  }
}

ubuntu-nxlog-server.ini

########################################
# Global directives                    #
########################################
User nxlog
Group nxlog
LogFile /var/log/nxlog/nxlog.log
LogLevel INFO

########################################
# Modules                              #
########################################

# NXlog JSON extension activation (needed to forward messages to Logstash)
<Extension json>
    Module      xm_json
</Extension>

# Grab Windows Event Logs From Snare
<Extension snare>
    Module  xm_csv
    Fields   $id1, $id2, $id3, $id4, $id5, $id6, $id7, $id8, $id9, $id10, $id11, $id12, $id13, $id14, $id15, $id16
    Delimiter   \t
</Extension>

# Convert The Event Logs
<Extension charconv>
    Module  xm_charconv
    AutodetectCharsets utf-8, utf-16, utf-32, iso8859-2, windows-1252
</Extension>

# Load The Gelf Exension
<Extension gelf>
    Module  xm_gelf
</Extension>

# Map the fields from the IIS log file (you can open the IIS log file to see the header and know what fields to map
<Extension w3c>
    Module      xm_csv
    Fields      $date, $time, $s-ip, $cs-method, $cs-uri-stem, $cs-uri-query, $s-port, $cs-username, $c-ip, $cs(User-Agent), $sc-status, $sc-substatus, $sc-win32-status, $time-taken
    FieldTypes  string, string, string, string, string, string, integer, string, string, string, integer, integer, integer, integer
    Delimiter   ' '
</Extension>

########################################
# Inputs                               #
########################################

# Input From Snare
<Input in_snare>
    Module      im_udp
    Host        0.0.0.0
    Port        6161
    Exec   convert_fields("windows-1252", "utf-8");
    Exec        snare->parse_csv(); to_json();
</Input>

#Accept IIS logs via tcp port 5141, drop comment lines, join the date+time fields into an EventTime field, convert to json
<Input in-iis>
    Module      im_tcp
    Host        0.0.0.0
    Port        5141
    InputType   LineBased
    Exec        if $raw_event =~ /^#/ drop();                    \
                else                                             \
                {                                                \
                    w3c->parse_csv();                            \
                    $EventTime = parsedate($date + " " + $time); \
                    to_json ();                                  \
}
</Input>

########################################
# Outputs                              #
########################################

# Output To LogStash Gelf Module
<Output out_gelf>
    Module  om_udp
    Host    localhost
    Port    12201
    OutputType  GELF
</Output>

# Output to a json file - only need to enable when you want to test
<Output iisfileout>
  #output iis in json to a temporary file
    Module      om_file
    File        "/tmp/iis.json"
</Output>

# TCP Output to LogStash
<Output iis>
    Module      om_tcp
    Host        localhost
    Port        5151
</Output>

########################################
# Routes                               #
########################################

# Windows Event Logs out to logstash via gelf
<Route _snare>
    Path  in_snare => out_gelf
</Route>

# IIS logs out to logstash via TCP
<Route out_iis>
    Path         in-iis => iis
</Route>

windows-nxlog-client.ini

# Define the path where NX Log is installed
define ROOT C:\Program Files (x86)\nxlog

# Standard config for logging etc
Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log

# Load the json extension
<Extension json>
    Module      xm_json
</Extension>

# Select the input folder where logs will be scanned
<Input w3c>
  Module	im_file
  File  "C:\inetpub\logs\LogFiles\W3SVC1\u_ex*.log"
  ReadFromLast True
  SavePos True
	
  #Drop comments from the log file
  Exec	if $raw_event =~ /^#/ drop();
</Input>

# Send the read log lines out to nxlog server
<Output out-5141>
  #Send to central nxlog listener on tcp port 5141, change host address
    Module      om_tcp
    Host        10.0.0.111
    Port        5141
    OutputType	LineBased
</Output>

# Build the route from nxlog on Windows to nxlog on server
<Route 1>
    Path        w3c => out-5141
</Route>