Logstash Lab Setup

elasticsearch.yml

# /etc/elasticsearch/elasticsearch.yml
#
# Remember the cluster name if you ever add extra nodes
cluster.name: logstash
# If you leave node.name blank, it'll autogenerate a node name each time you start ES, picking from 3000 marvel comicbook heroes. 
node.name: "log-indexer"
node.master: true
node.data: true

# Set the bind address specifically (IPv4 or IPv6)
# Bind To All The Things. 
# Change this if you want it to bind to a specific address. 
network.bind_host: 0.0.0.0

# Path to directory containing configuration (this file and logging.yml):
path.conf: /etc/elasticsearch

# Path to log files:
path.logs: /var/log/elasticsearch

# The number of shards determines how fast data is INDEXED (loaded in). If you're testing, you don't need many shards. 
index.number_of_shards: 2
# If you're testing, you won't really need replicas. You can add some later at no cost. Just change this value and restart ES
index.number_of_replicas: 0

# Allocate all memory at startup so Java doesn't swap. If Java starts swapping, ES performance nosedives. 
bootstrap.mlockall: true

input_file_syslog.conf

# /etc/logstash/conf.d/input_file_syslog.conf
#
input {
  file {
    path => [ "/var/log/syslog", "/var/log/messages" ]
    type => "syslog"
  }
}

input_redis.conf

# /etc/logstash/conf.d/input_redis.conf
#
input {
  redis {
    data_type => "list"
    host => "127.0.0.1"
    key => "logstash"
    type => "redis-input"
  }
}

output_elasticsearch.conf

# /etc/logstash/conf.d/output_elasticsearch.conf
#
output {
  elasticsearch_http {
    host => "localhost"
    manage_template => true
    template_overwrite => true
  }
}