Logstash setup

stash setup

From https://www.digitalocean.com/community/tutorials/how-to-install-elasticsearch-logstash-and-kibana-elk-stack-on-ubuntu-14-04

# ELK Server

## Install Java 8
sudo apt-get install software-properties-common
sudo add-apt-repository -y ppa:webupd8team/java
sudo apt-get update
sudo apt-get -y install oracle-java8-installer

## Elasticsearch

wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
sudo apt-get update
sudo apt-get -y install elasticsearch

sudo vi /etc/elasticsearch/elasticsearch.yml
network.host: localhost

sudo service elasticsearch restart
sudo update-rc.d elasticsearch defaults 95 10

## Kibana

sudo groupadd -g 102102 kibana
sudo useradd -u 102102 -g 102102 kibana

export KIBANA_VERSION=4.3.0
cd ~; wget https://download.elastic.co/kibana/kibana/kibana-${KIBANA_VERSION}-linux-x64.tar.gz
tar xvf kibana-*.tar.gz
rm kibana-*.tar.gz

vim ~/kibana-4*/config/kibana.yml
server.host: "localhost"

sudo mkdir -p /opt/kibana
sudo cp -R ~/kibana-4*/* /opt/kibana/
rm -rf ~/kibana-4.3.0-linux-x64/
sudo chown -R kibana: /opt/kibana

cd /etc/init.d && sudo curl -o kibana https://gist.githubusercontent.com/thisismitch/8b15ac909aed214ad04a/raw/fc5025c3fc499ad8262aff34ba7fde8c87ead7c0/kibana-4.x-init
cd /etc/default && sudo curl -o kibana https://gist.githubusercontent.com/thisismitch/8b15ac909aed214ad04a/raw/fc5025c3fc499ad8262aff34ba7fde8c87ead7c0/kibana-4.x-default

sudo chmod +x /etc/init.d/kibana
sudo update-rc.d kibana defaults 96 9
sudo service kibana start

## Nginx
sudo apt-get install nginx apache2-utils

sudo htpasswd -c /etc/nginx/htpasswd.users kibanaadmin
nL4lAcbT2X5u


$ curl -u kibanaadmin http://kibana.server1.com:8090
Enter host password for user 'kibanaadmin':
<script>var hashRoute = '/app/kibana';
var defaultRoute = '/app/kibana';

var hash = window.location.hash;
if (hash.length) {
  window.location = hashRoute + hash;
} else {
  window.location = defaultRoute;


Update firewall

sudo ufw allow 80/tcp
sudo ufw allow 5044
sudo ufw status verbose


# /etc/nginx/sites-available/default
server {
    listen 80;

    server_name example.com;

    auth_basic "Restricted Access";
    auth_basic_user_file /etc/nginx/htpasswd.users;

    location / {
        proxy_pass http://localhost:5601;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
    }
}


## Logstash

### Install

echo 'deb http://packages.elasticsearch.org/logstash/2.1/debian stable main' | sudo tee /etc/apt/sources.list.d/logstash.list
sudo apt-get update
sudo apt-get install logstash

### Configure

# /etc/logstash/conf.d/02-filebeat-input.conf
input {
  beats {
    port => 5044
    type => "logs"
    ssl => true
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}

# /etc/logstash/conf.d/10-syslog.conf
filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
}

# /etc/logstash/conf.d/30-elasticsearch-output.conf
output {
  elasticsearch { hosts => ["localhost:9200"] }
  stdout { codec => rubydebug }
}


sudo service logstash configtest
sudo service logstash restart
sudo update-rc.d logstash defaults 96 9


## Generate SSL Certificates

sudo mkdir -p /etc/pki/tls/certs
sudo mkdir /etc/pki/tls/private

cd /etc/pki/tls

# sudo openssl req -subj '/CN=kibana.server1.com/' -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt


www-deploy@server1:~$ sudo service filebeat restart
 * Restarting Sends log files to Logstash or directly to Elasticsearch. filebeat
2015/12/08 17:35:13.113349 transport.go:125: ERR SSL client failed to connect with: x509: certificate is valid for ${logstash_server_fqdn}, not kibana.server1.com

sudo service logstash configtest
sudo service logstash restart


next

2015/12/08 17:45:30.114761 transport.go:125: ERR SSL client failed to connect with: x509: certificate signed by unknown authority


# /etc/ssl/openssl.cnf
[ v3_ca ]
+ subjectAltName = IP: 78.129.243.45

sudo openssl req -config /etc/ssl/openssl.cnf -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt


Copy key from home

scp db2:/etc/pki/tls/certs/logstash-forwarder.crt .

Copy key from ELK server

scp logstash-forwarder.crt server1:/tmp
scp logstash-forwarder.crt server2:/tmp

## Filebeat

### Install

echo "deb https://packages.elastic.co/beats/apt stable main" | sudo tee -a /etc/apt/sources.list.d/beats.list
wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt-get update
sudo apt-get install filebeat

sudo mkdir -p /etc/pki/tls/certs
sudo mv /tmp/logstash-forwarder.crt /etc/pki/tls/certs/

### Configure

# /etc/filebeat/filebeat.yml


sudo service filebeat restart
sudo update-rc.d filebeat defaults 95 10


# /etc/hosts (server1)
-127.0.0.1       kibana.server1.com



## Better logging
on db2

sudo mkdir -p /opt/logstash/patterns
sudo chown logstash:logstash /opt/logstash/patterns

# /opt/logstash/patterns/nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent}

sudo chown logstash:logstash /opt/logstash/patterns/nginx