ELK Stack with Rails (Elasticsearch, Logstash, Kibana) on Ubuntu VPS

1. ELK.install

INSTALL JAVA
$ sudo apt-get update && sudo apt-get install default-jre

INSTALL ELASTIC SEARCH https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-repositories.html
$ wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
$ echo "deb https://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
$ sudo apt-get update && sudo apt-get install elasticsearch
$ sudo update-rc.d elasticsearch defaults 95 10
$ sudo service elasticsearch restart
$ sudo service elasticsearch status

INSTALL LOGSTASH https://www.elastic.co/guide/en/logstash/current/installing-logstash.html
$ wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
$ echo "deb https://packages.elastic.co/logstash/2.4/debian stable main" | sudo tee -a /etc/apt/sources.list
$ sudo apt-get update && sudo apt-get install logstash
$ sudo update-rc.d logstash defaults 95 10
$ sudo service logstash restart
$ sudo service logstash status

INSTALL KIBANA https://www.elastic.co/guide/en/kibana/current/setup-repositories.html
$ wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
$ echo "deb https://packages.elastic.co/kibana/4.6/debian stable main" | sudo tee -a /etc/apt/sources.list.d/kibana.list
$ sudo apt-get update && sudo apt-get install kibana
$ sudo update-rc.d kibana defaults 95 10
$ sudo service kibana restart
$ sudo service kibana status

2. Logstash.configure

$ cd /etc/logstash/conf.d
$ sudo touch 01-rails-<environment>.conf 02-sidekiq-<environment>.conf

PASTE AND EDIT EXAMPLE CONFIGS

RESTART LOGSTASH
$ sudo service logstash restart

3. 01-rails-<environment>.conf

input {
  file {
    path => "/home/deployer/apps/<app_name>/shared/log/logstash_staging.log"
    type => "rails"
    codec => json {
      charset => "UTF-8"
    }
  }
}
filter {
    ruby {
        init => "require 'digest/sha1'; require 'json'"
        code => "event['fingerprint'] = Digest::SHA1.hexdigest event.to_json"
    }
}
output {
  elasticsearch {
    hosts => ["localhost:9200"]
    document_id => "%{fingerprint}"
    index => "logstash-%{type}-%{+YYYY.MM.dd}"
  }
}

4. 02-sidekiq-<environment>.conf

input {
  file {
    path => "/home/deployer/apps/<app_name>/shared/log/logstash_sidekiq_staging.log"
    type => "sidekiq"
    codec => json {
      charset => "UTF-8"
    }
  }
}
filter {
    ruby {
        init => "require 'digest/sha1'; require 'json'"
        code => "event['fingerprint'] = Digest::SHA1.hexdigest event.to_json"
    }
}
output {
  elasticsearch {
    hosts => ["localhost:9200"]
    document_id => "%{fingerprint}"
    index => "logstash-%{type}-%{+YYYY.MM.dd}"
  }
}

5. Rails.configure

ADD TO GEMFILE
gem 'logstash-logger'

ADD TO <environment>.rb
  config.logstash.type = :file
  config.logstash.path = "log/logstash_#{Rails.env}.log"

IF U HAVE SIDEKIQ ADD TO ADD TO <environment>.rb
Sidekiq.configure_server do
  Sidekiq::Logging.logger = LogStashLogger.new(type: :file,
                                               path: "log/logstash_sidekiq_#{Rails.env}.log",
                                               sync: true)
  Sidekiq::Logging.logger.level = Logger::INFO
  Rails.logger = Sidekiq::Logging.logger
end

READ HOW TO ADD CUSTOM FIELDS HERE https://github.com/dwbutler/logstash-logger#custom-log-fields

6. Apache.configure

SETUP REVERSE PROXY FOR KIBANA ON APACHE

$ sudo apt-get install libapache2-mod-proxy-html
$ sudo a2enmod proxy
$ sudo a2enmod proxy_http

$ sudo nano /etc/apache2/sites-available/kibana.conf
PUT INTO FILE
<VirtualHost *:80>
ServerName kibana.mysite.com
ServerAdmin [email protected]
ProxyRequests Off
<Proxy *>
    Order Allow,Deny
    Allow from all
    AuthType Basic
    AuthName "Authenticated proxy"
    AuthUserFile /etc/apache2/kibana.htpwd
    Require valid-user
</Proxy>
  ProxyPass / http://127.0.0.1:5601/
  ProxyPassReverse / http://127.0.0.1:5601/
  ErrorLog ${APACHE_LOG_DIR}/kibana_error.log
  LogLevel warn
  CustomLog ${APACHE_LOG_DIR}/kibana_access.log combined
</VirtualHost>

GENERATE PASSWORD
$ sudo htpasswd -c /etc/apache2/kibana.htpwd <user_name>

ENABLE KIBANA
$ sudo a2ensite kibana.conf

RESTART APACHE
$ sudo service apache2 reload

7. Nginx.configure

SETUP REVERSE PROXY FOR KIBANA ON NGINX

$ sudo nano /etc/nginx/sites-enable/kibana.conf
PUT INTO FILE
server {
    listen 80;

    server_name example.com;

    auth_basic "Restricted Access";
    auth_basic_user_file /etc/nginx/kibana.htpwd;

    location / {
        proxy_pass http://localhost:5601/;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;        
    }
}

GENERATE PASSWORD
$ sudo htpasswd -c /etc/nginx/kibana.htpwd <user_name>

RESTART NGINX
$ sudo service nginx restart