asigatchov

upstream elastic {
  server elasticsearch:9200;
}
client_max_body_size 1G;
client_body_buffer_size 200M;
server {
  listen *:9200;
  server_name _;
  location / {

asigatchov

[nginx_access_logs]
type = "LogstreamerInput"
splitter = "TokenSplitter"
decoder = "nginx_access_decoder"
log_directory = "/opt/nginx/logs"
file_match = 'access\.log'


[nginx_access_decoder]
type = "SandboxDecoder"

asigatchov

<source>
  @type tail
  format /(?<schema>https?)://(?<domain>[^ ]*) (?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)") (?<message>.*) (?<resptime>\d+\.\d+)$/
  time_format %d/%b/%Y:%H:%M:%S %z
  types remote:string,host:string,user:string,time:time,method:string,path:string,code:integer,referer:string,agent:string,message:string,size:integer,resptime:float
  path /tmp/access.log
  pos_file /tmp/access_tail_pos.log
  tag es.demo.access
  read_lines_limit 1000
</source>

asigatchov

<match *.**>
    @type elasticsearch_dynamic
    buffer_type memory
    buffer_chunk_limit 100m
    buffer_queue_limit 128
     logstash_format true
    logstash_prefix   ${tag_parts[1]}
    include_tag_key true
    tag_key @log_name
    host 127.0.0.1

asigatchov

<filter es.**>
  @type record_transformer
  enable_ruby
  auto_typecast true
  <record>
    hostname ${hostname}
    bitps ${record["size"] * 8 / (record["resptime"] + 0.001)}
  </record>
</filter>

asigatchov

<source>
  @type tail
  format /(?<schema>https?)://(?<domain>[^ ]*) (?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)") (?<message>.*) (?<resptime>\d+\.\d+)$/
  time_format %d/%b/%Y:%H:%M:%S %z
  types remote:string,host:string,user:string,time:time,method:string,path:string,code:integer,referer:string,agent:string,message:string,size:integer,resptime:float

  path /data/access.log
  pos_file /tmp/access_tail_pos.log
  tag es.demo.access
  read_lines_limit 1000

asigatchov

apt-get install apt-transport-https ca-certificates &&  apt-key adv  --keyserver hkp://ha.pool.sks-keyservers.net:80  --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main"  > /etc/apt/sources.list.d/docke.list
apt-get update && apt-get install docker-engine  -y
curl -L "https://github.com/docker/compose/releases/download/1.9.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 
chmod +x /usr/local/bin/docker-compose 
mkdir /opt/efk/ 
curl "https://gist.githubusercontent.com/asigatchov/417ac1dad9b57ae2ad25d62e87604f28/raw/7cbb959d3b459f0fc51ae019f4fc42ff5fcf69a6/docker-compose-efk-16gb.yml" > /opt/efk/docker-compose.yml 
dd if=/dev/zero of=/swapfile bs=256M count=12 
chown root:root /swapfile 
chmod 0600 /swapfile 

asigatchov

version: '2'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:5.1.1
    container_name: elasticsearch
    environment:
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms7g -Xmx7g"
      - xpack.security.enabled=false
      - http.host=0.0.0.0

asigatchov

SELECT 
    floor(avg(resp_time), 2) AS avg_time, 
    ip
FROM access_log 
WHERE event_date = '2016-11-01'
GROUP BY ip
ORDER BY avg_time DESC
LIMIT 10

asigatchov

https://mysite.ru 5.255.253.63 - - [04/Nov/2016:16:31:28 +0300] "GET /pechene_s_m?page=563 HTTP/1.1" 200 25351 "-" "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" "-" "-" 0.213