elk elasticsearch logstash kinaba filebeat nginx basic_auth

elk-6.4.md

安装yum repo 然后通过repo安装

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

vi /etc/yum.repos.d/elasticsearch.repo

# 以下内容

[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md



# 安装
yum install elasticsearch   / kibana / logstash ...

6.4 的不同

filebeat

配置文件中 filebeat.prospectors: 变为 filebeat.inputs

elk.md

下载 rpm 包

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.0.rpm
wget https://artifacts.elastic.co/downloads/logstash/logstash-5.6.0.rpm
wget https://artifacts.elastic.co/downloads/kibana/kibana-5.6.0-x86_64.rpm
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.2.2-x86_64.rpm

安装

rpm -ivh elasticsearch-5.6.0.rpm
rpm -ivh logstash-5.6.0.rpm
rpm -ivh kibana-5.6.0-x86_64.rpm
rpm -ivh filebeat-6.2.2-x86_64.rpm # filebeat 收集日志

配置

elasticsearch

/etc/elasticsearch/elasticsearch.yml

cluster.name: erp #指定个集群名字，等下会用到
node.name: node0 #节点名称也指定下
path.data: /opt/data #指定ela生成数据的存放路径（注意读写权限问题）
path.logs: /opt/data/logs #指定ela执行日志的存放路径，好排查问题
network.host: *.*.*.* #指定监听地址
http.port: 9200 #指定端口

# 设置集群节点发现与沟通
transport.host: "**.**.**.**"
discovery.zen.ping.unicast.hosts: ["**.**.**.**"]
discovery.zen.minimum_master_nodes: 2    #(N/2)+1

启动命令

service elasticsearch start/stop/restart

logstash

logstash 单独生成启动文件

/usr/share/logstash/bin/system-install /etc/logstash/startup.options sysv

启动命令

service logstash start/stop/restart

日志配置

1. /etc/logstash/conf.d/ 下添加单独业务日志配置，如 nginx-access.conf somebusserv.conf

# 基本格式 输入过滤输出
input {
}
filter {
}
output {
}

# 获取本机的文件 打到本机 es
    input {
        file {
            path => "/www/busservlib/Log/**/*.log"
            start_position => "beginning"
            sincedb_path => "/www/busservlib/elk_progress"
            type => "file"
            # 多行合并
            codec=> multiline {
                pattern => "^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\|"
                negate => true
                what => "previous"
                # 多行日志 可以避免不处理最后一条日志
                auto_flush_interval => 1
            }
        }
    }

    filter {
        if [type] == "file" {
            grok {
                match => {
                    "message" => "(?m)(?<log_time>[^\|]+)\|(?<function>[^\|]+)\|(?<log_level>[^\|]+)\|(?<title>[^\|]+)(\|(?<description>.+))?"
                }
            }
        }
    }

    output {
        if [type] == "file" {
            elasticsearch {
                action => "index"
                hosts  => "*.*.*.*:30005"
                index  => "busserv"
            }
        }
    }

# 从 filebeat 获取 打到 本机 es
    input {
        beats {
            port => 30006
            type => "remote"
        }
    }

    filter {
        if [type] == "remote" {
            grok {
                match => {
                    "message" => "(?m)(?<log_time>[^\|]+)\|(?<function>[^\|]+)\|(?<log_level>[^\|]+)\|(?<title>[^\|]+)(\|(?<description>.+))?"
                }
            }
        }
    }

    output {
        if [type] == "remote" {
            elasticsearch {
                  action => "index"
                  hosts  => "*.*.*.*:30005"
                  index  => "remote-busserv"
            }
        }
    }

2. 详细的配置另查阅

kibana

/etc/kibana/kibana.yml

server.port: 5601
server.host: "*.*.*.*"
server.name: "*.*.*.*"
elasticsearch.url: "http://*.*.*.*:9200"

service kibana start/stop/restart

es 安装插件head和kopf

/usr/share/elasticsearch/bin/plugin install lmenezes/elasticsearch-kopf
/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head

filebeat

/etc/filebeat/filebeat.yml

filebeat.prospectors:
- type: log
  enabled: true
  paths:
    - /www/busservlib/Log/**/*.log
  exclude_files: ['^/www/busservlib/Log/Schema']
  multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\|'
  multiline.negate: true
  multiline.match: after
  multiline.timeout: 5s
  # try to optimize
  harvester_buffer_size: 409600
  # 初始化不收集之前的日志
  #tail_files: true
  # 一天前的日志忽略之后从 registry_file 删除
  close_inactive: 10m
  clean_inactive: 26h
  ignore_older: 25h
output:
  logstash:
    hosts: ["*.*.*.*:30006"]
    work: 2

service filebeat start/stop/restart

注意

• ignore_older 会在首次启动时就生效，在要收集新的一个日志目录的时候比较有用，相当于指定多少日期前日志不收集（目录中的文件）
• tail_files 启动之后 新文件出现只读取出现以后的内容，如果是新的日志目录，可以指定是否收集此文件之前的内容（文件中的内容）

nginx proxy kibana.md

使用nginx 作为前端 auth

upstream kibana {
    server 127.0.0.1:5601;
    keepalive 15;
}

server {
    listen 80;

    server_name kibana.pixara.com;
    access_log  /var/log/nginx/kibana.log;

    auth_basic "Kibana Login Auth";
    auth_basic_user_file /etc/nginx/basic_auth/kibana;

    location / {
        proxy_pass http://kibana;
        proxy_read_timeout 300;
        proxy_connect_timeout 300;
        proxy_redirect     off;

        proxy_set_header   X-Forwarded-Proto $scheme;
        proxy_set_header   Host              $http_host;
        proxy_set_header   X-Real-IP         $remote_addr;
        proxy_set_header   Connection "Keep-Alive";
        proxy_set_header   Proxy-Connection "Keep-Alive";
    }
}

添加auth用户

mkdir /etc/nginx/basic_auth

printf "USERNAME:$(openssl passwd -crypt USERPASS)\n" > /etc/nginx/basic_auth/kibana

x-pack 增加用户验证.md

安装 x-pack

cd /usr/share/kibana && ./bin/kibana-plugin install x-pack
cd /usr/share/elasticsearch && ./bin/elasticsearch-plugin install x-pack
cd /usr/share/logstash &&  bin/logstash-plugin install x-pack

elastic && kibana 的 x-pack 配置

添加 elasticsearch 用户（超级用户，后续可以控制台添加普通用户）

./bin/x-pack/users useradd YOURUSER # 添加用户
./bin/x-pack/users passwd YOURUSER #后续提示设置密码 YOURPASSWORD
./bin/x-pack/users roles YOURUSER -a superuser  #设为超级用户

elasticsearch .yml 添加配置

1. 启用用户验证
2. 关闭默认用户 u:kibana p:changme

xpack.security.enabled: true  
xpack.security.authc.accept_default_password: false

添加elastic密码到 kibana配置中

elasticsearch.username: YOURUSER
elasticsearch.password: YOURPASSWORD

最后重启服务 elastic 以及 kibana

为 logstash 添加 elastic 的写入权限

为 elastic 单独创建一个具有索引写入权限的角色 并加入用户

角色

POST _xpack/security/role/logstash_writer
{
  "cluster": ["manage_index_templates", "monitor"],
  "indices": [
    {
      "names": [ "*" ], 
      "privileges": ["write","delete","create_index"]
    }
  ]
}

用户

POST _xpack/security/user/logstash_internal
{
  "password" : "YOURPASSWORD",
  "roles" : [ "logstash_writer"],
  "full_name" : "Internal Logstash User"
}

将刚刚创建的用户加入到 logstash 的配置中（每个配置块都需要加上）

logstash.conf

output {
  elasticsearch {
    #...
    user => logstash_internal
    password => YOURPASSWORD
  }
}

重启 logstash

使用 elasticdump 导入导出数据.md

// 导出 kibana 配置
elasticdump --ignore-errors=true  --scrollTime=120m  --bulk=true --input=http://127.0.0.1:9200/.kibana  --output=data.json  --type=data

elasticdump --ignore-errors=true  --scrollTime=120m  --bulk=true --input=http://127.0.0.1:9200/.kibana --output=mapping.json  --type=mapping

// 导入 kibana 配置
elasticdump --input=mapping.json  --output=http://127.0.0.1:9200/.kibana --type=mapping

elasticdump --input=data.json  --output=http://127.0.0.1:9200/.kibana --type=data


// 导出 数据
elasticdump --ignore-errors=true  --scrollTime=120m  --bulk=true --input=http://127.0.0.1:9200/maidian* --output=mapping.json  --type=mapping

elasticdump --ignore-errors=true  --scrollTime=120m  --bulk=true --input=http://127.0.0.1:9200/maidian* --output=data.json  --type=data


// 导入 数据
elasticdump --input=mapping.json  --output=http://127.0.0.1:9200/maidian* --type=mapping

elasticdump --input=data.json  --output=http://127.0.0.1:9200/maidian* --type=data

可能出现的问题.md

elasticsearch

一

[node0] detected index data in default.path.data [/var/lib/elasticsearch/nodes/0/indices] where there should not be any

解决：启动前 默认数据地址应该被清空

rm -rf /var/lib/elasticsearch/*

二

system call filters failed to install; check the logs and fix your configuration or disable system call filters at your own risk

解决：centos6 不支持 bootstrap.system_call_filter

修改配置

bootstrap.system_call_filter: false
bootstrap.memory_lock: false

三

max file descriptors [65535] for elasticsearch process is too low, increase to at least [65536]
max number of threads [1024] for user [elasticsearch] is too low, increase to at least [2048]

解决：系统限制：文件描述符，最大线程数

vi /etc/security/limits.conf

elasticsearch soft nofile 65536
elasticsearch hard nofile 65536
elasticsearch soft nproc 4096
elasticsearch hard nproc 4096