ELK

ELK_STACK.sh

#!/bin/bash
###########
# Script install and configure ELK stack Server and can be used for adding new clients to this stack.
# Last modified 3/3/2016
# Author: Aljo Antony
###########

# Define colors in case we decide to use them
color() {
     Red='\e[0;31m'
     Blue='\e[0;34m'
     Cyan='\e[0;36m'
     Nc='\e[0m' # No Color
     Black='\e[0;30m'
     Dark_Gray='\e[1;30m'
     Light_Blue='\e[1;34m'
     Green='\e[0;32m'
     Light_Green='\e[1;32m'
     Light_Cyan='\e[1;36m'
     Light_Red='\e[1;31m'
     Purple='\e[0;35m'
     Light_Purple='\e[1;35m'
     Brown='\e[0;33m'
     Yellow='\e[1;33m'
     Light_Gray='\e[0;37m'
     White='\e[1;37m'

     return 0
}

# ==================== Configure_Repositories function ==============================
configure_repo() {
printf "${Green}Configuring Repositories...\n${Nc}"

# ---------------------------------------------------------------------
rpm --import http://packages.elastic.co/GPG-KEY-elasticsearch
# ---------------------------------------------------------------------
cat <<END >/etc/yum.repos.d/elasticsearch.repo
[elasticsearch-2.x]
name=Elasticsearch repository for 2.x packages
baseurl=http://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
END

cat <<END >/etc/yum.repos.d/kibana.repo
[kibana-4.4]
name=Kibana repository for 4.4.x packages
baseurl=http://packages.elastic.co/kibana/4.4/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
END

cat <<END >/etc/yum.repos.d/nginx.repo
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/rhel/6/x86_64/
gpgcheck=0
enabled=1
END

cat <<END >/etc/yum.repos.d/logstash.repo
[logstash-2.2]
name=logstash repository for 2.2 packages
baseurl=http://packages.elasticsearch.org/logstash/2.2/centos
gpgcheck=1
gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
enabled=1
END
# ---------------------------------------------------------------------
printf "${Green}Downloading prerequisites for ELK stack...\n${Nc}"
cd $ELK_DOWNLOAD_FILES
wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u65-b17/jdk-8u65-linux-x64.rpm"
curl -L -O https://download.elastic.co/beats/dashboards/beats-dashboards-1.1.0.zip
curl -O https://gist.githubusercontent.com/thisismitch/3429023e8438cc25b86c/raw/d8c479e2a1adcea8b1fe86570e42abab0f10f364/filebeat-index-template.json
}

# ==================== Installing Prerequisites function ==============================
install_components() {
printf "${Green}Installing required components for ELK stack...\n${Nc}"
sed -i s/SELINUX=enforcing/SELINUX=disabled/g /etc/selinux/config
yum clean all >>$INSTALL_LOG
cd $ELK_DOWNLOAD_FILES
yum install jdk-8u65-linux-x64.rpm -y 2>>$INSTALL_LOG >>$INSTALL_LOG

x=("elasticsearch" "kibana" "nginx" "httpd-tools" "logstash")
y=("elasticsearch-2.x" "kibana-4.4" "nginx" "null" "logstash-2.2")
for i in {0..4}
        do
                install_check
done

}

install_check() {
yum list installed ${x[$i]} 2>>$INSTALL_LOG >>$INSTALL_LOG

if [ "$?" = "0" ]; then
        tput setaf 2; echo "Application ${x[$i]} is already Installed...";tput sgr 0
else
        tput setaf 1; echo "Installing Application ${x[$i]} ...";tput sgr 0
        install_app
fi
}

install_app() {
if [ "$i" = "3" ] ; then
        yum install ${x[$i]} -y >>$INSTALL_LOG

else
        yum --enablerepo="${y[$i]}" install ${x[$i]} -y >>$INSTALL_LOG
fi

if [ "$?" = "0" ]; then
        tput setaf 2; echo "Application ${x[$i]} Installed successfully...";tput sgr 0
else
        error_exit
fi
}

# ==================== Configuring Components function ==============================
config_components() {
printf "${Green}Configuring ELK Components...\n${Nc}"
config_elastic
config_kibana
config_nginx
config_ssl
config_logstash
load_kibana
load_filebeat
config_firewall
}

# ==================== Configure Elastic Search function ==============================
config_elastic() {
sed -i_bac 's/#.*network.host.*$/network.host: localhost/' /etc/elasticsearch/elasticsearch.yml
chkconfig --add elasticsearch
service elasticsearch start
printf "${Green}Configured ElasticSearch successfully...\n${Nc}"
}

# ==================== Configure Kibana function ==============================
config_kibana() {
sed -i_bac 's/#.*server.host.*$/server.host: "localhost"/' /opt/kibana/config/kibana.yml
chkconfig --add kibana
service kibana start
printf "${Green}Configured Kibana successfully...\n${Nc}"
}

# ==================== Configure Nginx function ==============================
config_nginx() {
printf "${Yellow}Enter a password for Kibana Administrator User ${Green}(kibanaadmin):\n${Nc}"
htpasswd -c /etc/nginx/htpasswd.users kibanaadmin
cp -p /etc/nginx/nginx.conf{,.bak}
cat <<END >/etc/nginx/nginx.conf
user  nginx;
 worker_processes  1;
 error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;
 events {
     worker_connections  1024;
 }
 http {
     include       /etc/nginx/mime.types;
     default_type  application/octet-stream;
     log_format  main  ' -  []  '
                       '   '
                       ' ';
     access_log  /var/log/nginx/access.log  main;
     sendfile        on;
     #tcp_nopush     on;
     keepalive_timeout  65;
     #gzip  on;
     include /etc/nginx/conf.d/*.conf;
 }
END

cat <<END >/etc/nginx/conf.d/kibana.conf
    server {
        listen 80;

        server_name $SERVER_NAME;

        auth_basic "Restricted Access";
        auth_basic_user_file /etc/nginx/htpasswd.users;

        location / {
            proxy_pass http://localhost:5601;
            proxy_http_version 1.1;
            proxy_set_header Upgrade \$http_upgrade;
            proxy_set_header Connection 'upgrade';
            proxy_set_header Host \$host;
            proxy_cache_bypass \$http_upgrade;
        }
    }
END

chkconfig --add nginx
service nginx start
printf "${Green}Configured Nginx successfully...\n${Nc}"
}

# ==================== SSL Configuration function ==============================
config_ssl() {
cp -p /etc/pki/tls/openssl.cnf{,.bak}
sed -i_bac "/^\[ v3_ca \]/a \subjectAltName = IP: $SERVER_IP" /etc/pki/tls/openssl.cnf
cd /etc/pki/tls
openssl req -config /etc/pki/tls/openssl.cnf -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt >>$INSTALL_LOG
printf "${Green}Configured SSL successfully...\n${Nc}"
}

# ==================== Logstash Configuration function ==============================
config_logstash() {
cat <<END >/etc/logstash/conf.d/02-beats-input.conf
input {
   beats {
     port => 5044
     ssl => true
     ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
     ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
   }
 }
END

cat <<END >/etc/logstash/conf.d/10-syslog-filter.conf
filter {
   if [type] == "syslog" {
     grok {
       match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
       add_field => [ "received_at", "%{@timestamp}" ]
       add_field => [ "received_from", "%{host}" ]
     }
     syslog_pri { }
     date {
       match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
     }
   }
 }
END

cat <<END >/etc/logstash/conf.d/30-elasticsearch-output.conf
output {
   elasticsearch {
     hosts => ["localhost:9200"]
     sniffing => true
     manage_template => false
     index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
     document_type => "%{[@metadata][type]}"
   }
 }
END

service logstash configtest
chkconfig --add logstash
service logstash start
printf "${Green}Configured Logstash successfully...\n${Nc}"
}

# ==================== Load Kibana Dashboard function ==============================
load_kibana() {
printf "${Yellow}Load Kibana Dashboard...\n${Nc}"
cd $ELK_DOWNLOAD_FILES
unzip beats-dashboards-*.zip >>$INSTALL_LOG
cd beats-dashboards-*
sh ./load.sh >>$INSTALL_LOG
printf "${Light_Cyan}Kibana Dashboard loaded...\n${Nc}"
}

# ==================== Load Filebeat function ==============================
load_filebeat() {
printf "${Light_Cyan}Load File beat...\n${Nc}"
cd $ELK_DOWNLOAD_FILES
curl -XPUT 'http://localhost:9200/_template/filebeat?pretty' [email protected]
printf "${Light_Cyan}ELK Server is ready to receive Filebeat data, let's move onto setting up Filebeat on each client server.\n"
printf "${Light_Green}Connect to Kibana DashBoard using link ${Light_Red}\"http://`uname -n`\"\n${Light_Green}After entering the "kibanaadmin" credentials, you should see a page prompting you to configure a default index pattern.\n${Nc}"
}

# ==================== Configure firewall function ==============================
config_firewall() {
printf "${Light_Red}Configuring firewall on ELK Server...${Nc}\n"
for p in 5044 9200 5601 80
        do
                iptables -I INPUT -j ACCEPT -p tcp --dport $p >>$INSTALL_LOG
                iptables -I INPUT -j ACCEPT -p udp --dport $p >>$INSTALL_LOG
                iptables -I OUTPUT -j ACCEPT -p tcp --dport $p >>$INSTALL_LOG
                iptables -I OUTPUT -j ACCEPT -p udp --dport $p >>$INSTALL_LOG
        done
/etc/init.d/iptables save
/etc/init.d/iptables restart
printf "${Light_Green}Firewall is configured successfully...${Nc}"
}

# ==================== Configure Client function ==============================
config_client()
{
tput setaf 2; read -rp "Enter Client Server Private IP: " CLIENT_IP ; tput sgr 0
ssh-keygen -t rsa -f /root/.ssh/id_rsa -q -P ""
printf "${Green}Enter credentials of Client Server:${Nc}"
ssh-copy-id $CLIENT_IP
echo "Installation log of Client Server $CLIENT_IP:" >> $INSTALL_LOG
scp /etc/pki/tls/certs/logstash-forwarder.crt root@$CLIENT_IP:/tmp >> $INSTALL_LOG
ssh $CLIENT_IP "mkdir -p /etc/pki/tls/certs; cp /tmp/logstash-forwarder.crt /etc/pki/tls/certs/"
printf "${Green}Copied SSL Certficate...\n${Yellow}Installing Filebeat Package on Client Server...${Nc}"
ssh $CLIENT_IP "rpm --import http://packages.elastic.co/GPG-KEY-elasticsearch; echo -e '[beats]\nname=Elastic Beats Repository\nbaseurl=https://packages.elastic.co/beats/yum/el/\$basearch\nenabled=1\ngpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\ngpgcheck=1\n' > /etc/yum.repos.d/elastic-beats.repo; yum -y install filebeat" >> $INSTALL_LOG

if [ "$?" = "0" ]; then
cat <<END >/tmp/t.sh
#!/bin/bash
sed -i_bac -e '/- \/var/ s/^/#/' -e '/ paths:/a \        - /var/log/messages' -e '/ paths:/a \        - /var/log/secure' -e 's/#document_type:.*$/document_type: syslog/' -e '/elasticsearch:/,/# Logstash/ s/^/#/' -e 's/#logstash:/logstash:/' -e "/logstash:/,+2 s/#hosts: \[\"localhost:/hosts: \[\":/" -e '/hosts.*5044/a \    bulk_max_size: 1024' -e '/logstash:/,+30 s/#tls:/tls:/' -e '/ tls:/,+5 s/#certificate_authorities:.*/certificate_authorities: \["\/etc\/pki\/tls\/certs\/logstash-forwarder.crt"\]/' /etc/filebeat/filebeat.yml
exit
END
        scp /tmp/t.sh $CLIENT_IP:/tmp >> $INSTALL_LOG
        ssh $CLIENT_IP "sh /tmp/t.sh; service filebeat start; chkconfig --add filebeat;"
        printf "${Light_Green}Configured client successfully...${Nc}"
else
        error_exit
fi

}

# ==================== Exit function ==============================
function error_exit()
{
        tput setab 1; tput bold; tput setaf 7; echo "Unknown Error occured, check installation log located @ $INSTALL_LOG for more information";tput sgr 0
        exit 1
}

# ========================= BEGIN ==========================
# ========================================================
# ========================================================

# ========================= VARIABLES INITIALIZATION ==========================
color
mkdir /tmp/elk_downloads 2>/dev/null
ELK_DOWNLOAD_FILES=/tmp/elk_downloads
SERVER_NAME=$(uname -n)
touch /tmp/elk_downloads/install_log
INSTALL_LOG=/tmp/elk_downloads/install_log
tput setaf 2; read -rp "Enter ELK Server Private IP: " SERVER_IP; tput sgr 0

# ========================= FUNCTIONS INVOCATIONS ==========================

    BG_BLUE="$(tput setab 4)"
    BG_BLACK="$(tput setab 0)"
    FG_GREEN="$(tput setaf 2)"
    FG_WHITE="$(tput setaf 7)"

    # Screen size
    row=$(tput lines)
    col=$(tput cols)

    # Save screen
    tput smcup

    # Display menu until selection == 0
    while [[ $REPLY != 0 ]]; do
      echo -n ${BG_BLUE}${FG_WHITE}
      clear
      tput sc; tput cup $((row/3-1)) $((col/3)); tput setab 2; tput bold; tput setaf 7; printf "Check logs @$INSTALL_LOG to know about task progress...\n"; tput rc
      tput sc; tput cup $((row/3)) $((col/3)); tput setab 1; tput bold; tput setaf 7; printf "Improvisation: [email protected]\n"; tput rc
    cat<<EOF
    ==============================
      ELK Stack Installation Menu
    ------------------------------
    Please enter your choice:
    (1) Configure repo
    (2) Install Components
    (3) Configure Components
    (4) Configure Client
           (0)Quit
    ------------------------------
EOF
      read -p "Enter selection [0-4] > " selection

      # Clear area beneath menu
      tput cup 10 0
      echo -n ${BG_BLACK}${FG_GREEN}
      tput ed
      tput cup 11 0
      tput sc; tput cup $((row/3-1)) $((col/3)); tput setab 2; tput bold; tput setaf 7; printf "Check logs @$INSTALL_LOG to know about task progress...\n"; tput rc
      tput sc; tput cup $((row/3)) $((col/3)); tput setab 1; tput bold; tput setaf 7; printf "Improvisation: [email protected]\n"; tput rc
      # Act on selection
      case $selection in
        1)  configure_repo
            ;;
        2)  install_components
            ;;
        3)  config_components
            ;;
        4)  config_client
            ;;
        0)  break
            ;;
        *)  echo "Invalid entry."
            ;;
      esac
      printf "\n\nPress any key to continue."
      read -n 1
    done

    # Restore screen
    tput rmcup

# ========================= END ==========================
# =======================================================
# =======================================================