Install Filebeat

filebeat.sh

echo "Adding Filebeat"
DIRECTORY=""

sudo curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.3.3-linux-x86_64.tar.gz
sudo tar xzvf filebeat-8.3.3-linux-x86_64.tar.gz
sudo rm filebeat-8.3.3-linux-x86_64.tar.gz
DIRECTORY=filebeat-8.3.3-linux-x86_64


sudo rm -rf /etc/filebeat
sudo mkdir -p /etc/filebeat

sudo curl https://raw.githubusercontent.com/logzio/public-certificates/master/AAACertificateServices.crt --create-dirs -o /etc/pki/tls/certs/COMODORSADomainValidationSecureServerCA.crt

cd $DIRECTORY && sudo rm filebeat.yml && sudo bash -c  'cat <<EOF > filebeat.yml
############################# Filebeat #####################################
filebeat.inputs:
  - type: filestream
    paths:
      - /logs/*.log
      - /var/log/vm_agent/*.log
    fields:
      logzio_codec: json
      token: YOUR_LOGZ_TOKEN
      type: structured_logs
    fields_under_root: true
    parsers:
      - ndjson:
        target: ""
        message_key: msg
    encoding: utf-8 
    ignore_older: 3h

#The following processors are to ensure compatibility with version 7
processors:
  - add_cloud_metadata:
      providers:
        - aws
        - gcp
  - rename:
      fields:
        - from: "cloud.account.id"
          to: "resource_labels.account.id"
        - from: "cloud.availability_zone"
          to: "resource_labels.availability_zone"
        - from: "cloud.region"
          to: "resource_labels.region"
        - from: "cloud.instance.id"
          to: "resource_labels.instance.id"
        - from: "cloud.instance.name"
          to: "resource_labels.instance.name"
        - from: "cloud.image.id"
          to: "resource_labels.image.id"
        - from: "cloud.machine.type"
          to: "resource_labels.machine.type"
        - from: "cloud.project.id"
          to: "resource_labels.project.id"
        - from: "cloud.provider"
          to: "resource_provider"
        - from: "cloud.service.name"
          to: "resource_type"
        - from: "cloud.instance_name"
          to: "vm_name"
      ignore_missing: true
  - rename:
      fields:
        - from: "agent"
          to: "beat_agent"
      ignore_missing: true
  - rename:
      fields:
        - from: "log.file.path"
          to: "source"
      ignore_missing: true
  - copy_fields:
      when:
        equals:
          logger_name: application
      fields:
        - from: "text_payload"
          to: "message"
      ignore_missing: true
  - copy_fields:
      when:
        equals:
          logger_name: request
      fields:
        - from: "url"
          to: "message"
      ignore_missing: true

############################# Output ##########################################

output:
  logstash:
    hosts: [ "listener.logz.io:5015" ]
    ssl:
      certificate_authorities: [ "/etc/pki/tls/certs/COMODORSADomainValidationSecureServerCA.crt" ]
EOF' && cd ..

sudo mkdir filebeat
sudo mv $DIRECTORY/* filebeat/
sudo rm -rf $DIRECTORY
sudo mv filebeat /etc/

sudo chown root /etc/filebeat/filebeat.yml


cd /etc/systemd/system && sudo bash -c  "cat <<EOF > filebeat.service
[Unit]
Description=Filebeat by Elastic.
Documentation=https://www.elastic.co/products/beats/filebeat
Wants=network-online.target
After=network-online.target

[Service]

ExecStart=/usr/bin/sudo /etc/filebeat/filebeat -e -c /etc/filebeat/filebeat.yml
Restart=always
RestartSec=1

[Install]
WantedBy=multi-user.target
EOF"

cd ~
sudo systemctl stop filebeat && sudo systemctl disable filebeat
sudo systemctl enable filebeat && sudo systemctl start filebeat                 
sleep 5
sudo systemctl status filebeat