ls_config.sh

#!/bin/bash

usage="
This is a script used to configure your logstash tunables through delivery.rb, and to scale the number of \n
logstash processes running on your Automate server.\n
\n
You can specify the following options:\n
--workers                          The number of filter workers per logstash process. Defaults to number of virtual cores.\n
--total-procs                      The number of total logstash processes to run. Defaults to 2.\n
--heap-size                        The heap size allotted to each logstash process. Defaults to 2g.\n
--bulk-size                        The batch size for the bulk posts to ElasticSearch. Defaults to 25.\n
"

#if [ $# -eq 0 ]; then
#  echo -e $usage
#  exit 1
#fi

while [[ $# -gt 0 ]]
do
key="$1"
case $key in
    --workers)
    LS_WORKERS="$2"
    shift # past argument
    ;;
    --heap-size)
    LS_HEAP_SIZE="$2"
    shift # past argument
    ;;
    --total-procs)
    LS_TOTAL_PROCS="$2"
    shift # past argument
    ;;
    --bulk-size)
    LS_BULK_SIZE="$2"
    shift
    ;;
    -h|--help)
    echo -e $usage
    exit 0
    ;;
    *)
    echo "Unknown option $1"
    echo -e $usage
    exit 1
    ;;
esac
shift # past argument or value
done

if [ -z "$LS_HEAP_SIZE" ]; then
  LS_HEAP_SIZE="2g"
fi

if [ -z "$LS_TOTAL_PROCS" ]; then
  LS_TOTAL_PROCS="4"
fi

if [ -z "$LS_BULK_SIZE" ]; then
  LS_BULK_SIZE="25"
fi

if [ -z "$LS_WORKERS" ]; then
  LS_WORKERS="12"
fi

cat > /tmp/ls-config.rb <<EOF
ruby_block 'configure logstash settings in delivery.rb' do
  block do
    delivery_rb_lsconf = Chef::Util::FileEdit.new('/etc/delivery/delivery.rb')
    unless delivery_rb_lsconf.search_file_replace_line(
      /logstash\['config'\]/,
      "logstash['config'] = { 'pipeline' => { 'batch' => { 'size' => $LS_BULK_SIZE }, 'workers' => $LS_WORKERS } }"
    )
      delivery_rb_lsconf.insert_line_if_no_match(
      /logstash\['config'\]/,
      "logstash['config'] = { 'pipeline' => { 'batch' => { 'size' => $LS_BULK_SIZE }, 'workers' => $LS_WORKERS } }"
      )
    end
    delivery_rb_lsconf.write_file

    delivery_rb_lsheap = Chef::Util::FileEdit.new('/etc/delivery/delivery.rb')
    unless delivery_rb_lsheap.search_file_replace_line(
      /logstash\['heap_size'\]/,
      "logstash['heap_size'] = '$LS_HEAP_SIZE'"
    )
      delivery_rb_lsheap.insert_line_if_no_match(
      /logstash\['heap_size'\]/,
      "logstash['heap_size'] = '$LS_HEAP_SIZE'"
      )
    end
    delivery_rb_lsheap.write_file

    system('sudo automate-ctl reconfigure') if delivery_rb_lsconf.file_edited? || delivery_rb_lsheap.file_edited?
  end
end

ruby_block 'reap extra logstash workers' do
  block do
    logstash_dirs = Dir['/opt/delivery/sv/logstash*']
    if $LS_TOTAL_PROCS < logstash_dirs.length
      ($LS_TOTAL_PROCS + 1..logstash_dirs.length).each do |i|
        system("sudo automate-ctl stop logstash#{i}")
        FileUtils.rm_rf("/opt/delivery/sv/logstash#{i}")
        FileUtils.rm_rf("/opt/delivery/embedded/etc/logstash/conf.d#{i}")
      end
    end
  end
end

(2..$LS_TOTAL_PROCS).each do |i|
  unless Dir.exist?("/opt/delivery/sv/logstash#{i}")
    execute "copy logstash sv dir for logstash#{i}" do
      command "cp -r /opt/delivery/sv/logstash/ /opt/delivery/sv/logstash#{i}"
    end

    execute "copy logstash conf dir for logstash#{i}" do
      command "cp -r /opt/delivery/embedded/etc/logstash/conf.d /opt/delivery/embedded/etc/logstash/conf.d#{i}"
    end

    file "/opt/delivery/embedded/etc/logstash/conf.d#{i}/10-websocket-output.conf" do
      action :delete
    end
  end

  link "/opt/delivery/service/logstash#{i}" do
    to "/opt/delivery/sv/logstash#{i}"
  end

  link "/opt/delivery/init/logstash#{i}" do
    to '/opt/delivery/embedded/bin/sv'
  end

  directory "/var/log/delivery/logstash#{i}" do
    owner 'delivery'
    group 'root'
  end

  ruby_block "create directories for logstash#{i}" do
    block do
      ls_run_file = Chef::Util::FileEdit.new("/opt/delivery/sv/logstash#{i}/run")
      ls_run_file.search_file_replace(/conf\.d /, "conf.d#{i} ")
      ls_run_file.search_file_replace(/-w (?<!$LS_WORKERS)[0-9]+/, "-w $LS_WORKERS")
      ls_run_file.search_file_replace(/-b (?<!$LS_BULK_SIZE)[0-9]+/, "-b $LS_BULK_SIZE")
      ls_run_file.search_file_replace(/LS_HEAP_SIZE=(?<!$LS_HEAP_SIZE)[0-9]*m/, "LS_HEAP_SIZE=$LS_HEAP_SIZE")
      ls_run_file.write_file
      ls_log_run_file = Chef::Util::FileEdit.new("/opt/delivery/sv/logstash#{i}/log/run")
      ls_log_run_file.search_file_replace(/logstash((#{i}){2,}|(?![0-9]))/, "logstash#{i}")
      ls_log_run_file.write_file
      system("sudo automate-ctl restart logstash#{i}") if ls_run_file.file_edited? || ls_log_run_file.file_edited?
    end
  end
end
EOF

chef-apply /tmp/ls-config.rb

This is fantastic! 🎉

One small change is needed: line 149 is overly greedy and if run successive times will change logstash2 to logstash22 to logstash222 etc.

To fix this, replace line 149

ls_log_run_file.search_file_replace(/logstash/, "logstash#{i}")

with

ls_log_run_file.search_file_replace(/logstash((#{i}){2,}|(?![0-9]))/, "logstash#{i}")

This does two things:

1. corrects the overly greedy regex with a Negative lookahead to prevent the repeated characters issue
2. corrects any existing /opt/delivery/sv/logstash#{i}/log/run files that have repeated #{i} characters. ex. /var/log/delivery/logstash222 correctly becomes /var/log/delivery/logstash2