Workaround for https://github.com/elasticsearch/cookbook-elasticsearch/pull/103

1 - usage

To workaround this elasticsearch issue while it's not merged yet:

https://github.com/elasticsearch/cookbook-elasticsearch/pull/103


1. Paste everything in recipe.rb into your recipe, AFTER including elasticsearch recipe
2. Replace MY_COOKBOOK with your cookbook name
3. Put elasticsearch.init.erb to the templates/default/elasticsearch.init.erb of your cookbook

When you run your runlist with this in place, it will fix things and restart elasticsearch.

When pull/103 is finally released, simply update elasticsearch cookbook and remove this stuff

2 - recipe.rb

# Fix for issue where ulimit settings do not apply inside start-stop-daemon
# https://github.com/elasticsearch/cookbook-elasticsearch/pull/103
# =============================================================================

# Disable resource that increases limits via /etc/security/limits.conf
resources("bash[increase limits for the elasticsearch user]").action(:nothing)

# Override init template to come from this cookbook
resources("template[/etc/init.d/elasticsearch]").cookbook('MY_COOKBOOK')

# Add /etc/default/elasticsearch config
file '/etc/default/elasticsearch' do
  mode 0644
  content <<-TEXT
ulimit -n #{node.elasticsearch.limits.nofile}
ulimit -l #{node.elasticsearch.limits.memlock}
TEXT
end

# Clear out deprecated ulimit config
bash "clean elasticsearch user ulimits in /etc/security/limits.conf" do
  user 'root'

  code <<-EOH
    sed -i "/#{node.elasticsearch.fetch(:user, "elasticsearch")}     -    nofile/d" /etc/security/limits.conf
    sed -i "/#{node.elasticsearch.fetch(:user, "elasticsearch")}     -    memlock/d" /etc/security/limits.conf
  EOH

  only_if do
    file = ::File.read("/etc/security/limits.conf")
    file.include?("#{node.elasticsearch.fetch(:user, "elasticsearch")}     -    nofile") ||
      file.include?("#{node.elasticsearch.fetch(:user, "elasticsearch")}     -    memlock")
  end
end
# =============================================================================

3 - elasticsearch.init.erb

#!/usr/bin/env bash
#
# elasticsearch
#
# chkconfig:   - 57 47
# description: elasticsearch
# processname: elasticsearch
# config:      <%= node[:elasticsearch][:path][:conf] %>/elasticsearch.yml

# Source networking configuration
if [ -f /etc/sysconfig/network ]; then source /etc/sysconfig/network; fi
# Source defaults if found
if [ -f /etc/default/elasticsearch ]; then source /etc/default/elasticsearch; fi

# Exit if networking is not up
[ "$NETWORKING" = "no" ] && exit

PIDFILE='<%= node.elasticsearch[:pid_file] %>'
ES_INCLUDE='<%= node.elasticsearch[:path][:conf] %>/elasticsearch-env.sh'
CHECK_PID_RUNNING=$(ps ax | grep 'java' | grep -e "es.pidfile=$PIDFILE" | sed 's/^\s*\([0-9]*\)\s.*/\1/')

start() {
    if [ -f $PIDFILE ]; then
      # PIDFILE EXISTS -- ES RUNNING?
      echo -e "\033[31;1mPID file found in $PIDFILE, elasticsearch already running?\033[0m"
      es_pid="$(cat $PIDFILE)"
      pid_running="$( ps ax | grep 'java' | grep $es_pid )"

      if [ ! -z "$pid_running" ] ; then
        # EXIT IF ES IS ALREADY RUNNING
        echo -e "\033[31;1mPID $es_pid still alive, already running...\033[0m"
        return 1
      fi
    fi

    echo -e "\033[1mStarting elasticsearch...\033[0m"
    <% if node.platform_family == 'debian' %>
    ES_INCLUDE=$ES_INCLUDE start-stop-daemon --background --start --quiet --pidfile $PIDFILE --chuid <%= node[:elasticsearch][:user] %> --exec /usr/local/bin/elasticsearch -- -p $PIDFILE
    <% else %>
    su <%= node[:elasticsearch][:user] %> -c "ES_INCLUDE=$ES_INCLUDE /usr/local/bin/elasticsearch -p $PIDFILE"
    <% end %>

    return $?
}

stop() {
    if [[ -f $PIDFILE ]]; then
      echo -n -e "\033[1mStopping elasticsearch...\033[0m"

      # REMOVE PIDFILE AND EXIT IF PROCESS NOT RUNNING
      if [ ! $CHECK_PID_RUNNING ]; then
        echo -e "\033[1mPID file found, but no matching process running?\033[0m"
        echo    "Removing PID file..."
        su <%= node[:elasticsearch][:user] %> -m -c "rm $PIDFILE"
        exit 0
      fi

      # KILL PROCESS
      su <%= node[:elasticsearch][:user] %> -m -c "kill $(cat $PIDFILE)"
      r=$?
      timeout=0
      while [ -f $PIDFILE  ]; do
        echo -n '.'
        (( timeout++ ))
        if [ $timeout -gt '15' ]; then return; fi
        sleep 1
      done; echo
      return $r
    else
      echo -e "\033[1mNo PID file found -- elasticsearch not running?\033[0m"
    fi
}

restart() {
    stop
    timeout=30
    while ps aux | grep 'java' | grep -e "es.pidfile"; do
      echo -n '.'
      (( timeout-- ))
      if [ $timeout -lt '1' ]; then return; fi
      sleep 1
    done;
    start
}

status() {
  # GOT PIDFILE?
  [ -f $PIDFILE ] && pid=$(cat $PIDFILE)

  # RUNNING
  if [[ $pid && -d "/proc/$pid" ]]; then
    version=$(curl -s 'http://localhost:9200' | ruby -rubygems -e 'require "json"; print JSON.parse(STDIN.read)["version"]["number"]')
    echo -e "\033[1;37;46melasticsearch $version running with PID $pid\033[0m"
    # VERBOSE
    if [[ $pid && $1 == '-v' || $1 == '--verbose' ]]; then
      curl -s 'http://localhost:9200/_cluster/nodes/<%= node.elasticsearch[:node][:name] %>?os&process&jvm&network&transport&settings&pretty' | \
      ruby -rubygems -e '
        begin
          require "json"; h = JSON.parse(STDIN.read); id, node = h["nodes"].first;
          def e(name, value); puts %Q|\e[1;36m#{(name.to_s+":").ljust(20)}\e[0m #{value || "N/A" rescue "N/A"}|; end
          e "HTTP Address",  node["http_address"]
          e "Node Name",     node["name"]
          e "Cluster Name",  h["cluster_name"]
          e "Started",       Time.at(node["jvm"]["start_time"].to_i/1000)
          e "JVM",           "#{node["jvm"]["vm_name"]} (#{node["jvm"]["version"]})"
          e "Memory Total",  node["os"]["mem"]["total"]
          e "Open Files",    node["process"]["max_file_descriptors"]
          e "Configuration", node["settings"]["config"]
        rescue
          puts "Metadata cannot be retrieved."
        end
      '
    fi
    # INCORRECT PID?
    if [ $pid != $CHECK_PID_RUNNING ]; then
      echo -e "\033[1;31;40m[!] Incorrect PID found in $PIDFILE: $pid\033[0m"
      return 1
    fi
    return 0
  fi

  # NOT RUNNING
  if [[ ! $pid || ! -d "/proc/$pid" ]]; then
    echo -e "\033[1;33;40melasticsearch not running\033[0m"
    return 3
  fi

  # STALE PID FOUND
  if [[ ! -d "/proc/$pid" && -f $PIDFILE ]]; then
    echo -e "\033[1;31;40m[!] Stale PID found in $PIDFILE\033[0m"
    return 1
  fi
}


case "$1" in
  start)
        start
        ;;
  stop)
        stop
        ;;
  restart)
        restart
        ;;
  status)
        status $2
        ;;
  *)
        echo $"Usage: $0 {start|stop|restart|status [-v]|}"
        exit 1
esac

exit $?