Elasticsearch in Dallas

0-dependencies.sh

# Packages
sudo apt-get install -y make gcc g++ git rubygems screen ncdu iptraf unzip openjdk-7-jre-headless
 
# System-wide file descriptor limit
echo 'fs.file-max = 100000' | sudo tee -a /etc/sysctl.conf
 
# And unlimited memory locking
echo "dan soft memlock unlimited" | sudo tee -a /etc/security/limits.conf
echo "dan hard memlock unlimited" | sudo tee -a /etc/security/limits.conf
# Don't limit the number of processes
echo "dan soft nproc unlimited" | sudo tee -a /etc/security/limits.conf
echo "dan hard nproc unlimited" | sudo tee -a /etc/security/limits.conf
# Don't limit the number of files
echo 'dan soft nofile 65535' | sudo tee -a /etc/security/limits.conf
echo 'dan hard nofile 65535' | sudo tee -a /etc/security/limits.conf

# Block all the `unable to resolve hosts` message
echo "127.0.0.1 $HOSTNAME" | sudo tee -a /etc/hosts

1-raid.sh

# And make it rwx on
sudo mkdir /media/raid
sudo chmod -R a+rwx /media/raid

2-elasticsearch.sh

export ES_VERSION=0.20.1
 
# Download
curl -OL http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ES_VERSION.zip
 
# Unpack, link, cleanup
sudo unzip elasticsearch-$ES_VERSION.zip -d /usr/local/elasticsearch
sudo ln -s /usr/local/elasticsearch/elasticsearch-{$ES_VERSION,latest}
rm elasticsearch-$ES_VERSION.zip
 
# Some profile stuff
echo 'export ESHOME=/usr/local/elasticsearch/elasticsearch-latest/' >> ~/.bash_profile
echo 'export PATH=$PATH:/usr/local/elasticsearch/elasticsearch-latest/bin/service/' >> ~/.bash_profile
echo 'export ES_HEAP_SIZE=20480' >> ~/.bash_profile
echo "export JAVA=`which java`" >> ~/.bash_profile
echo "export HOSTNAME=`hostname`" >> ~/.bash_profile
source ~/.bash_profile
 
# Install paramedic
cd $ESHOME
sudo bin/plugin -install karmi/elasticsearch-paramedic
 
# Install this as a service
mkdir -p ~/git
cd ~/git && git clone https://github.com/elasticsearch/elasticsearch-servicewrapper.git
cp -r elasticsearch-servicewrapper/service $ESHOME/bin/
 
# Make the service honor ES_JAVA_OPTS
echo 'wrapper.java.additional.10=%ES_JAVA_OPTS%' | sudo tee -a /usr/local/elasticsearch/elasticsearch-latest/bin/service/elasticsearch.conf

3-config.sh

source ~/.bash_profile
 
export ES_CONFIG=$ESHOME/config/elasticsearch.yml
 
# Set the node name
sudo sed -i "s;# node.name: \"Franz Kafka\";node.name: \"\${HOSTNAME}\";" $ES_CONFIG
sudo sed -i "s;# cluster.name: elasticsearch;cluster.name: $CLUSTER;" $ES_CONFIG
sudo sed -i "s;# node.rack: rack314;node.rack: $RACK;" $ES_CONFIG
sudo sed -i 's;gateway.recover_after_time: 2m;gateway.recover_after_time: 10m;' $ES_CONFIG
 
export ES_LOGGING=$ESHOME/config/logging.yml
sudo sed -i 's;#gateway: DEBUG;gateway: DEBUG;' $ES_LOGGING
sudo sed -i 's;#index.gateway: DEBUG;index.gateway: DEBUG;' $ES_LOGGING
sudo sed -i 's;#indices.recovery: DEBUG;indices.recovery: DEBUG;' $ES_LOGGING
sudo sed -i 's;#discovery: TRACE;discovery: DEBUG;' $ES_LOGGING
 
export ES_LOGS_DIR="$ESDIR/logs"; mkdir -p $ES_LOGS_DIR
export ES_DATA_DIR="$ESDIR/data"; mkdir -p $ES_DATA_DIR
export ES_WORK_DIR="$ESDIR/work"; mkdir -p $ES_WORK_DIR
 
sudo sed -i "s;# path.data: /path/to/data$;path.data: $ES_DATA_DIR;" $ES_CONFIG
sudo sed -i "s;# path.work: /path/to/work$;path.work: $ES_WORK_DIR;" $ES_CONFIG
sudo sed -i "s;# path.logs: /path/to/logs$;path.logs: $ES_LOGS_DIR;" $ES_CONFIG
 
# Allow it to lock memory
sudo sed -i "s;# bootstrap.mlockall: true;bootstrap.mlockall: true;" $ES_CONFIG
 
# And allow large uploads
sudo sed -i "s;# http.max_content_length: 100mb;http.max_content_length: 1gb;" $ES_CONFIG
 
# Gateway configuration
sudo sed -i "s;# gateway.type: local;gateway.type: local;" $ES_CONFIG
sudo sed -i "s;# gateway.recover_after_time: 5m;gateway.recover_after_time: 2m;" $ES_CONFIG
sudo sed -i "s;# discovery.zen.minimum_master_nodes: 1;discovery.zen.minimum_master_nodes: 6;" $ES_CONFIG

# And we /don't/ want to dump the heap on failure
sudo sed -i s'#-XX:+HeapDumpOnOutOfMemoryError#-XX:-HeapDumpOnOutOfMemoryError#g' /usr/local/elasticsearch/elasticsearch-latest/bin/service/elasticsearch.conf
 
# Use TCP Keep-Alive
echo "network.tcp.keep_alive: true" | sudo tee -a /usr/local/elasticsearch/elasticsearch-latest/config/elasticsearch.yml

4-zabbix.sh

# Nothing yet