Installing Graphite on Ubuntu 11.10 take 2

gistfile1.txt

#number of seconds to use between flushing, default of 10, 1 for realtime
statsite_flush_interval=1
# 1 seconds * 1209600 datapoints = 2 week of 1 second granularity
# 10 seconds * 864000 datapoints   =  2 months of 10-second granularity
# 60 seconds * 259200 datapoints =   6 months of 1-minute granularity
# 10 minutes * 262974 datapoints = ~3 years of 10-minute granularity
graphite_retentions='1:1209600,10:864000,60:259200,600:262974'

####################################
# BASIC REQUIREMENTS
# http://graphite.wikidot.com/installation
# http://blog.jayway.com/2012/01/04/graphite-whisper-carbon-and-statsd-with-net-and-ubuntu-11-10/
# Last tested & updated 2012-02-27
# Refs:
# http://graphite.wikidot.com/
# https://gist.github.com/1360928
# http://blog.adku.com/2011/10/scalable-realtime-stats-with-graphite.html
# http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/
####################################
sudo add-apt-repository ppa:nginx/stable
sudo apt-get update -y

sudo apt-get install python-setuptools python-dev -y
sudo apt-get install supervisor -y

sudo easy_install pip

sudo apt-get install libgetopt-mixed-perl libmcrypt4 -y
sudo apt-get install libssl0.9.8 libxml2-dev -y
sudo apt-get install curl -y

###################################
# Install opsview which will tell 
# the state of the services
###################################
curl -o opsview-agent_3.13.2.7038-1lucid1_amd64.deb http://downloads.opsera.com/opsview-community/latest/apt/pool/main/o/opsview-base/opsview-agent_3.13.2.7038-1lucid1_amd64.deb
sudo dpkg -i opsview-agent_3.13.2.7038-1lucid1_amd64.deb

###################################
# Install nginx and uwsgi which is 
#  the web front end for Graphite
###################################
sudo apt-get -f install nginx-full -y
sudo update-rc.d -f nginx remove
sudo pip install uwsgi

####################################
# Install stats layer
####################################
 
sudo mkdir -p /tmp/installation
sudo pip install -d /tmp/installation graphite-web==0.9.9 carbon==0.9.9 whisper==0.9.9
sudo pip install statsite==0.4.0
sudo pip install Django==1.2.7 python-memcached==1.47 Twisted==11.0.0 txAMQP==0.5 zope.interface==3.8.0 django-tagging
pushd /tmp/installation
pushd whisper-0.9.9/
popd
pushd carbon-0.9.9/
popd
pushd graphite-web-0.9.9/
popd

####################################
# Configure graphite web in nginx
####################################
temp_path=/tmp/graphite
server=$(hostname)
echo "server {
  listen   80 default;
  server_name  $server;

  access_log  /var/log/nginx/graphite.access.log main;

  # http://flask.pocoo.org/docs/deploying/uwsgi/
  # http://readthedocs.org/docs/uwsgi/en/latest/features/magic-variables.html
  # http://readthedocs.org/docs/uwsgi/en/latest/examples.html

  location / {
      include uwsgi_params;
      uwsgi_pass  127.0.0.1:3031;
  }
}" | tee $temp_path
sudo cp $temp_path /etc/nginx/sites-enabled/graphite

####################
# nginx itself:
####################
temp_path=/tmp/nginx.conf
echo "daemon off;

user www-data;
worker_processes  4;

error_log  /var/log/nginx/error.log;
pid        /var/run/nginx.pid;

events {
    worker_connections  1024;
}

http {
    log_format main '\$remote_addr - \$remote_user [\$time_local] \"\$request\" '
                    '\$status \$body_bytes_sent \"\$http_referer\" '
                    '\"\$http_user_agent\" \"\$http_x_forwarded_for\" \$request_time \$upstream_response_time';

    include       /etc/nginx/mime.types;

    access_log  /var/log/nginx/access.log main;

    sendfile        on;

    keepalive_timeout  65;
    tcp_nodelay        on;

    gzip  on;
    gzip_disable \"MSIE [1-6]\.(?!.*SV1)\";

    include /etc/nginx/conf.d/*.conf;

    include /etc/nginx/sites-enabled/*;
}
" | tee $temp_path
sudo mv $temp_path /etc/nginx/nginx.conf

####################################
# /etc/supervisor/supervisord.conf
####################################
temp_path=/tmp/supervisord.conf
echo "; supervisor config file

[unix_http_server]
file=/var/run/supervisor.sock   ; (the path to the socket file)
chmod=0700                       ; sockef file mode (default 0700)

[supervisord]
logfile=/var/log/supervisor/supervisord.log ; (main log file;default \$CWD/supervisord.log)
pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default \$TEMP)

; the below section must remain in the config file for RPC
; (supervisorctl/web interface) to work, additional interfaces may be
; added by defining them in separate rpcinterface: sections
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///var/run//supervisor.sock ; use a unix:// URL  for a unix socket

; The [include] section can just contain the \"files\" setting.  This
; setting can list multiple files (separated by whitespace or
; newlines).  It can also contain wildcards.  The filenames are
; interpreted as relative to this file.  Included files *cannot*
; include files themselves.

[include]
files = /etc/supervisor/conf.d/*.conf

; Spinning up our web servers
[program:nginx]
command=/usr/sbin/nginx
process_name=%(program_name)s
autostart=true
autorestart=true
stopsignal=QUIT
priority=100

[program:uwsgi-graphite]
command=/usr/local/bin/uwsgi --socket 127.0.0.1:3031 --master --processes 4 --limit-as 512 --chdir=/opt/graphite/webapp --env DJANGO_SETTINGS_MODULE=graphite.settings --module='django.core.handlers.wsgi:WSGIHandler()'
process_name=%(program_name)s
autostart=true
autorestart=true
stopsignal=QUIT
priority=100

[program:carbon-cache]
command=python /opt/graphite/bin/carbon-cache.py --debug start
process_name=%(program_name)s
autostart=true
autorestart=true
stopsignal=TERM
priority=300

[program:statsite]
command=/usr/local/bin/statsite -c /etc/statsite.conf
process_name=%(program_name)s
autostart=true
autorestart=true
priority=999
" | tee $temp_path

sudo mv $temp_path /etc/supervisor/supervisord.conf

####################################
# /etc/statsite.conf
####################################
temp_path=/tmp/statsite.conf
echo "# Settings for the 'collector' which is the UDP listener
[collector]
host = 0.0.0.0
port = 8125

# Specify settings for the metrics 'store' which is where graphite is
[store]
host = 127.0.0.1
port = 2003

[flush]
interval = $statsite_flush_interval
" | tee $temp_path

sudo mv $temp_path /etc/statsite.conf

####################################
# /opt/graphite/conf/carbon.conf
# (note that this may be corrupted by some linux shells, 
# e.g. Xubuntu "Terminal" will corrupt it at about line 140)
####################################
line_prefix='# '
temp_path=/tmp/carbon.conf
echo "[cache]
# Configure carbon directories.
#
# OS environment variables can be used to tell carbon where graphite is
# installed, where to read configuration from and where to write data.
#
#   GRAPHITE_ROOT        - Root directory of the graphite installation.
#                          Defaults to ../
#   GRAPHITE_CONF_DIR    - Configuration directory (where this file lives).
#                          Defaults to \$GRAPHITE_ROOT/conf/
#   GRAPHITE_STORAGE_DIR - Storage directory for whipser/rrd/log/pid files.
#                          Defaults to \$GRAPHITE_ROOT/storage/
#
# To change other directory paths, add settings to this file. The following
# configuration variables are available with these default values:
#
#   STORAGE_DIR    = \$GRAPHITE_STORAGE_DIR
#   LOCAL_DATA_DIR = STORAGE_DIR/whisper/
#   WHITELISTS_DIR = STORAGE_DIR/lists/
#   CONF_DIR       = STORAGE_DIR/conf/
#   LOG_DIR        = STORAGE_DIR/log/
#   PID_DIR        = STORAGE_DIR/
#
# For FHS style directory structures, use:
#
#   STORAGE_DIR    = /var/lib/carbon/
#   CONF_DIR       = /etc/carbon/
#   LOG_DIR        = /var/log/carbon/
#   PID_DIR        = /var/run/
#
#LOCAL_DATA_DIR = 
#See /opt/graphite/webapp/local_settings.py

# Specify the user to drop privileges to
# If this is blank carbon runs as the user that invokes it
# This user must have write access to the local data directory
USER =

# Limit the size of the cache to avoid swapping or becoming CPU bound.
# Sorts and serving cache queries gets more expensive as the cache grows.
# Use the value \"inf\" (infinity) for an unlimited cache size.
MAX_CACHE_SIZE = inf

# Limits the number of whisper update_many() calls per second, which effectively
# means the number of write requests sent to the disk. This is intended to
# prevent over-utilizing the disk and thus starving the rest of the system.
# When the rate of required updates exceeds this, then carbon's caching will
# take effect and increase the overall throughput accordingly.
MAX_UPDATES_PER_SECOND = 1000

# Softly limits the number of whisper files that get created each minute.
# Setting this value low (like at 50) is a good way to ensure your graphite
# system will not be adversely impacted when a bunch of new metrics are
# sent to it. The trade off is that it will take much longer for those metrics'
# database files to all get created and thus longer until the data becomes usable.
# Setting this value high (like \"inf\" for infinity) will cause graphite to create
# the files quickly but at the risk of slowing I/O down considerably for a while.
MAX_CREATES_PER_MINUTE = 50

LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2003

# Set this to True to enable the UDP listener. By default this is off
# because it is very common to run multiple carbon daemons and managing
# another (rarely used) port for every carbon instance is not fun.
ENABLE_UDP_LISTENER = False
UDP_RECEIVER_INTERFACE = 0.0.0.0
UDP_RECEIVER_PORT = 2003

PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2004

# Per security concerns outlined in Bug #817247 the pickle receiver
# will use a more secure and slightly less efficient unpickler.
# Set this to True to revert to the old-fashioned insecure unpickler.
USE_INSECURE_UNPICKLER = False

CACHE_QUERY_INTERFACE = 0.0.0.0
CACHE_QUERY_PORT = 7002

# Set this to False to drop datapoints received after the cache
# reaches MAX_CACHE_SIZE. If this is True (the default) then sockets
# over which metrics are received will temporarily stop accepting
# data until the cache size falls below 95% MAX_CACHE_SIZE.
USE_FLOW_CONTROL = True

# By default, carbon-cache will log every whisper update. This can be excessive and
# degrade performance if logging on the same volume as the whisper data is stored.
LOG_UPDATES = False

# On some systems it is desirable for whisper to write synchronously.
# Set this option to True if youd like to try this. Basically it will
# shift the onus of buffering writes from the kernel into carbons cache.
WHISPER_AUTOFLUSH = False

# Enable AMQP if you want to receve metrics using an amqp broker
# ENABLE_AMQP = False

# Verbose means a line will be logged for every metric received
# useful for testing
# AMQP_VERBOSE = False

# AMQP_HOST = localhost
# AMQP_PORT = 5672
# AMQP_VHOST = /
# AMQP_USER = guest
# AMQP_PASSWORD = guest
# AMQP_EXCHANGE = graphite
# AMQP_METRIC_NAME_IN_BODY = False

# The manhole interface allows you to SSH into the carbon daemon
# and get a python interpreter. BE CAREFUL WITH THIS! If you do
# something like time.sleep() in the interpreter, the whole process
# will sleep! This is *extremely* helpful in debugging, assuming
# you are familiar with the code. If you are not, please dont
# mess with this, you are asking for trouble :)
#
# ENABLE_MANHOLE = False
# MANHOLE_INTERFACE = 127.0.0.1
# MANHOLE_PORT = 7222
# MANHOLE_USER = admin
# MANHOLE_PUBLIC_KEY = ssh-rsa AAAAB3NzaC1yc2EAAAABiwAaAIEAoxN0sv/e4eZCPpi3N3KYvyzRaBaMeS2RsOQ/cDuKv11dlNzVeiyc3RFmCv5Rjwn/lQ79y0zyHxw67qLyhQ/kDzINc4cY41ivuQXm2tPmgvexdrBv5nsfEpjs3gLZfJnyvlcVyWK/lId8WUvEWSWHTzsbtmXAF2raJMdgLTbQ8wE=

# Patterns for all of the metrics this machine will store. Read more at
# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
#
# Example: store all sales, linux servers, and utilization metrics
# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
#
# Example: store everything
# BIND_PATTERNS = #

# To configure special settings for the 'foo' carbon-cache instance, uncomment this:
#[cache:foo]
#LINE_RECEIVER_PORT = 2103
#PICKLE_RECEIVER_PORT = 2104
#CACHE_QUERY_PORT = 7102
# and any other settings you want to customize, defaults are inherited
# from [carbon] section.

[relay]
LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2013
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2014

# To use consistent hashing instead of the user defined relay-rules.conf,
# change this to:
# RELAY_METHOD = consistent-hashing
RELAY_METHOD = rules

# If you use consistent-hashing you may want to add redundancy
# of your data by replicating every datapoint to more than
# one machine.
REPLICATION_FACTOR = 1

# This is a list of carbon daemons we will send any relayed or
# generated metrics to. The default provided would send to a single
# carbon-cache instance on the default port. However if you
# use multiple carbon-cache instances then it would look like this:
#
# DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b
#
# The general form is IP:PORT:INSTANCE where the :INSTANCE part is
# optional and refers to the 'None' instance if omitted.
#
# Note that if the destinations are all carbon-caches then this should
# exactly match the webapp's CARBONLINK_HOSTS setting in terms of
# instances listed (order matters!).
DESTINATIONS = 127.0.0.1:2004

# This defines the maximum 'message size' between carbon daemons.
# You shouldn't need to tune this unless you really know what you're doing.
MAX_DATAPOINTS_PER_MESSAGE = 500
MAX_QUEUE_SIZE = 10000

# Set this to False to drop datapoints when any send queue (sending datapoints
# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
# default) then sockets over which metrics are received will temporarily stop accepting
# data until the send queues fall below 80% MAX_QUEUE_SIZE.
USE_FLOW_CONTROL = True

[aggregator]
LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2023

PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2024

DESTINATION_HOST = 127.0.0.1
DESTINATION_PORT = 2004
MAX_QUEUE_SIZE = 10000

# This defines the maximum 'message size' between carbon daemons.
# You shouldnt need to tune this unless you really know what youre doing.
MAX_DATAPOINTS_PER_MESSAGE = 500
" | tee $temp_path

sudo mv $temp_path /opt/graphite/conf/carbon.conf

####################################
# /opt/graphite/conf/dashboard.conf
####################################
temp_path=/tmp/dashboard.conf
echo '# This configuration file controls the behavior of the Dashboard UI, available
# at http://my-graphite-server/dashboard/.
#
# This file must contain a [ui] section that defines the following settings:
#
[ui]
default_graph_width = 650
default_graph_height = 350
automatic_variants = true
refresh_interval = 20
#
# These settings apply to the UI as a whole, all other sections in this file
# pertain only to specific metric types.
#
# The dashboard presents only metrics that fall into specified naming schemes
# defined in this file. This creates a simpler, more targetted view of the
# data. The general form for defining a naming scheme is as follows:
#
#[Metric Type]
#scheme = basis.path.<field1>.<field2>.<fieldN>
#field1.label = Foo
#field2.label = Bar
#
#
# Where each <field> will be displayed as a dropdown box
# in the UI and the remaining portion of the namespace
# shown in the Metric Selector panel. The .label options set the labels
# displayed for each dropdown.
#
# For example:
#
#[Sales]
#scheme = sales.<channel>.<type>.<brand>
#channel.label = Channel
#type.label = Product Type
#brand.label = Brand
#
# This defines a "Sales" metric type that uses 3 dropdowns in the Context Selector
# (the upper-left panel) while any deeper metrics (per-product counts or revenue, etc)
# will be available in the Metric Selector (upper-right panel).
' | tee $temp_path

sudo mv $temp_path /opt/graphite/conf/dashboard.conf

####################################
# /opt/graphite/conf/storage-schemas.conf
####################################
temp_path=/tmp/storage-schemas.conf
echo "[stats]
priority = 110
pattern = .*
retentions = $graphite_retentions
" | tee $temp_path
sudo mv $temp_path /opt/graphite/conf/storage-schemas.conf


# put some default local settings for django in place
sudo cp /opt/graphite/webapp/graphite/local_settings.py.example /opt/graphite/webapp/graphite/local_settings.py


####################################
# CREATE GRAPHITE-WEB DATABASE
# START supervisord (which starts carbon, graphite, nginx)
# ENSURE PROCESSES DIDNT BLOW
####################################
pushd /opt/graphite/webapp/graphite
yes no | sudo python manage.py syncdb

#nginx and wsgi runs under www-data user by default
sudo chown -R www-data:www-data /opt/graphite
sudo chown -R www-data:www-data /opt/graphite/storage


####################################
# Start Supervisor
####################################
# reread supervisord configuration, then start carbon, graphite, etc
sudo supervisorctl reread
sudo supervisorctl reload
sleep 2
sudo supervisorctl status


#should show all listening ports - 5666 for nrpe, 80 for nginx
#2003 for line receiver / 2004 for pickle receiver / 7002 for cache query for carbon (python)
#3031 for uwsgi, 8125 for statsite (udp)
sudo netstat -lntup

### DONE :) ###

# Have a look at the site
# firefox http://127.0.0.1

# Have a look at supervisor
# sudo supervisorctl
## supervisor> help [ENTER]

# SEND DATA TO GRAPHITE
#pushd examples
#sudo chmod +x example-client.py
# [optional] edit example-client.py to report data faster
# sudo vim example-client.py
#sudo ./example-client.py