IDC testing script

idc-init.sh

#!/bin/sh

## package install
cd /tmp
wget -c https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.4.1.linux-x64.tar.gz
tar zxvf grafana-4.4.1.linux-x64.tar.gz -C /usr/local && mv /usr/local/grafana{-4.4.1,}
wget -c https://dl.influxdata.com/influxdb/releases/influxdb-1.2.4_linux_amd64.tar.gz
tar xvfz influxdb-1.2.4_linux_amd64.tar.gz -C /usr/local && mv /usr/local/influxdb{-1.2.4-1,}
wget -c https://dl.influxdata.com/telegraf/releases/telegraf-1.3.4_linux_amd64.tar.gz
tar xvfz telegraf-1.3.4_linux_amd64.tar.gz -C /usr/local && mv /usr/local/telegraf{-1.3.4,}

## iptables allow all
iptables -I INPUT -j ACCEPT

curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" && python get-pip.py

pip install supervisor

cat << EOF > /etc/supervisord.conf
[unix_http_server]
file=/var/run/supervisor.sock

#[inet_http_server]
#port=127.0.0.1:9001
#username=user
#password=123

[supervisord]
logfile_maxbytes=10MB
logfile_backups=3
loglevel=info
nodaemon=false
minfds=1024
minprocs=200
pidfile=/var/run/supervisord.pid
logfile=/var/log/supervisord.log
childlogdir=/var/log/supervisor

[supervisorctl]
serverurl=unix:///var/run/supervisor.sock

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[include]
files = /etc/supervisor.d/*.conf
EOF

cat << EOF > /etc/init.d/supervisord
#!/bin/bash
# description:  supervisor is a process control utility.  It has a web based
#               xmlrpc interface as well as a few other nifty features.
# processname:  supervisord
# author: Delweng Zheng<[email protected]>
# config: /etc/supervisord.conf

ulimit -S -c 0
ulimit -n 65535
ulimit -f unlimited
ulimit -u unlimited

# source function library
. /etc/rc.d/init.d/functions

exec=/usr/bin/supervisord
prog=\$(basename \$exec)
conf="/etc/supervisord.conf"
lock="/var/lock/subsys/supervisord"
pidfile=\$(awk -F "=" '/pidfile/{print \$2}' \$conf)
log_dir=\$(awk -F "=" '/childlogdir/{print \$2}' \$conf)

[ -d \$log_dir ] || mkdir \$log_dir

RETVAL=0

start() {
    if [ -e \$pidfile ];then
        echo "\$prog already running...."
        return 1
    fi

    echo -n \$"Starting \$prog: "
    daemon \$exec -c \$conf
    RETVAL=\$?
    echo
    [ \$RETVAL -eq 0 ] && touch \$lock
}

stop() {
    echo -n \$"Stopping \$prog: "
    killproc -p \$pidfile \$prog
    echo
    [ \$RETVAL -eq 0 ] && rm -f \$lock
}

restart() {
    stop
    start
}

case "\$1" in
  start)
        start
        ;;
  stop)
        stop
        ;;
  restart|force-reload|reload)
        restart
        ;;
  condrestart)
        [ -f \$lock ] && restart
        ;;
  status)
        status -p \$pidfile \$prog
        RETVAL=\$?
        ;;
  *)
        echo \$"Usage: \$0 {start|stop|status|restart|reload|force-reload|condrestart}"
        exit 1
esac

exit \$RETVAL
EOF

chmod +x /etc/init./d/supervisord

cat <<EOF > /usr/local/telegraf/etc/telegraf/telegraf.conf
[agent]
  interval = "10s"
  round_interval = true
  hostname = ""
  omit_hostname = false


###############################################################################
#                            OUTPUT PLUGINS                                   #
###############################################################################

# Configuration for influxdb server to send metrics to
[[outputs.influxdb]]
  urls = ["http://localhost:8086"] # required
  database = "telegraf" # required

  retention_policy = ""
  write_consistency = "any"

  timeout = "5s"

###############################################################################
#                            INPUT PLUGINS                                    #
###############################################################################

[[inputs.cpu]]
  ## Whether to report per-cpu stats or not
  percpu = true
  ## Whether to report total system cpu stats or not
  totalcpu = true
  ## If true, collect raw CPU time metrics.
  collect_cpu_time = false


# Read metrics about disk usage by mount point
[[inputs.disk]]
  ## By default, telegraf gather stats for all mountpoints.
  ## Setting mountpoints will restrict the stats to the specified mountpoints.
  # mount_points = ["/"]

  ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually
  ## present on /run, /var/run, /dev/shm or /dev).
  ignore_fs = ["tmpfs", "devtmpfs", "devfs"]


# Read metrics about disk IO by device
[[inputs.diskio]]
  ## By default, telegraf will gather stats for all devices including
  ## disk partitions.
  ## Setting devices will restrict the stats to the specified devices.
  # devices = ["sda", "sdb"]
  ## Uncomment the following line if you need disk serial numbers.
  # skip_serial_number = false
  #
  ## On systems which support it, device metadata can be added in the form of
  ## tags.
  ## Currently only Linux is supported via udev properties. You can view
  ## available properties for a device by running:
  ## 'udevadm info -q property -n /dev/sda'
  # device_tags = ["ID_FS_TYPE", "ID_FS_USAGE"]
  #
  ## Using the same metadata source as device_tags, you can also customize the
  ## name of the device via templates.
  ## The 'name_templates' parameter is a list of templates to try and apply to
  ## the device. The template may contain variables in the form of '\$PROPERTY' or
  ## '\${PROPERTY}'. The first template which does not contain any variables not
  ## present for the device is used as the device name tag.
  ## The typical use case is for LVM volumes, to get the VG/LV name instead of
  ## the near-meaningless DM-0 name.
  # name_templates = ["\$ID_FS_LABEL","\$DM_VG_NAME/\$DM_LV_NAME"]


# Get kernel statistics from /proc/stat
[[inputs.kernel]]
  # no configuration


# Read metrics about memory usage
[[inputs.mem]]
  # no configuration


# Get the number of processes and group them by status
[[inputs.processes]]
  # no configuration


# Read metrics about swap memory usage
[[inputs.swap]]
  # no configuration


# Read metrics about system load & uptime
[[inputs.system]]
  # no configuration

[[inputs.net]]

[[inputs.netstat]]
EOF

cat << EOF > /usr/local/influxdb/etc/influxdb/influxdb.conf
[meta]
  # Where the metadata/raft database is stored
  dir = "/var/lib/influxdb/meta"

  # Automatically create a default retention policy when creating a database.
  # retention-autocreate = true

  # If log messages are printed for the meta service
  # logging-enabled = true

[data]
  dir = "/var/lib/influxdb/data"

  wal-dir = "/var/lib/influxdb/wal"

  # Settings for the TSM engine

  # CacheMaxMemorySize is the maximum size a shard's cache can
  # reach before it starts rejecting writes.
  # cache-max-memory-size = 1048576000

  # CacheSnapshotMemorySize is the size at which the engine will
  # snapshot the cache and write it to a TSM file, freeing up memory
  # cache-snapshot-memory-size = 26214400

  # CacheSnapshotWriteColdDuration is the length of time at
  # which the engine will snapshot the cache and write it to
  # a new TSM file if the shard hasn't received writes or deletes
  # cache-snapshot-write-cold-duration = "10m"

  # CompactFullWriteColdDuration is the duration at which the engine
  # will compact all TSM files in a shard if it hasn't received a
  # write or delete
  # compact-full-write-cold-duration = "4h"

  # The maximum series allowed per database before writes are dropped.  This limit can prevent
  # high cardinality issues at the database level.  This limit can be disabled by setting it to
  # 0.
  # max-series-per-database = 1000000

  # The maximum number of tag values per tag that are allowed before writes are dropped.  This limit
  # can prevent high cardinality tag values from being written to a measurement.  This limit can be
  # disabled by setting it to 0.
  # max-values-per-tag = 100000

###
### [coordinator]
###
### Controls the clustering service configuration.
###

[coordinator]
  # The default time a write request will wait until a "timeout" error is returned to the caller.
  # write-timeout = "10s"

  # The maximum number of concurrent queries allowed to be executing at one time.  If a query is
  # executed and exceeds this limit, an error is returned to the caller.  This limit can be disabled
  # by setting it to 0.
  # max-concurrent-queries = 0

  # The maximum time a query will is allowed to execute before being killed by the system.  This limit
  # can help prevent run away queries.  Setting the value to 0 disables the limit.
  # query-timeout = "0s"

  # The the time threshold when a query will be logged as a slow query.  This limit can be set to help
  # discover slow or resource intensive queries.  Setting the value to 0 disables the slow query logging.
  # log-queries-after = "0s"

  # The maximum number of points a SELECT can process.  A value of 0 will make the maximum
  # point count unlimited.
  # max-select-point = 0

  # The maximum number of series a SELECT can run.  A value of 0 will make the maximum series
  # count unlimited.

  # The maximum number of series a SELECT can run. A value of zero will make the maximum series
  # count unlimited.
  # max-select-series = 0

  # The maxium number of group by time bucket a SELECt can create.  A value of zero will max the maximum
  # number of buckets unlimited.
  # max-select-buckets = 0

[retention]
  # Determines whether retention policy enforcment enabled.
  # enabled = true

  # The interval of time when retention policy enforcement checks run.
  # check-interval = "30m"


[shard-precreation]
  # Determines whether shard pre-creation service is enabled.
  # enabled = true

  # The interval of time when the check to pre-create new shards runs.
  # check-interval = "10m"

  # The default period ahead of the endtime of a shard group that its successor
  # group is created.
  # advance-period = "30m"

[monitor]
  # Whether to record statistics internally.
  # store-enabled = true

  # The destination database for recorded statistics
  # store-database = "_internal"

  # The interval at which to record statistics
  # store-interval = "10s"

###
### [admin]
###
### Controls the availability of the built-in, web-based admin interface. If HTTPS is
### enabled for the admin interface, HTTPS must also be enabled on the [http] service.
###
### NOTE: This interface is deprecated as of 1.1.0 and will be removed in a future release.

[admin]
  # Determines whether the admin service is enabled.
  enabled = true

  # The default bind address used by the admin service.
  bind-address = ":8083"

  # Whether the admin service should use HTTPS.
  # https-enabled = false

  # The SSL certificate used when HTTPS is enabled.
  # https-certificate = "/etc/ssl/influxdb.pem"

###
### [http]
###
### Controls how the HTTP endpoints are configured. These are the primary
### mechanism for getting data into and out of InfluxDB.
###

[http]
  # Determines whether HTTP endpoint is enabled.
  # enabled = true

  # The bind address used by the HTTP service.
  # bind-address = ":8086"

  # Determines whether HTTP authentication is enabled.
  # auth-enabled = false

  # The default realm sent back when issuing a basic auth challenge.
  # realm = "InfluxDB"

  # Determines whether HTTP request logging is enable.d
  # log-enabled = true

  # Determines whether detailed write logging is enabled.
  # write-tracing = false

  # Determines whether the pprof endpoint is enabled.  This endpoint is used for
  # troubleshooting and monitoring.
  # pprof-enabled = true

  # Determines whether HTTPS is enabled.
  # https-enabled = false

  # The SSL certificate to use when HTTPS is enabled.
  # https-certificate = "/etc/ssl/influxdb.pem"

  # Use a separate private key location.
  # https-private-key = ""

  # The JWT auth shared secret to validate requests using JSON web tokens.
  # shared-secret = ""

  # The default chunk size for result sets that should be chunked.
  # max-row-limit = 0

  # The maximum number of HTTP connections that may be open at once.  New connections that
  # would exceed this limit are dropped.  Setting this value to 0 disables the limit.
  # max-connection-limit = 0

  # Enable http service over unix domain socket
  # unix-socket-enabled = false

  # The path of the unix domain socket.
  # bind-socket = "/var/run/influxdb.sock"

###
### [subscriber]
###
### Controls the subscriptions, which can be used to fork a copy of all data
### received by the InfluxDB host.
###

[subscriber]
  # Determines whether the subscriber service is enabled.
  # enabled = true

  # The default timeout for HTTP writes to subscribers.
  # http-timeout = "30s"

  # Allows insecure HTTPS connections to subscribers.  This is useful when testing with self-
  # signed certificates.
  # insecure-skip-verify = false

  # The path to the PEM encoded CA certs file. If the empty string, the default system certs will be used
  # ca-certs = ""

  # The number of writer goroutines processing the write channel.
  # write-concurrency = 40

  # The number of in-flight writes buffered in the write channel.
  # write-buffer-size = 1000


###
### [[graphite]]
###
### Controls one or many listeners for Graphite data.
###

[[graphite]]
  # Determines whether the graphite endpoint is enabled.
  # enabled = false
  # database = "graphite"
  # retention-policy = ""
  # bind-address = ":2003"
  # protocol = "tcp"
  # consistency-level = "one"

  # These next lines control how batching works. You should have this enabled
  # otherwise you could get dropped metrics or poor performance. Batching
  # will buffer points in memory if you have many coming in.

  # Flush if this many points get buffered
  # batch-size = 5000

  # number of batches that may be pending in memory
  # batch-pending = 10

  # Flush at least this often even if we haven't hit buffer limit
  # batch-timeout = "1s"

  # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max.
  # udp-read-buffer = 0

  ### This string joins multiple matching 'measurement' values providing more control over the final measurement name.
  # separator = "."

  ### Default tags that will be added to all metrics.  These can be overridden at the template level
  ### or by tags extracted from metric
  # tags = ["region=us-east", "zone=1c"]

  ### Each template line requires a template pattern.  It can have an optional
  ### filter before the template and separated by spaces.  It can also have optional extra
  ### tags following the template.  Multiple tags should be separated by commas and no spaces
  ### similar to the line protocol format.  There can be only one default template.
  # templates = [
  #   "*.app env.service.resource.measurement",
  #   # Default template
  #   "server.*",
  # ]


[[udp]]
  # enabled = false
  # bind-address = ":8089"
  # database = "udp"
  # retention-policy = ""

  # These next lines control how batching works. You should have this enabled
  # otherwise you could get dropped metrics or poor performance. Batching
  # will buffer points in memory if you have many coming in.

  # Flush if this many points get buffered
  # batch-size = 5000

  # Number of batches that may be pending in memory
  # batch-pending = 10

  # Will flush at least this often even if we haven't hit buffer limit
  # batch-timeout = "1s"

  # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max.
  # read-buffer = 0

###
### [continuous_queries]
###
### Controls how continuous queries are run within InfluxDB.
###

[continuous_queries]
  # Determiens whether the continuous query service is enabled.
  # enabled = true

  # Controls whether queries are logged when executed by the CQ service.
  # log-enabled = true

  # interval for how often continuous queries will be checked if they need to run
  # run-interval = "1s"
EOF

mkdir -p /etc/supervisor.d
mkdir -p /var/log/supervisor

cat <<EOF > /etc/supervisor.d/teleraf.conf
[program:telegraf]
command =/usr/local/telegraf/usr/bin/telegraf -config /usr/local/telegraf/etc/telegraf/telegraf.conf
autostart = true
autorestart = true
startsecs = 3
stdout_logfile = /var/log/supervisor/telegraf-info.log
stderr_logfile = /var/log/supervisor/telegraf-erro.log
EOF

cat <<EOF > /etc/supervisor.d/influxdb.conf
[program:influxdb]
command = /usr/local/influxdb/usr/bin/influxd  run -config /usr/local/influxdb/etc/influxdb/influxdb.conf
autostart = true
autorestart = true
startsecs = 3
stdout_logfile = /var/log/supervisor/influxdb-info.log
stderr_logfile = /var/log/supervisor/influxdb-erro.log
EOF

cat <<EOF > /etc/supervisor.d/grafana.conf
[program:grafana]
command = /usr/local/grafana/bin/grafana-server -homepath /usr/local/grafana -config /usr/local/grafana/conf/defaults.ini
autostart = true
autorestart = true
startsecs = 3
stdout_logfile = /var/log/supervisor/grafana-info.log
stderr_logfile = /var/log/supervisor/grafana-erro.log
EOF

/etc/init.d/supervisord restart