python code for hadoop metrics

app.py

import requests
import json
import boto
import socket

HOSTNAME = socket.gethostbyaddr(socket.gethostname())[0]
JOB_TRACKER_URL = "http://" + HOSTNAME + ":8088/ws/v1/cluster/metrics/"

config = json.load(open("config.json"))
METRIC_NAMESPACE = config['namespace']
METRIC_DIMENSION = config['dimensions']

response = requests.get(JOB_TRACKER_URL)
if (response.status_code == 200):
    yarn_cm = response.json()
else:
    print "Failed to fetch metrics, response.status_code %d" % response.status_code


containers_allocated = yarn_cm['clusterMetrics']['containersAllocated']
containers_pending = yarn_cm['clusterMetrics']['containersPending']
mem_health = yarn_cm['clusterMetrics']['totalMB']/yarn_cm['clusterMetrics']['allocatedMB'] if yarn_cm['clusterMetrics']['allocatedMB'] else 0

unhealthy_nodes = yarn_cm['clusterMetrics']['unhealthyNodes']
active_nodes = yarn_cm['clusterMetrics']['activeNodes']

# TODO - Compute AutoScaling Metric and push them since CloudWatch doesn't support
# creating alaram using multiple metrics

# Send hadoop metrics
watch = boto.connect_cloudwatch()
watch.put_metric_data(METRIC_NAMESPACE, "active_nodes", active_nodes, dimensions = METRIC_DIMENSION)
watch.put_metric_data(METRIC_NAMESPACE, "containers_allocated",
                      containers_allocated, dimensions=METRIC_DIMENSION)
watch.put_metric_data(METRIC_NAMESPACE, "containers_pending",
                      containers_pending, dimensions=METRIC_DIMENSION)
watch.put_metric_data(METRIC_NAMESPACE, "mem_health", mem_health, dimensions = METRIC_DIMENSION)
watch.put_metric_data(METRIC_NAMESPACE, "unhealthy_nodes", unhealthy_nodes, dimensions = METRIC_DIMENSION)