yarn_rest_solr.py

import sys
import requests
import json
import argparse
import pprint
import time
import datetime
import pysolr

rmServer = "seregiondev01.cloud.hortonworks.com"
rmPort = "8088"
url = "http://" + rmServer + ":" + rmPort + "/ws/v1/cluster/apps?states=accepted,running"

solr = pysolr.Solr('http://localhost:8983/solr/yarn')

print "start"


while True:

    resp = requests.get(url)
    resp.encoding = 'utf-8'
    applications = resp.json()

    now = datetime.datetime.utcnow()
    formatted_now = now.strftime("%Y-%m-%dT%H:%M:%SZ")

    totalRunningContainers = 0
    totalAllocatedMB = 0 
    totalAllocatedVCores = 0
    count = 0

    for application in applications['apps']['app']:
        docs = [{
            "id": formatted_now + application['id'], 
            "applicationId_s": application['id'],                
            "datetime_dt": formatted_now,
            "state_s": application['state'],            
            "applicationType_s": application['applicationType'],
            "name_s": application['name'],
            "user_s": application['user'],
            "runningContainers_i": application['runningContainers'],            
            "allocatedMB_i": application['allocatedMB'],            
            "allocatedVCores_i": application['allocatedVCores'],
            "rowType_s": "detail",
            }]

        solr.add(docs)

        totalRunningContainers += application['runningContainers']
        totalAllocatedMB += application['allocatedMB']
        totalAllocatedVCores += application['allocatedVCores']
        count += 1

        print "added" + formatted_now  + '=' +  application['id']

    docs = [{
        "id": formatted_now + 'summary',            
        "datetime_dt": formatted_now,
        "rowType_s": 'summary',            
        "runningContainers_i": totalRunningContainers,            
        "allocatedMB_i": totalAllocatedMB,            
        "allocatedVCores_i": totalAllocatedVCores,
        "applications_s": count
        }]

    solr.add(docs)

    #break
    
    time.sleep(3)