Hadoop utilities with python.

hadoop_utils.py

# 汇总hdfs web上文件大小
def hdfs_file_size(path):
    s_data = open(path).read()
    data = s_data.split('\n')
    s_size = tuple(r.split('\t')[3].split(' ')[0] for r in data if r)
    return sum(map(float, s_size))

# 打开yarn任务页面
def open_yarn_application_page(app_name):
    import requests
    import webbrowser

    host = '127.0.0.1:8088'
    apps = requests.get('http://{host}/ws/v1/cluster/apps'.format(host=host)).json()
    app = tuple(filter(lambda app: app['state'] == 'RUNNING' and app_name in app['name'], apps['apps']['app']))[0]
    app_id = app['id']
    app_url = 'http://{host}/proxy/{app_id}/'.format(host=host, app_id=app_id)
    webbrowser.open_new_tab(app_url)