collect spark job duration from yarn application web page.

spark_job_duration_on_yarn.py

import bs4
import requests


def parse_ts(s):
    s = s.strip()
    v, unit = s.split(' ')[:2]
    v = float(v)
    if unit == 'ms':
        return v
    elif unit == 's':
        return v * 1000
    elif unit == 'min':
        return v * 1000 * 60
    
    raise ValueError(f'unknown time unit {unit}')


def get_page_duration(url, i):
    data = requests.get(url, params={'completedJob.page': i}).text
    sp = bs4.BeautifulSoup(data)
    table = sp.find('table', attrs={'id': 'completedJob-table'})
    return sum(parse_ts(tr.find_all('td')[3].text) for tr in table.find_all('tr'))