scrape_sos_grant_apps.py

import os
import requests
from bs4 import BeautifulSoup


def get_app_data(app_url):
    app_page = requests.get(app_url, cookies=cookies)
    parsed = BeautifulSoup(app_page.text, 'html.parser')
    result = []

    for tr in parsed.table.find_all('tr'):
        first, second = tr.find_all('td')
        result.append((first.string, second.string))

    return result


def data_to_md(data):
    md = u''

    for key, item in data:
        md += u"### {} \n\n {} \n\n".format(key, item)

    return md


cookies = {
    'sessionid': os.environ['SESSIONID'],
    'cosign-sin': os.environ['COSIGNSIN']
}

index_page = requests.get('http://sin.reed.edu/webapps2/sos_grant/admin/sos_apps', cookies=cookies)

app_links = []

for button in BeautifulSoup(index_page.text, 'html.parser').find_all('button'):
    app_links.append("http://sin.reed.edu{}".format(button.get('onclick')[19:-3]))

for app_link in app_links:
    data = get_app_data(app_link)
    md = data_to_md(data)
    applicant = dict(data)['applicant']

    print("Creating PDF for {}".format(applicant))

    with open("markdown_files/{}.md".format(applicant), 'w') as f:
        f.write(md.encode('utf-8'))

    os.system("pandoc -V geometry:margin=1in --latex-engine=xelatex -o \"pdf_files/{}.pdf\" \"markdown_files/{}.md\"".format(applicant, applicant))