danhammer · November 23, 2015 17:25 · danhammer · Nov 23, 2015
diff --git a/jobs.py b/jobs.py
 # Clean jobs data

 import requests
 import pandas as pd
 from bs4 import BeautifulSoup


 def collect_links(category_idx=231):
    # Collect the links to individual jobs given by the supplied category index
    LIST_BASE = "https://www.higheredjobs.com/faculty/search.cfm"
    links = []
    i = 0
    row_count = 25

    def _link(row):
        base = "https://www.higheredjobs.com/faculty/"
        [a] = row.find_all('a')
        return base + a['href']

    while row_count > 1:
        payload = {'JobCat': category_idx, 'StartRow': (25 * i) + 1}
        job_list = requests.get(LIST_BASE, params=payload)
        soup = BeautifulSoup(job_list.text, 'html.parser')
        rows = soup.find_all("div", class_="row record")
        row_count = len(rows)
        links += [_link(r) for r in rows]
        i += 1

    return list(set(links))


 def process(link):
    # Extract the attributes from a job link.  Accepts a string link for a
    # specific job and returns a dictionary with job attributes.

    details = BeautifulSoup(requests.get(link).text, 'html.parser')
    title = details.find(id="jobtitle-header").string
    attr_rows = details.find(id="jobAttrib").find_all(class_="row field")

    def _clean(s):
        # A wrapper function to clean text within the job description
        if s is None:
            return " "
        else:
            s = s.replace("\n", " ")
            s = s.replace("\r", " ")
            return s

    def _keyval(row):
        # convert an HTML table row into key value pair
        [k] = row.find_all("div", class_="field-label col-sm-3")
        key = k.text.strip('\n\t\r: ').lower()
        [v] = row.find_all("div", class_="field-value col-sm-9")
        val = v.text.strip('\n\t\r: ').lower()
        return key, val

    description = "".join(
        [_clean(d.string) for d in details.find(id="jobDesc")]
    )

    clean = ' '.join(description.split())  # replace multiple spaces with one

    words = {
        # count the number of specified strings
        "data": clean.lower().count("data"),
        "data science": clean.lower().count("data science")
    }

    meta = dict([_keyval(row) for row in attr_rows])
    meta.update(words)
    meta.update({'title': title})

    return meta


 def grab_info(n=20):
    links = collect_links(category_idx=231)
    data = [process(link) for link in links[0:n]]
    return pd.DataFrame(data)
diff --git a/output_sample.csv b/output_sample.csv
	# Clean jobs data

	import requests
	import pandas as pd
	from bs4 import BeautifulSoup


	def collect_links(category_idx=231):
	# Collect the links to individual jobs given by the supplied category index
	LIST_BASE = "https://www.higheredjobs.com/faculty/search.cfm"
	links = []
	i = 0
	row_count = 25

	def _link(row):
	base = "https://www.higheredjobs.com/faculty/"
	[a] = row.find_all('a')
	return base + a['href']

	while row_count > 1:
	payload = {'JobCat': category_idx, 'StartRow': (25 * i) + 1}
	job_list = requests.get(LIST_BASE, params=payload)
	soup = BeautifulSoup(job_list.text, 'html.parser')
	rows = soup.find_all("div", class_="row record")
	row_count = len(rows)
	links += [_link(r) for r in rows]
	i += 1

	return list(set(links))


	def process(link):
	# Extract the attributes from a job link. Accepts a string link for a
	# specific job and returns a dictionary with job attributes.

	details = BeautifulSoup(requests.get(link).text, 'html.parser')
	title = details.find(id="jobtitle-header").string
	attr_rows = details.find(id="jobAttrib").find_all(class_="row field")

	def _clean(s):
	# A wrapper function to clean text within the job description
	if s is None:
	return " "
	else:
	s = s.replace("\n", " ")
	s = s.replace("\r", " ")
	return s

	def _keyval(row):
	# convert an HTML table row into key value pair
	[k] = row.find_all("div", class_="field-label col-sm-3")
	key = k.text.strip('\n\t\r: ').lower()
	[v] = row.find_all("div", class_="field-value col-sm-9")
	val = v.text.strip('\n\t\r: ').lower()
	return key, val

	description = "".join(
	[_clean(d.string) for d in details.find(id="jobDesc")]
	)

	clean = ' '.join(description.split()) # replace multiple spaces with one

	words = {
	# count the number of specified strings
	"data": clean.lower().count("data"),
	"data science": clean.lower().count("data science")
	}

	meta = dict([_keyval(row) for row in attr_rows])
	meta.update(words)
	meta.update({'title': title})

	return meta


	def grab_info(n=20):
	links = collect_links(category_idx=231)
	data = [process(link) for link in links[0:n]]
	return pd.DataFrame(data)