Python script that retrieves a pandas dataframe containing fundamental stock market valuation ratios

globalValues.py

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import sys

def globalValues:
	url = 'http://www.starcapital.de/research/stockmarketvaluation?SortBy=Shiller_PE'
	r = requests.get(url)
	if r.status_code != 200:
	    print('Could not retrieve funds page. Code %d' % r.status_code)
	    sys.exit()
	soup = BeautifulSoup(r.text)
	img_header_str = soup.find(attrs={'class':'img_header'}).text
	m = re.search('(?<=per)(.*)', img_header_str)
	date_str = m.groups()[0].strip()
	tmp = soup.findAll(attrs={'class':'years', 'scope':'col'})
	headers = []
	for t in tmp:
	    headers.append(t.text)
	nCols = len(headers)
	data = []
	tmp = soup.findAll(attrs={'scope':'row'})
	for t in tmp: # for each country
	    country = t.text
	    p = t.parent # traverse up the node
	    matches = p.findAll(attrs={'class':re.compile("^(mz|mzm)$")})
	    vals = {}
	    for i in range(nCols):
	        val = matches[i].text
	        vals[headers[i]] = val # these are strings, not numbers
	    data.append(vals)
	df = pd.DataFrame(data, columns=headers)
	return df