Debugging baseball-reference standings scrape

baseball_reference_scraping.py

from bs4 import BeautifulSoup
import requests

# Get the data
raw = requests.get('http://www.baseball-reference.com/leagues/MLB/2016-standings.shtml').text

# Parse it
soup = BeautifulSoup(raw,'html.parser')

# Find the table
table = soup.find_all('table',{'id':'expanded_standings_overall'})

# Should be true, but there's nothing in there
len(table) > 0

It turns out the data is hidden in a comment field.

from bs4 import BeautifulSoup, Comment
import pandas as pd

comments = soup.find_all(text=lambda e: isinstance(e, Comment))

# It's the 14th comment
table_soup = BeautifulSoup(comments[14],'lxml')
table_html = table_soup.find_all('table',{'id':'expanded_standings_overall'})[0]

pd.read_html(str(table_html),index_col=0)[0]