parse_wikipedia.py

import re
import wikipedia


def parse(raw_content):
    section_title_re = re.compile("^=+\s+.*\s+=+$")
    content = []
    skip = False
    for l in raw_content.splitlines():
        line = l.strip()
        if "= references =" in line.lower():
            skip = True  # replace with break if this is the last section
            continue
        if "= further reading =" in line.lower():
            skip = True  # replace with break if this is the last section
            continue
        if section_title_re.match(line):
            skip = False
            continue
        if skip:
            continue
        content.append(line)

    return '\n'.join(content) + '\n'


ny = wikipedia.page("New York")
print(parse(ny.content))