from urllib.request import urlopen from urllib.parse import urljoin, unquote_plus import re from random import choice from bs4 import BeautifulSoup BASE_URL = 'https://en.wikipedia.org/' def get_links(page): soup = BeautifulSoup(page, 'html.parser') content = soup.find('div', {'id': 'bodyContent'}) link_regex = re.compile('^/wiki/[^:]*$') return [link.attrs['href'] for link in content.find_all('a', href=link_regex)] if __name__ == '__main__': links = ['/wiki/Kevin_Bacon'] while links: url = urljoin(BASE_URL, choice(links)) print(unquote_plus(url), flush=True) links = get_links(urlopen(url))