Sample Web Scraping function

scraper.py

import re
import requests
from bs4 import BeautifulSoup

def scrape(max_visits=10_000):
    seed, articles, visited = ['www.malaymail.com'], [], []
    for _ in range(max_visits):
        if not seed:
            break
        link = seed.pop(0)
        url = urllib.parse.urlparse(link)._replace(scheme='https', netloc='www.malaymail.com').geturl()
        if url in visited:
            continue
        try:
            response = requests.get(url, timeout=5)
        except:
            continue
        finally:
            visited.append(url)
        soup = BeautifulSoup(response.content)
        seed.extend([a['href'] for a in soup.select('a') if re.search(r'news/\w+', a['href']) is not None])
        if 'www.malaymail.com/news' in url:
            elem = soup.find("div", class_="article-body")
            if elem:
                article = elem.get_text()
                articles.append(article)
    return articles, visited