scraper.py

import requests
from requests.exceptions import HTTPError
from bs4 import BeautifulSoup


def scrape_url(url, headers=None):
    try:
        page = requests.get(url, headers)
        page.raise_for_status()
    except HTTPError as e:
        print(e)
    except Exception as err:
        print(err)
    else:
        soup = BeautifulSoup(page.content, 'html.parser')
        soup = BeautifulSoup(soup.prettify(), 'html.parser')
        return soup

    return BeautifulSoup("", 'html.parser')


if __name__ == '__main__':
    soup = scrape_url("https://somoynews.tv")
    all_links = soup.find_all('a')
    training_samples = ['করোনা', 'কোভিড-১৯', 'উহান-ভাইরাস']
    for link in all_links:
        title_words = link.get_text().strip().split()
        if set(training_samples).intersection(set(title_words)):
            print("Title: {}, Link: {}".format(link.get_text().strip(), link.get('href')))