Medium Scraper

medium.py

from selenium import webdriver
from selenium.webdriver.firefox.options import Options

from bs4 import BeautifulSoup

class Scraper:
    def __init__(self):
        self.options = Options()
        self.options.add_argument('--headless')
        self.driver = webdriver.Firefox(options=self.options)


    def get(self,link):
        self.driver.get(link)
        data = self.driver.execute_script('return document.body.innerHTML')
        self.driver.close()
        soup = BeautifulSoup(data, "lxml")
        all_sections = soup.find_all("section")
        for section in all_sections:
            for paragraph in section.find_all({"h1","p"})[1:-1]:
                print(paragraph.text)