Created
April 23, 2018 21:11
-
-
Save calthoff/2d7f36e483a1afda4e1a2551de807235 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import urllib.request | |
| from bs4 import BeautifulSoup | |
| class Scraper: | |
| def __init__(self, site): | |
| self.site = site | |
| def scrape(self): | |
| r = urllib.request.urlopen(self.site) | |
| html = r.read() | |
| parser = "html.parser" | |
| sp = BeautifulSoup(html, parser) | |
| for tag in sp.find_all("a"): | |
| url = tag.get("href") | |
| print("\n" + url) | |
| scrape = Scraper('https://news.google.com') | |
| scrape.scrape() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment