Scrape Top Stories URLs and titles from Google SERPs.

Google Top Stories.py

#Scrape Google SERPs for Top Stories URLs and titles
import requests
from bs4 import BeautifulSoup

keyword = 'keyword goes here'
url = 'https://www.google.com/search?q=%s' % keyword

source = requests.get(url).content

soup = BeautifulSoup(source, 'lxml')

result_url = soup.find_all("div", {"class":"HCUNre"})
result_text = soup.find_all("div", {"class":"nDgy9d"})

i=0

output_urls = []
output_text = []
output_rows = []

for r in result_url:
	output_urls.append(r.find('a')['href'].strip())
	
for r in result_text:
	output_text.append(r.text.strip())

for i in range(len(output_urls)):
	output_rows.append('%s;%s' % (output_urls[i], output_text[i]))
	print('%s;%s' % (output_urls[i], output_text[i]))