Skip to content

Instantly share code, notes, and snippets.

@SnowyPainter
Last active June 24, 2020 16:22
Show Gist options
  • Save SnowyPainter/00e4c210776536029b8feed4ee856868 to your computer and use it in GitHub Desktop.
Save SnowyPainter/00e4c210776536029b8feed4ee856868 to your computer and use it in GitHub Desktop.
import os
import time
from selenium import webdriver
from collections import defaultdict
def getOptions(select):
return select.find_elements_by_tag_name("option")
def getOptgroups(select):
return select.find_elements_by_tag_name("optgroup")
def getYears(select):
selyears = getOptions(select)[1:]
years = []
for y in selyears:
years.append(y.text)
return years
url = 'https://www.edmunds.com/car-reviews/'
driver = webdriver.Chrome("/usr/lib/chromium-browser/chromedriver")
driver.get(url)
time.sleep(2)
selmake_element = driver.find_element_by_name("select-make")
selmake_element.click()
options = getOptions(selmake_element)[1:]
allcars = defaultdict(dict)
urls = defaultdict(dict)
for o in options:
brand = o.text
cars = {}
o.click()
time.sleep(0.2)
selmodel_element = driver.find_element_by_name("select-model")
time.sleep(0.2)
optgroups = getOptgroups(selmodel_element)
selyear_element = driver.find_element_by_name("select-year")
if(len(optgroups) > 0):
popular = getOptions(optgroups[0])
others = getOptions(optgroups[1])
for p in popular:
p.click()
selyear_element.click()
cars[o.text] = getYears(selyear_element)
for o in others:
o.click()
selyear_element.click()
cars[o.text] = getYears(selyear_element)
else:
models = getOptions(selmodel_element)[1:]
for m in models:
m.click()
selyear_element.click()
cars[o.text] = getYears(selyear_element)
allcars[brand] = cars
for brand in allcars:
urls[brand] = {}
for car in allcars[brand]:
urls[brand][car] = {}
urls[brand][car]["reviews"] = []
for year in allcars[brand][car]:
urls[brand][car]["reviews"].append("https://www.edmunds.com/{}/{}/{}/review/".format(brand, car, year))
print(allcars.items())
print(urls.items())
driver.quit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment