Skip to content

Instantly share code, notes, and snippets.

@scrapehero
Created May 2, 2019 13:31
Show Gist options
  • Save scrapehero/100da6c4cf9a57b6242b3624aca128b7 to your computer and use it in GitHub Desktop.
Save scrapehero/100da6c4cf9a57b6242b3624aca128b7 to your computer and use it in GitHub Desktop.
A quick and easy tutorial to scrape car details from cars.com based on location, new/used cars, deal rating, year, make, model,and trim.
{
"_id":"cars",
"startUrl":[
"https://www.cars.com/for-sale/searchresults.action/?mdId=22162&mkId=20053&page=1&perPage=20&rd=10&searchSource=GN_REFINEMENT&shippable-dealers-checkbox=true&showMore=false&sort=relevance&stkTypId=28881&trId=24731&trId=24209&yrId=35797618&yrId=36362520&zc=20005&localVehicles=false"
],
"selectors":[
{
"id":"car_links",
"type":"SelectorElementClick",
"parentSelectors":[
"_root"
],
"selector":"a.shop-srp-listings__listing",
"multiple":true,
"delay":"6000",
"clickElementSelector":"a.button.next-page",
"clickType":"clickMore",
"discardInitialElements":"do-not-discard",
"clickElementUniquenessType":"uniqueText"
},
{
"id":"car_page",
"type":"SelectorLink",
"parentSelectors":[
"car_links"
],
"selector":"_parent_",
"multiple":false,
"delay":0
},
{
"id":"product_name",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"h1.cui-heading-2--secondary",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"price",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"span.vehicle-info__price-display",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"mileage",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"div.vdp-cap-price__mileage--mobile",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"sold_by",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"div.page-section--seller-details h2.page-section__title",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"location",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"a.get-directions-link__link",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"rating",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"div.rating__seller-details-top--reviews-text p.rating__link",
"multiple":false,
"regex":"\\(([^)]+)\\)",
"delay":0
},
{
"id":"no_of_reviews",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"a.rating__link--has-reviews-count",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"fuel_type",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(1) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"city_mpg",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(3) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"highway_mpg",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(5) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"drivetrain",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(7) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"engine",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(9) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"exterior_color",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(2) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"interior_color",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(4) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"stock",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(6) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"transmission",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(8) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"vin",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"li.vdp-details-basics__item:nth-of-type(10) span",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"contact_info",
"type":"SelectorText",
"parentSelectors":[
"car_page"
],
"selector":"div.vdp-cap-seller-exp__phone",
"multiple":false,
"regex":"[\\+]?[(]?[0-9]{3}[)]?[-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4,6}",
"delay":0
}
]
}
@ranakashif123
Copy link

WoW!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment