scraper for vehicle counts from autoscout.ch https://twitter.com/philshem/status/1295371670321070080

get_data.py

import requests
from bs4 import BeautifulSoup
import json
from collections import defaultdict
from random import randint
from time import sleep

'''
scraper for counts of each make of car found at autoscout.ch
https://twitter.com/philshem/status/1295371670321070080

when it prints ERROR, you have to run it again and again, many times
(because I'm too lazy to write a proper sleep routine)
'''

def main():
	url_base = 'https://www.autoscout24.ch/de/?make={}'

	s = requests.Session()
	s.headers.update({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'})

	# make first request to get all possible page ids
	r = s.get(url_base.format(str(391)))

	# quick function to take url
	data = get_data(r)

	#print(data.get('search'))
	d = defaultdict(int)
	for x,y in enumerate(data.get('search').get('items')):
		if y.get('id','') == 'make':
			for z in y.get('options'):
				if z.get('value',None) not in [None,'']:
					d[int(z.get('value'))] = {'make' : z.get('text', None), 'pop' : z.get('isPopular', False)}

	print('INFO:',len(d),'auto makes found')

	done = []
	with open('results.csv','r') as fp:
		tmp = fp.readlines()
		for t in tmp:
			done.append(int(t.split(',')[0]))
	
	# loop over all make ids
	for j in d.keys():
		
		# exclude already found
		if j in done:
			print('skipping:',j,d.get(j).get('make',None))
			continue

		print ('running:',j,d.get(j).get('make',None))

		if not d.get(j).get('make',None):
			continue

		r = s.get(url_base.format(str(j)))

		try:
			data = get_data(r)
		except:
			print('ERROR:',j,d.get(j))
			# no json found in html, meaning no search results	
			exit(0)
			#fp.write(','.join((make,str(0),str(popular)))+'\n')
			#continue

		# dive into dict
		data = data.get('search',None)
		
		if not data:
			continue
		data = data.get('stats',None)

		if not data:
			continue

		#slug = data.get('listSlug',None)
		count = data.get('count',0)
		popular = d.get(j).get('pop')
		make = d.get(j).get('make')
#			print(j,slug,count,d.get(j).get('pop'))

		# append to one csv
		with open('results.csv','a') as fp:
			fp.write(','.join((str(j),make,count,str(popular)))+'\n')

		sleep(randint(3,13))

#		break

def get_data(t):
	
	soup = BeautifulSoup(t.content,features='html5lib')
	script = soup.find_all('script')

	data = str(script[2]).replace('<script id="initial-state">window.INITIAL_STATE = ','').replace(';</script>','')
	data = json.loads(data)

	return data

if __name__ == "__main__":
	main()

results.csv