scrape_the_scrapers.sh

# fetches raw data from https://shopsensor-272815.appspot.com/
#
# requires multiple queries for each city's bounding box
#
# if you actually run this, pipe each curl statement to a seperate file

# bern, 27 results
curl 'https://shopsensor-272815.appspot.com/searchresults' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/json; charset=utf-8' -H 'X-Requested-With: XMLHttpRequest' -H 'Origin: https://shopsensor-272815.appspot.com' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://shopsensor-272815.appspot.com/' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data '{"_southWest":{"lat":46.92687938388263,"lng":7.421092987060548},"_northEast":{"lat":46.96941802389087,"lng":7.471647262573243}}'
# > bern.json

# biel, 19 results
curl 'https://shopsensor-272815.appspot.com/searchresults' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/json; charset=utf-8' -H 'X-Requested-With: XMLHttpRequest' -H 'Origin: https://shopsensor-272815.appspot.com' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://shopsensor-272815.appspot.com/' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data '{"_southWest":{"lat":47.11505823645416,"lng":7.221021652221681},"_northEast":{"lat":47.15744715642642,"lng":7.271575927734375}}'
# > biel.json

# zug, 6 results
curl 'https://shopsensor-272815.appspot.com/searchresults' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/json; charset=utf-8' -H 'X-Requested-With: XMLHttpRequest' -H 'Origin: https://shopsensor-272815.appspot.com' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://shopsensor-272815.appspot.com/' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: Trailers' --data '{"_southWest":{"lat":47.14991770950615,"lng":8.488397598266603},"_northEast":{"lat":47.19227884418124,"lng":8.538951873779299}}'
# > zug.json

# zurich, 43 results
curl 'https://shopsensor-272815.appspot.com/searchresults' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/json; charset=utf-8' -H 'X-Requested-With: XMLHttpRequest' -H 'Origin: https://shopsensor-272815.appspot.com' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://shopsensor-272815.appspot.com/' --data '{"_southWest":{"lat":47.363245989634336,"lng":8.487625122070314},"_northEast":{"lat":47.40543674680085,"lng":8.53817939758301}}' 
# > zurich.json

# to count the results, append the curl statement with this:
#         | jq '. | length'