Created
February 23, 2017 15:02
-
-
Save nnnikolay/4ca6646dfb4cb458ff3fcaf519e8977d to your computer and use it in GitHub Desktop.
With this module it's possible to crawl google places API in a stream
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import stream from 'stream' | |
import https from 'https' | |
export default class googleServiceStream extends stream.Transform { | |
/** | |
* Initialize the stream | |
*/ | |
constructor() { | |
super({objectMode: true}) | |
} | |
/** | |
* Take data and call Googl Service API | |
* to fetch all related data | |
* @param {data} JSON string (from another stream) | |
* @param {String} encoding The encoding of the passed data chunk | |
* @param {Function} done A callback function for async operations | |
*/ | |
_transform(data, encoding, done) { | |
try { | |
data = JSON.parse(data) | |
fetchFromGoogleServices.call(this, null, data, done) | |
} catch (e) { | |
done() | |
} | |
} | |
} | |
// Google API Response for each item looks like this: | |
// { | |
// geometry: [ | |
// Object | |
// ], | |
// icon: 'https: //maps.gstatic.com/mapfiles/place_api/icons/generic_business-71.png', | |
// id: '', //<---- deprected don't use it, | |
// name: '', | |
// place_id: '', | |
// reference: '', //<---- deprected don't use it | |
// scope: 'GOOGLE', | |
// types: [ | |
// Object | |
// ], | |
// vicinity: '' | |
// } | |
// | |
// place_id — a textual identifier that uniquely identifies a place. | |
// To retrieve information about the place, pass this identifier in the placeId field of a Places API request. | |
// For more information about place IDs, see the place ID overview (https://developers.google.com/places/web-service/place-id). | |
function fetchFromGoogleServices(nextPageToken = null, placeObject = null, done = null) { | |
// request google service API with specific lat/lon | |
let url = 'https://maps.googleapis.com/maps/api/place/nearbysearch/json?key=' + process.env.GOOGLE_PLACES_API_KEY + '&' | |
const options = (nextPageToken === null) | |
? [ | |
'type=funeral_home', | |
'rankby=prominence', | |
'radius=50000', | |
'location=' + placeObject['Latitude'] + ',' + placeObject['Longitude'] | |
].join('&') | |
: 'pagetoken=' + nextPageToken | |
url = url + options | |
https.get(url, (res) => { | |
let rawData = '', parsedData = {} | |
res.setEncoding('utf8') | |
res.on('data', (chunk) => rawData += chunk) | |
res.on('end', () => { | |
try { | |
parsedData = JSON.parse(rawData) | |
console.log(parsedData.results.length) | |
} catch (e) { | |
throw e.message | |
} | |
// there are more pages in the result | |
if (parsedData.next_page_token !== undefined) { | |
// transform it | |
parsedData.results.map((r) => { | |
this.push(JSON.stringify(transformIt(r))) | |
}, this) | |
// call the next page only in 2 seconds !! Google limitation | |
setTimeout(function() { | |
fetchFromGoogleServices.call(this, parsedData.next_page_token, null, done) | |
}.bind(this), 2000) | |
} else { | |
try { | |
parsedData.results.map((r) => { | |
this.push(JSON.stringify(transformIt(r))) | |
}, this) | |
done() | |
} catch (e) { | |
console.log(e) | |
} | |
} | |
}) | |
}).on('error', (e) => { | |
console.error(e) | |
}) | |
} | |
function transformIt(data) { | |
return { | |
id: data.place_id, | |
name: data.name, | |
address: data.vicinity, | |
lat: data.geometry.location.lat, | |
lng: data.geometry.location.lng | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment