Last active
May 17, 2021 22:01
-
-
Save sylvainmetayer/f4374861c79c669271e8734d8e7d9411 to your computer and use it in GitHub Desktop.
Split Google location file into smaller chuncks
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const data = JSON.parse(fs.readFileSync('./google.json', | |
{ encoding: 'utf8', flag: 'r' })); | |
const locations = data.locations; | |
let iterations = locations.length; | |
let dataFile = []; | |
const fileName = 'google.'; | |
let fileSuffix = 0; | |
console.log(`${locations.length} locations`); | |
for (location of locations) { | |
if (dataFile.length % 100000 === 0 && dataFile.length > 0) { | |
const filename = `google.${fileSuffix}.json`; | |
fs.writeFileSync(filename, JSON.stringify({ locations: dataFile })); | |
console.log(`Need to write file ${fileSuffix}`); | |
fileSuffix++; | |
dataFile = []; | |
} | |
dataFile.push(location); | |
} | |
const filename = `google.999.json`; | |
fs.writeFileSync(filename, JSON.stringify({ locations: dataFile })); | |
console.log(`Wrote last data file with ${dataFile.length} locations`); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment