Last active
November 19, 2019 15:57
-
-
Save ntkog/37f14a572383ef0dfdd481d28d73e5c9 to your computer and use it in GitHub Desktop.
GeoGrep - Para cuando quieres filtrar csv's espacialmente por países sin una geodatabase.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require("fs"); | |
const {chain} = require("stream-chain"); | |
const {parser} = require("stream-csv-as-json"); | |
const {asObjects} = require("stream-csv-as-json/AsObjects"); | |
const {streamValues} = require("stream-json/streamers/StreamValues"); | |
const PolygonLookup = require('polygon-lookup'); | |
const COUNTRYCODES = require('./data/countries_iso_codes.json'); | |
const COUNTRY = process.argv[5]; | |
function validCountry(countryCode) { | |
return COUNTRYCODES.includes(countryCode); | |
} | |
function getCountryBBDD(countryCode) { | |
let country = require(`./data/countries/${countryCode}.json`); | |
return new PolygonLookup(country); | |
} | |
function setupPipeline(countryCode) { | |
let BBDD = getCountryBBDD(countryCode); | |
return chain([ | |
parser({ separator: ","}), | |
asObjects(), | |
streamValues(), | |
data => { | |
let lat = parseFloat(data.value[process.argv[3]]); | |
let lon = parseFloat(data.value[process.argv[4]]); | |
let found = BBDD.search(lon,lat); | |
return found ? data : null; | |
}, | |
data => console.log(Object.values(data.value).join(",")) | |
]) | |
} | |
if (validCountry(COUNTRY)) { | |
let rs = fs.createReadStream(`${process.argv[2]}`); | |
rs.pipe(setupPipeline(COUNTRY)); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
cd data/countries | |
time parallel -j+0 curl -sS -o '{}.json' '"https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/World__Countries_Generalized_analysis_trim/FeatureServer/0/query?where=ISO_3DIGIT+%3D+%27{}%27&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=ISO_3DIGIT&returnGeometry=true&returnCentroid=false&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=4326&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=false&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=geojson&token="' :::: <(cat list.txt) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
cd data | |
curl -sS "https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/World__Countries_Generalized_analysis_trim/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=ISO_3DIGIT&returnGeometry=false&returnCentroid=false&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=true&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pjson&token=" | jq ' map(.attributes.ISO_3DIGIT)' > countries_iso_codes.json |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3 | |
// You should install jq for getting a plain text list of countries | |
mkdir -p data/countries | |
cd data/countries | |
curl -sS "https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/World__Countries_Generalized_analysis_trim/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=ISO_3DIGIT&returnGeometry=false&returnCentroid=false&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=true&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pjson&token=" | jq -r '.features[] | .attributes.ISO_3DIGIT' > list.txt |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment