Brute-force ArcGIS Feature Layer to GeoJSON exporter

featureserver-geojson.py

#!/usr/bin/python3

import json
import os
import sys

import math

import requests
import fiona

import argparse

from collections import OrderedDict

REQUEST_SIZE=1000

GEOMETRY_TYPE_MAP = {
    'esriGeometryPoint': 'Point',
    'esriGeometryPolygon': 'Polygon',
    'esriGeometryPolyline': 'MultiLineString'
}

ATTRIBUTE_TYPE_MAP = {
    'esriFieldTypeInteger': 'int',
    'esriFieldTypeString': 'str',
    'esriFieldTypeDouble': 'float',
    'esriFieldTypeDate': 'str'
}

class Converter(object):
    def __init__(self):
        self._features = {}

        self._schema = {
            'properties': OrderedDict({})
        }

    def _add(self, feature):
        object_id = feature['id']
        if object_id not in self._features:
            self._features[object_id] = feature

    def parse(self, fs_json):
        if not self._schema['properties']:
            if 'geometryType' in fs_json:
                self._schema['geometry'] = GEOMETRY_TYPE_MAP[fs_json['geometryType']]

            if 'fields' not in fs_json:
                return

            for item in fs_json['fields']:
                print(item)
                if item['type'] in ATTRIBUTE_TYPE_MAP:
                    type_ = ATTRIBUTE_TYPE_MAP[item['type']]
                else:
                    type_ = 'str'

                if 'length' in item:
                    type_ += ':' + str(item['length'])

                self._schema['properties'][item['name']] = type_

        if 'features' not in fs_json:
            return

        for item in fs_json['features']:
            item_geometry = item['geometry']

            if self._schema['geometry'] == 'Point':
                coordinates = [item_geometry['x'], item_geometry['y']]
            elif self._schema['geometry'] == 'Polygon':
                coordinates = item['geometry']['rings']
            elif self._schema['geometry'] == 'MultiLineString':
                coordinates = item['geometry']['paths']

            for id_attr in ['OBJECTID', 'OID']:
                if id_attr in item['attributes']:
                    id_ = item['attributes'][id_attr]
                    break

            feature = {
                'geometry': {
                    'type': self._schema['geometry'],
                    'coordinates': coordinates
                },
                'properties': {
                    str(k): str(v) for k,v in item['attributes'].items()
                },
                'id': id_
            }

            self._add(feature)

    def serialize(self, output):
        print(self._schema)
        with fiona.open(output, 'w', schema=self._schema,
                driver='GeoJSON') as sink:
            for feature in self._features.values():
                sink.write(feature)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='Convert FeatureServer JSON to GeoJSON'
    )
    parser.add_argument('url', type=str,
                        help='FeatureServer endpoint URL')
    parser.add_argument('--layer', '-l', dest='layer', type=str,
                        default='0', help='FeatureServer layer id or name')
    parser.add_argument('output', type=str,
                        help='Output GeoJSON file')
    parser.add_argument('--force', '-f', dest='force', action='store_true',
                        default=False, help='Force overwriting the file')
    parser.add_argument('--size', '-s', dest='request_size',
                        default=REQUEST_SIZE, type=int,
                        help='Set request size for X and Y')
    parser.add_argument('--one', '-o', dest='one',
                        default=False,
                        action='store_true',
                        help='Run conversion on the first batch of data only')

    args = parser.parse_args()

    if os.path.exists(args.output) and not args.force:
        print("Output file {} exists, will not overwrite without -f".format(
            args.output))
        sys.exit(1)

    params = {
        'f': 'json'
    }

    response = requests.get(args.url, params)

    metadata = response.json()

    layer_id = None
    for layer in metadata['layers']:
        if layer['name'] == args.layer or str(layer['id']) == args.layer:
            layer_id = layer['id']

    if layer_id is None:
        print("Layer '{}' was not found in metadata".format(args.layer))
        print('Available layers:')
        for layer in metadata['layers']:
            print('\t{}: {}'.format(layer['id'], layer['name']))
        sys.exit(1)

    url = '{}/{}'.format(args.url, layer_id)

    response = requests.get(url, params)
    layer_metadata = response.json()

    extent = layer_metadata['extent']

    ranges = {
        'y': range(
            math.floor(extent['ymin']),
            math.ceil(extent['ymax']),
            args.request_size
        ),
        'x': range(
            math.floor(extent['xmin']),
            math.ceil(extent['xmax']),
            args.request_size
        )
    }

    c = Converter()

    url = '{}/{}/query'.format(args.url, layer_id)

    params = {
        'f': 'json',
        'geometry': '',
        'geometryType': 'esriGeometryEnvelope',
        'inSR': extent['spatialReference']['wkid'],
        'spatialRel': 'esriSpatialRelIntersects',
        'returnGeometry': 'true',
        'outFields':'*',
        'outSR': 4326
    }

    total = len(ranges['y']) * len(ranges['x'])
    counter = 0

    print("Ranges:\tx={}\n\ty={}".format(ranges['x'], ranges['y']))

    geometry = {
        "xmin": 0,
        "ymin": 0,
        "xmax": 0,
        "ymax": 0,
        "spatialReference": extent['spatialReference']
    }

    overlap = args.request_size / 100 * 10

    for y in ranges['y']:
        geometry['ymin'] = y - overlap
        geometry['ymax'] = y + overlap + args.request_size

        for x in ranges['x']:
            geometry['xmin'] = x - overlap
            geometry['xmax'] = x + overlap + args.request_size

            params['geometry'] = json.dumps(geometry)

            counter += 1

            print('Fetching data... {:8.2f}% ({}/{})'.format(
                counter / total * 100,
                counter, total))

            response = requests.get(url, params)

            c.parse(response.json())

            if args.one:
                break

        if args.one:
            break

    if os.path.exists(args.output):
        os.unlink(args.output)

    c.serialize(args.output)