split gps JSON file

json_split.py

import json
from os.path import isfile, isdir, join
import os
import sys

def msg(s): print (s)
def dashes(): msg(40*'-')
def msgt(s): dashes(); msg(s); dashes()


def split_json_into_files(data_fname, output_dir):
    """Split json file into multiple files"""
    msgt(split_json_into_files.__doc__)

    assert isfile(data_fname), "File not found: %s" % data_fname
    if not isdir(output_dir):
        os.makedirs(output_dir)
        msg('directory created: %s' % output_dir)

    # -----------------------
    # Iterate through the file lines
    # -----------------------
    jlines = []
    fline_num = 0
    file_cnt = 0

    for line in open(data_fname, 'r').readlines():
        fline_num += 1
        jlines.append(line)

        # Is this the end of a JSON dict?
        #
        if line and line[0] == '}':
            file_cnt += 1

            # Join the JSON lines back together
            #
            json_block = ''.join(jlines)

            # Make sure it's valid JSON
            #
            try:
                json.loads(json_block)
            except:
                msg('This was not valid JSON:\n%s' % json_block)
                msgt('ERROR')
                msg('Last file line loaded: %s' % fline_num)
                sys.exit(0)

            # Write the JSON lines to a file
            #
            fout_name = join(output_dir, 'gps_data_%s.json' % (`file_cnt`.zfill(8)))
            open(fout_name, 'w').write(json_block)
            msg('(%s) file written: %s' % (file_cnt, fout_name))

            # Reset
            #
            jlines = []

if __name__ == '__main__':
    gps_data_fname = 'gpsData.json'
    output_file_dir = 'output'

    split_json_into_files(gps_data_fname, output_file_dir)