qjson.py

#!/usr/bin/env python

import json
import sys
import os
from collections import OrderedDict

"""
QJson

A short utility class and script that allows you to parse JSON from stdin or a
file and output deeply nested values in terminal friendly formats. It allows you
to chain recursive lookups into one call, look up multiple keys, and separate by
spaces or newlines to fit the utility you're piping it into. It runs on both
Python 2.7 and Python 3, and requires a minimal set of modules that should run
on pretty much any Linux install out of the box.

---

Copyright 2017 Derrek Bertrand

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
class QJson:
    key_delimiter = '.'
    value_delimiter = ' '
    only_unique = False
    json_data = {}

    # we change IO behavior by changing which file descriptors we use
    fd_out = sys.stdout
    fd_err = sys.stderr

    def getValue(self, keys, data=None):
        # ensure that we have some sensible data set
        if data == None:
            data = self.json_data

        # if we have no key, assume we're returning the whole document
        if len(keys) == 0:
            return data

        key = keys.pop(0)

        if isinstance(data, list):
            if key == '*':
                tmp = []
                for index in range(len(data)):
                    tmp.append(self.getValue(list(keys), data[index]))
                return tmp
            else:
                try:
                    data = data[int(key)]
                except IndexError as e:
                    raise IndexError('Index "'+str(key)+'" does not exist in array')
                except ValueError as e:
                    raise IndexError('Index "'+str(key)+'" is not an integer in array')
        elif isinstance(data, OrderedDict):
            if key == '*':
                tmp = []
                for sub in data:
                    tmp.append(self.getValue(list(keys), sub))
                return tmp
            else:
                try:
                    data = data[str(key)]
                except KeyError as e:
                    raise IndexError('Index "'+str(key)+'" does not exist in object')
        else:
            raise IndexError('Cannot get index "'+str(key)+'" from scalar value')

        if len(keys):
            data = self.getValue(list(keys), data)

        return data

    def serializeValues(self, values, final=True):
        if isinstance(values, OrderedDict):
            return json.dumps(values, separators=(',', ':'))

        if isinstance(values, list):
            out = []
            for index in range(len(values)):
                value = self.serializeValues(values[index], False)
                if isinstance(value, list):
                    for tmp in value:
                        out.append(tmp)
                else:
                    out.append(value)

            if final:
                if self.only_unique:
                    out = list(set(out))

                out = self.value_delimiter.join(out)

            return out

        return str(values)

    def parseArgs(self, args):
        if len(args) == 0:
            raise RuntimeError('No keys supplied for lookup')

        keys = args.pop(0).split(self.key_delimiter)
        value = self.getValue(keys)

        if len(args):
            if isinstance(value, list):
                for index in range(len(value)):
                    tmp_args = list(args)
                    tmp_args[0] = self.key_delimiter.join([tmp_args[0], str(value[index])])
                    value[index] = self.parseArgs(tmp_args)
            else:
                args[0] = self.key_delimiter.join([args[0], str(value)])
                value = self.parseArgs(args)

        return value

    def main(self, args, opts, longopts):
        status = 0
        try:
            self.parseOpts(opts)
            self.parseLongopts(longopts)
            values = self.parseArgs(args)
            values = self.serializeValues(values)
            self.writeOut(values)
        except (RuntimeError, IndexError) as e:
            self.writeErr(e)
            status = 1
        except Exception as e:
            raise e
        finally:
            self.destroy()

        return status

    def parseOpts(self, opts):
        # check quiet first
        if 'q' in opts:
            self.fd_err = None
            opts.remove('q')
        if 'l' in opts:
            self.value_delimiter = '\n'
            opts.remove('l')
        if 'u' in opts:
            self.only_unique = True
            opts.remove('u')

        # options should now be empty
        if len(opts):
            raise RuntimeError('Unknown options: ['+','.join(opts)+']')

    def parseLongopts(self, longopts):
        # stdin or a provided file
        try:
            if 'source' in longopts:
                with open(os.path.expanduser(longopts['source'])) as fd:
                    self.json_data = json.load(fd, object_pairs_hook=OrderedDict)
                longopts.pop('source')
            else:
                self.json_data = json.load(sys.stdin, object_pairs_hook=OrderedDict)
        except IOError:
            raise RuntimeError('Unable to open "'+longopts['source']+'" for input')

        # stdout or a provided file
        try:
            if 'dest' in longopts:
                with open(os.path.expanduser(longopts['dest'], 'x')) as fd:
                    self.fd_out = fd
                longopts.pop('dest')
        except IOError:
            raise RuntimeError('Unable to open "'+longopts['dest']+'" for output')

        # longopts should now be empty
        if len(longopts):
            raise RuntimeError('Unkown options: ['+
                ','.join(list(longopts.keys()))+']')

    def destroy(self):
        if self.fd_out != sys.stdout:
            self.fd_out.close()
        if self.fd_err is not None:
            self.fd_err.close()

    """
        Prints to stdout without an excessive trailing newline on Python 2 and
        Python 3.
    """
    def writeOut(self, output, term='\n'):
        self.fd_out.write(str(output)+term)

    def writeErr(self, output, term='\n'):
        if self.fd_err is not None:
            self.fd_err.write(str(output)+term)

    """
        Display the application help listing. Exiting with a bad status is the
        job of other parts of the code, but we do output to stderr so that we
        don't conflate this with actual output.
    """
    def displayHelp(self):
        self.writeErr("""\
qjson something.json path.to.key path.to.key2 [options]
cat something.json | qjson path.to.key path.to.key2 -i [options]
Parses JSON and echoes deeply nested and recursive values.
Derrek Bertrand (c) 2017; see comments for legal.
OPTIONS:
    --dest : use this as the resource to write values to. Default is to output
        to stdout.
    --source : use this as the resource to pull values from. Default is to use
        stdin, which will block if you're interactive. Piping data is perfectly
        fine. (CTRL+D will end stream if interactive)
    -l : use newlines when separating values, default is spaces
    -q : do not print errors, only print data""")

"""
    Run as an application.
    Parse the arguments, set class settings, and try to get the values.
"""
if __name__ == "__main__":
    qj = None
    args = []
    opts = []
    longopts = {}

    # Prepare args and options as separate lists
    for arg in sys.argv[1:]:
        if arg.startswith('--'):
            tmp = arg[2:].split('=')
            if len(tmp) > 1:
                longopts[tmp[0]] = '='.join(tmp[1:])
            else:
                longopts[tmp[0]] = ''
        elif arg.startswith('-'):
            for ch in arg[1:]:
                opts.append(ch)
        else:
            args.append(arg)
    opts = list(set(opts))

    qj = QJson()
    if len(args) == 0:
        qj.displayHelp()
        sys.exit(0)
    else:
        sys.exit(qj.main(args, opts, longopts))

A note on OrderedDict: Why?

I became aware that implementations of dict may vary and may or may not be ordered, and may or may not reorder your keys. This DOES vary from version to version. I didn't want anyone relying on one behavior only to have it change when using Python 3.6, so I opted to guarantee the original order as a compromise.