jq-based script to parse an Ansible json logfile obtained with `ANSIBLE_STDOUT_CALLBACK=json`

parseansiblelog.py

#!/usr/bin/env python
"""
Usage: parseansiblelog.py [OPTIONS] [HOSTNAME] LOGFILE  

  parse an Ansible json formatted log

  required parameters:   - HOSTNAME :  hostname/IP as it appears in log i.e.
  `192.169.1.70` to address task results from `"hosts": {"192.169.1.70":
  {"_ansible_no_log": false`... - LOGFILE  : log file

Options:
  --taskname TEXT  - this accepts a regex as filter
  --failed BOOLEAN
  --help            Show this message and exit.

"""
import pdb
import json

# pylint: disable=unused-import
from traceback import print_exc as xp
# pylint: enable=unused-import

#################################################################
# Dependencies
#################################################################
import click
import pyjq

#################################################################
# these are debugging-related and are not needed for operation
# I've left them here 
#################################################################

try:
    # pylint: disable=unused-import
    from bemyerp.lib.utils import set_cpdb, set_rpdb, ppp, set_breakpoints3
    # pylint: enable=unused-import
#pragma: no cover pylint: disable=unused-variable
except (ImportError,) as e: 
    def set_cpdb(*args, **kwargs):
        "basically return a do-nothing function"
        return cpdb
    ppp = set_rpdb = set_breakpoints3 = set_cpdb

#################################################################




@click.command()

#regex-filter on task.name
@click.option("--taskname", type=str, default="", help="regex filter on task names")

#filter for failed tasks (only 1 typically)
@click.option("--failed", type=bool, default=False)

# @"192.169.1.70"
@click.argument("hostname", type=str, default=False)
@click.argument('logfile', type=click.Path(exists=True, dir_okay=False))

def run_via_click(**kwargs):
    """parse an Ansible json formatted log 

required parameters:  
- HOSTNAME :  hostname/IP as it appears in log i.e. `192.169.1.70` 
  to address task results from `"hosts": {"192.169.1.70": {"_ansible_no_log": false`...
- LOGFILE  : log file
    """
    try:

        mgr = Main(**kwargs)
        mgr.process()
    # pragma: no cover pylint: disable=unused-variable
    except (Exception,) as e:  # pragma: no cover
        if cpdb():
            pdb.set_trace()
        raise

# pylint doesn't get the click & kwargs tricks
# pylint: disable=no-member
class Main:
    """ parser """

    def __repr__(self):  # pylint: disable=missing-function-docstring
        """ repr """
        return "%s" % (self.__class__.__name__)


    def __init__(self, **kwargs):
        """note that the incoming `kwargs` are determined 
        by `run_via_click`s arguments 
        """
        self.__dict__.update(**kwargs)

    def process(self):
        """ take the parameters from click and run """
        try:

            di = None
            with open(self.logfile) as fi:

                # expect a JSONDecodeError from the header stuff
                try:
                    di = json.load(fi)
                except (json.decoder.JSONDecodeError,) as e: 
                    pass


            if di is None:

                # get rid of the headers stuff like 
                #`2020-03-12 13:41:24,041 p=88609 u=myuser n=ansible | {`
                with open(self.logfile) as fi:

                    text = fi.read()
                    start = "{\n"

                    pos = text.find(start)

                    di = json.loads(text[pos:])


            #################################################################
            # having a tough time replacing the hardcoded IP with a jq variable 
            # and getting Python crashes when passing
            # in more than 1 variable in the `vars` parameter to `pyjq.all`
            # so building host filtering in Python
            #################################################################

            has_host = f'has("{self.hostname}")'
            host_addressing = f'"{self.hostname}"'

            # pylint: disable=line-too-long

            #hardcode version that works
            # jqryhard = '.plays[].tasks[]| select(.hosts | has("192.169.1.70"))| .task + (.hosts."192.169.1.70" | {msg, action, changed, stderr_lines, stdout_lines, failed, cmd}) | select(.name | startswith($taskname))'


            jqry = f""".plays[].tasks[]| select(.hosts | {has_host})| .task + (.hosts.{host_addressing} | {{msg, action, changed, stderr_lines, stdout_lines, failed, cmd}}) | select(.name | test($taskname))"""

            #################################################################

            if self.failed:
                jqry += " | select(.failed == true) "

            # having more than 1 variable in the vars dictionary causes Python crashes and seg faults
            data = pyjq.all(jqry, di, vars=dict(taskname=self.taskname))#, hostname=self.hostname))

            #makes the name show up on top rather than in the middle
            for entry in data:
                try:
                    entry["_name"] = entry["name"]
                #pragma: no cover pylint: disable=unused-variable
                except (KeyError,) as e: 
                    pass
 
            print(json.dumps(data, sort_keys=True, indent=2))


        # pragma: no cover pylint: disable=unused-variable
        except (Exception,) as e:
            if cpdb(): 
                pdb.set_trace()
            raise

def cpdb(*args, **kwargs):
    "disabled"

rpdb = breakpoints = cpdb

if __name__ == "__main__":
    cpdb = set_cpdb()
    rpdb = set_rpdb()
    run_via_click()