A parallel logger that allows for an optional verbose mode and still calls the actual logger

parallel_logger.py

import json
import yaml
import logging as real_logger

class ParallelLogger:
    def __init__(self, output_file=None, verbose=False):
        self.data = []
        self._output_set = False
        self.set_file(output_file)
        self.verbose = verbose

    def _store_and_log(self, log_meth, message=None, verbose_message=None):
        if self.verbose and verbose_message:
            message = verbose_message
        if not logging._nameToLevel[log_meth.__name__.upper()] >= real_logger.level:
            # lower log level, ignore it
            return
        if not self._output_set:
            # if an output file hasn't been specified, store the data for later
            self.data.append(message)
        else:
            self._update_output_file(message)
        if isinstance(message, (list, dict)):
            message = yaml.dump(message, default_flow_style=False, sort_keys=False)
        log_meth(message)

    def _update_output_file(self, data):
        curr_data = json.loads(self.output_file.read_text() or "[]")
        if isinstance(data, list):
            curr_data.extend(data)
        else:
            curr_data.append(data)
        self.output_file.write_text(json.dumps(curr_data, indent=4))

    def set_file(self, output_file=None):
        if not output_file:
            return
        self.output_file = Path(output_file)
        self.output_file.parent.mkdir(exist_ok=True, parents=True)
        if self.output_file.exists():
            self.output_file.unlink()
        self.output_file.touch()
        self._output_set = True
        if self.data:
            self._update_output_file(self.data)
            self.data = []

    def __getattribute__(self, name):
        # pass along calls to the actual logger
        if name.lower() in ("debug", "info", "warning", "error"):
            log_meth = getattr(real_logger, name)
            return partial(self._store_and_log, log_meth)
        elif (found := getattr(real_logger, name, None)):
            return found
        return super().__getattribute__(name)
      
logger = ParallelLogger()