Run pep8 in parallel, adapted from https://github.com/jcrocholl/pep8/pull/230 by @jdahlin

pep8q.py

#!python
# -*- coding: utf-8 -*-
# Adapted from a contribution of @jdahlin on jcrocholl/pep8/pull/230

import collections
import multiprocessing

import pep8


class BaseQReport(pep8.BaseReport):
    """Base Queue Report."""

    def __init__(self, options):
        assert options.jobs > 0
        super(BaseQReport, self).__init__(options)
        self.counters = collections.defaultdict(int)
        self.n_jobs = options.jobs

        # init queues
        self.task_queue = multiprocessing.Queue()
        self.result_queue = multiprocessing.Queue()

    def start(self):
        super(BaseQReport, self).start()
        # spawn processes
        for i in range(self.n_jobs):
            p = multiprocessing.Process(target=self.process_main)
            p.start()

    def stop(self):
        # collect queues
        for i in range(self.n_jobs):
            self.task_queue.put('DONE')
            self.update_state(self.result_queue.get())
        super(BaseQReport, self).stop()

    def process_main(self):
        for filename in iter(self.task_queue.get, 'DONE'):
            self.input_file(filename)
        self.result_queue.put(self.get_state())

    def get_state(self):
        return {'total_errors': self.total_errors,
                'counters': self.counters,
                'messages': self.messages}

    def update_state(self, state):
        self.total_errors += state['total_errors']
        for key, value in state['counters'].items():
            self.counters[key] += value
        self.messages.update(state['messages'])


class QueueReport(pep8.StandardReport, BaseQReport):
    """Standard Queue Report."""


def main():
    """Parse options and run checks on Python source."""
    # Prepare
    parser = pep8.get_parser()
    parser.config_options.append('jobs')
    parser.add_option('-j', '--jobs', type='int', default=1,
                      help="number of jobs to run simultaneously")

    pep8style = pep8.StyleGuide(
        parse_argv=True, config_file=True, parser=parser)
    options = pep8style.options

    if options.jobs > 1:
        # Initialize the queue reporter
        reporter = BaseQReport if options.quiet else QueueReport
        report = pep8style.init_report(reporter)

        # Replace the "runner" with the multiprocessing task queue
        report.input_file = pep8style.input_file
        pep8style.runner = report.task_queue.put

    # Run the checkers
    report = pep8style.check_files()

    if options.statistics:
        report.print_statistics()
    if options.benchmark:
        report.print_benchmark()
    if report.total_errors:
        raise SystemExit(1)

if __name__ == '__main__':
    main()