Multi process logging with os.fork() and logging module

multi-process-logging.py

import os
import logging


def generate_primes(number):
    logger = logging.getLogger(str(os.getpid()) + ':logger')

    for num in range(number + 1):
       # prime numbers are greater than 1
       if num > 1:
           for i in range(2,num):
               if (num % i) == 0:
                   break
           else:
               logger.info('child:' + str(os.getpid()) + ':' + str(num))

def start_workers():
    logger = logging.getLogger('parent:prime')
    logger.info('parent forking')
    children = []

    for child in range(10):
        pid = os.fork()

        if not pid:

            logger = logging.getLogger(str(os.getpid()) + ':logger'))
            fh = logging.FileHandler('logs/child_primes_' + str(os.getpid()) + '.log')
            logger.addHandler(fh)

            generate_primes(10000)

            os._exit(0)
        else:
            children.push(pid)
            
    for child_pid in children:
        os.waitpid(child_pid, 0)


if __name__ == "__main__":

    logging.basicConfig(filename='logs/parent_primes.log', level=logging.INFO)
    logger = logging.getLogger('parent:prime')

    logger.info('Fork parent/child logging example.')
    logger.info('Starting children')

    start_workers()

In your new approach it looks like you are going to wait for each child to finish before starting a new one, since you have os.waitpid() in the same loop as your os.fork() call. Maybe add pid in the else: block to an array, and then wait on them outside the first loop? That way you can exercise all the CPU cores on your machine.

Ah yes! So the real script that this is in, I have looping through the children pids just like the updated one here now shows. Also been running the original script with the new logging and it seems to be keeping the logging separate now which is the big win!