Report Python memory usage

log_memory.py

import os
from subprocess import check_output, CalledProcessError

from logging import getLogger, basicConfig, INFO

logger = getLogger('main')

def report_progress(message='Progress', progress=0, limit=None):
    try:
        process_ids = list(map(
            int,
            check_output(['pgrep', '-P', str(os.getpid())]).splitlines()))
    except CalledProcessError:
        process_ids = []
    process_ids.append(os.getpid())
    memory = sum(get_process_memory(process_id) for process_id in process_ids)
    memory /= 1024*1024.0
    limit = limit and ' of ' + str(limit) or ''
    logger.info('{}: {}{} using {:.2f}MB.'.format(message,
                                                progress,
                                                limit,
                                                memory))


def get_process_memory(process_id):
    status_file = '/proc/{}/status'.format(process_id)
    with open(status_file) as f:
        status = f.read()
    try:
        i = status.index('VmRSS:')
        fields = status[i:].split(None, 3)
        memory_sizes = {'kB': 1024}
        memory = int(fields[1]) * memory_sizes[fields[2]]
        name = status.splitlines(False)[0].split()[-1]
        logger.info('memory for %s: %dMB', name, memory/1024/1024)
        return memory
    except ValueError:
        return 0


def main():
    basicConfig(level=INFO)
    report_progress('Starting')
    data = []
    loop_count = 10
    for i in range(loop_count):
        data.append('*' * 100000)
        report_progress('looping', i, loop_count)

main()