benchmarking lasio for speed and memory usage

mem.py

from ctypes import *
from ctypes.wintypes import *
from collections import namedtuple

__all__ = ['query_working_set', 'working_set_size']

kernel32 = WinDLL('kernel32', use_last_error=True)
psapi = WinDLL('psapi', use_last_error=True)

PROCESS_VM_READ           = 0x0010
PROCESS_QUERY_INFORMATION = 0x0400

ERROR_ACCESS_DENIED = 0x0005
ERROR_BAD_LENGTH    = 0x0018

ULONG_PTR = WPARAM
SIZE_T = c_size_t

class PSAPI_WORKING_SET_BLOCK(Union):
    class _FLAGS(Structure):
        _fields_ = (('Protection',  ULONG_PTR,  5),
                    ('ShareCount',  ULONG_PTR,  3),
                    ('Shared',      ULONG_PTR,  1),
                    ('Reserved',    ULONG_PTR,  3),
                    ('VirtualPage', ULONG_PTR, 20))
    _anonymous_ = '_flags',
    _fields_ = (('Flags', ULONG_PTR),
                ('_flags', _FLAGS))

class PSAPI_WORKING_SET_INFORMATION(Structure):
    _fields_ = (('NumberOfEntries',  ULONG_PTR),
                ('_WorkingSetInfo', PSAPI_WORKING_SET_BLOCK * 1))
    @property
    def WorkingSetInfo(self):
        array_t = PSAPI_WORKING_SET_BLOCK * self.NumberOfEntries
        offset = PSAPI_WORKING_SET_INFORMATION._WorkingSetInfo.offset
        return array_t.from_buffer(self, offset)

PPSAPI_WORKING_SET_INFORMATION = POINTER(PSAPI_WORKING_SET_INFORMATION)

def errcheck_bool(result, func, args):
    if not result:
        raise WinError(get_last_error())
    return args

psapi.QueryWorkingSet.errcheck = errcheck_bool
psapi.QueryWorkingSet.argtypes = (
    HANDLE,                         # _In_  hProcess
    PPSAPI_WORKING_SET_INFORMATION, # _Out_ pv
    DWORD)                          # _In_  cb

kernel32.GetCurrentProcess.restype = HANDLE

kernel32.OpenProcess.errcheck = errcheck_bool
kernel32.OpenProcess.restype = HANDLE
kernel32.OpenProcess.argtypes = (
    DWORD, # _In_ dwDesiredAccess
    BOOL,  # _In_ bInheritHandle
    DWORD) # _In_ dwProcessId

def query_working_set(pid=None):
    """Return the PSAPI_WORKING_SET_BLOCK array for the target process."""
    if pid is None:
        hprocess = kernel32.GetCurrentProcess()
    else:
        access = PROCESS_VM_READ | PROCESS_QUERY_INFORMATION
        hprocess = kernel32.OpenProcess(access, False, pid)
    info = PSAPI_WORKING_SET_INFORMATION()
    base_size = sizeof(info)
    item_size = sizeof(PSAPI_WORKING_SET_BLOCK)
    overshoot = 0
    while True:
        overshoot += 4096
        n = info.NumberOfEntries + overshoot
        resize(info, base_size + n * item_size)
        try:
            psapi.QueryWorkingSet(hprocess, byref(info), sizeof(info))
            break
        except OSError as e:
            if e.winerror != ERROR_BAD_LENGTH:
                raise
    return info.WorkingSetInfo

class PERFORMANCE_INFORMATION(Structure):
    _fields_ = (('cb',                DWORD),
                ('CommitTotal',       SIZE_T),
                ('CommitLimit',       SIZE_T),
                ('CommitPeak',        SIZE_T),
                ('PhysicalTotal',     SIZE_T),
                ('PhysicalAvailable', SIZE_T),
                ('SystemCache',       SIZE_T),
                ('KernelTotal',       SIZE_T),
                ('KernelPaged',       SIZE_T),
                ('KernelNonpaged',    SIZE_T),
                ('PageSize',          SIZE_T),
                ('HandleCount',       DWORD),
                ('ProcessCount',      DWORD),
                ('ThreadCount',       DWORD))
    def __init__(self, *args, **kwds):
        super(PERFORMANCE_INFORMATION, self).__init__(*args, **kwds)
        self.cb = sizeof(self)

PPERFORMANCE_INFORMATION = POINTER(PERFORMANCE_INFORMATION)

psapi.GetPerformanceInfo.errcheck = errcheck_bool
psapi.GetPerformanceInfo.argtypes = (
    PPERFORMANCE_INFORMATION, # _Out_ pPerformanceInformation
    DWORD)                    # _In_  cb

WorkingSetSize = namedtuple('WorkingSetSize', 'total shared private')

def working_set_size(pid=None):
    """Return the total, shared, and private working set sizes
       for the target process.
    """
    wset = query_working_set(pid)
    pinfo = PERFORMANCE_INFORMATION()
    psapi.GetPerformanceInfo(byref(pinfo), sizeof(pinfo))
    pagesize = pinfo.PageSize        
    total = len(wset) * pagesize
    shared = sum(b.Shared for b in wset) * pagesize
    private = total - shared
    return WorkingSetSize(total, shared, private)

memory_test.py

import psutil
import os
import subprocess
import sys

import lasio
import mem


def memory():
    total, shared, private = mem.working_set_size()
    total = float(total)
    return total / (1000 * 1000)


def file_test(fn, m0):
    print('!\t%.3f\t\t' % (os.stat(fn).st_size / (1000 * 1000), ), end='')
    l = lasio.read(fn)
    print('%.3f' % (memory() - m0, ))
    return memory() - m0


def test(n):
    # m = psutil.virtual_memory()
    # m0 = m.used / (1000 * 1000)
    print('!\tLAS file (MB)\tUsed Mem. (MB)')
    dm = 0
    for i in range(n):
        m0 = memory()
        dm += file_test('5MB.las', m0)
    print('!\t\t\t-----------------')
    print('!\t\t\taverage=%.3f MB\n!' % (dm / n, ))
    # l2 = file_test('9MB.las', m0)
    # l3 = file_test('16MB.las', m0)
    # l4 = file_test('21MB.las', m0)
    # l5 = file_test('52MB.las', m0)
    # l6 = file_test('193MB.las', m0)


if __name__ == '__main__':
    d = os.getcwd()
    os.chdir(r'..\lasio')
    git_cmd = 'git show --name-status --oneline'
    out = subprocess.check_output(git_cmd).decode('ascii')
    print('!$ %s\n! %s\n!' % (git_cmd, out.split('\n')[0]))
    os.chdir(d)
    test(int(sys.argv[1]))

result.txt

!$ git show --name-status --oneline
! f0d95fd Add command-line version script
!
!	LAS file (MB)	Used Mem. (MB)
!	5.112		10.985
!	5.112		9.282
!	5.112		9.101
!	5.112		9.101
!	5.112		9.097
!	5.112		9.105
!	5.112		9.101
!	5.112		9.101
!	5.112		9.101
!	5.112		7.004
!			-----------------
!			average=9.098 MB
!
   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
       10    0.001    0.000   27.455    2.745 memory_test.py:16(file_test)

run_memory_test.bat

python -m cProfile memory_test.py 10 | grep -E 'file_test^|!^|percall' > result.txt