Python Snippets

shm_dict.py

            pointer, read_only_flag = attr_mmap_data.__array_interface__['data']
            print(pointer)

            pointer, read_only_flag = attr_data.__array_interface__['data']
            print(pointer)

            attr_data2 = np.ndarray(attr_mmap_data.shape, dtype=np.uint8)

            pointer, read_only_flag = attr_data2.__array_interface__['data']
            print(pointer)

shm_test.py

def print_mem_consumption(label):
    current_process = psutil.Process(os.getpid())
    rss = current_process.memory_info().rss / (1024 * 1024)
    vms = current_process.memory_info().vms / (1024 * 1024)

    print(label)
    print("Resident Set Size: % 0.2d (MB), Virtual Memory Size: % 0.2f (MB)" %(rss, vms)) 

def shm_test():
    print_mem_consumption("start")

    gb_1 = 1024*1024*1024
    with open('large_file', 'wb') as fout:
        fout.write(os.urandom(gb_1))
    print_mem_consumption("file Open")

    mmap_data = np.memmap('large_file', dtype=np.uint8, mode='r')
    print_mem_consumption("mmap")

    attr_data = np.ndarray(mmap_data.shape, dtype=np.uint8, buffer=mmap_data)
    print_mem_consumption("assign buffer")

if __name__ == "__main__":
    # main()
    shm_test()

# start
# Resident Set Size:  54 (MB), Virtual Memory Size:  598.82 (MB)
# file Open
# Resident Set Size:  54 (MB), Virtual Memory Size:  598.82 (MB)
# mmap
# Resident Set Size:  54 (MB), Virtual Memory Size:  1622.82 (MB)
# assign buffer
# Resident Set Size:  54 (MB), Virtual Memory Size:  1622.82 (MB)