test_dict_mem.py

import sys
from six.moves import range
import time
import psutil
import resource
import random

M = 1024*1024
p = psutil.Process()
d = {}

def timing(f):
    return f
    def _():
        t1 = time.time()

        for i in range(1000):
            r = f()
        t2 = time.time()
        return r, (t2-t1)
    return _


@timing
def dmem():
    return sys.getsizeof(d)/M


@timing
def pmem():
    return p.memory_info().rss/M


@timing
def statm():
    with open('/proc/self/statm','r') as f:
        v = f.readline()
        return int(v.split()[1])*4096/M

@timing
def length():
    return len(d)


def max_rss():
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024


def f1():
    for i in range(M * 1000):
        #d[i] = i
        d[i] = str(i) * 10
        if i % M == 0:
            print(i>>20,  statm(), pmem(), dmem(), length())


def f2():
    last_rss = 0
    last_dsize = 0
    for i in range(M * 1000):
        d[i] = i
        # d[int(i/2)] = i
        d[i] = str(i) * 10
        rss = statm()
        dsize = sys.getsizeof(d)
        if dsize != last_dsize:
            print("%d %s:  mem %d + %d -> %d, max %d; size %d + %d -> %d" % (i, hex(i), last_rss, rss-last_rss, rss, max_rss() /M, last_dsize/M, (dsize - last_dsize)/M, dsize/M))

        last_dsize = dsize
        last_rss = rss

def f2():
    last_rss = 0
    last_dsize = 0
    for i in range(M * 1000):
        d[i] = i
        # d[int(i/2)] = i
        d[i] = str(i) * 10
        rss = statm()
        dsize = sys.getsizeof(d)
        if dsize != last_dsize:
            print("%d %s:  mem %d + %d -> %d, max %d; size %d + %d -> %d" % (i, hex(i), last_rss, rss-last_rss, rss, max_rss() /M, last_dsize/M, (dsize - last_dsize)/M, dsize/M))

        last_dsize = dsize
        last_rss = rss


def f2_2():
    last_rss = 0
    last_dsize = 0

    S = 0x1aaaaaa + 1
    vs = list(range(S))
    random.shuffle(vs)

    for i in vs:
        d[i] = i
        rss = statm()
        dsize = sys.getsizeof(d)
        if dsize != last_dsize:
            print("%d %s:  mem %d + %d -> %d, max %d; size %d + %d -> %d" % (i, hex(i), last_rss, rss-last_rss, rss, max_rss() /M, last_dsize/M, (dsize - last_dsize)/M, dsize/M))

        last_dsize = dsize
        last_rss = rss


def f2_3():
    last_rss = 0
    last_dsize = 0
    for i in range(M * 1000):
        k = random.randint(0, 1<<30)
        d[k] = i
        rss = statm()
        dsize = sys.getsizeof(d)
        if dsize != last_dsize:
            print("%d %s:  mem %d + %d -> %d, max %d; size %d + %d -> %d" % (i, hex(i), last_rss, rss-last_rss, rss, max_rss() /M, last_dsize/M, (dsize - last_dsize)/M, dsize/M))

        last_dsize = dsize
        last_rss = rss


def f3():
    import random
    d = dict()
    t = time.time()
    for i in range(S):
        #k = (i, i, i, i)
        k = i
        d[k] = i
    t = time.time() - t
    print("dict ", t, statm())
    del d


def f4():
    S = 0xaaaaaa-1
    N = 100
    dd = [dict() for i in range(N)]
    t = time.time()
    for i in range(S):
        #k = (i, i, i, i)
        k = i
        h = hash(k) % N
        dd[h][k] = i
    t = time.time() - t
    print("%d dicts " % N, t, statm())


def f5():
    import random

    S = 0xaaaaaa-1 + 100
    vs = list(range(S))
    random.shuffle(vs)

    d = dict()
    t = time.time()
    for i in vs:
        #k = i
        #k = str(i)
        k = (i, i, i, i)
        d[k] = i
    t = time.time() - t
    print("dict ", t, statm())
    del d



def f6():
    import random

    S = 0xaaaaaa-1+100
    vs = list(range(S))
    random.shuffle(vs)

    N = 100
    dd = [dict() for i in range(N)]
    t = time.time()
    for i in vs:
        #k = i
        #k = str(i)
        k = (i, i, i, i)
        h = hash(k) % N
        dd[h][k] = i
    t = time.time() - t
    print("dict ", t, statm())


def f7():
    import random
    sizes0 = [0xaaaaaa, 0x1555555, 0x2aaaaaa]
    sizes = []
    for s in sizes0:
        sizes.append(s-1)
        sizes.append(s)
    print(sizes)

    def _(n):
        random.seed(1)
        d = {}
        dd = [dict() for i in range(n)]
        target_idx = 0
        target_count = sizes[target_idx]
        time_mems = []
        t0 = time.time()
        msg = "%3d" % n
        for i in range(sizes[-1] + 1):
            k = "%07x" % (i, )
            if n == 1:
                d[k] = i
            else:
                h = hash(k) % n
                dd[h][k] = i
            if i == target_count:
                m = statm()
                #print(hex(i), m)
                t = time.time() - t0
                time_mems.append((i, t, m))
                msg += " (0x%07x %.2fs %5dM)" % (i,t,m)
                target_idx += 1
                if target_idx < len(sizes):
                    target_count = sizes[target_idx]
        del d
        del dd
        print(msg)
        return time_mems

    _(1)
    _(139)


f7()

f1

(0, (11, 0.014364004135131836), (11, 0.02672886848449707), (0, 0.0004909038543701172), (1, 0.00017499923706054688))
(1, (83, 0.014297962188720703), (83, 0.02685713768005371), (48, 0.0004858970642089844), (1048577, 0.00017786026000976562))
(2, (156, 0.014461040496826172), (156, 0.0269320011138916), (96, 0.00048804283142089844), (2097153, 0.00018286705017089844))
(3, (277, 0.014409065246582031), (277, 0.026877880096435547), (192, 0.0004899501800537109), (3145729, 0.0001881122589111328))
(4, (301, 0.014414072036743164), (301, 0.026740074157714844), (192, 0.0004968643188476562), (4194305, 0.0001819133758544922))
(5, (326, 0.014369964599609375), (326, 0.026819944381713867), (192, 0.0004909038543701172), (5242881, 0.00017786026000976562))
(6, (542, 0.014636993408203125), (542, 0.026791810989379883), (384, 0.0004940032958984375), (6291457, 0.0001800060272216797))
(7, (567, 0.014297008514404297), (567, 0.026976823806762695), (384, 0.00048804283142089844), (7340033, 0.00017905235290527344))
(8, (592, 0.014339208602905273), (592, 0.026766061782836914), (384, 0.0004899501800537109), (8388609, 0.0001800060272216797))

f2: resize 位置 二进制 10101...

0 0x0:  mem 0 + 11 -> 11; size 0 + 0 -> 0
5 0x5:  mem 11 + 0 -> 11; size 0 + 0 -> 0
21 0x15:  mem 11 + 0 -> 11; size 0 + 0 -> 0
85 0x55:  mem 11 + 0 -> 11; size 0 + 0 -> 0
341 0x155:  mem 11 + 0 -> 11; size 0 + 0 -> 0
1365 0x555:  mem 11 + 0 -> 11; size 0 + 0 -> 0
5461 0x1555:  mem 11 + 1 -> 12; size 0 + 0 -> 0
21845 0x5555:  mem 12 + 3 -> 15; size 0 + 2 -> 3
87381 0x15555:  mem 16 + 3 -> 19; size 3 + 3 -> 6
174762 0x2aaaa:  mem 21 + 6 -> 27; size 6 + 6 -> 12
349525 0x55555:  mem 31 + 12 -> 43; size 12 + 12 -> 24
699050 0xaaaaa:  mem 51 + 25 -> 76; size 24 + 24 -> 48
1398101 0x155555:  mem 92 + 48 -> 140; size 48 + 48 -> 96
2796202 0x2aaaaa:  mem 173 + 96 -> 269; size 96 + 96 -> 192
5592405 0x555555:  mem 334 + 192 -> 526; size 192 + 192 -> 384
11184810 0xaaaaaa:  mem 657 + 384 -> 1041; size 384 + 384 -> 768
22369621 0x1555555:  mem 1304 + 768 -> 2072; size 768 + 768 -> 1536
44739242 0x2aaaaaa:  mem 2596 + 1536 -> 4132; size 1536 + 1536 -> 3072
89478485 0x5555555:  mem 5181 + 3072 -> 8253; size 3072 + 3072 -> 6144

randint 做key 值就变了！

0 0x0:  mem 0 + 12 -> 12, max 12; size 0 + 0 -> 0
5 0x5:  mem 12 + 0 -> 12, max 12; size 0 + 0 -> 0
21 0x15:  mem 12 + 0 -> 12, max 12; size 0 + 0 -> 0
85 0x55:  mem 12 + 0 -> 12, max 12; size 0 + 0 -> 0
341 0x155:  mem 12 + 0 -> 12, max 12; size 0 + 0 -> 0
1365 0x555:  mem 12 + 0 -> 12, max 12; size 0 + 0 -> 0
5461 0x1555:  mem 12 + 1 -> 13, max 13; size 0 + 0 -> 0
21846 0x5556:  mem 14 + 2 -> 16, max 17; size 0 + 2 -> 3
87386 0x1555a:  mem 19 + 3 -> 22, max 25; size 3 + 3 -> 6
174775 0x2aab7:  mem 26 + 6 -> 32, max 38; size 6 + 6 -> 12
349575 0x55587:  mem 40 + 12 -> 52, max 64; size 12 + 12 -> 24
699257 0xaab79:  mem 69 + 24 -> 93, max 117; size 24 + 24 -> 48
1399064 0x155918:  mem 126 + 48 -> 174, max 221; size 48 + 48 -> 96