tile processing

results.txt

0 171556606.0
1 10728346.0
2 5160568.0
3 2201743.0
4 934465.0
5 492517.0
6 626750.0
7 1387458.0
8 5171130.0
9 1478714.0
10 958935.0
11 1530604.0
12 5167445.0
13 1498443.0
14 992019.0
15 1566097.0
16 5181424.0
17 1793285.0
18 1803736.0
19 5291919.0
20 1826697.0
21 1875015.0
22 5468056.0
23 2554443.0
24 5357592.0
25 1806703.0
26 1800779.0
27 5205168.0
28 2361447.0
29 5025088.0
30 1646963.0
31 1612457.0
32 4705647.0
33 2112078.0
34 4512719.0
35 1966536.0
36 4085025.0
37 1753077.0
38 3671925.0
39 1560369.0
40 3316492.0
41 1464506.0
42 3209117.0
43 2930640.0
44 1156552.0
45 2299725.0
46 932613.0
47 2048042.0
48 885444.0
49 2031138.0
50 1909978.0
51 770132.0
52 1637935.0
53 701988.0
54 1643607.0
55 1583151.0
56 685797.0
57 1495822.0
58 1427013.0
59 626399.0
60 1336249.0
61 1282510.0
62 578942.0
63 1212300.0
64 1180637.0
65 562595.0
66 1181496.0
67 1226790.0
68 1137266.0
69 549510.0
70 1108695.0
71 1158742.0
72 1081303.0
73 535606.0
74 1074973.0
75 1132340.0
76 1045287.0
77 525925.0
78 1036442.0
79 1115066.0
80 1104211.0
81 991495.0
82 499929.0
83 963072.0
84 1043674.0
85 1051350.0
86 1006987.0
87 889822.0
88 453469.0
89 858756.0
90 925213.0
91 925449.0
92 894341.0
93 839730.0
94 731411.0
95 378914.0
96 699339.0
97 748971.0
98 733946.0
99 714459.0
100 690160.0
101 663663.0
102 627516.0
103 591503.0
104 558578.0
105 520830.0
106 468192.0
107 395822.0
108 200790.0
109 339115.0
110 338067.0
111 299228.0
112 260397.0
113 217555.0
114 178452.0
115 144354.0
116 109973.0
117 84280.0
118 59917.0
119 39860.0
120 22920.0
121 11915.0
122 5823.0
123 2550.0
124 1124.0
125 404.0
126 176.0
127 52.0
128 11.0
129 2.0
130 1.0
208 1.0
209 5.0
210 63.0
211 973.0
212 6206.0
213 22129.0
214 78781.0
215 244201.0
216 629076.0
217 1439400.0
218 3684176.0
219 216604456.0
220 9255949.0
221 8357504.0
222 231716144.0
223 153241440.0
224 6358619.0
225 2717989.0
226 1252963.0
227 486858.0
228 152962.0
229 39821.0
230 6810.0
231 1028.0
232 95.0
233 6.0

tile_hist.py

import sys
import Image
from os import walk 
from os.path import join
from itertools import chain
from collections import defaultdict
from multiprocessing import Pool

def image_hist(image_path):
    hist = defaultdict(float)
    try:
        i = Image.open(image_path)
    except IOError:
        print "failed to open %s" % image_path
        return hist

    pixels = list(i.getdata())
    # optimize not repeating if inside  
    if len(i.getbands()) == 1:
        for p in range(0, len(pixels)):
            u = pixels[p]
            hist[u] += 1.0
    else:
        for p in range(0, len(pixels)):
            u = pixels[p][0]
            hist[u] += 1.0
        
    return hist

def accum_hist(image_hist):
    curr = 0
    hist = defaultdict(float)
    for h in image_hist:
        curr+=1
        if curr%100 == 0:
            print curr
        for x in h:
            hist[x] += h[x]
    return hist
    
def files(folder):
    for path, dirs, files in walk(sys.argv[1]):
        for f in files:
            if ".png" in f:
                yield join(path, f)

if __name__ == '__main__':
    pool = Pool(processes=2)
    # map
    proccessed = pool.imap_unordered(image_hist, files(sys.argv[1]), chunksize=10)
    # reduce
    hist = accum_hist(proccessed)
    for x in hist:
        print x, hist[x]