Two scripts for creating heatmap of OSM data size, e.g. https://tiles.mapbox.com/migurski/map/osmsize

measure-sizes.py

from sys import argv, stdout, stderr
from subprocess import Popen
from os import stat, unlink
from math import log, ceil
from stat import ST_SIZE
from time import time

from ModestMaps.OpenStreetMap import Provider
from ModestMaps.Core import Coordinate

osm = Provider()

if __name__ == '__main__':

    stdout.write('coord\tfile\tsize\ttime\n')
    stdout.flush()
    
    tile, original_file = argv[1:]
    
    z, x, y = [int(part) for part in tile.split('/')]
    coords = [(Coordinate(y, x, z), original_file)]
    
    while len(coords):
        coord, file = coords.pop(0)
        size = stat(file)[ST_SIZE]
        
        if size < 1024 or coord.zoom >= 9:
            # nothing useful fits inside 1KB, and don't descend past z9

            stdout.write('%d/%d/%d\t%s\t%d\t\n' % (coord.zoom, coord.column, coord.row, file, size))
        
        else:

            ul = coord.zoomBy(1)
            ur = ul.right()
            lr = ur.down()
            ll = lr.left()
    
            osmosis = '/usr/local/bin/osmosis --rb ___ --log-progress interval=60 --tee outputCount=4'.split()
            osmosis[2] = file
            
            for other_coord in (ul, ll, ur, lr):
                ne = osm.coordinateLocation(other_coord.right())
                sw = osm.coordinateLocation(other_coord.down())
                
                zoom = other_coord.zoom
                length = '%d' % max(1, ceil(log(2**zoom) / log(10)))
                x, y = [('%0' + length + 'd') % val for val in (other_coord.column, other_coord.row)]
                other_file = 'planet-%(zoom)d-%(x)s-%(y)s.osm.pbf' % locals()
    
                osmosis += ('--bb top=%.6f right=%.6f bottom=%.6f left=%.6f --wb ___' % (ne.lat, ne.lon, sw.lat, sw.lon)).split()
                osmosis[-1] = other_file
                
                coords.insert(0, (other_coord, other_file))
            
            start_time = time()
            osmosis = Popen(osmosis)
            osmosis.wait()
            
            stdout.write('%d/%d/%d\t%s\t%d\t%.1f\n' % (coord.zoom, coord.column, coord.row, file, size, time() - start_time))
        
        if file != original_file:
            unlink(file)

        stdout.flush()

sample shell interaction

from psycopg2 import connect
db = connect(user='osm', database='planet_osm')
cur = db.cursor()
cur.execute('''select SUM(ST_NumPoints(way)), COUNT(osm_id)
               from planet_osm_line
               where way && ST_Transform(ST_SetSRID(ST_MakeBox2D(ST_MakePoint(-123, 37), ST_MakePoint(-122, 38)), 4326), 900913)''')

vertex_count, way_count = cur.fetchone()

tile-renderer.py

from math import log, floor, ceil

from psycopg2 import connect
from PIL.ImageDraw import ImageDraw
from PIL import Image

from ModestMaps.Core import Coordinate
from ModestMaps.Tiles import toMicrosoft

colors = [
    (8,29,88),
    (37,52,148),
    (34,94,168),
    (29,145,192),
    (65,182,196),
    (127,205,187),
    (199,233,180),
    (237,248,177),
    (255,255,217)
    ]

min_size, max_size = 126, 16206851105
log_base = (max_size - min_size) ** (1./len(colors))

def size_color(size):
    """ Return an interpolated color for a given byte size.
    """
    try:
        index = log(size - min_size) / log(log_base)
    except ValueError:
        index = 0
    
    low_index = int(index)
    high_index = low_index + 1
    
    high_mix = index - low_index
    low_mix = 1 - high_mix
    
    r1, g1, b1 = colors[low_index]

    try:
        r2, g2, b2 = colors[high_index]
    except IndexError:
        return colors[-1]
    
    return (int(r1 * low_mix + r2 * high_mix),
            int(g1 * low_mix + g2 * high_mix),
            int(b1 * low_mix + b2 * high_mix))

class Provider:

    def __init__(self, layer):
        """
        """
        pass
    
    def renderTile(self, width, height, srs, coord):
        """
        """
        db = connect(user='osm', database='planet_osm').cursor()
        quad = toMicrosoft(coord.column, coord.row, coord.zoom)
        
        db.execute("""SELECT z, x, y, size
                      FROM data_sizes
                      WHERE quad LIKE '%s%%'
                        AND z = %d""" 
                    % (quad, min(9, coord.zoom + 4)))
        
        img = Image.new('RGB', (width, height), colors[0])
        draw = ImageDraw(img)
        
        coord = coord.zoomBy(8)
        
        for (zoom, column, row, size) in db.fetchall():
            sub_coord = Coordinate(row, column, zoom)
            
            ul = sub_coord.zoomTo(coord.zoom)
            lr = sub_coord.right().down().zoomTo(coord.zoom)
            
            x1 = int(ul.column - coord.column)
            x2 = int(lr.column - coord.column - 1)
            y1 = int(ul.row - coord.row)
            y2 = int(lr.row - coord.row - 1)
            
            draw.rectangle((x1, y1, x2, y2), size_color(size))
        
        return img

tilestache.json

{
    "cache": {"name": "test"},
    "layers":
    {
        "size":
        {
            "provider": {"class": "sizerender:Provider"},
            "preview": {"zoom": 5},
            "bounds": {"high": 9}
        }
    }
}