Strava archiver

README.txt

1. install postgres
2. run makedb.rb >tiles.csv
3. run tiles.sql
4. run archive.rb
5. enjoy

archive.rb

#!/usr/bin/env ruby

require 'pg'
require 'fileutils'
require 'net/http'

db = {}
nthr = 64

def domain
  s = %w{a b c}.sample
  "heatmap-external-#{s}.strava.com"
end

def url(z, x, y)
  "http://#{domain}/tiles/all/hot/#{z}/#{x}/#{y}@2x.png?v=19"
end

queue = []

nthr.times do
  Thread.new do
    conn = PG::Connection.open(db)
    Net::HTTP.start domain do |http|
      loop do
        z, x, y = nil
        loop do
          task = queue.pop
          if task.nil?
            sleep rand
          else
            z, x, y = task
            break
          end
        end

        tile = http.get(url(z, x, y)).body
        FileUtils.mkdir_p "data/#{z}/#{x}"
        File.write("data/#{z}/#{x}/#{y}.png", tile)

        conn.exec %{UPDATE tiles SET done = 't', size = #{tile.length}
                    WHERE z = #{z} AND x = #{x} AND y = #{y}}

        puts "#{z} #{x.rjust(4)} #{y.rjust(4)}"
      end
    end
  end
end

conn = PG::Connection.open(db)
loop do
  sleep 1 while queue.length > nthr
  conn.exec %{SELECT z, x, y FROM tiles
              WHERE done = 'f' ORDER BY z
              LIMIT 1000} do |rows|
    break unless rows.any?

    rows.each do |row|
      queue.push([row["z"], row["x"], row["y"]])
    end
  end
end

makedb.rb

#!/usr/bin/env ruby

14.times do |z|
  (2**z).times do |x|
    (2**z).times do |y|
      puts %{#{z},#{x},#{y},0,}
    end
  end
end

stats.rb

#!/usr/bin/env ruby

require 'pg'

conn = PG::Connection.open

total_count = 0
total_size  = 0

last_z   = 0
last_avg = 0
14.times do |z|
  size, = *conn.exec(%{SELECT MIN(size), MAX(size), AVG(size)
                       FROM tiles WHERE size > 0 AND z = #{z}})
  break if size["avg"].nil?

  count    = 2**z*2**z
  avg_size = size["avg"].to_i

  puts("Z=#{z}:")
  puts("  File size:")
  puts("    min    #{size["min"]}")
  puts("    max    #{size["max"]}")
  puts("    mean   #{size["avg"]}")
  puts("    median #{size["med"]}")
  puts("  Count:")
  puts("    #{count}")
  puts("  Predicted size:")
  puts("    #{count*avg_size}")
  puts

  total_count += count
  total_size  += count*avg_size

  last_z = z
  last_avg = avg_size
end

(last_z+1...14).each do |z|
  count = 2**z*2**z

  puts("Z=#{z}: (extrapolated)")
  puts("  File size:")
  puts("    mean   #{last_avg}")
  puts("  Count:")
  puts("    #{count}")
  puts("  Predicted size:")
  puts("    #{count*last_avg}")
  puts

  total_count += count
  total_size  += count*last_avg
end

puts("Grand total:")
puts("  Count:")
puts("    #{total_count}")
puts("  Predicted size:")
puts("    #{total_size}")

tiles.sql

CREATE TABLE tiles (z INT, x INT, y INT, done BOOLEAN, size INT);
CREATE INDEX coords ON tiles (z, x, y);
COPY tiles FROM 'tiles.csv' WITH (FORMAT csv);