Storing product activity from net-a-porter

backup.rb

require 'sqlite3'
sdb = SQLite3::Database.new('net-a-porter.sqlite3')
ddb = SQLite3::Database.new('net-a-porter-backup.sqlite3')

b = SQLite3::Backup.new(ddb, 'main', sdb, 'main')

begin
  b.step(1)
end while b.remaining > 0

b.finish

net-a-porter-product-activity.rb

require 'digest/md5'
require 'open-uri'
require 'logger'
require 'json'
require 'sequel'

LOG = Logger.new($stdout)
DB = Sequel.sqlite("net-a-porter.sqlite3", loggers: [LOG])

Sequel::Model.plugin(:schema)

class Activity < Sequel::Model
  set_schema do
    String :id, primary_key: true

    Time :activityCreated
    Float :price
    Float :latitude
    Float :longitude
    String :activityChannel
    String :activityType
    String :city
    String :country
    String :description
    String :designer
    String :hash
    String :imgURI
    String :pid
    String :productChannel
    String :productPageURI
    String :title
  end

  create_table?
end

url = 'http://www.net-a-porter.com/webapi/feed/productactivity.json?channel=NAP_AM&size=100'

loop do
  begin
    open(url) do |io|
      JSON.parse(io.read)['productactivity'].each do |given|
        id = Digest::MD5.hexdigest(given.values.map(&:to_s).sort.join)

        unless Activity[id: id]
          activity = Activity.new
          given.each do |key, value|
            case key
            when 'activityCreated'
              value = Time.at(value / 1000)
            end

            activity[key] = value
          end

          activity.id = id
          activity.save
        end
      end

      puts "Currently #{Activity.count} product activities"
    end
  rescue Exception => error
    puts "#{error} (#{error.class})"
    puts error.backtrace
  end

  sleep 60
end