test.rb

=begin

Tokyo Cabinet vs. SQLite indexed string selection & inserting test

Install gems "tokyocabinet" and "sqlite-ruby"

% ruby test.rb
Creating SQlite db..                    user     system      total        real
Generating test data...           100.700000   8.500000 109.200000 (110.876452)
Generating random selection set...  0.160000   0.050000   0.210000 (  0.202189)
Inserting into SQLite...           16.450000  20.430000  36.880000 (143.351468)
Selecting from SQLite...            7.360000   0.670000   8.030000 (  8.103382)
Creating Tokyo db...
Inserting into Tokyo DB...          1.820000   1.570000   3.390000 ( 17.297461)
Selecting from Tokyo DB...          0.710000   0.290000   1.000000 (  1.288947)


=end

require 'rubygems'
require 'sqlite3'
require 'tokyocabinet'
require 'benchmark'
require 'fileutils'

STDOUT.sync = true

def genstring(size)
	s = ""
	symbols = [('a'..'z'), ('A'..'Z'), ('0'..'9')].map(&:to_a).flatten
	size.times do
		s << symbols[rand(symbols.length)]
	end
	s
end

`rm test.db 2> /dev/null`
`rm test.tdb 2>/dev/null`

puts "Creating SQlite db..               "

sqdb = SQLite3::Database.new("test.db")

begin
sqdb.execute("CREATE TABLE test (key VARCHAR(666) PRIMARY KEY, value VARCHAR(666))")
rescue
end

print "Generating test data...           "

data = []
time = Benchmark.measure do
	66666.times do
		data << [genstring(250), genstring(250)]
	end
end

puts time

print "Generating random selection set..."

selects = []
time = Benchmark.measure do
	data.sort_by{ rand 50 }.each do |key, value|
		selects << key if rand(2) == 1
	end
end

puts time

print "Inserting into SQLite...          "

time = Benchmark.measure do
	data.each do |key, value|
		sqdb.execute("REPLACE INTO test VALUES('?', '?')", key, value)
	end
end

puts time

print "Selecting from SQLite...          "

time = Benchmark.measure do
	selects.each do |key|
		sqdb.execute("SELECT value FROM test WHERE key='?'", key)
	end
end

puts time

puts "Creating Tokyo db...              "

tdb = TokyoCabinet::BDB.new
# tdb.tune(128, 256, 32749, 4, 10, TokyoCabinet::BDB::TDEFLATE)
tdb.open("test.tdb", TokyoCabinet::BDB::OREADER | TokyoCabinet::BDB::OCREAT | TokyoCabinet::BDB::OWRITER)

print "Inserting into Tokyo DB...        "

time = Benchmark.measure do
	data.each do |key, value|
		tdb[key] = value
	end
end

puts time

print "Selecting from Tokyo DB...        "

time = Benchmark.measure do
	selects.each do |key|
		tdb[key]
	end
end

puts time


tdb.close
sqdb.close



puts "Creating FS-based DB...              "

FileUtils.mkdir_p "test_data"

print "Inserting into FS...        "

time = Benchmark.measure do
	data.each do |key, value|
		File.open("test_data/#{key}", "w") do |f| f.write value end
	end
end

puts time

print "Selecting from FS...        "

time = Benchmark.measure do
	selects.each do |key|
		File.read("test_data/#{key}")
	end
end

puts time

`rm -rf test_data`