ActiveRecord vs Machinist vs FactoryGirl

fixture_replacement_benchmark.rb

#!/usr/bin/env ruby
require "rubygems"
require "benchmark"

########################################################################
# ### Without association ###
#                                user     system      total        real
# # baseline
# 10k Hash.new               0.010000   0.000000   0.010000 (  0.009023)
# 10k INSERT                 0.260000   0.000000   0.260000 (  0.267696)
# # creation
# 10k ActiveRecord.new       0.560000   0.010000   0.570000 (  0.561774)
# 10k Machinist.make         0.940000   0.020000   0.960000 (  0.958489)
# 10k Factory.build          2.100000   0.090000   2.190000 (  2.190935)
# # persistence
# 10k ActiveRecord.create    5.470000   0.080000   5.550000 (  5.549147)
# 10k Machinist.make!        9.860000   0.190000  10.050000 ( 10.052224)
# 10k Factory.create         7.160000   0.180000   7.340000 (  7.337876)
# 
# ### With belongs_to association ###
#                                user     system      total        real
# # baseline
# 10k Hash.new               0.010000   0.000000   0.010000 (  0.012028)
# # creation
# 10k ActiveRecord.new       1.260000   0.020000   1.280000 (  1.282245)
# 10k Machinist.make         2.360000   0.050000   2.410000 (  2.411901)
# 10k Factory.build         10.980000   0.350000  11.330000 ( 11.320351)
# # persistence
# 10k ActiveRecord.create   10.890000   0.160000  11.050000 ( 11.056770)
# 10k Machinist.make!       16.220000   0.300000  16.520000 ( 16.517063)
# 10k Factory.create        16.480000   0.440000  16.920000 ( 16.932647)
########################################################################

require 'active_record'

ActiveRecord::Base.establish_connection(
  :adapter  => 'sqlite3',
  :database => ':memory:'
)

ActiveRecord::Base.connection.execute %{
  CREATE TABLE depts (
    id INTEGER PRIMARY KEY ASC,
    name VARCHAR(255)
  )
}

ActiveRecord::Base.connection.execute %{
  CREATE TABLE emps (
    id INTEGER PRIMARY KEY ASC,
    name VARCHAR(255),
    dept_id INTEGER
  )
}

class Dept < ActiveRecord::Base
  attr_accessible :name
  has_many :emps
end

class Emp < ActiveRecord::Base
  attr_accessible :name
  belongs_to :dept
end

##################################################
require 'factory_girl'

FactoryGirl.define do
  factory :dept do
    name "Development"
  end

  factory :emp do
    name "John Doe"
    dept
  end
end

##################################################
require 'machinist/active_record'

Dept.blueprint do
  name {"Development"}
end

Emp.blueprint do
  name {"John Doe"}
  dept
end

##################################################

n = 10
m = 1000 * n

def with_clean_db
  ActiveRecord::Base.connection.execute "DELETE FROM depts"
  ActiveRecord::Base.connection.execute "DELETE FROM emps"
  yield
end

# warm up
with_clean_db do
  100.times { Dept.create(:name => 'Development') }
  100.times { FactoryGirl.create :dept }
  100.times { Dept.make! }
  100.times { Emp.create(:name => 'John Doe') }
  100.times { FactoryGirl.create :emp }
  100.times { Emp.make! }
end

puts "### Without association ###"
Benchmark.bm(25) do |x|
puts "# baseline"

  x.report("#{n}k Hash.new") do
    m.times { Hash.new(:name => 'Development') }
  end

  with_clean_db do
    conn = Emp.connection.raw_connection
    x.report("#{n}k INSERT") do
      m.times { conn.execute 'INSERT INTO "emps" ("name") VALUES ("John Doe")' }
    end
  end

puts "# creation"

  with_clean_db do
    x.report("#{n}k ActiveRecord.new") do
      m.times { Dept.new(:name => 'Development') }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make") do
      m.times { Dept.make }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.build") do
      m.times { FactoryGirl.build :dept }
    end
  end

puts "# persistence"

  with_clean_db do
    x.report("#{n}k ActiveRecord.create") do
      m.times { Dept.create(:name => 'Development') }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make!") do
      m.times { Dept.make! }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.create") do
      m.times { FactoryGirl.create :dept }
    end
  end
end
puts

puts "### With belongs_to association ###"
Benchmark.bm(25) do |x|
puts "# baseline"

  x.report("#{n}k Hash.new") do
    m.times { Hash.new(:name => 'John Doe', :dept => {:name => 'Development'}) }
  end

puts "# creation"

  with_clean_db do
    x.report("#{n}k ActiveRecord.new") do
      m.times { Emp.new(:name => 'John Doe', :dept => Dept.new(:name => 'Development')) }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make") do
      m.times { Emp.make }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.build") do
      m.times { FactoryGirl.build :emp }
    end
  end

puts "# persistence"

  with_clean_db do
    x.report("#{n}k ActiveRecord.create") do
      m.times { Emp.create(:name => 'John Doe', :dept => Dept.create(:name => 'Development')) }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make!") do
      m.times { Emp.make! }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.create") do
      m.times { FactoryGirl.create :emp }
    end
  end
end
puts

Gemfile

source 'http://rubygems.org'
gem 'activerecord'
gem 'sqlite3'
gem 'factory_girl', '= 2.6.4'
gem 'machinist'

Gemfile.lock

GEM
  remote: http://rubygems.org/
  specs:
    activemodel (3.2.6)
      activesupport (= 3.2.6)
      builder (~> 3.0.0)
    activerecord (3.2.6)
      activemodel (= 3.2.6)
      activesupport (= 3.2.6)
      arel (~> 3.0.2)
      tzinfo (~> 0.3.29)
    activesupport (3.2.6)
      i18n (~> 0.6)
      multi_json (~> 1.0)
    arel (3.0.2)
    builder (3.0.0)
    factory_girl (2.6.4)
      activesupport (>= 2.3.9)
    i18n (0.6.0)
    machinist (2.0)
    multi_json (1.3.6)
    sqlite3 (1.3.6)
    tzinfo (0.3.33)

PLATFORMS
  ruby

DEPENDENCIES
  activerecord
  factory_girl (= 2.6.4)
  machinist
  sqlite3

@jsanders I agree we could do this manually but (except for the performance hit) I like a little help to do it all in one place. I think the organization aspect is a win. That said, a quick browse through the FactoryGirl code and these benchmarks show the thing is totally wasteful. A little work to improve its performance would be a good investment IMO.

@thinkerbot - I'm fully in favor of having a set of methods in a common place to build commonly-built object networks, but I don't see why you need a library to do that.

I agree with @thinkerbot that the first step should be to see if a little work to improve FactoryGirl might help. It's easier to tune an existing thing than to replace it.

Another related thing we could do is to abstract away from the implementation so that we could replace the implementation in the future. It's not clear if this is possible but if we did FactoryDude.create(...) and had FactoryDude call FactoryGirl now and maybe Machinist in the future. But would that just be add more overhead and make it slower? Hmmm.

Oh, another reason to use FactoryGirl - we use it everywhere already!

Edit. Yeah, what @tra said.

I am now going to start an object-building framework called SurferDude as an antidote to FactoryGirl.

tldr I found I could make FactoryGirl a lot faster, and that it really doesn't matter.

Before (thinkerbot/factory_girl@f9ab6c7549942c610ba8dba29ff3228a7d2b1ec2):

$ bundle exec ruby -Ilib benchmark/factory_girl_benchmark.rb 
                               user     system      total        real
# baseline
10k Hash.new               0.010000   0.000000   0.010000 (  0.009226)
# creation
10k ActiveRecord.new       0.610000   0.010000   0.620000 (  0.628761)
10k Factory.build          2.090000   0.100000   2.190000 (  2.188184)
# peristence
10k ActiveRecord.create    4.750000   0.090000   4.840000 (  4.842905)
10k Factory.create         6.540000   0.200000   6.740000 (  6.743012)

After (thinkerbot/factory_girl@416977fc4e14b53e9a0381e6dc654ac70382e602):

$ bundle exec ruby -Ilib benchmark/factory_girl_benchmark.rb 
                               user     system      total        real
# baseline
10k Hash.new               0.010000   0.000000   0.010000 (  0.009660)
# creation
10k ActiveRecord.new       0.640000   0.010000   0.650000 (  0.645614)
10k Factory.build          0.890000   0.010000   0.900000 (  0.907428)
# peristence
10k ActiveRecord.create    4.920000   0.100000   5.020000 (  5.012222)
10k Factory.create         5.320000   0.120000   5.440000 (  5.433966)

Largely the improvements were won by ensuring FactoryGirl doesn't recalculate known information for every build/create, and to a lesser degree creating fewer Procs. Unfortunately the improvements don't matter. The percent improvement is pretty good but in absolute terms it's nothing. If you look at the persistence numbers you can see 1.2s are saved over 10k creates. That's not a lot. Indeed over the whole spec suite it's only about 2 min.

What is interesting is that these tests are run vs an in-memory sqlite3 database so the database time is minimal (~ 0.27s), and yet ActiveRecord is still taking it's sweet time to create records. Perhaps it's ActiveRecord and not FactoryGirl that's slow.