ActiveRecord vs Machinist vs FactoryGirl

fixture_replacement_benchmark.rb

#!/usr/bin/env ruby
require "rubygems"
require "benchmark"

########################################################################
# ### Without association ###
#                                user     system      total        real
# # baseline
# 10k Hash.new               0.010000   0.000000   0.010000 (  0.009023)
# 10k INSERT                 0.260000   0.000000   0.260000 (  0.267696)
# # creation
# 10k ActiveRecord.new       0.560000   0.010000   0.570000 (  0.561774)
# 10k Machinist.make         0.940000   0.020000   0.960000 (  0.958489)
# 10k Factory.build          2.100000   0.090000   2.190000 (  2.190935)
# # persistence
# 10k ActiveRecord.create    5.470000   0.080000   5.550000 (  5.549147)
# 10k Machinist.make!        9.860000   0.190000  10.050000 ( 10.052224)
# 10k Factory.create         7.160000   0.180000   7.340000 (  7.337876)
# 
# ### With belongs_to association ###
#                                user     system      total        real
# # baseline
# 10k Hash.new               0.010000   0.000000   0.010000 (  0.012028)
# # creation
# 10k ActiveRecord.new       1.260000   0.020000   1.280000 (  1.282245)
# 10k Machinist.make         2.360000   0.050000   2.410000 (  2.411901)
# 10k Factory.build         10.980000   0.350000  11.330000 ( 11.320351)
# # persistence
# 10k ActiveRecord.create   10.890000   0.160000  11.050000 ( 11.056770)
# 10k Machinist.make!       16.220000   0.300000  16.520000 ( 16.517063)
# 10k Factory.create        16.480000   0.440000  16.920000 ( 16.932647)
########################################################################

require 'active_record'

ActiveRecord::Base.establish_connection(
  :adapter  => 'sqlite3',
  :database => ':memory:'
)

ActiveRecord::Base.connection.execute %{
  CREATE TABLE depts (
    id INTEGER PRIMARY KEY ASC,
    name VARCHAR(255)
  )
}

ActiveRecord::Base.connection.execute %{
  CREATE TABLE emps (
    id INTEGER PRIMARY KEY ASC,
    name VARCHAR(255),
    dept_id INTEGER
  )
}

class Dept < ActiveRecord::Base
  attr_accessible :name
  has_many :emps
end

class Emp < ActiveRecord::Base
  attr_accessible :name
  belongs_to :dept
end

##################################################
require 'factory_girl'

FactoryGirl.define do
  factory :dept do
    name "Development"
  end

  factory :emp do
    name "John Doe"
    dept
  end
end

##################################################
require 'machinist/active_record'

Dept.blueprint do
  name {"Development"}
end

Emp.blueprint do
  name {"John Doe"}
  dept
end

##################################################

n = 10
m = 1000 * n

def with_clean_db
  ActiveRecord::Base.connection.execute "DELETE FROM depts"
  ActiveRecord::Base.connection.execute "DELETE FROM emps"
  yield
end

# warm up
with_clean_db do
  100.times { Dept.create(:name => 'Development') }
  100.times { FactoryGirl.create :dept }
  100.times { Dept.make! }
  100.times { Emp.create(:name => 'John Doe') }
  100.times { FactoryGirl.create :emp }
  100.times { Emp.make! }
end

puts "### Without association ###"
Benchmark.bm(25) do |x|
puts "# baseline"

  x.report("#{n}k Hash.new") do
    m.times { Hash.new(:name => 'Development') }
  end

  with_clean_db do
    conn = Emp.connection.raw_connection
    x.report("#{n}k INSERT") do
      m.times { conn.execute 'INSERT INTO "emps" ("name") VALUES ("John Doe")' }
    end
  end

puts "# creation"

  with_clean_db do
    x.report("#{n}k ActiveRecord.new") do
      m.times { Dept.new(:name => 'Development') }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make") do
      m.times { Dept.make }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.build") do
      m.times { FactoryGirl.build :dept }
    end
  end

puts "# persistence"

  with_clean_db do
    x.report("#{n}k ActiveRecord.create") do
      m.times { Dept.create(:name => 'Development') }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make!") do
      m.times { Dept.make! }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.create") do
      m.times { FactoryGirl.create :dept }
    end
  end
end
puts

puts "### With belongs_to association ###"
Benchmark.bm(25) do |x|
puts "# baseline"

  x.report("#{n}k Hash.new") do
    m.times { Hash.new(:name => 'John Doe', :dept => {:name => 'Development'}) }
  end

puts "# creation"

  with_clean_db do
    x.report("#{n}k ActiveRecord.new") do
      m.times { Emp.new(:name => 'John Doe', :dept => Dept.new(:name => 'Development')) }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make") do
      m.times { Emp.make }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.build") do
      m.times { FactoryGirl.build :emp }
    end
  end

puts "# persistence"

  with_clean_db do
    x.report("#{n}k ActiveRecord.create") do
      m.times { Emp.create(:name => 'John Doe', :dept => Dept.create(:name => 'Development')) }
    end
  end

  with_clean_db do
    x.report("#{n}k Machinist.make!") do
      m.times { Emp.make! }
    end
  end

  with_clean_db do
    x.report("#{n}k Factory.create") do
      m.times { FactoryGirl.create :emp }
    end
  end
end
puts

Gemfile

source 'http://rubygems.org'
gem 'activerecord'
gem 'sqlite3'
gem 'factory_girl', '= 2.6.4'
gem 'machinist'

Gemfile.lock

GEM
  remote: http://rubygems.org/
  specs:
    activemodel (3.2.6)
      activesupport (= 3.2.6)
      builder (~> 3.0.0)
    activerecord (3.2.6)
      activemodel (= 3.2.6)
      activesupport (= 3.2.6)
      arel (~> 3.0.2)
      tzinfo (~> 0.3.29)
    activesupport (3.2.6)
      i18n (~> 0.6)
      multi_json (~> 1.0)
    arel (3.0.2)
    builder (3.0.0)
    factory_girl (2.6.4)
      activesupport (>= 2.3.9)
    i18n (0.6.0)
    machinist (2.0)
    multi_json (1.3.6)
    sqlite3 (1.3.6)
    tzinfo (0.3.33)

PLATFORMS
  ruby

DEPENDENCIES
  activerecord
  factory_girl (= 2.6.4)
  machinist
  sqlite3

Ex-Coworker of mine who has a nasty case of NIH wrote this: https://github.com/soveran/spawn which is a fairly thin wrapper around AR.create if I remember right. Would this kind of syntax be good? (basically you end up with a spawn block inline with each class that is possible to spawn).

@jsanders - Regarding #1 I think that it's useful in integration tests: "give me a valid working control event" is a big damn question. But almost useless in the spec directory where you don't want to incur that cost. But maybe this whole thing is a code smell that we're way to interdependent in our data model. I'm not sure.

@cschneid - Yes, that thing you said about smelliness. I'm not saying I'm against having methods that create complex sets of data in some common place, I just don't really see that these fully-generic object-aggregation-and-creation systems buy us much besides magic and complexity.

@jsanders I agree we could do this manually but (except for the performance hit) I like a little help to do it all in one place. I think the organization aspect is a win. That said, a quick browse through the FactoryGirl code and these benchmarks show the thing is totally wasteful. A little work to improve its performance would be a good investment IMO.

@thinkerbot - I'm fully in favor of having a set of methods in a common place to build commonly-built object networks, but I don't see why you need a library to do that.

I agree with @thinkerbot that the first step should be to see if a little work to improve FactoryGirl might help. It's easier to tune an existing thing than to replace it.

Another related thing we could do is to abstract away from the implementation so that we could replace the implementation in the future. It's not clear if this is possible but if we did FactoryDude.create(...) and had FactoryDude call FactoryGirl now and maybe Machinist in the future. But would that just be add more overhead and make it slower? Hmmm.

Oh, another reason to use FactoryGirl - we use it everywhere already!

Edit. Yeah, what @tra said.

I am now going to start an object-building framework called SurferDude as an antidote to FactoryGirl.

tldr I found I could make FactoryGirl a lot faster, and that it really doesn't matter.

Before (thinkerbot/factory_girl@f9ab6c7549942c610ba8dba29ff3228a7d2b1ec2):

$ bundle exec ruby -Ilib benchmark/factory_girl_benchmark.rb 
                               user     system      total        real
# baseline
10k Hash.new               0.010000   0.000000   0.010000 (  0.009226)
# creation
10k ActiveRecord.new       0.610000   0.010000   0.620000 (  0.628761)
10k Factory.build          2.090000   0.100000   2.190000 (  2.188184)
# peristence
10k ActiveRecord.create    4.750000   0.090000   4.840000 (  4.842905)
10k Factory.create         6.540000   0.200000   6.740000 (  6.743012)

After (thinkerbot/factory_girl@416977fc4e14b53e9a0381e6dc654ac70382e602):

$ bundle exec ruby -Ilib benchmark/factory_girl_benchmark.rb 
                               user     system      total        real
# baseline
10k Hash.new               0.010000   0.000000   0.010000 (  0.009660)
# creation
10k ActiveRecord.new       0.640000   0.010000   0.650000 (  0.645614)
10k Factory.build          0.890000   0.010000   0.900000 (  0.907428)
# peristence
10k ActiveRecord.create    4.920000   0.100000   5.020000 (  5.012222)
10k Factory.create         5.320000   0.120000   5.440000 (  5.433966)

Largely the improvements were won by ensuring FactoryGirl doesn't recalculate known information for every build/create, and to a lesser degree creating fewer Procs. Unfortunately the improvements don't matter. The percent improvement is pretty good but in absolute terms it's nothing. If you look at the persistence numbers you can see 1.2s are saved over 10k creates. That's not a lot. Indeed over the whole spec suite it's only about 2 min.

What is interesting is that these tests are run vs an in-memory sqlite3 database so the database time is minimal (~ 0.27s), and yet ActiveRecord is still taking it's sweet time to create records. Perhaps it's ActiveRecord and not FactoryGirl that's slow.