test case for eager loading of nested associations with DataMapper

.gitignore

Gemfile.lock

dm_el.rb

#!/usr/bin/env ruby
# encoding: UTF-8

# test case for eager loading of nested associations with DataMapper

require 'rubygems'
require 'dm-core'
require 'dm-constraints'
require 'dm-migrations'
require 'eager_loading'

DataMapper::Logger.new($stdout, :debug)
DataMapper.setup(:default, "sqlite3://#{Dir.pwd}/db.sqlite")

class Person
  include DataMapper::Resource

  property :id, Serial
  property :name, String, :required => true

  has n, :vehicles

end

class Vehicle
  include DataMapper::Resource

  property :id, Serial
  property :name, String, :required => true

  has n, :components

end

class Component
  include DataMapper::Resource

  property :id, Serial
  property :name, String, :required => true

  belongs_to :manufacturer

end

class Manufacturer
  include DataMapper::Resource

  property :id, Serial
  property :name, String, :required => true

end

DataMapper.auto_migrate!

# generate test data

Person.create(:name => "FND", :vehicles => [
  Vehicle.create(:name => "Taurus", :components => [
    Component.create(:name => "engine",
        :manufacturer => Manufacturer.create(:name => "Ford")),
    Component.create(:name => "radio",
        :manufacturer => Manufacturer.create(:name => "Bose"))
  ]),
  Vehicle.create(:name => "fixie", :components => [
    Component.create(:name => "frame",
        :manufacturer => Manufacturer.create(:name => "Campagnolo")),
    Component.create(:name => "breaks",
        :manufacturer => Manufacturer.create(:name => "Shimano"))
  ])
])

Person.create(:name => "tillsc", :vehicles => [
  Vehicle.create(:name => "Golf", :components => [
    Component.create(:name => "engine",
        :manufacturer => Manufacturer.create(:name => "VW"))
  ])
])

# retrieve data

puts "", "[INFO] test case A"
person = Person.get!(1)
puts person.vehicles.components.manufacturer.map(&:name).join(", ")

puts "", "[INFO] test case B"
people = Person.all
people.each do |person|
  person.vehicles.each do |vehicle|
    puts sprintf("%-10s %-10s", person.name, vehicle.name)
  end
end

puts "", "[INFO] test case C    ===== /!\ n+1 hazard ===="
people = Person.all
people.each do |person|
  person.vehicles.each do |vehicle|
    vehicle.components.each do |component|
      puts sprintf("%-10s %-10s %-10s", person.name, vehicle.name, component.name)
    end
  end
end

puts "", "[INFO] test case D"
people = Person.all
people.eager_load(Person.vehicles.components).each do |person|
  person.vehicles.each do |vehicle|
    vehicle.components.each do |component|
      puts sprintf("%-10s %-10s %-10s", person.name, vehicle.name, component.name)
    end
  end
end

eager_loading.rb

# manual eager loading for DataMapper
# adapted from Chris Corbyn: https://gist.github.com/1244491#gistcomment-56797
module EagerLoading

  def eager_load(query_path)
    scope = self

    query_path.relationships.each do |relation|
      source_key = relation.source_key.first # TODO: rename
      target_key = relation.target_key.first # TODO: rename

      # for each level in the query path, collect all the resources referencing
      # keys at the current scope
      next_scope = relation.target_model.all(target_key.name => scope.
          collect(&:"#{source_key.name}"))

      # map target keys to the resources that exist for them
      links = next_scope.inject({}) do |map, resource|
        map.merge(target_key.get(resource) => [resource]) { |k, v1, v2| v1 + v2 }
      end

      # now pre-load those from the map
      scope.each do |parent|
        if links.key?(source_key.get(parent))
          parent.instance_variable_set(:"@#{relation.name}",
              links[source_key.get(parent)])
        end
      end

      # and step into the next nesting level
      scope = next_scope
    end

    self
  end

end

DataMapper::Collection.send(:include, EagerLoading)

Gemfile

source :rubygems

DM_VERSION    = '~> 1.2.0.rc2'

gem 'dm-core', DM_VERSION
gem 'dm-constraints', DM_VERSION
gem 'dm-migrations', DM_VERSION
gem 'dm-sqlite-adapter', DM_VERSION

run.sh

#!/usr/bin/env sh

rm db.sqlite

# reformat SQL queries for readability
bundle exec ./dm_el.rb | perl -pe 's#SELECT .*? (FROM ".*?" )(.*)#\1SELECT ... \2#'

I agree, this does look a little more complicated than the last iteration, but I'll chalk that up to the collections and composite key support.

Did it not work to use a Relationship as a query predicate? (eg., instead of Hash[relationship.target_key.collect(&:name).zip(relationship.source_key.get(resource))], do relationship => resource). There's definitely code in place to support that API (DataMapper::Query::Conditions::Comparison::RelationshipHandler#foreign_key_mapping), but I'm not sure if it will work here.

One teeny thing: I think you can use Relationship#set instead of directly setting the instance variable on resource in #load_into_collection. Currently, it's:

            resource.instance_variable_set(
              :"@#{relationship.name}",
              relationship.collection_for(resource).set(map[relationship.source_key.get(resource)])
            )

But I think it can be (I also introduced a couple of explanatory temporary variables):

            collection = relationship.collection_for(resource)
            eager_loaded_resources = map[relationship.source_key.get(resource)]
            collection.set(eager_loaded_resources)
            relationship.set(resource, collection)

Re: relationship as a predicate, I tried something (I forget what) and while it didn't error, it didn't produce the correct results. I almost certainly did the wrong thing though. Are you able to check out the original gist, drop in my code then see if you get the correct output with your suggested simplification?

You should see the following under the "test case C" heading:

[INFO] test case C    ===== /!\ n+1 hazard ====
 ~ (0.000043) SELECT "id", "name" FROM "people" ORDER BY "id"
 ~ (0.000055) SELECT "id", "name", "person_id" FROM "vehicles" WHERE "person_id" IN (1, 2) ORDER BY "id"
 ~ (0.000086) SELECT "id", "name", "manufacturer_id", "vehicle_id" FROM "components" WHERE "vehicle_id" IN (1, 2, 3) ORDER BY "id"
FND        Taurus     engine    
FND        Taurus     radio     
FND        fixie      frame     
FND        fixie      breaks    
tillsc     Golf       engine 

I always get a bit lost poking around in dm-core, but I always learn something new. It would be good to clear that "FIXME" :)

Awesome advice in Relationship#set! Will try that. Classic example of getting a bit lost :P (Excuse the fragmented response... I'm a bit distracted with food and another coding problem simultaneously ;))

Hmm, I was just looking through some of Relationship, and I noticed #eager_load, which looks like exactly the API for this situation. So if I'm reading it correctly, you could simplify the whole thing to:

def load(query_path)
  query_path.relationships.inject(@collection) do |scope, relationship|
    relationship.eager_load(scope)
  end
end

(BTW, I've vacillated, and I now prefer passing the Query::Path to #load instead of #initialize (reverting to how you had it before). It seems more coherent, even though EagerLoader is probably still a single-use object.)

It's too late here for me to start hacking on this right now, but I'm going to try to make time to poke at it tomorrow.

@d11wtq—looks like Relationship#eager_load does work; check out my fork of this gist: https://gist.github.com/1297105

Hey guys, I know this is an old thread, but check this out: https://gist.github.com/3100034