Sequel multi_insert with fallback to single insert

gistfile1.rb

module Vulcan
  module DataInterface
    module RawLoader

      def max_id
        max(:id) || 0
      end

      # Rows are selected in chunks specified by page_size
      # and inserted in slices by slice_size
      PAGE_SIZE = 50000
      SLICE_SIZE = 1000

      INSERT_MODE_MULTI  = 0
      INSERT_MODE_SINGLE = 1
      
      # yields page if block provided
      # Use this code to alter page records in caller's block
      # page.each do |row|
      #  row[:name] = 'something'
      # end
      def raw_loader src, table, range
        @logger.info "Loading #{range} Count: #{range.count}"
        num_inserts = 0
        insert_mode = INSERT_MODE_MULTI
        current_row = nil
        src[table].filter(:id => range).each_page(@page_size||PAGE_SIZE) do |page_ds|
          data = page_ds.all
          data.each_slice(@slice_size||SLICE_SIZE)  do |slice|
            begin
              yield slice  if block_given?
              if insert_mode == INSERT_MODE_MULTI
                multi_insert(slice)
              else
                slice.each { |row| current_row = row ; insert(row) }
              end
            rescue Exception => e
              if insert_mode == INSERT_MODE_MULTI
                @logger.info(e.message)
                @logger.info('Falling back to single record insert in order to detect offending record')
                insert_mode = INSERT_MODE_SINGLE
                retry
              end
              @logger.error(current_row.inspect)
              @logger.error(e)
              raise LoadingException.new(self, current_row, e)
            end
          end
          num_inserts += page_ds.current_page_record_range.count
        end
        @logger.info "Inserted #{num_inserts} into #{table}"
        num_inserts
      end

    end
  end
end