safer_import.rb

preprocess {
  # mark the current data for deletion - do not delete right away
  # to avoid ending up with no data on the web-site
  Item.update_all('marked_for_deletion = 1')
}

source(:csv_file) {
  FasterCSV.open(get_var(:source_file),"rb", :col_sep => ';', :headers => true)
}

each_row { |row|
  Item.create!(row[:data].to_hash)
}
  
postprocess {
  my_assert "Item.count(:conditions => 'marked_for_deletion <> 1') > 5000"

  # remove stale data - TODO : this should only be carried out after reindexing
  Item.delete_all('marked_for_deletion = 1')  
  
  # at this point we should have valid, new data not yet indexed by sphinx - index it
  Rake::Task['ts:index'].invoke      
}