gistfile1.rb

# encoding: utf-8
require 'mechanize'
require 'stringex'
require 'tempfile'
require Rails.root.join('app/models/category')
require Rails.root.join('app/models/product')

class Parser
  BASE_URL     = "http://asiaopt.com"
  URL_MASK     = "#{BASE_URL}/products/page:%s"
  PAGES_RANGE  = 1..44

  def initialize
    Category.delete_all
    Product.delete_all

    parse_catalogue
  end

protected

  def parse_catalogue
    PAGES_RANGE.each do |page_num|
      with_each_product_on(url_mask(page_num)) do |product_uri|
        parse_product(product_uri)
      end
    end
  end

  def with_each_product_on(page_url, &block)
    agent.get page_url do |page|
      page.parser.css('.products .item a').each do |link|
        block.call(link[:href])
      end
    end
  end

  def parse_product(product_uri)
    agent.get("#{BASE_URL}#{product_uri}") do |page|
      category = parse_breadcrumbs_and_find_category(page)
      next unless category

      product_name     = page.parser.css('div#h1-wrap h1').first.try(:text)
      product_img      = page.parser.css('div#cip-main > a').first
      product_announce = page.parser.css('#cat-item > p').first.try(:text)
      product_descr    = page.parser.css('#cat-item > div.text').first.try(:inner_html)

      product = category.products.build(
        :name        => product_name.strip,
        :description => product_descr,
        :announce    => product_announce,
      )

      if product_img && product_img.has_attribute?('href')
        agent.get("#{BASE_URL}#{product_img.attribute('href')}") do |page|
          
          photo = Tempfile.new "binary"
          photo.binmode # switch to binary mode
          photo.write(page.content.read)
          photo.close

          product.build_photo
          product.photo.attach = photo
        end
      end

      unless product.save
        puts "IMAGE #{product_img.attribute('href')}"
      end
    end
  end

  def parse_breadcrumbs_and_find_category(page)
    crumbs = []
    page.parser.css('#breadcrumbs a').each_with_index do |link, i|
      next if i < 2
      crumbs << link.text
    end

    find_or_create_category(crumbs)
  end

  # Квадроциклы / Спортивные
  def find_or_create_category(crumbs)
    slugs = []
    categories = []

    crumbs.each do |crumb|
      slugs << crumb.to_url#find_or_create_slug(crumb)
      category_uri = slugs.join('/')
      category = Category.find_by_uri(category_uri)

      unless category
        category = Category.create!(:name => crumb, :slug => crumb.to_url)
        parent_category = categories.last
        # Перемещаем в родительскую категорию
        category.move_to_child_of(parent_category) if parent_category
      end

      # Помещаем с список категорий
      categories << category
    end

    categories.last
  end

  def find_or_create_slug(crumb)
    slug    = crumb.to_url
    db_slug = Category.where("slug REGEXP '#{slug}\-[0-9]+' OR slug = '#{slug}'").pluck(:slug).first

    if db_slug.present? && db_slug.match(/-[0-9]+$/)
      db_slug.sub(/-([0-9]+)$/) { "-#{$1.to_i + 1}" }
    elsif db_slug.present?
      "#{db_slug}-1"
    else
      slug
    end
  end

  def url_mask(page_num)
    URL_MASK % page_num.to_s
  end

  def agent
    @agent ||= Mechanize.new { |agent| agent.user_agent_alias = 'Mac Safari' }
  end

end

namespace :parser do
  task :all => :environment do
    Parser.new
  end
end