演歌スクレイプ

require 'open-uri'
require 'nokogiri'

def get_content(url)
  charset = nil
  html = open(url) do |f|
    charset = f.charset
    f.read
  end

  articles = []

  doc = Nokogiri::HTML.parse(html, nil, charset)
  doc.css('.entryTd').each_with_index do |ent, i|
    ent_lines = ent.text.split("\r\n")
    title = ent_lines[0].strip
    lines = ent_lines.slice(1..-1)
    hashs = title.split("　")
    hashs.delete("")
    hashed = hashs.map{ |l| "#" + l }
    lines = lines + hashed
    lines = lines.map {|l| l.gsub(/(\s|　)+/, '')}
    lines.delete("")
    lines = lines.map {|l| l.include?("youtube.com") ? "[" + l + "]" : l}

    #source url
    source = doc.css('.entryTitle')[i].css('a')[0].attribute('href').value
    lines.push("ソース" + source) if source
    articles << { title: title, lines: lines }
  end
  articles
end

articles = []
(1..213).to_a.each do |i|
  url = 'http://blogs.yahoo.co.jp/oya_ji3/MYBLOG/yblog.html?m=lc&p='
  url = url + i.to_s
  articles += get_content(url)
  sleep 1
end

scrapbox = {
  pages: articles
}