d3_api_scraping.rb

# encoding: utf-8
require 'net/http'
require 'open-uri'
require 'nokogiri'
require 'json'

GITHUB_URL = "https://github.com"
WIKI_URL = GITHUB_URL + "/" + "mbostock/d3/wiki/"
API_URL = WIKI_URL + "API-Reference"


html = Nokogiri::HTML(open(API_URL))

json = {}
json.store(:name, "D3.js API")
#json.store(:url, URL)
json.store(:children, [])

def get_link(url)
  if /\/mbostock/.match(url)
    url = GITHUB_URL + url
  else
    url = WIKI_URL + url
  end
end

html.css(".markdown-body").first.children.each do |element|
  if element.name == "h2"
    url = get_link(element.css("a")[1].attributes["href"].value)
    json[:children].push({name: element.css("a").text, url: url, children: []})
  elsif element.name == "h3"
    url = get_link(element.css("a")[1].attributes["href"].value)
    json[:children].last[:children].push({name: element.css("a").text, url: url, children: []})
  elsif element.name == "ul"
    element.css("li a").each do |li|
      url = get_link(li.attributes["href"].value)
      json[:children].last[:children].last[:children].push({name: li.text, url: url})
    end
  end
end

File.open("d3_api.json", "w") do |f|
  f.write(json.to_json)
end

D3.jsのAPI-ReferenceからAPI一覧をスクレイピングし、JSON化する。