Backlog Wiki crawler

backlog_wiki_crawler.rb

#!/usr/bin/ruby
#-*- coding: utf-8; -*-

require "pp"
require "mechanize"

STDOUT.sync = true


# config

AUTH_FROM = {:space => 'fromspace', :id => 'xxx', :password => 'xxx', :project => 'FROM'}


# API class

def create_mechanize_agent(auth_info)
  agent = Mechanize.new
  page_login = agent.get("https://#{auth_info[:space]}.backlog.jp/LoginDisplay.action")
  page_dashboard = page_login.form_with(:name => 'Login') do |form|
    form.userId = auth_info[:id]
    form.password = auth_info[:password]
  end.submit
  
  return agent
end

class BacklogAPI

  def initialize
    @fagent = create_mechanize_agent(AUTH_FROM)
  end

  def from_agent
    @fagent
  end

end


# main

def convert_wiki
  api = BacklogAPI.new
  home = api.from_agent.get("https://#{AUTH_FROM[:space]}.backlog.jp/wiki/#{AUTH_FROM[:project]}/Home")
  list = home.search("//div[@id='wikipagelist']/ul/li/a")
  list.each do |i|
    convert_one_wiki(i, api)
    sleep 2 # yasashisa
  end
end

def convert_one_wiki(wiki, api)
  print "#{wiki.text}"
  vpage = api.from_agent.get(wiki["href"])
  edit_link = vpage.search("//a[@class='editWikiButton']")[0]
  return unless edit_link
  epage = api.from_agent.get(edit_link["href"])

  title = epage.search("//input[@id='page.name']")[0]["value"]
  content = epage.search("//textarea[@id='page.content']")[0].text

  # do something...

  puts "  OK"
end

convert_wiki