履修選抜.死ぬ.jp クローラー。SFC-SFSの情報を取得する。

config.yaml

---
:username: t000000aa
:password: hogehogefugafuga

crawler.rb

#coding:UTF-8
require 'mechanize'
require 'kconv'
require 'yaml'
require './model.rb'


def login a,nm,pw
  a.get('https://gc.sfc.keio.ac.jp/sfc-sfs/')
  a.page.forms.first['u_login'] = nm
  a.page.forms.first['u_pass']  = pw
  a.page.forms.first.submit
  a.get(a.page.meta[0].href)
  return a
end

@agent =  Mechanize.new
@agent.max_history = 3
@encode = nil
# 受信するテキストをすべてUTF-8にする
@agent.post_connect_hooks << Proc.new do |prm|
  if prm[:response]["Content-Type"] =~ /^text\/.*$/
    @encode = Kconv.guess(prm[:response_body])
    prm[:response_body] = Kconv.kconv(prm[:response_body],Kconv::UTF8,Kconv::AUTO)
    prm[:response]["content-Type"] = "text/html; charset=utf-8"
    prm[:response_body].gsub!(/<meta[^>]*>/i) do |m|
      m.gsub(/S(?:hift)?[-|_]JIS|EUC[-_]JP|Windows-31J/i,"UTF-8")
    end
  end
end

# 送信するテキストをUTF-8からもとの文字コードに戻す
@agent.pre_connect_hooks << Proc.new do |prm|
  if @encode
    np = []
    prm[:params].each do |m|
      np.push URI.encode(Kconv::kconv(URI.decode(m),@encode))
    end
    prm[:params] = np
  end
end

config = YAML.load(File.open("./config.yaml"))
@agent = login(@agent,config[:username],config[:password])

@agent.page.link_with(:text => /MY時間割/).click
@timetable = @agent.page.uri + @agent.page.search('iframe').first.attributes['src'].value
@agent.get(@timetable)
@agent.page.links_with(:href=>/s_class_top\.cgi/).each do |l|
  puts l.href

  serial = nil
  l.href.split('&').each{|e| if a = e.match(/yc=(.+)/);serial = a[1];break;end}
  next if !serial

  lecture = Lectures.first(:serial => serial)
  next if lecture && lecture.finished

  l.click
  selection = [email protected](/《履修人数を制限しない》/)
  list_link = @agent.page.link_with(:text=>/履修許可者確認/)
  finished = !selection || !!list_link || [email protected](/「履修者選抜なし」となりました/)
  title = @agent.page.title.split(/SFC-SFS\s(.*)/)[1]

  puts "#{serial} : #{title} (Selection:#{selection} / Finished:#{finished})"

  @agent.page.uri.query.split('&').each{|e| if a = e.match(/yc=(.+)/);serial = a[1];break;end}

  if lecture
    lecture.update(:selection => selection,
                    :finished => finished,
                    :url => @agent.page.uri.to_s )
  else
    Lectures.create(:serial => serial,
                    :title => @agent.page.title.split(/SFC-SFS\s(.*)/)[1],
                    :selection => selection,
                    :finished => finished,
                    :url => @agent.page.uri.to_s )
  end

  next if !list_link

  list_link.click
  list = @agent.page.search('tr[bgcolor="#efefef"] td').map{|e| e.text}
  puts "permissions #{list.length}"
  list.each do |n|
    next if n !~ /^\d{8}/
    Permissions.create(
      :number => n,
      :lecture_serial => serial,
      :lecture_title => title)
  end
end

model.rb

require 'sequel'
Sequel::Model.plugin(:schema)
Sequel.connect("sqlite://senbatsu.db")

class Lectures < Sequel::Model
  set_schema do
    primary_key :id
    string :serial
    string :title
    bool :selection
    bool :finished
    string :url
  end
end
class Permissions < Sequel::Model
  set_schema do
    primary_key :id
    string :number
    string :lecture_serial
    string :lecture_title
  end
end

Lectures.create_table if !Lectures.table_exists?
Permissions.create_table if !Permissions.table_exists?