vunetscraper.rb

require 'capybara/dsl'
require 'capybara-webkit'

include Capybara::DSL
Capybara.current_driver = :webkit

Capybara.app_host = "http://vunet.vu.nl"

module Runner
  extend self

  def run 
    page.visit('/')
    fill_in "username", with: "YOURUSER"
    fill_in "password", with: "SECRETTTWACHTWOORD"
    click_button "Log On"

    page.visit("https://portal.login.vu.nl/Pages/moduleoverview.aspx")
    wait_for_completion
    (2008..2013).each do |year|
      if File.exists?("results/#{year}")
        puts "Skipping #{year}"
      else
        fetch_year year.to_s 
      end 
    end 
  end 

  def wait_for_completion
    sleep 2
    while true
      style = find(".progress").find(:xpath, "..")[:style]
      styles = style.split(";").collect { |x| x.split(":").collect(&:strip) }.flatten.reject { |x| x == ""} 
      if styles.first != "display"
        p styles
        raise "Hm, didn't expect this"
      else
        break if styles.last == "none"
        sleep 1
      end 
    end 
  end 


  def fetch_year year
    p "Fetching #{year}"
    find(".filterSelect").select year
    wait_for_completion


    results = {}
    if all(".yearWrap .program .periods").length < 1 
      puts "No entries for year #{year}"
    else
      within(".yearWrap .program .periods") do
        all(".moduleLine").each do |course|
          passed = course[:class].split.member?("Passed")
          name  = course.find(".name").text
          ects  = course.find(".ects").text
          grade = course.find(".grade").text
          date  = course.find(".examdate").text
          code  = course.find(".moduleCode span:last").text.split("_").last

          results[code] = { name: name, ects: ects, grade: grade, passed: passed, exam_at: date }
          p [name, ects, grade, code, date]
        end 
      end 

      File.open("results/#{year}", "w") do |f| 
        f.write Marshal.dump(results)
      end 
    end 
  end 
end

Runner.run