wikidump.rb

# handle the wikipedia dump format

module WikiDump

  def self.yield_page_strings(stream)
    buf = ""
    stream.each do |line|
      if line =~ /^\s* <page> \s*$/x
        buf = ""
      end
      buf << line

      if line =~ /^\s* <\/page> \s*$/x
        yield buf
      end
    end
  end

  def self.yield_pages(stream)
    yield_page_strings(stream) do |pagestr|
      yield WikiPage.new(pagestr)
    end
  end

end