A generator that creates a list of all available titles in Simple English Wiktionary

simple_english.rb

require 'rest-client'

base_url = "http://simple.wiktionary.org/w/api.php?action=query&aplimit=23294&list=allpages&format=json"
filename = "simple_english_titles.yml"
count    = nil
apfrom   = nil
num      = 0 
titles   = []

begin
  url = if apfrom
          base_url + "&apfrom=#{apfrom}"
        else
          base_url
        end

  puts "processing page #{num}: #{url}"

  json   = RestClient.get(url)
  json   = JSON.load(json)
  count  = json["query"]["allpages"].size
  apfrom = json["query"]["allpages"].last['title'] if count > 0

  titles += json["query"]["allpages"].map {|hash| hash["title"] }
  num    += 1
end while count == 500

require 'json'
require 'yaml'

puts "Number of titles: #{titles.uniq.size}"

File.open(filename, 'w') do |file|
  file.write(titles.uniq.to_yaml)
end