gistfile1.txt

#!/usr/bin/env ruby  
require 'open-uri'
require 'nokogiri'

#ideas: sinatra app with sphinx (http://sphinxsearch.com/) or elastic search backend


books = ["OR","ZGB","SchKG","URG","URV","DSG","VDSG","UWG","KG","StGB","ZPO","StPO","BV","TSchG","TSchV","MG","DR04","KMG","ZSG","ZSV","KVG","MWSTG","MWSTGV","RPG","RPV","SVG","OBG","OBV","SSV","ArG","AHVG","AHVV","BVG","UVG","UVV","EOG","EOV"]

save_dir = "./html/" 


def search_query_url(query)
    #curl -i -d "q=BV&lang=i&pg=q&i=0" http://www.admin.ch/chp/redir_sr2.pl
    language='d'     
    pg='q' #god knows...
    i_param="0" #god knows...
    search_url = "http://www.admin.ch/chp/redir_sr2.pl?q=#{query}&lang=#{language}&pg=#{pg}&i=#{i_param}"
end

def get_toc(book)  
    url = search_query_url(book)  
    #TODO: throw error if its not a TOC and multiple results show up
    return Nokogiri::HTML(open(url))
end
    

books.each do |book|

    doc = get_toc(book)

    toc = doc.css('#spalteContentPlus')
    #TODO cleanup html: remove pdf links, remove footer content

    #TODO wrap into template
    File.open( save_dir + bookname + ".html")

    layer1_links = toc.css('a').collect{|lnk|  lnk["href"]}.select{|lnk|  lnk.to_s.start_with? "./"}

    toplink = url.gsub(/index.html$/,'') 

    layer1_links.each do |lnk|   
        lnk.gsub!(/^.\//,'')
        subsite = toplink + lnk  
        puts "fetching #{subsite}"
        sdoc =  Nokogiri::HTML(open(subsite))           
        text = sdoc.css('#spalteContentPlus') 
        filename = save_dir + lnk 
        #TODO wrap into template        
        puts "writing to #{filename}"
        File.open(  filename, 'w'){ |file|
            file.write text.to_html 
        }


    end
end

superclaw.rb

#!/usr/bin/env ruby  
require 'open-uri'
require 'nokogiri'

#ideas: sinatra app with sphinx (http://sphinxsearch.com/) or elastic search backend


books = ["OR","ZGB","SchKG","URG","URV","DSG","VDSG","UWG","KG","StGB","ZPO","StPO","BV","TSchG","TSchV","MG","DR04","KMG","ZSG","ZSV","KVG","MWSTG","MWSTGV","RPG","RPV","SVG","OBG","OBV","SSV","ArG","AHVG","AHVV","BVG","UVG","UVV","EOG","EOV"]

save_dir = "./html/" 


def search_query_url(query)
    #curl -i -d "q=BV&lang=i&pg=q&i=0" http://www.admin.ch/chp/redir_sr2.pl
    language='d'     
    pg='q' #god knows...
    i_param="0" #god knows...
    search_url = "http://www.admin.ch/chp/redir_sr2.pl?q=#{query}&lang=#{language}&pg=#{pg}&i=#{i_param}"
end

def get_toc(book)  
    url = search_query_url(book)  
    #TODO: throw error if its not a TOC and multiple results show up
    return Nokogiri::HTML(open(url))
end
    

books.each do |book|

    doc = get_toc(book)

    toc = doc.css('#spalteContentPlus')
    #TODO cleanup html: remove pdf links, remove footer content

    #TODO wrap into template
    File.open( save_dir + bookname + ".html")

    layer1_links = toc.css('a').collect{|lnk|  lnk["href"]}.select{|lnk|  lnk.to_s.start_with? "./"}

    toplink = url.gsub(/index.html$/,'') 

    layer1_links.each do |lnk|   
        lnk.gsub!(/^.\//,'')
        subsite = toplink + lnk  
        puts "fetching #{subsite}"
        sdoc =  Nokogiri::HTML(open(subsite))           
        text = sdoc.css('#spalteContentPlus') 
        filename = save_dir + lnk 
        #TODO wrap into template        
        puts "writing to #{filename}"
        File.open(  filename, 'w'){ |file|
            file.write text.to_html 
        }


    end
end