A simple crawler and indexer based on Collective Intelligence chapter 4

crawler.R

library(RCurl)
library(XML)

joinURL <- function(base, u){
  sbase <- parseURI(base)
  su <- parseURI(u)
  if( su$scheme != "")
    return(u)

  su$scheme <- sbase$scheme
  su$server <- sbase$server
  if(is.na(su$path))
    su$path <- ""
  su$fragment <- ""
  
  path <- sbase$path
  if(path == "")
    path <- "/"
  if(substring(path,nchar(path),nchar(path)) == "/")
    path <- paste(path, "index.html", sep="")
  path <- strsplit(path,"/")[[1]]
  path <- path[-length(path)]
  
  if(substring(su$path,1,3) == "../") {
    if(length(path) > 1)
      path <- path[- length(path)]
    else
      path <- ""
    su$path <- substring(su$path,4,nchar(su$path))
  }
  if(substring(su$path,1,2) == "./") 
    su$path <- substring(su$path,3,nchar(su$path))
  if(substring(su$path,1,1) == "/") {
    path <- ""
    su$path <- substring(su$path,2,nchar(su$path))
  }
  
  path <- paste(c(path, su$path), collapse="/")
  su$path <- path
  return( as(su, "character") )
}  

trataPage <- function(url, dir, debug = FALSE) {
  getLinks <- function() { 
    links <- character() 
    a <- function(node, ...) { 
      links <<- c(links, xmlGetAttr(node, "href"))
      node
    }
    getLinks <- function() links
    return( list(a = a, links = getLinks) )
  }

  getChar <- function() {
    txt <- NULL
    text <- function(node,...) {
      txt <<- c(txt, xmlValue(node))
    }
    getText <- function() txt
    return(list(text = text, getText = getText))
  }
  error <- FALSE
  h1 <- getLinks()
  cat("Parsing...", url, "\n")
  tryCatch(doc <- htmlTreeParse(getURL(url), handlers = h1), error = function(e) error <<- TRUE)

  if( !error ) {
    links.0 <- h1$links()
    links.1 <- sapply(links.0, function(x) try(joinURL(url, x), TRUE))
    names(links.1) <- NULL
    
    h2 <- getChar()
    tryCatch(doc <- htmlParse(url, handlers = h2, encoding = "UTF-8", ignoreBlanks = TRUE),
             error = function(e) error <<- TRUE)
    if(! error){
      cat(paste(h2$getText(), collapse = " "),
          file = file(tempfile(tmpdir = dir), encoding="UTF-8"))
    
      if(debug)
        list(links = links.1, tmp = links.0)
      else
        list(links = links.1)
    }
  }
  else 
    list( links = c() )
}


VISITED <- c()
start <- "http://arademaker.github.com/CA-2012-1/aulas.html"

is.visited <- function(u){
  if(u %in% VISITED)
    TRUE
  else
    FALSE
}

crawler <- function(u, dir, depth = 3){
  if(depth == 0)
    return(0)
  VISITED <<- c(VISITED, u)
  links <- trataPage(u, dir)$links
  for(l in links)
    if( !is.visited(l) )
      crawler(l, dir, depth - 1)
}