Simple Web Crawler for Text

gistfile1.r

# Load RCurl package
library(RCurl)


addresses <- read.csv("~/links.csv") # Create a .csv file with all of the links you want to crawl

for (i in addresses) full.text <- getURL(i) 

text.sub <- gsub("<.+?>", "", full.text) # Removes HTML tags
text <- data.frame(text.sub)

outpath <- "~/text.indv/"
x <- 1:nrow(text)
for(i in x) {
   write(as.character(text[i,1]), file = paste(outpath,"/",i,".txt",sep=""))
} # Note: this is for Mac OS paths