crawler preparation for NEWS analysis

udn_news_crawler.rb

require 'nokogiri'
require 'open-uri'

# Let's try to fetch and parse HTML document

books = Nokogiri::HTML(open('https://udn.com/rank/pv/2/0/1'))

news = []

i = 0 
books.css('dt h2 a').each do |link|
	news.push link['href']
	i+=1
end

string = ''

news.each do |n|
	article = Nokogiri::HTML(open(n))
	
	article.css('p').each do |link|
		string += link.content
	end
end

File.open('article.txt', 'w') { |file| file.write(string) }