Skip to content

Instantly share code, notes, and snippets.

@kozy4324

kozy4324/node_api_only.js

Created June 29, 2012 12:24
Show Gist options
  • Save kozy4324/3017635 to your computer and use it in GitHub Desktop.
Save kozy4324/3017635 to your computer and use it in GitHub Desktop.
Download ZIP
wiki scraping
Raw
node_api_only.js
var START_PAGE = 'http://ja.wikipedia.org/wiki/Mapion';
function getLinks(url_string, callback) {
var http = require('http');
var url = require('url');
http.get(url.parse(url_string), function(res) {
var data = [];
res.on('data', function(chunk) {
data.push(chunk);
});
res.on('end', function() {
var content = data.join('');
var result = content.match(/<a[^>]+>/g);
var links = result.map(function (a) {
if (a.match(/href="([^"]+)"/)) {
return RegExp.$1;
}
}).filter(function (href) {
if (!href) return false;
if (href.indexOf('/wiki/') == 0) return true;
return false;
});
callback(links);
});
}).on('error', function(e) {
console.log('Got error: %s', e.message);
});
}
getLinks(START_PAGE, function(links) {
console.log(links.sort());
});
Raw
result
$ node node_api_only.js
[ '/wiki/%E3%82%AD%E3%83%A3%E3%83%A9%E3%82%AF%E3%82%BF%E3%83%BC',
'/wiki/%E3%82%B1%E3%83%BC%E3%82%BF%E3%82%A4%E5%9B%BD%E7%9B%97%E3%82%8A%E5%90%88%E6%88%A6',
'/wiki/%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF',
'/wiki/%E3%82%B7%E3%83%A3%E3%83%BC%E3%83%97',
'/wiki/%E3%82%B9%E3%82%AF%E3%83%AD%E3%83%BC%E3%83%AB',
'/wiki/%E3%83%8E%E3%83%BC%E3%83%88:Mapion',
'/wiki/%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB:Computer-blue.svg',
'/wiki/%E3%83%96%E3%83%AD%E3%83%BC%E3%83%89%E3%83%90%E3%83%B3%E3%83%89',
'/wiki/%E3%83%97%E3%83%AD%E3%82%B8%E3%82%A7%E3%82%AF%E3%83%88:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF',
'/wiki/%E3%83%A1%E3%82%A4%E3%83%B3%E3%83%9A%E3%83%BC%E3%82%B8',
'/wiki/%E3%83%A1%E3%82%A4%E3%83%B3%E3%83%9A%E3%83%BC%E3%82%B8',
'/wiki/%E4%BB%A3%E8%A1%A8%E5%8F%96%E7%B7%A0%E5%BD%B9',
'/wiki/%E4%BC%9A%E7%A4%BE',
'/wiki/%E5%87%B8%E7%89%88%E5%8D%B0%E5%88%B7',
'/wiki/%E5%8D%94%E5%AE%9A%E4%B8%96%E7%95%8C%E6%99%82',
'/wiki/%E5%9C%B0%E5%9B%B3',
'/wiki/%E5%A4%A9%E6%B0%97',
'/wiki/%E6%97%A5%E6%9C%AC',
'/wiki/%E6%97%A5%E6%9C%AC',
'/wiki/%E6%9D%B1%E4%BA%AC%E9%83%BD',
'/wiki/%E6%9D%B1%E6%97%A5%E6%9C%AC%E9%9B%BB%E4%BF%A1%E9%9B%BB%E8%A9%B1',
'/wiki/%E6%A0%AA%E5%BC%8F%E4%BC%9A%E7%A4%BE',
'/wiki/%E6%A4%9C%E7%B4%A2',
'/wiki/%E6%A5%AD%E7%A8%AE',
'/wiki/%E6%B8%AC%E5%9C%B0%E7%B3%BB',
'/wiki/%E6%B8%AF%E5%8C%BA_(%E6%9D%B1%E4%BA%AC%E9%83%BD)',
'/wiki/%E7%89%B9%E5%88%A5:%E3%81%8A%E3%81%BE%E3%81%8B%E3%81%9B%E8%A1%A8%E7%A4%BA',
'/wiki/%E7%89%B9%E5%88%A5:%E3%82%AB%E3%83%86%E3%82%B4%E3%83%AA',
'/wiki/%E7%89%B9%E5%88%A5:%E3%83%AA%E3%83%B3%E3%82%AF%E5%85%83/Mapion',
'/wiki/%E7%89%B9%E5%88%A5:%E5%80%8B%E4%BA%BA%E8%A8%AD%E5%AE%9A',
'/wiki/%E7%89%B9%E5%88%A5:%E6%96%B0%E3%81%97%E3%81%84%E3%83%9A%E3%83%BC%E3%82%B8',
'/wiki/%E7%89%B9%E5%88%A5:%E6%9C%80%E8%BF%91%E3%81%AE%E6%9B%B4%E6%96%B0',
'/wiki/%E7%89%B9%E5%88%A5:%E7%89%B9%E5%88%A5%E3%83%9A%E3%83%BC%E3%82%B8%E4%B8%80%E8%A6%A7',
'/wiki/%E7%89%B9%E5%88%A5:%E9%96%A2%E9%80%A3%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E6%9B%B4%E6%96%B0%E7%8A%B6%E6%B3%81/Mapion',
'/wiki/%E7%A4%BE%E9%95%B7',
'/wiki/%E8%8A%9D%E6%B5%A6',
'/wiki/%E8%A6%B3%E5%85%89',
'/wiki/%E9%9B%BB%E9%80%9A',
'/wiki/1%E6%9C%8810%E6%97%A5',
'/wiki/1%E6%9C%8815%E6%97%A5',
'/wiki/1%E6%9C%8820%E6%97%A5',
'/wiki/1%E6%9C%8820%E6%97%A5',
'/wiki/10%E6%9C%881%E6%97%A5',
'/wiki/10%E6%9C%8813%E6%97%A5',
'/wiki/11%E6%9C%8811%E6%97%A5',
'/wiki/11%E6%9C%8812%E6%97%A5',
'/wiki/12%E6%9C%88',
'/wiki/12%E6%9C%881%E6%97%A5',
'/wiki/1997%E5%B9%B4',
'/wiki/1997%E5%B9%B4',
'/wiki/2%E6%9C%8824%E6%97%A5',
'/wiki/2002%E5%B9%B4',
'/wiki/2004%E5%B9%B4',
'/wiki/2006%E5%B9%B4',
'/wiki/2007%E5%B9%B4',
'/wiki/2009%E5%B9%B4',
'/wiki/4%E6%9C%881%E6%97%A5',
'/wiki/4%E6%9C%8821%E6%97%A5',
'/wiki/4%E6%9C%8821%E6%97%A5',
'/wiki/4%E6%9C%8822%E6%97%A5',
'/wiki/4%E6%9C%887%E6%97%A5',
'/wiki/6%E6%9C%881%E6%97%A5',
'/wiki/7%E6%9C%8830%E6%97%A5',
'/wiki/8%E6%9C%882%E6%97%A5',
'/wiki/8%E6%9C%8824%E6%97%A5',
'/wiki/Ajax',
'/wiki/Category:%E3%82%AA%E3%83%B3%E3%83%A9%E3%82%A4%E3%83%B3%E6%83%85%E5%A0%B1%E6%BA%90',
'/wiki/Category:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF%E9%96%A2%E9%80%A3%E3%81%AE%E3%82%B9%E3%82%BF%E3%83%96%E9%A0%85%E7%9B%AE',
'/wiki/Category:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF%E9%96%A2%E9%80%A3%E3%81%AE%E3%82%B9%E3%82%BF%E3%83%96%E9%A0%85%E7%9B%AE',
'/wiki/Category:%E5%9C%B0%E5%9B%B3%E4%BC%9A%E7%A4%BE',
'/wiki/Category:%E5%9C%B0%E7%90%86%E6%8A%80%E8%A1%93',
'/wiki/Category:%E6%97%A5%E6%9C%AC%E3%81%AE%E5%9C%B0%E5%9B%B3',
'/wiki/Category:%E6%97%A5%E6%9C%AC%E3%81%AE%E6%83%85%E5%A0%B1%E3%83%BB%E9%80%9A%E4%BF%A1%E6%A5%AD',
'/wiki/Category:%E6%97%A5%E6%9C%AC%E3%81%AE%E6%83%85%E5%A0%B1%E3%83%BB%E9%80%9A%E4%BF%A1%E6%A5%AD',
'/wiki/Category:%E6%B8%AF%E5%8C%BA_(%E6%9D%B1%E4%BA%AC%E9%83%BD)%E3%81%AE%E4%BC%81%E6%A5%AD',
'/wiki/Category:PlaceEngine',
'/wiki/Google_%E3%83%9E%E3%83%83%E3%83%97',
'/wiki/Help:%E7%9B%AE%E6%AC%A1',
'/wiki/Mapion',
'/wiki/Mapion',
'/wiki/MapionBB',
'/wiki/NTT%E3%82%B3%E3%83%9F%E3%83%A5%E3%83%8B%E3%82%B1%E3%83%BC%E3%82%B7%E3%83%A7%E3%83%B3%E3%82%BA',
'/wiki/Portal:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF',
'/wiki/Portal:%E6%9C%80%E8%BF%91%E3%81%AE%E5%87%BA%E6%9D%A5%E4%BA%8B',
'/wiki/Template:%E5%9F%BA%E7%A4%8E%E6%83%85%E5%A0%B1_%E4%BC%9A%E7%A4%BE',
'/wiki/Web_of_the_Year',
'/wiki/Wikipedia:%E3%81%8A%E7%9F%A5%E3%82%89%E3%81%9B',
'/wiki/Wikipedia:%E3%82%A6%E3%82%A3%E3%82%AD%E3%83%9A%E3%83%87%E3%82%A3%E3%82%A2%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6',
'/wiki/Wikipedia:%E3%82%B3%E3%83%9F%E3%83%A5%E3%83%8B%E3%83%86%E3%82%A3%E3%83%BB%E3%83%9D%E3%83%BC%E3%82%BF%E3%83%AB',
'/wiki/Wikipedia:%E3%82%B5%E3%83%B3%E3%83%89%E3%83%9C%E3%83%83%E3%82%AF%E3%82%B9',
'/wiki/Wikipedia:%E3%82%B9%E3%82%BF%E3%83%96',
'/wiki/Wikipedia:%E3%83%90%E3%82%B0%E3%81%AE%E5%A0%B1%E5%91%8A',
'/wiki/Wikipedia:%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%81%AE%E3%82%A2%E3%83%83%E3%83%97%E3%83%AD%E3%83%BC%E3%83%89',
'/wiki/Wikipedia:%E4%BA%95%E6%88%B8%E7%AB%AF',
'/wiki/Wikipedia:%E5%85%8D%E8%B2%AC%E4%BA%8B%E9%A0%85',
'/wiki/Wikipedia:%E9%80%A3%E7%B5%A1%E5%85%88',
'/wiki/Yahoo!%E3%83%9E%E3%83%83%E3%83%97',
'/wiki/Yahoo!_JAPAN' ]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment