Skip to content

Instantly share code, notes, and snippets.

@stefanoverna
Last active December 21, 2015 23:15
Show Gist options
  • Select an option

  • Save stefanoverna/b7173f1f736209073c3c to your computer and use it in GitHub Desktop.

Select an option

Save stefanoverna/b7173f1f736209073c3c to your computer and use it in GitHub Desktop.
var request = require("request");
var FeedParser = require("feedparser");
var sanitizeHtml = require('sanitize-html');
var Iconv = require('iconv').Iconv;
var Promise = require('bluebird').Promise;
function feedItems(url) {
return new Promise(function(resolve, reject) {
var req = request(url, { timeout: 10000, pool: false });
req.setMaxListeners(50);
req.setHeader('user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');
req.setHeader('accept', 'text/html,application/xhtml+xml');
var feedparser = new FeedParser();
req.on('error', function(error) { reject(error) });
req.on('response', function(res) {
if (res.statusCode != 200) return this.emit('error', new Error('Bad status code'));
var charset = getParams(res.headers['content-type'] || '').charset;
res = maybeTranslate(res, charset);
res.pipe(feedparser);
});
var items = [];
feedparser.on('error', function(error) { reject(error) });
feedparser.on('readable', function() {
var item;
while (item = this.read()) {
items.push({
title: item.title,
description: sanitizeHtml(item.summary || item.description, { allowedAttributes: [], allowedTags: [] }).trim(),
url: item.link || item.permalink,
published_at: item.date,
image_url: item.image && item.image.url
});
}
});
feedparser.on('end', function() { resolve(items) });
});
}
function getParams(str) {
var params = str.split(';').reduce(function (params, param) {
var parts = param.split('=').map(function (part) { return part.trim(); });
if (parts.length === 2) {
params[parts[0]] = parts[1];
}
return params;
}, {});
return params;
}
function maybeTranslate(res, charset) {
var iconv;
// Use iconv if its not utf8 already.
if (!iconv && charset && !/utf-*8/i.test(charset)) {
try {
iconv = new Iconv(charset, 'utf-8');
iconv.on('error', done);
// If we're using iconv, stream will be the output of iconv
// otherwise it will remain the output of request
res = res.pipe(iconv);
} catch(err) {
res.emit('error', err);
}
}
return res;
}
feedItems('http://www.ft.com/rss/home/asia')
.then(function(items) {
console.log(items);
})
.catch(function(error) {
console.log(error);
});
Admin
id
email
bcrypt_password
POST /admin/admins
-> { email, password }
<- { id, email }
201
DELETE /admin/admins/:id
204
———————
Feeds
id
name
url
website_url
GET /admin/feeds
<- [ { id, name, website_url, url } ]
POST /admin/feeds
-> { name, website_url, url }
<- { id, name, website_url, url }
201
DELETE /admin/feeds/:id
204
———————
FeedItems
id
feed_id
title
description
url
published_at:datetime
image_url
seen:boolean
GET /admin/feed-items?max_id&limit&unseen_only
[ { id, feed_id, title, description, url, published_at, image_url, seen } ]
PATCH /admin/feed-items/:id
{ seen }
200
————————
Category
id
name
GET /admin/categories
[ { id, name } ]
200
POST /admin/categories
-> { name }
<- { id, name }
201
PATCH /admin/categories/:id
-> { name }
<- { id, name }
200
DELETE /admin/categories/:id
204
————————
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment