Skip to content

Instantly share code, notes, and snippets.

@hakatashi
Last active August 29, 2015 14:06
Show Gist options
  • Save hakatashi/7584c64e643a48084587 to your computer and use it in GitHub Desktop.
Save hakatashi/7584c64e643a48084587 to your computer and use it in GitHub Desktop.
Twitpic download script. Create 'twitpic' directory and make a list of ids as 'twitpics.json' like "["/6jbgg1","/5ryufc","/5hpuoq"]". Copyleft.
var cheerio = require('cheerio');
var request = require('request');
var async = require('async');
var fs = require('fs');
var querystring = require('querystring');
var url = require('url');
var pics = JSON.parse(fs.readFileSync('twitpics.json'));
var skipped = [];
var twitpic = function (method, resource, params, callback) {
var paramString = querystring.stringify(params);
request({
url: 'http://api.twitpic.com/2/' + resource + '.json' + '?' + paramString,
json: true,
method: method,
}, callback);
};
async.eachLimit(pics, 3, function (pic, done) {
var id = pic.slice(1);
var viewPage = 'http://twitpic.com' + pic;
var fullPage = viewPage + '/full';
async.parallel([
function (done) {
twitpic('GET', 'media/show', {
id: id
}, function (error, response, data) {
if (error) return done(error);
console.log(id + ': info downloaded, ratelimit remains ' + response.headers['x-ratelimit-remaining']);
// validate json file
if (typeof data === 'undefined' || !data.id || !data.user || !data.timestamp) {
return done(new Error(id + ': Invalid JSON'));
}
fs.writeFile('twitpic/' + id + '.json', JSON.stringify(data), done);
});
},
function (done) {
request(fullPage, function (error, response, body) {
if (error) return done(error);
try {
var $ = cheerio.load(body);
var imageURL = $('#media-full > img').attr('src');
var resolvedURL = url.resolve(fullPage, imageURL);
var parsedURL = url.parse(resolvedURL);
var extension = parsedURL.pathname.match('\\.[^.]+$')[0];
var ws = fs.createWriteStream('twitpic/' + id + extension);
ws.on('error', done);
ws.on('finish', function () {
console.log(id + ': image downloaded');
return done();
});
request(imageURL).pipe(ws);
} catch (error) {
return done(error);
}
});
}
], function (error) {
if (error) {
console.error(id + ': ' + error);
skipped.push(pic);
}
return done();
});
}, function (error) {
if (error) {
console.error(error);
} else {
console.log('Successful. Skipped: ' + JSON.stringify(skipped));
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment