[Node] Simple Tumblr photo downloader for NodeJS. Really basic, no options, no error handling.

tumblr-downloader.js

/**
 * TUMBLR downloader
 *
 * Usage:
 *        - enter the blog name: URL part before `.tumblr.com`
 *        - enter what you want to download: `posts` or `likes`
 *        - enter your OAuth public key (https://www.tumblr.com/settings/apps)
 *        - optionally enter the minimum id to retrieve (ignored if `[blog]-[mode].last` file exists)
 *        - if you have a small internet connection, set the queue size to 1
 *        - you can ask the downloader to overwrite existing files
 *        - launch with `node tumblr-downloader.js`
 */


// CONF
var conf = {
    blog: '',
    mode: 'posts',
    api_key: '',

    // advanced
    last: 0,
    queue: 1,
    overwrite: false,
};


// QUEUE MANAGER
var Queue = function() {
    this.done = 0;
    this.running = 0;
    this.queue = [];
};

Queue.prototype.add = function(url, cb) {
    this.queue.push([url, cb]);
    this.init();
};

Queue.prototype.init = function() {
    if (this.running>0 || !this.queue.length) {
        return;
    }

    while (this.running<conf.queue && this.queue.length) {
        this.start();
    }
};

Queue.prototype.start = function() {
    this.running++;

    var item = this.queue.shift(),
        that = this;

    console.log('Download ' + item[0]);

    http.get(item[0], function(rep) {
        that.done++;
        that.running--;

        item[1](rep);

        that.init();
    });
};


// DOWNLOAD MANAGER
var Downloader = function() {
    this.url = 'http://api.tumblr.com/v2/blog/' + conf.blog + '.tumblr.com/' + conf.mode + '?limit=20&api_key=' + conf.api_key;
    this.offset = 0;
    this.first = true;
    this.queue = new Queue();

    try {
        fs.mkdirSync(conf.blog + '-' + conf.mode);
    } catch(e) {}

    try {
        var last = fs.readFileSync(conf.blog + '-' + conf.mode + '.last', {encoding: 'utf8'});
        if (!!last) {
            conf.last = parseInt(last);
        }
    } catch(e) {}
};

// load 20 posts at current offset
Downloader.prototype.init = function() {
    var that = this;

    http.get(this.url + '&offset=' + this.offset, function(rep) {
        rep.setEncoding('utf8');
        var data = "";

        rep
          .on('data', function (chunk) {
              data+= chunk;
          })
          .on('end', function() {
              that.parse.call(that, JSON.parse(data));
          });
    });
};

// parse loaded posts
// download images until conf.last is reached
Downloader.prototype.parse = function(data) {
    var posts = conf.mode=='posts' ? data.response.posts : data.response.liked_posts,
        that = this;

    if (this.first) {
        fs.writeFileSync(conf.blog + '-' + conf.mode + '.last', posts[0].id, {encoding: 'utf8', flags: 'w'});
        this.first = false;
    }

    var next = posts.every(function(post) {
        if ((conf.mode=='posts' && post.id > conf.last) || (conf.mode=='likes' && post.id != conf.last)) {
            if (post.photos) {
                post.photos.forEach(function(photo, i) {
                    var file = photo.original_size.url,
                        filename = that.getFilename(file, post, i);

                    that.download(file, filename);
                });
            }
            return true;
        }
        else {
            return false;
        }
    });

    if (next && posts.length >= 20) {
        this.offset+= 20;
        this.init();
    }
};

// download an image file
Downloader.prototype.download = function(file, filename) {
    var that = this;

    fs.exists(filename, function(e) {
        if (e && !conf.overwrite) {
            console.log(filename + ' already exists');
            return;
        }

        that.queue.add(file, function(rep) {
            var stream = fs.createWriteStream(filename, {flags: 'w'});
            rep.pipe(stream);
        });
    });
};

// build local filename
Downloader.prototype.getFilename = function(file, post, idx) {
    var frag = file.split('/');
    return conf.blog + '-' + conf.mode + '/' + frag.pop();
};


// START
var http = require('http'),
    fs = require('fs'),
    downloader = new Downloader();

downloader.init();