Flickr archive bulk downloader

flickr_archive_downloader_browser.js

var PAD = '00000';

var count = {
  all: 0,
  day: 0,
  single: 0
};

var timer, isFirst, imgs, current;

var days = Array().slice.call(document.querySelectorAll('.FullDay a'));

var iframe = document.createElement('iframe');
document.body.appendChild(iframe);

var link = document.createElement('a');
document.body.appendChild(link);

function save(src) {
  this.onload = null;
  link.href = src;
  link.download = (PAD + count.all++).slice(PAD.length * -1) + '_' + 
    ('00' + count.day).slice(-2) + '_' + ('000' + count.single++).slice(-3) + '.jpg';
  link.click();
  next();
}

function next() {
  if (current < imgs.length) {
    var src = imgs[current].src.replace(/_\w.jpg/, '_z.jpg');
    current++;
    var image = new Image();
    image.onload = save.bind(image, src);
    image.src = src;
  } else {
    nextPage();
  }
}

function nextPage() {
  var prev = iframe.contentWindow.document.querySelector('.Prev');
  if (prev) {
    prev.click();
  } else {
    var day = days.shift();
    if (day) {
      count.day++;
      count.single = 0;
      timer = setTimeout(function() {
        console.log('reload ', iframe.src);
        iframe.contentWindow.location.reload();
      }, 20000);
      isFirst = true;
      iframe.src = day.href;
    } else {
      link.parentNode.removeChild(link);
      iframe.parentNode.removeChild(iframe);
    }
  }
}
 
iframe.onload = function() {
  clearTimeout(timer);
  
  if (isFirst) {
    isFirst = false;
    var last = iframe.contentWindow.document.querySelector('.pages a:last-child');
    if (last) {
      return last.click();
    }
  }

  imgs = Array().slice.call(iframe.contentWindow.document.querySelectorAll('.pc_img')).reverse();
  current = 0;
  next();
};

isFirst = true;
iframe.src = days.shift().href;

flickr_archive_downloader_node.js

var Flickr = require('flickrapi');
var async = require('async');
var https = require('https');
var fs = require('fs');
var path = require('path');

var FLICKR_OPTIONS = {
  api_key: '',
  secret: '',
  user_id: '',
  access_token: '',
  access_token_secret: '',
};

var SEARCH_OPTIONS = {
  user_id: '',
  authenticated: true,
  min_taken_date: '2016-11-01 00:00:00',
  max_taken_date: '2016-11-30 23:59:59',
  sort: 'date-posted-asc',
  per_page: 500,
};

var SAVE_DIR = 'jpg';
var PHOTOS = [];
var FLICKR;

Flickr.authenticate(FLICKR_OPTIONS, function(error, flickr) {
  console.log('authenticate');
  FLICKR = flickr;
  getPhotos(1);
});

function getPhotos(page) {
  console.log('getPhotos', page);
  SEARCH_OPTIONS.page = page;
  FLICKR.photos.search(SEARCH_OPTIONS, checkResults);
}

function checkResults(err, result) {
  console.log('checkResults page:', result.photos.page,
      'total:', result.photos.total);
  PHOTOS.push({page: result.photos.page, photos: urlify(result.photos.photo)});
  if (result.photos.page + 1 <= result.photos.pages) {
    getPhotos(result.photos.page + 1);
  } else {
    unifyPhotos();
    downloadPhotos();
  }
}

function urlify(photos) {
  console.log('urlify', photos.length);
  return photos.map(function(p) {
    return 'https://farm' + p.farm + '.staticflickr.com/' + p.server +
        '/' + p.id + '_' + p.secret + '_b.jpg';
  });
}

function unifyPhotos() {
  PHOTOS = PHOTOS.reduce(function(p, c) {
    return p.concat(c.photos);
  }, []);
}

function downloadPhotos() {
  async.forEachOfLimit(PHOTOS, 16, function(url, index, cb) {
    downloadFile(url, ('00000' + index).slice(-5) + '.jpg', cb);
  }, function(err) {
    if (err) {
      console.log(err);
    } else {
      console.log('done', PHOTOS.length, 'photos');
    }
  });
}

function downloadFile(url, filename, cb) {
  console.log('downloading', filename, url);
  var file = fs.createWriteStream(path.join(SAVE_DIR, filename));
  https.get(url, function(response) {
    response.pipe(file);
    file.on('finish', function() {
      file.close(cb);
    });
  }).on('error', function(err) {
    fs.unlink(path.join(SAVE_DIR, filename));
    cb(err);
  });
}

fps

Math.floor(number_of_images / 90)
Math.floor(Math.floor(number_of_images / 90) / 2)

non_zero_bytes_files.sh

find . -size +0 -type f -name \*.jpg | wc -l

renamer.sh

x=1; for i in `find . -size +0 -type f -name \*.jpg`; do counter=$(printf %05d $x); cp "$i" jpg/img"$counter".jpg; x=$(($x+1)); done

# or

x=1; for i in `ls -lUm1`; do counter=$(printf %05d $x); cp "$i" jpg/img"$counter".jpg; x=$(($x+1)); done

timelapse.sh

ffmpeg -r 50 -i img%05d.jpg -r 100 -vcodec libx264 -crf 20 -g 15 timelapse.mp4

# or

avconv -r 50 -i %05d.jpg -r 100 -vcodec libx264 -crf 20 -g 15 ../timelapse.mp4