joshwand · June 28, 2024 14:24 · chufucious · Nov 10, 2013 · dannberg · Nov 12, 2013
diff --git a/tumblr-likes-downloader.rb b/tumblr-likes-downloader.rb
 require 'tumblr_client'
 require 'mechanize'
 require 'date'
 require 'yaml'
 require 'uri'

 Tumblr.configure do |config|
  config.consumer_key = "consumer_key"
  config.consumer_secret = "consumer_secret"
  config.oauth_token = "oath_token"
  config.oauth_token_secret = "token_secret"
 end

 THREADPOOL_SIZE = 4
 SEGMENT_SIZE = 25

 HISTORY_FILE = File.join(File.dirname(__FILE__), "previous.yml")


 previous_dled_ids = YAML.load(File.open(HISTORY_FILE)) rescue []

 directory = "tumblr-likes"

 client = Tumblr::Client.new
 likes = client.likes
 liked_count = likes["liked_count"]

 puts liked_count

 likes = []

 (0...liked_count).each do |i|
  if i==0 or i%SEGMENT_SIZE==0
    p "getting #{SEGMENT_SIZE} more likes starting at #{i}: #{likes.count} thus far"
    client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do |like|
     likes << like if like['type'] == 'photo' and !previous_dled_ids.include?(like['id'])
   end
 end

 end

 if likes.empty? 
  p "no new likes!"
  exit 0
 end 

 puts "#{likes.count} new likes!"



 # some of this code comes from https://github.com/jamiew/tumblr-photo-downloader

 already_had = 0

 threads = []
 likes.each_slice(likes.count / THREADPOOL_SIZE ).each do |group|

  threads << Thread.new {
    begin
      p "launching thread #{threads.size + 1}"

      group.each do |like|

        i = 0
        like["photos"].each do |photo|
          url = photo["original_size"]["url"]

          filename = "#{like["blog_name"]}-#{like["slug"]}-"
          filename += "#{i}-" if i > 0
          filename += File.basename(URI.parse(url).path.split('?')[0])
          if File.exists?("#{directory}/#{filename}")
            puts "Already have #{url}"
            already_had += 1
          else
            begin
              puts "Saving photo #{url}"
              file = Mechanize.new.get(url)
              file.save_as("#{directory}/#{filename}")
            rescue Mechanize::ResponseCodeError => e
              puts "Error #{e.response_code} getting file for #{url}"
            end
          end
          i += 1  
          previous_dled_ids << like['id']            
        end
      end
    rescue Exception => e
      puts "unhandled exception:, #{$!}"
    end
    p "closing thread"
  }
  
 end

 threads.each{|t| t.join }

 YAML.dump(previous_dled_ids, File.open(HISTORY_FILE, "w"))
	require 'tumblr_client'
	require 'mechanize'
	require 'date'
	require 'yaml'
	require 'uri'

	Tumblr.configure do \|config\|
	config.consumer_key = "consumer_key"
	config.consumer_secret = "consumer_secret"
	config.oauth_token = "oath_token"
	config.oauth_token_secret = "token_secret"
	end

	THREADPOOL_SIZE = 4
	SEGMENT_SIZE = 25

	HISTORY_FILE = File.join(File.dirname(__FILE__), "previous.yml")


	previous_dled_ids = YAML.load(File.open(HISTORY_FILE)) rescue []

	directory = "tumblr-likes"

	client = Tumblr::Client.new
	likes = client.likes
	liked_count = likes["liked_count"]

	puts liked_count

	likes = []

	(0...liked_count).each do \|i\|
	if i==0 or i%SEGMENT_SIZE==0
	p "getting #{SEGMENT_SIZE} more likes starting at #{i}: #{likes.count} thus far"
	client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do \|like\|
	likes << like if like['type'] == 'photo' and !previous_dled_ids.include?(like['id'])
	end
	end

	end

	if likes.empty?
	p "no new likes!"
	exit 0
	end

	puts "#{likes.count} new likes!"



	# some of this code comes from https://github.com/jamiew/tumblr-photo-downloader

	already_had = 0

	threads = []
	likes.each_slice(likes.count / THREADPOOL_SIZE ).each do \|group\|

	threads << Thread.new {
	begin
	p "launching thread #{threads.size + 1}"

	group.each do \|like\|

	i = 0
	like["photos"].each do \|photo\|
	url = photo["original_size"]["url"]

	filename = "#{like["blog_name"]}-#{like["slug"]}-"
	filename += "#{i}-" if i > 0
	filename += File.basename(URI.parse(url).path.split('?')[0])
	if File.exists?("#{directory}/#{filename}")
	puts "Already have #{url}"
	already_had += 1
	else
	begin
	puts "Saving photo #{url}"
	file = Mechanize.new.get(url)
	file.save_as("#{directory}/#{filename}")
	rescue Mechanize::ResponseCodeError => e
	puts "Error #{e.response_code} getting file for #{url}"
	end
	end
	i += 1
	previous_dled_ids << like['id']
	end
	end
	rescue Exception => e
	puts "unhandled exception:, #{$!}"
	end
	p "closing thread"
	}

	end

	threads.each{\|t\| t.join }

	YAML.dump(previous_dled_ids, File.open(HISTORY_FILE, "w"))