Gemfile

source "https://rubygems.org"

gem 'faraday'
gem 'faraday_middleware'
gem 'hashie'

gif_getter.rb

#!/usr/bin/env ruby

# Usage: TUMBLR_TOKEN="XYZ" bundle exec ruby gif_getter.rb | sed 's/\\\\/\\/g' > library.gifwit
# Gifwit does something weird with escaping characters in urls, it transforms '/' into '\/' which Ruby doesn't like.
# So I pipe the output through sed to sanitize it.
#
# Requirements:
# - the following dependencies installed via `bundle install`
# - setting ENV variable "TUMBLR_TOKEN" for your Tumbler API Token

require 'faraday'
require 'faraday_middleware'
require 'faraday_middleware/response/mashify'
require 'hashie'

CONN = Faraday.new(:url => 'https://api.tumblr.com/v2/blog/gif-database.tumblr.com/') do |faraday|
  faraday.params['api_key'] = ENV['TUMBLR_TOKEN']
  faraday.request  :url_encoded             # form-encode POST params
  faraday.response :json
  faraday.adapter  Faraday.default_adapter  # make requests with Net::HTTP
end

def get_by_offset(offset)
  result = CONN.get("posts", {offset: offset})
  mash = Hashie::Mash.new(result.body)
  posts = mash.response.posts
end

# Gets 20 posts at a time, can use offset to get next page


def sanitize_tags(tags)
  Array(tags).reject { |i| i[/[ \.]?gif$/]}
             .map { |i| i.gsub(/ /, '-')}
             .join(" ")
end

def sanitize_link(link)
  link.gsub(/\d+\.media\.tumblr/, 'media.tumblr')
      .gsub(%r(/), '\/')
end

def url_and_tags_from_post(post)
  begin
  plain_link = post.photos.first.alt_sizes.first.url
  # transform this.
  link = sanitize_link(plain_link)
  tags = sanitize_tags(post.tags)
  {
    url: link,
    keywords: tags
  }

  rescue => e
    {}
  end
end

steps = 1.upto(20).map { |i| i * 20 }
posts = steps.flat_map { |i| get_by_offset(i) }
all_of_em = posts.map { |i| url_and_tags_from_post(i)}.reject(&:empty?)
output = {images: all_of_em, version: "1"}
puts JSON.pretty_generate(output)

scrape.sh

#!/usr/bin/env bash

# NOTE, set TUMBLR_TOKEN="XYZ" with your token

OUTPUT_FILE="library.gifwit"

bundle exec ruby gif_getter.rb | sed 's/\\\\/\\/g' > $OUTPUT_FILE