filter your twitter friends

twitter_friends_with_caching.rb

#!/usr/bin/env ruby
require 'twitter'
require 'dbm'


# https://dev.twitter.com/apps
# https://dev.twitter.com/apps/{some_id}/show -> (OAuth settings + Your access token) 
CONSUMER_KEY       = "_"
CONSUMER_SECRET    = "_"
OAUTH_TOKEN        = "_"
OAUTH_TOKEN_SECRET = "_"





USERNAME = 'mindreframer'
REGEX = /Opscode|devops|webops|Ops|Sysa|Pup/i


Twitter.configure do |config|
  config.consumer_key       = CONSUMER_KEY
  config.consumer_secret    = CONSUMER_SECRET
  config.oauth_token        = OAUTH_TOKEN
  config.oauth_token_secret = OAUTH_TOKEN_SECRET
end


MyCursor = Struct.new(:collection, :next)
class MyDB
  class << self
    def connection
      @connection ||= DBM.open('cache', 777, DBM::WRCREAT)
    end

    def save(key, v)
      connection[key] = dump(v)
    end

    def get(key)
      if data = connection[key]
        load(data)
      else
        nil
      end
    end

    def dump(v)
      if v.is_a?(Twitter::Cursor)
        v = MyCursor.new(v.collection, v.next)
      end
      Marshal.dump(v)
    end

    def load(v)
      Marshal.load(v)
    end
  end
end


class CachedTwitter
  def self.friends(user, params ={})
    key = key_from_params(user, params)
    if r = MyDB.get(key)
      r
    else
      r = Twitter.friends(user, params)
      MyDB.save(key, r)
      r
    end
  end

  def self.key_from_params(*params)
    params.map{|x| x.to_s}.join("_")
  end
end

class SleepyCachedTwitter < CachedTwitter
  def self.friends(user, params ={})
    key = key_from_params(user, params)
    if r = MyDB.get(key)
      r
    else
      begin
        r = Twitter.friends(user, params)
        sleep 10
        MyDB.save(key, r)
        r
      rescue Twitter::Error::TooManyRequests => e
        puts "TooManyRequests"
        MyCursor.new([], 0)
      end
    end
  end
end


class ShortUrlResolver
  def self.resolve(url)
    unless r = MyDB.get(key_from_url(url))
      r = `curl --silent --head #{url} |grep location|awk '{print $2}'`.strip
      MyDB.save(key_from_url(url), r)
      r
    end
    r
  end

  def self.key_from_url(url)
    url.tr('^A-Za-z0-9', '_')
  end
end

all_friends = []
friends     = CachedTwitter.friends(USERNAME)
all_friends += friends.collection
next_cursor = friends.next
while next_cursor != 0
  puts "getting #{next_cursor}"
  friends = SleepyCachedTwitter.friends(USERNAME, :cursor => next_cursor)
  all_friends += friends.collection
  next_cursor = friends.next
end; nil

# now filter
filtered = all_friends.select{|x|  x[:description] =~ REGEX }; nil
filtered.sort_by!{|x| x[:screen_name].downcase}

puts "resolving URLs"
filtered.each do |x|
  if x[:url]
    print "."
    full_url = ShortUrlResolver.resolve(x[:url])
    x.attrs[:url] = full_url
  end
end;nil

require 'pp'
puts "FOUND #{filtered.size} devops peeps"
pp filtered.map{|u| {:twitter => u[:screen_name], :name => u[:name], :blog => u[:url], :description => u[:description]}}