twitter_to_neo.rb

BEARER='...'
def load_tweets(query,since_id=nil,lang="en",page=1,rpp=100)
  res=RestClient.get('https://api.twitter.com/1.1/search/tweets.json',
   {:params=> {:q=>query, :lang=>lang,:count=>rpp,:result_type=>:recent,:since_id=>since_id},
    :accept=>:json,
    :Authorization => "Bearer #{BEARER}"})
  puts "query '#{query}'\n since id #{since_id} result #{res.code}"
  return [] unless res.code==200

  data=JSON.parse(res.to_str)

  data['statuses']
end

add_tweet_query = "
UNWIND {tweets} as t
WITH t, t.entities as e, t.user as u
MERGE (tweet:Tweet {id:t.id}) 
  ON CREATE SET tweet.text = t.text, tweet.created_at = t.created_at,tweet.short = t.short
MERGE (user:User {screen_name:u.screen_name}) 
  ON CREATE SET user.name = u.name,user.name = u.name, user.location = u.location, user.profile_image_url = u.profile_image_url, 
                user.followers_count = u.followers_count, user.friends_count = u.friends_count, user.statuses_count = u.statuses_count 
MERGE (user)-[:POSTED]->(tweet)
FOREACH (h IN e.hashtags | 
  MERGE (tag:Tag {name:h.text}) MERGE (tag)<-[:TAGGED]-(tweet))
FOREACH (u IN e.urls | 
  MERGE (url:Url {url:u.expanded_url}) MERGE (url)<-[:CONTAINS]-(tweet))
FOREACH (m IN e.user_mentions | 
  MERGE (mentioned:User {screen_name:m.screen_name}) ON CREATE SET mentioned.name = m.name MERGE (mentioned)<-[:MENTIONS]-(tweet))
FOREACH (r IN [r in [t.in_reply_to_status_id] WHERE r IS NOT NULL] | 
  MERGE (reply:Tweet {id:r}) MERGE (tweet)-[:REPLY_TO]->(reply))
FOREACH (sn IN [sn in [t.in_reply_to_screen_name] WHERE sn IS NOT NULL] | 
  MERGE (replied:User {screen_name:sn}) MERGE (tweet)-[:REPLY_TO_USER]->(replied))
RETURN count(*) as added_tweets
"
neo = Neography::Rest.new

[[:t, :Tweet, :id],[:u, :User, :screen_name],[:u, :Url, :url],[:t, :Tag, :name]]
.each do |(id,label,prop)| 
   neo.commit_transaction("create constraint on (#{id}:#{label}) assert #{id}.#{prop} is unique")
end

while true
  twitter_id = nil
  begin
    query = ENV['TWITTER_SEARCH']||'oscon OR oreillymedia OR graphalchemist OR graphconnect OR neo4j'
    tweets=load_tweets(query,twitter_id)
    twitter_id = tweets.first["id"]
    puts "Found #{tweets.length} new tweets"
    tweets.each { |tweet| tweet["short"] = tweet["text"].gsub(/https?:\S+/i,"(link)").split(/\s/)[0..10].join(" ")+" ..." }
    res = neo.commit_transaction([add_tweet_query,{:tweets=>tweets}])
    puts res
  rescue => e
    puts e
  end
  sleep(300)
end