Twitter API - timeline 3200 limit

gistfile1.txt

{
 "metadata": {
  "name": "",
  "signature": "sha256:db2443c6f9fcf3519206f186f3f594b1c87a884c732a90e8704be69561e29b64"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import twitter\n",
      "from twitter_settings import CONSUMER_KEY, CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET\n",
      "\n",
      "# CONSUMER_KEY = ''\n",
      "# CONSUMER_SECRET = ''\n",
      "# OAUTH_TOKEN = ''\n",
      "# OAUTH_TOKEN_SECRET = ''\n",
      "\n",
      "auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,\n",
      "                           CONSUMER_KEY, CONSUMER_SECRET)\n",
      "\n",
      "twitter_api = twitter.Twitter(auth=auth)\n",
      "\n",
      "print twitter_api"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<twitter.api.Twitter object at 0x0000000003AC4080>\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "from pandas import Series, DataFrame"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ischooler_list = twitter_api.friends.ids(screen_name=\"BerkeleyISchool\", count=2500)['ids']    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def get_timeline(user, batch_count = 200):\n",
      "    '''Recursively get 3200 most recent tweets of given user.'''\n",
      "    tl = twitter_api.statuses.user_timeline(user_id=user, count=batch_count, trim_user = 1, include_rts = 1)\n",
      "    timeline = tl\n",
      "    while True:\n",
      "        if len(tl) < batch_count:\n",
      "            return timeline\n",
      "        smallest = min(tl, key=lambda k: k['id'])['id']-1\n",
      "        tl = twitter_api.statuses.user_timeline(user_id=user, count=batch_count, trim_user = 1, include_rts = 1, max_id = smallest)\n",
      "        timeline += tl\n",
      "\n",
      "\n",
      "def get_tweetID_list(timeline):\n",
      "    '''Replace the retweeted with original tweet id. Return a list of tweet ID in long.'''\n",
      "    tweetID_list = []\n",
      "    for tweet in timeline:\n",
      "        if 'retweeted_status' in tweet.keys():\n",
      "            tweetID_list.append(tweet['retweeted_status']['id'])\n",
      "        else:\n",
      "            tweetID_list.append(tweet['id'])\n",
      "    return tweetID_list\n",
      "# test_user = \"41123\"        \n",
      "# test_tl = get_timeline(test_user)\n",
      "# len(test_tl)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "599"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def get_friends(user):\n",
      "#     if(twitter_api.users.lookup(user_id = user)[0][\"protected\"] == False):\n",
      "    return twitter_api.friends.ids(user_id = user)['ids']\n",
      "def get_followers(user):\n",
      "    return twitter_api.followers.ids(user_id = user)['ids']\n",
      "# get_follower(ischooler_list[171])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def __main__():\n",
      "    dic = {}\n",
      "    for user in ischooler_list[170:175]:\n",
      "        if(twitter_api.users.lookup(user_id = user)[0][\"protected\"] == False):\n",
      "            tl = get_timeline(user)\n",
      "#             print len(tl)\n",
      "            if len(tl) != 0:\n",
      "                dic[user] = [get_tweetID_list(tl), get_friends(user), get_followers(user)]#4, 5, 6\n",
      "    return dic\n",
      "__main__()        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "22\n",
        "382"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "0"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "90"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "599"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}