twitter-search-aiit.py

# -*- coding:utf-8 -*-
import json, config, time, MeCab
from requests_oauthlib import OAuth1Session
from jinja2 import Template

CK = config.CONSUMER_KEY
CS = config.CONSUMER_SECRET
AT = config.ACCESS_TOKEN
ATS = config.ACCESS_TOKEN_SECRET
twitter = OAuth1Session(CK, CS, AT, ATS)

url = 'https://api.twitter.com/1.1/search/tweets.json'

params = {
    'q': 'ビッグデータ',
    'result_type': 'recent',
    'lang': 'ja',
    'count': 100,
}
excluded_pos = ["名詞-サ変接続", "名詞-数", "名詞-接尾-助数詞", "名詞-非自立-一般", "名詞-接尾-一般", "名詞-接尾-人名", "名詞-接尾-サ変接続", "名詞-副詞可能", "名詞-接尾-特殊"]
max_id = None

mecab = MeCab.Tagger("-Ochasen")

while True:
    if max_id is not None:
        params['max_id'] = max_id

    time.sleep(10)  # Rate Limit: 180 requests per 15 minutes
    response = twitter.get(url, params=params)
    if response.status_code == 200:
        search_timeline = json.loads(response.text)
        for tweet in search_timeline['statuses']:
            parsed = mecab.parse(tweet['text'])
            tokens = parsed.split("\n")
            for token in tokens:
                mecab_structure = token.split("\t")
                if (3 < len(mecab_structure) and mecab_structure[3].startswith("名詞") and mecab_structure[3] not in excluded_pos):
                    print("{}\t{}\t{}".format(mecab_structure[0], mecab_structure[3], tweet['created_at']))

        next_params = search_timeline['search_metadata']['next_results'][1:].split('&')
        max_id = [pair for pair in next_params if pair.startswith('max_id')][0].split('=')[1]