Tweets from streaming API

crawl_tweets.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json

from tweepy.streaming import StreamListener
from tweepy import Stream, OAuthHandler

######## Application settings
## API key
consumer_key=''
## API secret
consumer_secret=''
######## Your access token
## Access token
access_token=''
## Access token secret
access_secret=''

search_query=[u'крим',
              u'крым',
              u'война',
              u'війна', 
              u'украина',
              u'україна',
              u'pocсия',
              u'pociя',
              'crimea',
              'CrimeaInvasion',
              'ukraine',
              'putin',
              'russia']



import logging
# create logger with 'spam_application'
logger = logging.getLogger('tweetstream_war')
logger.setLevel(logging.DEBUG)

# create file handler which logs debug messages
fh = logging.FileHandler('tweetstream_war.log')
fh.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
fh.setFormatter(formatter)

# create console handler with a higher log level
from logging import StreamHandler
class DebugStreamHandler(StreamHandler):
    def __init__(self, stream=None):
        StreamHandler.__init__(self, stream)
    def emit(self, record):
        if not record.levelno == logging.DEBUG:
            return
        StreamHandler.emit(self, record)

ch = DebugStreamHandler() #logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)

# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)


class MyStreamListener(StreamListener):
    def __init__(self):
        self.tweet_count = 0
        self.received_friend_ids = False

    def on_data(self, data):
        try:
            tweet = json.loads(data)
        except Exception:
            logger.debug("Failed to parse tweet data")
        if tweet:
            text = json.dumps(tweet,
                              ensure_ascii=False,
                              sort_keys=True)
                              #indent=2)
            logger.info(text)
            try:
                logger.debug(tweet['text'] + tweet['created_at'])
            except:
                pass
        return True

def main():
    auth = OAuthHandler(consumer_key,
                        consumer_secret)

    auth.set_access_token(access_token,
                          access_secret)

    listener = MyStreamListener()

    stream = Stream(auth, listener)

    try:
        stream.filter(track=search_query, stall_warnings=True) #, count=10)
    except KeyboardInterrupt:
        logger.debug("Total tweets received: %d" % listener.tweet_count)

if __name__ == "__main__":
    while 1:
        try:
            main()
        except Exception as e:
            logger.debug("Unexpected exception! %s", e)