repeats fetching of follower lists from twitter for a small number of users

followers.py

#!/usr/bin/env python

"""
Simple tool to fetch follower lists every n seconds and store them
with time-based filenames.  Can be later merged, deduped, and fed to
users/lookup method to extract full user info.
"""

import argparse
import datetime
import logging
import subprocess
import time

from twarc import Twarc

logging.basicConfig(
    filename='followers.py.log',
    level=logging.INFO,
    format="%(asctime)s %(levelname)s %(message)s"
)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--interval', dest='interval', type=int, default=150,
                        help='number of seconds to wait before fetching again')
    parser.add_argument('screen_names', nargs='+', metavar='screen_name',
                        help='users whose follower lists to fetch')
    options = parser.parse_args()

    if not options.screen_names:
        parser.error('please add at least one valid Twitter user name')

    # Verify the screen names and stash their user info
    fp = open('user_profiles.json', 'w')
    call_list = ['./twarc.py', '--lookup_screen_names']
    call_list.extend(options.screen_names)
    subprocess.run(call_list, stdout=fp)
    fp.close()
    logging.info('saved user profiles')

    while True:
        for screen_name in options.screen_names:
            dt_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
            fname = '%s-followers-%s.txt' % (dt_str, screen_name)
            fp = open(fname, 'w')
            call_list = ['./twarc.py', '--follower_ids', screen_name]
            subprocess.run(call_list, stdout=fp)
            fp.close()
            logging.info('saved follower list %s', fname)
        time.sleep(options.interval)