twspider2.py

from urllib.request import urlopen
import twurl  # This has a red underscore, because PyCharm doesn't like imports that have no class(es)
import json
from mm_db import mm_db
import mm_file

TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
DB_PATH = '../../databases/'

db = mm_db(DB_PATH + 'spider.sqlite', 'sqlite3')
con = db.connection
cur = db.cursor

cdc = mm_file.from_http(False).cdc

cur.execute('CREATE TABLE IF NOT EXISTS Twitter (name TEXT, retrieved INTEGER, friends INTEGER)')

while True:
    account = input('Enter Twitter account, \'quit\' or hit enter: ')
    if account == 'quit':
        break
    if len(account) < 1:
        cur.execute('SELECT name FROM Twitter WHERE retrieved = 0 LIMIT 1')
        try:
            account = cur.fetchone()[0]
        except:
            print('No unretrieved Twitter accounts found.')
            continue

    url = twurl.augment(TWITTER_URL, {'screen_name': account, 'count': '5'})
    # print('Retrieving', url)
    connection = urlopen(url, context=cdc)
    data = connection.read().decode()
    headers = dict(connection.getheaders())

    # How many remaining API-calls left, until API-calls will be suspended
    print('Remaining', headers['x-rate-limit-remaining'])
    js = json.loads(data)

    cur.execute('UPDATE Twitter SET retrieved=1 WHERE name = ?', (account,))

    count_friends = {'old': 0, 'new': 0}

    for user in js['users']:
        friend = user['screen_name']
        print(friend)
        cur.execute('SELECT friends FROM Twitter WHERE name = ? LIMIT 1', (friend, ))
        try:
            count = cur.fetchone()[0]
            cur.execute('UPDATE Twitter SET friends = ? WHERE name = ?', (count+1, friend))
            count_friends['old'] += 1
        except:
            cur.execute('INSERT INTO Twitter (name, retrieved, friends) VALUES(?, 0, 1)', (friend, ))
            count_friends['new'] += 1
    print('New accounts=', count_friends['new'], ' revisited', count_friends['old'])
    con.commit()
cur.close()