Interactive tool for finding missinging YNMP twitter handles

gistfile1.py

import time
import requests
import twitter
import re
from bs4 import BeautifulSoup


consumer_key = '######'
consumer_secret = '######'
access_token = '######'
access_token_secret = '######'

ynmp_login = '######'
ynmp_password = '######'

tmpl = 'http://yournextmp.popit.mysociety.org/api/v0.1/search/persons?q=_exists_:standing_in.2015.post_id&page=%d'
more = True
page = 0
candidates = []
while more:
    page += 1
    print 'Current page: %d' % page
    j = requests.get(tmpl % page).json()
    more = j['has_more']
    candidates += j['result']
    time.sleep(0.5)

no_twitter = [x for x in candidates if x['contact_details'] == []]
print 100 - (100.* len(no_twitter) / len(candidates))

api = twitter.Api(
    consumer_key=consumer_key,
    consumer_secret=consumer_secret,
    access_token_key=access_token,
    access_token_secret=access_token_secret,
)
done_list = []
found_handles = []

for person in no_twitter:
    if person['id'] in done_list:
        continue
    print 'Searching twitter for: %s (%s, %s)' % (person['name'], person['standing_in']['2015']['name'], person['party_memberships']['2015']['name'])
    search = api.GetUsersSearch(person['name'])
    if len(search) == 0 or ([search[0].location, search[0].description] == ['', '']):
        done_list.append(person['id'])
        continue
    best_guess = search[0]
    print '%s (%s): %s' % (best_guess.name, best_guess.location, best_guess.description)
    matches = raw_input()
    if matches == 'y':
        found_handles.append([person['id'], best_guess.screen_name])
    done_list.append(person['id'])

with requests.Session() as s:
    time.sleep(0.5)
    # Log in to yournextmp
    login_page = 'https://yournextmp.com/accounts/login/'
    r = s.get(login_page, verify=False)
    token = re.search('csrfmiddlewaretoken\' value=\'(.*?)\'', r.text).group(1)
    r = s.post(login_page, data={'login': ynmp_login, 'password': ynmp_password, 'csrfmiddlewaretoken': token}, verify=False, headers={'Referer': login_page})    
    for handle in found_handles:
        time.sleep(0.5)
        update_url = 'https://yournextmp.com/person/%s/update' % handle[0]
        r = s.get(update_url, verify=False)
        form_soup = BeautifulSoup(r.text).form
        # faff about populating forms.
        # Mechanize is preferable for this sort of thing.
        payload = {x.attrs['name']: x.attrs.get('value', '') for x in form_soup.find_all('input') if 'name' in x.attrs}
        for x in form_soup.find_all('select'):
            selected = x.find('option', {'selected': 'selected'})
            if selected is not None:
                payload[x.attrs.get('name')] = selected.attrs.get('value', '')
            else:
                payload[x.attrs.get('name')] = ''
        payload['twitter_username'] = handle[1]
        payload['source'] = 'Added twitter handle'
        print 'Updating %s with @%s' % (update_url, handle[1])
        # Update!!!
        r = s.post(update_url, data=payload, verify=False, headers={"Referer": update_url})