vote_scraper.py

import time
import datetime
import pprint

from django.core.management.base import BaseCommand, CommandError
from django.db import IntegrityError, transaction

from finder.api import recent_votes
from finder.models import Legislator, Vote, Bill


class Command(BaseCommand):
    def handle(self, *args, **options):
        count = 0
        bills = []
        bill_objects = []
        vote_objects = []

        for i in range(10):
            results = recent_votes(i)
            count += len(results)
            bills.append(results)
        self.stdout.write("Appended results of the API calls")
        self.stdout.write("{} bills ready to create.".format(count))

        for entitylist in bills:
            for bill in entitylist:
                try:
                    bill_objects.append(
                        Bill(
                            title = bill['bill']['official_title'],
                            short_title = bill['bill']['short_title'],
                            number = bill['bill']['number'],
                            chamber = bill['bill']['chamber'],
                            bill_type = bill['bill']['bill_type'],
                            bill_id = bill['bill']['bill_id'],
                            opencongress_url = bill['bill']['urls']['opencongress'],
                            sponsor = Legislator.objects.get(bioguide_id=bill['bill']['sponsor_id']))
                        )
                except KeyError:
                    pass

        with transaction.atomic():
            for bill in bill_objects:
                Bill.objects.get_or_create(
                        bill_id=bill.bill_id,
                        defaults={
                            'title': bill.title,
                            'short_title': bill.short_title,
                            'number': bill.number,
                            'chamber': bill.chamber,
                            'bill_type': bill.bill_type,
                            'bill_id': bill.bill_id,
                            'opencongress_url': bill.opencongress_url,
                            'sponsor': bill.sponsor}
                        )

        self.stdout.write("{} bill objects".format(Bill.objects.count()))
        self.stdout.write("Created the Bill objects.")

        for entitylist in bills:
            for bill in entitylist:
                status = ""
                for voter, vote in bill['voter_ids'].items():
                    try:
                        date = time.strptime(bill['last_vote_at'], "%Y-%m-%dT%H:%M:%SZ")
                    except KeyError:
                        date = None
                        status = "{} had no last_vote_at entry.".format(bill['bill']['bill_id'])

                    try:
                        vote_obj = Vote(
                            bill=Bill.objects.get(bill_id=bill['bill']['bill_id']),
                            legislator=Legislator.objects.get(bioguide_id=voter),
                            vote=vote)
                        if date:
                            vote_obj.date = date
                    except KeyError:
                        pass
                    vote_objects.append(vote_obj)
                self.stdout.write(status)

        with transaction.atomic():
            for vote in vote_objects:
                Vote.objects.get_or_create(
                    bill=vote.bill,
                    legislator=vote.legislator,
                    vote=vote.vote,
                    date=vote.date)

        self.stdout.write("Created {} Vote objects.".format(Vote.objects.count()))