US Governors Contact Information

governors.csv

scrape_governors.py

# inspired by / updates https://github.com/spacedogXYZ/us_governors_contact
# scraped from https://www.nga.org/cms/governors/addresses
# needs: pip install beautifulsoup4 requests unicodecsv

from bs4 import BeautifulSoup
import unicodecsv as csv
import requests, re


req = requests.get('https://www.nga.org/cms/governors/addresses')
soup = BeautifulSoup(req.text, "html5lib")


with open('governors.csv', 'w') as f:

    writer = csv.DictWriter(f, fieldnames=['state', 'first_name', 'last_name',
                                            'address1', 'address2',
                                            'city', 'state_abbr', 'zip',
                                            'phone', 'fax', 'url'])

    writer.writeheader()

    # would prefer something more semantic but whaddyagonna do.
    for gov in soup.select('.article-body .col-sm-6.col-md-4'):

        lines = gov.find('p').text.split('\n')
        gov_data = {'state': gov.find('h3').text}

        if lines[0].startswith('Office of Governor'):
            gov_data['first_name'], gov_data['last_name'] = lines.pop(0).replace('Office of Governor ', '').rsplit(' ', 1)

        if 'Governor\'s website' in lines[-1]:
            gov_data['url'] = gov.find('a')['href']
            lines.pop(-1)

        if 'Fax:' in lines[-1]:
            gov_data['fax'] = lines.pop(-1).replace('Fax: ', '').replace('/', '-')

        if 'Phone:' in lines[-1]:
            gov_data['phone'] = lines.pop(-1).replace('Phone: ', '').replace('/', '-')

        if re.search(r'\d{5}', lines[-1]):
            gov_data['city'], gov_data['state_abbr'], gov_data['zip'] = re.match(r'(?P<city>.*)\,\s(?P<state_abbr>[A-Z]{2})\s(?P<zip>[\d\-]+)$', lines.pop(-1)).groups()

        if len(lines) == 1:
            gov_data['address1'] = lines[0]
        else:
            gov_data['address1'], gov_data['address2'] = lines
        
        writer.writerow(gov_data)