// fetch status tester Python

scraper.py

import re

import bs4
import aiohttp
import asyncio

HOST = 'http://dev-api.wyzant.com'

# Holds fetched API links and their status.
api_status = {}

async def get_http_status(url):
    with aiohttp.Timeout(5):
        with aiohttp.ClientSession() as session:
            async with session.get(url) as resp:
                return resp.status

async def extract_links_and_test_urls():
    url = '{}/Help'.format(HOST)

    with aiohttp.ClientSession() as session:
        async with session.get(url) as resp:
            page = await resp.text()

    #  Extract links.
    links_finder = bs4.SoupStrainer('a', href=re.compile('.*/Help/Api/.*'))
    links = bs4.BeautifulSoup(page, 'html.parser', parse_only=links_finder)

    # Build a list of links
    links = ['{}{}'.format(HOST, link['href']) for link in links]

    # Get HTTP status for each link.
    for link in links[:100]:
        api_status[link] = await get_http_status(link)


if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    conn = aiohttp.connector.TCPConnector()
    loop.run_until_complete(extract_links_and_test_urls())
    print(api_status)