Python BS4 sitemap validator - checks HTTP Response for all links inside <loc> .. </loc> tags

validate_robots.py

#! /usr/bin/env python

# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import requests
import time
from time import sleep


def process_sitemap(s):
    soup = BeautifulSoup(s, "lxml")

    return [loc.text for loc in soup.findAll('loc')]

  
def main():

    with requests.Session() as s:  

        s.headers.update(
            {
                'User-Agent': 'Mozilla/5.0'
            }
        )

        robots = s.get('http://www.yoursite.com/robots.txt')

        for line in robots.text.splitlines():

            if line.strip().endswith('.xml') and 'sitemap' in line:
                sitemap = 'http://' + line.split()[-1]

                for url in process_sitemap(s.get(sitemap).text):
                    try:
                        print ('{url} {status_code}'.format(
                            url=url, 
                            status_code=s.get(url).status_code
                            )
                        )
                    except requests.exceptions.ConnectionError as e:
                        print('Error --> {}'.format(e))

main()