Read and parse the sitemap of a site.

parseSitemap.py

#!/usr/bin/env python

import requests
from xml.etree.ElementTree import XML, SubElement, Element, tostring
import sys
from bs4 import BeautifulSoup

url = sys.argv[1]

user_agent = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0' }

try:
	requests.packages.urllib3.disable_warnings()
	r = requests.get(url, verify=False, headers=user_agent)
	soup = BeautifulSoup(r.text, "html5lib")
	urls = soup.findAll('url')
	for link in urls:
		print link.find('loc').string
		
	

except Exception as e:
	print "[ERR] %s"%str(e)