Python script to Extract Href's from the list pages.

HrefExtractionScript.py

from bs4 import BeautifulSoup
import urllib.request

a={}
a = set()
f = open("output.txt","w")

with open('url.txt') as lines:
  for line in lines:
    #print(''.join(['URL:  ', line]))
    resp = urllib.request.urlopen(line)
    soup = BeautifulSoup(resp, from_encoding=resp.info().get_param('charset'),features='html.parser')
    for link in soup.find_all('a', href=True):
    #print(link.prettify())
      b = ''.join([link['href'], ' -- ' ,  link.get_text(' ', strip=True)])
      if b not in a:
        a.add(b)
        #print(''.join([line.rstrip(), ',', link['href'], ',' ,  link.get_text('', strip=True)]))
        f.write(''.join([line.rstrip(), ',', link['href'], ',' ,  link.get_text('', strip=True),'\n']))
f.close()