Scraping bot for oba bibliotheek

oba-bot.py

import json
import cookielib
import mechanize
import urllib
import urllib2
from bs4 import BeautifulSoup
from pymongo import MongoClient

# Database stuff - JBG
#client = MongoClient()
#db = client.library
#collection = db.books

# File to write to - JBG
txt_file = open('library.txt', 'w')

# Library url - JBG
url = 'http://zoeken.oba.nl'

br = mechanize.Browser()
br.addheaders = [
  ('Upser-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:40.0) Gecko/20100101 Firefox/40.0'),
  ('Accept', '*/*'),
  ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'),
  ('Accept-Language', 'en-US,en;q=0.5'),
  ('Connection', 'keep-alive')
]

# Enable cookie support for urllib2 
cookiejar = cookielib.LWPCookieJar()
br.set_cookiejar(cookiejar)

# Browser options
br.set_handle_equiv(True)
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)

def get_text(child):
  try:
    link = child.find('a')
    return link.contents[0].strip()
  except Exception, e:
    try:
      return child.contents[0].strip()
    except Exception, e:
      return None

count = 0
a = 97 # is a - JBG
for i in range(0, 26):
  page = 1 
  q = str(unichr(97+i))
  print 'q: ' + q 
  j = 1 
  page_count = 1
  while True:
    print 'j: ' + str(j) + ', page_count: ' + str(page_count)
    query_url = url + '?curpage=' + str(j) + '&q=' + q + '&dim=dcterms:type.uri%28http://dbpedia.org/ontology/Book%29'
    print query_url
    try:
      res = br.open(query_url)
      soup = BeautifulSoup(res.read(), 'html.parser')
      articles = soup.findAll('article', class_='record')

      # Check number of pages - JBG
      pages = soup.find('div', class_='pagination')
      pages_list = pages.findAll('a')
      returned_page_count = pages_list[len(pages_list) - 3].contents[0]

      if int(returned_page_count) != page_count:
        page_count = returned_page_count
        print 'PAGE_COUNT: ' + page_count

      for article in articles:
        try:
          subpath = article.findAll('a', class_='classiclink')[0]['href']
          link = url + subpath
          try:
            #print link
            res = br.open(link)
            soup = BeautifulSoup(res.read(), 'html.parser')
            article = soup.findAll('article', class_='fullrecord-block')[0]
            details = article.find('dl')
            children = details.findChildren()
            key = None 
            book_dict = {}
            for child in children:
              if child.name == 'dt':
                text = get_text(child)
                if text != None:
                  key = text
              elif child.name == 'dd':
                text = get_text(child)
                if text != None and key != None:
                  book_dict[key] = text
                elif text != None and key == None:
                  obj = book_dict[key]
                  if isinstance(obj, list):
                    obj.append(text)
                  else:
                    obj = [ obj ]
                    obj.append(text)
                    book_dict[key] = obj 
            # Write to DB - JBG
            #collection.insert_one(book_dict)
          
            # Write to File - JBG
            json_str = json.dumps(book_dict, separators=(',',':'))
            txt_file.write(json_str)
            txt_file.write('\n') 

            count += 1
            print 'Inserted ' + str(count) + ' books.'
          except Exception, e:
            print e 
        except Exception, e:
          print e 
    except Exception, e:
      #print e.code
      print e
      #print e.read()
    j += 1
    if j == page_count:
      break
# Close database - JBG
#client.close()

# Close text file - JBG
txt_file.close()

#res = br.open(url)

#print br.title()
#print res.info()  # headers
#print res.read()  # body