newestRMP

newestRMP.py

import sys
# sys.setdefaultencoding() does not exist, here!
reload(sys)  # Reload does the trick!
sys.setdefaultencoding('UTF8')
import requests
import threading
import json
#import sqlite3

def sanitize(value, is_value=True):
    if isinstance(value, dict):
        value = {sanitize(k,False):sanitize(v,True) for k, v in value.items()}
    elif isinstance(value, list):
        value = [sanitize(v, True) for v in value]
    elif isinstance(value, str):
        if not is_value:
            value = re.sub(r"[.]", "", value)
    return value

url = "http://search.mtvnservices.com/typeahead/suggest/?solrformat=true&rows=500&q=*&defType=edismax&qf=teacherfirstname_t%5E2000+teacherlastname_t%5E2000+teacherfullname_t%5E2000+autosuggest&bf=pow(total_number_of_ratings_i,2.1)&start={0}&siteName=rmp&fl=*&fq="
for i in range(0, 1105000, 750):
	try:
		tempUrl = url.format(i)
		headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0'}
		res = requests.get(tempUrl, headers=headers)
		#print res.text
		with open('listOfRMP.text', 'a') as outfile:
			json.dump(str(res.text) + "\n\n", outfile)
		print(i)
	except Exception as exp:
		print exp
		print("Error on {}".format(i))