rebuild.py

from pymongo import MongoClient
import datetime

topics = MongoClient('10.8.8.111:27017')['onions35']['topics']
users = MongoClient('10.8.8.111:27017')['onions']['users']
finish_state = MongoClient('10.8.8.111:27017')['ronfedb']['finishStateNew']
finish_state.drop()

DATE_DEFAULT_HOLD = datetime.datetime(2016,5,5,1)
DATE_INIT = datetime.datetime(1970,1,1)

# STEP 1 Get topic - type mapping
topic_to_type = {}

x = topics.find()
for doc in x:
    topic_to_type[str(doc['_id'])] = doc['type']

# STEP 2 deal with user data
write_queue = []
x = users.find()
for doc in x:
    if 'finishState' in doc.keys() and 'math' in doc['finishState'].keys() and 'topics' in doc['finishState']['math'].keys() and doc['finishState']['math']['topics'] != {}:
        cur_topics = doc['finishState']['math']['topics']
        for k, v in cur_topics.iteritems():                
            if type(v) != unicode and 'practices' in v.keys() and v['practices'] != {}:
                for kx, vx in v['practices'].iteritems():
                    # try:
                    #     print vx['upgradeTime']
                    #     print type(vx['upgradeTime'])
                    # except:
                    #     pass
                    if type(vx['finishTime']) == float:
                        finish_time = datetime.datetime.fromtimestamp(vx['finishTime'] / 1000)
                    else:
                        finish_time = vx['finishTime']
                    try:
                        if type(vx['upgradeTime']) == float:
                            upgrade_time = datetime.datetime.fromtimestamp(vx['upgradeTime'] / 1000)
                        else: 
                            upgradeTime = vx['upgradeTime']
                    except:
                        upgrade_time = None

                    if ('state' in vx.keys()) and (finish_time >= DATE_DEFAULT_HOLD) and ((upgrade_time == None) or (upgrade_time < DATE_DEFAULT_HOLD)): 
                        
                        cur_record = {
                            "user": str(doc['_id']),
                            "serverTime": finish_time,
                            "finishState": vx["state"],
                            "topic": k, 
                            "practice": kx,
                            "type": topic_to_type[k]
                        }
                        write_queue.append(cur_record)
                    if ('state' in v.keys()) and (finish_time < DATE_DEFAULT_HOLD) and ((upgrade_time == None) or (upgrade_time < DATE_DEFAULT_HOLD)):
                        cur_record = {
                            "user": str(doc['_id']),
                            "serverTime": DATE_INIT,
                            "finishState": vx["state"],
                            "topic": k, 
                            "practice": kx,
                            "type": topic_to_type[k]
                        }
                        write_queue.append(cur_record)
                    elif 'state' in vx.keys():
                        cur_record = {
                            "user": str(doc['_id']),
                            "serverTime": finish_time,
                            "finishState": "imperfect",
                            "topic": k, 
                            "practice": kx,
                            "type": topic_to_type[k]
                        }
                        write_queue.append(cur_record)
                        cur_record = {
                            "user": str(doc['_id']),
                            "serverTime": upgrade_time,
                            "finishState": vx["state"],
                            "topic": k, 
                            "practice": kx,
                            "type": topic_to_type[k]
                        }
                        write_queue.append(cur_record)
    print len(write_queue)
    if len(write_queue) >= 100:
        finish_state.insert_many(write_queue)
        write_queue = []