transfer.py

# _*_ coding:utf-8 _*_

from pymongo import MongoClient
from bson.objectid import ObjectId
import subprocess
import jwt

'''
This script aims to generate finishPractice log from daily
dumped httplogs 
'''

# deocde token
# @ input token
# @ output userid and user role
def parseToken(token):
    secret = "follow your heart&intuition"
    thisToken = token.split(' ')[-1]
    decode = {}
    try:
        decode = jwt.decode(thisToken, secret, verify=False)
        return decode
    except Exception as e:
        return None

# db = MongoClient('10.8.8.111:27017')['eventsV35testuse']

# STEP 1 query rehttplogs
# TODO change the bson directory to real bson

# restore_cmd = 'mongorestore --db ronfedb --collection tempLog ./httplogs.bson'
# subprocess.call(restore_cmd)

# STEP 2 connect to db and get all the practice final level id
logs = MongoClient('10.8.8.111:27017')['ronfedb']['httplogs']
topics = MongoClient('10.8.8.111:27017')['onions35']['topics']
finish_state = MongoClient('10.8.8.111:27017')['ronfedb']['finishState']

x = topics.find()
final_level_ids = []
topic_type_dict = {}
for doc in x:
    if 'modules' in doc.keys() and len(doc['modules']) > 0:
        for module in doc['modules']:
            if 'practice' in module.keys():
                if 'levels' in module['practice'].keys() and len(module['practice']['levels']) > 0:
                    # remove hook level
                    if module['practice']['levels'][-1]['pool'] != 'hanger':
                        final_level_ids.append(module['practice']['levels'][-1]['_id'])
                    else:
                        final_level_ids.append(module['practice']['levels'][-2]['_id'])

                    # add topic type to map
                    topic_type_dict[str(doc['_id'])] = doc['type']


# STEP 3 get all the fitable logs
x = logs.find({"url": {"$regex": r'/levels/.{24}$'}, "status": "200"})
for doc in x:
    doc_level = doc['url'].split('/')
    if ObjectId(doc_level[-1]) in final_level_ids:
        if doc['request']['tryTimes'] != 0:
            
            result_doc = {
                "eventTime": doc['eventTime'],
                "serverTime": doc['serverTime'],
                "topic": doc_level[doc_level.index('topics') + 1],
                "finishState": doc['response']['topicState']['state']
            }


            # insert user id
            user_info = parseToken(doc['token'])
            result_doc['user'] = user_info['id']

            # insert topic type
            if doc_level[doc_level.index('topics') + 1] in topic_type_dict.keys():
                result_doc['type'] = topic_type_dict[doc_level[doc_level.index('topics') + 1]]

            # add to db
            finish_state.insert_one(result_doc)