config.csv

/signup$,POST,,signupSuccess,,channel,from,,_id,type
/signup/qq,POST,,signupSuccess,,channel,from,,_id,type
/login$,POST,,loginSuccess,,,,_id,type
/login/qq,POST,,loginSuccess,,,,_id,type
/bind/qq,PUT,,bindQQSuccess,,,,_id
/unbind/qq,PUT,,unbindQQSuccess,,,,
/logout,POST,,logoutSuccess,,,,
/user/password$,PUT,,modifyPassword,,,,_id
/me$,PUT,,updateProfile,,**,,
/me$,GET,,getProfile,,,,_id,allowed,partialOpen
/me/rooms,PUT,,addRoom,,user,,
/me/verify/phone$,POST,,bindPhoneSuccess,,user,phone,,
/me/avatars/purchase$,PUT,,buyAvatarSuccess,,user,avatarId,,
/me/password$,PUT,,changePasswordSuccess,,user,,
/me/progress/\S+/entered$,PUT,,startTopic,,user,,
/me/progress/\S+/practice/,PUT,,finishLevel,,**,,**
/me/progress/\S+/hyperVideo/,PUT,,finishHyperVideo,,**,,**
/questionnaire$,POST,,submitPollSuccess,,user,tag,body,,
/teacher/rooms$,POST,,createRoomSuccess,,user,,name
/teacher/rooms,DELETE,,deleteRoomSuccess,,id,,
/teacher/rooms/\S+/students/batch$,POST,,batchAddStudentsSuccess,,**,,
/teacher/rooms/\S+/students/password$,PUT,,updateStudentPassword,,students,,

service.py

# _*_ coding:utf-8 _*_

import re
from pymongo import MongoClient
from bson.objectid import ObjectId 
import jwt

topics_db = MongoClient('10.8.8.8')['onions35']
topics_collection = topics_db['topics']
pay_topics = topics_collection.distinct('_id', {"pay": True})

# Functions for xiaomi's requirement
def generate_record(doc, jwt_info):
    req_url = doc['url'].split('/')
    topic_id = req_url[req_url.index('topics')+1]

    if not topic_id in pay_topics:
        return None

    meta_result = {
        "userId": jwt_info['userId'],
        "topicId": ObjectId(topic_id)
        "time": doc['serverTime']
    }

    if req_url[-1] == 'entered':
        meta_result['from'] = "unentered"
        meta_result['to'] = "unfinished"
    else:
        meta_result['from'] = "unfinished"
        meta_result['to'] = doc['response']['topicState']['state']
        if meta_result['to'] == 'unfinished':
            meta_result['to'] = 'unchanged'

    if meta_result['to'] == 'unchanged':
        return None
    else:
        return meta_result


def convert_state(coll):
    pipeline = [
        {"$match": {"to": "imperfect"}},
        {"$group": {"_id": "$topicId", "users": {"$addToSet": "$userId"}}}
    ]
    x = coll.aggregate(pipeline)
    for each in list(x):
        coll.update_many(
            {"topicId": each['_id'], "userId": {"$in": each["users"]}, "to": "perfect"},
            {"$set": {"from": "imperfect"}}
        )
# block end


# deocde token
# @ input token
# @ output userid and user role
def parseToken(token):
    secret = "follow your heart&intuition"
    thisToken = token.split(' ')[-1]
    decode = {}
    decode = jwt.decode(thisToken, secret)
    return decode

def merge_two_dicts(x, y):
    '''Given two dicts, merge them into a new dict as a shallow copy.'''
    z = x.copy()
    z.update(y)
    return z

def generate_config(file_path):

    # result = {
    #     "req": [
    #         {"url": re.compile("signup"), "m": "POST"}
    #     ],
    #     "res": [
    #         {
    #             "eK": "signupSuccess", 
    #             "req": ["channel", "from"],
    #             "res": []
    #         }
    #     ]
    # }

    f = open(file_path).readlines()
    result = {"req": [], "res": []}

    for each in f:
        txt = each.strip()
        txt_array = txt.split(',,')

        unit_req = {}
        unit_res = {}
        unit_req['url'] = re.compile(txt_array[0].split(',')[0])
        unit_req['m'] = txt_array[0].split(',')[1]

        unit_res['eK'] = txt_array[1].split(',')[0]
        unit_res['req'] = txt_array[2].split(',')
        unit_res['res'] = txt_array[3].split(',')

        result['req'].append(unit_req)
        result['res'].append(unit_res)

    return result

def get_meta_data(doc_list, res_info):
    result = []
    for doc in doc_list:
        event_obj = {
            "eventKey": "",
            "eventValue": {},
            "ua": "",
            "url": "",
            "category": "",
            "platform": "",
            "ip": "",
            "location": "",
            "user": "",
            "role": "",
            "eventTime": "",
            "serverTime": ""
        }

        event_obj['eventKey'] = res_info['eK']
        event_obj['ua'] = doc['ua']                  # httplog ua filed
        event_obj['url'] = doc['url']                # httplog url filed
        event_obj['category'] = doc['apptag']        # 没法找？ parse url有可能拿到
        event_obj['platform'] = "backend"            # 一律backend还是根据不同的apptag传不同的platform
        if doc['ip'] != None:
            event_obj['ip'] = doc['ip']                  # read from httplog this.request.ip
        if doc['token'] != None and doc['token'] != "":
            token_obj = parseToken(doc['token'])
            event_obj['user'] = ObjectId(token_obj['id'])  # read from httplog
            event_obj['role'] = token_obj['role']          # read from httplog
        event_obj['eventTime'] = doc['eventTime']                   # http2mongo实时写入unixtimestamp
        event_obj['serverTime'] = doc['serverTime']                 # read from token

        event_value = {}
        if res_info['req'][0] != '':
            event_value_req = dict((k, doc['request'][k]) for k in res_info['req'])
        else:
            event_value_req = {}

        if res_info['res'][0] != '':
            event_value_res = dict((k, doc['response'][k]) for k in res_info['res'])
        else:
            event_value_res = {}

        event_value = merge_two_dicts(event_value_req, event_value_res)
        event_obj['eventValue'] = event_value   # 根据不同的情况解析req/res填充eventValue/ read from httplog res/req

        result.append(event_obj)

        state_chage_url = ['']
        if 'progress' in doc['url']:
            xiaomi = generate_record(doc, token_obj)
            if xiaomi:
                # insert xiaomi collection

    return result

def convert(file_path, http_log):
    config = generate_config(file_path)
    requests = config['req']
    responses = config['res']

    for i in range(len(requests)):
        # STEP 1: get all the http doc
        condition = {
            "url": {"$regex": unit_info['url']},
            "method": unit_info['m'],
            "status": "200"
        }
        unit_info = requests[i]
        x = list(http_log.find(condition))

        result = get_meta_data(x, responses[i])
        # xxx.insert_many(result)
        http_log.delete_many(condition)

xiaomi.py

from pymongo import MongoClient
from bson.objectid import ObjectId 


topics_db = MongoClient('10.8.8.8')['onions35']
topics_collection = topics_db['topics']
pay_topics = topics_collection.distinct('_id', {"pay": True})

def generate_record(doc, jwt_info):
    req_url = doc['url'].split('/')
    topic_id = req_url[req_url.index('topics')+1]

    if not topic_id in pay_topics:
        return None

    meta_result = {
        "userId": jwt_info['userId'],
        "topicId": ObjectId(topic_id)
        "time": doc['serverTime']
    }

    if req_url[-1] == 'entered':
        meta_result['from'] = "unentered"
        meta_result['to'] = "unfinished"
    else:
        meta_result['from'] = "unfinished"
        meta_result['to'] = doc['response']['topicState']['state']
        if meta_result['to'] == 'unfinished':
            meta_result['to'] = 'unchanged'

    if meta_result['to'] == 'unchanged':
        return None
    else:
        return meta_result


def convert_state(coll):
    pipeline = [
        {"$match": {"to": "imperfect"}},
        {"$group": {"_id": "$topicId", "users": {"$addToSet": "$userId"}}}
    ]
    x = coll.aggregate(pipeline)
    for each in list(x):
        coll.update_many(
            {"topicId": each['_id'], "userId": {"$in": each["users"]}, "to": "perfect"},
            {"$set": {"from": "imperfect"}}
        )