tsData.py

__author__ = 'ronfe'

import pymongo
from pymongo import MongoClient
from time import strftime
import re
from tabulate import tabulate

output = []

# Calculate UV
def calcUV(eventName):
    pipeLine = [
        {"$match": {"eventKey": eventName}},
        {"$group": {"_id": "$user"}}
    ]
    return len(list(points.aggregate(pipeLine)))

# Generate Logged in User id
def generatgeLoggedInUser():
    pipeLine = [
        {"$match": {"eventKey": "enterTchrSharing", "eventValue.isLoggedIn": "1"}},
        {"$group": {"_id": "$user"}}
    ]
    return list(points.aggregate(pipeLine))


# Calculate User dist
def calcUserDist(matchPipe,  loggedInList, loggedOutList, noAccountList, eventName):
    # result = [PV, UV, Loggedin, notLoggedIn, noUser]
    result = []
    pipeLine = []
    tempPipe = []
    pipeLine.append(matchPipe)
    tempPipe = list(pipeLine)

    # calc PV
    step = list(points.aggregate(tempPipe))
    result.append(len(step))

    # calc UV
    tempPipe.append({"$group": {"_id": "$user"}})
    step = list(points.aggregate(tempPipe))
    result.append(len(step))
    tempPipe = list(pipeLine)

    # calc logged in
    tempPipe.append({"$match": {"user": {"$in": loggedInList}}})
    tempPipe.append({"$group": {"_id": "$user"}})
    step = list(points.aggregate(tempPipe))
    result.append(len(step))
    tempPipe = list(pipeLine)

    # calc not logged in
    tempPipe.append({"$match": {"user": {"$in": loggedOutList}}})
    tempPipe.append({"$group": {"_id": "$user"}})
    step = list(points.aggregate(tempPipe))
    result.append(len(step))
    tempPipe = list(pipeLine)

    # calc no account
    tempPipe.append({"$match": {"user": {"$in": noAccountList}}})
    tempPipe.append({"$group": {"_id": "$user"}})
    step = list(points.aggregate(tempPipe))
    result.append(len(step))

    # return
    return [eventName] + result

# eventKey
eventKeys = {
    "enter": "enterTchrSharing",
    "play": "playVideo",
    "clickVideo": "clickVideoZone",
    "pause": "pauseVideo",
    "switch": "clickVideoTab",
    "share": "pushShareBtn",
    "navSignup": "clickNavSignup",
    "modalSignup": "clickModalSignup",
    "footerSignup": "clickFooterSignup",
    "login": "clickNavLogin"
}

# Step 0, Connect database
host = 'mongodb://10.8.8.8:27017'
dbName = 'matrix-yangcong-prod25'
colName = 'tsPoint'

client = MongoClient(host)
db = client[dbName]
points = db[colName]

# Step 1, Give title/time/author
output.append('Teacher Sharing Data Report')
output.append(strftime("%Y-%m-%d %H:%M:%S"))
output.append('by ronfe')
output.append('Total records: ' + str(points.find({}).count()))
output.append('')

# Step 2, General data
output.append('Section A: Key Data')
output.append('===================')

# #2.1 enter page
enterPV = points.find({"eventKey": eventKeys["enter"]}).count()
enterUV = calcUV(eventKeys["enter"])
output.append('Enter page: ' + str(enterPV) + '/' + str(enterUV) + ' (PV/UV, hereafter)')

# #2.2 play video
playPV = points.find({"eventKey": eventKeys["play"]}).count()
playUV = calcUV(eventKeys["play"])
output.append('Play video: ' + str(playPV) + '/' + str(playUV))

# #2.3 share
sharePV = points.find({"eventKey": eventKeys["share"]}).count()
shareUV = calcUV(eventKeys["share"])
output.append('Share page: ' + str(sharePV) + '/' + str(shareUV))

# #2.4 signup
navbarSignupPV = points.find({"eventKey": eventKeys["navSignup"]}).count()
modalSignupPV = points.find({"eventKey": eventKeys["modalSignup"]}).count()
footerSignupPV = points.find({"eventKey": eventKeys["footerSignup"]}).count()
navbarSignupUV = calcUV(eventKeys["navSignup"])
modalSignupUV = calcUV(eventKeys["modalSignup"])
footerSignupUV = calcUV(eventKeys["footerSignup"])
output.append('Click Signup: ' + str(navbarSignupPV + modalSignupPV + footerSignupPV) + '/' + str(navbarSignupUV + modalSignupUV + footerSignupUV))

output.append('')

# STEP 3 detailed page
allUsers = []
temp = list(points.aggregate([
    {"$group": {"_id": "$user"}}
]))
for each in temp:
    allUsers.append(each[u'_id'])

temp = generatgeLoggedInUser()
loggedInUsers = []
for each in temp:
    loggedInUsers.append(each[u'_id'])

loggedOutUsers = list(set(allUsers) - set(loggedInUsers))
noAccountUsers = []
for each in loggedOutUsers:
    m = re.match(r'ffffff', str(each))
    if m:
        noAccountUsers.append(each)



bigList = []
# enter page
matchPipe ={"$match": {"eventKey": eventKeys["enter"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "Enter Page"))

# play video
matchPipe ={"$match": {"eventKey": eventKeys["play"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "Play Video"))

# play video 1
matchPipe ={"$match": {"eventKey": eventKeys["play"], "eventValue.videoIndex": "1"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Play V1"))

# play video 2
matchPipe ={"$match": {"eventKey": eventKeys["play"], "eventValue.videoIndex": "2"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Play V2"))

# play video 3
matchPipe ={"$match": {"eventKey": eventKeys["play"], "eventValue.videoIndex": "3"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Play V3"))

# click video
matchPipe ={"$match": {"eventKey": eventKeys["clickVideo"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "Click Video Zone"))

# play video 1
matchPipe ={"$match": {"eventKey": eventKeys["clickVideo"], "eventValue.videoIndex": "1"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Click V1"))

# play video 2
matchPipe ={"$match": {"eventKey": eventKeys["clickVideo"], "eventValue.videoIndex": "2"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Click V2"))

# play video 3
matchPipe ={"$match": {"eventKey": eventKeys["clickVideo"], "eventValue.videoIndex": "3"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Click V3"))

# pause video
matchPipe ={"$match": {"eventKey": eventKeys["pause"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "Pause Video"))

# pause video 1
matchPipe ={"$match": {"eventKey": eventKeys["pause"], "eventValue.videoIndex": "1"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Pause V1"))

# pause video 2
matchPipe ={"$match": {"eventKey": eventKeys["pause"], "eventValue.videoIndex": "2"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Pause V2"))

# pause video 3
matchPipe ={"$match": {"eventKey": eventKeys["pause"], "eventValue.videoIndex": "3"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Pause V3"))

# switch video
matchPipe ={"$match": {"eventKey": eventKeys["switch"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "Switch Video"))

# switch video 1
matchPipe ={"$match": {"eventKey": eventKeys["switch"], "eventValue.videoIndex": "1"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Switch V1"))

# switch video 2
matchPipe ={"$match": {"eventKey": eventKeys["switch"], "eventValue.videoIndex": "2"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Switch V2"))

# switch video 3
matchPipe ={"$match": {"eventKey": eventKeys["switch"], "eventValue.videoIndex": "3"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Switch V3"))

# share page
matchPipe ={"$match": {"eventKey": eventKeys["share"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "Share"))

# share video 1 QQ
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "1", "eventValue.sharePltform": "qq"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V1 QQ"))
# share video 1 QZone
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "1", "eventValue.sharePltform": "qzone"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V1 QZone"))
# share video 1 Weibo
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "1", "eventValue.sharePltform": "weibo"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V1 Weibo"))
# share video 2 QQ
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "2", "eventValue.sharePltform": "qq"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V2 QQ"))
# share video 2 QZone
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "2", "eventValue.sharePltform": "qzone"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V2 QZone"))
# share video 2 Weibo
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "2", "eventValue.sharePltform": "weibo"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V2 Weibo"))
# share video 3 QQ
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "3", "eventValue.sharePltform": "qq"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V3 QQ"))
# share video 3 QZone
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "3", "eventValue.sharePltform": "qzone"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V3 QZone"))
# share video 3 Weibo
matchPipe ={"$match": {"eventKey": eventKeys["share"], "eventValue.videoIndex": "3", "eventValue.sharePltform": "weibo"}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Share V3 Weibo"))


bigList.append(['Click Signup', '', '', '', '', ''])
# click nav signup
matchPipe ={"$match": {"eventKey": eventKeys["navSignup"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Navbar Signup"))
# click modal signup
matchPipe ={"$match": {"eventKey": eventKeys["modalSignup"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Modal Signup"))
# click footer signup
matchPipe ={"$match": {"eventKey": eventKeys["footerSignup"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "-- Footer Signup"))

# login
matchPipe ={"$match": {"eventKey": eventKeys["login"]}}
bigList.append(calcUserDist(matchPipe, loggedInUsers, loggedOutUsers, noAccountUsers, "Click Login"))

# Step X, output to file
for each in output:
    print each

print ''
print 'Section B: Detailed Data'
print '========================'
print tabulate(bigList, headers=['Event', 'PV', 'UV', 'Logged', 'Not Logged', 'No Account'], tablefmt='orgtbl')

print ''
print '-- END --'