cache_debug.py

from pymongo import MongoClient, DESCENDING
import datetime
import threading

START_DATE = datetime.datetime(2015,7,20)
END_DATE = datetime.datetime(2015, 7,31)
NUM_OF_WORKERS = 10

db = MongoClient('10.8.8.111:27017')['yangcong-prod25']
user_attr = db['userAttr']
points = db['points']
users = db['users']

total_points = points.distinct('user', filter={"createdBy": {"$gte": START_DATE, "$lt": END_DATE}})
points_count = len(total_points)

# # Task assigner
def assign_tasks(worker_num):
   global total_points
   if len(total_points) > 0:
      result = total_points.pop(len(total_points) / (worker_num + 2))
      return result
   else:
      print 'done'
      return 'done'

def worker(worker_num):
   finish_marker = False
   while not finish_marker:
      myWork = assign_tasks(worker_num)
      if myWork == 'done':
         finish_marker = True
      else:
         client_id = myWork
         myWork = points.find({"user": client_id, "createdBy": {"$gte": START_DATE, "$lt": END_DATE}})
         first_work = myWork[0]
         last_work = myWork[myWork.count() - 1]
         print "[Worker " + str(worker_num) + "] Start " + str(first_work['createdBy'].day) + ":" + str(first_work['createdBy'].hour)
         client_result = {
            "userId": client_id,
            "isRegistered": False,
            "isActivated": True,
            "os": {
               "pc": False,
               "android": False,
               "ios": False
            }
         }
         # Step 1 trying to find it in userattrs
         is_user_attr = user_attr.find_one({"userId": client_id})
         # if found, update prior os info
         if is_user_attr != None:
            client_result['os'] = is_user_attr['os']
            client_result["recentSession"] = max(is_user_attr['recentSession'], last_work['createdBy'])
            client_result
         else:
            client_result["recentSession"] = last_work['createdBy']
            client_result['activatedTime'] = first_work['createdBy']

         client_result['os'][last_work['from']] = True

         # Step 2 trying to find it in users
         is_our_user = users.find_one({"_id": client_id})
         if is_our_user != None:
            client_result['isRegistered'] = True
            if is_user_attr == None or (is_user_attr != None and is_user_attr['isRegistered'] == False):
               client_result['activatedTime'] = first_work['createdBy']
               # client_result["recentSession"] = first_work['createdBy']
            else:
               # Returning users
               client_result['activatedTime'] = min(is_user_attr['activatedTime'], first_work['createdBy'])



            # role/q/schoolid
            client_result['role'] = is_our_user['role']
            if 'usefulData' in is_our_user.keys() and 'q' in is_our_user['usefulData'].keys():
               client_result['q'] = is_our_user['usefulData']['q']
            if 'school' in is_our_user.keys():
               client_result['schoolId'] = is_our_user['school']

         # upsert user attr
         user_attr.find_one_and_replace({"userId": client_id}, client_result, upsert = True)

threads = []
for i in range(NUM_OF_WORKERS):
   t = threading.Thread(target=worker, args=[i, ])
   threads.append(t)
   t.start()