用户地址位置信息脚本

userid_address_cache.py

# _*_ coding:utf-8 _*_

"""
    s0. dump fully event collection by getProfile 得出所有登录过的注册用户的埋点
    s1. run this programm
"""

from pymongo import MongoClient
from bson.objectid import ObjectId

print("[REMEMBER] Dump fully event collection by getProfile")

"""
    DB connection configure
"""
event_db_instance = MongoClient('10.8.8.111', 27017)['eventsV35']

# the getProfile dump db
event_collection = event_db_instance['userLocationCacheEvent']
location_collection = event_db_instance['userLocation']

"""
    count user id that have been count
"""
# user_id_counted_location = []
# for user in location_collection.find({}):
#     user_id_counted_location.append(user['_id'])
# print("%d users was recorded location")%(len(user_id_counted_location))

"""
    mongo aggregate pipeline
    use API GET /me eventKey:getProfile
    Returns:
        _id         : user ObjectId
        role        : user role enum['teacher','student','visitor','editor']
        channel     : user came from which channel
        location    : user location
    Raises:
        empty list  : active_user_id_list == []
"""
pipeline = [
    {"$match": {
        "eventKey": "getProfile"
    }},
    {"$group": {
        "_id": "$user",
        "role": {"$last":"$role"},
        "channel": {"$last":"$channel"},
        "location": {"$first": "$location"}
    }}
]
active_user_id_list = list(event_collection.aggregate(pipeline))

"""
    save to db
"""
location_collection.delete_many({"_id": {"$exists": True}})
location_collection.insert_many(active_user_id_list)
print("%s users cached with filed location/role/channel")%len(active_user_id_list)

#!/usr/bin/env bash

LOGFILE=/tmp/cache_user_location_dump.log

echo "`date`" >> $LOGFILE && \
echo "************************************************" >> $LOGFILE && \
echo "*     dump user location with other fileds     *" >> $LOGFILE && \
echo "************************************************" >> $LOGFILE && \

echo "1. start to dump" >> $LOGFILE && \
cd /home/master/yangcongDatabase/v3_5collections/service
echo "{'eventKey':'getProfile'}" >> ./getProfile.json
mongodump --db eventsV35 --collection eventV35 --queryFile ./getProfile.json
echo "2. end to dump" >> $LOGFILE && \

echo "3. start to restore" >> $LOGFILE && \
mongorestore --drop --db eventsV35 --collection userLocationCacheEvent ./dump/eventsV35/*.bson
echo "4. end to restore" >> $LOGFILE && \

echo "5. start to cache" >> $LOGFILE && \
python ./cache_user_location.py >> $LOGFILE && \
echo "6. end to cache" >> $LOGFILE && \

echo "code done" >> $LOGFILE