Synchronization of the local MongoDB with Vimeo Channel

vimeosync.py

import sys
import logging
from math import ceil
from random import random
from datetime import datetime

from pymongo import ASCENDING
from pymongo.mongo_client import MongoClient
from pymongo.errors import OperationFailure

# 'vimeo' package was downloaded from
# https://github.com/dkm/python-vimeo
from vimeo import VimeoClient as VimeoAPI


# Vimeo authorization tokens are present on the page
# https://developer.vimeo.com/apps/<your app id, ex. 26810>
VIMEO_CLIENT_ID = 'YOUR_VIMEO_CLIENT_ID'
VIMEO_CLIENT_SECRET = 'YOUR_VIMEO_CLIENT_SECRET'
VIMEO_ACCESS_TOKEN = 'YOUR_VIMEO_ACCESS_TOKEN'
VIMEO_TOKEN_SECRET = 'YOUR_VIMEO_TOKEN_SECRET'

# per_page Vimeo API param: number of items to show on each page. Max 50.
VIMEO_CHANNEL_FEED_PAGESIZE = 50
VIMEO_CHANNEL = 'mychannel'
VIMEO_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'

DB_CONNECTION_URI = 'mongodb://localhost:27017'
DB_NAME = 'mydatabase'
DB_COLLECTION = 'videos'
DB_RANDOM_CHOICE_INDEX_NAME = 'random_choice'


logger = logging.getLogger(__name__)


class VimeoService(object):

    def __init__(self, vimeo_client):
        self.client = vimeo_client

    def channel_feed(self, channel_name):
        page = 1
        pagesize = VIMEO_CHANNEL_FEED_PAGESIZE
        params = dict(channel_id=channel_name, page=page, per_page=pagesize)
        bunch = self.client.channels_getVideos(**params)
        for video in bunch['video']:
            yield video
        total = float(bunch['total'])
        if total <= pagesize:
            return
        total_pages = int(ceil(total/pagesize))
        for page in range(2, total_pages+1):
            params['page'] = page
            bunch = self.client.channels_getVideos(**params)
            for video in bunch['video']:
                yield video

    def video_info(self, video_id):
        return self.client.videos_getInfo(video_id=video_id)[0]


class VideoDatabase(object):

    def __init__(self, collection):
        self.videos = collection
        self.logcounter = 0

    def turn_off_random_choice_index(self):
        try:
            self.videos.drop_index(DB_RANDOM_CHOICE_INDEX_NAME)
            logger.info("The index '{0}' was dropped."
                .format(DB_RANDOM_CHOICE_INDEX_NAME))
        except OperationFailure:
            logger.info("The index '{0}' does not exists."
                .format(DB_RANDOM_CHOICE_INDEX_NAME))
        
    def turn_on_random_choice_index(self):
        idx = [('active', ASCENDING), ('random', ASCENDING)]
        self.videos.ensure_index(idx, name=DB_RANDOM_CHOICE_INDEX_NAME)
        logger.info("The index '{0}' was created."
            .format(DB_RANDOM_CHOICE_INDEX_NAME))

    def ensure_vimeo_id_trigger(self):
        idx = 'vimeoid'
        name = self.videos.ensure_index(idx, unique=True)
        if name:
            logger.info("The '{0}' index was created.".format(name))
        else:
            logger.info("The index for 'vimeoid' field already exists.")

    def deactivate_all_videos(self):
        spec = {'active': True}
        document = {'$set': {'active': False}}
        res = self.videos.update(spec, document, multi=True)
        logger.info("{0} videos were deactivated.".format(res['n']))

    def activate_video(self, video):
        spec = {'_id': video['_id']}
        document = {'$set': {'active': True}}
        res = self.videos.update(spec, document)
        self.logcounter += 1
        logger.info("[{0}] The video '{1}' was activated without updating."
            .format(self.logcounter, video['title'].encode('utf-8')))
    
    def upsert_video(self, video):
        isnewstr1 = 'existing' if '_id' in video else 'new'
        isnewstr2 = 'updated' if '_id' in video else 'inserted'
        self.videos.save(video)
        self.logcounter += 1
        logger.info("[{0}] The {1} video '{2}' was {3} and activated."
            .format(self.logcounter, isnewstr1, 
                    video['title'].encode('utf-8'), isnewstr2))

    def get_video(self, vimeo_id):
        return self.videos.find_one({'vimeoid': vimeo_id})


class VideoSync(object):
    
    def __init__(self, database, vimeo):
        self.database = database
        self.vimeo = vimeo

    def run_sync(self):

        def check_for_modification(video, vimeo_record):
            datestring = vimeo_record['modified_date']
            modified = datetime.strptime(datestring, VIMEO_TIME_FORMAT)
            if 'modified' not in video:
                video['modified'] = datetime.utcfromtimestamp(0)
            return modified > video['modified']

        def update_video(video, vimeo_record):
            video['vimeoid'] = int(vimeo_record['id'])
            video['duration'] = int(vimeo_record['duration'])
            video['title'] = vimeo_record['title']
            video['description'] = vimeo_record['description']
            video['random'] = random()
            video['active'] = True
            datestring = vimeo_record['modified_date']
            modified = datetime.strptime(datestring, VIMEO_TIME_FORMAT)
            video['modified'] = modified

        self.database.turn_off_random_choice_index()
        self.database.ensure_vimeo_id_trigger()
        self.database.deactivate_all_videos()
        feed = self.vimeo.channel_feed(VIMEO_CHANNEL)
        for record in feed:
            vimeo_id = int(record['id'])
            video = self.database.get_video(vimeo_id)
            if video:
                modified = check_for_modification(video, record)
                if not modified:
                    self.database.activate_video(video)
                    continue
            else:
                video = {}
            # Sadly but this is only way to get 
            # duration and description attributes
            record = self.vimeo.video_info(vimeo_id)
            update_video(video, record)
            self.database.upsert_video(video)
        self.database.turn_on_random_choice_index()


def get_mongodb_connection():
    return MongoClient(DB_CONNECTION_URI)[DB_NAME]


def logger_config():
    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler(sys.stdout)
    # ch.setLevel(logging.ERROR)
    logger.addHandler(ch)


def main():
    logger_config()
    vimeo_client_params = dict(
        key=VIMEO_CLIENT_ID,
        secret=VIMEO_CLIENT_SECRET,
        token=VIMEO_ACCESS_TOKEN,
        token_secret=VIMEO_TOKEN_SECRET,
        format='json')
    vimeo_api = VimeoAPI(**vimeo_client_params)
    vimeo = VimeoService(vimeo_api)
    connection = get_mongodb_connection()
    collection = connection[DB_COLLECTION]
    database = VideoDatabase(collection)
    sync = VideoSync(database, vimeo)
    sync.run_sync()


if __name__ == '__main__':
    sys.exit(main())