gistfile1.py

"""Gets list of panel members referenced in interaction json documents in
specified date range, parse and add to redis.

Usage:
    get_panel_members.py START_DATE END_DATE

Arguments:
    START_DATE     ISO format: YYYY-MM-DD
    END_DATE       ISO format: YYYY-MM-DD

Options:
  -h --help     Show this screen.

"""
from datetime import timedelta
from itertools import izip_longest

import dateutil.parser as dateparser
from docopt import docopt

from tasks import process_keys_for_date


def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return izip_longest(fillvalue=fillvalue, *args)


def process_panel_members(start, end):

    # make sure that the end value represents the last second of that day,
    # not 00:00:00
    end += timedelta(hours=23, minutes=59, seconds=59)

    # generator of datetime objects for every second between start and end dates
    r = (start + timedelta(seconds=i)
         for i in range(int((end - start).total_seconds()) + 1))

    # Parcel work into hour chunks and queue for processing.
    # A single task will be queued for each hour, which will execute 3600 times
    #process_keys_for_date.chunks(r, 3600).apply_async()
    for chunk in grouper(r, 3600):
        process_keys_for_date.map(chunk).delay()


def main():
    args = docopt(__doc__)
    start = dateparser.parse(args['START_DATE'])
    end = dateparser.parse(args['END_DATE'])

    process_panel_members(start, end)

if __name__ == '__main__':
    main()