Download Manager Example

fetch.py

import gevent
import gevent.monkey
gevent.monkey.patch_socket()

from gevent.queue import Queue
import requests
from datacat import client_from_config_file
from jinja2 import Template

"""
This example gets the first 100 datasets in the target directory, using their
theoretical SLAC.HTTP site, whose resources is a URL for the file.

It uses gevent to simultaneously download 4 at a time.
"""

name_template = Template("{{ parent.name }}/{{ ds.name }}.{{ ds.fileFormat }}")

q = Queue()

target = "/LSST/mirror/BNL3/workarea/ccdtest/e2v/113-03/flat/20140709-112014"
parent = client.path(target)
for i in client.search(target, max=100, site="SLAC.HTTP")
    q.put(i)

def worker(thread_id):
    while not q.empty():
        dataset = q.get()
        name = name_template.render(ds=dataset, parent=parent)
        response = requests.get(ds.resource, stream=True)
        if not response.status_code == 200:
            return True # Swallow for now
        with open(dataset.name, 'wb') as handle:
            for block in response.iter_content(32768):
            handle.write(block)
    return True

def start_workers():
    threads = []
    for i in range(4):
        threads.append(gevent.spawn(worker, i))
    gevent.joinall(threads)

start_workers()