Created
July 11, 2012 01:06
-
-
Save jlyu/3087266 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from Queue import Queue | |
from threading import Thread | |
import time | |
import feedparser | |
num_fetch_threads = 2 | |
enclosure_queue = Queue() | |
feed_urls = ['http://www.castsampler.com/cast/feed/rss/guest',] | |
def downloadEnclosures(i, q): | |
while True: | |
print '%s: Looking for the next enclosure' % i | |
url = q.get() | |
print '%s: Downloading:' % i, url | |
time.sleep(i + 2) | |
q.task_done() | |
for i in range(num_fetch_threads): | |
worker = Thread(target=downloadEnclosures, args=(i, enclosure_queue,)) | |
worker.setDaemon(True) | |
worker.start() | |
for url in feed_urls: | |
response = feedparser.parse(url, agent='fetch_podcasts.py') | |
for entry in response['entries']: | |
for enclosure in entry.get('enclosures', []): | |
print 'Queuing:', enclosure['url'] | |
enclosure_queue.put(enclosure['url']) | |
print '*** Main thread waiting' | |
enclosure_queue.join() | |
print '*** Done' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment