Created
April 26, 2011 13:22
-
-
Save juanriaza/942247 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-8 -*- | |
from gevent import monkey | |
monkey.patch_all() | |
from gevent.pool import Group | |
import urllib2 | |
# Estos urls estan en una lista, pero podrias leerlos de un archivo o obtenerlos de cualquier lado | |
urllist = [ | |
'http://stackoverflow.com/questions/2397804/java-string-searching-ignoring-accents', | |
'http://django.es/blog/lista-de-paises-django-countries/', | |
'http://www.ip2location.com/flagsoftheworld.aspx', | |
] | |
workers_group = Group() | |
def save_callback(param): | |
print "save_callback ->", "Aqui se guardaria lo que haga falta:", type(param) | |
def worker(url): | |
data = urllib2.urlopen(url) | |
print "Worker:", url | |
workers_group.spawn(save_callback, data) | |
for url in urllist: | |
workers_group.spawn(worker, url) | |
workers_group.join() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment