wot_task.py

from __future__ import division
import requests
from urlparse import urlparse
from celery import task
from celery.contrib.batches import Batches
from celery import current_app
from six.moves import zip

wot_api_target = "https://api.mywot.com/0.4/public_link_json"


@task(base=Batches, flush_every=100, flush_interval=1)
def wot_malware_scan(requests):
    reponses = wot_malware_scan_real(
        (request.kwargs["url"] for request in requests)
    )
    for response, request in zip(reponses, requests):
        current_app.backend.mark_as_done(request.id, response)


def wot_malware_scan_real(urls):
    domains = [urlparse(url).netloc for url in urls]
    response = requests.get(
        wot_api_target,
        params={"hosts": ('/').join(set(domains)) + '/'}
    )
    return [response.json[domain] for domain in domains]


if __name__ == "__main__":
    print wot_malware_scan_real(("http://google.com", "http://yahoo.com"))