snippet

sample.py

#!/usr/bin/python


#### WEB
from aiocache import cached, RedisCache
from aiocache.serializers import PickleSerializer
import aiohttp


class Session():
    def __init__(self):
        self.aiohttp = aiohttp.ClientSession()


    @cached(ttl=604800, cache=RedisCache, serializer=PickleSerializer(),
            port=6379, timeout=0, noself=True)
    async def get_page(self, url):
        async with self.aiohttp.get(url) as resp:
            return await resp.text()


    def close(self):
        return self.aiohttp.close()
#### WEB


#### PARALLELISM
from concurrent import futures


class Pool():
    def __init__(self, ncpus):
        self.pool = futures.ProcessPoolExecutor(ncpus)


    def submit(self, fn, *args):
        return self.pool.submit(fn, *args)


pool   = Pool(8)
future = []


def submit_job(func, *args):
    f = pool.submit(func, *args)
    future.append(f)


def wait():
    for f in futures.as_completed(future):
        pass
#### PARALLELISM


#### MULTI
import asyncio


def start_loop():
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    webSession = Session()
    tasks = []
    URL = 'https://www.github.com/'
    for i in range(100):
        tasks.append(asyncio.ensure_future(webSession.get_page(URL + str(i)),
                                           loop=loop))
    loop.run_until_complete(asyncio.gather(*tasks, loop=loop))
    loop.run_until_complete(webSession.close())
    return


def get_infos():
    for i in range(8):
        submit_job(start_loop)
    wait()
#### MULTI


import logging

if __name__ == '__main__':
    logging.basicConfig(filename='mylog.log',
                        filemode = 'w',
                        level=logging.DEBUG,
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
    loop = asyncio.get_event_loop()
    session = Session()
    tasks = [asyncio.ensure_future(session.get_page('https://www.github.com'))]
    loop.run_until_complete(asyncio.gather(*tasks))
    get_infos()
    loop.run_until_complete(session.close())