Kiollpt

sqs= boto3.resource('sqs', aws_access_key_id = AWS_ACCESS_KEY_ID,
                           aws_secret_access_key = AWS_SECRET_ACCESS_KEY,
                           region_name = REGION_NAME)


def receive_message_from_sqs(spider,delete=True):
    queue = sqs.get_queue_by_name(QueueName=spider)
    rows = []

Kiollpt

[uwsgi]
module = application:app
master = true
processes = 3


chdir = YOUR_PROJECT_PATH
socket = YOUR_SOCK_FILE_PATH
chmod-socket = 777
vacuum = true

Kiollpt

print('I am hungry Friday and Saturday is beautiful day')
 

Kiollpt

from aiohttp import gather, create_task
from string import Template
from aiohttp import web, ClientSession
from bs4 import BeautifulSoup

async def news(request):
    sites = [
        ('http://edition.cnn.com',cnn_articles),
        ('http://www.aljazeera.com',aljazeera_articles)
    ]

Kiollpt

reference

1. https://juejin.im/post/6844903632534503437

2. https://www.educative.io/blog/python-concurrency-making-sense-of-asyncio

3. https://realpython.com/async-io-python/#async-io-explained

4. A good point why uses third party of asyncio packages

Kiollpt

AWS API move to local server

9 month data need 1. t3a.large 2 Variable 8 GiB EBS Only $0.0752 per Hour 54/month t3a.xlarge 4 Variable 16 GiB EBS Only $0.1504 per Hour 108/month

VPN tunnel in prod 2. $37/month

Kiollpt

c = (
    "\033[0m",   # End of color
    "\033[36m",  # Cyan
    "\033[91m",  # Red
    "\033[32m",  # Green
) 

Kiollpt

from threading import Thread
from threading import current_thread
from asyncio import Future
import asyncio
import time

c = (
    "\033[0m",   # End of color
    "\033[36m",  # Cyan
    "\033[91m",  # Red

Kiollpt

Indexs:

Secondary Index: not prmiary key Unique Index: the performace is almost same compared to (Secondary) index

Kiollpt

import asyncio
import itertools as it
import os
import random
import time

async def makeitem(size: int = 5) -> str:
    return os.urandom(size).hex()

async def randsleep(a: int = 1, b: int = 5, caller=None) -> None: