This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import asyncio | |
import aio_pika | |
async def read(reader_id): | |
connection = await aio_pika.connect_robust(**RMQ_CONFIG) | |
async with connection: | |
channel = await connection.channel() | |
queue = await channel.declare_queue('test_queue', durable=True) | |
message = await queue.get() | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import asyncio | |
import aio_pika | |
async def read(reader_id): | |
connection = await aio_pika.connect_robust(**RMQ_CONFIG) | |
async with connection: | |
channel = await connection.channel() | |
queue = await channel.declare_queue('test_queue', durable=True) | |
message = await queue.get() | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf8 -*- | |
from gevent import monkey | |
from app.pack import Pack | |
monkey.patch_socket() | |
monkey.patch_ssl() | |
monkey.patch_thread() | |
from app import redis | |
from app.page import Page |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf8 -*- | |
from datetime import datetime | |
from libs.common.Date import Date | |
import logging | |
import time | |
from app import redis | |
from app.redis_keys import SITE_PACK_PREFIX, URL_QUEUE, SITE_QUEUE_LEN_PREFIX, \ | |
SITE_LAST_DOWNLOAD_PREFIX, PACK_META_PREFIX | |
from app.site import Site, NoPacks |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf8 -*- | |
from datetime import datetime, timedelta | |
import json | |
from libs.common.Date import Date | |
from libs.common.os_tools import create_file, remove_file, create_gzip, \ | |
read_gzip | |
from libs.common.string import md5, encode_from_page_charset, hash_string | |
import logging | |
import os |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf8 -*- | |
from datetime import datetime | |
from libs.common.Date import Date | |
import logging | |
from app import redis | |
from app.pack import Pack | |
from app.page import Page | |
from app.tasks.parse import parse_content_wrapper | |
import config |