Created
April 2, 2015 12:21
-
-
Save maizy/663f9f93b591618a2206 to your computer and use it in GitHub Desktop.
curl async client problems
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# coding=utf-8 | |
import httplib | |
import importlib | |
import logging | |
import re | |
import time | |
from lxml import etree | |
import tornado.autoreload | |
import tornado.web | |
import tornado.ioloop | |
import tornado.curl_httpclient | |
from tornado.options import options | |
from frontik import frontik_logging | |
from frontik.globals import global_stats | |
import frontik.producers.json_producer | |
import frontik.producers.xml_producer | |
from frontik.handler import ErrorHandler | |
import frontik.sentry | |
from frontik.util import make_get_request | |
# ... | |
class StatusHandler(tornado.web.RequestHandler): | |
@tornado.web.asynchronous | |
def get(self): | |
if self.get_argument('no_network_check', 'false') == 'true': | |
self.set_header('Content-Type', 'application/json; charset=UTF-8') | |
result = { | |
'pages served': global_stats.page_count, | |
'http requests made': global_stats.http_reqs_count, | |
'bytes from http requests': global_stats.http_reqs_size_sum, | |
} | |
cur_uptime = time.time() - global_stats.start_time | |
if cur_uptime < 60: | |
uptime_value = '{:.2f} seconds'.format(cur_uptime) | |
elif cur_uptime < 3600: | |
uptime_value = '{:.2f} minutes'.format(cur_uptime / 60) | |
else: | |
uptime_value = '{:.2f} hours and {:.2f} minutes'.format(cur_uptime / 3600, (cur_uptime % 3600) / 60) | |
result['uptime'] = uptime_value | |
import datetime | |
import functools | |
tornado.ioloop.IOLoop.current().add_timeout( | |
datetime.timedelta(seconds=15), # long response | |
functools.partial(self.finish, result) | |
) | |
# self.finish(result) | |
else: | |
request = make_get_request( | |
'http://{host}:{port}/status'.format(host='127.0.0.1' if options.host == '0.0.0.0' else options.host, | |
port=options.port), | |
data={'no_network_check': 'true'}, | |
connect_timeout=0.5, | |
request_timeout=0.5, # do not wait for long responses | |
follow_redirects=False | |
) | |
def _request_ready(result): | |
if result.error is not None: | |
raise tornado.web.HTTPError(httplib.SERVICE_UNAVAILABLE) | |
self.finish(result.body) | |
self.application.curl_http_client.fetch(request, callback=_request_ready) | |
# ... | |
# ... | |
class FrontikApplication(tornado.web.Application): | |
class DefaultConfig(object): | |
pass | |
def __init__(self, **settings): | |
# ... | |
# set limit to 1 | |
self.curl_http_client = tornado.curl_httpclient.CurlAsyncHTTPClient(max_clients=1) | |
# ... | |
Тот же самый код с подменой на AsyncHTTPClient (встроенный написанный на python) работает правильно:
class FrontikApplication(tornado.web.Application):
def __init__(self, **settings):
# ...
import tornado.httpclient
self.curl_http_client = tornado.httpclient.AsyncHTTPClient(max_clients=1)
# ...
[I 15.04.02 16:55:30 tornado_util.server server:114] starting server on 127.0.0.1:8080
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 1 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 2 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 3 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 4 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 5 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 6 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 7 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 8 queued requests.
[D 15.04.02 16:55:41 tornado.general simple_httpclient:107] max_clients limit reached, request queued. 1 active, 9 queued requests.
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 507.20ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 501.84ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 501.06ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 501.31ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 501.82ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 501.06ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 501.29ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 502.04ms
[E 15.04.02 16:55:41 tornado.access web:1854] 503 GET /status (127.0.0.1) 502.12ms
[E 15.04.02 16:55:42 tornado.access web:1854] 503 GET /status (127.0.0.1) 1010.47ms
[I 15.04.02 16:55:56 tornado.access web:1854] 200 GET /status?no_network_check=true (127.0.0.1) 15007.35ms
[I 15.04.02 16:55:56 tornado.access web:1854] 200 GET /status?no_network_check=true (127.0.0.1) 15005.84ms
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
in logs: