Skip to content

Instantly share code, notes, and snippets.

@menghan
Created July 4, 2014 05:32
Show Gist options
  • Save menghan/17a370f16e5fefd068f3 to your computer and use it in GitHub Desktop.
Save menghan/17a370f16e5fefd068f3 to your computer and use it in GitHub Desktop.
menghan@MenghanMac:~$ cat /tmp/diffs
--- httpclient.py 2013-11-15 11:21:29.000000000 +0800
+++ httpclient.pyc_dis 2014-07-04 13:30:27.000000000 +0800
@@ -1,3 +1,4 @@
+#Embedded file name: /Users/menghan/.virtualenvs/common/lib/python2.7/site-packages/tornado/httpclient.py
"""Blocking and non-blocking HTTP client interfaces.
This module defines a common interface shared by two implementations,
@@ -26,20 +27,16 @@
you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum
supported version is 7.18.2, and the recommended version is 7.21.1 or newer.
"""
-
from __future__ import absolute_import, division, print_function, with_statement
-
import functools
import time
import weakref
-
from tornado.concurrent import Future
from tornado.escape import utf8
from tornado import httputil, stack_context
from tornado.ioloop import IOLoop
from tornado.util import Configurable
-
class HTTPClient(object):
"""A blocking HTTP client.
@@ -55,6 +52,7 @@
print "Error:", e
http_client.close()
"""
+
def __init__(self, async_client_class=None, **kwargs):
self._io_loop = IOLoop()
if async_client_class is None:
@@ -81,8 +79,7 @@
If an error occurs during the fetch, we raise an `HTTPError`.
"""
- response = self._io_loop.run_sync(functools.partial(
- self._async_client.fetch, request, **kwargs))
+ response = self._io_loop.run_sync(functools.partial(self._async_client.fetch, request, **kwargs))
response.rethrow()
return response
@@ -110,6 +107,7 @@
deprecated. The implementation subclass as well as arguments to
its constructor can be set with the static method `configure()`
"""
+
@classmethod
def configurable_base(cls):
return AsyncHTTPClient
@@ -130,8 +128,7 @@
io_loop = io_loop or IOLoop.current()
if io_loop in cls._async_clients() and not force_instance:
return cls._async_clients()[io_loop]
- instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop,
- **kwargs)
+ instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, **kwargs)
if not force_instance:
cls._async_clients()[io_loop] = instance
return instance
@@ -169,9 +166,6 @@
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
- # We may modify this (to add Host, Accept-Encoding, etc),
- # so make sure we don't modify the caller's object. This is also
- # where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
future = Future()
@@ -183,12 +177,11 @@
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
- response = HTTPResponse(
- request, 599, error=exc,
- request_time=time.time() - request.start_time)
+ response = HTTPResponse(request, 599, error=exc, request_time=time.time() - request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
+
future.add_done_callback(handle_future)
def handle_response(response):
@@ -196,6 +189,7 @@
future.set_exception(response.error)
else:
future.set_result(response)
+
self.fetch_impl(request, handle_response)
return future
@@ -227,32 +221,9 @@
class HTTPRequest(object):
"""HTTP client request object."""
+ _DEFAULTS = dict(connect_timeout=20.0, request_timeout=20.0, follow_redirects=True, max_redirects=5, use_gzip=True, proxy_password='', allow_nonstandard_methods=False, validate_cert=True)
- # Default values for HTTPRequest parameters.
- # Merged with the values on the request object by AsyncHTTPClient
- # implementations.
- _DEFAULTS = dict(
- connect_timeout=20.0,
- request_timeout=20.0,
- follow_redirects=True,
- max_redirects=5,
- use_gzip=True,
- proxy_password='',
- allow_nonstandard_methods=False,
- validate_cert=True)
-
- def __init__(self, url, method="GET", headers=None, body=None,
- auth_username=None, auth_password=None, auth_mode=None,
- connect_timeout=None, request_timeout=None,
- if_modified_since=None, follow_redirects=None,
- max_redirects=None, user_agent=None, use_gzip=None,
- network_interface=None, streaming_callback=None,
- header_callback=None, prepare_curl_callback=None,
- proxy_host=None, proxy_port=None, proxy_username=None,
- proxy_password=None, allow_nonstandard_methods=None,
- validate_cert=None, ca_certs=None,
- allow_ipv6=None,
- client_key=None, client_cert=None):
+ def __init__(self, url, method = 'GET', headers = None, body = None, auth_username = None, auth_password = None, auth_mode = None, connect_timeout = None, request_timeout = None, if_modified_since = None, follow_redirects = None, max_redirects = None, user_agent = None, use_gzip = None, network_interface = None, streaming_callback = None, header_callback = None, prepare_curl_callback = None, proxy_host = None, proxy_port = None, proxy_username = None, proxy_password = None, allow_nonstandard_methods = None, validate_cert = None, ca_certs = None, allow_ipv6 = None, client_key = None, client_cert = None):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
@@ -318,8 +289,7 @@
if headers is None:
headers = httputil.HTTPHeaders()
if if_modified_since:
- headers["If-Modified-Since"] = httputil.format_timestamp(
- if_modified_since)
+ headers['If-Modified-Since'] = httputil.format_timestamp(if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
@@ -379,15 +349,14 @@
plus ``queue``, which is the delay (if any) introduced by waiting for
a slot under `AsyncHTTPClient`'s ``max_clients`` setting.
"""
- def __init__(self, request, code, headers=None, buffer=None,
- effective_url=None, error=None, request_time=None,
- time_info=None, reason=None):
+
+ def __init__(self, request, code, headers = None, buffer = None, effective_url = None, error = None, request_time = None, time_info = None, reason = None):
if isinstance(request, _RequestProxy):
self.request = request.request
else:
self.request = request
self.code = code
- self.reason = reason or httputil.responses.get(code, "Unknown")
+ self.reason = reason or httputil.responses.get(code, 'Unknown')
if headers is not None:
self.headers = headers
else:
@@ -410,10 +379,9 @@
def _get_body(self):
if self.buffer is None:
- return None
- elif self._body is None:
+ return
+ if self._body is None:
self._body = self.buffer.getvalue()
-
return self._body
body = property(_get_body)
@@ -424,8 +392,8 @@
raise self.error
def __repr__(self):
- args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items()))
- return "%s(%s)" % (self.__class__.__name__, args)
+ args = ','.join(('%s=%r' % i for i in sorted(self.__dict__.items())))
+ return '%s(%s)' % (self.__class__.__name__, args)
class HTTPError(Exception):
@@ -442,11 +410,12 @@
and you can look at ``error.response.headers['Location']`` to see the
destination of the redirect.
"""
+
def __init__(self, code, message=None, response=None):
self.code = code
- message = message or httputil.responses.get(code, "Unknown")
+ message = message or httputil.responses.get(code, 'Unknown')
self.response = response
- Exception.__init__(self, "HTTP %d: %s" % (self.code, message))
+ Exception.__init__(self, 'HTTP %d: %s' % (self.code, message))
class _RequestProxy(object):
@@ -454,6 +423,7 @@
Used internally by AsyncHTTPClient implementations.
"""
+
def __init__(self, request, defaults):
self.request = request
self.defaults = defaults
@@ -465,33 +435,33 @@
elif self.defaults is not None:
return self.defaults.get(name, None)
else:
- return None
+ return
def main():
from tornado.options import define, options, parse_command_line
- define("print_headers", type=bool, default=False)
- define("print_body", type=bool, default=True)
- define("follow_redirects", type=bool, default=True)
- define("validate_cert", type=bool, default=True)
+ define('print_headers', type=bool, default=False)
+ define('print_body', type=bool, default=True)
+ define('follow_redirects', type=bool, default=True)
+ define('validate_cert', type=bool, default=True)
args = parse_command_line()
client = HTTPClient()
for arg in args:
try:
- response = client.fetch(arg,
- follow_redirects=options.follow_redirects,
- validate_cert=options.validate_cert,
- )
+ response = client.fetch(arg, follow_redirects=options.follow_redirects, validate_cert=options.validate_cert)
except HTTPError as e:
if e.response is not None:
response = e.response
else:
raise
+
if options.print_headers:
print(response.headers)
if options.print_body:
print(response.body)
+
client.close()
-if __name__ == "__main__":
+
+if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment