Created
July 4, 2014 05:32
-
-
Save menghan/17a370f16e5fefd068f3 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
menghan@MenghanMac:~$ cat /tmp/diffs | |
--- httpclient.py 2013-11-15 11:21:29.000000000 +0800 | |
+++ httpclient.pyc_dis 2014-07-04 13:30:27.000000000 +0800 | |
@@ -1,3 +1,4 @@ | |
+#Embedded file name: /Users/menghan/.virtualenvs/common/lib/python2.7/site-packages/tornado/httpclient.py | |
"""Blocking and non-blocking HTTP client interfaces. | |
This module defines a common interface shared by two implementations, | |
@@ -26,20 +27,16 @@ | |
you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum | |
supported version is 7.18.2, and the recommended version is 7.21.1 or newer. | |
""" | |
- | |
from __future__ import absolute_import, division, print_function, with_statement | |
- | |
import functools | |
import time | |
import weakref | |
- | |
from tornado.concurrent import Future | |
from tornado.escape import utf8 | |
from tornado import httputil, stack_context | |
from tornado.ioloop import IOLoop | |
from tornado.util import Configurable | |
- | |
class HTTPClient(object): | |
"""A blocking HTTP client. | |
@@ -55,6 +52,7 @@ | |
print "Error:", e | |
http_client.close() | |
""" | |
+ | |
def __init__(self, async_client_class=None, **kwargs): | |
self._io_loop = IOLoop() | |
if async_client_class is None: | |
@@ -81,8 +79,7 @@ | |
If an error occurs during the fetch, we raise an `HTTPError`. | |
""" | |
- response = self._io_loop.run_sync(functools.partial( | |
- self._async_client.fetch, request, **kwargs)) | |
+ response = self._io_loop.run_sync(functools.partial(self._async_client.fetch, request, **kwargs)) | |
response.rethrow() | |
return response | |
@@ -110,6 +107,7 @@ | |
deprecated. The implementation subclass as well as arguments to | |
its constructor can be set with the static method `configure()` | |
""" | |
+ | |
@classmethod | |
def configurable_base(cls): | |
return AsyncHTTPClient | |
@@ -130,8 +128,7 @@ | |
io_loop = io_loop or IOLoop.current() | |
if io_loop in cls._async_clients() and not force_instance: | |
return cls._async_clients()[io_loop] | |
- instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, | |
- **kwargs) | |
+ instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, **kwargs) | |
if not force_instance: | |
cls._async_clients()[io_loop] = instance | |
return instance | |
@@ -169,9 +166,6 @@ | |
""" | |
if not isinstance(request, HTTPRequest): | |
request = HTTPRequest(url=request, **kwargs) | |
- # We may modify this (to add Host, Accept-Encoding, etc), | |
- # so make sure we don't modify the caller's object. This is also | |
- # where normal dicts get converted to HTTPHeaders objects. | |
request.headers = httputil.HTTPHeaders(request.headers) | |
request = _RequestProxy(request, self.defaults) | |
future = Future() | |
@@ -183,12 +177,11 @@ | |
if isinstance(exc, HTTPError) and exc.response is not None: | |
response = exc.response | |
elif exc is not None: | |
- response = HTTPResponse( | |
- request, 599, error=exc, | |
- request_time=time.time() - request.start_time) | |
+ response = HTTPResponse(request, 599, error=exc, request_time=time.time() - request.start_time) | |
else: | |
response = future.result() | |
self.io_loop.add_callback(callback, response) | |
+ | |
future.add_done_callback(handle_future) | |
def handle_response(response): | |
@@ -196,6 +189,7 @@ | |
future.set_exception(response.error) | |
else: | |
future.set_result(response) | |
+ | |
self.fetch_impl(request, handle_response) | |
return future | |
@@ -227,32 +221,9 @@ | |
class HTTPRequest(object): | |
"""HTTP client request object.""" | |
+ _DEFAULTS = dict(connect_timeout=20.0, request_timeout=20.0, follow_redirects=True, max_redirects=5, use_gzip=True, proxy_password='', allow_nonstandard_methods=False, validate_cert=True) | |
- # Default values for HTTPRequest parameters. | |
- # Merged with the values on the request object by AsyncHTTPClient | |
- # implementations. | |
- _DEFAULTS = dict( | |
- connect_timeout=20.0, | |
- request_timeout=20.0, | |
- follow_redirects=True, | |
- max_redirects=5, | |
- use_gzip=True, | |
- proxy_password='', | |
- allow_nonstandard_methods=False, | |
- validate_cert=True) | |
- | |
- def __init__(self, url, method="GET", headers=None, body=None, | |
- auth_username=None, auth_password=None, auth_mode=None, | |
- connect_timeout=None, request_timeout=None, | |
- if_modified_since=None, follow_redirects=None, | |
- max_redirects=None, user_agent=None, use_gzip=None, | |
- network_interface=None, streaming_callback=None, | |
- header_callback=None, prepare_curl_callback=None, | |
- proxy_host=None, proxy_port=None, proxy_username=None, | |
- proxy_password=None, allow_nonstandard_methods=None, | |
- validate_cert=None, ca_certs=None, | |
- allow_ipv6=None, | |
- client_key=None, client_cert=None): | |
+ def __init__(self, url, method = 'GET', headers = None, body = None, auth_username = None, auth_password = None, auth_mode = None, connect_timeout = None, request_timeout = None, if_modified_since = None, follow_redirects = None, max_redirects = None, user_agent = None, use_gzip = None, network_interface = None, streaming_callback = None, header_callback = None, prepare_curl_callback = None, proxy_host = None, proxy_port = None, proxy_username = None, proxy_password = None, allow_nonstandard_methods = None, validate_cert = None, ca_certs = None, allow_ipv6 = None, client_key = None, client_cert = None): | |
r"""All parameters except ``url`` are optional. | |
:arg string url: URL to fetch | |
@@ -318,8 +289,7 @@ | |
if headers is None: | |
headers = httputil.HTTPHeaders() | |
if if_modified_since: | |
- headers["If-Modified-Since"] = httputil.format_timestamp( | |
- if_modified_since) | |
+ headers['If-Modified-Since'] = httputil.format_timestamp(if_modified_since) | |
self.proxy_host = proxy_host | |
self.proxy_port = proxy_port | |
self.proxy_username = proxy_username | |
@@ -379,15 +349,14 @@ | |
plus ``queue``, which is the delay (if any) introduced by waiting for | |
a slot under `AsyncHTTPClient`'s ``max_clients`` setting. | |
""" | |
- def __init__(self, request, code, headers=None, buffer=None, | |
- effective_url=None, error=None, request_time=None, | |
- time_info=None, reason=None): | |
+ | |
+ def __init__(self, request, code, headers = None, buffer = None, effective_url = None, error = None, request_time = None, time_info = None, reason = None): | |
if isinstance(request, _RequestProxy): | |
self.request = request.request | |
else: | |
self.request = request | |
self.code = code | |
- self.reason = reason or httputil.responses.get(code, "Unknown") | |
+ self.reason = reason or httputil.responses.get(code, 'Unknown') | |
if headers is not None: | |
self.headers = headers | |
else: | |
@@ -410,10 +379,9 @@ | |
def _get_body(self): | |
if self.buffer is None: | |
- return None | |
- elif self._body is None: | |
+ return | |
+ if self._body is None: | |
self._body = self.buffer.getvalue() | |
- | |
return self._body | |
body = property(_get_body) | |
@@ -424,8 +392,8 @@ | |
raise self.error | |
def __repr__(self): | |
- args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items())) | |
- return "%s(%s)" % (self.__class__.__name__, args) | |
+ args = ','.join(('%s=%r' % i for i in sorted(self.__dict__.items()))) | |
+ return '%s(%s)' % (self.__class__.__name__, args) | |
class HTTPError(Exception): | |
@@ -442,11 +410,12 @@ | |
and you can look at ``error.response.headers['Location']`` to see the | |
destination of the redirect. | |
""" | |
+ | |
def __init__(self, code, message=None, response=None): | |
self.code = code | |
- message = message or httputil.responses.get(code, "Unknown") | |
+ message = message or httputil.responses.get(code, 'Unknown') | |
self.response = response | |
- Exception.__init__(self, "HTTP %d: %s" % (self.code, message)) | |
+ Exception.__init__(self, 'HTTP %d: %s' % (self.code, message)) | |
class _RequestProxy(object): | |
@@ -454,6 +423,7 @@ | |
Used internally by AsyncHTTPClient implementations. | |
""" | |
+ | |
def __init__(self, request, defaults): | |
self.request = request | |
self.defaults = defaults | |
@@ -465,33 +435,33 @@ | |
elif self.defaults is not None: | |
return self.defaults.get(name, None) | |
else: | |
- return None | |
+ return | |
def main(): | |
from tornado.options import define, options, parse_command_line | |
- define("print_headers", type=bool, default=False) | |
- define("print_body", type=bool, default=True) | |
- define("follow_redirects", type=bool, default=True) | |
- define("validate_cert", type=bool, default=True) | |
+ define('print_headers', type=bool, default=False) | |
+ define('print_body', type=bool, default=True) | |
+ define('follow_redirects', type=bool, default=True) | |
+ define('validate_cert', type=bool, default=True) | |
args = parse_command_line() | |
client = HTTPClient() | |
for arg in args: | |
try: | |
- response = client.fetch(arg, | |
- follow_redirects=options.follow_redirects, | |
- validate_cert=options.validate_cert, | |
- ) | |
+ response = client.fetch(arg, follow_redirects=options.follow_redirects, validate_cert=options.validate_cert) | |
except HTTPError as e: | |
if e.response is not None: | |
response = e.response | |
else: | |
raise | |
+ | |
if options.print_headers: | |
print(response.headers) | |
if options.print_body: | |
print(response.body) | |
+ | |
client.close() | |
-if __name__ == "__main__": | |
+ | |
+if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment