Created
March 16, 2015 21:01
-
-
Save khibma/65acf00f85275c0c477c to your computer and use it in GitHub Desktop.
Py3 Request maker
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import urllib.parse as parse | |
import urllib.request as request | |
import http.client as client | |
#Force connections to the previous HTTP 1.0 standard as this is faster with chunked responses, which is typically what you'll get | |
client.HTTPConnection._http_vsn= 10 | |
client.HTTPConnection._http_vsn_str='HTTP/1.0' | |
def sendReq(url, qDict=None, headers=None): | |
"""Robust request maker""" | |
#Need to handle chunked response / incomplete reads. 2 solutions here: http://stackoverflow.com/questions/14442222/how-to-handle-incompleteread-in-python | |
#This function sends a request and handles incomplete reads. However its found to be very slow. It adds 30 seconds to chunked | |
#responses. Forcing the connection to HTTP 10 (1.0) at the top, for some reason makes it faster. | |
qData = parse.urlencode(qDict).encode('UTF-8') if qDict else None | |
reqObj = request.Request(url) | |
if headers != None: | |
for k, v in headers.items(): | |
reqObj.add_header(k, v) | |
try: | |
if qDict == None: #GET | |
r = request.urlopen(reqObj) | |
else: #POST | |
r = request.urlopen(reqObj, qData) | |
responseJSON="" | |
while True: | |
try: | |
responseJSONpart = r.read() | |
except client.IncompleteRead as icread: | |
responseJSON = responseJSON + icread.partial.decode('utf-8') | |
continue | |
else: | |
responseJSON = responseJSON + responseJSONpart.decode('utf-8') | |
break | |
return (json.loads(responseJSON)) | |
except Exception as RESTex: | |
print("Exception occurred making REST call: " + RESTex.__str__()) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment