Skip to content

Instantly share code, notes, and snippets.

@khibma
Created March 16, 2015 21:01
Show Gist options
  • Save khibma/65acf00f85275c0c477c to your computer and use it in GitHub Desktop.
Save khibma/65acf00f85275c0c477c to your computer and use it in GitHub Desktop.
Py3 Request maker
import urllib.parse as parse
import urllib.request as request
import http.client as client
#Force connections to the previous HTTP 1.0 standard as this is faster with chunked responses, which is typically what you'll get
client.HTTPConnection._http_vsn= 10
client.HTTPConnection._http_vsn_str='HTTP/1.0'
def sendReq(url, qDict=None, headers=None):
"""Robust request maker"""
#Need to handle chunked response / incomplete reads. 2 solutions here: http://stackoverflow.com/questions/14442222/how-to-handle-incompleteread-in-python
#This function sends a request and handles incomplete reads. However its found to be very slow. It adds 30 seconds to chunked
#responses. Forcing the connection to HTTP 10 (1.0) at the top, for some reason makes it faster.
qData = parse.urlencode(qDict).encode('UTF-8') if qDict else None
reqObj = request.Request(url)
if headers != None:
for k, v in headers.items():
reqObj.add_header(k, v)
try:
if qDict == None: #GET
r = request.urlopen(reqObj)
else: #POST
r = request.urlopen(reqObj, qData)
responseJSON=""
while True:
try:
responseJSONpart = r.read()
except client.IncompleteRead as icread:
responseJSON = responseJSON + icread.partial.decode('utf-8')
continue
else:
responseJSON = responseJSON + responseJSONpart.decode('utf-8')
break
return (json.loads(responseJSON))
except Exception as RESTex:
print("Exception occurred making REST call: " + RESTex.__str__())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment