Skip to content

Instantly share code, notes, and snippets.

@michaelhelmick

michaelhelmick/iter_lines.py

Created May 28, 2013 19:05
Show Gist options
  • Save michaelhelmick/5665249 to your computer and use it in GitHub Desktop.
Save michaelhelmick/5665249 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
iter_lines.py
ITER_CHUNK_SIZE = 512
def iter_lines(response, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None):
"""Iterates over the response data, one line at a time. When
stream=True is set on the request, this avoids reading the
content at once into memory for large responses.
"""
pending = None
for chunk in response.iter_content(chunk_size=chunk_size,
decode_unicode=decode_unicode):
if pending is not None:
chunk = pending + chunk
#lines = chunk.splitlines()
print 'chunk is', chunk
lines = chunk.split("\r\n")
print lines
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
pending = lines.pop()
else:
pending = None
for line in lines:
yield line
if pending is not None:
yield pending
Raw
response.json
chunk is {
"created_at": "Tue May 28 19:03:46 +0000 2013",
"id": 339456985797890049,
"id_str": "339456985797890049",
"text": "@mikehelmick whut wut",
"source": "web",
"truncated": false,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": 29251354,
"in_reply_to_user_id_str": "29251354",
"in_reply_to_screen_name": "mikehelmick",
"user": {
"id": 1431865928,
"id_str": "1431865928",
"name": "Twython",
"screen_name": "__twython__",
"location": "",
"url": null,
"description": null,
"protected": false,
"followers_count": 1,
"fri
['{"
created_at ":"
Tue May 28 19: 03: 46 + 0000 2013 ","
id ":339456985797890049,"
id_str ":"
339456985797890049 ","
text ":"@
mikehelmick whut wut ","
source ":"
web ","
truncated ":false,"
in_reply_to_status_id ":null,"
in_reply_to_status_id_str ":null,"
in_reply_to_user_id ":29251354,"
in_reply_to_user_id_str ":"
29251354 ","
in_reply_to_screen_name ":"
mikehelmick ","
user ":{"
id ":1431865928,"
id_str ":"
1431865928 ","
name ":"
Twython ","
screen_name ":"
__twython__ ","
location ":"
","
url ":null,"
description ":null,"
protected ":false,"
followers_count ":1,"
fri ']
chunk is {"created_at":"Tue May 28 19:03:46 +0000 2013","id":339456985797890049,"id_str":"339456985797890049","text":"@mikehelmick whut wut","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":29251354,"in_reply_to_user_id_str":"29251354","in_reply_to_screen_name":"mikehelmick","user":{"id":1431865928,"id_str":"1431865928","name":"Twython","screen_name":"__twython__","location":"","url":null,"description":null,"protected":false,"followers_count":1,"friends_count":1,"listed_count":0,"created_at":"Thu May 16 01:11:09 +0000 2013","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":8,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"3D3D3D","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_i
[' {
"created_at": "Tue May 28 19:03:46 +0000 2013",
"id": 339456985797890049,
"id_str": "339456985797890049",
"text": "@mikehelmick whut wut",
"source": "web",
"truncated": false,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": 29251354,
"in_reply_to_user_id_str": "29251354",
"in_reply_to_screen_name": "mikehelmick",
"user": {
"id": 1431865928,
"id_str": "1431865928",
"name": "Twython",
"screen_name": "__twython__",
"location": "",
"url": null,
"description": null,
"protected": false,
"followers_count": 1,
"friends_count": 1,
"listed_count": 0,
"created_at": "Thu May 16 01:11:09 +0000 2013",
"favourites_count": 0,
"utc_offset": null,
"time_zone": null,
"geo_enabled": false,
"verified": false,
"statuses_count": 8,
"lang": "en",
"contributors_enabled": false,
"is_translator": false,
"profile_background_color": "3D3D3D",
"profile_background_image_url": "http:\\/\\/a0.twimg.com\\/images\\/themes\\/theme1\\/bg.png",
"profile_background_image_url_https": "https:\\/\\/si0.twimg.com\\/images\\/themes\\/theme1\\/bg.png",
"profile_background_tile": false,
"profile_i']
chunk is {"
created_at ":"
Tue May 28 19: 03: 46 + 0000 2013 ","
id ":339456985797890049,"
id_str ":"
339456985797890049 ","
text ":"@
mikehelmick whut wut ","
source ":"
web ","
truncated ":false,"
in_reply_to_status_id ":null,"
in_reply_to_status_id_str ":null,"
in_reply_to_user_id ":29251354,"
in_reply_to_user_id_str ":"
29251354 ","
in_reply_to_screen_name ":"
mikehelmick ","
user ":{"
id ":1431865928,"
id_str ":"
1431865928 ","
name ":"
Twython ","
screen_name ":"
__twython__ ","
location ":"
","
url ":null,"
description ":null,"
protected ":false,"
followers_count ":1,"
friends_count ":1,"
listed_count ":0,"
created_at ":"
Thu May 16 01: 11: 09 + 0000 2013 ","
favourites_count ":0,"
utc_offset ":null,"
time_zone ":null,"
geo_enabled ":false,"
verified ":false,"
statuses_count ":8,"
lang ":"
en ","
contributors_enabled ":false,"
is_translator ":false,"
profile_background_color ":"
3D3D3D ","
profile_background_image_url ":"
http: \/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/sticky\/default_profile_images\/default_profile_3_normal.png","profile_image_url_https":"https:\/\/si0.twimg.com\/sticky\/default_profile_images\/default_profile_3_normal.png","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":true,"following":null,"follow_request_sent":null,"notifications":null},"geo":nul
['{"created_at":"Tue May 28 19:03:46 +0000 2013","id":339456985797890049,"id_str":"339456985797890049","text":"@mikehelmick whut wut","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":29251354,"in_reply_to_user_id_str":"29251354","in_reply_to_screen_name":"mikehelmick","user":{"id":1431865928,"id_str":"1431865928","name":"Twython","screen_name":"__twython__","location":"","url":null,"description":null,"protected":false,"followers_count":1,"friends_count":1,"listed_count":0,"created_at":"Thu May 16 01:11:09 +0000 2013","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":8,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"3D3D3D","profile_background_image_url":"http:\\/\\/a0.twimg.com\\/images\\/themes\\/theme1\\/bg.png","profile_background_image_url_https":"https:\\/\\/si0.twimg.com\\/images\\/themes\\/theme1\\/bg.png","profile_background_tile":false,"profile_image_url":"http:\\/\\/a0.twimg.com\\/sticky\\/default_profile_images\\/default_profile_3_normal.png","profile_image_url_https":"https:\\/\\/si0.twimg.com\\/sticky\\/default_profile_images\\/default_profile_3_normal.png","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":true,"following":null,"follow_request_sent":null,"notifications":null},"geo":nul']
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment