Read JSON directly from AWS S3

read_from_s3.py

import io
import json
import gzip
import boto3

def read_gzipped_json_file_from_s3(bucket_name, key_name):
    s3 = boto3.client('s3')
    retr = s3.get_object(Bucket=bucket_name, Key=key_name)

    bytestream = io.BytesIO(retr['Body'].read())
    got_text = gzip.GzipFile(None, 'rb', fileobj=bytestream).read().decode('utf-8')
    dat = json.loads(got_text)
    return dat

def listdir_s3(bucket_name, prefix):
    out = !aws s3 ls $bucket_name/$prefix | sed 's/.* //'
    for line in out:
        yield line