How to store and retrieve gzip-compressed objects in AWS S3

jsonlines_gzip_bytesio_bytestream_serialize_serialise.py

# Totally ~plagiarised~ inspired from the forked file by Vince Veselosky in this gist, extended to work with jsonlines format 
# and skipping the S3 write so the serialisation can be tested without mocking S3

import re
import io
import json
import gzip
import dateutil.parser
from json import JSONEncoder
from pprint import pprint
from datetime import datetime, date

class DatetimePlusEncoder(JSONEncoder):
    """
    The default serialiser and datetime_hook deserialiser are
    intended to reverse each other
    """
    def default(self, obj):
        """
        JSON serializer for datetimes & dates
        from https://stackoverflow.com/a/22238613/1335793
        """
        if isinstance(obj, (datetime, date)):
            return obj.isoformat(sep="T", timespec="milliseconds")
        raise TypeError(f"Type {type(obj)} is not serializable")

    @staticmethod
    def datetime_hook(json_dict):
        patty = re.compile(r"\d\d\d\d-[0-1]\d-[0-3]\dT[0-2]\d:[0-6]\d:[0-6]\d.*")
        for k, v in json_dict.items():
            if isinstance(v, str):
                p_match = re.match(patty, v)
                if p_match:
                    json_dict[k] = dateutil.parser.parse(v)
        return json_dict


data = [
    {"hey": "thing", "yo": "stuff", "what": 3.00, "time success": datetime(2018, 11, 28, 20, 54, 0)},
    {"hey": "more thing", "yo": "less stuff", "what": 2, "time success": datetime(2018, 11, 29, 12, 54, 47)}
]

data_transformed = [
    {"hey": "thing", "yo": "stuff", "what": 3, "time success": '2018-11-28T20:54:00.000'},
    {"hey": "more thing", "yo": "less stuff", "what": 2, "time success": '2018-11-29T12:54:47.000'},
]

def serialise_to_json_lines_gzip_bytestream(list_of_dicts_to_be_serialised):
    """
    Serialise to jsonlines text compressed as gzip, as a byte stream

    :param list_of_dicts_to_be_serialised: what it says on the can
    :return: a BytesIO byte stream
    """
    gzip_byte_stream = io.BytesIO()
    with gzip.open(gzip_byte_stream, mode="wt", encoding="utf-8") as gzip_file_handle:
        for row in list_of_dicts_to_be_serialised:
            gzip_file_handle.write(json.dumps(row, cls=DatetimePlusEncoder))
            gzip_file_handle.write("\n")
    return gzip_byte_stream

def deserialise_from_json_lines_gzip_bytestream(gzip_byte_stream, datetime_hook=DatetimePlusEncoder.datetime_hook):
    """
    Deserialise a gzipped bytestream, rehydrating the original list of dicts
    Doesn't deserialise complex objects
    Will convert datetime strings back into datetime objects if desired:
        ISO8601-ish, strict must start with `YYYY-mm-ddThh:mi:ss` back to datetimes.
    If you don't want to convert datetime strings back to datetime objects then pass in
    datetime_hook=None

    :param gzip_byte_stream: A gzip compressed bytestream containing jsonlines text
    :return: list of dicts
    """
    gzip_body = gzip_byte_stream.getvalue()
    new_bytes_io = io.BytesIO(gzip_body)
    with gzip.open(new_bytes_io, mode="rt", encoding="utf-8") as unzipped_file:
        decoded_data = [json.loads(row, object_hook=datetime_hook) for row in unzipped_file.readlines()]
    return decoded_data

serialised_data = serialise_to_json_lines_gzip_bytestream(data)

deserialised_data = deserialise_from_json_lines_gzip_bytestream(serialised_data)

deserialised_data_no_datetimes = deserialise_from_json_lines_gzip_bytestream(serialised_data, datetime_hook=None)

pprint(data)
pprint(deserialised_data)
print()
pprint(data_transformed)
pprint(deserialised_data_no_datetimes)
assert data == deserialised_data

assert data_transformed == deserialised_data_no_datetimes

s3gzip.py

# vim: set fileencoding=utf-8 :
#
# How to store and retrieve gzip-compressed objects in AWS S3
###########################################################################
#
#   Copyright 2015 Vince Veselosky and contributors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
from __future__ import absolute_import, print_function, unicode_literals
from io import BytesIO
from gzip import GzipFile

import boto3
s3 = boto3.client('s3')
bucket = 'bluebucket.mindvessel.net'

# Read in some example text, as unicode
with open("utext.txt") as fi:
    text_body = fi.read().decode("utf-8")

# A GzipFile must wrap a real file or a file-like object. We do not want to
# write to disk, so we use a BytesIO as a buffer.
gz_body = BytesIO()
gz = GzipFile(None, 'wb', 9, gz_body)
gz.write(text_body.encode('utf-8'))  # convert unicode strings to bytes!
gz.close()
# GzipFile has written the compressed bytes into our gz_body
s3.put_object(
    Bucket=bucket,
    Key='gztest.txt',  # Note: NO .gz extension!
    ContentType='text/plain',  # the original type
    ContentEncoding='gzip',  # MUST have or browsers will error
    Body=gz_body.getvalue()
)
retr = s3.get_object(Bucket=bucket, Key='gztest.txt')
# Now the fun part. Reading it back requires this little dance, because
# GzipFile insists that its underlying file-like thing implement tell and
# seek, but boto3's io stream does not.
bytestream = BytesIO(retr['Body'].read())
got_text = GzipFile(None, 'rb', fileobj=bytestream).read().decode('utf-8')
assert got_text == text_body