Created
January 23, 2020 13:55
-
-
Save viewpointsa/b6be251d26af18456ad23841e56ddcb8 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import struct, time, zlib | |
def generate_gzip(): | |
# Migration Python2 to 3 https://stackoverflow.com/a/44387566/2137364 | |
# Yield a gzip file header first. | |
yield ( | |
b'\037\213\010\000' + # Gzip file, deflate, no filename | |
struct.pack('<L', int(time.time())) + # compression start time | |
b'\002\377' # maximum compression, no OS specified | |
) | |
# bookkeeping: the compression state, running CRC and total length | |
compressor = zlib.compressobj( | |
9, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) | |
crc = zlib.crc32(b'') | |
length = 0 | |
for x in range(10000): | |
data = "this is my line: {}\n".format(x).encode() | |
chunk = compressor.compress(data) | |
if chunk: | |
yield chunk | |
crc = zlib.crc32(data, crc) & 0xffffffff | |
length += len(data) | |
# Finishing off, send remainder of the compressed data, and CRC and length | |
yield compressor.flush() | |
yield struct.pack("<2L", crc, length & 0xffffffff) | |
with open('out.gz', 'wb') as f: | |
for data in generate_gzip(): | |
f.write( data ) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment