Skip to content

Instantly share code, notes, and snippets.

@mndrake
mndrake / parquet_split.py
Created May 27, 2018 13:20
read/write to split parquet files
import os
from io import BytesIO
import pyarrow as pa
import pyarrow.parquet as pq
kilobytes = 1024
megabytes = kilobytes * 1000
chunksize = int(10 * megabytes)
import json
import boto3
firehose_client = boto3.client('firehose')
def lambda_handler(event, context):
resultString = ""
for record in event['Records']: