A quick and easy CSV to Parquet converter from one bucket to another. This may be attached to a lambda function that can be triggered whenever a new s3:PutObject is executed.

csv_to_parquet.py

from io import BytesIO
import boto3
import pandas as pd
from os import environ


def convert(bucket, key):
    s3_client = boto3.client('s3', region_name=environ['REGION'])
    s3_object = s3_client.get_object(Bucket=bucket, Key=key)

    df = pd.read_csv(s3_object['Body'])
    df.columns.astype(str)

    target_bucket = f"{bucket}-parquet"
    target_key = f"{key.split('-')[0]}.parquet"

    parquet_out_buffer = BytesIO()
    df.to_parquet(parquet_out_buffer, index=False, engine='fastparquet')

    s3_res = boto3.resource('s3')
    s3_res.Object(target_bucket, target_key).put(Body=parquet_out_buffer.getvalue())

    return {
        'bucket': target_bucket,
        'key': target_key
    }

Also using fastparquet instead of pyarrow as pyarrow goes over the 50mb limit even with auto layers on.