Write out a CSV file with PyArrow

write_csv_with_pyarrow.py

import os
from dotenv import load_dotenv
import pyarrow as pa
from pyarrow.fs import S3FileSystem
from pyarrow import csv


# Load environment
load_dotenv(dotenv_path="env.txt")

# Create a dummy table
table = pa.table({"a": [1, 2, 3], "b": [4, 5, 6]})

# Setup an S3 file system
s3 = S3FileSystem(access_key=os.environ['AWS_ACCESS_KEY_ID'],
                  secret_key=os.environ['AWS_SECRET_ACCESS_KEY'],
                  session_token=os.environ['AWS_SESSION_TOKEN'],
                  region=os.environ['AWS_REGION']
                  )

s3_path = "voltrondata-demo-data-us-east-2/phil_test/joe.csv"
with s3.open_output_stream(path=s3_path) as s3_file:
    csv.write_csv(data=table, output_file=s3_file)