Wrangling files on AWS S3 with the awswrangler python module (in Colab)

awswrangler_in_colab.py

# Commented out IPython magic to ensure Python compatibility.
# %%capture
# !pip install awswrangler --quiet

import awswrangler as wr
import pandas as pd
import boto3
from google.colab import data_table, userdata

data_table.enable_dataframe_formatter()

# refresh the temporary credentials via the key symbol in the left hand tab
boto3.setup_default_session(
    region_name="us-east-2",
    aws_access_key_id=userdata.get('AWS_ACCESS_KEY_ID'),
    aws_secret_access_key=userdata.get('AWS_SECRET_ACCESS_KEY'),
    aws_session_token=userdata.get('AWS_SESSION_TOKEN')
)

path = 's3://path/do/prefix/with/partitioned/parquet/dataset'
df = wr.s3.read_parquet(path, dataset=True)

df.head()