Read entire content of s3 bucket to pandas dataframe

s3_bucket_to_pandas.py

import boto3 
import pandas as pd 
from io import BytesIO 
 
bucket = "innovation-mapping-general" 
directory = "nih_all_processed_data/" 
s3 = boto3.resource('s3') 
dfs = [] 
for key in s3.Bucket(bucket).objects.all(): 
    if not key.key.startswith(directory): 
        continue 
    if key.key == directory: 
        continue 
    #print(key.key, str(key.key)) 
     
    obj = s3.Object(bucket, key.key)     
    with BytesIO(obj.get()['Body'].read()) as bio:
        df = pd.read_json(bio) 
    dfs.append(df) 
    if len(dfs) > 3: 
        break 
df = pd.concat(dfs)