boto3 s3 pagination example

boto3_s3_paginate.py

/usr/bin/env python3

import boto3
from boto3.session import Session
from io import BytesIO
from sys import argv

pn = argv[1]
bucket = argv[2]
sub_folder_path = argv[3]

session = Session(profile_name=pn)
s3c = session.client("s3")

gimme = s3c.list_objects_v2(Bucket=bucket,Prefix=sub_folder_path)
my_keys = [x['Key'] for x in gimme['Contents']]

if gimme['IsTruncated']:
    pages = paginator.paginate(Bucket=bucket,Prefix=sub_folder_path, PaginationConfig={'MaxItems':1000000,'PageSize':1000,'StartingToken':gimme['NextContinuationToken']})
    for page in pages:
        for obj in page['Contents']:
            my_keys.append(obj['Key'])

accts = {}
my_keys = sorted(set(my_keys))
for this in my_keys:
    f = BytesIO()
    s3c.download_fileobj(bucket, this, f)
    j = json.loads(f.getvalue().decode('utf-8'))
    accts[j['awsAccountId']] = j