01_raw_data_to_s3.py

import boto3
import requests


def extract_and_upload(file: str, bucket: str = "kestraio") -> None:
    url = f"https://raw.githubusercontent.com/kestra-io/datasets/main/{file}"

    response = requests.get(url)
    data = response.content.decode("utf-8")
    s3 = boto3.resource("s3")
    s3.Bucket(bucket).put_object(Key=file, Body=data)
    print(f"{url} downloaded and saved to {bucket}/{file}")


if __name__ == "__main__":
    for month in range(1, 13):
        filename = f"monthly_orders/2023_{str(month).zfill(2)}.csv"
        extract_and_upload(filename)