example using dlt to ingest s3

dlt_assets.py

# after running dlt init filesystem duckdb

from dagster import AssetExecutionContext
from dagster_embedded_elt.dlt import DagsterDltResource, dlt_assets
import dlt
from dlt_sources.filesystem_pipeline import s3_locations_data
from dlt_sources.filesystem import readers
from pathlib import Path 
import os 

@dlt_assets(
    dlt_source=
    readers(
        bucket_url="s3://hooli-demo/embedded-elt/", file_glob="*.csv"
    ),
    dlt_pipeline=dlt.pipeline(
        pipeline_name="locations_data_2",
        dataset_name="locations_2",
        destination=dlt.destinations.duckdb(os.path.join(DBT_PROJECT_DIR, "example.duckdb")), #"duckdb",
        progress="log",
    ),
    name="locations_2",
    group_name="dlt_testing",
)
def dagster_s3_assets_2(context: AssetExecutionContext, dlt: DagsterDltResource):
    yield from dlt.run(context=context)