Gist to write large parquet files to S3 on M1 (avoid blosc issues)

write_large_parquet_m1

# ...spin up cluster...connect Dask...etc.

# use client.submit() to write large parquet files to S3 (to avoid blosc issues on M1)
def submit_jobs():
    from distributed import get_client
    
    with get_client() as client:        
        large = dask.datasets.timeseries(start="2000", end="2015", freq="10s", partition_freq="1M")

        large.to_parquet(
            's3://coiled-datasets/dask-merge/large.parquet',
            engine="fastparquet"
        )

client.submit(submit_jobs).result()