Download CSV directly to parquet

convert_csv_to_parquet.py

import aiohttp
import asyncio
import polars as pl
import tempfile

async def convert_csv_to_parquet(url: str, output_file: str):
  """
  A script that rapidly streams a CSV url to a parquet file
  
  Args:
    url: a string representing the URL of the CSV to be downloaded
    output_file: the location where the parquet file should be saved (be sure to include .parquet at the end)
  """
  async with aiohttp.ClientSession() as session:
    async with session.get(url) as resp:
      with tempfile.NamedTemporaryFile() as fp:
        fp.write(await resp.read())
        fp.seek(0)
        pl.scan_csv(fp.name, encoding="utf8-lossy").sink_parquet(output_file)

asnycio.run(convert_csv_to_parquet(url))