Skip to content

Instantly share code, notes, and snippets.

View djouallah's full-sized avatar

Mimoune djouallah

View GitHub Profile
import time
# Make sure you have duckdb==0.7.0. Earlier versions might fail with GIL problems ( https://twitter.com/mr_le_fox/status/1620535141675433986 )
import duckdb
import s3fs
from fsspec.implementations.cached import SimpleCacheFileSystem
# Create the s3 file system. This one does not have caching
@ML-engineer
ML-engineer / duckdb_bq_storage_api.py
Created July 4, 2022 22:08
Read BQ table to DuckDB directly from storage read api
import duckdb
from google.cloud import bigquery
bqclient = bigquery.Client()
table = bigquery.TableReference.from_string(
"bigquery-public-data.utility_us.country_code_iso"
)
rows = bqclient.list_rows(table)
country_code_iso = rows.to_arrow(create_bqstorage_client=True)
cursor = duckdb.connect()