miniparquet-altrep-demo.R

# install like so:
# remotes::install_github("hannesmuehleisen/miniparquet", ref="altrep")
options(tibble.print_max = 10, tibble.print_min = 10) 

# parquet file from https://archive.luftdaten.info/parquet/2019-08/sds011/part-00000-54e23417-8f54-4a91-9b6b-b8724706a9a7-c000.snappy.parquet

f <- "pqtest/big_data.snappy.parquet"
system(sprintf("ls -lah %s", f))

# the read_parquet() function only reads metadata and sets up ALTREP
df <- miniparquet::read_parquet(f)

# head() and print.tibble take lazy subsets and are thus fast
system.time(str(head(df)))
system.time(print(tibble::tibble(df)))