[Pandas DataFrame storage with Apache Parquet] using Apache Arrow (from https://tech.blue-yonder.com/efficient-dataframe-storage-with-apache-parquet/) #python #bigdata #pandas #datascience #parquet

pandas_and_parquet.py

# READING PARQUET FILES TO PANDAS
import pyarrow.parquet as pq
df = pq.read_table('<filename>').to_pandas()
# Only read a subset of the columns
df = pq.read_table('<filename>', columns=['A', 'B']).to_pandas()

# WRITING PARQUET FILES WITH PANDAS
import pyarrow as pa
import pyarrow.parquet as pq
table = pa.Table.from_pandas(data_frame, timestamps_to_ms=True)
pq.write_table(table, '<filename>')