Count the number of rows in a sharded parquet dataset without loading the shards. Works by reading just the metadata headers. Works with S3 datasets

count_parquet_rows.py

import pyarrow.dataset as ds

def count_parquet_rows(dataset_path: str) -> int:
    """
    Count the number of rows in a parquet file without reading the data into memory.
    
    https://stackoverflow.com/a/79118602/4212158
    """
    dataset = ds.dataset(dataset_path, format="parquet")
    row_count = sum(row_group.num_rows for fragment in dataset.get_fragments() for row_group in fragment.row_groups)
    return row_count