arrow-parquet-performance-rendered.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f93cd8a9-4fb9-45a4-9096-455b1120c049",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyarrow as pa\n",
    "import pyarrow.parquet as pq\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27d172d4-0aeb-456b-bd01-4b21960721a6",
   "metadata": {},
   "source": [
    "### Create Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fb0e955e-baf8-4274-8298-3872ba33b2e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.random.randint(0, 100000, size=(1000000, 100))\n",
    "df = pd.DataFrame(x)\n",
    "t = pa.Table.from_pandas(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f636cf9-3604-4b01-bba2-96a4aa5e4cb8",
   "metadata": {},
   "source": [
    "### Write to parquet locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1dac9b6f-6318-4aa8-afd4-ad37d8ad4ac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "pq.write_table(t, \"foo.parquet\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08e03332-f664-49aa-abea-31f1ae528eff",
   "metadata": {},
   "source": [
    "### Time Disk Speeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "442c9c4b-6dfe-4abb-be78-001b225f4a7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disk Bandwidth: 1266 MiB/s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "start = time.time()\n",
    "with open(\"foo.parquet\", mode=\"rb\") as f:\n",
    "    bytes = f.read()\n",
    "    nbytes = len(bytes)\n",
    "    \n",
    "stop = time.time()\n",
    "\n",
    "print(\"Disk Bandwidth:\", int(nbytes / (stop - start) / 2**20), \"MiB/s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d11c4c98-b970-4aa9-9b23-937c0c39fb09",
   "metadata": {},
   "source": [
    "### Time Arrow Parquet Speeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0e974144-7119-4a99-b02d-113b3750407a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyArrow Read Bandwidth: 232 MiB/s\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "_ = pq.read_table(\"foo.parquet\")\n",
    "stop = time.time()\n",
    "\n",
    "print(\"PyArrow Read Bandwidth:\", int(nbytes / (stop - start) / 2**20), \"MiB/s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fcd44d24-2844-4671-ba4a-42989cf62409",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyArrow In-Memory Bandwidth: 183 MiB/s\n"
     ]
    }
   ],
   "source": [
    "import io\n",
    "\n",
    "start = time.time()\n",
    "pq.read_table(io.BytesIO(bytes))\n",
    "stop = time.time()\n",
    "\n",
    "print(\"PyArrow In-Memory Bandwidth:\", int(nbytes / (stop - start) / 2**20), \"MiB/s\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pyarrow]",
   "language": "python",
   "name": "conda-env-pyarrow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}