配列の読み書きのスピードテスト

MatfileSpeedTest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CPU Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25l\u001b[?7l\u001b[0m\u001b[32m\u001b[1m                    'c.\n",
      "                 ,xNMM.\n",
      "               .OMMMMo\n",
      "               OMMM0,\n",
      "     .;loddo:' loolloddol;.\n",
      "   cKMMMMMMMMMMNWMMMMMMMMMM0:\n",
      "\u001b[0m\u001b[33m\u001b[1m .KMMMMMMMMMMMMMMMMMMMMMMMWd.\n",
      " XMMMMMMMMMMMMMMMMMMMMMMMX.\n",
      "\u001b[0m\u001b[31m\u001b[1m;MMMMMMMMMMMMMMMMMMMMMMMM:\n",
      ":MMMMMMMMMMMMMMMMMMMMMMMM:\n",
      "\u001b[0m\u001b[31m\u001b[1m.MMMMMMMMMMMMMMMMMMMMMMMMX.\n",
      " kMMMMMMMMMMMMMMMMMMMMMMMMWd.\n",
      " \u001b[0m\u001b[35m\u001b[1m.XMMMMMMMMMMMMMMMMMMMMMMMMMMk\n",
      "  .XMMMMMMMMMMMMMMMMMMMMMMMMK.\n",
      "    \u001b[0m\u001b[34m\u001b[1mkMMMMMMMMMMMMMMMMMMMMMMd\n",
      "     ;KMMMMMMMWXXWMMMMMMMk.\n",
      "       .cooc,.    .,coo:.\u001b[0m\n",
      "\u001b[17A\u001b[9999999D\u001b[33C\u001b[0m\u001b[1m\u001b[32m\u001b[1mhidehisa\u001b[0m@\u001b[32m\u001b[1maraihidenaonoMacBook-Pro.local\u001b[0m \n",
      "\u001b[33C\u001b[0m---------------------------------------\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mOS\u001b[0m\u001b[0m:\u001b[0m macOS Catalina 10.15.7 19H2 x86_64\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mHost\u001b[0m\u001b[0m:\u001b[0m MacBookPro13,3\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mKernel\u001b[0m\u001b[0m:\u001b[0m 19.6.0\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mUptime\u001b[0m\u001b[0m:\u001b[0m 18 hours, 51 mins\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mPackages\u001b[0m\u001b[0m:\u001b[0m 184 (brew)\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mShell\u001b[0m\u001b[0m:\u001b[0m fish 2.7.1\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mResolution\u001b[0m\u001b[0m:\u001b[0m 1680x1050\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mDE\u001b[0m\u001b[0m:\u001b[0m Aqua\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mWM\u001b[0m\u001b[0m:\u001b[0m Quartz Compositor\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mWM Theme\u001b[0m\u001b[0m:\u001b[0m Blue (Dark)\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mTerminal\u001b[0m\u001b[0m:\u001b[0m vscode\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mCPU\u001b[0m\u001b[0m:\u001b[0m Intel i7-6820HQ (8) @ 2.70GHz\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mGPU\u001b[0m\u001b[0m:\u001b[0m Intel HD Graphics 530, AMD Radeon Pro 455\u001b[0m \n",
      "\u001b[33C\u001b[0m\u001b[33m\u001b[1mMemory\u001b[0m\u001b[0m:\u001b[0m 10414MiB / 16384MiB\u001b[0m \n",
      "\n",
      "\u001b[33C\u001b[30m\u001b[40m   \u001b[31m\u001b[41m   \u001b[32m\u001b[42m   \u001b[33m\u001b[43m   \u001b[34m\u001b[44m   \u001b[35m\u001b[45m   \u001b[36m\u001b[46m   \u001b[37m\u001b[47m   \u001b[m\n",
      "\u001b[33C\u001b[38;5;8m\u001b[48;5;8m   \u001b[38;5;9m\u001b[48;5;9m   \u001b[38;5;10m\u001b[48;5;10m   \u001b[38;5;11m\u001b[48;5;11m   \u001b[38;5;12m\u001b[48;5;12m   \u001b[38;5;13m\u001b[48;5;13m   \u001b[38;5;14m\u001b[48;5;14m   \u001b[38;5;15m\u001b[48;5;15m   \u001b[m\n",
      "\n",
      "\n",
      "\u001b[?25h\u001b[?7h"
     ]
    }
   ],
   "source": [
    "!neofetch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "import joblib\n",
    "import numpy as np\n",
    "\n",
    "from scipy.sparse import csr_matrix\n",
    "from scipy.io import savemat, loadmat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Case1: 乱数行列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.random.randn(256, 256, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### np.save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.45 ms ± 576 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.save(\"a.npy\", a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363 µs ± 10 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.load(\"a.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   1.5M 11 25 09:56 \u001b[01;31m\u001b[Ka.npy\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep a.npy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f a.npy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### np.savez_compressed\n",
    "\n",
    "圧縮あり、書き出しは遅めだが読み込みが速い"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68.5 ms ± 2.66 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.savez_compressed(\"a.npy\", a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65.6 µs ± 3.02 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.load(\"a.npy.npz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   1.4M 11 25 09:56 \u001b[01;31m\u001b[Ka.npy.npz\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep a.npy.npz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "取り出し方に癖がある"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['arr_0']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aa = np.load(\"a.npy.npz\")\n",
    "aa.files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.20239102,  0.37383372, -0.79941785],\n",
       "        [-0.85540574,  0.17240746, -0.98539357],\n",
       "        [-0.87391825, -0.86141655,  0.34941402],\n",
       "        ...,\n",
       "        [-0.51699032,  2.93163159,  0.36189887],\n",
       "        [-0.51422701, -0.19698399,  1.31475487],\n",
       "        [ 1.70792745, -0.2228165 , -0.41474589]],\n",
       "\n",
       "       [[ 1.18903922, -0.46300179, -0.49396092],\n",
       "        [ 0.40682345, -2.53154819, -1.53560004],\n",
       "        [-1.29830327, -1.27992439, -0.72188917],\n",
       "        ...,\n",
       "        [ 0.18440747, -0.32896796,  0.44594645],\n",
       "        [-1.42360919, -0.10762281, -2.2276768 ],\n",
       "        [ 1.70897435, -0.81933952, -0.20320236]],\n",
       "\n",
       "       [[ 0.25962003,  1.48201361, -1.22802066],\n",
       "        [-1.42737811,  0.61574337,  0.1009645 ],\n",
       "        [-0.25704913, -0.82313048,  0.82107737],\n",
       "        ...,\n",
       "        [ 0.27376284,  0.30428687, -1.10083257],\n",
       "        [-0.91431796, -0.38424368, -1.4240811 ],\n",
       "        [ 0.64929156, -0.40993174,  0.95160479]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[ 0.21310458,  0.364045  ,  0.840645  ],\n",
       "        [ 1.05534974,  0.07496168,  1.76842987],\n",
       "        [-0.02294783,  0.68710474,  0.96094913],\n",
       "        ...,\n",
       "        [-1.45985302, -0.35835899, -0.49365589],\n",
       "        [ 0.54136375,  0.4323057 ,  1.35250782],\n",
       "        [ 0.7536575 ,  1.32436637, -0.42235029]],\n",
       "\n",
       "       [[-0.37174953,  0.77438195,  1.12398299],\n",
       "        [-0.10213659, -0.8338731 , -0.7724271 ],\n",
       "        [-0.35257496,  1.22800204, -0.99073415],\n",
       "        ...,\n",
       "        [-0.62863125, -0.81999371,  0.82252486],\n",
       "        [-0.62148623,  0.09642251,  1.51298272],\n",
       "        [-0.16196264,  0.9259189 ,  0.78179996]],\n",
       "\n",
       "       [[ 0.72380544,  1.18419219, -1.72712512],\n",
       "        [ 0.89111329,  0.0299966 , -2.50459415],\n",
       "        [-0.56849416, -0.99600025, -1.77963208],\n",
       "        ...,\n",
       "        [-0.42582803, -0.04800713, -0.32263953],\n",
       "        [ 1.0303978 ,  0.98871888,  0.09722896],\n",
       "        [ 2.34839695,  1.02545279,  0.10599268]]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aa[\"arr_0\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f a.npy.npz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pickle\n",
    "\n",
    "圧縮なし、書き出しがはやい"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.58 ms ± 47.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"a.pkl\", \"wb\") as f:\n",
    "    pickle.dump(a, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157 µs ± 46.7 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"a.pkl\", \"rb\") as f:\n",
    "    pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   1.5M 11 25 09:56 \u001b[01;31m\u001b[Ka.pkl\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep a.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f a.pkl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### joblib\n",
    "\n",
    "圧縮あり、書き出しは`savez_compressed`並だが読み込みが遅いぞ？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63.8 ms ± 1.73 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"a.joblib\", \"wb\") as f:\n",
    "    joblib.dump(a, f, compress=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.67 ms ± 206 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"a.joblib\", \"rb\") as f:\n",
    "    joblib.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   1.4M 11 25 09:57 \u001b[01;31m\u001b[Ka.joblib\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep a.joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f a.joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### csr_matrix + savemat\n",
    "\n",
    "疎行列なら圧縮率が高い、読み込みが遅いが圧縮あり系の中だと圧縮は一番速いかも"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "二次元配列しか扱えないのでややめんどい"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.1 ms ± 400 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "mat0 = csr_matrix(a[:, :, 0])\n",
    "mat1 = csr_matrix(a[:, :, 1])\n",
    "mat2 = csr_matrix(a[:, :, 2])\n",
    "savemat(\"a.mat\", {\"mat0\": mat0, \"mat1\": mat1, \"mat2\": mat2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.58 ms ± 130 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "aa = loadmat(\"a.mat\")\n",
    "np.stack([\n",
    "    aa[\"mat0\"].toarray(), \n",
    "    aa[\"mat1\"].toarray(), \n",
    "    aa[\"mat2\"].toarray()\n",
    "], axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   2.3M 11 25 09:58 \u001b[01;31m\u001b[Ka.mat\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep a.mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f a.mat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Case2: 疎な行列\n",
    "\n",
    "セグメンテーション用のラベルなどを配列で保存する場合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = np.zeros([256, 256, 3])\n",
    "for _ in range(10):\n",
    "    ystart = np.random.randint(0, 256 - 20)\n",
    "    xstart = np.random.randint(0, 256 - 20)\n",
    "    color = np.random.randint(0, 255, [10, 10, 3])\n",
    "    b[ystart:ystart + 10, xstart:xstart + 10, :] = color"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### np.save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.43 ms ± 451 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.save(\"b.npy\", b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "420 µs ± 25.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.load(\"b.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   1.5M 11 25 10:00 \u001b[01;31m\u001b[Kb.npy\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep b.npy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f b.npy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### np.savez_compressed\n",
    "\n",
    "めっちゃ圧縮されてる、書き出しも速くなってる"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.16 ms ± 52.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.savez_compressed(\"b.npy\", b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70.3 µs ± 3.24 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.load(\"b.npy.npz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   9.5K 11 25 10:00 \u001b[01;31m\u001b[Kb.npy.npz\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep b.npy.npz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f b.npy.npz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.69 ms ± 45.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"b.pkl\", \"wb\") as f:\n",
    "    pickle.dump(b, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "132 µs ± 10.2 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"b.pkl\", \"rb\") as f:\n",
    "    pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff   1.5M 11 25 10:01 \u001b[01;31m\u001b[Kb.pkl\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep b.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f b.pkl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### joblib\n",
    "\n",
    "割と圧縮される。相変わらず読み込みが遅い"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.49 ms ± 231 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"b.joblib\", \"wb\") as f:\n",
    "    joblib.dump(b, f, compress=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.23 ms ± 158 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "with open(\"b.joblib\", \"rb\") as f:\n",
    "    joblib.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff    15K 11 25 10:02 \u001b[01;31m\u001b[Kb.joblib\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep b.joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### csr_matrix + savemat\n",
    "\n",
    "圧縮率は前の二つよりは低い。書き出しが速い。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.34 ms ± 256 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "mat0 = csr_matrix(b[:, :, 0])\n",
    "mat1 = csr_matrix(b[:, :, 1])\n",
    "mat2 = csr_matrix(b[:, :, 2])\n",
    "savemat(\"b.mat\", {\"mat0\": mat0, \"mat1\": mat1, \"mat2\": mat2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "583 µs ± 10.6 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "bb = loadmat(\"b.mat\")\n",
    "np.stack([\n",
    "    bb[\"mat0\"].toarray(), \n",
    "    bb[\"mat1\"].toarray(), \n",
    "    bb[\"mat2\"].toarray()\n",
    "], axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 hidehisa  staff    38K 11 25 10:04 \u001b[01;31m\u001b[Kb.mat\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!ls -lh | grep b.mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -f b.mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}