Bitshuffle/LZ4 compression speed for Eiger images

bitshuffle_LZ4_speed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Timing for decompressing and compressing bitshuffle-LZ4 data\n",
    "\n",
    "Processor: 2x Intel(R) Xeon(R) Gold 6134 CPU @ 3.20GHz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"HDF5_PLUGIN_PATH\"] = \"/usr/lib/x86_64-linux-gnu/hdf5/plugins/\"\n",
    "import h5py\n",
    "import bitshuffle, lz4, bitshuffle.h5\n",
    "from lz4 import block, frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HDF5 dataset \"data\": shape (50, 3269, 3110), type \"<u2\">\n"
     ]
    }
   ],
   "source": [
    "h = h5py.File(\"/tmp/200Hz_0.1dpf_1_data_000001.h5\")\n",
    "ds = h[\"entry/data/data\"]\n",
    "print(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size of the raw data: 1016659000\n"
     ]
    }
   ],
   "source": [
    "raw_size = ds.size * 2\n",
    "print(\"size of the raw data:\", raw_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17.4 s, sys: 2.45 s, total: 19.8 s\n",
      "Wall time: 1.33 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#read all images into a list of 2D array\n",
    "raw_images = [i for i in ds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of images: 50\n",
      "one image: [[    0     0     0 ...     0     0 65535]\n",
      " [    0     0     0 ...     0     0     0]\n",
      " [    0     0     0 ...     0     0     0]\n",
      " ...\n",
      " [    0     0     0 ...     0     0     0]\n",
      " [    0     1     0 ...     0     0     0]\n",
      " [    0 65535     0 ...     0     0     0]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of images:\", len(raw_images))\n",
    "print(\"one image:\", raw_images[25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18.8 s, sys: 734 ms, total: 19.5 s\n",
      "Wall time: 1.32 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#compress every frame independently:\n",
    "compressed = []\n",
    "for idx,data in enumerate(raw_images):\n",
    "    compressed.append(frame.compress(bitshuffle.bitshuffle(data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compressed size: 104798605\n",
      "Compression ratio:  9.70107378814823\n"
     ]
    }
   ],
   "source": [
    "compressed_size = sum(len(i) for i in compressed)\n",
    "print(\"Compressed size:\", compressed_size)\n",
    "print(\"Compression ratio: \",raw_size/compressed_size) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compression time per frame: 26.400ms\n"
     ]
    }
   ],
   "source": [
    "print(\"compression time per frame: %.3fms\"%(1000*1.32/50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}