Blosc.jl-based summation benchmark

blosc-sum.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Blosc, BenchmarkTools\n",
    "Blosc.set_num_threads(4)\n",
    "Blosc.set_compressor(\"blosclz\") # blosclz is the default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "clevel = 6\n",
    "N = 100_000_000\n",
    "csize = 500_000\n",
    "nchunks = N ÷ csize\n",
    "a = [Int64(n) for n in 0:N-1];\n",
    "a_chunked = [a[(i-1)*csize+1:i*csize] for i in 1:nchunks];\n",
    "a_uncompressed = [Blosc.compress(a[(i-1)*csize+1:i*csize], level=0) for i in 1:nchunks];\n",
    "a_compressed = [Blosc.compress(a[(i-1)*csize+1:i*csize], level=clevel) for i in 1:nchunks];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sum_blocks (generic function with 1 method)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_cblocks(compressed, buf) = sum(sum(Blosc.decompress!(buf, c)) for c in compressed)\n",
    "sum_blocks(chunks) = sum(sum(u) for u in chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buff = copy(a[1:csize]) \n",
    "sum(a) == sum_blocks(a_chunked) == sum_cblocks(a_compressed, buff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  68.780 ms (0 allocations: 0 bytes)\n",
      "  68.887 ms (3 allocations: 48 bytes)\n",
      "  131.108 ms (604 allocations: 9.45 KiB)\n",
      "  286.563 ms (604 allocations: 9.45 KiB)\n"
     ]
    }
   ],
   "source": [
    "# benchmark the functions … use $ to interpolate global variables\n",
    "# to avoid an artificial slowdown for benchmarking globals.\n",
    "@btime sum($a) \n",
    "@btime sum_blocks($a_chunked) \n",
    "@btime sum_cblocks($a_uncompressed, $buff) \n",
    "@btime sum_cblocks($a_compressed, $buff);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function sum_cblocks2(compressed, buf)\n",
    "    s = 0\n",
    "    for c in compressed\n",
    "        s += sum(Blosc.decompress!(buf, c))\n",
    "    end\n",
    "    return s\n",
    "end\n",
    "sum(a) == sum_cblocks2(a_compressed, buff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  283.613 ms (600 allocations: 9.38 KiB)\n"
     ]
    }
   ],
   "source": [
    "@btime sum_cblocks2($a_compressed, $buff);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function sum_cblocks3(compressed, buf)\n",
    "    s = 0\n",
    "    for c in compressed\n",
    "        n = Blosc.blosc_decompress(c, buf, sizeof(buf)) ÷ sizeof(eltype(buf))\n",
    "        @simd for i = 1:n\n",
    "            @inbounds s += buf[i]\n",
    "        end\n",
    "    end\n",
    "    return s\n",
    "end\n",
    "sum(a) == sum_cblocks3(a_compressed, buff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  282.500 ms (0 allocations: 0 bytes)\n"
     ]
    }
   ],
   "source": [
    "@btime sum_cblocks3($a_compressed, $buff);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151.48800991337538"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sizeof(a) / sum(sizeof, a_compressed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.0",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}