Quantization error simulation of SiLU (Swish) activation

quantization_error_simulation_of_silu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fake_quantize(x):\n",
    "    min_ = np.percentile(x, 0)\n",
    "    max_ = np.percentile(x, 100)\n",
    "    x_clip = np.clip(x, a_min=min_, a_max=max_)\n",
    "    x_int8 = ((x_clip - min_) / (max_ - min_) * 255).astype(np.uint8)\n",
    "    return (x_int8.astype(float) / 255) * (max_ - min_) + min_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(x, quantize=False):\n",
    "    quantize_fn = fake_quantize if quantize else (lambda x: x)\n",
    "    h = quantize_fn(x)  # quantize input\n",
    "    h = h.clip(min=0)  # relu operation\n",
    "    return quantize_fn(h)  # quantize output\n",
    "\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "\n",
    "def silu(x, quantize=False):\n",
    "    quantize_fn = fake_quantize if quantize else (lambda x: x)\n",
    "    h1 = quantize_fn(x)\n",
    "    h2 = quantize_fn(sigmoid(h1))\n",
    "    return quantize_fn(h1 * h2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_sqnr(x, q):\n",
    "    return 20 * np.log10(np.linalg.norm(x) / np.linalg.norm(x - q))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ReLU SQNR[dB]: 34.1037243352085\n",
      "SiLU SQNR[dB]: 31.78201463229342\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGxCAYAAABIjE2TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5EElEQVR4nO3deVxVZeLH8S+LIAr3EopcSFwqN8olaURsGq1INDMdabEcl36m1aClpjk2btlMmC1mpVlNiePUaM5PrWwxI3cJlaTcNUOhkcUlwCUB4fz+8MX5eQOV5SIH/Lxfr/vSe85zznmee87lfu9znnOum2EYhgAAACzEvaYrAAAA8FsEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDmeNV2ByiguLtaRI0fk5+cnNze3mq4OAAAoB8MwdPLkSYWEhMjd/dJ9JLUyoBw5ckShoaE1XQ0AAFAJ6enpatq06SXL1MqA4ufnJ+l8A202Ww3XBgAAlEdeXp5CQ0PNz/FLqZUBpeS0js1mI6AAAFDLlGd4BoNkAQCA5RBQAACA5RBQAACA5dTKMSgAALhacXGxCgoKaroatVq9evXk4eHhknURUAAAV72CggKlpqaquLi4pqtS6/n7+8vhcFT5PmUEFADAVc0wDGVkZMjDw0OhoaGXvYEYymYYhs6cOaPs7GxJUnBwcJXWR0ABAFzVzp07pzNnzigkJEQNGjSo6erUaj4+PpKk7OxsNWnSpEqne4iJAICrWlFRkSTJy8urhmtSN5SEvMLCwiqth4ACAIDKd/MwXJ6rXkcCCgAAsBwCCgAAsBwGyQIAUIZJy3Zc0e3FDWhfofLDhg3TwoULJUmenp5q2rSp7r//fs2YMUP169e/7PKHDh1Sy5YttX37dnXq1Mlp3tq1a3X77bfrl19+kb+/v9O8Fi1aaMyYMRozZkyF6ltRBBQAAGqpXr16acGCBSosLFRycrKGDh0qNzc3vfjiizVdtSrjFA8AALWUt7e3HA6HQkND1b9/f0VFRWn16tWSzt8ZNy4uTi1btpSPj486duyo//znPzVc4/KjBwUAgDpg586d2rx5s5o3by5JiouL07/+9S/Nnz9frVq10vr16/WnP/1JgYGB6t69ew3X9vIIKGUoz3nHip4rBADA1VauXClfX1+dO3dO+fn5cnd315tvvqn8/Hy98MIL+vrrrxUZGSlJuu6667Rx40a9/fbbBBQAAFB9br/9dr311ls6ffq0Zs+eLU9PT8XExGjXrl06c+aM7rrrLqfyBQUFuvnmm2uothVDQAEAoJZq2LChbrjhBknS+++/r44dO+q9997TTTfdJEn67LPPdO211zot4+3tfdn12mw2SVJubm6pq3hycnJkt9tdUPtLI6AAAFAHuLu769lnn9W4ceO0f/9+eXt7Ky0trVKnc1q1aiV3d3clJyebY1ok6aefflJubq5at27tyqqXqUJX8UyfPl1ubm5Oj7Zt25rzz549q9jYWDVq1Ei+vr6KiYlRVlaW0zrS0tLUp08fNWjQQE2aNNGECRN07tw517QGAICr2P333y8PDw+9/fbbGj9+vMaOHauFCxfq4MGD+u677/TGG2+Y904psW/fPqWkpDg96tevr0cffVRPP/20PvnkE6Wmpmr9+vUaNGiQunbtqm7dulV7Wyrcg3LjjTfq66+//v8VeP7/KsaOHavPPvtMS5culd1u16hRozRgwABt2rRJ0vkfZOrTp48cDoc2b96sjIwMDRkyRPXq1dMLL7zgguYAAOAatfFiCE9PT40aNUqzZs1SamqqAgMDFRcXp59++kn+/v7q3Lmznn32WadlBg4cWGo96enpmjNnjmbOnKmJEyfq8OHDcjgcuuuuu/T3v//9ivxukZthGEZ5C0+fPl0rVqxQSkpKqXm5ubkKDAzUhx9+qPvuu0+StHfvXrVr106JiYnq2rWrvvjiC91zzz06cuSIgoKCJEnz58/XxIkTdfTo0XL/kmReXp7sdrtyc3PN82SuxFU8AHD1OHv2rFJTU9WyZcty3YEVl3ap17Min98VvlHbgQMHFBISouuuu06DBg1SWlqaJCk5OVmFhYWKiooyy7Zt21bNmjVTYmKiJCkxMVHt27c3w4kkRUdHKy8vT7t27apoVQAAQB1VoVM8ERERio+PV5s2bZSRkaHnnntOt912m3bu3KnMzEx5eXmVGu0bFBSkzMxMSVJmZqZTOCmZXzLvYvLz85Wfn28+z8vLq0i1AQBALVOhgNK7d2/z/x06dFBERISaN2+ujz76SD4+Pi6vXIm4uDg999xz1bZ+AABgLVX6LR5/f3+1bt1aP/74oxwOhwoKCpSTk+NUJisrSw6HQ5LkcDhKXdVT8rykTFkmTZqk3Nxc85Genl6VagMAAIurUkA5deqUDh48qODgYIWHh6tevXpKSEgw5+/bt09paWnmbXYjIyO1Y8cOZWdnm2VWr14tm82msLCwi27H29tbNpvN6QEAAOquCp3iGT9+vPr27avmzZvryJEjmjZtmjw8PPTQQw/Jbrdr+PDhGjdunAICAmSz2TR69GhFRkaqa9eukqSePXsqLCxMgwcP1qxZs5SZmanJkycrNja2XHe2AwAAV4cKBZSff/5ZDz30kI4fP67AwED9/ve/17fffqvAwEBJ0uzZs+Xu7q6YmBjl5+crOjpa8+bNM5f38PDQypUr9cQTTygyMlINGzbU0KFDNWPGDNe2CgAA1GoVCiiLFy++5Pz69etr7ty5mjt37kXLNG/eXJ9//nlFNgsAAK4yVRqDAgAArMvNzU0rVqyo6WpUCgEFAIBa6ujRo3riiSfUrFkzeXt7y+FwKDo62vyJmYyMDKdbhFwssBw6dEhubm5l3im+R48eGjNmTDW14OL4NWMAAMry6VNXdnt951R4kZiYGBUUFGjhwoW67rrrlJWVpYSEBB0/flzSpW/hYXUEFAAAaqGcnBxt2LBBa9euVffu3SWdH+fZpUsXs4ybm5uWL1+u/v3711AtK49TPAAA1EK+vr7y9fXVihUrnH4Opq4goAAAUAt5enoqPj5eCxculL+/v2699VY9++yz+uGHH2q6ai5BQAEAoJaKiYnRkSNH9Mknn6hXr15au3atOnfurPj4+JquWpURUAAAqMXq16+vu+66S1OmTNHmzZs1bNgwTZs2rULrKPkJmdzc3FLzcnJyZLfbXVLXiiCgAABQh4SFhen06dMVWiYgIECNGzdWcnKy0/S8vDz9+OOPat26tSurWC5cxQMAQC10/Phx3X///fqf//kfdejQQX5+ftq2bZtmzZqlfv36XXS51NTUUvc7adWqlcaNG6cXXnhBQUFB6tq1q44fP67nn39egYGBGjBgQDW3pjQCCgAAtZCvr68iIiI0e/ZsHTx4UIWFhQoNDdWIESP07LPPXnS5cePGlZq2YcMGPfPMM/L19dWLL76ogwcPKiAgQLfeeqvWrFkjHx+f6mxKmdwMwzCu+FarKC8vT3a7Xbm5ueZ5M1eatGzHZcvEDWjv8u0CAK68s2fPKjU1VS1btlT9+vVrujq13qVez4p8fjMGBQAAWA4BBQAAWA4BBQAAWA4BBQAAWA4BBQAAWA4BBQAASbXwolZLctXrSEABAFzVPDw8JEkFBQU1XJO64cyZM5KkevXqVWk93KgNAHBV8/T0VIMGDXT06FHVq1dP7u58d68MwzB05swZZWdny9/f3wx+lUVAAQBc1dzc3BQcHKzU1FQdPny4pqtT6/n7+8vhcFR5PQQUAMBVz8vLS61ateI0TxXVq1evyj0nJQgoAABIcnd351b3FsKJNgAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDlVCigzZ86Um5ubxowZY047e/asYmNj1ahRI/n6+iomJkZZWVlOy6WlpalPnz5q0KCBmjRpogkTJujcuXNVqQoAAKhDKh1Qtm7dqrffflsdOnRwmj527Fh9+umnWrp0qdatW6cjR45owIAB5vyioiL16dNHBQUF2rx5sxYuXKj4+HhNnTq18q0AAAB1SqUCyqlTpzRo0CC9++67uuaaa8zpubm5eu+99/Tqq6/qjjvuUHh4uBYsWKDNmzfr22+/lSR99dVX2r17t/71r3+pU6dO6t27t55//nnNnTtXBQUFrmkVAACo1SoVUGJjY9WnTx9FRUU5TU9OTlZhYaHT9LZt26pZs2ZKTEyUJCUmJqp9+/YKCgoyy0RHRysvL0+7du0qc3v5+fnKy8tzegAAgLrLs6ILLF68WN999522bt1aal5mZqa8vLzk7+/vND0oKEiZmZlmmQvDScn8knlliYuL03PPPVfRqgIAgFqqQj0o6enpeuqpp/TBBx+ofv361VWnUiZNmqTc3FzzkZ6efsW2DQAArrwKBZTk5GRlZ2erc+fO8vT0lKenp9atW6fXX39dnp6eCgoKUkFBgXJycpyWy8rKksPhkCQ5HI5SV/WUPC8p81ve3t6y2WxODwAAUHdVKKDceeed2rFjh1JSUszHLbfcokGDBpn/r1evnhISEsxl9u3bp7S0NEVGRkqSIiMjtWPHDmVnZ5tlVq9eLZvNprCwMBc1CwAA1GYVGoPi5+enm266yWlaw4YN1ahRI3P68OHDNW7cOAUEBMhms2n06NGKjIxU165dJUk9e/ZUWFiYBg8erFmzZikzM1OTJ09WbGysvL29XdQsAABQm1V4kOzlzJ49W+7u7oqJiVF+fr6io6M1b948c76Hh4dWrlypJ554QpGRkWrYsKGGDh2qGTNmuLoqAACglnIzDMOo6UpUVF5enux2u3Jzc6tlPMqkZTsuWyZuQHuXbxcAgLqsIp/f/BYPAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwnAoFlLfeeksdOnSQzWaTzWZTZGSkvvjiC3P+2bNnFRsbq0aNGsnX11cxMTHKyspyWkdaWpr69OmjBg0aqEmTJpowYYLOnTvnmtYAAIA6oUIBpWnTppo5c6aSk5O1bds23XHHHerXr5927dolSRo7dqw+/fRTLV26VOvWrdORI0c0YMAAc/mioiL16dNHBQUF2rx5sxYuXKj4+HhNnTrVta0CAAC1mpthGEZVVhAQEKCXXnpJ9913nwIDA/Xhhx/qvvvukyTt3btX7dq1U2Jiorp27aovvvhC99xzj44cOaKgoCBJ0vz58zVx4kQdPXpUXl5e5dpmXl6e7Ha7cnNzZbPZqlL9Mk1atuOyZeIGtHf5dgEAqMsq8vld6TEoRUVFWrx4sU6fPq3IyEglJyersLBQUVFRZpm2bduqWbNmSkxMlCQlJiaqffv2ZjiRpOjoaOXl5Zm9MAAAAJ4VXWDHjh2KjIzU2bNn5evrq+XLlyssLEwpKSny8vKSv7+/U/mgoCBlZmZKkjIzM53CScn8knkXk5+fr/z8fPN5Xl5eRasNAABqkQr3oLRp00YpKSlKSkrSE088oaFDh2r37t3VUTdTXFyc7Ha7+QgNDa3W7QEAgJpV4YDi5eWlG264QeHh4YqLi1PHjh01Z84cORwOFRQUKCcnx6l8VlaWHA6HJMnhcJS6qqfkeUmZskyaNEm5ubnmIz09vaLVBgAAtUiV74NSXFys/Px8hYeHq169ekpISDDn7du3T2lpaYqMjJQkRUZGaseOHcrOzjbLrF69WjabTWFhYRfdhre3t3lpc8kDAADUXRUagzJp0iT17t1bzZo108mTJ/Xhhx9q7dq1WrVqlex2u4YPH65x48YpICBANptNo0ePVmRkpLp27SpJ6tmzp8LCwjR48GDNmjVLmZmZmjx5smJjY+Xt7V0tDQQAALVPhQJKdna2hgwZooyMDNntdnXo0EGrVq3SXXfdJUmaPXu23N3dFRMTo/z8fEVHR2vevHnm8h4eHlq5cqWeeOIJRUZGqmHDhho6dKhmzJjh2lYBAIBarcr3QakJ3AcFAIDa54rcBwUAAKC6EFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlVCigxMXF6Xe/+538/PzUpEkT9e/fX/v27XMqc/bsWcXGxqpRo0by9fVVTEyMsrKynMqkpaWpT58+atCggZo0aaIJEybo3LlzVW8NAACoEyoUUNatW6fY2Fh9++23Wr16tQoLC9WzZ0+dPn3aLDN27Fh9+umnWrp0qdatW6cjR45owIAB5vyioiL16dNHBQUF2rx5sxYuXKj4+HhNnTrVda0CAAC1mpthGEZlFz569KiaNGmidevW6Q9/+INyc3MVGBioDz/8UPfdd58kae/evWrXrp0SExPVtWtXffHFF7rnnnt05MgRBQUFSZLmz5+viRMn6ujRo/Ly8rrsdvPy8mS325WbmyubzVbZ6l/UpGU7LlsmbkB7l28XAIC6rCKf31Uag5KbmytJCggIkCQlJyersLBQUVFRZpm2bduqWbNmSkxMlCQlJiaqffv2ZjiRpOjoaOXl5WnXrl1lbic/P195eXlODwAAUHdVOqAUFxdrzJgxuvXWW3XTTTdJkjIzM+Xl5SV/f3+nskFBQcrMzDTLXBhOSuaXzCtLXFyc7Ha7+QgNDa1stQEAQC1Q6YASGxurnTt3avHixa6sT5kmTZqk3Nxc85Genl7t2wQAADXHszILjRo1SitXrtT69evVtGlTc7rD4VBBQYFycnKcelGysrLkcDjMMlu2bHFaX8lVPiVlfsvb21ve3t6VqSoAAKiFKtSDYhiGRo0apeXLl+ubb75Ry5YtneaHh4erXr16SkhIMKft27dPaWlpioyMlCRFRkZqx44dys7ONsusXr1aNptNYWFhVWkLAACoIyrUgxIbG6sPP/xQH3/8sfz8/MwxI3a7XT4+PrLb7Ro+fLjGjRungIAA2Ww2jR49WpGRkerataskqWfPngoLC9PgwYM1a9YsZWZmavLkyYqNjaWXBAAASKpgQHnrrbckST169HCavmDBAg0bNkySNHv2bLm7uysmJkb5+fmKjo7WvHnzzLIeHh5auXKlnnjiCUVGRqphw4YaOnSoZsyYUbWWAACAOqNK90GpKdwHBQCA2ueK3QcFAACgOhBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5VQ4oKxfv159+/ZVSEiI3NzctGLFCqf5hmFo6tSpCg4Olo+Pj6KionTgwAGnMidOnNCgQYNks9nk7++v4cOH69SpU1VqCAAAqDsqHFBOnz6tjh07au7cuWXOnzVrll5//XXNnz9fSUlJatiwoaKjo3X27FmzzKBBg7Rr1y6tXr1aK1eu1Pr16zVy5MjKtwIAANQpnhVdoHfv3urdu3eZ8wzD0GuvvabJkyerX79+kqR//vOfCgoK0ooVKzRw4EDt2bNHX375pbZu3apbbrlFkvTGG2/o7rvv1ssvv6yQkJAqNAcAANQFLh2DkpqaqszMTEVFRZnT7Ha7IiIilJiYKElKTEyUv7+/GU4kKSoqSu7u7kpKSnJldQAAQC1V4R6US8nMzJQkBQUFOU0PCgoy52VmZqpJkybOlfD0VEBAgFnmt/Lz85Wfn28+z8vLc2W1AQCAxdSKq3ji4uJkt9vNR2hoaE1XCQAAVCOXBhSHwyFJysrKcpqelZVlznM4HMrOznaaf+7cOZ04ccIs81uTJk1Sbm6u+UhPT3dltQEAgMW4NKC0bNlSDodDCQkJ5rS8vDwlJSUpMjJSkhQZGamcnBwlJyebZb755hsVFxcrIiKizPV6e3vLZrM5PQAAQN1V4TEop06d0o8//mg+T01NVUpKigICAtSsWTONGTNGf/vb39SqVSu1bNlSU6ZMUUhIiPr37y9JateunXr16qURI0Zo/vz5Kiws1KhRozRw4ECu4AEAAJIqEVC2bdum22+/3Xw+btw4SdLQoUMVHx+vZ555RqdPn9bIkSOVk5Oj3//+9/ryyy9Vv359c5kPPvhAo0aN0p133il3d3fFxMTo9ddfd0FzAABAXeBmGIZR05WoqLy8PNntduXm5lbL6Z5Jy3ZctkzcgPYu3y4AAHVZRT6/a8VVPAAA4OpCQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJZDQAEAAJbjWdMVsKL+P88qR6lF1V6PGvPpU5cv03dO9dcDAHDVIqDg6kL4AoBagVM8AADAcggoAADAcggoAADAchiDgrqjPONLrvT2GM8CAJVCDwoAALAcelBQOa7sPbjSPR8AAMujBwUAAFgOPSioPvSMAAAqiYBSnRhECQBApRBQgN+i5wcAahwBpTYo7wdmeXpj+PAFPXsAagEGyQIAAMuhBwWoTleyt4LeMQB1CD0oAADAcuhBqUv4Bl13sW8BXGXoQQEAAJZDD0pl8Y0WrlJLj6VJy3ZctkzcgPZXoCYA6iICSk2rpR9OcJ2k1BOXLRPRMuAK1OQC5TouH632agC4ehFQgFrAkiEGAKoRY1AAAIDl0IMCoFL6/zyrHKUWVXs9ANRNBBSgkqx22qU89Skvl9W7HGNZJhWWbyxLuQbccht/oM4goOCqYrVQAWsqzxVK5cWVTEDlEFBQKZb8tu4irmzb1a5cr2XT6q/HhVwZPgBUHwIKSuED2nV4LS+vfGNZJH165YJseeu0oukz1VyTasKpMNQCBJSrTG39wKyt9Qau5A3tyts7FFfPJZsDqhUBBUAptTUQWvGUEoDKIaCgxtXWD0NcWa46Tsp9SslF6yrPaSDGxQClEVAA4CpTrqvZyrEefo8J1alGA8rcuXP10ksvKTMzUx07dtQbb7yhLl261GSVLKm83xytdjUMANf22FhNedqW9Prl1xPxZPlu6Jf0+mCXrQvWV2MBZcmSJRo3bpzmz5+viIgIvfbaa4qOjta+ffvUpEmTmqqWS13pUxecKgHgKuUJA0B1cjMMw6iJDUdEROh3v/ud3nzzTUlScXGxQkNDNXr0aP3lL3+55LJ5eXmy2+3Kzc2VzWZzed3KldLL0VtBYACAK8tlPclcZl0tKvL5XSM9KAUFBUpOTtakSZPMae7u7oqKilJiYmJNVKnCCB8AYD1X9G7R5bmfjETYqaQaCSjHjh1TUVGRgoKCnKYHBQVp7969pcrn5+crPz/ffJ6bmyvpfBKrDqfPFlTLegEANS/vTP7lCy153IUbLMdn1RfluOlf79o/nqnkc7s8J29qxVU8cXFxeu6550pNDw0NrYHaAABQEW9bbD017+TJk7Lb7ZcsUyMBpXHjxvLw8FBWVpbT9KysLDkcjlLlJ02apHHjxpnPi4uLdeLECTVq1Ehubm4urVteXp5CQ0OVnp5eLeNbUDHsD2thf1gL+8Na2B+XZxiGTp48qZCQkMuWrZGA4uXlpfDwcCUkJKh///6SzoeOhIQEjRo1qlR5b29veXt7O03z9/ev1jrabDYOMAthf1gL+8Na2B/Wwv64tMv1nJSosVM848aN09ChQ3XLLbeoS5cueu2113T69Gk98sgjNVUlAABgETUWUB588EEdPXpUU6dOVWZmpjp16qQvv/yy1MBZAABw9anRQbKjRo0q85ROTfL29ta0adNKnVJCzWB/WAv7w1rYH9bC/nCtGrtRGwAAwMW413QFAAAAfouAAgAALIeAAgAALKfOB5S5c+eqRYsWql+/viIiIrRly5ZLll+6dKnatm2r+vXrq3379vr888+d5huGoalTpyo4OFg+Pj6KiorSgQMHqrMJdYqr98eyZcvUs2dP86Z9KSkp1Vj7usmV+6SwsFATJ05U+/bt1bBhQ4WEhGjIkCE6cuRIdTejznD1e2T69Olq27atGjZsqGuuuUZRUVFKSkqqzibUKa7eHxd6/PHH5ebmptdee83Fta4jjDps8eLFhpeXl/H+++8bu3btMkaMGGH4+/sbWVlZZZbftGmT4eHhYcyaNcvYvXu3MXnyZKNevXrGjh07zDIzZ8407Ha7sWLFCuP777837r33XqNly5bGr7/+eqWaVWtVx/745z//aTz33HPGu+++a0gytm/ffoVaUze4ep/k5OQYUVFRxpIlS4y9e/caiYmJRpcuXYzw8PAr2axaqzreIx988IGxevVq4+DBg8bOnTuN4cOHGzabzcjOzr5Szaq1qmN/lFi2bJnRsWNHIyQkxJg9e3Y1t6R2qtMBpUuXLkZsbKz5vKioyAgJCTHi4uLKLP/AAw8Yffr0cZoWERFhPPbYY4ZhGEZxcbHhcDiMl156yZyfk5NjeHt7G//+97+roQV1i6v3x4VSU1MJKJVQnfukxJYtWwxJxuHDh11T6TrsSuyP3NxcQ5Lx9ddfu6bSdVh17Y+ff/7ZuPbaa42dO3cazZs3J6BcRJ09xVNQUKDk5GRFRUWZ09zd3RUVFaXExMQyl0lMTHQqL0nR0dFm+dTUVGVmZjqVsdvtioiIuOg6cV517A9UzZXaJ7m5uXJzc6v2n6eo7a7E/igoKNA777wju92ujh07uq7ydVB17Y/i4mINHjxYEyZM0I033lg9la8j6mxAOXbsmIqKikrdmTYoKEiZmZllLpOZmXnJ8iX/VmSdOK869geq5krsk7Nnz2rixIl66KGH+G2Sy6jO/bFy5Ur5+vqqfv36mj17tlavXq3GjRu7tgF1THXtjxdffFGenp568sknXV/pOqbOBhQANauwsFAPPPCADMPQW2+9VdPVuardfvvtSklJ0ebNm9WrVy898MADys7OrulqXXWSk5M1Z84cxcfHy83NraarY3l1NqA0btxYHh4eysrKcpqelZUlh8NR5jIOh+OS5Uv+rcg6cV517A9UTXXuk5JwcvjwYa1evZrek3Kozv3RsGFD3XDDDeratavee+89eXp66r333nNtA+qY6tgfGzZsUHZ2tpo1ayZPT095enrq8OHDevrpp9WiRYtqaUdtVmcDipeXl8LDw5WQkGBOKy4uVkJCgiIjI8tcJjIy0qm8JK1evdos37JlSzkcDqcyeXl5SkpKuug6cV517A9UTXXtk5JwcuDAAX399ddq1KhR9TSgjrmS75Hi4mLl5+dXvdJ1WHXsj8GDB+uHH35QSkqK+QgJCdGECRO0atWq6mtMbVXTo3Sr0+LFiw1vb28jPj7e2L17tzFy5EjD39/fyMzMNAzDMAYPHmz85S9/Mctv2rTJ8PT0NF5++WVjz549xrRp08q8zNjf39/4+OOPjR9++MHo168flxmXU3Xsj+PHjxvbt283PvvsM0OSsXjxYmP79u1GRkbGFW9fbeTqfVJQUGDce++9RtOmTY2UlBQjIyPDfOTn59dIG2sTV++PU6dOGZMmTTISExONQ4cOGdu2bTMeeeQRw9vb29i5c2eNtLE2qY6/Wb/FVTwXV6cDimEYxhtvvGE0a9bM8PLyMrp06WJ8++235rzu3bsbQ4cOdSr/0UcfGa1btza8vLyMG2+80fjss8+c5hcXFxtTpkwxgoKCDG9vb+POO+809u3bdyWaUie4en8sWLDAkFTqMW3atCvQmrrBlfuk5HLvsh5r1qy5Qi2q3Vy5P3799Vfjj3/8oxESEmJ4eXkZwcHBxr333mts2bLlSjWn1nP136zfIqBcHL9mDAAALKfOjkEBAAC1FwEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFtdbatWvl5uamnJycat1OixYt9Nprr1XrNuoSV+2XKVOmaOTIkeZzwzA0cuRIBQQEyM3NTSkpKerRo4fGjBlTtQpXs+nTpysoKEhubm5asWJFuZapSNm6Jj4+Xv7+/tW6jWHDhql///7m84EDB+qVV16p1m2i4ggoqBXK+iDq1q2bMjIyZLfbXbKNi/1h3Lp1q9MHJapfZmam5syZo7/+9a/mtC+//FLx8fFauXKlMjIydNNNN1XLtl0Zevbs2aPnnntOb7/9tjIyMtS7d2+XrLeuKCv8P/jgg9q/f/8VrcfkyZP197//Xbm5uVd0u7g0AgpqLS8vLzkcDrm5uVXrdgIDA9WgQYNq3UZlFBQUlJpWVFSk4uLiCq+rsstVl3/84x/q1q2bmjdvbk47ePCggoOD1a1bNzkcDnl6etZgDcvn4MGDkqR+/frJ4XDI29u7hmtkfT4+PmrSpMkV3eZNN92k66+/Xv/617+u6HZxaQQUXNTp06c1ZMgQ+fr6Kjg4WK+88kqpb5dldUX7+/srPj7efD5x4kS1bt1aDRo00HXXXacpU6aosLDQnD99+nR16tRJixYtUosWLWS32zVw4ECdPHlS0vnu2HXr1mnOnDlyc3OTm5ubDh06VOpUQo8ePcz5Fz4OHTokSXr11VfVvn17NWzYUKGhofrzn/+sU6dOSTp/WuKRRx5Rbm6uudz06dMllf6Wl5aWpn79+snX11c2m00PPPCAsrKyyt2ei9m4caNuu+02+fj4KDQ0VE8++aROnz5tzm/RooWef/55DRkyRDabTSNHjjR7fT755BOFhYXJ29tbaWlp+uWXXzRkyBBdc801atCggXr37q0DBw6Y67rYcuWxaNEi3XLLLfLz85PD4dDDDz+s7OzsUuU2bdqkDh06qH79+uratat27txpzjt8+LD69u2ra665Rg0bNtSNN96ozz//3Jy/ePFi9e3b13w+bNgwjR49WmlpaXJzc1OLFi3KrNvl2n38+HE99NBDuvbaa9WgQQO1b99e//73v522U9axVhnTp0832+Du7m4G6a1bt+quu+5S48aNZbfb1b17d3333XeXXNe0adMUHBysH374QdLlj5VLyc7OVt++feXj46OWLVvqgw8+cDrGDx06ZJ5CK5GTkyM3NzetXbtW0vlAO3z4cLVs2VI+Pj5q06aN5syZ47SdktMoL7/8soKDg9WoUSPFxsaa7/0ePXro8OHDGjt2rPlaS6V7Mlu0aFHm+7pEenq6HnjgAfn7+ysgIED9+vVz2mdFRUUaN26c/P391ahRIz3zzDMq6yfo+vbtq8WLF5frNcSVQUDBRU2YMEHr1q3Txx9/rK+++kpr16697B/Ssvj5+Sk+Pl67d+/WnDlz9O6772r27NlOZQ4ePKgVK1Zo5cqVWrlypdatW6eZM2dKkubMmaPIyEiNGDFCGRkZysjIUGhoaKntLFu2zJyfkZGhAQMGqE2bNgoKCpJ0/kPi9ddf165du7Rw4UJ98803euaZZySdP1302muvyWazmcuPHz++1DaKi4vVr18/nThxQuvWrdPq1av1008/6cEHHyx3e8py8OBB9erVSzExMfrhhx+0ZMkSbdy4UaNGjXIq9/LLL6tjx47avn27pkyZIkk6c+aMXnzxRf3jH//Qrl271KRJEw0bNkzbtm3TJ598osTERBmGobvvvtspGJa13PTp0y/64V+isLBQzz//vL7//nutWLFChw4d0rBhw0qVmzBhgl555RVt3bpVgYGB6tu3r7n92NhY5efna/369dqxY4defPFF+fr6SpJOnDih3bt365ZbbjHXNWfOHM2YMUNNmzZVRkaGtm7dWmbdLtfus2fPKjw8XJ999pl27typkSNHavDgwdqyZYu5nfIca+Uxfvx4LViwQJLMdUnSyZMnNXToUG3cuFHffvutWrVqpbvvvrvMAGsYhkaPHq1//vOf2rBhgzp06FDuY+Vihg0bpvT0dK1Zs0b/+c9/NG/evDID5qUUFxeradOmWrp0qXbv3q2pU6fq2Wef1UcffeRUbs2aNTp48KDWrFmjhQsXKj4+3vzysmzZMjVt2lQzZsxwen1+a+vWreb8n3/+WV27dtVtt90m6fyxGB0dLT8/P23YsEGbNm2Sr6+vevXqZfYwvvLKK4qPj9f777+vjRs36sSJE1q+fHmp7XTp0kVbtmxRfn5+hV4LVKMa/CVlWNjJkycNLy8v46OPPjKnHT9+3PDx8TGeeuopc5okY/ny5U7L2u12Y8GCBRdd90svvWSEh4ebz6dNm2Y0aNDAyMvLM6dNmDDBiIiIMJ93797dabuGYRhr1qwxJBm//PJLqW28+uqrhr+/v7Fv376L1mPp0qVGo0aNzOcLFiww7HZ7qXIX/hz6V199ZXh4eBhpaWnm/F27dhmSzJ+wL097fmv48OHGyJEjnaZt2LDBcHd3N3799VezHv3793cqs2DBAkOSkZKSYk7bv3+/IcnYtGmTOe3YsWOGj4+PuT/LWs4wzv+0/B133HHRepZl69athiTj5MmThmH8/35ZvHixWabk2FmyZIlhGIbRvn17Y/r06WWub/v27YYkp9fYMAxj9uzZRvPmzZ2mXXhclKfdZenTp4/x9NNPl7nOqlq+fLlxuT+zRUVFhp+fn/Hpp5+a0yQZS5cuNR5++GGjXbt2xs8//2zOK8+xcjH79u1zOlYNwzD27NljSDKP8dTUVEOSsX37drPML7/8Ykgy1qxZc9F1x8bGGjExMebzoUOHGs2bNzfOnTtnTrv//vuNBx980Hx+4XurxMXeh4ZhGE8++aTRvHlzIzs72zAMw1i0aJHRpk0bo7i42CyTn59v+Pj4GKtWrTIMwzCCg4ONWbNmmfMLCwuNpk2bGv369XNa9/fff29IMg4dOnTRNuLKogcFZTp48KAKCgoUERFhTgsICFCbNm0qvK4lS5bo1ltvlcPhkK+vryZPnlzqdEKLFi3k5+dnPg8ODq7wt7oSX3zxhf7yl79oyZIlat26tTn966+/1p133qlrr71Wfn5+Gjx4sI4fP64zZ86Ue9179uxRaGio07fqsLAw+fv7a8+ePZVuz/fff6/4+Hj5+vqaj+joaBUXFys1NdUsd2GvQgkvLy916NDBqY6enp5O+65Ro0Zq06aNUx1/u5wkjRo1SgkJCZd8DZKTk9W3b181a9ZMfn5+6t69uySV2qeRkZHm/0uOnZLtP/nkk/rb3/6mW2+9VdOmTTNPXUjSr7/+KkmqX7/+JevxW+Vpd1FRkZ5//nm1b99eAQEB8vX11apVq8p9eqvECy+84LSvKrJ8VlaWRowYoVatWslut8tms+nUqVOl1jF27FglJSVp/fr1uvbaa83p5T1WylLyGoWHh5vT2rZtW6mrZubOnavw8HAFBgbK19dX77zzTqk23HjjjfLw8DCfV+V9/c477+i9997TJ598osDAQEnnX4sff/xRfn5+5msREBCgs2fP6uDBg8rNzVVGRobTMeHp6Vnm+8jHx0eSKvT3ANWLgIIqcXNzK3U+98LTCImJiRo0aJDuvvturVy5Utu3b9df//rXUgM869WrV2q9lRm0uXv3bg0cOFAzZ85Uz549zemHDh3SPffcow4dOuh///d/lZycrLlz50oqe7BpVVW0PadOndJjjz2mlJQU8/H999/rwIEDuv76681yDRs2LLWsj49PpQYKV2a506dPKzo6WjabTR988IG2bt1qdpdX5HV89NFH9dNPP2nw4MHasWOHbrnlFr3xxhuSpMaNG0s6P57E1V566SXNmTNHEydO1Jo1a5SSkqLo6OgKHwOPP/64074KCQkp97JDhw5VSkqK5syZo82bNyslJUWNGjUqVYe77rpL//3vf7Vq1Sqn6eU9VirL3f38x8KF7+sL39PS+TFC48eP1/Dhw/XVV18pJSVFjzzySLW9r9esWWOe6rowVJ86dUrh4eFOr0VKSor279+vhx9+uELbOHHihCSZ4Qc1z/rD4FEjrr/+etWrV09JSUlq1qyZpPMfGPv37ze/MUvn38wXnjs+cOCA0zeQzZs3q3nz5k6Xix4+fLjC9fHy8lJRUdElyxw7dkx9+/ZVTEyMxo4d6zQvOTlZxcXFeuWVV8w/wL89X16ebbRr107p6elKT083e1F2796tnJwchYWFVbRZps6dO2v37t264YYbKr2OC+t47tw5JSUlqVu3bpLODw7dt29fleooSXv37tXx48c1c+ZMs/3btm0rs+y3335b6thp166dOT80NFSPP/64Hn/8cU2aNEnvvvuuRo8ereuvv142m027d+926gG7nPK0e9OmTerXr5/+9Kc/STo/lmL//v1Or0t5joOAgAAFBASUu24X2rRpk+bNm6e7775b0vlBnseOHStV7t5771Xfvn318MMPy8PDQwMHDpRUtWOlbdu2OnfunJKTk/W73/1OkrRv3z6ne9aUfEBnZGTo5ptvliSnAbMlbejWrZv+/Oc/m9NKrliqiPK81j/++KPuu+8+PfvssxowYIDTvM6dO2vJkiVq0qSJbDZbmcsHBwcrKSlJf/jDHyTJbH/nzp2dyu3cuVNNmzY1AzJqHj0oKJOvr6+GDx+uCRMm6JtvvtHOnTs1bNgw88O9xB133KE333xT27dv17Zt2/T44487fWtq1aqV0tLStHjxYh08eFCvv/56mQPULqdFixZKSkrSoUOHdOzYsTK/hcXExKhBgwaaPn26MjMzzUdRUZFuuOEGFRYW6o033tBPP/2kRYsWaf78+aW2cerUKSUkJOjYsWNldvVGRUWpffv2GjRokL777jtt2bJFQ4YMUffu3cvsNi6viRMnavPmzRo1apRSUlJ04MABffzxx+Ue+HihVq1aqV+/fhoxYoQ2btyo77//Xn/605907bXXql+/fpdc9s0339Sdd9550fnNmjWTl5eX+Tp+8sknev7558ssO2PGDCUkJJjHTuPGjc2bY40ZM0arVq1SamqqvvvuO61Zs8YML+7u7oqKitLGjRtd3u5WrVpp9erV2rx5s/bs2aPHHnvM6QosqXzHWlW0atVKixYt0p49e5SUlKRBgwaZpxd+649//KMWLVqkRx55RP/5z38kVe1YadOmjXr16qXHHntMSUlJSk5O1qOPPuq0fR8fH3Xt2lUzZ87Unj17tG7dOk2ePLlUG7Zt26ZVq1Zp//79mjJlykUHLl9KixYttH79ev33v/8tM6T9+uuv6tu3r26++WaNHDnS6X0tSYMGDVLjxo3Vr18/bdiwQampqVq7dq2efPJJ/fzzz5Kkp556SjNnztSKFSu0d+9e/fnPfy7zJoIbNmxw6nVFzSOg4KJeeukl3Xbbberbt6+ioqL0+9//3unctXR+hHxoaKhuu+02Pfzwwxo/frzTPUPuvfdejR07VqNGjVKnTp20efNm8+qTihg/frw8PDwUFhamwMDAMs/5r1+/Xjt37lTz5s0VHBxsPtLT09WxY0e9+uqrevHFF3XTTTfpgw8+UFxcnNPy3bp10+OPP64HH3xQgYGBmjVrVqltuLm56eOPP9Y111yjP/zhD4qKitJ1112nJUuWVLhNF+rQoYPWrVun/fv367bbbtPNN9+sqVOnVujUwYUWLFig8PBw3XPPPYqMjJRhGPr8889Ldbn/1rFjxy75TTgwMFDx8fFaunSpwsLCNHPmTL388stllp05c6aeeuophYeHKzMzU59++qm8vLwknR8LEhsbq3bt2qlXr15q3bq15s2bZy776KOPavHixRUOB5dr9+TJk9W5c2dFR0erR48ecjgcTncUlcp3rFXFe++9p19++UWdO3fW4MGD9eSTT17yvh/33XefFi5cqMGDB2vZsmVVPlYWLFigkJAQde/eXQMGDNDIkSNLbf/999/XuXPnFB4erjFjxuhvf/ub0/zHHntMAwYM0IMPPqiIiAgdP37cqTelvGbMmKFDhw7p+uuvL/PUSlZWlvbu3auEhASFhIQ4va8lqUGDBlq/fr2aNWumAQMGqF27dho+fLjOnj1r9qg8/fTTGjx4sIYOHarIyEj5+fnpj3/8o9N2zp49qxUrVmjEiBEVbgOqj5vx2wEEwCX06NFDnTp14tbvqFaGYSgiIkJjx47VQw89VNPVqfNatGihMWPGWP5nA6rLW2+9peXLl+urr76q6argAvSgALAcNzc3vfPOOzp37lxNVwVXgXr16pmDtGEdDJIFYEmdOnVSp06daroatcqGDRsu+Xs/JXdOhrNHH320pquAMnCKBwDqiF9//VX//e9/LzrfFVeJAVcKAQUAAFgOY1AAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDl/B+6rI3pvhW2QQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mu = 0\n",
    "sigma = 1.0\n",
    "x = np.random.normal(mu, sigma, 1000)\n",
    "\n",
    "bins = np.linspace(0, 0.045, 50)\n",
    "plt.hist(np.abs(relu(x) - relu(x, quantize=True)), bins, label='ReLU', alpha=0.6)\n",
    "plt.hist(np.abs(silu(x) - silu(x, quantize=True)), bins, label='SiLU', alpha=0.6)\n",
    "plt.legend()\n",
    "plt.xlabel('quantization error: abs(float - fake_quantized)');\n",
    "\n",
    "print('ReLU SQNR[dB]:', compute_sqnr(relu(x), relu(x, quantize=True)))\n",
    "print('SiLU SQNR[dB]:', compute_sqnr(silu(x), silu(x, quantize=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}