mish_2nd_grad_opt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sympy import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_printing(use_unicode=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, sp, tsp, grad_sp, grad_tsp = symbols('x sp tsp grad_sp grad_tsp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softplus(x):\n",
    "    return log(1+exp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "mish = x * tanh(softplus(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad = diff(mish, x)\\\n",
    "    .subs(log(exp(x)+1), sp)\\\n",
    "    .subs(tanh(sp), tsp)\\\n",
    "    .subs(-tsp**2 + 1, grad_tsp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAIMAAAAmCAYAAADnTCg7AAAABHNCSVQICAgIfAhkiAAABp5JREFUeJztmmtsVEUUx3/FAhpBQVCUAsWCoihUEYkgCL4QFMQ3okZXE5EPFRMkEcH4BoLBBJqAUrTWiJqAL/QDQYhBXsEqKKAi+Cob0YDIUxAoUD/8z6XTyy1ddre72+z9Jzf3zuvM2Z2ZM/85cyBEiAxBEbAh3Uo4WAFcZd8lwNg06pJynJLm/h8FdgAfplkPD38CY4CzgTzgqfSq07DQOMH25WTeH74aWAI0SbMeKUcjX7obsAw4AKwH+gCVQH+gHVAF3AcsBQ4CI6zds8A64F/gb6AUONUnuwvwhSO7F3Ap8F0Sfsfdpk++kzcd+Bmtckz/d5El2oWs0Tk+OVcCZwF7gENJ0KvBoisazCnAxcAQIIomQAtLVwFrgEFAJ/TH5QAvAFejwbgB2AKMc2RfCOxGA9QZuAXYbPLaJEH3HOAbYLalxwJbTUeAAktPtt92GfAl8IEjIw9Ya3W/tjpZi8Ucv3e/Dfxu3xOA/ejPqgslQJmTXgS876tTCvzlpDsCw2JTNRADkRUbh1Z2T6fsc2CSr/5NwE77Pg1YiSwgwO1kDo9JOTqgVVroy58NfGzf84C5AW3bA8XI9O9A1qUSeMkpr0Im2EUJsMBJPwY8HZ/6x7ASOIwsl4d863+/6eY9/yFrEcKQa+/L0Z+4zld+CbDQvguBab7yVsikLkWm+Q/gqOWtdWQfcdIerkArFrQiJwP/AMOBt4CI6bcJuNPqfYK4QYH1fS8ioQDXmY6NgG1OP4Voi3IthYeDAXlZjyFoEE938nqhFTXM8o8AvX3tHkRkLMfJi1g7b78eaunmTp3+ljfcyVsCnA+0RBPHm6gtnDq/AePtewSyVqAB3wU8hCaMa3EGo4nu9h/iBDgPsfyZaBAHAxvRgOWjSXCEmpMFRAQPoz22M/A44gHuBGlrsmeZbJc8dnFkVVib5lb+KjrdeGiGiKk3SbojnpNv+ROc/KNAP0u3RCec+chKdQJuBGZQcxJnGsZQfXpLOUagQdgHfIrO/x7BGwX8FNAmB/2pe5BpLgamolXu4n50MtlpZS9aP97Rth2wyql/BrI6P1C9RfShelsBWYFS5MGc5etvLrDcSfdEx9pdwF50nB1HZmMOxy+YtCAH/fHFKeqvN/CRfV/g5BdTvTJGIf7QGK32cjSJMhERNJADEpCxAU1cvy+o3tEXrcACxPrfQ1bh3BT1fyby/K1HR7qNyJ8xg2qX+UxEMr+yejenSLd4ECH+yTDF2gY9Dzj1+qEF9CvahrehBeIeoftYu2Jk+ZchMn3A6l4bpMBdjtAo8i/kxfFD6hPLEcFsCIgQ/2S4B/loqtDF2fPO097qjLfyzeiIPgl4ExHvNY6sIqu3AXG7+cAryO9ThY7XHeLQMe2IktmEz0WExLaJkdZ+ZEBZGzSwywi+P2ntfJeanN3I+rsotrKpXkbK96ME4DnGsgE97L0moOwitHVuIvj+ZHuAnCJqEmqAN+zdNU4dQwSggtr3+KCnLAaZ5WigmwaUtUanoip06huOCLUfTU1GlOBFn2cyFnkZuQGVQpwcplHTMQa65BqGuFeFr6yuW9pc5F/5kWAP6XZk8p9DJHoo8gEtQr4Wz5p0Qyevhcjv4kdHe0fr0CdEgogQP2fobm1LY6jbBN0Sz7U226m2Jh7vmBDclEn4vMCeZciWvTgRpIq8elfn38ZQ9xDywi5GhLIvIphRqvmC32qBQg9Goruk+V6mt5fkhE+dT6rQyt57Aso8d7ofnVGgUBQNMOgiEOQ2cK8RmiE/UivgCeROAFLLGVYATyK3cwliw1NP2CI7sdreE9EA70Nu+XnAaOSGL7e8bcj3cqu1eQTxg1xruxY59NYhstkUcZm2KFzA8/qmHINModHoijqZuAb4DAW0VqHVkE5ESMzPUIS8sAdMzkTLvw14x8q80LwKdEx03fiF1u41y19g9feiLWVgnHolFScbbFqGPG91YTDwMnAHmTEZ0o2Hqd1pVSuS6XSqK+C0PoNNFwDPkEazl2HwyGMsJPQYkjUZCtCqj6LA2AHIOTLTyvOQKbsexU5kdbBpCtAD+R7Wp6PzEwWcJhJsWkZs24SLbN8mGqEYz+/T0XkyA07H+2RUoi3FzetXa2sh2ydD3EjG0TKZAaevUzMCewoKaXODbLacpMwQMSIZk6ESOTK2oqNLIthhj4e9lv4lQbkhYkAyCOQqxA3mkL6A02aIlHrEtKN9N8jAjYaO+go4LSM2AjmA+K+LQ4QIESJEiBAhQoQIEQ/+B1SWurWC+46fAAAAAElFTkSuQmCC\n",
      "text/latex": [
       "$$\\frac{grad_{tsp} x e^{x}}{e^{x} + 1} + tsp$$"
      ],
      "text/plain": [
       "            x      \n",
       "grad_tsp*x*e       \n",
       "------------- + tsp\n",
       "     x             \n",
       "    e  + 1         "
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "gradgrad = diff(mish, x, 2)\\\n",
    "    .subs(log(1+exp(x)), sp)\\\n",
    "    .subs(tanh(sp), tsp)\\\n",
    "    .subs(-tsp**2 + 1, grad_tsp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOsAAAAsCAYAAACE50/ZAAAABHNCSVQICAgIfAhkiAAACFhJREFUeJzt3XuMXFUdwPHPFnDRFoSiYOnTloJS+5JKKA874AORigFRglZQEyLRavyDhAI+KhgbfEUbrQJal0qjPHwUVBQrqVUIgtIC8Y3YVqigpjxFpWj943cnMzs7Mzs7c2d2xj3fZLL3nnvOfZx7fuf3O7/zu2dJJBKJHueo7O9UbMKvcQ/OKMszA2/swL28CF/EbKxBXweumWiOOaN9A2ONZTg8256E+dn2wfgznpftvxsXdeiePo516O/Q9RLNsRBvaMeJx7XjpD3OS3Aofp/t/0VoVPgrHsULsASr8C5sxf44D1twH76ZlfkOrsVdeABHZ+m34Zhs+0pckG3PxnfxC/xUaNX9hIbfjX/n9aA9SrdbGVtE5z5ztG9kLPANIXjVWITfKDWQTXhxtn2gEOq9s/0Dsr8P4OJs+2xcn22/Djfi/fhqltaPH2Natn+e6BC+jCm4FC8f+SN1PSPpuOh+K+MgfGW0b6LbWC6EJy/mYG2NYweJceuxZWnblAR3P2zHpzE3S5uAh5QEeB42lpX/pRD452T7Z2Gn0NRbxbN9qJkH6TEa7bguE/X8Nd0vDOuUhk8joVbH1fNcha/neL5P4PQq6f3YjLeXpU3BHRX59sc5+BXeJAT7lrLj5+Iz2fYr8CfRSIt8DG9t8t57nUY7rtG0Mi4Sw5kn8DfchJfVyHuaMNdHSrWOq2vYp4Wyd+LCvG5ENIhDKtL6RIewsiJ9Mb5Vtj+7bHu1ELrzxdh3H2Em3yka2mRhMs8UL39BVm45BpS09Vy9wYCh9TMSeqXj+iHeKQR0Lr6NhzGxSt5DhL+jGSo7rrYwV4wt/iWcLMcKp8iS7PgU7BEvYbNwlpyTHfsw7sVTotdai33Lzn0Ebi0799F4GifndO8ThQOpkuPxX6Uefqt4zueLSr0v278av8Pd+AL2Eg6QVfh5lu/1eC5uV6qT05WcUeNFA/htdp2iFm43bxbvYnpZ2ufwB7ywgfIDmhfWXu64JuA/ant/t2NWRdoUrMcuPCbe/cFlx6t1XLlzpBC0y/FSLMUOIZxFZ8vSbP9uoe5nCSHpw0dxnGgwrxZjvRVZucPxuGhAh+FUURF7DNWEzXKM8OTlyc+UHFDdTJ9w4lyV7V+ARwxtaLUY0JywdmPHNRImiTZ4XI3jt4m2WmSmqNdVQkYW4Ce4ITteq+PKnY1KFV3katFLFLlEaMNG3NpXikYAPzJ0bLpW82ZGNc4QmjtPdui+qYVavFZYQSvEmGzRCMoOaM0M7lWuFR38XjWOf084xorcIjzZ5ZwspgPrdVy5Mk30MJXer6tE71jkelxXpfxUMc67T5gHT4mGc1l2bI8wD8q5EjeX7c/QWjTRMny/hfLdwEpRV/V+hTrlb8ezwuqpx8XiHRV/u/FMRdoJzT1CR1iptXqCTwplcVidPOvxgWx7enbepw2up38KbTssew+fpSEWipd8b0X6HDEoLzIfn63Ic5BQ+ZuF+fWgGCPeJUyChWJccE9FuaMM9rSeLEzqDU0+w7NKkUm9yufFPHE9dtRIP0m8n3Gqj93L+ZLBne7lYtiyuiztoWHOMZq0Uk/E9NwynIj76+TbV3RkRN0+rrrF0tFAl6VCwMaXpR0tepKithsvhG5xRdlzxEC73Fx8R1Z2lhi87xHza0WWZGlnle3vEg6RZqKJCG2ytcqzDdcDj/YvD+aLd3CuqKOb62cfwoDhzeBer6Miq4UmPLKBvLcqebRPEQphv9rZO8Mk4aVdIwTsFOEZ3aPkYVwshHV8RdlTxUOcLkyK9wnzoijAh2bnviI7d7lz6Yiy82zSfDSR7FzbR/DM7eBGMX65YbiMOTJdaMFLsv15ouMdiRk7YPTHrJ2ouzViPH+SiKYq/ibUyL9FSTkdKGY5NghrcRZeI2YOOu7XOFs09n+IirvQYAfQ+cKzV0mfuOEnhPm1Gp8SwlfkbcIseTRLvzS7Tnls8zatRRONEx3EAZqjkZjVgvqN+kRhSXRKWCeKueUrKtKvE57sRhnQvLA2GutbGOYanai7Wpq72n2Nw98NFuRFQts+hieFJbdiaNHO0ifGk6uHy5gTrUYTFdkgvKLD0WzMasHwjbqgs5q1k7QS61vQW3W3QDjsciOvr26OFwIxU3ht1wuNVummbhdTRWhakdlCU68T8aX9QpPOUIomeq+hwrpe9ND16BcOlveInnJddq70ZUx9xlq9naD6zMeocyb+KMaWO8T86uQOXr/ZaKJK9hKmcT0veTMxqxuzvPeL8LRi2RlVzl/QPdohT5qN9e3VurtJKIVEEzQaTXQa3lLneCsxqwW9ZcrlSauxvgW9U3fz8JG8TzqWPj6fJpxQw3Gj8NbVcrE/LMa13Ryz2o2MpXpbJuaeEx2gX23t2krMakF97bBRuPefFsEhlXPSvUyrsb4FvVF3Sw3+ICLRAcaS1ZHIj9RuEonEYLYZWRjXNaNyl4nEGKRyiqI4/dIoO4fPkkgk8iCvmMS8g6UTiUQikehNan3hnug9bhNLpT4oPsw/Qs6xqYlEIh/avYTlK0UI3U4x7DmzDddI1CHNCf3/8AMRj32G+B88jTCg8U/bxotvhJeP9MYS+ZCEtXdoZAnLieJro2facP2b8UGD10pOdJAkrL3BTPFV0Q6x7GVB/HOsNdnxyeLrlVeJVTvatoRlIpGoT7uWsBzQ3AoPacyaSFSh5SUsy8hrCdEkrKNAXkuRJtpHnktY9voSomOaJKzdz26x6NYjYpGtVtiV/Yo8me3XW/s20SUkB1P3c4cYm15jdJewnCAcV0Xn1Yxse1qtAonEWKRdS1gOaNzBVFD9y6uBHO4jkUgkEolEIpFIJBKJRCKRSLSb/wErG5jinFK3qgAAAABJRU5ErkJggg==\n",
      "text/latex": [
       "$$- \\frac{grad_{tsp} \\left(\\frac{2 tsp x e^{x}}{e^{x} + 1} - x + \\frac{x e^{x}}{e^{x} + 1} - 2\\right) e^{x}}{e^{x} + 1}$$"
      ],
      "text/plain": [
       "          /         x           x     \\    \n",
       "          |2*tsp*x*e         x*e      |  x \n",
       "-grad_tsp*|---------- - x + ------ - 2|*e  \n",
       "          |   x              x        |    \n",
       "          \\  e  + 1         e  + 1    /    \n",
       "-------------------------------------------\n",
       "                    x                      \n",
       "                   e  + 1                  "
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradgrad"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}