Parametric Exponential Linear Unit in TensorFlow

Parametric Exponential Linear Unit in TensorFlow.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true,
        "collapsed": true
      },
      "cell_type": "code",
      "source": "import tensorflow as tf\n\n\ndef pelu(x):\n    \"\"\"Parametric Exponential Linear Unit (https://arxiv.org/abs/1605.09332v1).\"\"\"\n    with tf.variable_scope(x.op.name + '_activation', \n                           initializer=tf.constant_initializer(1.0)):\n        shape = x.get_shape().as_list()[1:]\n        alpha = tf.get_variable('alpha', shape)\n        beta = tf.get_variable('beta', shape)\n        positive = tf.nn.relu(x) * alpha / (beta + 1e-9)\n        negative = alpha * (tf.exp((-tf.nn.relu(-x)) / (beta + 1e-9)) - 1)\n        return negative + positive",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.0",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "gist": {
      "id": "",
      "data": {
        "description": "Parametric Exponential Linear Unit in TensorFlow",
        "public": true
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}

Thanks! This looks like it's defined unit-wise though, while the original paper has the parameters constant across the layer?