Created
March 12, 2018 18:26
-
-
Save justheuristic/1445f58339528d07481e025337fbdb3d to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "from __future__ import print_function, division\n", | |
| "from IPython.core import display\n", | |
| "import matplotlib.pyplot as plt\n", | |
| "%matplotlib inline\n", | |
| "import numpy as np\n", | |
| "\n", | |
| "#If you are running on a server, launch xvfb to record game videos\n", | |
| "#Please make sure you have xvfb installed\n", | |
| "import os\n", | |
| "if type(os.environ.get(\"DISPLAY\")) is not str or len(os.environ.get(\"DISPLAY\"))==0:\n", | |
| " !bash ../xvfb start\n", | |
| " %env DISPLAY=:1" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### KungFuMaster env" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n", | |
| "Observation shape: (1, 42, 42)\n", | |
| "Num actions: 14\n", | |
| "Action names: ['NOOP', 'UP', 'RIGHT', 'LEFT', 'DOWN', 'DOWNRIGHT', 'DOWNLEFT', 'RIGHTFIRE', 'LEFTFIRE', 'DOWNFIRE', 'UPRIGHTFIRE', 'UPLEFTFIRE', 'DOWNRIGHTFIRE', 'DOWNLEFTFIRE']\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "import gym\n", | |
| "from atari_util import PreprocessAtari\n", | |
| "\n", | |
| "def make_env():\n", | |
| " env = gym.make(\"KungFuMasterDeterministic-v0\")\n", | |
| " env = PreprocessAtari(env, height=42, width=42,\n", | |
| " crop = lambda img: img[60:-30, 15:],\n", | |
| " color=False, n_frames=1)\n", | |
| " return env\n", | |
| "\n", | |
| "env = make_env()\n", | |
| "\n", | |
| "obs_shape = env.observation_space.shape\n", | |
| "n_actions = env.action_space.n\n", | |
| "\n", | |
| "print(\"Observation shape:\", obs_shape)\n", | |
| "print(\"Num actions:\", n_actions)\n", | |
| "print(\"Action names:\", env.env.env.get_action_meanings())" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": [ | |
| "/home/jheuristic/anaconda3/lib/python3.6/site-packages/scipy/misc/pilutil.py:482: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n", | |
| " if issubdtype(ts, int):\n", | |
| "/home/jheuristic/anaconda3/lib/python3.6/site-packages/scipy/misc/pilutil.py:485: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", | |
| " elif issubdtype(type(size), float):\n" | |
| ] | |
| }, | |
| { | |
| "data": { | |
| "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAEICAYAAADBfBG8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFmVJREFUeJzt3XvUHHV9x/H3hyBoASHcEgi3wAGO4CVGxFTKRbyFVAXa\nqsFWUWkJlVA80FMIKFLUAirQKBUImnIRQSqi1BNQCnhpEeRiCJcIJIAQckMIBAVpE7/9Y2Zhstl9\nnnl2dp+Z2f28ztmzszOzu99J5ru/3/xmnu8oIjCzzm1QdgBmdeckMivISWRWkJPIrCAnkVlBTiKz\ngpxEfUjSTpJ+J2lM2bEMAidRAZKmS7pd0u8lrUynPyVJZcYVEY9HxKYRsbbMOAaFk6hDkk4EZgNf\nBsYD44BjgP2AjUoMzUZbRPgxwgewOfB74C+HWe/PgV8Bq4EngNMzy3YBAvhEumwVSRK+FVgAPAuc\n3/R5nwQWpuv+CNi5zfc2PnvD9PVPgC8AtwK/A/4T2Aq4Io3tDmCXzPtnpzGtBu4C9s8sew1waRrD\nQuCfgCWZ5dsD1wBPAY8C/1D2/1fP94eyA6jjA5gKrGnspEOsdxDwBpIW/43ACuCwdFljR78QeDXw\nHuAPwPeBbYEJwErgwHT9w4BFwOuADYHPALe2+d5WSbQI2C39AXgAeAh4V/pZlwH/nnn/36RJtiFw\nIrAceHW67Czgp8BYYIc04ZekyzZIk+40ktZ4V+AR4L1l/5/1dH8oO4A6PtKdbHnTvFvT1uNF4IA2\n7/tX4Lx0urGjT8gsfxr4cOb1NcCn0+nrgaMyyzYAXqBFa9QmiU7NLD8HuD7z+v3A/CG2dxXwpnR6\nnaQA/jaTRG8DHm9676xsgvbjw8dEnXka2FrSho0ZEfH2iNgiXbYBgKS3SbpF0lOSniPprm3d9Fkr\nMtMvtni9aTq9MzBb0rOSngWeAUTSYuWR93uQdKKkhZKeS79r80zc25N09Rqy0zsD2zdiTN97Csnx\nYt9yEnXmF8BLwKHDrPdt4Dpgx4jYnKTr1unI3RPAjIjYIvN4TUTc2uHntSRpf+Ak4EPA2PSH4Tle\niXsZSTeuYcemGB9tinGziJjWzRirxknUgYh4Fvhn4OuS/krSppI2kDQJ2CSz6mbAMxHxB0n7Ah8p\n8LUXArMk7Q0gaXNJHyzwee1sRnK89xSwoaTTgNdmll+dxjFW0gRgZmbZL4HVkk6S9BpJYyS9XtJb\nexBnZTiJOhQRXwJOIBmdWknSPbqI5Fe80Tp8CjhD0vMkB9tXF/i+a4GzgaskrQbuAw7peAPa+xHJ\n8ddDwG9IBjuyXbYzgCUkI2//BXyXpFUmkvNS7wcmpct/C3yDpDvYt5Qe/Jl1RNLfA9Mj4sCyYymL\nWyIbEUnbSdov7b7uSTIEfm3ZcZVpw+FXMVvHRiTd1okkQ/pXAV8vNaKS9aw7J2kqyZnvMcA3IuKs\nnnyRWcl6kkTp1cMPAe8mOQi9AzgiIh7o+peZlaxX3bl9gUUR8QiApKtIzqm0TCJJHt2wKvptRGwz\n3Eq9GliYwLrDoktoOrMu6WhJd0q6s0cxmBX1mzwr9aolanVWfp3WJiLmAHPALZHVW69aoiWseznI\nDsDSHn2XWal6lUR3ALtLmihpI2A6yTVkZn2nJ925iFgjaSbJJSRjgLkRcX8vvsusbJW47MfHRFZR\nd0XEPsOt5Mt+zAqqxWU/xx9/fNkh2ACaPXt2rvXcEpkVVIuWaLTMmDEDgIsuuqjtsqzm9ZrXGely\nqye3RKlWSdJq2UUXXfTyzp+dn03ATpZbfTmJUm4VrFNOohyyCTZjxowhu3btllv/chKZFeSBhZyG\nGyRoXset0eBwS5RDnoRw0gyuWlz2MxonW0c6PJ1nHQ9x19vs2bNzXfbjJDJrI28SuTtnVpCTyKwg\nj85VyNhZY9ebt+rMVSVEYiPhlqgiGgm06sxVLz+y8626nERmBXWcRJJ2TG9gtVDS/ZKOT+efLulJ\nSfPTR1/fm8asyDHRGuDEiLhb0mbAXZJuTJedFxFfKR6eWfV1nEQRsYzkrmlExPOSFpL/1odmfaMr\nx0SSdgHeDNyezpopaYGkuZJaHhm7Auq6sgMJjUd2vlVX4SFuSZvyyl2uV0u6APg8ScXTz5PcqfqT\nze9zBdT1OWHqqVBLJOlVJAl0RUR8DyAiVkTE2oj4I3AxSXF7s75VZHROwDeBhRFxbmb+dpnVDie5\nt6hZ3yrSndsP+Chwr6T56bxTgCPSu2gH8BjgvxGwvlZkdO6/aX33h3mdh2NV5D/hGNrAXjt374NH\nrPP6DXteOaLl3fiMPN9RthkzZrSsMeFEeoUv+7EhOVmG5ySy3IYqbjnInESWm4tOtuYksiE5YYbn\nGgs2rEEdnctbY2FgR+csv0FJmk65O2dWkJPIrCAnkVlBA3NM1HyPoVZn4lstzz5nNc9rfNasWQ/3\nahO64swzdy87hL4zUC3RcAfIeQ6gszfpyvse628DlUTDnfNoXt5q/Tzr2GAZqCRqbkVaLW+ebl6/\n1fvdGg22gUqiZp3c1a75Pa2Ol2yw+IoFszZG7YoFSY8BzwNrgTURsY+kLYHvALuQ/HXrhyLCVTis\nL3WrO/eOiJiUydqTgZsiYnfgpvS1WV/q1XmiQ4GD0ulLgZ8AJ/Xou0ZkJOeDWs1v9Z6sQ37+89HZ\nkA5dv//+ZYfQd7qRRAH8OD2uuSitJzcurZBKRCyTtG0Xvqdrit4m0iyrG925/SJiMnAIcKykA/K8\nqcwKqCM9X9TpOjYYCidRRCxNn1cC15IUa1zRqD+XPq9s8b45EbFPntGPbhvplQvtXvv8kEHxCqib\npHeEQNImwHtIijVeBxyZrnYk8IMi39Ntrc71DLXcbCiFzhNJ2pWk9YHk+OrbEfFFSVsBVwM7AY8D\nH4yIZ4b4HJ8nssoZlfNEEfEI8KYW858G3lnks83qohZXLJiVpH9qLEz+wuSyQ7ABdPdn7s61Xi2S\naNsdKnWayWwdtUiiDa4e6IvNreJqkUTzd5g//EpmJalFEo3faXzZIdgAWsrSXOu5n2RWUC1aIg8s\nWJX5PJFZe7nOE7k7Z1aQk8isoFocE90w2Vcs2Oibene+KxbcEpkV5CQyK8hJZFZQLY6JJs3zFQtW\ngpy7nVsis4I6bokk7UlS5bRhV+A0YAvg74Cn0vmnRMS8jiMEPvLx04ZdZ9aJxwFw5jlfK/JVhTiG\nfosh327bcRJFxIPAJABJY4AnSeotfAI4LyK+0ulnd2LtSWuTiRKvEHIMgxlDt46J3gksjojfSOrS\nR47MmLPHJBPnlPL1jmGAY+hWEk0Hrsy8ninpY8CdwImjUcx+0H79HEN1Yig8sCBpI+ADwH+ksy4A\ndiPp6i2jzW9Btyugjjl7zCu/PiVxDIMZQzdaokOAuyNiBUDjGUDSxcAPW70prdk9J12v8FXcg/br\n5xiqE0M3kugIMl05Sds1itkDh5NURO25QeuHO4bqxFAoiST9CfBuIFtz90uSJpHcLeKxpmU9M2i/\nfo6hOjEUrYD6ArBV07yPFoqoQ4P26+cYqhNDLS77yWPQfv0cQ3Vi6JskGrRfP8dQnRj6JokG7dfP\nMVQnhr5JokH79XMM1Ymhb5Jo0H79HEN1YuibJBq0Xz/HUJ0Y+iaJBu3XzzFUJ4ZaFG9cvnzaaIVi\n9rLx4+e5eKPZaKhFd+6Wyb61ilWXWyKzgpxEZgU5icwKqsUx0TvunlR2CDaIxvtOeWajohYtUZ66\nc2bdl6/unFsis4JyJZGkuZJWSrovM29LSTdKejh9HpvOl6SvSlokaYEk31zI+lrelugSYGrTvJOB\nmyJid+Cm9DUk1X92Tx9Hk5TQMutbuZIoIn4GPNM0+1Dg0nT6UuCwzPzLInEbsIWk7boRrFkVFTkm\nGtcojZU+N66XnQA8kVlvSTpvHd0u3mhWll6MzrUqxr3eVdrdLt5oVpYiLdGKRjctfV6Zzl8C7JhZ\nbwcg31krsxoqkkTXAUem00cCP8jM/1g6SjcFeC5TEdWs7+Tqzkm6EjgI2FrSEuBzwFnA1ZKOAh4H\nPpiuPg+YBiwCXiC5X5FZ38qVRBFxRJtF72yxbgDHFgnKrE58xYJZQU4is4KcRGYFOYnMCnISmRXk\nJDIryElkVpCTyKwgJ5FZQU4is4KcRGYFOYnMCnISmRXkJDIryElkVpCTyKwgJ5FZQcMmUZvqp1+W\n9Ou0wum1krZI5+8i6UVJ89PHhb0M3qwK8rREl7B+9dMbgddHxBuBh4BZmWWLI2JS+jimO2GaVdew\nSdSq+mlE/Dgi1qQvbyMpi2U2kLpxTPRJ4PrM64mSfiXpp5L2b/cmV0C1flGoAqqkU4E1wBXprGXA\nThHxtKS3AN+XtHdErG5+bzcroN58w5SXpw+eeluRj6p1DEOpenx11nFLJOlI4H3AX6dlsoiIlyLi\n6XT6LmAxsEc3Am0nu3OUpQoxjETd4q26jpJI0lTgJOADEfFCZv42ksak07uS3F7lkW4EmlcVdpAq\nxJBVtXj6zbDduTbVT2cBGwM3SgK4LR2JOwA4Q9IaYC1wTEQ035KlJxpdlDJ3mCrE0E6VY6u7YZOo\nTfXTb7ZZ9xrgmqJBdaKxc5TZ369CDK0cPPU2J08P1eLGx0M5eOptfO3tZ7z8+rhbBzOG4Sz41rSX\npz/9Ld9Iupt82Y9ZQX2RRMfdeto6z4Maw1AarY9boe6rfXcOYI97FnAc5e4cZcVw/rmvBWDmCeud\nimux3lc4P72X+3DrW361b4n2uGfBOs+DFEMjgZqnh1ovz/o2MrVPoqwyE6lKMTScf+5rnSyjoLbd\nuarsrGXG0eiSNRJluIRpXt+6oy9aoofe9MayQyg1huzxzcwTVrd83ZxAPibqntq2RNZacyvjVqf3\n+qIlstYtS3OrNNS61rnaJ9Ggd+WympOjMbCQTSYnUPfVPomyB/Zl7cxViGEo2WSy7qt9Etm6nCij\nr/YDC1X45a9CDFl77bXXeleS33zDlMpdXd4v3BKZFVTbJFo790DWzj1wnddlxVF2DMNxK9Rbte/O\nAex2/NiyQ6hEDA0HT71t3fND5z7gY6Ue6rQC6umSnsxUOp2WWTZL0iJJD0p6b68Cb6UKO3IVYmjm\nBOqtTiugApyXqXQ6D0DSXsB0YO/0PV9vFC7ptsWzV7F49ip2O34si2ev6sVX5I6j7BisXHlqLPxM\n0i45P+9Q4KqIeAl4VNIiYF/gFx1HmEMVduIqxGDlKDKwMDMtaD9XUqMPMwF4IrPOknTeerpVAbWx\n45bZjapCDFaeTpPoAmA3YBJJ1dNz0vlqsW7L6qYRMSci9omIfTqMYT1V2ImrEIMvOh1dHSVRRKyI\niLUR8UfgYpIuGyQtz46ZVXcAlhYL0YrwoELvdVoBdbvMy8OBxsjddcB0SRtLmkhSAfWXxUIcWhV+\n+asQg5Wn0wqoB0maRNJVewyYARAR90u6GniApND9sRGxtjehWyvuyo2+rlZATdf/IvDFIkHlUZVf\n/6rEYeWp7WU/rVRhiLkKMdjoUnpXlHKDGOb+RENd97Xf8icB+J/xLUfSR0UVYsiqak3wurn5hil3\n5Rk9rsW1cydMbn/r19vnfRZIduS3Tfv8aIVUuRiybr4heR7q382G1/h3HE7tu3NV2GmrEEMr7/uX\n+WWHMBBq0Z0zK0n/dOd+eMqkskOwAZS3Ja99d86sbE4is4KcRGYFeWDBrD0PLJgV4YEFs1FSi+7c\n8uXThlps1hPjx8/rn+7cLZN95t2qy905s4KcRGYFOYnMCuq0Aup3MtVPH5M0P52/i6QXM8su7GXw\nZlWQZ2DhEuB84LLGjIj4cGNa0jnAc5n1F0dEV0/svONunyeyEozPV6iqUAVUSQI+BBw8gtBGbPz4\neb38eLNCig5x7w+siIiHM/MmSvoVsBr4TET8vNUbJR0NHJ3nS67cfvuCYZqN3BFLu9QSDfc9wJWZ\n18uAnSLiaUlvAb4vae+IWK+CYETMAeaAr52zeus4iSRtCPwF8JbGvLSQ/Uvp9F2SFgN7AIXqbeeV\nPXZqnKBtNc8xlB/DaMTR7vu6/W9RZIj7XcCvI2JJY4akbRq3UpG0K0kF1EeKhTgyrf5RRvuKB8dQ\nrRh6HUeeIe4rSW6NsqekJZKOShdNZ92uHMABwAJJ9wDfBY6JiGe6Fq1ZBXVaAZWI+HiLedcA1xQP\ny6w+fMWCWUF9mUTZ/m5ZV4A7hurE0Os4avGnECNRhasbHMNgxVCLP8rzyVYrwxFLl+b6o7xaJJFZ\nSfrnL1uT619H5vI//WcAPvqLz3U7GMdQwxg6i2NmrrX6cmDBbDQ5icwKchKZFVSLY6Lx229Vynu7\nxTFUJwbIH8fyfH8J4ZbIrKhatETbjB/ZHbrPPfuznHDS5QBcfulnOeGk0b+TnWOoTgydxjGwLdEV\nl5zFuHGbvPx63LhNuOKSsxzDAMfQ6zjq0RJtu8WI39P8j9TJZxTlGKoTQy/jqMUVCyO9lfy3Lzlj\nndcf+fhpIw+qIMdQnRg6jePmG6b0z2U/I00is27Im0R9d0xkNtry/Hn4jpJukbRQ0v2Sjk/nbynp\nRkkPp89j0/mS9FVJiyQtkDS51xthVqY8LdEa4MSIeB0wBThW0l7AycBNEbE7cFP6GuAQkgIlu5PU\nlbug61GbVciwSRQRyyLi7nT6eWAhMAE4FLg0Xe1S4LB0+lDgskjcBmwhabuuR25WESMa4k7LCb8Z\nuB0YFxHLIEk0Sdumq00Ansi8bUk6b1nTZ+WugHrzDVNGEqbZqMqdRJI2Jank8+mIWJ2U4W69aot5\n642+uQKq9Ytco3OSXkWSQFdExPfS2Ssa3bT0eWU6fwmwY+btOwA5L6Awq588o3MCvgksjIhzM4uu\nA45Mp48EfpCZ/7F0lG4K8Fyj22fWlyJiyAfwZyTdsQXA/PQxDdiKZFTu4fR5y3R9Af8GLAbuBfbJ\n8R3hhx8VfNw53L4bEfW4YsGsJL5iwWw0OInMCnISmRXkJDIrqCp/lPdb4Pfpc7/Ymv7Znn7aFsi/\nPTvn+bBKjM4BSLozz0hIXfTT9vTTtkD3t8fdObOCnERmBVUpieaUHUCX9dP29NO2QJe3pzLHRGZ1\nVaWWyKyWnERmBZWeRJKmSnowLWxy8vDvqB5Jj0m6V9J8SXem81oWcqkiSXMlrZR0X2ZebQvRtNme\n0yU9mf4fzZc0LbNsVro9D0p674i/MM+l3r16AGNI/mRiV2Aj4B5grzJj6nA7HgO2bpr3JeDkdPpk\n4Oyy4xwi/gOAycB9w8VP8mcw15P8ycsU4Pay48+5PacD/9hi3b3S/W5jYGK6P44ZyfeV3RLtCyyK\niEci4n+Bq0gKnfSDdoVcKicifgY80zS7toVo2mxPO4cCV0XESxHxKLCIZL/MrewkalfUpG4C+LGk\nu9ICLNBUyAXYtu27q6ld/HX+P5uZdkHnZrrXhben7CTKVdSkBvaLiMkkNfeOlXRA2QH1UF3/zy4A\ndgMmkVSeOiedX3h7yk6ivihqEhFL0+eVwLUk3YF2hVzqoq8K0UTEiohYGxF/BC7mlS5b4e0pO4nu\nAHaXNFHSRsB0kkIntSFpE0mbNaaB9wD30b6QS130VSGapuO2w0n+jyDZnumSNpY0kaRy7y9H9OEV\nGEmZBjxEMipyatnxdBD/riSjO/cA9ze2gTaFXKr4AK4k6eL8H8kv81Ht4qeDQjQV2Z7L03gXpImz\nXWb9U9PteRA4ZKTf58t+zAoquztnVntOIrOCnERmBTmJzApyEpkV5CQyK8hJZFbQ/wPTMFRqoBLr\nRQAAAABJRU5ErkJggg==\n", | |
| "text/plain": [ | |
| "<matplotlib.figure.Figure at 0x7f3da1e3e0f0>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "data": { | |
| "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFz5JREFUeJzt3X20HHV9x/H35z5xQxIeEkIMJAUfooItpi1G6sORolhE\nFGzViihpy7HtsfRYH9qqfcJWrZ6K2HP06EFFUquAjzVVasmJIIVaHsSIQagBBBMTEhACuXm6T9/+\nMXPL3jtzc/fe3Z3dze/zOmfP3f3N7M539u53Z+a3M7+vIgIzS09PuwMws/Zw8pslyslvlignv1mi\nnPxmiXLymyXKyZ8wSSdKCkl97Y5lNiRdIOm6dsfR7Zz8TSTpBkmPSTqswmWGpGdUtbyqlX1BRcQX\nIuLl7YzrUODkbxJJJwIvBgJ4dVuD6SDK+HPWgfxPaZ4Lgf8BrgTW1E6QtFjSv0t6QtJtkt4v6aaa\n6c+WtF7So5L+V9Lra6ZdKekTkr4labekWyQ9PZ92Yz7bDyUNSfrdqUFJ6pH015IelLRT0r9IOnLK\nbH8gaZuk7ZLeWfPc1ZJuz+PeIemjNdNOk/TfknZJ+qGk02um3SDpA5JuBvYC75V0+5S43i5pXX7/\nlZJ+kC9ni6RLamadWMdd+Tr+hqTfm/L+vSB/Xx/P/75gSiz/IOnm/P27TtIxU9+nJEWEb024AfcC\nbwV+HRgBltZMuzq/HQ6cDGwBbsqnzc8f/z7QB/wa8AjwnHz6lcCjwOp8+heAq2teO4BnHCSuP8hj\nexqwAPga8Pl82on586/K4/gV4GHgZfn07wFvzu8vAE7L7x8P/AI4m2wDcmb+eEk+/QbgZ8Bz8piP\nBHYDK2viug14Q37/9HzZPcApwA7gvCkx9tU89/dq3r9FwGPAm/NlnZ8/XlwTy33AM4F5+eMPtfvz\n0gk3b/mbQNKLgBOAL0XE98k+bG/Mp/UCvwP8XUTsjYgfA2trnn4O8EBEfC4iRiPiDuCrwGtr5vla\nRNwaEaNkyb9qFuFdAHw0Iu6PiCHgPcAbpnTyvS8i9kTEj4DPkSUQZF9iz5B0TEQMRcT/5O1vAq6N\niGsjYjwi1gO3k30ZTLgyIu7K1+lx4BsTrytpJfBsYB1ARNwQET/KX+tOsi+jl9S5fq8ENkfE5/Nl\nXQXcA7yqZp7PRcRPImIf8CVm9/4dspz8zbEGuC4iHskff5End/2XkG2RttTMX3v/BOD5+e7zLkm7\nyBL2KTXzPFRzfy/ZVrhexwEP1jx+MI9n6TTxPJg/B+Aisi3mPfnu9Dk1Mb9uSswvApZN85qQvScT\nXypvBP4tIvYCSHq+pOslPSzpceCPgXp3zaeu38Q6HF/zuJH375DVVT/xdCJJ84DXA72SJj5khwFH\nSXousAkYBZYDP8mnr6h5iS3AdyPizBaFuI0sWSf8Uh7PjjymiXjuqZm+DSAiNgPn5x12vw18RdLi\nPObPR8RbDrLcqZeLXgccI2kV2ZfA22umfRH4OPCKiNgv6WM8mfwzXXY6df0m1uHbMzwved7yN+48\nYIzsWH5VfjsJ+C/gwogYIzvOvkTS4ZKeTdY5OOGbwDMlvVlSf357nqST6lz+DrLj+elcBbxd0lMl\nLQA+CFyTH0JM+Js8tueQ9T1cAyDpTZKWRMQ4sCufdwz4V+BVkn5LUq+kQUmnS1rONPLlfQX4J7Lj\n9PU1kxcCj+aJv5r8kCn3MDB+kHW8luz9e6OkvrzT82Sy99UOwsnfuDVkx5Q/i4iHJm5kW7IL8mPr\ni8k6vR4CPk+WkAcAImI38HLgDWRbsYeAD5PtPdTjEmBtvvv9+pLpV+TLvBH4KbAf+NMp83yXrFNw\nA/CRiJg4geYs4C5JQ8A/k3XQ7Y+ILcC5wHvJknML8OfM/Hn6IvAy4MtTvnzeCvy9pN3A35IdlwOQ\nHxp8ALg5X8fTal8wIn5B1m/yTrJOx78Azqk5BLNpKO8RtQpJ+jDwlIhYM+PMZi3iLX8F8t/xT8nO\nd9Fqso60r7c7LkubO/yqsZBsV/84YCdwKdlPX2Zt491+s0R5t98sUQ3t9ks6i6wXuBf4TER86GDz\n9w/Mj8HBoxtZpJkdxP79jzEyvEf1zDvn5M9PW/0E2XndW4HbJK3LT18tNTh4NKeuvniuizSzGdx+\n68frnreR3f7VwL35OePDZBeunNvA65lZhRpJ/uOZfP72ViafTw2ApD/MLwu9fWRkTwOLM7NmaiT5\ny44rCj8dRMTlEXFqRJza3z+/gcWZWTM10uG3lckXqCwnvyBkOhraR//NmxpYpJkdjA7sq3veRrb8\ntwEr8wtGBsjOTV/XwOuZWYXmvOWPiFFJFwP/SfZT3xURcVfTIjOzlmrod/6IuJbskkoz6zI+w88s\nUZVe2BNHzGP/i0+pcpFmSYn/uqHueb3lN0uUk98sUU5+s0Q5+c0S5eQ3S1Slvf3Di4It549Oaovx\n4iUCkkcXMoio/7Mxm3k7SbPjHr6r/ud6y2+WKCe/WaKc/GaJcvKbJaracftDxNjkDo4YKX7/lHUC\ntpsGxgttU9cFgLK2duor7wBST7E9hjtsW1ASI/3F/wNADPcW2zqsv6/0MzRa/Lw09PmfxXM77L9t\nZlVx8pslyslvlignv1miGq3Y8wCwGxgDRiPi1IPOPyL6fj657LzGGomgQmVfk2UdSh3WyTStsvUp\n70vrLNNtrro19ibHrZH6O/ya0dv/mxHxSBNex8wq5N1+s0Q1mvwBXCfp+5L+sGyG2oo9Y3tcsces\nUzS62//CiNgm6VhgvaR7IuLG2hki4nLgcoDB5Su65YjY7JDX6NDd2/K/OyV9nax4543Tzg/ElBOx\nVNbh0YFfEVPjhmk6Kzss9rK4gdJ9vtL/RTuV9F2NT7M+PZ0We4my2MtOYqzqMzTn3X5J8yUtnLgP\nvBxwLS6zLtHIln8p8HVJE6/zxYj4dlOiMrOWa6Rc1/3Ac5sYi5lVyD/1mSWq0kt6RUknWYd1kE2n\nGzr3ykx7BmUXdJCVvb893XJGaInS2Nv4GfKW3yxRTn6zRDn5zRLl5DdLlJPfLFGV9vYHEFO+bnx6\nb2tNe3pvyamzGi22tdUhdnpv6WeoG0/vNbPu5uQ3S5ST3yxRTn6zRFVbsacvGF1cR69Sh3WaAaWd\nTx0Z51SzKf7SaevTaPGjblifJscY/S7RbWYzcPKbJcrJb5YoJ79Zombs8JN0BXAOsDMifjlvWwRc\nA5wIPAC8PiIem3Fp40J7p47g2Wm9MmZdrMkluq8EzprS9m5gQ0SsBDbkj82si8yY/Pk4/I9OaT4X\nWJvfXwuc1+S4zKzF5nrMvzQitgPkf4+dbsZJFXuGXLHHrFO0vMMvIi6PiFMj4tTeBfNbvTgzq9Nc\nz/DbIWlZRGyXtAzYWdezAnpGpjY2ehqXmf2/WfSfz3XLvw5Yk99fA3xjjq9jZm0yY/JLugr4HvAs\nSVslXQR8CDhT0mbgzPyxmXWRGXf7I+L8aSa9tMmxmFmFfIafWaIqv6R3bHGhx8/MmqXPl/Sa2Qyc\n/GaJcvKbJcrJb5aoakt0D4uBrQNVLtIsKRpu7iW9ZnYIcvKbJcrJb5YoJ79Zopz8Zoly8pslyslv\nlignv1minPxmiapnJJ8rJO2UtKmm7RJJP5e0Mb+d3dowzazZ5lq0A+CyiFiV365tblhm1mpzLdph\nZl2ukWP+iyXdmR8WHN20iMysEnNN/k8CTwdWAduBS6ebcVLFnj2u2GPWKeaU/BGxIyLGImIc+DSw\n+iDzPlmxZ74r9ph1ijklf16lZ8JrgE3TzWtmnWnGwTzyoh2nA8dI2gr8HXC6pFVkxYEeAP6ohTGa\nWQvMtWjHZ1sQi5lVyGf4mSXKyW+WKCe/WaKc/GaJcvKbJcrJb5YoJ79Zopz8Zoly8pslyslvlign\nv1minPxmiXLymyXKyW+WKCe/WaKc/GaJcvKbJaqeij0rJF0v6W5Jd0l6W96+SNJ6SZvzvx6+26yL\n1LPlHwXeGREnAacBfyLpZODdwIaIWAlsyB+bWZeop2LP9oi4I7+/G7gbOB44F1ibz7YWOK9VQZpZ\n883qmF/SicCvArcASyNiO2RfEMCx0zzHRTvMOlDdyS9pAfBV4M8i4ol6n+eiHWadqa7kl9RPlvhf\niIiv5c07Jop35H93tiZEM2uFenr7RTZO/90R8dGaSeuANfn9NcA3mh+embXKjEU7gBcCbwZ+JGlj\n3vZe4EPAlyRdBPwMeF1rQjSzVqinYs9NgKaZ/NLmhmNmVfEZfmaJcvKbJaqeY/6m0Tj07Z18BDE2\nLwrzxXQHGW3Ut68Y1PhAcb7x3uL6dIve4fre+LGB7l1He5K3/GaJcvKbJcrJb5YoJ79Zoirt8Ot/\naA/L//G/J7Vte9cLCvMNH9neDqWBJ4odX8ddekuhbdebVhfbVrYkpKZTyVu8Yv1Qoa1vZ/Eyjvsv\nPK7Q1s0dnanylt8sUU5+s0Q5+c0S5eQ3S5ST3yxR1Z7ee9hh9J749Mlt41VGUJ+ekWJb39Ilhbbx\n3gqCaRGNF3/RGD7qsEJb7xNl5zCXvGAXvxep8pbfLFFOfrNEOfnNEtVIxZ5LJP1c0sb8dnbrwzWz\nZqmnw2+iYs8dkhYC35e0Pp92WUR8pN6F7V/Sx0/eMnl4/94DJaeFtvlM0QOLigFsfttTC20aK3ly\nGzswy8ZGmHZwhP3FeR84r2TeviMKTQMPdd7/zGavnjH8tgMTxTl2S5qo2GNmXayRij0AF0u6U9IV\n0xXqdMUes87USMWeTwJPB1aR7RlcWvY8V+wx60xzrtgTETsiYiwixoFPA8XrW82sY814zD9dxR5J\nyyYKdQKvATbN9Fo9IzBv5+ROpZGFnTeAp8aKAQw+XJxvZGGxrZ3XtZ+y+r5C29ED+0rn/e59xYEH\nLlv95ULbkt7i9fwXrntroa1vdweOumoH1UjFnvMlrSLr530A+KOWRGhmLdFIxZ5rmx+OmVXFZ/iZ\nJcrJb5aoSi/pBegZnfy47NLSaPNgkBqts63sDL82Xtq6cePTCm1/dWZ55fS3/MZ3C233DC8rtG3a\nt7zQ1lNnZR/rbN7ymyXKyW+WKCe/WaKc/GaJqrbDTzDeP7mp3Wfz1Wtq3NB5sfc/Xvwu/8Tml5TO\ne8ep1xTaHhrbW2j74I/PKrT1HJhDcNZxvOU3S5ST3yxRTn6zRDn5zRLl5DdLVKW9/dELw0dOOXW3\nAyv2jA0WTy8em1cyYxcMWrnnzkWl7c/ce2GhbXS4+HHof7BYxccODd7ymyXKyW+WKCe/WaLqqdgz\nKOlWST/MK/a8L29/qqRbJG2WdI2kknKuZtap6unwOwCcERFD+Si+N0n6D+AdZBV7rpb0KeAisuG8\np6XBMfpPmjwg5MjdxYow7e4EHDmi2JO38ITHC2177j2y0Na7r7PO+e0ZLY/nWct2FNru2XFsoS1w\nh9+hasYtf2SG8of9+S2AM4Cv5O1rgfNaEqGZtUS94/b35iP37gTWA/cBuyJiYnybrUxTwqu2Ys/o\nE8ULR8ysPepK/rw4xypgOVlxjpPKZpvmuf9fsafviMPnHqmZNdWsevsjYhdwA3AacJSkiT6D5cC2\n5oZmZq1UT8WeJcBIROySNA94GfBh4HrgtcDVwBqgfKTIGhFibKzzf10s67QbGy+Ju9Mu6C8x3l9+\nGuJLj7mn0HbvL44ptI00PSLrFPX09i8D1krqJdtT+FJEfFPSj4GrJb0f+AFZSS8z6xL1VOy5k6ws\n99T2+3FxTrOu1fn74GbWEk5+s0RVeknvQN8oJyx+dFLb/b0LCvOpAy/zXXrE7kLbTw8rxt67v7O+\nT0fnl3f4/crglkLbnkeKP8X6nO1DV2d9Us2sMk5+s0Q5+c0S5eQ3S1SlHX6j4z3sHJrSSdaBnXtj\nC4tB9ajYcdYz0gVn+C0oqS0O3HWgeB2W9rWxvrhVzlt+s0Q5+c0S5eQ3S5ST3yxR1RbtGOpj+ObF\nk9oGy/uj2mrgiWLH17atKwpthw9XEU1j5u0sP0fvW5eeVmg74szitqCsNLl1Lo3VP6+3/GaJcvKb\nJcrJb5YoJ79Zohqp2HOlpJ9K2pjfVrU+XDNrlkYq9gD8eUR85SDPnaRnBOZv74K61ocQjZe/30PP\nOrrQNvho8bTm6On8U5jtST2z+PWsnjH8Aiir2GNmXWxOFXsi4pZ80gck3SnpMkmlRd0mVezZv6dJ\nYZtZo+ZUsUfSLwPvAZ4NPA9YBPzlNM99smLP4PwmhW1mjZprxZ6zImJ7XsTzAPA5PIy3WVeZc8Ue\nScsiYrskkVXo3TTTa4VgzCNCVqy8w2500NfuH4pmU0SqkYo938m/GARsBP54DrGaWZs0UrHnjJZE\nZGaV8Bl+Zoly8pslqtLr+cf7Ye9SnzFm1iqzGX/BW36zRDn5zRLl5DdLlJPfLFGVdvj1jMDhO3xB\noFmr9IzMYt7WhWFmnczJb5YoJ79Zopz8ZomqtMNPMbuKImY2OyWV5KflLb9Zopz8Zoly8pslyslv\nlqi6kz8fvvsHkr6ZP36qpFskbZZ0jSSPzmfWRWbT2/824G7giPzxh4HLIuJqSZ8CLgI+ebAX6BkO\nFm6ZUtS+pCLMeG/5Nf8Du4YLbRorVplphfGB4oCXIwuLF0+X9bb2DRXPuewZruZnj+gt/34fPqr4\nXd0zVgxeo8W2vqHi/6Eqw0eXlodAJbFHX3HdB3YdKD55mqpGzTY2v/h5GRsoxjhdlaR6Pv+9++vP\nh3qLdiwHXgl8Jn8s4AxgolTXWrIRfM2sS9S72/8x4C+Aia+VxcCuiJioDLYVOL7sibUVe0ZGXLHH\nrFPUU6X3HGBnRHy/trlk1tJ9p9qKPf39rthj1inqOeZ/IfBqSWcDg2TH/B8DjpLUl2/9lwPbWhem\nmTVbPeP2v4esLh+STgfeFREXSPoy8FrgamAN8I2ZXmt0vtixenKHzciC4g7D+EB5B8xxNxU7e/qf\nqKbjbOj4YgfZL1aVxFnStPjO4tu8YGs1nWYjC8sr82x7cXGnr/dAcYdu4PFi27G3Nx7XXP38JeUj\nVPbtK8ZZ9tlasaG+Ts1WePTk4ud3aEVJR2VveTxP+d5goe2wxybX5J5NSfVGfuf/S+Adku4l6wP4\nbAOvZWYVm9WFPRFxA1mhTiLiflyc06xr+Qw/s0Q5+c0SpYjqBtSU9DDwYP7wGOCRyhbeWofSuoDX\np9MdbH1OiIgl9bxIpck/acHS7RFxalsW3mSH0rqA16fTNWt9vNtvlignv1mi2pn8l7dx2c12KK0L\neH06XVPWp23H/GbWXt7tN0uUk98sUZUnv6SzJP2vpHslvbvq5TdK0hWSdkraVNO2SNL6fEiz9ZKO\nbmeMsyFphaTrJd0t6S5Jb8vbu26dJA1KulXSD/N1eV/e3tVDzrVqCL1Kk19SL/AJ4BXAycD5kk6u\nMoYmuBI4a0rbu4ENEbES2JA/7hajwDsj4iTgNOBP8v9JN67TAeCMiHgusAo4S9JpPDnk3ErgMbIh\n57rJxBB6E5qyPlVv+VcD90bE/RExTHY58LkVx9CQiLgReHRK87lkQ5lBlw1pFhHbI+KO/P5usg/Z\n8XThOkVmKH/Yn9+CLh5yrpVD6FWd/McDW2oeTzv8V5dZGhHbIUsm4Ng2xzMnkk4EfhW4hS5dp3wX\neSOwE1gP3EedQ851qDkPoTeTqpO/7uG/rFqSFgBfBf4sIp5odzxzFRFjEbGKbHSp1cBJZbNVG9Xc\nNDqE3kwqLdRJ9i21oubxoTL81w5JyyJiu6RlZFudriGpnyzxvxARX8ubu3qdImKXpBvI+jG6dci5\nlg6hV/WW/zZgZd5bOQC8AVhXcQytsI5sKDOoc0izTpEfQ34WuDsiPlozqevWSdISSUfl9+cBLyPr\nw7iebMg56JJ1gWwIvYhYHhEnkuXKdyLiApq1PhFR6Q04G/gJ2bHYX1W9/CbEfxWwHRgh25O5iOw4\nbAOwOf+7qN1xzmJ9XkS223gnsDG/nd2N6wScAvwgX5dNwN/m7U8DbgXuBb4MHNbuWOewbqcD32zm\n+vj0XrNE+Qw/s0Q5+c0S5eQ3S5ST3yxRTn6zRDn5zRLl5DdL1P8B8FPBd33wU/8AAAAASUVORK5C\nYII=\n", | |
| "text/plain": [ | |
| "<matplotlib.figure.Figure at 0x7f3dd8d382b0>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "output_type": "display_data" | |
| } | |
| ], | |
| "source": [ | |
| "s = env.reset()\n", | |
| "for _ in range(100):\n", | |
| " s, _, _, _ = env.step(env.action_space.sample())\n", | |
| "\n", | |
| "plt.title('Game image')\n", | |
| "plt.imshow(env.render('rgb_array'))\n", | |
| "plt.show()\n", | |
| "\n", | |
| "plt.title('Agent observation')\n", | |
| "plt.imshow(s.reshape([42,42]))\n", | |
| "plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Simple agent for fully-observable MDP\n", | |
| "\n", | |
| "Here's a code for an agent that only uses feedforward layers. Please read it carefully: you'll have to extend it later!" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "import torch\n", | |
| "import torch.nn as nn\n", | |
| "import torch.nn.functional as F\n", | |
| "from torch.autograd import Variable\n", | |
| "\n", | |
| "# a special module that converts [batch, channel, w, h] to [batch, units]\n", | |
| "class Flatten(nn.Module):\n", | |
| " def forward(self, input):\n", | |
| " return input.view(input.size(0), -1)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "class FeedforwardAgent(nn.Module):\n", | |
| " def __init__(self, obs_shape, n_actions):\n", | |
| " \"\"\"A simple actor-critic agent\"\"\"\n", | |
| " super(self.__class__, self).__init__()\n", | |
| " \n", | |
| " # Note: number of units/filters is arbitrary, you can and should change it at your will\n", | |
| " self.conv0 = nn.Conv2d(1, 32, kernel_size=(3,3), stride=(2,2))\n", | |
| " self.conv1 = nn.Conv2d(32, 32, kernel_size=(3,3), stride=(2,2))\n", | |
| " self.conv2 = nn.Conv2d(32, 32, kernel_size=(3,3), stride=(2,2))\n", | |
| " self.flatten = Flatten()\n", | |
| " \n", | |
| " self.hid = nn.Linear(512, 128)\n", | |
| " self.logits = nn.Linear(128, n_actions)\n", | |
| " self.state_value = nn.Linear(128, 1)\n", | |
| " \n", | |
| " \n", | |
| " def forward(self, prev_state, obs_t):\n", | |
| " \"\"\"\n", | |
| " Takes agent's previous step and observation (np.array), \n", | |
| " returns next state and whatever it needs to learn (torch variables)\n", | |
| " \"\"\" \n", | |
| " nn = F.elu(self.conv0(obs_t))\n", | |
| " nn = F.elu(self.conv1(nn))\n", | |
| " nn = F.elu(self.conv2(nn))\n", | |
| " nn = self.flatten(nn)\n", | |
| " nn = F.elu(self.hid(nn))\n", | |
| " logits = self.logits(nn)\n", | |
| " state_value = self.state_value(nn)[:,0]\n", | |
| " \n", | |
| " # feedforward agent has no state\n", | |
| " new_state = []\n", | |
| " \n", | |
| " return new_state, (logits, state_value)\n", | |
| " \n", | |
| " def get_initial_state(self, batch_size):\n", | |
| " \"\"\"\n", | |
| " Return a list of agent memory states at game start (Variables).\n", | |
| " Each state is a np array of shape [batch_size, ...]\n", | |
| " \"\"\"\n", | |
| " # feedforward agent has no state\n", | |
| " return []\n", | |
| " \n", | |
| " def sample_actions(self, agent_outputs):\n", | |
| " \"\"\"pick actions given numeric agent outputs (np arrays)\"\"\"\n", | |
| " logits, state_values = agent_outputs\n", | |
| " probs = F.softmax(logits)\n", | |
| " return torch.multinomial(probs, 1)[:, 0].data.numpy()\n", | |
| " \n", | |
| " def step(self, prev_state, obs_t):\n", | |
| " \"\"\" like forward, but obs_t is not Variable \"\"\"\n", | |
| " obs_t = Variable(torch.FloatTensor(np.array(obs_t)))\n", | |
| " return self.forward(prev_state, obs_t)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "n_parallel_games = 5\n", | |
| "gamma = 0.99\n", | |
| "\n", | |
| "agent = FeedforwardAgent(obs_shape, n_actions)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 55, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "action logits:\n", | |
| " Variable containing:\n", | |
| "\n", | |
| "Columns 0 to 9 \n", | |
| "-0.0325 0.0137 0.0580 0.0096 -0.1176 -0.0218 -0.0212 0.0147 -0.0366 -0.0625\n", | |
| "\n", | |
| "Columns 10 to 13 \n", | |
| " 0.0484 0.0207 -0.0204 -0.0532\n", | |
| "[torch.FloatTensor of size 1x14]\n", | |
| "\n", | |
| "state values:\n", | |
| " Variable containing:\n", | |
| "1.00000e-02 *\n", | |
| " -3.0021\n", | |
| "[torch.FloatTensor of size 1]\n", | |
| "\n" | |
| ] | |
| }, | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": [ | |
| "/home/jheuristic/anaconda3/lib/python3.6/site-packages/scipy/misc/pilutil.py:482: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n", | |
| " if issubdtype(ts, int):\n", | |
| "/home/jheuristic/anaconda3/lib/python3.6/site-packages/scipy/misc/pilutil.py:485: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", | |
| " elif issubdtype(type(size), float):\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "state = [env.reset()]\n", | |
| "_, (logits, value) = agent.step(agent.get_initial_state(1), state)\n", | |
| "print(\"action logits:\\n\", logits)\n", | |
| "print(\"state values:\\n\", value)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Let's play!\n", | |
| "Let's build a function that measures agent's average reward." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def evaluate(agent, env, n_games=1):\n", | |
| " \"\"\"Plays an entire game start to end, returns session rewards.\"\"\"\n", | |
| "\n", | |
| " game_rewards = []\n", | |
| " for _ in range(n_games):\n", | |
| " # initial observation and memory\n", | |
| " observation = env.reset()\n", | |
| " prev_memories = agent.get_initial_state(1)\n", | |
| "\n", | |
| " total_reward = 0\n", | |
| " while True:\n", | |
| " new_memories, readouts = agent.step(prev_memories, observation[None, ...])\n", | |
| " action = agent.sample_actions(readouts)\n", | |
| "\n", | |
| " observation, reward, done, info = env.step(action[0])\n", | |
| "\n", | |
| " total_reward += reward\n", | |
| " prev_memories = new_memories\n", | |
| " if done: break\n", | |
| " \n", | |
| " game_rewards.append(total_reward)\n", | |
| " return game_rewards\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "env_monitor = gym.wrappers.Monitor(env, directory=\"kungfu_videos\", force=True)\n", | |
| "rw = evaluate(agent, env_monitor, n_games=3,)\n", | |
| "env_monitor.close()\n", | |
| "print (rw)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "#show video\n", | |
| "from IPython.display import HTML\n", | |
| "import os\n", | |
| "\n", | |
| "video_names = list(filter(lambda s:s.endswith(\".mp4\"),os.listdir(\"./kungfu_videos/\")))\n", | |
| "\n", | |
| "HTML(\"\"\"\n", | |
| "<video width=\"640\" height=\"480\" controls>\n", | |
| " <source src=\"{}\" type=\"video/mp4\">\n", | |
| "</video>\n", | |
| "\"\"\".format(\"./kungfu_videos/\"+video_names[-1])) #this may or may not be _last_ video. Try other indices\n", | |
| "\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Training on parallel games\n", | |
| "\n", | |
| "We introduce a class called EnvPool - it's a tool that handles multiple environments for you. Here's how it works:\n", | |
| "" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "from env_pool import EnvPool\n", | |
| "pool = EnvPool(agent, make_env, n_parallel_games)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# for each of n_parallel_games, take 10 steps\n", | |
| "rollout_obs, rollout_actions, rollout_rewards, rollout_mask = pool.interact(10)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "print(\"Actions shape:\", rollout_actions.shape)\n", | |
| "print(\"Rewards shape:\", rollout_rewards.shape)\n", | |
| "print(\"Mask shape:\", rollout_mask.shape)\n", | |
| "print(\"Observations shape: \",rollout_obs.shape)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "# Actor-critic\n", | |
| "\n", | |
| "Here we define a loss function that uses rollout above to train" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def to_one_hot(y, n_dims=None):\n", | |
| " \"\"\" Take an integer vector (tensor of variable) and convert it to 1-hot matrix. \"\"\"\n", | |
| " y_tensor = y.data if isinstance(y, Variable) else y\n", | |
| " y_tensor = y_tensor.type(torch.LongTensor).view(-1, 1)\n", | |
| " n_dims = n_dims if n_dims is not None else int(torch.max(y_tensor)) + 1\n", | |
| " y_one_hot = torch.zeros(y_tensor.size()[0], n_dims).scatter_(1, y_tensor, 1)\n", | |
| " return Variable(y_one_hot) if isinstance(y, Variable) else y_one_hot" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def train_on_rollout(states, actions, rewards, is_not_done, prev_memory_states, gamma = 0.99):\n", | |
| " \"\"\"\n", | |
| " Takes a sequence of states, actions and rewards produced by generate_session.\n", | |
| " Updates agent's weights by following the policy gradient above.\n", | |
| " Please use Adam optimizer with default parameters.\n", | |
| " \"\"\"\n", | |
| " \n", | |
| " # cast everything into a variable\n", | |
| " states = Variable(torch.FloatTensor(np.array(states))) # shape: [batch_size, time, c, h, w]\n", | |
| " actions = Variable(torch.IntTensor(np.array(actions))) # shape: [batch_size, time]\n", | |
| " rewards = Variable(torch.FloatTensor(np.array(rewards))) # shape: [batch_size, time]\n", | |
| " is_not_done = Variable(torch.FloatTensor(is_not_done.astype('float32'))) # shape: [batch_size, time]\n", | |
| "\n", | |
| " \n", | |
| "\n", | |
| " # predict logits, probas and log-probas using an agent. \n", | |
| " memory = prev_memory_states\n", | |
| " \n", | |
| " logits = [] # append logit sequence here\n", | |
| " state_values = [] #append state values here\n", | |
| " for t in range(rewards.shape[1]):\n", | |
| " obs_t = states[:, t]\n", | |
| " \n", | |
| " # use agent to comute logits_t and state values_t.\n", | |
| " # append them to logits and state_values array\n", | |
| " \n", | |
| " < YOUR CODE HERE >\n", | |
| " \n", | |
| " \n", | |
| " logits = torch.stack(logits, dim=1)\n", | |
| " state_values = torch.stack(state_values, dim=1)\n", | |
| " \n", | |
| " probas = <your code here>\n", | |
| " logprobas = <your code here>\n", | |
| " \n", | |
| " # select log-probabilities for chosen actions, log pi(a_i|s_i)\n", | |
| " actions_one_hot = to_one_hot(actions, n_actions).view(actions.shape[0], actions.shape[1], n_actions)\n", | |
| " logprobas_for_actions = torch.sum(logprobas * actions_one_hot,\n", | |
| " dim = -1) \n", | |
| " # Policy gradient objective. \n", | |
| " # Notes: Please don't forget to call .detach() on advantage term. Also please use mean, not sum.\n", | |
| " # it's okay to use loops if you want\n", | |
| " J_hat = <policy objective as in the formula for J_hat.>\n", | |
| " \n", | |
| " # Temporal difference MSE for state values\n", | |
| " # Notes: Please don't forget to call on V(s') term. Also please use mean, not sum.\n", | |
| " # it's okay to use loops if you want\n", | |
| " TD_loss = <Temporal difference MSE for state values>\n", | |
| " \n", | |
| " #regularize with entropy\n", | |
| " entropy = <compute mean entropy of probas. Don't forget the sign!>\n", | |
| " \n", | |
| " loss = - J_hat - 0.01 * entropy_reg + TD_loss\n", | |
| " \n", | |
| " # Gradient descent step\n", | |
| " < your code >\n", | |
| " \n", | |
| " return loss.data.numpy()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# let's test it\n", | |
| "memory = pool.prev_memory_states\n", | |
| "rollout_obs, rollout_actions, rollout_rewards, rollout_mask = pool.interact(10)\n", | |
| "\n", | |
| "train_on_rollout(rollout_obs, rollout_actions, rollout_rewards, rollout_mask, prev_memory_states)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "# Train \n", | |
| "\n", | |
| "just run train step and see if agent learns any better" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "from IPython.display import clear_output\n", | |
| "from tqdm import trange\n", | |
| "from pandas import ewma\n", | |
| "rewards_history = []" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVPe9+P/Xh2ETBRFFRAbFBaMigkDcErOYaFySmEbJ\n0iymN439fbO0vWnTLDe3adPkNm2atOm9TZs9ms3EJWqiidEYq4m4gIL7ggqyKQiKILLO5/fHOSgq\nCsowh5l5Px+PeTDzOWdm3gfxvM/y+XzeSmuNEEII7+NjdQBCCCGsIQlACCG8lCQAIYTwUpIAhBDC\nS0kCEEIILyUJQAghvJQkACGE8FKSAIQQwktJAhBCCC/la3UAF9OjRw8dExNjdRhCCOFWMjIyjmqt\nw1tar0MngJiYGNLT060OQwgh3IpSKrc168klICGE8FKSAIQQwktJAhBCCC/Voe8BNKeuro78/Hyq\nq6utDkVYLDAwELvdjp+fn9WhCOGW3C4B5OfnExwcTExMDEopq8MRFtFaU1paSn5+Pv369bM6HCHc\nkttdAqqurqZ79+6y8/dySim6d+8uZ4JCtIHbJQBAdv4CkL8DIdrKLROAEMJ7ZBdXsHLnEavD8EiS\nANxQTEwMR48etToMIVziqQXb+P8+zKC0ssbqUDyOJIA20lrjcDja7fPr6+vb7bOF6Oi2F5STnnuM\neodmUWah1eF4HEkAlyEnJ4chQ4bw8MMPk5SUxAcffMCYMWNISkoiNTWVyspKNm7cyO233w7A4sWL\n6dSpE7W1tVRXV9O/f38A3nrrLa688koSEhKYPn06VVVVADzwwAM8/vjjXH/99Tz55JOUlpYyceJE\nRowYwc9+9jO01gCcPHmSqVOnkpCQwLBhw/j000+t+YUI0U4+SMulk5+NKyKCmZeed/pvXziH23UD\nber3X+xgZ+EJp37m0N4hPHdLXIvr7dmzh/fee4/nn3+e22+/nZUrV9K5c2f+9Kc/8eqrr/LMM8+w\nZcsWANauXcuwYcPYtGkT9fX1jBo1CoDbb7+dhx56CIBnn32Wd955h8ceewyAvXv3snLlSmw2Gz//\n+c+5+uqr+e1vf8vSpUt58803Afj666/p3bs3S5cuBaC8vNypvwshrHS8qpZFmQVMT7YzJDKE/160\nnR2FJxgW1dXq0DyGnAFcpr59+zJ69GjWr1/Pzp07ueqqq0hMTGT27Nnk5ubi6+vLwIED2bVrFxs3\nbuTxxx9nzZo1rF27lnHjxgGwfft2xo0bR3x8PB999BE7duw4/fmpqanYbDYA1qxZw7333gvA1KlT\n6datGwDx8fGsXLmSJ598krVr19K1q/zHEJ7js/Q8auod3D+mL7cO742/rw+fpedZHZZHceszgNYc\nqbeXzp07A8Y9gAkTJvDJJ5+ct864ceP46quv8PPz48Ybb+SBBx6goaGBv/zlL4BxqWfRokUkJCTw\n/vvvs3r16vM+v1FzXR4HDRpERkYGy5Yt4+mnn2bixIn89re/deJWCmGNBofmg/W5jOoXxuBeIQDc\nFNeLxZmFPDNlCIF+Nosj9AxyBtBGo0eP5ocffiA7OxuAqqoq9u7dC8A111zD3/72N8aMGUN4eDil\npaXs3r2buDgjcVVUVBAZGUldXR0fffTRBb/jmmuuOb38q6++4tixYwAUFhYSFBTEvffey69//Ws2\nb97cnpsqhMus3lNMXtkpZo6NOd12R4qd8lN1rNwlXUKdxa3PADqC8PBw3n//fe6++25qaoxuai+8\n8AKDBg1i1KhRHDlyhGuuuQaA4cOH07Nnz9NH83/4wx8YNWoUffv2JT4+noqKima/47nnnuPuu+8m\nKSmJa6+9lj59+gCwbds2nnjiCXx8fPDz8+Of//ynC7ZYiPY3Oy2XXiGBTBgacbpt7IAe9O4ayLz0\nfG4e3tvC6DyH6sh31VNSUvS5BWF27drFkCFDLIpIdDTy9+B5DpRUMv6Vf/OrCYN47IbYs5a98s0e\n/u+7bNY9NZ7Irp0sirDjU0plaK1TWlpPLgEJITqUD9bn4mdT3DWyz3nLZiTb0RoWbi6wIDLPIwlA\nCNFhnKypZ356PlPjIwkPDjhved/unRnVL0zGBDiJJAAhRIfx+ZYCKmrqub/Jzd9zpaZEk1NaRXru\nMdcF5qEkAQghOgStNXPScoiP6sqI6NALrjclvhed/W18tknGBLRVqxOAUsqmlNqilPrSfN1PKbVB\nKbVPKfWpUsrfbA8wX2eby2OafMbTZvsepdRNzt4YIYT7Wn+gjL1HKrl/TN+LTvUd5O/L1OGRLN1W\nxMkamSurLS7lDOAXwK4mr/8E/FVrHQscAx402x8EjmmtBwJ/NddDKTUUuAuIAyYBryulZDSHEAKA\nOWk5dAvy45aElrt4pqZEU1XbwLJtRe0fmAdrVQJQStmBqcDb5msFjAfmm6vMBm4zn08zX2Muv8Fc\nfxowV2tdo7U+CGQDI52xEUII91Z4/BTf7DzCnVf2adUo35S+3ejXozPzMvJdEJ3nau0ZwN+A3wCN\n8x53B45rrRvPv/KBKPN5FJAHYC4vN9c/3d7Me05TSs1SSqUrpdJLSkouYVO8h7vUA3A4HLz99ttc\nffXVJCQkMGHCBL788suz1pk3bx5xcXH4+Phw7piPP/7xjwwcOJArrriC5cuXuzJ04WIfbziE1pp7\nRp3f9bM5SilmJNvZeLCMnKMn2zk6z9ViAlBK3QwUa60zmjY3s6puYdnF3nOmQes3tdYpWuuU8PDw\nlsKzVHvXAgD3rQegteaee+5h+/btLFiwgKysLN5//30+/PBDXnvttdPrDRs2jIULF54eLd1o586d\nzJ07lx07dvD111/z8MMP09DQ4OrNEC5QU9/AJxsPccOQCKLDglr9vulJdnwUzJezgMvWmqkgrgJu\nVUpNAQKBEIwzglCllK95lG8HGqs15APRQL5SyhfoCpQ1aW/U9D2X56un4PC2Nn3EeXrFw+SXLrg4\nJyeHyZMnc/3115OWlsaiRYvYs2cPzz33HDU1NQwYMID33nuPnTt38tJLL7Fw4UIWL17MXXfdRXl5\nOQ6Hg6FDh3LgwAHeeust3nzzTWpraxk4cCAffPABQUFBPPDAA4SFhbFlyxaSkpJ45plnuPvuuykp\nKWHkyJFn1QO44447yM/Pp6Ghgf/+7//mzjvvPCve/fv388gjj1BSUkJQUBBvvfUWsbGxxMbGsn//\nfsrLywkLC2P16tVcc801jBs3jvfee48PP/yQgwcPUlRUxN69e3n11VdZv349X331FVFRUXzxxRf4\n+fnx/PPP88UXX3Dq1CnGjh3LG2+8gVKK2bNn07dvX1566czvMioqio8//pibbrqJGTNmEBUVdcFR\nvI2/s4CAAPr168fAgQPZuHEjY8aMccI/suhIlm0rovRkLTPHxFzS+3p1DWRcbDgLNufznxMGYfOR\nGtGXqsUzAK3101pru9Y6BuMm7iqt9T3Ad8AMc7WZwGLz+RLzNebyVdrYYy0B7jJ7CfUDYoGNTtsS\nF9qzZw/3338/W7ZsoXPnzrzwwgusXLmSzZs3k5KSwquvvkpSUlKz9QA2bNhwVj2ATZs2kZWVxZAh\nQ3jnnXdOf0djPYBXXnmF3//+91x99dVs2bKFW2+9lUOHDgFn6gFkZWWxfft2Jk2adF6ss2bN4n//\n93/JyMjgL3/5Cw8//DA2m41Bgwaxc+dOvv/+e5KTk1m7di01NTXk5+czcOBAwEgeS5cuZfHixdx7\n771cf/31bNu2jU6dOp2uQfDoo4+yadMmtm/fzqlTp05f4pkzZw7PPPMMJSUlTJkyhbFjx/LEE08w\nb948HnnkkRaL1xQUFBAdfeZ4wW63U1Agoz890ex1ufQP78xVA7tf8nvvSImmqLyaH7I7/iXRjqgt\nk8E9CcxVSr0AbAEa917vAB8opbIxjvzvAtBa71BKfQbsBOqBR7TWbTunv8iRentqrAUAnFUPAKC2\ntpYxY8ZcsB5AQ0PDWfUAnn32WY4fP05lZSU33XSmZ+y59QAWLlwInF8P4Ne//jVPPvkkN9988+nP\nbVRZWcm6detITU093dY4Yd24ceNYs2YNBw8e5Omnn+att97i2muv5corrzy97uTJk/Hz8yM+Pp6G\nhobTCSY+Pp6cnBwAvvvuO/785z9TVVVFWVkZcXFx3HLLLdTX1xMSEsJ//ud/MmvWLG655RZmzJhB\nXFwcw4cPZ8WKFRf9HTc3yvNiXQOFe8rKO05m3nF+f2vcZf373ji0J6FBfszLyOeaQR37knFHdEkJ\nQGu9GlhtPj9AM714tNbVQOq57eayF4EXLzXIjqbpXP0duR6Aw+EgNDSUzMzMZmP717/+RWFhIc8/\n/zwvv/zy6ctAjQICjKH4jbONNsbg4+NDfX091dXVPPzww6SnpxMdHc3vfvc7qqurAU4nr927d/PH\nP/4Rm83GxIkTASguLqZnz54X+Q0bR/x5eWf6DOTn59O7t8wA6WnmpOXS2d/G7Unn9QdplQBfG9MS\nevPJpjzKq+roGuTn5Ag9m4wEbqOOXA8gJCSEfv36MW/ePMBIVllZWQCMGjWKdevW4ePjQ2BgIImJ\nibzxxhvnnUVcTOPOvkePHlRWVjJ//vyzlldUVHDFFVfwzTff4HA4WLFiBdXV1bzyyivn3as41623\n3srcuXOpqanh4MGD7Nu3j5EjpdewJymtrOGLrYVMT7YTHHj5O+7UlGhq6x0syZJLhJdKEkAbNa0H\nMHz4cEaPHs3u3bsBmq0HMHz48PPqAUyYMIHBgwdf8Duee+451qxZQ1JSEt98881Z9QBGjhxJYmIi\nL774Is8+++x57/3oo4945513SEhIIC4ujsWLjVs1AQEBREdHn76UNW7cOCoqKoiPj2/1toeGhvLQ\nQw8RHx/Pbbfddtblo7vvvpvf/va3PP3007z++utcffXVxMbGMnfuXB555JHT2/v5559jt9tJS0tj\n6tSppy+DxcXFcccddzB06FAmTZrEP/7xj9NnFcIzfJqeR61Z8rEt4nqHMCQyRMYEXAapByDahcPh\nYPr06SQmJvL4448THBxMSUkJCxcu5MEHH8TX1zm1iOTvwT3VNzi49uXVxPQI4qOfjm7z5737/UGe\n/3InX/9y3OkSkt5M6gEIS/n4+DB//nzCwsK46aabSEpK4ic/+QmxsbFO2/kL9/Xt7mIKjp/i/kvs\n+nkht42Iws+mmJcuZwGXwi3/J2qtpUeIG7DZbDz22GM89thj7fL5HfnsVVzcnLQcokI7ccPgi3cG\naK2wzv7cMDiCRVsKeGryYPxscmzbGm73WwoMDKS0tFT+83s5rTWlpaUEBgZaHYq4RNnFFfyQXco9\no/vg68QddWqKndKTtazaXey0z/R0bncGYLfbyc/PR+YJEoGBgdjtdqvDEJdo9rpc/H19uDMluuWV\nL8G1g8IJDw5gXno+N8X1cupneyq3SwB+fn7069fP6jCEEJfhRHUdCzbnc8vw3nTvcn7Jx7bwtflw\ne1IUb689SHFFNT2D5eywJW53CUgI4b4WZuRTVdvAAxcp+dgWqcnRNDg0i7bImIDWkAQghHAJh0Mz\nJy2XEX1Cibd3bZfvGNizCyP6hDIvPV/uE7aCJAAhhEv8sP8oB46evORZPy/VHSnR7CuuJCu/vF2/\nxxNIAhBCuMTsdbn06OLP5Pj2vUF78/BIAv18mJcuReNbIglACNHu8sqq+Hb3Ee4e2YcA3/ad0iM4\n0I/JwyJZklVIdZ0UEboYSQBCiHb34YZcfJTix60s+dhWqcl2KqrrWb7jsEu+z11JAhBCtKvqugY+\n3ZTHTXERRHbt5JLvHN2/O/ZunWRqiBZIAhBCtKslWYUcr6pz2rw/reHjYxSN/2H/UfKPVbnse92N\nJAAhRLvRWjN7XQ5XRAQzql+YS797epIdrWFBhowJuJAWE4BSKlAptVEplaWU2qGU+r3Z/r5S6qBS\nKtN8JJrtSin1d6VUtlJqq1IqqclnzVRK7TMfMy/0nUIIz7D50HF2FJ7g/rF9XT6BY3RYEGMHdGf+\n5jwcDhkT0JzWnAHUAOO11glAIjBJKdU4gfcTWutE89FYd3AyRsH3WGAW8E8ApVQY8BwwCqOU5HNK\nqW7O2xQhREczJy2H4EBfbku8vJKPbXVHSjR5ZafYcLDMku/v6FpMANpQab70Mx8XS6fTgDnm+9YD\noUqpSOAmYIXWukxrfQxYAUxqW/hCiI6quKKaZduKSE2OpnOANdOO3RTXi+AAX+ZlyJiA5rTqHoBS\nyqaUygSKMXbiG8xFL5qXef6qlGqc2SkKaPrbzjfbLtQuhPBAczfmUdegua+NJR/bopO/jZsTerNs\nWxEV1XWWxdFRtSoBaK0btNaJgB0YqZQaBjwNDAauBMKAJ83Vm7vQpy/Sfhal1CylVLpSKl2mfBbC\nPdU1OPhoQy7XDgqnX4/OlsaSmmKnus7B0q1FlsbREV1SLyCt9XFgNTBJa11kXuapAd7DuK4PxpF9\n04m+7UDhRdrP/Y43tdYpWuuU8PDwSwlPCNFBfLPjCEdO1DBzrHVH/41GRIcyILyzFI1vRmt6AYUr\npULN552AG4Hd5nV9lHFr/zZgu/mWJcD9Zm+g0UC51roIWA5MVEp1M2/+TjTbhBAeZnZaDn3Cgrh2\nkHNKPraFUoo7UqLJyD3G/pLKlt/gRVpzBhAJfKeU2gpswrgH8CXwkVJqG7AN6AG8YK6/DDgAZANv\nAQ8DaK3LgD+Yn7EJeN5sE0J4kF1FJ9h4sIz7RvfF5tMxanf/KCkKm49ivpwFnKXFW/Na663AiGba\nx19gfQ08coFl7wLvXmKMQgg3Mictl0A/H1JTOk65zp7BgVw3KJwFGfn8asIgp9YidmfyWxBCOE15\nVR2LthRwW2IUoUH+VodzltQUO8UVNazdd9TqUDoMSQBCCKeZl5HHqboGS7t+Xsj4wRGEdfaXMQFN\nSAIQQjiFw6H5YH0uV8Z0I653+5R8bAt/Xx9uS4xixc4jlJ2stTqcDkESgBDCKf69r4Tc0iqXzvp5\nqVJT7NQ1aBZnygRxIAlACOEkc9bl0DM4gJvi2rfkY1sMiQxhWFSI1AkwSQIQQrRZztGTrN5bwo9H\n9cHft2PvVu5IiWZn0Ql2FErR+I79LyWEcAsfrs/FphQ/Humako9tcWtCb/xtPnIWgCQAIUQbVdXW\n81l6HpPjI+kZEmh1OC0KDfJnQlwEizILqKn37qLxkgCEEG2yOLOQE9X1zOyAXT8vJDXZzvGqOr7d\nVWx1KJaSBCCEuGyNJR+HRoaQ3Nd96juNiw2nV0gg89K9e0yAJAAhxGXbeLCM3YcrmGlByce2sPko\npidH8e+9JRw5UW11OJaRBCCEuGxz0nLp2smPWxPcr7bTjORoHBoWbvbeMQGSAIQQl+VweTVf7zjM\nnVdG08nfZnU4l6xfj85cGdONeel5GHNYeh9JAEKIy/LxhlwcWnPvKPe5+Xuu1ORoDhw9yeZDx6wO\nxRKSAIQQl6ymvoGPNx7ihsE96dM9yOpwLtuU4ZEE+du8dkyAJAAhxCX7evthjlbWduh5f1qjS4Av\nU+Ij+XJrEVW19VaH43KSAIQQl2z2uhz69+jM1QN7WB1Km6Um26msqeerbYetDsXlJAEIIS7Jtvxy\nNh86zn1j+uLTQUo+tsXIfmH07R7klXUCWlMUPlAptVEplaWU2qGU+r3Z3k8ptUEptU8p9alSyt9s\nDzBfZ5vLY5p81tNm+x6l1E3ttVFCiPYzJy2HIH8b05M7TsnHtlBKkZpsZ/2BMg6VVlkdjku15gyg\nBhivtU4AEoFJSqnRwJ+Av2qtY4FjwIPm+g8Cx7TWA4G/muuhlBoK3AXEAZOA15VS7td3TAgvduxk\nLYuzCrk9KYqQQD+rw3Ga25PsKAXzN3vXzeAWE4A2VJov/cyHBsYD88322cBt5vNp5mvM5TcoY4jg\nNGCu1rpGa30QyAZGOmUrhBAu8Wl6HrX1Dre/+Xuu3qGduHpgDxZk5ONweM+YgFbdA1BK2ZRSmUAx\nsALYDxzXWjfeNs8HGocCRgF5AObycqB70/Zm3iOE6OAaHJoP0nIZ0787gyKCrQ7H6VJToik4fop1\n+0utDsVlWpUAtNYNWutEwI5x1D6kudXMn83dFdIXaT+LUmqWUipdKZVeUlLSmvCEEC6wancxBcdP\nMXOs+w78upiJQyMICfT1qpvBl9QLSGt9HFgNjAZClVK+5iI7UGg+zweiAczlXYGypu3NvKfpd7yp\ntU7RWqeEh4dfSnhCiHY0Jy2HyK6B3DgkwupQ2kWgn41piVF8vf0w5afqrA7HJVrTCyhcKRVqPu8E\n3AjsAr4DZpirzQQWm8+XmK8xl6/SxkQbS4C7zF5C/YBYYKOzNkQI0X6yiytZu+8o947ui6/Nc3uP\np6bYqal38OXW845NPVJr/iUjge+UUluBTcAKrfWXwJPA40qpbIxr/O+Y678DdDfbHweeAtBa7wA+\nA3YCXwOPaK29uxyPEG7iw/W5+Nt8uPPK6JZXdmPxUV25IiKYz7xkagjfllbQWm8FRjTTfoBmevFo\nrauB1At81ovAi5cephDCKpU19czPyOfm4ZH06BJgdTjtSilFaoqdF5buYt+RCmI98GZ3U557LieE\ncIrPN+dTWVPP/WNjrA7FJW4bEYWvj2JehuefBUgCEEJckNaa2Wm5JNi7khgdanU4LtGjSwDjB/dk\n4eYC6hocVofTriQBCCEuKG1/KdnFlR438KslqSnRHK2sYfUez+6KLglACHFBs9NyCOvsz9ThkVaH\n4lLXXRFOjy7+Hl80XhKAEKJZBcdPsWLnEe66MppAP++atsvP5sPtSXZW7S7maGWN1eG0G0kAQohm\nfbQ+F4B7RnvmyN+WpCbbqXdoFm3x3KLxkgCEEOeprmtg7qY8JgyNICq0k9XhWCI2IpiE6FDmped7\nbNF4SQBCiPMs3VpE2claZnrZzd9zpSbb2XOkgm0F5VaH0i4kAQghzjMnLYeBPbswZkB3q0Ox1C0J\nvQnw9fHYovGSAIQQZ8nMO05Wfjkzx/TFKOXhvbp28mPSsF4sziygus7zZq6RBCCEOMucdTl0CfDl\nR0meUfKxrVKTozlRXc+KnUesDsXpJAEIIU47WlnDl1uLmJFsp0tAi1OFeYWxA7oTFdqJzzxwTIAk\nACHEaZ9uyqO2wcG9Xtr1szk+PorpSVF8n32UwuOnrA7HqSQBCCEAqG9w8OH6XMbF9mBgzy5Wh9Oh\nzEiORmtY6GFF4yUBCCEAWLHzCEXl1V43709r9OkexOj+YczL8KwxAZIAhBCAMe9PVGgnxg/uaXUo\nHVJqcjS5pVVsPFhmdShOIwlACMGewxWsP1DGfWP6YvPx7q6fFzI5vhddAnw9qk6AJAAhBHPScgjw\n9eHOFM8u+dgWQf6+3Dw8kmXbijhZU291OE7RmqLw0Uqp75RSu5RSO5RSvzDbf6eUKlBKZZqPKU3e\n87RSKlsptUcpdVOT9klmW7ZS6qn22SQhxKUoP1XHws0FTEvsTbfO/laH06Glptipqm1g6bYiq0Nx\nitacAdQDv9JaDwFGA48opYaay/6qtU40H8sAzGV3AXHAJOB1pZRNKWUD/gFMBoYCdzf5HCGERRZk\n5HOqrkFu/rZCUp9u9A/v7DF1AlpMAFrrIq31ZvN5BbALiLrIW6YBc7XWNVrrg0A2RvH4kUC21vqA\n1roWmGuuK4SwiMOh+WB9Lsl9uzEsqqvV4XR4SilmJNvZlHOMg0dPWh1Om13SPQClVAwwAthgNj2q\nlNqqlHpXKdXNbIsCmqbHfLPtQu3nfscspVS6Uiq9pMSzy7EJYbW12Uc5ePQk94+RgV+tNT3Jjo+C\n+RnufxbQ6gSglOoCLAB+qbU+AfwTGAAkAkXAK42rNvN2fZH2sxu0flNrnaK1TgkPD29teEKIyzBn\nXQ49ugQweZh3lXxsi4iQQK4dFM6CjAIaHO49JqBVCUAp5Yex8/9Ia70QQGt9RGvdoLV2AG9hXOIB\n48i+aVcCO1B4kXYhhAUOlVaxak8xPx7VB39f6RB4KVJTojl8oprvs49aHUqbtKYXkALeAXZprV9t\n0t70kOFHwHbz+RLgLqVUgFKqHxALbAQ2AbFKqX5KKX+MG8VLnLMZQohL9eGGXGxKcc+oPlaH4nZu\nGNKT0CA/t58grjXT/V0F3AdsU0plmm3PYPTiScS4jJMD/AxAa71DKfUZsBOjB9EjWusGAKXUo8By\nwAa8q7Xe4cRtEUK00qnaBj7dlMdNw3oRERJodThuJ8DXxm2JUXy84RDHq2oJDXLP7rMtJgCt9fc0\nf/1+2UXe8yLwYjPtyy72PiGEayzJKqD8VJ3Xl3xsi9QUO++vy2FJVqHbdqGVC39CeBmtNbPX5TK4\nVzBXxnRr+Q2iWXG9uzI0MsSty0VKAhDCy2TkHmNn0Qlmjo3x+pKPbZWaYmdbQTm7ik5YHcplkQQg\nhJeZnZZLSKAv0xJ7Wx2K25uWGIWfTbntWYAkACG8SPGJar7aVsQdKdEE+UvJx7YK6+zPhKERLMos\noLbeYXU4l0wSgBBe5OONh2jQWko+OlFqcjRlJ2tZtbvY6lAumSQAIbxEbb2DjzYc4rpB4cT06Gx1\nOB5jXGwPegYHuOUEcZIAhPASy3ccpqSihvvHxlgdikfxtflwe5Kd1XtLKK6otjqcSyIJQAgvMSct\nh77dg7g2VubYcrbUFDsNDs3nmwusDuWSSAIQwgvsKCxnU84x7hvdFx8p+eh0A8K7kNy3m9sVjZcE\nIIQX+CAtl05+NlKTpeRje0lNtpNdXElm3nGrQ2k1SQBCeLjjVbUsyizgthFRdA3yszocjzV1eCSB\nfj585kZjAiQBCOHh5qXnU13nkKIv7Sw40I8pwyL5MquQU7UNVofTKpIAhPBgDWbJx5H9whgSGWJ1\nOJdOa9jwBnzyY1j1Auz6Ao7lGu0dUGpKNBU19SzfcdjqUFpFhgIK4cH+vbeYQ2VVPDlpsNWhXLpT\nx2DRI7BnKXSNhr1fgzaPrANDITLh7EfYAPCx9ph2VL8wosM6MS8jj9tGXKx0escgCUAIDzZ7XS4R\nIQFMjIvQzZKnAAAd80lEQVSwOpRLU7gFPpsJJwph0p9g1M+gvhqO7ISiTDi8FYqyYMO/oKHWeI9/\nF+gVD72Gn0kK4VeAzXX3PXx8FDOSovnryr3klVURHRbksu++HJIAhPBQB4+e5N97S3h8wiD8bG5y\ntVdr2PQ2LH8GukTAf3wN9hRjmV8nsCcbj0YNdVCyG4rMhFCUBVs+hI1vGMttARAx9OwzhZ5x4Nd+\nRXCmJ0fxt2/3smBzPr+8cVC7fY8zSAIQwkPNScvBz6a4a6SbdP2sqYAvfgHbF0DsRPjRGxAUdvH3\n2PzMo/54GHGP0eZogLIDZkLINH7uWAQZ7xvLlQ3CB5sJwTxb6BUPAcFO2Qx7tyCuGtCD+Rn5/Hx8\nbIced9FiAlBKRQNzgF6AA3hTa/2aUioM+BSIwSgJeYfW+phZQ/g1YApQBTygtd5sftZM4Fnzo1/Q\nWs927uYIIQBO1tQzPz2fKfGR9Ax2g5KPR3bAZ/cbO+4bnoOrfnn51/N9bNAj1njEzzDatIbjh86c\nJRzeCtkrIetj800Kug84+/JRZELLCegCUlPs/GJuJusPljJ2QI/L2w4XaM0ZQD3wK631ZqVUMJCh\nlFoBPAB8q7V+SSn1FPAU8CQwGaMQfCwwCvgnMMpMGM8BKRh1hDOUUku01secvVFCeLvPtxRQUVPv\nHqUKt3wES38FgV1h5hcQc7Xzv0Mp6NbXeAy99Ux7xeEml48yIT8ddiw8s7xr9NkJoddwCO5lfN5F\n3BTXi+BAX+al57t3AtBaFwFF5vMKpdQuIAqYBlxnrjYbWI2RAKYBc7QxHnq9UipUKRVprrtCa10G\nYCaRScAnTtweIbye1po5aTkMiwohqU+o1eFcWG0VLHsCMj+EftfC9LehS0/XxhDcy3gMmnimrars\nzE3moiwjQexeinHcCnTuefblo8gECO17VlII9LNxS0JvFm7O5/fT4ggJ7JgD8C7pHoBSKgYYAWwA\nIszkgNa6SCnV+C8XBTSdFzXfbLtQuxDCidYfKGPvkUr+PGN4xy35eHSf0cuneCdc+6Tx8LFZHZUh\nKAz6X2c8GtVUwOHtZy4fFWXB/lVNuqV2bXL5KBEiE7gjqTcfbzjE0q1F3D2yj+u3oxVanQCUUl2A\nBcAvtdYnLvKH1dwCfZH2c79nFjALoE+fjvlLE6Ijm5OWQ2iQH7cmdNCSj9sXwJKfg28A3LsABt5g\ndUQtCwiGvmOMR6O6aiOBnT5TyIKNb0FDDQAJfkEs7dyH3O9iwW+SkSDCB4Ovv0Ubcb5WJQCllB/G\nzv8jrXXjBbIjSqlI8+g/Emgsh5MPNO12YAcKzfbrzmlffe53aa3fBN4ESElJ6ZjD/YTooAqPn+Kb\nnUf46bh+BPp1kCPqRvU1RvfOTW9D9GiY8S50deOLAH6BEJVkPBo11MHRvVCUhSrKImz3BvoeXwGL\nvzCW2/yh59Aml5ASISLO6OJqgdb0AlLAO8AurfWrTRYtAWYCL5k/Fzdpf1QpNRfjJnC5mSSWA/+j\nlOpmrjcReNo5myGEAPh4wyG01tw7qoPN+3Msx7jkU5QJY38ON/zWpQO0XMbmZ+zQI+Ig8cfYrq4m\n8Y8reSLFj5/FVpy5fLRrCWw2O0EqG/QYdM7N5ngIbP+pO1pzBnAVcB+wTSmVabY9g7Hj/0wp9SBw\nCEg1ly3D6AKajdEN9CcAWusypdQfgE3mes833hAWQrRdTX0Dn2w8xA1DIjrWCNTdS2HR/zOe3/UJ\nDJ5ibTwu1DM4kOuuiODtXeU8OO12fJt2Sy3PO3sA24HVsHXumTf3vw7uX9zMpzpPa3oBfU/z1+8B\nzrt4Z/b+eeQCn/Uu8O6lBCiEaJ1l24ooPVnLzI7S9bOhDlb+DtL+D3qPgNT3oVuMxUG5XmpKNCt3\nFbNmXwnjB5tTcigFoX2Mx5Cbz6xcccQ8S8gE3/a/LCQjgYXwELPX5dI/vDNXDexudShQXgDzfwJ5\nG2DkLJj4gnHT1wuNH9yT7p39mZeefyYBXEhwBARPgNgJLonNTSYIEUJcTFbecTLzjjNzTIz1XT+z\nv4U3xhmje2e8C1Ne9tqdP4CfzYfbRkSxctcRyk7WWh3OWSQBCOEB5qTl0tnfxu1JFvaqcTTAqhfh\nw+nQpRfM+jcMm25dPB1IaoqdugbNoi0dq2i8JAAh3FxpZQ1fbC1kerKdYKtGnFYcgTnTYM2fjUnZ\nfroSegy0JpYOaHCvEIbbuzIvo2OVi5QEIISb+zQ9j9p6C0s+5nxvXPLJT4dpr8O0f4B/B+qF1EGk\nJtvZVXSC7QXlVodymiQAIdxYfYODj9Yf4qqB3RnY0znTGbeawwFrX4HZt0BACDy06syUzOI8tyZE\n4e/rw7z0vJZXdhFJAEK4sW93F1Nw/JTrZ/2sKoNP7oRvn4e4H8Gs74zCK+KCugb5MXFoBIuzCqmp\n7xhF4yUBCOHG5qTlEBXaiRsGu3AWzbxN8K9xxsClqa/A9HecVkzF092REs3xqjpW7ixueWUXkAQg\nhJvKLq7gh+xS7hndB19XlHzUGtJeh/cmGTN3PvgNXPnTFufGF2dcNbAHkV0DmZfRMS4DSQIQwk3N\nScvF39eHO1NcUPKxuhw+uw+WPw2DJsHP1hije8UlsfkopifZWbO3hMPl1VaHIwlACHdUUV3Hgox8\nbhnem+5d2nmQVVEWvHEt7PkKJr4Id34InTpwoZkObkayHYeGBZut7xIqCUAIN7RwcwEnaxuYObYd\nu35qDenvwtsToKEWHlgGYx+VSz5tFNOjMyP7hTE/Ix9j6jTrSAIQws1orZmdlkNidCjD7e10JF5T\nCQtnwZf/Cf3Gwc/WQp9R7fNdXig12c7BoyfJyLW2JLokACHczA/ZpRwoOdl+R//Fu+Gt8bB9Pox/\nFn48Dzp3gAnmPMiU+EiC/G3MS7f2MpAkACHczOy0HLp39mdKfKTzPzxrLrx1PZw6ZsxFf80T4CO7\nCWfrHODL1PhIvtxaSFVtvWVxyHTQwm3VNTjYcug49Q0Oq0NxmYqaer7ddYSHrxtIgK8TSz7WnYKv\nfgOb50Dfq2HGOxDcy3mfL86TmhLNvIx8lm07zIxkuyUxSAIQbsfh0HyxtZBXV+wlt7TK6nBczs+m\n+PGoPs77wNL9RrnGI9tg3K/huqfBJruG9nZlTDdiugcxLz1PEoAQLdFa8+2uYv7yzR52H65gSGQI\n//fjEYS3dzfIDqZHcAC9Q51ULWrHIlj8qFHL9p75LitEIkApRWpKNC8v30Nu6Un6du/s8hhaUxT+\nXeBmoFhrPcxs+x3wEFBirvaM1nqZuexp4EGgAfi51nq52T4JeA2wAW9rrV9y7qYIT7b+QCkvL99D\nRu4xYroH8fe7R3BzfCQ+PtIl8bLU18KK/4YN/wL7lUa5xq7WHIV6s9uTonjlmz3Mz8jnVxOvcPn3\nt+YM4H3g/4A557T/VWv9l6YNSqmhwF1AHNAbWKmUGmQu/gcwAcgHNimllmitd7YhduEFtuWX8/I3\ne1izt4SIkAD+50fxpKbY8XPF1Aee6vghmPcAFGTA6Efgxt+Br7/FQXmnyK6duDo2nAUZ+fzyxkHY\nXHxA05qi8GuUUjGt/LxpwFytdQ1wUCmVDYw0l2VrrQ8AKKXmmutKAhDNyi6u5NUVe1i27TChQX48\nM2Uw94+JIdDPiTc+vdGer+HznxmDvO78EIbcYnVEXu+OFDuPfryFdfuPMi423KXf3ZZ7AI8qpe4H\n0oFfaa2PAVHA+ibr5JttAHnntMuoEnGeguOneG3lXuZn5BPoZ+Pn4wfy02v6E2JVpStP0VAPq/4A\nP/wNeg2HO2ZDWH+roxLAjUMi6NrJj3np+W6TAP4J/AHQ5s9XgP8Amjt/0TQ/3qDZMdBKqVnALIA+\nfZzY00F0aEcra/jHd9l8tP4QAA+M7cfD1w+gh5fd4G0XJwph/oNwaB2k/Afc9EfwC7Q6KmEK9LMx\nLbE3czflUV5VR9cg1x3sXFYC0FofaXyulHoL+NJ8mQ80nZrQDhSazy/Ufu5nvwm8CZCSkmLtRBmi\n3Z2oruPtNQd45/uDnKprIDU5mp/fGEuUs3q5eLv938GCnxr9/G9/G4anWh2RaEZqcjRz0nJZsrWQ\n+0a7rrTnZSUApVSk1rrIfPkjYLv5fAnwsVLqVYybwLHARowzg1ilVD+gAONG8Y/bErhwb9V1DcxJ\ny+H11fs5XlXH1PhIHp84iAHhXawOzTM4GmDNy7D6JQgfDHfMgfBBLb9PWGJYVAiDewUzPz2vYyUA\npdQnwHVAD6VUPvAccJ1SKhHjMk4O8DMArfUOpdRnGDd364FHtNYN5uc8CizH6Ab6rtZ6h9O3RnR4\ndQ0OPkvP4+/f7uPIiRquGRTOExOvIN7e1erQPEdlCSx8CA58Bwl3G1W7/F3fx1y0XuOYgD98uZO9\nRyoYFOGaCmvK6ulILyYlJUWnp6dbHYZwgsbRu39dsZec0iqS+oTym0mDGd1fJhlzqtx1MP8/jLl8\npvwFRtwr0ze7idLKGkb9z7f85KoY/mtq2+orK6UytNYpLa0nI4FFu9Jas2p3MS8vN0bvDu4VzDsz\nUxg/uCdKdkzO43DAur8bRdq7xcA986BXvNVRiUvQvUsANwzpyedbCvjNpMEuGesiCUC0mw3m6N30\n3GP07R7Ea3clcsvw3jJ619mqymDRw7D3Kxh6G9z6vxAYYnVU4jKkJkezfMcRvttdzMS49p+MTxKA\ncLrtBeW8vHwP/95bQs/gAF64bRh3Xhkto3fbQ36GMaq3oggmvwwjH5JLPm7suivCCQ8OYF5GviQA\n4V72l1Ty6oq9LN1aRGiQH09PNkbvdvKX0btOpzVsfBOW/xcER8KDyyEq2eqoRBv52ny4fUQU73x/\nkJKKGsKD23ccjCQA0WaFx0/x2sp9zN+cT4CvD4+NH8hDMnq3/VSfgCWPwc5FMGgS3PZPCAqzOirh\nJKkpdt5Yc4BFWwp46Jr2Ha0tCUBcttLKGl5fvZ8P1ueChvvH9OWR6wfK6F1nq6mAYznGo+wgZLxv\nPJ/wPIx5TCp2eZiBPYNJjA5lfka+JADR8VRU1/HW2oO8s/YAp+oamJ5k5xc3xmLvFmR1aO7J4YDK\nw2d28Mdy4NjBM6+rjp69fmhfeOBL6DvWgmCFK/zu1ji6dmr/M2hJAKLVqusa+CAtl9dXZ3Osqo7J\nw3rxq4mDGNjTNYNW3FpdNRzPbX4nfywH6qvPrKt8IMQOYTEweKrRrTOsn/GzWwx06ub6+IVLJUaH\nuuR7JAGIFtU1OJiXns/fv93H4RPVjIvtwRM3XcFwu2v+SN2C1lBVeuGj+Ipzpr7y62zs1LsPhIE3\nNtnJ94Ou0TI/v3AJSQDighwOzZfbivjrir0cPHqSEX1C+eudiYwZ4KWjdxvqoDyvmR18jvGztuLs\n9YMjjR17/+vOOYrvB517SHdNYTlJAOI8WmtW7ynh5eV72Fl0gisignnr/hRuHOIFo3eryy98FF+e\nD8bUVgZbAHTra+zU+449ewcf2gf85Z6I6NgkAYizbMop489f72ZTzjH6hAXxtzsTuSWht8tL1bUb\nh8O4HNPcDv5YDpwqO3v9oO7GTt1+JcSnnr2TD46UHjjCrUkCEIAxevcv3+xh9R5j9O4fbhvGnSnR\n+Pu64Q6utsq44Vp28Pwd/PFcaKg9s66yQWi0sUMfOu3sHXy3vhAos5QKzyUJwMsdMEfvfrm1iK6d\n/Hhq8mBmusvo3eN5cCgNyg6cvZOvPHz2ev7BRo+ankPgisln7+S7RoNN/hsI7yR/+V6qqNwYvTsv\nIx9/mw+PXm+M3nVF3+PLVlMJuT9A9rewfxWU7jMXKAjpbezQB95o7Oy7mT1qusUYo2Q9/d6FEJdB\nEoCXKTtZy+vfZTNnfS5aa+4bbYzebe85Ry6LwwFHtp3Z4R9aD4468O0EMVcZ9W37XWN0pZQat0Jc\nMkkAXqKypp631x7g7bUHqaqt50cj7PzyxliiwzpYT5WKw0Yd2/2rjIpWJ0uM9ohhMPr/wYDx0GeM\n7PCFcAJJAB6uuq6BD9fn8vrq/ZSdrGVSnDF6N9ZFJedaVFdtXMff/62x4z9ilpcO6mHs7AeMhwHX\nQ3D7T40rhLdpTU3gd4GbgWKt9TCzLQz4FIjBqAl8h9b6mDI6ib8GTAGqgAe01pvN98wEnjU/9gWt\n9Wznbopoqr7BwfyMfF77dh9F5cbo3V9PvIIEFw0xvyCtoWSPucNfBTk/QP0p8PGDPqPhxt8ZO/2I\neOliKUQ7a80ZwPvA/wFzmrQ9BXyrtX5JKfWU+fpJYDIQaz5GAf8ERpkJ4zkgBaOQfIZSaonW+piz\nNkQYHA7Nsu1FvPrNXg4cPUlidCiv3JHA2AE9rAuqqsy4nLN/lXGUf6LAaO8eC8kzjR1+36sgoIt1\nMQrhhVpMAFrrNUqpmHOapwHXmc9nA6sxEsA0YI42Ks2vV0qFKqUizXVXaK3LAJRSK4BJwCdt3gIB\nGDv+f+8t4S/f7GFH4QkGRXThzfuSmTA0wvWjdxvqIH+TscPP/hYKtwDa6FPf/zoY8KRxWSe0j2vj\nEkKc5XLvAURorYsAtNZFSqmeZnsUkNdkvXyz7ULtog201uwqqmBxVgFfZBZSWF5NdFgnXr0jgWmJ\nUa4dvVt2wOyt8x0cXGPMi6NsYE+B6542jvKjksDHDcYXCOElnH0TuLk9jr5I+/kfoNQsYBZAnz5y\nhNicQ6VVLMkqYHFmIfuKK/H1UVwzKJzfTBrMlPhI14zerT5h7Oj3rzKu5x/LMdpD+0D8DGOH3+8a\n6CQzhgrRUV1uAjiilIo0j/4jgWKzPR+IbrKeHSg02687p311cx+stX4TeBMgJSWl2SThjYorqlm6\ntYjFmYVk5h0HYGRMGC/cNowp8ZGEdW7n6YMdDVCYeWaHn7fRmBjNvwvEjIMxjxo7/bD+MuhKCDdx\nuQlgCTATeMn8ubhJ+6NKqbkYN4HLzSSxHPgfpVRjJYuJwNOXH7Z3OFFdx/Lth1mSVcgP2UdxaBga\nGcJTkwdzS0JvokI7tW8A5fnmDn8VHFgNp44BCiIT4OpfGjt8+0iZu14IN9WabqCfYBy991BK5WP0\n5nkJ+Ewp9SBwCEg1V1+G0QU0G6Mb6E8AtNZlSqk/AJvM9Z5vvCEszlZd18B3u4tZnFnIqj3F1NY7\n6BMWxCPXD+TWhN7t23+/tsqYaqHx5u3RPUZ7cCRcMcXY4fe/zpjLXgjh9pTRYadjSklJ0enp6VaH\n0e7qGxykHShlcWYhy7cfpqKmnh5dArh5eCTTEnuTGB3aPj15tDYGXjXu8A+lGTNl+gYa89sPGA8D\nbjAmUZPLOkK4DaVUhtY6paX1ZCSwRbTWZOYdZ3FmIV9uLeJoZQ3BAb5MGtaLaYlRjO4fhq+tHW7m\nVhafmWph/yo4ad6+6TkURs4y++SPBb92vrwkhLCcJAAX23ekgsWZhSzJKuRQWRX+vj7cMLgn0xJ7\nc90VPQn0c3I3yfoaYxK1xpu3h7cZ7UHdof/1Z6ZbCIl07vcKITo8SQAuUHD8FF9kFbI4s5BdRSfw\nUXDVwB48Nn4gNw3rRUigE6dg1hqO7juzw8/5HuqqwMcXokfDDb81dvi9EmSqBSG8nCSAdlJ2spZl\n24pYklnIxhzjfveIPqH87pahTB3e23nTL9dVQ/EOKMqCgs1Gb51yc8xd2AAYca+xw4+5GgI6yARw\nQogOQRKAE52sqWfFziMszixg7b6j1Ds0A3t24dcTB3FrQhR9urdx6uXqE8YlnKIsOLzV+Fmy50yh\n8sBQ6DcOxj1u7PS7xbR5m4QQnksSQBvV1jtYs7eExVmFrNh5mOo6B727BvLguH5MS4hiSGTw5fXg\nqSyBw1nGTr7I3NkfO3hmeZdeEDkcBk+FXsON56F9pbeOEKLVJAFcBodDs+FgGUuyCli27TDlp+ro\nFuTHjGQ70xKjSO7TDZ/WzsOjtXHJpnEnf3ir8byi8Mw63WKMnfyIeyAy0XgeHNEu2yaE8B6SAFpJ\na82OwhMszizgi6wiDp+oJsjfxsShEUxLjOLq2B74tdRt09EApfvNnXymsaM/vNUcYQsoH+gxyLiM\n02u4MeK2V7zMpyOEaBeSAFpw8OhJlmQWsjirgAMlJ/GzKa4d1JP/mjqEG4b0JMj/Ar/C+loo2XX2\nkf3h7VB30lhu8zf63g+51bh80ysBIuLAv4OVaBRCeCxJAM04cqKaL7IK+SKrkKz8cpSCUf3CeGhc\nfyYP60Vo0Dlz39SeNHbujTdmi7KgeJdRwByMCdN6xRs9ciITjB1++GCwObH7pxBCXCJJAKbyqjq+\n3mHMtpl2oBStIT6qK89OHcLNw3vTq6tZhLyqDA5sPXP5pijL6HffOLt1pzBjJz/mYfMSToIxQ6b0\nuRdCdDBenQCq6xr4dlcxizMLWL2nhNoGB/16dObn42O5NSGSAYGVxg4+c/6Z3jjlh858QEiUsZOP\nu/3MkX1IlPTEEUK4Ba9LAPUNDr7PPsqSzEKW7zjMydoGenbx5+cjbNwSUUafmnTU4a0wOwtOlpx5\nY9gAsCfDlf9x5gatzIophHBjXpEAtNZsPnSMxZmFfJ2VT+ipHFIC8ngj/AjxtkOElO9CbT8B2zGm\nTAgfDAMnnDmqjxgGgSFWb4YQQjiVRyeAvfklrF+/lsN7NhJ1ai/TbTk865OHf0CtsUJ5oLFzj59h\nXq8fbvTM8Qu0NnAhhHABj0wAhXn7qXnvdvo3HGKQcgBQGxiMT+8EfHtPMo7qIxOgeyzYPPJXIIQQ\nLfLIvV/PCDs7AiLYFXkjMXFjCO6XjH+3GLk5K4QQTXhkAvD1DyDhyW+sDkMIITq0NnVOV0rlKKW2\nKaUylVLpZluYUmqFUmqf+bOb2a6UUn9XSmUrpbYqpZKcsQFCCCEujzNGJ12vtU5sUn/yKeBbrXUs\n8K35GmAyEGs+ZgH/dMJ3CyGEuEztMTx1GjDbfD4buK1J+xxtWA+EKqWkDqEQQlikrQlAA98opTKU\nUrPMtgitdRGA+bOn2R4F5DV5b77Zdhal1CylVLpSKr2kpOTcxUIIIZykrTeBr9JaFyqlegIrlFK7\nL7Juc11w9HkNWr8JvAmQkpJy3nIhhBDO0aYzAK11ofmzGPgcGAkcaby0Y/4sNlfPB6KbvN0ONKl6\nIoQQwpUuOwEopTorpYIbnwMTMSZTWALMNFebCSw2ny8B7jd7A40GyhsvFQkhhHC9tlwCigA+N+vd\n+gIfa62/VkptAj5TSj0IHAJSzfWXAVOAbKAK+EkbvlsIIUQbKa077mV2pVQJkNuGj+gBHHVSOO7C\n27bZ27YXZJu9RVu2ua/WOryllTp0AmgrpVR6k/EJXsHbttnbthdkm72FK7ZZylQJIYSXkgQghBBe\nytMTwJtWB2ABb9tmb9tekG32Fu2+zR59D0AIIcSFefoZgBBCiAvwyASglJqklNpjTj39VMvvcG9K\nqXeVUsVKqe1Wx+IqSqlopdR3SqldSqkdSqlfWB1Te1NKBSqlNiqlssxt/r3VMbmCUsqmlNqilPrS\n6lhcpbmp9tvlezztEpBSygbsBSZgTD+xCbhba73T0sDakVLqGqASY7bVYVbH4wrmNCORWuvN5oj0\nDOA2D/93VkBnrXWlUsoP+B74hTm7rsdSSj0OpAAhWuubrY7HFZRSOUCK1rpdxz544hnASCBba31A\na10LzMWYitpjaa3XAGVWx+FKWusirfVm83kFsItmZpf1JOZU6pXmSz/z4VlHcOdQStmBqcDbVsfi\niTwxAbRq2mnhOZRSMcAIYIO1kbQ/83JIJsYkiyu01p6+zX8DfgM4rA7ExZqbat/pPDEBtGraaeEZ\nlFJdgAXAL7XWJ6yOp71prRu01okYs+mOVEp57CU/pdTNQLHWOsPqWCxwldY6CaOS4iPmZV6n88QE\nINNOewnzOvgC4COt9UKr43ElrfVxYDUwyeJQ2tNVwK3m9fC5wHil1IfWhuQaF5hq3+k8MQFsAmKV\nUv2UUv7AXRhTUQsPYt4QfQfYpbV+1ep4XEEpFa6UCjWfdwJuBC5WhMmtaa2f1lrbtdYxGP+PV2mt\n77U4rHZ3kan2nc7jEoDWuh54FFiOcWPwM631Dmujal9KqU+ANOAKpVS+ORW3p7sKuA/jqDDTfEyx\nOqh2Fgl8p5TainGgs0Jr7TVdI71IBPC9UioL2Ags1Vp/3R5f5HHdQIUQQrSOx50BCCGEaB1JAEII\n4aUkAQghhJeSBCCEEF5KEoAQQngpSQBCCOGlJAEIIYSXkgQghBBe6v8Hx0u6aIFv9tIAAAAASUVO\nRK5CYII=\n", | |
| "text/plain": [ | |
| "<matplotlib.figure.Figure at 0x7f14cbf28860>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": [ | |
| " 6%|▌ | 584/10000 [02:24<38:50, 4.04it/s]" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "for i in trange(5000): \n", | |
| " \n", | |
| " memory = pool.prev_memory_states\n", | |
| " rollout_obs, rollout_actions, rollout_rewards, rollout_mask = pool.interact(10)\n", | |
| " train_on_rollout(rollout_obs, rollout_actions, rollout_rewards, rollout_mask, prev_memory_states) \n", | |
| " \n", | |
| " if i % 100 == 0: \n", | |
| " rewards_history.append(np.mean(evaluate(agent, env, n_games=1)))\n", | |
| " clear_output(True)\n", | |
| " plt.plot(rewards_history, label='rewards')\n", | |
| " plt.plot(ewma(np.array(rewards_history),span=10), label='rewards ewma@10')\n", | |
| " plt.legend()\n", | |
| " plt.show()\n", | |
| " if rewards_history[-1] >= 10000:\n", | |
| " print(\"Your agent has just passed the minimum homework threshold\")\n", | |
| " break" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### \"Final\" evaluation" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "env_monitor = gym.wrappers.Monitor(env, directory=\"kungfu_videos\", force=True)\n", | |
| "final_rewards = evaluate(agent, env_monitor, n_games=20,)\n", | |
| "env_monitor.close()\n", | |
| "print(\"Final mean reward\": np.mean(final_rewards))\n", | |
| "\n", | |
| "video_names = list(filter(lambda s:s.endswith(\".mp4\"),os.listdir(\"./kungfu_videos/\")))\n", | |
| "HTML(\"\"\"\n", | |
| "<video width=\"640\" height=\"480\" controls>\n", | |
| " <source src=\"{}\" type=\"video/mp4\">\n", | |
| "</video>\n", | |
| "\"\"\".format(\"./kungfu_videos/\"+video_names[-1])) #this may or may not be _last_ video. Try other indices\n", | |
| "\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### POMDP setting\n", | |
| "\n", | |
| "The atari game we're working with is actually a POMDP: your agent needs to know timing at which enemies spawn and move, but cannot do so unless it has some memory. \n", | |
| "\n", | |
| "Let's design another agent that has a recurrent neural net memory to solve this.\n", | |
| "\n", | |
| "__Note:__ it's also a good idea to scale rollout_len up to learn longer sequences. You may wish set it to >=20 or to start at 10 and then scale up as time passes." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "class SimpleRecurrentAgent:\n", | |
| " def __init__(self, obs_shape, n_actions, reuse=False):\n", | |
| " \"\"\"A simple actor-critic agent\"\"\"\n", | |
| " <YOUR CODE>\n", | |
| " \n", | |
| " def forward(self, prev_state, obs_t):\n", | |
| " \"\"\"Takes agent's previous step and observation, returns next state and whatever it needs to learn (tf tensors)\"\"\"\n", | |
| " \n", | |
| " <YOUR CODE>\n", | |
| " \n", | |
| " # YOUR CODE: apply recurrent neural net for one step here.\n", | |
| " # See docs on self.rnn0(...)\n", | |
| " # the recurrent cell should take the last feedforward dense layer as input\n", | |
| " raise NotImplementedError(\"Please implement rnn step\")\n", | |
| " \n", | |
| " new_state = <YOUR CODE>\n", | |
| " \n", | |
| " return new_state, (logits, state_value)\n", | |
| " \n", | |
| " def get_initial_state(self, batch_size):\n", | |
| " \"\"\"Return a list of agent memory states at game start. Each state is a np array of shape [batch_size, ...]\"\"\"\n", | |
| " return <YOUR CODE>\n", | |
| "\n", | |
| " \n", | |
| " def sample_actions(self, agent_outputs):\n", | |
| " \"\"\"pick actions given numeric agent outputs (np arrays)\"\"\"\n", | |
| " logits, state_values = agent_outputs\n", | |
| " probs = F.softmax(logits)\n", | |
| " return torch.multinomial(probs, 1)[:, 0].data.numpy()\n", | |
| " \n", | |
| " def step(self, prev_state, obs_t):\n", | |
| " \"\"\" like forward, but obs_t is not Variable \"\"\"\n", | |
| " obs_t = Variable(torch.FloatTensor(np.array(obs_t)))\n", | |
| " return self.forward(prev_state, obs_t)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 52, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "agent = SimpleRecurrentAgent(obs_shape, n_actions)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Now let's train it!" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "### A whole lot of your code here: train the new agent with GRU memory.\n", | |
| "# - create pool\n", | |
| "# - write loss functions and training op\n", | |
| "# - train\n", | |
| "# You can reuse most of the code with zero to few changes" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "```\n", | |
| "\n", | |
| "```\n", | |
| "```\n", | |
| "\n", | |
| "```\n", | |
| "```\n", | |
| "\n", | |
| "```\n", | |
| "```\n", | |
| "\n", | |
| "```\n", | |
| "```\n", | |
| "\n", | |
| "```\n", | |
| "```\n", | |
| "\n", | |
| "```\n", | |
| "```\n", | |
| "\n", | |
| "```\n", | |
| "```\n", | |
| "\n", | |
| "```\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "### Homework assignment is TBA" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "anaconda-cloud": {}, | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.6.2" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 1 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment