colab-gym-keras-rl.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"gym-keras-rl.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"GGIkB5ypTFMq","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":161},"outputId":"4971f504-443d-4ad0-e891-21d5f1a08873","executionInfo":{"status":"ok","timestamp":1536514366566,"user_tz":-540,"elapsed":4907,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["!df -h"],"execution_count":9,"outputs":[{"output_type":"stream","text":["Filesystem      Size  Used Avail Use% Mounted on\n","overlay         359G  8.2G  332G   3% /\n","tmpfs           6.4G     0  6.4G   0% /dev\n","tmpfs           6.4G     0  6.4G   0% /sys/fs/cgroup\n","tmpfs           6.4G  249M  6.2G   4% /opt/bin\n","/dev/sda1       365G   11G  355G   3% /etc/hosts\n","shm              64M     0   64M   0% /dev/shm\n","tmpfs           6.4G     0  6.4G   0% /sys/firmware\n"],"name":"stdout"}]},{"metadata":{"id":"aT-Os6_6cSPp","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"0ef70099-54e9-4eae-efeb-d8170b3f1e2c","executionInfo":{"status":"ok","timestamp":1536550018978,"user_tz":-540,"elapsed":2826,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["!mkdir project\n","!ls"],"execution_count":11,"outputs":[{"output_type":"stream","text":["project  sample_data\n"],"name":"stdout"}]},{"metadata":{"id":"UrPTQwoghMv2","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"8c80948c-e5d0-4605-95a7-b0b0b8748419","executionInfo":{"status":"ok","timestamp":1536594908912,"user_tz":-540,"elapsed":4082,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["!cd sample_data"],"execution_count":1,"outputs":[{"output_type":"stream","text":["\u001b[0m\u001b[01;34msample_data\u001b[0m/\n"],"name":"stdout"}]},{"metadata":{"id":"z5P17iZChSrF","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"54510d79-ff3b-40e7-d265-4677efe306ca","executionInfo":{"status":"ok","timestamp":1536550723527,"user_tz":-540,"elapsed":585,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["import keras\n","keras.__version__"],"execution_count":29,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'2.1.6'"]},"metadata":{"tags":[]},"execution_count":29}]},{"metadata":{"id":"jSH21eMuZY_0","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":287},"outputId":"6616d27c-4b99-499b-f1df-e54332b98868","executionInfo":{"status":"ok","timestamp":1536514607275,"user_tz":-540,"elapsed":1731,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["from tensorflow.python.client import device_lib\n","device_lib.list_local_devices()"],"execution_count":13,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[name: \"/device:CPU:0\"\n"," device_type: \"CPU\"\n"," memory_limit: 268435456\n"," locality {\n"," }\n"," incarnation: 16719986220226534363, name: \"/device:GPU:0\"\n"," device_type: \"GPU\"\n"," memory_limit: 11281989632\n"," locality {\n","   bus_id: 1\n","   links {\n","   }\n"," }\n"," incarnation: 11146876127476096945\n"," physical_device_desc: \"device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7\"]"]},"metadata":{"tags":[]},"execution_count":13}]},{"metadata":{"id":"euItnxKYca3Q","colab_type":"code","colab":{"resources":{"http://localhost:8080/nbextensions/google.colab/files.js":{"data":"Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=","ok":true,"headers":[["content-type","application/javascript"]],"status":200,"status_text":""}},"base_uri":"https://localhost:8080/","height":74},"outputId":"a7690dd7-d0c4-4378-d567-c3e04668e891","executionInfo":{"status":"ok","timestamp":1536532177761,"user_tz":-540,"elapsed":172566,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["from google.colab import files\n","uploaded = files.upload()"],"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/html":["\n","     <input type=\"file\" id=\"files-51915982-403d-4088-9b97-7c8d7c41f4d3\" name=\"files[]\" multiple disabled />\n","     <output id=\"result-51915982-403d-4088-9b97-7c8d7c41f4d3\">\n","      Upload widget is only available when the cell has been executed in the\n","      current browser session. Please rerun this cell to enable.\n","      </output>\n","      <script src=\"/nbextensions/google.colab/files.js\"></script> "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["Saving dqn_BreakoutDeterministic-v4_weights_1750000.h5f to dqn_BreakoutDeterministic-v4_weights_1750000.h5f\n"],"name":"stdout"}]},{"metadata":{"id":"1H73-hB7eJGM","colab_type":"code","colab":{}},"cell_type":"code","source":["!pip install gym keras-rl pyglet==1.2.4\n","!apt-get install -y cmake zlib1g-dev libjpeg-dev xvfb ffmpeg xorg-dev python-opengl libboost-all-dev libsdl2-dev swig\n","!pip install 'gym[all]'"],"execution_count":0,"outputs":[]},{"metadata":{"id":"hRy1xdunfxry","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":486},"outputId":"dec05cac-cc93-4ad0-915a-daf06e444adb","executionInfo":{"status":"ok","timestamp":1536533985819,"user_tz":-540,"elapsed":4,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["import gym\n","from IPython import display\n","import matplotlib.pyplot as plt\n","%matplotlib inline\n","\n","env = gym.make('Breakout-v0')\n","env.reset()\n","\n","plt.figure(figsize=(6,8))\n","img = plt.imshow(env.render(mode='rgb_array'))\n","\n","for _ in range(100):\n","    img.set_data(env.render(mode='rgb_array'))\n","    display.display(plt.gcf())\n","    display.clear_output(wait=True)\n","    action = env.action_space.sample()\n","    env.step(action)\n","env.close()"],"execution_count":26,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXEAAAHVCAYAAAAHCstcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGR9JREFUeJzt3X+w5XV93/HnCSSVLiOI2wSkJsQm\n+3aSZdoJoWCRskSCSGmYEYx/rLeKODoBHDm3JtVYEWimdqTcmwQYW0aCetVRhDEs6iAVmsBA+CGT\nxDVx3orT2I6LhYW6ASTLLpz+cb4XD8v9cfbc77nnvJfnY2aHc77f7/l+X3su93U/9/P9nu92er0e\nkqSafmrSASRJo7PEJakwS1ySCrPEJakwS1ySCrPEJamwg9veYUTMAycCPeB9mflA28eQJPW1OhKP\niFOAX87M1wHnA3/c5v4lSS/U9nTKG4A/BcjMbwOviIiXt3wMSdKiXq/X2p9NmzZdu2nTprMHnt+1\nadOmTcttD/S2b9/eoz/1MtV/KuQ040srZ4WMVXJWyLhcj7Y+J76Pzkort2/fzubNmxcLfepVyGnG\n9lTIWSEj1Mg5zRk7neWrtO0S3wEcOfD8VcDDy2187LHH0uv1Vgw4LSrkNGN7KuSskBFq5KyQcTlt\nz4nfBpwLEBG/BuzIzCdaPoYkqdFqiWfmPcCDEXEP/StTLmxz/5KkF+pMch6o0+n0qvwaUyGnGdtT\nIWeFjFAjZ5GMSwb0E5uSVJglLkmFWeKSVJglLkmFjfvDPmrJ3Nxcq9stZXZ2dmz7HnY/65VhJatl\nmAaL78M4348K78MwVnuPqv89HYlLUmGWuCQVZolLUmHOiR8gZmdn6Xa7y87vTcNc8jRkWK8c47ba\n13sYB8L7IEfiklSaJS5JhVniklSYJS5JhXliUzpAeeLypcGRuCQVZolLUmGWuCQV5py4dICahg9f\nafwciUtSYZa4JBVmiUtSYc6JHyDW4x8JGDbDJE1DhvUwDV9vTQdH4pJUmCUuSYVZ4pJUWKfX603u\n4J1Or9fr0el0JpZhWBVymrE9FXJWyAg1chbJuGRAR+KSVJglLkmFWeKSVNhErxOvdq1rhZxmbE+F\nnBUyQo2cFTIuxZG4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4\nJBVmiUtSYZa4JBU28l0MI+JjwMnNPj4K/BZwHPBYs8kVmfmVNSeUJC1rpBKPiFOBzZn5uoh4JfCX\nwB3ABzPzy20GlCQtb9SR+J3A/c3jHwEbgINaSSRJGtpIJZ6ZzwJPNU/PB74KPAtcFBGzwCPARZm5\ns5WUkqQlrelfu4+Is4HfB04Hfh14LDP/KiI+APzTzLxopdfv3Lmzt3HjxpGPL0kvBfPz83S73SX/\ntfu1nNh8I/Ah4IzM3AXcPrB6G/Dx1faxsLBAt9tlfn5+1BjrpkJOM7anQs4KGaFGzgoZlzPSJYYR\ncRhwBXBWZj7eLLspIl7TbLIF+FYrCSVJyxp1JP5WYCNwQ0QsLrse+EJE/Bh4Ejhv7fEkSSsZ9cTm\ntcC1S6z61NriSJL2h5/YlKTCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKmxNN8Baq/n5\n+d5a71kwOzvbYqLl9Xo9Op0l7z8zNczYngo5K2SEGjnHnXFubm7N+1juBliOxCWpMEtckgqzxCWp\nMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtc\nkgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqz\nxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgo7eJQXRcQW4IvA3zSLtgMfAxaAg4CHgZnM3N1CRknS\nMtYyEv/zzNzS/HkvcDlwTWaeDDwEvLOVhJKkZbU5nbIF2NY8vgU4rcV9S5KWMNJ0SuNXImIbcARw\nGbBhYPrkEeCo1XYwMzMDQLfbHTnEWl67v3q93roda1RmbE+FnBUyQo2c05xxfn5+2XWjlvh36Rf3\nDcBrgP+5z746w+xkYWGBbre7YsDVzM7Ojvza/dHr9eh0hvprTYwZ21MhZ4WMUCPnuDPOzc2Nbd8j\nlXhm/gD4QvP0exHxQ+D4iDgkM58GjgZ2tJRRkrSMkebEI2JrRLy/eXwk8HPA9cA5zSbnALe2klCS\ntKxRp1O2AZ+LiLOBnwF+B/hL4NMR8R7g+8Cn2okoSVrOqNMpTwD/dolVv7m2OJKk/eEnNiWpMEtc\nkgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgpby10Mp8K9Z5xxQB5rVGZsT4WcFTJCjZzjzHjP\n2PbsSFySSrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCit/75Tn\nfunvD8hjjcqM7amQs0JGqJGzQsalOBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkq\nzBKXpMIscUkqzBKXpMIscUkqrPwNsB5/+Y8PyGONyoztqZCzQkaokbNCxqU4EpekwixxSSrMEpek\nwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwkb6xGZEnA/MDCz6deAbwAbgqWbZv8/MB9cWT5K0\nkpFKPDOvA64DiIhTgN8GfhU4LzO/1V48SdJK2rh3yiXAVuDzLexrvz3+2mcOyGONyoztqZCzQkao\nkXOsGXeOb9edXq838osj4njgwsx8R0T8GfA4sBH4NnBxZj690ut37tzZ27hx48jHl6SXgvn5ebrd\nbmepdWsdib8L+GTz+I+Ab2bm9yLi48CFwH9d6cULCwt0u13m5+dHDrBx4ydGfu3+mJn5GxYWfnVd\njjUqM7anQs4KGaFGznFn3LnzXWPb91pLfAvwXoDM/NLA8luAt65x35KkVYxc4hHxKuDJzHwmIjrA\n/wDOzcwf0S93T3BK0pit5Trxo4BHADKzB1wL3B4RdwKvBq5ZezxJ0kpGHok314C/aeD5DcANbYSS\nJA3HT2xKUmGWuCQVZolLUmGWuCQVZolLUmGWuCQV1sYNsCbqc8/9/LocZ2YdjzUqM7anQs4KGaFG\nznFnPH1se3YkLkmlWeKSVJglLkmFWeKSVJglLkmFWeKSVJglLkmFWeKSVJglLkmFWeKSVJglLkmF\nlb93yjOfv3R9DvT2dTzWqMzYngo5K2SEGjnHnfH0e8a2a0fiklSYJS5JhVniklSYJS5JhVniklSY\nJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhZW/AdYdt564TkfqreOxRmXG9lTIWSEj1Mg5\n3oxnnT43tn07EpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSpsqI/d\nR8Rm4GZgPjOvjohXAwvAQcDDwExm7o6IrcDFwHPAtZl53ZhyS5IYYiQeERuAq4DbBxZfDlyTmScD\nDwHvbLa7BDgN2AJ0I+KI1hNLkp43zHTKbuBMYMfAsi3AtubxLfSL+wTggczclZlPA3cDJ7UXVZK0\nr1WnUzJzL7A3IgYXb8jM3c3jR4CjgCOBRwe2WVwuSRqTNm5F29nP5c+bmZkBoNvtjnzwtbx2f/V6\nvXU71qjM2J4KOStkhBo5pznj/Pz8sutGLfEnI+KQZtrkaPpTLTvoj8YXHQ3cu9JOFhYW6Ha7KwZc\nzezs7Miv3R+9Xo9OZ9WfSxNlxvZUyFkhI9TIOe6Mc3PTdz/xrwPnNI/PAW4F7gOOj4jDI+JQ+vPh\nd609oiRpOauOxCPiOOBK4BhgT0ScC2wFPhkR7wG+D3wqM/dExAeArwE94LLM3DW25JKkoU5sPkj/\napR9/eYS294I3Lj2WJKkYfiJTUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkq\nzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKX\npMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIs\ncUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkq7OBhNoqIzcDNwHxmXh0Rrwau\nB34a2AO8LTN/GBF7gLsHXvqGzHy27dCSpL5VSzwiNgBXAbcPLP4D4NrMvCEiLgRmgd8DdmXmlnEE\nlSS92DDTKbuBM4EdA8suAG5qHj8KvLLlXJKkIaw6Es/MvcDeiBhc9hRARBwEXAhc3qx6WUR8DvgF\n4KbMnGs9sSTpeUPNiS+lKfAF4I7MXJxqeT/wGaAH3BkRd2bmN5bbx8zMDADdbnfUGGt67f7q9Xrr\ndqxRmbE9FXJWyAg1ck5zxvn5+WXXjVzi9E9sfjczL1tckJn/bfFxRNwOHAssW+ILCwt0u90VA65m\ndnZ25Nfuj16vR6fTWZdjjcqM7amQs0JGqJFz3Bnn5sY3KTFSiUfEVuCZzPzIwLIAPgJsBQ4CTgJu\nbCOkJGlpw1ydchxwJXAMsCcizgV+FviHiPizZrO/zcwLIuL/APcDzwHbMvP+saSWJAHDndh8ENgy\nzM4y8z+sNZAkaXh+YlOSCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakw\nS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1yS\nCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPE\nJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCjt4mI0iYjNwMzCfmVdHxCeB44DHmk2u\nyMyvRMRW4GLgOeDazLxuDJklSY1VSzwiNgBXAbfvs+qDmfnlfba7BPiXwDPAAxHxpcx8vMW8kqQB\nw0yn7AbOBHasst0JwAOZuSsznwbuBk5aYz5J0gpWHYln5l5gb0Tsu+qiiJgFHgEuAo4EHh1Y/whw\nVEs5JUlLGGpOfAkLwGOZ+VcR8QHgUuCefbbprLaTmZkZALrd7ogx1vba/dXr9dbtWKMyY3sq5KyQ\nEWrknOaM8/Pzy64bqcQzc3B+fBvwceBG+qPxRUcD9660n4WFBbrd7ooBVzM7Ozvya/dHr9ej01n1\n59JEmbE9FXJWyAg1co4749zc3Nj2PdIlhhFxU0S8pnm6BfgWcB9wfEQcHhGH0p8Pv6uVlJKkJQ1z\ndcpxwJXAMcCeiDiX/tUqX4iIHwNPAudl5tPN1MrXgB5wWWbuGltySdJQJzYfpD/a3tdNS2x7I/1p\nFUnSOvATm5JUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUu\nSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ\n4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJU\nmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJU2MHDbBQRm4GbgfnMvDoivgj8k2b1EcC9wH8GtgMP\nNssfzcy3tJxXkjRg1RKPiA3AVcDti8sGyzki/gT4xE9W5ZaWM0qSljHMdMpu4Exgx74rIiKAwzPz\n/raDSZJWt+pIPDP3Anv7ff0i76M/Sl90ZETcCLwKuCYzP7vSvmdmZgDodrvD5n2Rtbx2f/V6vXU7\n1qjM2J4KOStkhBo5pznj/Pz8suuGmhNfSkT8DPD6zLygWfQY8GHgM8BhwP0RcUdmPrzcPhYWFuh2\nuysGXM3s7OzIr90fvV6PTqezLscalRnbUyFnhYxQI+e4M87NzY1t3yOXOHAK8Pw0SmY+AVzfPN0Z\nEd8AXgssW+KSpLVZyyWGxwN/vfgkIk6NiLnm8QbgXwDfWVs8SdJKhrk65TjgSuAYYE9EnAu8GTgK\n+N7ApncBb4+IvwAOAj6amT9oPbEk6XnDnNh8ENiyxKr37rPdXuAdraSSJA3FT2xKUmGWuCQVZolL\nUmGWuCQVZolLUmGWuCQVtpZPbEovKfeeccYLnp94660TSiL9hCNxSSrMEpekwixxSSrMEpekwixx\nSSrMEpekwixxSSrM68SlIXlduKaRI3FJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKmyi14l/\n+fAn6Tb/1frY957Yk8rgNdd6KflXt9229p10u0sudiQuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJU\nmCUuSYVZ4pJUmP8oxEvMpD9k05uCDNKBxJG4JBVmiUtSYZa4JBXmnLgkjVkb54F6yyx3JC5JhXV6\nveX6fR0O3un0er0enU5nYhmGVSGnGdtTIWeFjFAjZ5GMSwZ0JC5JhVniklSYJS5JhVniklTYUJcY\nRsTHgJOb7T8KPAAsAAcBDwMzmbk7IrYCFwPPAddm5nVjSS1JAoYYiUfEqcDmzHwdcAbwh8DlwDWZ\neTLwEPDOiNgAXAKcBmwBuhFxxLiCS5KGm065E3hL8/hHwAb6Jb2tWXYL/eI+AXggM3dl5tPA3cBJ\nraaVJL3AqtMpmfks8FTz9Hzgq8AbM3N3s+wR4CjgSODRgZcuLpckjcnQH7uPiLPpl/jpwHcHVi13\nhfyqV85v374d6F9oX0GFnGZsT4WcFTJCjZzTnHGlDyINe2LzjcCHgDMyc1dEPBkRhzTTJkcDO5o/\nRw687Gjg3pX2e+yxx5b4pBSU+USXGVtSIWeFjFAjZ4WMyxnmxOZhwBXAWZn5eLP468A5zeNzgFuB\n+4DjI+LwiDiU/nz4Xe1HliQtWvXeKRHxbuBS4DsDi98OfAJ4GfB94LzM3BMR5wK/S/+GW1dl5mdX\nPLj3TmmVGdtTIWeFjFAjZ5GMSwb0BlhDqpDTjO2pkLNCRqiRs0hGb4AlSQcaS1ySCrPEJakwS1yS\nCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJamwid5PXJK0\nNo7EJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCjt4kgePiHngRKAHvC8zH5hknkUR8THgZPrvz0eB\nB4AF4CDgYWAmM3dPLuFPRMQhwLeA/wTczpTljIitwO8Be4FLgG8yfRkPBT4NvAL4R8BlwA+Bj9P/\nf/Obmfk7E8q2GbgZmM/MqyPi1Szx/jXv88XAc8C1mXndFOS8HvhpYA/wtsz84SRz7ptxYPkbgVsz\ns9M8n+h7ub8mNhKPiFOAX87M1wHnA388qSyDIuJUYHOT6wzgD4HLgWsy82TgIeCdE4y4r/8IPN48\nnqqcEfFK4CPA64GzgLOZsoyNdwCZmacC5wJ/RP/r/r7MPAk4LCLetN6hImIDcBX9H86LXvT+Ndtd\nApwGbAG6EXHEhHP+Af0CPAX4EjA7yZzLZCQiXgZ8kP4PRCb9Xo5iktMpbwD+FCAzvw28IiJePsE8\ni+4E3tI8/hGwgf4Xc1uz7Bb6X+CJi4jXAr8CfKVZtIXpynka8PXMfCIzH87MdzN9GQF2Aq9sHr+C\n/g/FXxz4zXBSOXcDZwI7BpZt4cXv3wnAA5m5KzOfBu4GTppwzguAm5rHj9J/fyeZc6mMAL8PXAM8\n0zyf9Hu53yZZ4kfS/+IuerRZNlGZ+WxmPtU8PR/4KrBh4Ff+R4CjJhLuxa4EZgeeT1vOY4B/HBHb\nIuKuiHgD05eRzPw88PMR8RD9H+LvB/7fwCYTyZmZe5siGbTU+7fv99K65l0qZ2Y+lZnPRsRBwIXA\n5yaZc6mMEbEJ+OeZ+cWBxRN9L0cxTSc2O5MOMCgizqZf4hfts2oqckbEvwP+IjP/1zKbTEPODv0R\n2JvpT1lczwtzTUNGIuJtwP/OzF8CfgP4zD6bTEXOJSyXayryNgW+ANyRmbcvscmkc87zwkHQUiad\ncVWTLPEdvHDk/SqaealJa050fAh4U2buAp5sTiACHM2LfyWbhH8DnB0R9wLvAj7M9OX8v8A9zSjo\ne8ATwBNTlhH6vy5/DSAz/xo4BNg4sH5acsLSX+N9v5emJe/1wHcz87Lm+dTkjIijgdcCn22+h46K\niD9nijIOa5Ilfhv9k0hExK8BOzLziQnmoclyGHAFcFZmLp4w/DpwTvP4HODWSWQblJlvzczjM/NE\n4BP0r06Ztpy3Ab8RET/VnOQ8lOnLCP0ThCcARMQv0P9h8+2IeH2z/s1MR05Y+v27Dzg+Ig5vrrQ5\nCbhrQvmA56/weCYzPzKweGpyZuYPMvOfZeaJzffQw81J2KnJOKyJ3sUwIv4L8K/pX8pzYTMKmqiI\neDdwKfCdgcVvp1+ULwO+D5yXmXvWP93SIuJS4O/ojyY/zRTljIj30J+Wgv4VCw8wfRkPBf4E+Dn6\nl5V+mP4lhv+d/kDnvsxc7dfuceQ6jv55j2PoX6b3A2Ar8En2ef8i4lzgd+lfEnlVZn52wjl/FvgH\n4O+bzf42My+YVM5lMr55caAWEX+Xmcc0jyf2Xo7CW9FKUmHTdGJTkrSfLHFJKswSl6TCLHFJKswS\nl6TCLHFJKswSl6TCLHFJKuz/Ax192n9Wlo21AAAAAElFTkSuQmCC\n","text/plain":["<matplotlib.figure.Figure at 0x7fd507b155c0>"]},"metadata":{"tags":[]}}]},{"metadata":{"id":"fsmDh_Tmixr2","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":269},"outputId":"04461ce6-6c1a-4183-99c6-69a2f30d48cb","executionInfo":{"status":"ok","timestamp":1536533748589,"user_tz":-540,"elapsed":1367,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["# import gym\n","# from IPython import display\n","# import matplotlib.pyplot as plt\n","# %matplotlib inline\n","\n","env = gym.make('Breakout-v0')\n","env.reset()\n","\n","plt.imshow(env.render(mode='rgb_array'))"],"execution_count":22,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAMsAAAD8CAYAAADZhFAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAD19JREFUeJzt3X+MHPV5x/H31S0qMgHHuTYGlzZC\niZ4o8VVVHBcT6nAUAm5Ei1q7zR/uiYKrRg1GcFfamkYitluVCDe+KgaRWnFNuJSWH1YSA5FDY5oY\nJQUbK9CDoqfgP1CEjeyzhcFA/QNv/5jZernsj+dm5jyze5+XhNj93uzs8wV97jszu/NcX61WQ0Q6\n+7myCxDpFgqLSJDCIhKksIgEKSwiQQqLSNDPF71DMxsFFgM14GZ33130e4iUodCVxcwuAz7i7pcA\nK4GvFrl/kTIVfRh2BfBtAHd/EXi/mZ1b8HuIlKLow7B5wJ6G5wfTsTeabdzX11cbHx9nYGCg4DLK\n0UtzgZk7n1qt1tdsvPBzlkmavmnd+Pg4CxYsoJe+ctNLcwHNp1HRYdlHspLUXQDsb7XxwMAAtVqN\nvr62meoavTQXmLnzaRWoos9ZHgeWA5jZJ4B97v5mwe8hUoq+opdZM/sy8GngFHCjuz/X8s37+mq9\n9Nurl+YCM3c+rc5ZCg/LVFQxLBs2bJjya0ZGRoDT/zOmuo/66/PU0WwfedVqNUZHR0utYbLJ/12m\n8p55w6JP8EWCpvtqWNebjt/6eVavPPvI60ysHFWmlUUkSCuLhHVazXp95dHKIhKklUXaardalHHe\nVCatLCJBWlk6KOK3Z1X20U3vW0VaWUSCFBaRIH3dpUC9NBeYufPR111Ecir1BL9+8thLJ5G9NBfQ\nfBppZREJUlhEghQWkSCFRSQo8wm+md0JLEn3cQfwe8BC4FC6yXp3fyx3hSIVkSksZnY5sMDdLzGz\nDwA/AZ4AbnP3R4ssUKQqsq4sO4Fd6ePXgdnArEIqEqmoTGFx93eBt9KnK4HvAu8Cq8xsBDgArHL3\niUKqFKmAXF93MbNrgb8BrgI+CRxy92fNbDXwK+6+qt3rJyYmav39/ZnfX2SaFNu+1cyuBr4ILHX3\nI8COhh9vA+7ptI+xsTGGh4en1G6nynppLjBz5zM8PNx0PNOlYzM7D1gPXOPuh9OxrWZ2UbrJIPB8\nln2LVFXWleVzQD/woJnVx7YAD5jZ28BR4Pr85YlUR9YT/E3ApiY/+ka+ckSqS5/giwSVevPX6Oho\nrd1JV7f1oZqpN0t1i2gv6uHhYd38JZKHwiISpLCIBCksIkEKi0iQwiISpLCIBCksIkEKi0iQwiIS\npLCIBCksIkEKi0iQwiISpLCIBGVtsjcIPAS8kA6NA3cCYyT9w/YDQ+5+rIAaRSohz8ryQ3cfTP+5\nCVgH3O3uS4CXgRsKqVCkIoo8DBskaYEE8AhwZYH7Fildnr/89TEz2wbMBdYCsxsOuw4A5+ctTqRK\nMt2Db2bzgd8CHgQuAv4DOMfd56Y//zBwn7t/qt1+1JFSKqq4jpTu/irwQPp0r5m9Biwys7Pd/R1g\nPrCv0346daRUw4py9ep8Ag0rmo5n7Ui5wsxuTR/PAz5I0mRvWbrJMmB7ln2LVFXWc5ZtwP1pY/Cz\ngD8n+Rst95nZ54FXUMM96TFZD8PeBH63yY8+k68ckerSJ/giQXkuHU+7p5YuLbuEKevGmtvpxfn8\nOONrtbKIBCksIkEKi0iQwiISpLCIBFX6atipD79RdglT1o01t6P5nKaVRSRIYREJUlhEghQWkSCF\nRSRIYREJqvSl48Pnvl12CVPWjTW3o/mcppVFJEhhEQnK2pFyJTDUMPRJ4BlgNvBWOvYX7r4nX3ki\n1ZH1tuLNwGYAM7sM+CPg48D17v58ceWJVEcRh2G3A39bwH5EKi3X1TAzWwT81N1fMzOAdWbWD7wI\n3JL2EMvs8EeP53l5Kbqx5nZ6cj4T2V6bqSNlnZn9E/Cv7v4DM/t94L/cfa+Z3QPsdfd/aPd6daSU\niiquI2WDQeAmAHf/VsP4I8DnOr24U0fK/v6v5yzvzBoaeoGxsY+XXUZhenU+ExN/2na7Vh0pM4fF\nzC4Ajrr7cTPrA/4dWO7ur5OESCf60lPynOCfT9ItH3evAZuAHWa2E7gQuDt/eSLVkXllST9D+Z2G\n5w+SdNUX6UmV/m7Y/ad+tewSpmSI7qu5nV6dz1UZX6+vu4gEKSwiQQqLSJDCIhKksIgEVfpq2PF/\nW1N2CVNzXRfW3E6vzueqbH90QiuLSJDCIhKksIgEKSwiQQqLSJDCIhJU6UvHT2xfXHYJU1Trwprb\n6c35XHPVhkyv1soiEqSwiAQpLCJBoXMWM1sAfAcYdfe7zOxCYAyYBewHhtz9mJmtAG4BTgGb0mZ8\nIj2h48piZrOBjcCOhuF1wN3uvgR4Gbgh3e524EqShhXDZja38IpFShI5DDsGfBbY1zA2CGxLHz9C\nEpCLgd3ufiRtrvcj4NLiShUpV8fDMHc/CZxMO07WzXb3Y+njAySdXuYBBxu2qY+3NDSU9BZv1aep\n1XiV5WlaWEWaz2lFfM7StHtfm/H/16nJ3sjISJ66zrharUZfX8dpd41enc+GDe0/Z2n1Szrr1bCj\nZnZ2+ng+ySHaPpLVhUnjIj0ha1i+DyxLHy8DtgNPA4vMbI6ZnUNyvvJk/hJFqqHjYZiZLQS+AnwI\nOGFmy4EVwL1m9nngFeAb7n7CzFYD3wNqwFp3PzJtlYucYZET/D0kV78m+0yTbR8GHs5flkj16BN8\nkSCFRSRIYREJUlhEghQWkSCFRSRIYREJUlhEghQWkSCFRSRIYREJUlhEghQWkSCFRSRIYREJUlhE\nghQWkaA8HSm3AL8AnAD+2N1fM7MTJP3C6q5w93eLLlqkDJF78Jt1pPw7kvasD5rZjcAI8FfAEXcf\nnI5CRcqWtSPlF4Ct6eODwAcKrkukcvqiHfrMbA0w4e53NYzNAp4A1rn7DjM7StLW9deAre7etpvZ\nxMRErb+/P2vtItOlaWfBzB0p06CMAU+4e/0Q7VbgmyStkHaa2U53f6bVPtSRstp6dT5ZO1Lmad+6\nBXjJ3dfWB9z9a/XHZrYDGABahkWkm2QKS/p3WI67+5caxgz4EkkDvlkkHSnVQ0x6RtaOlL8M/K+Z\n/SDd7L/d/Qtm9lNgF8kfM9rm7rumpWqREuTpSNls27/OW5BIVekTfJEghUUkSGERCVJYRIIUFpEg\nhUUkSGERCVJYRIIUFpEghUUkSGERCVJYRIIUFpEghUUkSGERCVJYRIIUFpGgrB0p7wUWAofSTda7\n+2Ppvfm3kNxWvMndN09DzSKlyNqREuA2d3900na3A78JHAd2m9m33P1wgfWKlCZrR8pmLgZ2u/sR\nd3+HpOfxpTnrE6mMSMOKk8DJpNPRe6wysxHgALAKmEfSyrXuAHB+u30PDQ0BrZuatRqvsmiHz26h\n+ZyWtcneGHDI3Z81s9XAGuDHk7bp2MpQHSmrrVfnc0Y7Uja0a4Wkt/E9JA315jWMzweeyrJ/kSrK\ndOnYzLaa2UXp00HgeeBpYJGZzTGzc0jOV54spEqRCsjakXIj8ICZvQ0cBa5393fSQ7LvkTQGX+vu\nR6atcpEzLE9Hyq2TB9z9YdTfWHqUPsEXCVJYRIIUFpEghUUkSGERCVJYRIIUFpEghUUkSGERCVJY\nRIIUFpEghUUkSGERCVJYRIIUFpEghUUkKGuTvYeAX0p/PJfkXvu/B8aBPen4QXf/w4LrFSlNpiZ7\njSEws38Gvn76Rz5YcI0ilZCryZ4lzcTmuPuuogsTqZo8TfYAbiZZdermmdnDwAXA3e7+L4VUKVIB\nfdEOfWa2Bphw97vS52cBz7j7r6fP3wcsB74JnAfsAi519/2t9jkxMVHr7+/PNQGRadC0s2DWjpQA\nl5EEAgB3fxPYkj6dMLNngI8CLcOijpTV1qvzydqRMs+l40XAc/UnZna5mW1IH88GfgP4nxz7F6mU\nrE32/oCk6ffehk2fBK4zs/8EZgF3uPurhVcsUpI8TfZumrTdSeBPCqlKpIL0Cb5IkMIiEqSwiAQp\nLCJBCotIkMIiEqSwiAQpLCJBCotIUJ4vUub26JyjDKf/lmp5aunS9/x7qhZv315kOYX61OOPt99g\nGr5IKTKjKCwiQQqLSFCp5yxSXYu3b6dGtc89zjStLCJBWllkxum0WrbqShFuWDEd+vr6ar10n3cv\nzQVm7nxqtVr2hhVmdiewJN3+DmA3MEZy+/B+YMjdj5nZCuAW4BSwyd03R/Yv0g06nrOY2eXAAne/\nBFgK/COwjqQv2BLgZeCGtEnF7cCVJLchD5vZ3OkqXORMi5zg7wTq7VpfB2aThGFbOvYISUAuBna7\n+xF3fwf4EXBpodWKlCjSsOJd4K306Urgu8DV7n4sHTtA0ullHnCw4aX1cZGeEL4aZmbXkoTlKuCl\nhh+1OmPqeCY1Pj4OJCdevaKX5gKaT6PoCf7VwBeBpe5+xMyOmtnZ6eHWfJKm4ftIVpe6+SR/iqKl\ngYGBnrri0ktzgZk7n1aBipzgnwesB65x98Pp8PeBZenjZcB24GlgkZnNMbNzSM5XnuxYmUiX6Pg5\ni5n9GbCG97ZivY7kb7L8IvAKcL2717tV/iXJ5zobO3XR1+cs1TZT59PqcxZ9KFmgXpoLzNz5tAqL\nvhsmEqSwiAQpLCJBCotIkMIiElTq1TCRbqKVRSRIYREJUlhEghQWkSCFRSRIYREJKq0VkpmNAotJ\nvqF8s7vvLquWLMxsEHgIeCEdGgfupEkjj1IKDDKzBcB3gFF3v8vMLqSLm5E0mc+9wELgULrJend/\nLMt8SllZzOwy4CNpE4yVwFfLqKMAP3T3wfSfm2jSyKPc8tpLm4xsBHY0DHdtM5IW8wG4reH/02NZ\n51PWYdgVwLcB3P1F4P1mdm5JtRRpkJ9t5FFlx4DPktzlWjdI9zYjaTafZjLNp6zDsHnAnobnB9Ox\nN8opJ7OPmdk2YC6wFpjdpJFHZbn7SeCkmTUON5tDVzQjaTEfgFVmNkJS9yoyzqcqJ/jdeIfRSyQB\nuZbkztHNvPeXTzfOabLMzUgqZAxY7e6/DTxLctfvZKH5lBWWyc0tLiA5mewa7v6quz/g7jV33wu8\nRnI4eXa6Sb2RR7c52mQOzZqRdMXc3H2Huz+bPt0GDJBxPmWF5XFgOYCZfQLY5+5vllRLJma2wsxu\nTR/PAz4IbOFnG3l0m55qRmJmW83sovTpIPA8GedT2reOzezLwKdJLt3d6O7PlVJIRmb2PuB+YA5w\nFskh2U+A+5jUyKO0Ijsws4XAV4APASeAV4EVwL3kbEZShhbz2QisBt4GjpLM50CW+egr+iJBVTnB\nF6k8hUUkSGERCVJYRIIUFpEghUUkSGERCVJYRIL+D4kJ167OcBoUAAAAAElFTkSuQmCC\n","text/plain":["<matplotlib.figure.Figure at 0x7fd507ac0080>"]},"metadata":{"tags":[]}}]},{"metadata":{"id":"c0ghaIuqTyvc","colab_type":"code","colab":{}},"cell_type":"code","source":["from PIL import Image\n","import numpy as np\n","import gym\n","\n","from keras.models import Sequential\n","from keras.layers import Dense, Activation, Flatten, Convolution2D, Permute\n","from keras.optimizers import Adam\n","import keras.backend as K\n","\n","from rl.agents.dqn import DQNAgent\n","from rl.policy import LinearAnnealedPolicy, BoltzmannQPolicy, EpsGreedyQPolicy\n","from rl.memory import SequentialMemory\n","from rl.core import Processor\n","from rl.callbacks import FileLogger, ModelIntervalCheckpoint\n","\n","\n","INPUT_SHAPE = (84, 84)\n","WINDOW_LENGTH = 4\n","\n","class AtariProcessor(Processor):\n","    def process_observation(self, observation):\n","        assert observation.ndim == 3\n","        img = Image.fromarray(observation)\n","        img = img.resize(INPUT_SHAPE).convert('L')\n","        processed_observation = np.array(img)\n","        assert processed_observation.shape == INPUT_SHAPE\n","        return processed_observation.astype('uint8')\n","\n","    def process_state_batch(self, batch):\n","        processed_batch = batch.astype('float32') / 255.\n","        return processed_batch\n","\n","    def process_reward(self, reward):\n","        return np.clip(reward, -1., 1.)\n","\n","ENV_NAME = 'BreakoutDeterministic-v4'\n","env = gym.make(ENV_NAME)\n","np.random.seed(123)\n","env.seed(123)\n","nb_actions = env.action_space.n\n","input_shape = (WINDOW_LENGTH,) + INPUT_SHAPE\n","\n","model = Sequential()\n","model.add(Permute((2, 3, 1), input_shape=input_shape))\n","model.add(Convolution2D(32, (8, 8), strides=(4, 4)))\n","model.add(Activation('relu'))\n","model.add(Convolution2D(64, (4, 4), strides=(2, 2)))\n","model.add(Activation('relu'))\n","model.add(Convolution2D(64, (3, 3), strides=(1, 1)))\n","model.add(Activation('relu'))\n","model.add(Flatten())\n","model.add(Dense(512))\n","model.add(Activation('relu'))\n","model.add(Dense(nb_actions))\n","model.add(Activation('linear'))\n","#print(model.summary())\n","\n","memory = SequentialMemory(limit=1000000, window_length=WINDOW_LENGTH)\n","processor = AtariProcessor()\n","\n","policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,\n","                              nb_steps=1000000)\n","\n","dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, memory=memory,\n","               processor=processor, nb_steps_warmup=50000, gamma=.99, target_model_update=10000,\n","               train_interval=4, delta_clip=1.)\n","\n","dqn.compile(Adam(lr=.00025), metrics=['mae'])"],"execution_count":0,"outputs":[]},{"metadata":{"id":"oOZ1AX3DdZxW","colab_type":"text"},"cell_type":"markdown","source":["# TRAIN"]},{"metadata":{"id":"zOevRHKqUU8a","colab_type":"code","colab":{}},"cell_type":"code","source":["weights_filename = 'dqn_{}_weights.h5f'.format(ENV_NAME)\n","checkpoint_weights_filename = 'dqn_' + ENV_NAME + '_weights_{step}.h5f'\n","log_filename = 'dqn_{}_log.json'.format(ENV_NAME)\n","callbacks = [ModelIntervalCheckpoint(checkpoint_weights_filename, interval=250000)]\n","callbacks += [FileLogger(log_filename, interval=100)]\n","dqn.fit(env, callbacks=callbacks, nb_steps=1750000, log_interval=10000)\n","\n","dqn.save_weights(weights_filename, overwrite=True)\n","dqn.test(env, nb_episodes=1, visualize=False)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"6EeLQhNRdhcr","colab_type":"text"},"cell_type":"markdown","source":["# TEST:1"]},{"metadata":{"id":"26Cc_GzxdoBy","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":521},"outputId":"99a02654-aa41-471b-ddbd-e1e924f89a05","executionInfo":{"status":"ok","timestamp":1536534342474,"user_tz":-540,"elapsed":8,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["from rl.callbacks import Callback\n","from IPython import display\n","import matplotlib.pyplot as plt\n","%matplotlib inline\n","\n","# Subclass to visualize on Jupyter Notebook, inherits from rl.callbacks.Callback Class\n","class Render(Callback):\n","    def on_step_end(self, step, logs={}):\n","        plt.clf()\n","        plt.imshow(env.render(mode='rgb_array'))\n","        display.display(plt.gcf())\n","        display.clear_output(wait=True)\n","\n","# weights_filename = 'dqn_{}_weights.h5f'.format(ENV_NAME)\n","# weights_filename = 'dqn_{}_weights_1750000.h5f'.format(ENV_NAME) # @check point\n","weights_filename = 'dqn_BreakoutDeterministic-v4_weights_1750000.h5f'\n","dqn.load_weights(weights_filename)\n","\n","callbacks = Render()\n","plt.figure(figsize=(6,8))\n","dqn.test(env, nb_episodes=1, visualize=False, callbacks=[callbacks])\n","\n","env.close()\n","print('END')"],"execution_count":28,"outputs":[{"output_type":"stream","text":["Episode 1: reward: 37.000, steps: 1185\n","END\n"],"name":"stdout"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXEAAAHVCAYAAAAHCstcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGZRJREFUeJzt3X+Q5HV95/FnB5KTW0oQ9xKQMyFe\nsm/LLHWpEA4UORYluHLcUSV4/rEOilhaASzpOZPzx4nCpfRKj5kkQHlHSUBGqYhQhkWtlRMugYIA\nKxXjGq23Yl28KxcPFs4NIFl2oe+P7sFmmB+93d+Z7vfyfFRN0f3tb3+/r+2eec1nPt9vf2l1Oh0k\nSTX9wrgDSJKGZ4lLUmGWuCQVZolLUmGWuCQVZolLUmEHN73BiJgFTgQ6wPszc3vT+5AkdTU6Eo+I\nU4DfzMzXAucDf9rk9iVJz9f0dMobgb8AyMzvAS+LiJc2vA9J0rxOp9PY14YNG67esGHDWX3379qw\nYcOGpdYHOjt27OjQnXqZ6K8KOc344spZIWOVnBUyLtWjjc+JL9Ba7sEdO3awcePG+UKfeBVymrE5\nFXJWyAg1ck5yxlZr6SptusR3Akf23X8F8NBSKx977LF0Op1lA06KCjnN2JwKOStkhBo5K2RcStNz\n4rcB5wBExO8AOzPz8Yb3IUnqabTEM/Me4IGIuIfumSkXNrl9SdLztcY5D9RqtTpV/oypkNOMzamQ\ns0JGqJGzSMZFA/qJTUkqzBKXpMIscUkqzBKXpMJW+8M+asjMzEyj6y1menp61bY96HZGzbDS819M\nfK26DvTXwZG4JBVmiUtSYZa4JBXmnPgBYnp6mna7veT8XhPz2aPOHTY1py7p5xyJS1JhlrgkFWaJ\nS1JhlrgkFeaBTR1QJuEArrSWHIlLUmGWuCQVZolLUmHOieuAMsh8th860oHEkbgkFWaJS1Jhlrgk\nFeac+AFifp73xT7f+2L/9+vFx5G4JBVmiUtSYZa4JBXW6nQ649t5q9XpdDq0Wq2xZRhUhZxmbE6F\nnBUyQo2cRTIuGtCRuCQVZolLUmGWuCQVNtbzxKud21whpxmbUyFnhYxQI2eFjItxJC5JhVniklSY\nJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhQ19FcOI+BRw\ncm8bnwT+HXAc8GhvlU9n5ldHTihJWtJQJR4RpwIbM/O1EfFy4G+AO4APZeZXmgwoSVrasCPxO4H7\ne7d/CqwDDmokkSRpYEOVeGY+AzzZu3s+8DXgGeCiiJgGHgYuysxdjaSUJC1qpP/bfUScBXwYOB34\nXeDRzPxWRHwQ+OeZedFyz9+1a1dn/fr1Q+9fkl4MZmdnabfbi/7f7kc5sPkm4CPA5szcDdze9/BW\n4DMrbWNubo52u83s7OywMdZMhZxmbE6FnBUyQo2cFTIuZahTDCPiMODTwJmZ+Vhv2c0R8areKpuA\n7zSSUJK0pGFH4m8D1gM3RsT8smuBL0bEz4AngPNGjydJWs6wBzavBq5e5KHPjRZHkrQ//MSmJBVm\niUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYSNdAGtUs7OznVGvWTA9Pd1goqV1Oh1arUWv\nPzMxzNicCjkrZIQaOVc748zMzMjbWOoCWI7EJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakw\nS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1yS\nCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPEJakwS1ySCrPE\nJakwS1ySCjt4mCdFxCbgS8Df9RbtAD4FzAEHAQ8BU5m5p4GMkqQljDIS/6vM3NT7eh9wGXBVZp4M\nPAi8q5GEkqQlNTmdsgnY2rt9K3Bag9uWJC1iqOmUntdExFbgCOBSYF3f9MnDwFErbWBqagqAdrs9\ndIhRnru/Op3Omu1rWGZsToWcFTJCjZyTnHF2dnbJx4Yt8R/QLe4bgVcB/3PBtlqDbGRubo52u71s\nwJVMT08P/dz90el0aLUG+meNjRmbUyFnhYxQI+dqZ5yZmVm1bQ9V4pn5Y+CLvbs/jIifAMdHxCGZ\n+RRwNLCzoYySpCUMNSceEVsi4gO920cCvwJcC5zdW+VsYFsjCSVJSxp2OmUrcENEnAX8EvD7wN8A\n10fEe4EfAZ9rJqIkaSnDTqc8DvzbRR76vdHiSJL2h5/YlKTCLHFJKswSl6TCLHFJKswSl6TCLHFJ\nKswSl6TCLHFJKmyUqxhOhHs3bz4g9zUsMzanQs4KGaFGztXMeM+qbdmRuCSVZolLUmGWuCQVZolL\nUmGWuCQVZolLUmGWuCQVZolLUmGWuCQVZolLUmGWuCQVVv7aKc/+xj8ckPsalhmbUyFnhYwwWs7X\nXTnalUfuueh1A61X5bVcyJG4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVm\niUtSYZa4JBVmiUtSYeUvgPXYS392QO5rWOPMeOYnvrXiOp0rRr+g0Wr7yod/G6j/Ws7/O8bpzE98\na+T3fNR/x2MM9j5W+PlejCNxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSps\nqE9sRsT5wFTfot8FvgmsA57sLfsPmfnAaPEkScsZqsQz8xrgGoCIOAX498BvAedl5neaiydJWk4T\n1065BNgC/HkD29pvj7366QNyX8OqkHHSzb+G1V/LQa6/spzrr39NQ0lGs1bvw6ruZ9fqbbrV6XSG\nfnJEHA9cmJnvjIi/BB4D1gPfAy7OzKeWe/6uXbs669evH3r/kvRiMDs7S7vdbi322Kgj8XcD1/Vu\n/wnw7cz8YUR8BrgQ+K/LPXlubo52u83s7OzQAdav/+zQz90fU1N/x9zcb63JvoY17oznnvvdFdfp\ndDq0Wot+L06M669/ja8lzYzEzz33uyPnXIu/CFb7/d61692rtu1RS3wT8D6AzPxy3/JbgbeNuG1J\n0gqGLvGIeAXwRGY+HREt4H8A52TmT+mWuwc4JWmVjXKe+FHAwwCZ2QGuBm6PiDuBVwJXjR5PkrSc\noUfivXPA39x3/0bgxiZCSZIG4yc2JakwS1ySCrPEJakwS1ySCrPEJakwS1ySCmviAlhjdcOzv7om\n+5laZl/b3rltTTKsZGpqsI9rH8g2X7d55G3c8Ozy7/faGP/7OCnfS6PmGOR7YrXf79NXbcuOxCWp\nNEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpsPLXTlmz65a8Y+l9\nvWHzvSNt+o5tJ470/P4Mo2RpIse4Nfb9sMz7PYhRvydgMq7HcyAY6H1c4f0e+f08/Z7Rnr8MR+KS\nVJglLkmFWeKSVJglLkmFWeKSVJglLkmFWeKSVJglLkmFWeKSVJglLkmFWeKSVJglLkmFlb8A1iSY\nhAtHdTN0JiKLJuN7Qs0Z9f088/SZhpK8kCNxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrM\nEpekwixxSSrMEpekwgb62H1EbARuAWYz88qIeCUwBxwEPARMZeaeiNgCXAw8C1ydmdesUm5JEgOM\nxCNiHXAFcHvf4suAqzLzZOBB4F299S4BTgM2Ae2IOKLxxJKk5wwynbIHOAPY2bdsE7C1d/tWusV9\nArA9M3dn5lPA3cBJzUWVJC204nRKZu4D9kVE/+J1mbmnd/th4CjgSOCRvnXml0uSVkkTl6Jt7efy\n50xNTQHQbreH3vkoz91fnU5nzfY1LDM2p0LOChmhRs5Jzjg7O7vkY8OW+BMRcUhv2uRoulMtO+mO\nxucdDdy73Ebm5uZot9vLBlzJ9PT00M/dH51Oh1Zrxd9LY2XG5lTIWSEj1Mi52hlnZibveuLfAM7u\n3T4b2AbcBxwfEYdHxKF058PvGj2iJGkpK47EI+I44HLgGGBvRJwDbAGui4j3Aj8CPpeZeyPig8DX\ngQ5waWbuXrXkkqSBDmw+QPdslIV+b5F1bwJuGj2WJGkQfmJTkgqzxCWpMEtckgqzxCWpMEtckgqz\nxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWp\nMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtc\nkgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgo7\neJCVImIjcAswm5lXRsQrgWuBXwT2Am/PzJ9ExF7g7r6nvjEzn2k6tCSpa8USj4h1wBXA7X2L/wi4\nOjNvjIgLgWngD4HdmblpNYJKkl5okOmUPcAZwM6+ZRcAN/duPwK8vOFckqQBrDgSz8x9wL6I6F/2\nJEBEHARcCFzWe+glEXED8GvAzZk503hiSdJzBpoTX0yvwOeAOzJzfqrlA8DngQ5wZ0TcmZnfXGob\nU1NTALTb7WFjjPTc/dXpdNZsX8MyY3Mq5KyQEWrknOSMs7OzSz42dInTPbD5g8y8dH5BZv63+dsR\ncTtwLLBkic/NzdFut5cNuJLp6emhn7s/Op0OrVZrTfY1LDM2p0LOChmhRs7Vzjgzs3qTEkOVeERs\nAZ7OzI/1LQvgY8AW4CDgJOCmJkJKkhY3yNkpxwGXA8cAeyPiHOCXgX+MiL/srfbdzLwgIv4PcD/w\nLLA1M+9fldSSJGCwA5sPAJsG2Vhm/sdRA0mSBucnNiWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqz\nxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWp\nMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtc\nkgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpsIMHWSki\nNgK3ALOZeWVEXAccBzzaW+XTmfnViNgCXAw8C1ydmdesQmZJUs+KJR4R64ArgNsXPPShzPzKgvUu\nAf4V8DSwPSK+nJmPNZhXktRnkOmUPcAZwM4V1jsB2J6ZuzPzKeBu4KQR80mSlrHiSDwz9wH7ImLh\nQxdFxDTwMHARcCTwSN/jDwNHNZRTkrSIgebEFzEHPJqZ34qIDwIfB+5ZsE5rpY1MTU0B0G63h4wx\n2nP3V6fTWbN9DcuMzamQs0JGqJFzkjPOzs4u+dhQJZ6Z/fPjW4HPADfRHY3POxq4d7ntzM3N0W63\nlw24kunp6aGfuz86nQ6t1oq/l8bKjM2pkLNCRqiRc7UzzszMrNq2hzrFMCJujohX9e5uAr4D3Acc\nHxGHR8ShdOfD72okpSRpUYOcnXIccDlwDLA3Is6he7bKFyPiZ8ATwHmZ+VRvauXrQAe4NDN3r1py\nSdJABzYfoDvaXujmRda9ie60iiRpDfiJTUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKX\npMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIs\ncUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkq\nzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkqzBKXpMIscUkq7OBBVoqIjcAtwGxm\nXhkRXwL+We/hI4B7gU8AO4AHessfycy3NpxXktRnxRKPiHXAFcDt88v6yzki/gz47M8fyk0NZ5Qk\nLWGQ6ZQ9wBnAzoUPREQAh2fm/U0HkyStbMWReGbuA/Z1+/oF3k93lD7vyIi4CXgFcFVmfmG5bU9N\nTQHQbrcHzfsCozx3f3U6nTXb17DM2JwKOStkhBo5Jznj7Ozsko8NNCe+mIj4JeD1mXlBb9GjwEeB\nzwOHAfdHxB2Z+dBS25ibm6Pdbi8bcCXT09NDP3d/dDodWq3WmuxrWGZsToWcFTJCjZyrnXFmZmbV\ntj10iQOnAM9No2Tm48C1vbu7IuKbwKuBJUtckjSaUU4xPB742/k7EXFqRMz0bq8Dfhv4/mjxJEnL\nGeTslOOAy4FjgL0RcQ7wFuAo4Id9q94FvCMi/ho4CPhkZv648cSSpOcMcmDzAWDTIg+9b8F6+4B3\nNpJKkjQQP7EpSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ\n4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJUmCUuSYVZ4pJU2MHj3PlXDn+C\ndu+/Gs69mzcve38tnLht25rvU6rkdbfdNvpG2u1FFzsSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJ\nKswSl6TCLHFJKmysH/bR6Po/aNPBD95ILzaOxCWpMEtckgqzxCWpMOfEJWmVNXGsqrPEckfiklRY\nq9NZqt/XYOetVqfT6dBqtcaWYVAVcpqxORVyVsgINXIWybhoQEfiklSYJS5JhVniklSYJS5JhQ10\nimFEfAo4ubf+J4HtwBxwEPAQMJWZeyJiC3Ax8CxwdWZesyqpJUnAACPxiDgV2JiZrwU2A38MXAZc\nlZknAw8C74qIdcAlwGnAJqAdEUesVnBJ0mDTKXcCb+3d/imwjm5Jb+0tu5VucZ8AbM/M3Zn5FHA3\ncFKjaSVJz7PidEpmPgM82bt7PvA14E2Zuae37GHgKOBI4JG+p84vlyStkoE/dh8RZ9Et8dOBH/Q9\ntNQZ8iueOb9jxw6ge6J9BRVymrE5FXJWyAg1ck5yxuU+iDTogc03AR8BNmfm7oh4IiIO6U2bHA3s\n7H0d2fe0o4F7l9vuscceW+KTUlDmE11mbEiFnBUyQo2cFTIuZZADm4cBnwbOzMzHeou/AZzdu302\nsA24Dzg+Ig6PiEPpzoff1XxkSdK8Fa+dEhHvAT4OfL9v8TuAzwIvAX4EnJeZeyPiHOAP6F5w64rM\n/MKyO/faKY0yY3Mq5KyQEWrkLJJx0YBeAGtAFXKasTkVclbICDVyFsnoBbAk6UBjiUtSYZa4JBVm\niUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBVmiUtSYZa4JBU21uuJ\nS5JG40hckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpsIPHufOImAVOBDrA+zNz+zjzzIuITwEn0319\nPglsB+aAg4CHgKnM3DO+hD8XEYcA3wH+M3A7E5YzIrYAfwjsAy4Bvs3kZTwUuB54GfBPgEuBnwCf\nofu9+e3M/P0xZdsI3ALMZuaVEfFKFnn9eq/zxcCzwNWZec0E5LwW+EVgL/D2zPzJOHMuzNi3/E3A\ntsxs9e6P9bXcX2MbiUfEKcBvZuZrgfOBPx1Xln4RcSqwsZdrM/DHwGXAVZl5MvAg8K4xRlzoPwGP\n9W5PVM6IeDnwMeD1wJnAWUxYxp53ApmZpwLnAH9C931/f2aeBBwWEW9e61ARsQ64gu4v53kveP16\n610CnAZsAtoRccSYc/4R3QI8BfgyMD3OnEtkJCJeAnyI7i9Exv1aDmOc0ylvBP4CIDO/B7wsIl46\nxjzz7gTe2rv9U2Ad3Tdza2/ZrXTf4LGLiFcDrwG+2lu0icnKeRrwjcx8PDMfysz3MHkZAXYBL+/d\nfhndX4q/3veX4bhy7gHOAHb2LdvEC1+/E4Dtmbk7M58C7gZOGnPOC4Cbe7cfofv6jjPnYhkBPgxc\nBTzduz/u13K/jbPEj6T75s57pLdsrDLzmcx8snf3fOBrwLq+P/kfBo4aS7gXuhyY7rs/aTmPAf5p\nRGyNiLsi4o1MXkYy88+BX42IB+n+Ev8A8P/6VhlLzszc1yuSfou9fgt/ltY072I5M/PJzHwmIg4C\nLgRuGGfOxTJGxAbgX2bml/oWj/W1HMYkHdhsjTtAv4g4i26JX7TgoYnIGRHnAn+dmf9riVUmIWeL\n7gjsLXSnLK7l+bkmISMR8Xbgf2fmbwBvAD6/YJWJyLmIpXJNRN5egc8Bd2Tm7YusMu6cszx/ELSY\ncWdc0ThLfCfPH3m/gt681Lj1DnR8BHhzZu4GnugdQAQ4mhf+STYO/wY4KyLuBd4NfJTJy/l/gXt6\no6AfAo8Dj09YRuj+ufx1gMz8W+AQYH3f45OSExZ/jxf+LE1K3muBH2Tmpb37E5MzIo4GXg18ofcz\ndFRE/BUTlHFQ4yzx2+geRCIifgfYmZmPjzEPvSyHAZ8GzszM+QOG3wDO7t0+G9g2jmz9MvNtmXl8\nZp4IfJbu2SmTlvM24A0R8Qu9g5yHMnkZoXuA8ASAiPg1ur9svhcRr+89/hYmIycs/vrdBxwfEYf3\nzrQ5CbhrTPmA587weDozP9a3eGJyZuaPM/NfZOaJvZ+hh3oHYScm46DGehXDiPgvwL+meyrPhb1R\n0FhFxHuAjwPf71v8DrpF+RLgR8B5mbl37dMtLiI+Dvw93dHk9UxQzoh4L91pKeiesbCdyct4KPBn\nwK/QPa30o3RPMfzvdAc692XmSn92r0au4+ge9ziG7ml6Pwa2ANex4PWLiHOAP6B7SuQVmfmFMef8\nZeAfgX/orfbdzLxgXDmXyPiW+YFaRPx9Zh7Tuz2213IYXopWkgqbpAObkqT9ZIlLUmGWuCQVZolL\nUmGWuCQVZolLUmGWuCQVZolLUmH/Hwih3vSAUzTRAAAAAElFTkSuQmCC\n","text/plain":["<matplotlib.figure.Figure at 0x7fd5018a0198>"]},"metadata":{"tags":[]}}]},{"metadata":{"id":"HJOS8esKkJ0L","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":485,"output_embedded_package_id":"1YZIFkFyPX-SJHYKGK4agi_3MRlpij2Be"},"outputId":"db59e244-c38d-425b-acdd-d72fb173ecfa","executionInfo":{"status":"ok","timestamp":1536534609257,"user_tz":-540,"elapsed":37654,"user":{"displayName":"mirror nerror","photoUrl":"//lh4.googleusercontent.com/-gGETofzEVi8/AAAAAAAAAAI/AAAAAAAADss/kscD3IWnpJ4/s50-c-k-no/photo.jpg","userId":"110024841757699787861"}}},"cell_type":"code","source":["from rl.callbacks import Callback\n","import matplotlib.pyplot as plt\n","import matplotlib.animation as animation\n","from IPython.display import HTML\n","%matplotlib inline\n","\n","ims = []  # list for animation\n","\n","class Render(Callback):\n","    def on_step_end(self, step, logs={}):\n","        im = plt.imshow(env.render(mode='rgb_array'))\n","        ims.append([im])\n","\n","#weights_filename = 'dqn_{}_weights.h5f'.format(ENV_NAME)\n","weights_filename = 'dqn_BreakoutDeterministic-v4_weights_1750000.h5f'\n","dqn.load_weights(weights_filename)\n","\n","callbacks = Render()\n","fig = plt.figure(figsize=(4,5)) # output size\n","plt.axis('off')                 # no frame\n","dqn.test(env, nb_episodes=1, visualize=False, callbacks=[callbacks])\n","\n","ani = animation.ArtistAnimation(fig=fig, artists=ims, interval=10)\n","# ani.save(\"anim.gif\", writer = \"imagemagick\") # save as gif animation\n","plt.close()\n","\n","# animation on Jupyter Notebook\n","HTML(ani.to_jshtml())        # JavascriptHTML output\n","#HTML(ani.to_html5_video())  # HTML5 Video output"],"execution_count":30},{"metadata":{"id":"xZITVZmBmENZ","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]}]}