ugo-nama-kun

val=42
for i in $(seq 1 5); do
  echo $1
  echo $(($val+$2))
done

ugo-nama-kun

# From : https://stackoverflow.com/questions/10003143/how-to-slice-a-deque

class sliceable_deque(collections.deque):
    def __getitem__(self, index):
        try:
            return collections.deque.__getitem__(self, index)
        except TypeError:
            return type(self)(itertools.islice(self, index.start,
                                               index.stop, index.step))

ugo-nama-kun

import matplotlib.pyplot as plt
import numpy as np

N = 100

x = np.linspace(0, 6*np.pi, N)

y = np.sin(x)

upper = y + 0.2

ugo-nama-kun

gamma = 0.05

def f(x):
    # x = [y, v]^T
    a = np.array(
        [
            [0, 1],
            [-1, -2*gamma]
        ]
    )

ugo-nama-kun

from numpy import std, mean, sqrt

#correct if the population S.D. is expected to be equal for the two groups.
def cohen_d(x,y):
    nx = len(x)
    ny = len(y)
    dof = nx + ny - 2
    return (mean(x) - mean(y)) / sqrt(((nx-1)*std(x, ddof=1) ** 2 + (ny-1)*std(y, ddof=1) ** 2) / dof)

ugo-nama-kun

import torch
import numpy as np
import matplotlib.pyplot as plt

# Assuming some environment that render image...
im = env.render(mode="rgb_array",
                height=84,
                width=84,
                camera_id=0) / 255.
im = torch.tensor(im.astype(np.float32))

ugo-nama-kun

# From: https://stackoverflow.com/questions/14313510/how-to-calculate-rolling-moving-average-using-python-numpy-scipy 

def moving_average(x: np.array, w: int):
    return np.convolve(x, np.ones(w), 'valid') / w

ugo-nama-kun

Deep RL + Continuous Control with Vision

• 深層強化学習で 連続行動 と 視覚入力 を使ったものをまとめる
• 特に重要なテクニックが書かれていればそれも書き出す
• マルチモーダルな強化学習もあれば書いておく

まとめた後の画像エージェントの構成パターン

• SAC のような形で、完全に actor と critic でネットワークを分けて CNN を2つ利用する
• actor と critic で CNN は共有するが、CNNの更新はcriticでのみしてactorはそれを利用する

ugo-nama-kun

from dm_control import suite
from collections import deque

# deepmind control suite を使う場合
env_dmc = suite.load("cartpole", "balance")

# dm_control で得た場合の画像の形式は (84, 84, 3)．これは mujoco_py の場合も同じ
im = self.env_dmc.physics.render(camera_id=0, height=84, width=84)

# frame stack [(84, 84, 3), (84, 84, 3), (84, 84, 3)]

ugo-nama-kun

import tensorflow as tf  # chacked @ tensorflow==2.6.0

available_gpus = tf.config.experimental.list_physical_devices('GPU')
print("Num GPUs Available: ", len(available_gpus))
if available_gpus:
    try:
        tf.config.experimental.set_visible_devices(available_gpus[gpu_id], "GPU")
        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
        print(len(available_gpus), "Physical GPUs,", len(logical_gpus), "Logical GPU")
    except RuntimeError as e: