BipedalWalker-v3-replay

BipedalWalker-v3-replay.py

def replay(self, states, actions, rewards, dones, next_states, logp_ts):
    # reshape memory to appropriate shape for training
    states = np.vstack(states)
    next_states = np.vstack(next_states)
    actions = np.vstack(actions)
    logp_ts = np.vstack(logp_ts)

    # Get Critic network predictions 
    values = self.Critic.predict(states)
    next_values = self.Critic.predict(next_states)

    # Compute discounted rewards and advantages
    #discounted_r = self.discount_rewards(rewards)
    #advantages = np.vstack(discounted_r - values)
    advantages, target = self.get_gaes(rewards, dones, np.squeeze(values), np.squeeze(next_values))
    '''
    pylab.plot(adv,'.')
    pylab.plot(target,'-')
    ax=pylab.gca()
    ax.grid(True)
    pylab.subplots_adjust(left=0.05, right=0.98, top=0.96, bottom=0.06)
    pylab.show()
    if str(episode)[-2:] == "00": pylab.savefig(self.env_name+"_"+self.episode+".png")
    '''
    # stack everything to numpy array
    # pack all advantages, predictions and actions to y_true and when they are received
    # in custom loss function we unpack it
    y_true = np.hstack([advantages, actions, logp_ts])

    # training Actor and Critic networks
    a_loss = self.Actor.Actor.fit(states, y_true, epochs=self.epochs, verbose=0, shuffle=self.shuffle)
    c_loss = self.Critic.Critic.fit([states, values], target, epochs=self.epochs, verbose=0, shuffle=self.shuffle)

    # calculate loss parameters (should be done in loss, but couldn't find working way how to do that with disabled eager execution)
    pred = self.Actor.predict(states)
    log_std = -0.5 * np.ones(self.action_size, dtype=np.float32)
    logp = self.gaussian_likelihood(actions, pred, log_std)
    approx_kl = np.mean(logp_ts - logp)
    approx_ent = np.mean(-logp)

    self.writer.add_scalar('Data/actor_loss_per_replay', np.sum(a_loss.history['loss']), self.replay_count)
    self.writer.add_scalar('Data/critic_loss_per_replay', np.sum(c_loss.history['loss']), self.replay_count)
    self.writer.add_scalar('Data/approx_kl_per_replay', approx_kl, self.replay_count)
    self.writer.add_scalar('Data/approx_ent_per_replay', approx_ent, self.replay_count)
    self.replay_count += 1