ikatsov

num_episodes = 100
return_trace = []
q_values_rewards_trace = np.zeros((num_episodes, T, 2, ))
for i_episode in range(num_episodes):        # modified copy of the simulation loop
    state = env_intial_state()
    for t in range(T):
        # Select and perform an action
        with torch.no_grad():
          q_values = policy_net(to_tensor(state)).detach().numpy()
        action = policy.select_action(q_values)

ikatsov

sample_state = [170, 0, ... ]
Q_s = policy_net(to_tensor(sample_state))
a_opt = Q_s.max(0)[1].detach()        # Optimal price action is price_grid[a_opt]

plt.bar(price_grid, Q_s.detach().numpy(), color='crimson',  width=6, alpha=0.8)

ikatsov

policy_net = PolicyNetworkDQN(2*T, len(price_grid)).to(device)
target_net = PolicyNetworkDQN(2*T, len(price_grid)).to(device)
optimizer = optim.AdamW(policy_net.parameters(), lr = 0.005)
policy = AnnealedEpsGreedyPolicy()
memory = ReplayMemory(10000)

TARGET_UPDATE = 20

num_episodes = 1000
return_trace = []

ikatsov

def env_intial_state():
  return np.repeat(0, 2*T)

def env_step(t, state, action):
  next_state = np.repeat(0, len(state))
  next_state[0] = price_grid[action]  # first element is the current price
  next_state[1:T] = state[0:T-1]      # price history
  next_state[T+t] = 1                 # one-hot encoding of time t
  reward = profit_t_response(next_state[0], next_state[1])
  return next_state, reward

ikatsov

GAMMA = 1.00
BATCH_SIZE = 512

def update_model(memory, policy_net, target_net):
    if len(memory) < BATCH_SIZE:
        return
    transitions = memory.sample(BATCH_SIZE)
    batch = Transition(*zip(*transitions))

    non_final_mask = torch.tensor(tuple(                    

ikatsov

class AnnealedEpsGreedyPolicy(object):
  def __init__(self, eps_start = 0.9, eps_end = 0.05, eps_decay = 400):
    self.eps_start = eps_start
    self.eps_end = eps_end
    self.eps_decay = eps_decay
    self.steps_done = 0

  def select_action(self, q_values):
    sample = random.random()
    eps_threshold = self.eps_end +                 # exponential decay

ikatsov

class PolicyNetworkDQN(nn.Module):
    def __init__(self, state_size, action_size, hidden_size=128):
      super(PolicyNetworkDQN, self).__init__()
      layers = [
              nn.Linear(state_size, hidden_size),       # simple network with 4 layers
              nn.ReLU(),
              nn.Linear(hidden_size, hidden_size),
              nn.ReLU(),
              nn.Linear(hidden_size, hidden_size),
              nn.ReLU(),

ikatsov

Transition = namedtuple('Transition', ('state', 'action', 'next_state', 'reward'))

class ReplayMemory(object):                             
    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0

    def push(self, *args):                                     # push new transition to the buffer
        if len(self.memory) < self.capacity:

ikatsov

def find_optimal_price_t(p_baseline, price_grid, t):   # evalutes all possible price schedules 
  p_grid = np.tile(p_baseline, (len(price_grid), 1))   # derived from the baseline by  
  p_grid[:, t] = price_grid                            # changing the price at time t
  profit_grid = np.array([ profit_response(p) for p in p_grid ])
  return price_grid[ np.argmax(profit_grid) ]

p_opt = np.repeat(price_opt_const, T)                  # start with the constant price schedule
for t in range(T):                                     # and optimize one price at a time
  price_t = find_optimal_price_t(p_opt, price_grid, t)
  p_opt[t] = price_t

ikatsov

profits = np.array([ profit_response(np.repeat(p, T)) for p in price_grid ])
p_idx = np.argmax(profits)                 # index of the profit-maximizing price
price_opt_const = price_grid[p_idx]        # profit-maximizing price
print(f'Optimal price is {price_opt_const}, achieved profit is {profits[p_idx]}')

#> Optimal price is 170, achieved profit is 2352000.0