From b8689acb5d78bcca0ed82c9db56c7830311aad6d Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 31 Jul 2020 02:37:36 +0900 Subject: [PATCH 01/16] add MDQN agent and example script --- examples/atari/train_mdqn_ale.py | 308 +++++++++++++++++++++++++++++++ pfrl/agents/__init__.py | 1 + pfrl/agents/mdqn.py | 130 +++++++++++++ 3 files changed, 439 insertions(+) create mode 100644 examples/atari/train_mdqn_ale.py create mode 100644 pfrl/agents/mdqn.py diff --git a/examples/atari/train_mdqn_ale.py b/examples/atari/train_mdqn_ale.py new file mode 100644 index 000000000..fd7ab1a28 --- /dev/null +++ b/examples/atari/train_mdqn_ale.py @@ -0,0 +1,308 @@ +import argparse + +import torch +import torch.nn as nn +import torch.optim as optim +import numpy as np + +import pfrl +from pfrl.q_functions import DiscreteActionValueHead +from pfrl.agents import MDQN +from pfrl import experiments +from pfrl import explorers +from pfrl import nn as pnn +from pfrl import utils +from pfrl.q_functions import DuelingDQN +from pfrl import replay_buffers + +from pfrl.wrappers import atari_wrappers +from pfrl.initializers import init_chainer_default + + +class SingleSharedBias(nn.Module): + """Single shared bias used in the Double DQN paper. + + You can add this link after a Linear layer with nobias=True to implement a + Linear layer with a single shared bias parameter. + + See http://arxiv.org/abs/1509.06461. + """ + + def __init__(self): + super().__init__() + self.bias = nn.Parameter(torch.zeros([1], dtype=torch.float32)) + + def __call__(self, x): + return x + self.bias.expand_as(x) + + +def parse_arch(arch, n_actions): + if arch == "nature": + return nn.Sequential( + pnn.LargeAtariCNN(), + init_chainer_default(nn.Linear(512, n_actions)), + DiscreteActionValueHead(), + ) + elif arch == "doubledqn": + return nn.Sequential( + pnn.LargeAtariCNN(), + init_chainer_default(nn.Linear(512, n_actions, bias=False)), + SingleSharedBias(), + DiscreteActionValueHead(), + ) + elif arch == "nips": + return nn.Sequential( + pnn.SmallAtariCNN(), + init_chainer_default(nn.Linear(256, n_actions)), + DiscreteActionValueHead(), + ) + elif arch == "dueling": + return DuelingDQN(n_actions) + else: + raise RuntimeError("Not supported architecture: {}".format(arch)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--env", + type=str, + default="BreakoutNoFrameskip-v4", + help="OpenAI Atari domain to perform algorithm on.", + ) + parser.add_argument( + "--outdir", + type=str, + default="results", + help=( + "Directory path to save output files." + " If it does not exist, it will be created." + ), + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed [0, 2 ** 31)") + parser.add_argument( + "--gpu", type=int, default=0, help="GPU to use, set to -1 if no GPU." + ) + parser.add_argument("--demo", action="store_true", default=False) + parser.add_argument("--load", type=str, default=None) + parser.add_argument( + "--final-exploration-frames", + type=int, + default=250000, + help="Timesteps after which we stop " + "annealing exploration rate", + ) + parser.add_argument( + "--final-epsilon", + type=float, + default=0.01, + help="Final value of epsilon during training.", + ) + parser.add_argument( + "--eval-epsilon", + type=float, + default=0.001, + help="Exploration epsilon used during eval episodes.", + ) + parser.add_argument("--noisy-net-sigma", type=float, default=None) + parser.add_argument( + "--arch", + type=str, + default="doubledqn", + choices=["nature", "nips", "dueling", "doubledqn"], + help="Network architecture to use.", + ) + parser.add_argument( + "--steps", + type=int, + default=5 * 10 ** 7, + help="Total number of timesteps to train the agent.", + ) + parser.add_argument( + "--max-frames", + type=int, + default=30 * 60 * 60, # 30 minutes with 60 fps + help="Maximum number of frames for each episode.", + ) + parser.add_argument( + "--replay-start-size", + type=int, + default=5 * 10 ** 4, + help="Minimum replay buffer size before " + "performing gradient updates.", + ) + parser.add_argument( + "--target-update-interval", + type=int, + default=3 * 10 ** 4, + help="Frequency (in timesteps) at which " + "the target network is updated.", + ) + parser.add_argument( + "--eval-interval", + type=int, + default=10 ** 5, + help="Frequency (in timesteps) of evaluation phase.", + ) + parser.add_argument( + "--update-interval", + type=int, + default=4, + help="Frequency (in timesteps) of network updates.", + ) + parser.add_argument("--eval-n-runs", type=int, default=10) + parser.add_argument("--no-clip-delta", dest="clip_delta", action="store_false") + parser.add_argument("--num-step-return", type=int, default=1) + parser.set_defaults(clip_delta=True) + parser.add_argument( + "--log-level", + type=int, + default=20, + help="Logging level. 10:DEBUG, 20:INFO etc.", + ) + parser.add_argument( + "--render", + action="store_true", + default=False, + help="Render env states in a GUI window.", + ) + parser.add_argument( + "--monitor", + action="store_true", + default=False, + help=( + "Monitor env. Videos and additional information are saved as output files." + ), + ) + parser.add_argument("--lr", type=float, default=5e-4, help="Learning rate.") + parser.add_argument( + "--prioritized", + action="store_true", + default=False, + help="Use prioritized experience replay.", + ) + parser.add_argument( + "--checkpoint-frequency", + type=int, + default=None, + help="Frequency at which agents are stored.", + ) + args = parser.parse_args() + + import logging + + logging.basicConfig(level=args.log_level) + + # Set a random seed used in PFRL. + utils.set_random_seed(args.seed) + + # Set different random seeds for train and test envs. + train_seed = args.seed + test_seed = 2 ** 31 - 1 - args.seed + + args.outdir = experiments.prepare_output_dir(args, args.outdir) + print("Output files are saved in {}".format(args.outdir)) + + def make_env(test): + # Use different random seeds for train and test envs + env_seed = test_seed if test else train_seed + env = atari_wrappers.wrap_deepmind( + atari_wrappers.make_atari(args.env, max_frames=args.max_frames), + episode_life=not test, + clip_rewards=not test, + ) + env.seed(int(env_seed)) + if test: + # Randomize actions like epsilon-greedy in evaluation as well + env = pfrl.wrappers.RandomizeAction(env, args.eval_epsilon) + if args.monitor: + env = pfrl.wrappers.Monitor( + env, args.outdir, mode="evaluation" if test else "training" + ) + if args.render: + env = pfrl.wrappers.Render(env) + return env + + env = make_env(test=False) + eval_env = make_env(test=True) + + n_actions = env.action_space.n + q_func = parse_arch(args.arch, n_actions) + + if args.noisy_net_sigma is not None: + pnn.to_factorized_noisy(q_func, sigma_scale=args.noisy_net_sigma) + # Turn off explorer + explorer = explorers.Greedy() + else: + explorer = explorers.LinearDecayEpsilonGreedy( + 1.0, + args.final_epsilon, + args.final_exploration_frames, + lambda: np.random.randint(n_actions), + ) + + # Use the Nature paper's hyperparameters + opt = optim.Adam( q_func.parameters(), lr=args.lr) + + # Select a replay buffer to use + if args.prioritized: + # Anneal beta from beta0 to 1 throughout training + betasteps = args.steps / args.update_interval + rbuf = replay_buffers.PrioritizedReplayBuffer( + 10 ** 6, + alpha=0.6, + beta0=0.4, + betasteps=betasteps, + num_steps=args.num_step_return, + ) + else: + rbuf = replay_buffers.ReplayBuffer(10 ** 6, args.num_step_return) + + def phi(x): + # Feature extractor + return np.asarray(x, dtype=np.float32) / 255 + + agent = MDQN( + q_func, + opt, + rbuf, + gpu=args.gpu, + gamma=0.99, + explorer=explorer, + replay_start_size=args.replay_start_size, + target_update_interval=args.target_update_interval, + clip_delta=args.clip_delta, + update_interval=args.update_interval, + batch_accumulator="sum", + phi=phi, + ) + + if args.load: + agent.load(args.load) + + if args.demo: + eval_stats = experiments.eval_performance( + env=eval_env, agent=agent, n_steps=None, n_episodes=args.eval_n_runs + ) + print( + "n_runs: {} mean: {} median: {} stdev {}".format( + args.eval_n_runs, + eval_stats["mean"], + eval_stats["median"], + eval_stats["stdev"], + ) + ) + else: + experiments.train_agent_with_evaluation( + agent=agent, + env=env, + steps=args.steps, + eval_n_steps=None, + checkpoint_freq=args.checkpoint_frequency, + eval_n_episodes=args.eval_n_runs, + eval_interval=args.eval_interval, + outdir=args.outdir, + save_best_so_far_agent=False, + eval_env=eval_env, + ) + + +if __name__ == "__main__": + main() diff --git a/pfrl/agents/__init__.py b/pfrl/agents/__init__.py index b8cd64644..22f6762af 100644 --- a/pfrl/agents/__init__.py +++ b/pfrl/agents/__init__.py @@ -10,6 +10,7 @@ from pfrl.agents.dpp import DPP # NOQA from pfrl.agents.dqn import DQN # NOQA from pfrl.agents.iqn import IQN # NOQA +from pfrl.agents.mdqn import MDQN # NOQA from pfrl.agents.pal import PAL # NOQA from pfrl.agents.ppo import PPO # NOQA from pfrl.agents.reinforce import REINFORCE # NOQA diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py new file mode 100644 index 000000000..195ab832f --- /dev/null +++ b/pfrl/agents/mdqn.py @@ -0,0 +1,130 @@ +from logging import getLogger + +import torch + +from pfrl.agents import dqn +from pfrl.utils.batch_states import batch_states +from pfrl.utils.recurrent import pack_and_forward + + +class MDQN(dqn.DQN): + """Munchausen Deep Q-Network algorithm. + + Args: + q_function (StateQFunction): Q-function + optimizer (Optimizer): Optimizer that is already setup + replay_buffer (ReplayBuffer): Replay buffer + gamma (float): Discount factor + explorer (Explorer): Explorer that specifies an exploration strategy. + gpu (int): GPU device id if not None nor negative. + replay_start_size (int): if the replay buffer's size is less than + replay_start_size, skip update + minibatch_size (int): Minibatch size + update_interval (int): Model update interval in step + target_update_interval (int): Target model update interval in step + clip_delta (bool): Clip delta if set True + phi (callable): Feature extractor applied to observations + target_update_method (str): 'hard' or 'soft'. + soft_update_tau (float): Tau of soft target update. + n_times_update (int): Number of repetition of update + batch_accumulator (str): 'mean' or 'sum' + episodic_update_len (int or None): Subsequences of this length are used + for update if set int and episodic_update=True + logger (Logger): Logger used + batch_states (callable): method which makes a batch of observations. + default is `pfrl.utils.batch_states.batch_states` + recurrent (bool): If set to True, `model` is assumed to implement + `pfrl.nn.Recurrent` and is updated in a recurrent + manner. + max_grad_norm (float or None): Maximum L2 norm of the gradient used for + gradient clipping. If set to None, the gradient is not clipped. + temperature (float): entropy temperature + scaling_term (float): Munchausen scaling term + clip_l0 (float): log-policy clipping value + """ + + saved_attributes = ("model", "target_model", "optimizer") + + def __init__( + self, + q_function, + optimizer, + replay_buffer, + gamma, + explorer, + gpu=None, + replay_start_size=50000, + minibatch_size=32, + update_interval=1, + target_update_interval=10000, + clip_delta=True, + phi=lambda x: x, + target_update_method="hard", + soft_update_tau=1e-2, + n_times_update=1, + batch_accumulator="mean", + episodic_update_len=None, + logger=getLogger(__name__), + batch_states=batch_states, + recurrent=False, + max_grad_norm=None, + temperature=0.03, + scaling_term=0.9, + clip_l0=-1.0, + ): + super(MDQN, self).__init__( + q_function, + optimizer, + replay_buffer, + gamma, + explorer, + gpu=gpu, + replay_start_size=replay_start_size, + minibatch_size=minibatch_size, + update_interval=update_interval, + target_update_interval=target_update_interval, + clip_delta=clip_delta, + phi=phi, + target_update_method=target_update_method, + soft_update_tau=soft_update_tau, + n_times_update=n_times_update, + batch_accumulator=batch_accumulator, + episodic_update_len=episodic_update_len, + logger=logger, + batch_states=batch_states, + recurrent=recurrent, + max_grad_norm=max_grad_norm, + ) + + self.temperature = temperature + self.scaling_term = scaling_term + self.clip_l0 = clip_l0 + + def _compute_target_values(self, exp_batch): + batch_next_state = exp_batch["next_state"] + + if self.recurrent: + target_next_qout, _ = pack_and_forward( + self.target_model, batch_next_state, exp_batch["next_recurrent_state"], + ) + else: + target_next_qout = self.target_model(batch_next_state) + next_q_max = target_next_qout.max + + # log-sum-exp-trick + advantages = target_next_qout.q_values - next_q_max.unsqueeze(1) + t_ln_pi = advantages - self.temperature * ( + advantages / self.temperature + ).exp().sum(dim=1).log().unsqueeze(1) + pi = (t_ln_pi / self.temperature).exp() + + batch_actions = exp_batch["action"].unsqueeze(1) + batch_rewards = ( + exp_batch["reward"] + t_ln_pi.gather(dim=1, index=batch_actions).flatten() + ) + batch_terminal = exp_batch["is_state_terminal"] + discount = exp_batch["discount"] + + next_value = torch.sum(pi * (target_next_qout.q_values - t_ln_pi), dim=1) + + return batch_rewards + discount * (1.0 - batch_terminal) * next_value From 73f5e98c5ad0c1b6f1311ec79fe24e7c681ecf59 Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 31 Jul 2020 03:48:56 +0900 Subject: [PATCH 02/16] cast to long --- pfrl/agents/mdqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index 195ab832f..81aa1053f 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -118,7 +118,7 @@ def _compute_target_values(self, exp_batch): ).exp().sum(dim=1).log().unsqueeze(1) pi = (t_ln_pi / self.temperature).exp() - batch_actions = exp_batch["action"].unsqueeze(1) + batch_actions = exp_batch["action"].long().unsqueeze(1) batch_rewards = ( exp_batch["reward"] + t_ln_pi.gather(dim=1, index=batch_actions).flatten() ) From 87c298c02020c4424c017612a07928333c953bac Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 31 Jul 2020 10:29:13 +0900 Subject: [PATCH 03/16] add scaling and clipping --- examples/atari/train_mdqn_ale.py | 2 +- pfrl/agents/mdqn.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/atari/train_mdqn_ale.py b/examples/atari/train_mdqn_ale.py index fd7ab1a28..c700d9b1b 100644 --- a/examples/atari/train_mdqn_ale.py +++ b/examples/atari/train_mdqn_ale.py @@ -239,7 +239,7 @@ def make_env(test): ) # Use the Nature paper's hyperparameters - opt = optim.Adam( q_func.parameters(), lr=args.lr) + opt = optim.Adam(q_func.parameters(), lr=args.lr) # Select a replay buffer to use if args.prioritized: diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index 81aa1053f..208b047b4 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -119,8 +119,9 @@ def _compute_target_values(self, exp_batch): pi = (t_ln_pi / self.temperature).exp() batch_actions = exp_batch["action"].long().unsqueeze(1) - batch_rewards = ( - exp_batch["reward"] + t_ln_pi.gather(dim=1, index=batch_actions).flatten() + chosen_t_ln_pi = t_ln_pi.gather(dim=1, index=batch_actions).flatten() + batch_rewards = exp_batch["reward"] + self.scaling_term * torch.max( + chosen_t_ln_pi, torch.tensor(self.clip_l0) ) batch_terminal = exp_batch["is_state_terminal"] discount = exp_batch["discount"] From 64d4742ad05e4244d5f7d46d65cddf9590a439fa Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 31 Jul 2020 02:11:15 +0000 Subject: [PATCH 04/16] add tensor device --- pfrl/agents/mdqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index 208b047b4..d499b72ab 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -121,7 +121,7 @@ def _compute_target_values(self, exp_batch): batch_actions = exp_batch["action"].long().unsqueeze(1) chosen_t_ln_pi = t_ln_pi.gather(dim=1, index=batch_actions).flatten() batch_rewards = exp_batch["reward"] + self.scaling_term * torch.max( - chosen_t_ln_pi, torch.tensor(self.clip_l0) + chosen_t_ln_pi, torch.tensor(self.clip_l0, device=self.device) ) batch_terminal = exp_batch["is_state_terminal"] discount = exp_batch["discount"] From f33dfa56e01ab42d4e29523172fe93423a34fa8b Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 31 Jul 2020 11:35:53 +0900 Subject: [PATCH 05/16] add reference --- pfrl/agents/mdqn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index d499b72ab..b735eadff 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -10,6 +10,8 @@ class MDQN(dqn.DQN): """Munchausen Deep Q-Network algorithm. + See https://arxiv.org/abs/2007.14430. + Args: q_function (StateQFunction): Q-function optimizer (Optimizer): Optimizer that is already setup From b72db9a7c3a079e05e25e2ccd6a83cc13ab9aad3 Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 31 Jul 2020 18:41:39 +0900 Subject: [PATCH 06/16] fix scaled log policy bonus --- pfrl/agents/mdqn.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index b735eadff..1effeb6f9 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -103,31 +103,50 @@ def __init__( self.clip_l0 = clip_l0 def _compute_target_values(self, exp_batch): - batch_next_state = exp_batch["next_state"] + # Compute Q-values for current states using the target network + batch_state = exp_batch["state"] if self.recurrent: - target_next_qout, _ = pack_and_forward( - self.target_model, batch_next_state, exp_batch["next_recurrent_state"], + qout, _ = pack_and_forward( + self.target_model, batch_state, exp_batch["recurrent_state"] ) else: - target_next_qout = self.target_model(batch_next_state) - next_q_max = target_next_qout.max + qout = self.target_model(batch_state) # log-sum-exp-trick - advantages = target_next_qout.q_values - next_q_max.unsqueeze(1) + advantages = qout.q_values - qout.max.unsqueeze(1) t_ln_pi = advantages - self.temperature * ( advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) pi = (t_ln_pi / self.temperature).exp() + # add scaled log policy batch_actions = exp_batch["action"].long().unsqueeze(1) chosen_t_ln_pi = t_ln_pi.gather(dim=1, index=batch_actions).flatten() - batch_rewards = exp_batch["reward"] + self.scaling_term * torch.max( + exp_batch["reward"] += self.scaling_term * torch.max( chosen_t_ln_pi, torch.tensor(self.clip_l0, device=self.device) ) + + # value of next state (entropy-augmented) using the target network + batch_next_state = exp_batch["next_state"] + + if self.recurrent: + target_next_qout, _ = pack_and_forward( + self.target_model, batch_next_state, exp_batch["next_recurrent_state"], + ) + else: + target_next_qout = self.target_model(batch_next_state) + next_q_max = target_next_qout.max + + # log-sum-exp-trick + next_advantages = target_next_qout.q_values - next_q_max.unsqueeze(1) + next_t_ln_pi = next_advantages - self.temperature * ( + advantages / self.temperature + ).exp().sum(dim=1).log().unsqueeze(1) + next_value = torch.sum(pi * (target_next_qout.q_values - next_t_ln_pi), dim=1) + + batch_rewards = exp_batch["reward"] batch_terminal = exp_batch["is_state_terminal"] discount = exp_batch["discount"] - next_value = torch.sum(pi * (target_next_qout.q_values - t_ln_pi), dim=1) - return batch_rewards + discount * (1.0 - batch_terminal) * next_value From c73de3e6ec5f0dad87b2ef70b2f58e1a57bbce4b Mon Sep 17 00:00:00 2001 From: marioyc Date: Mon, 3 Aug 2020 14:32:58 +0900 Subject: [PATCH 07/16] change default target update interval according to the paper --- examples/atari/train_mdqn_ale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/atari/train_mdqn_ale.py b/examples/atari/train_mdqn_ale.py index c700d9b1b..19f57ec8a 100644 --- a/examples/atari/train_mdqn_ale.py +++ b/examples/atari/train_mdqn_ale.py @@ -132,7 +132,7 @@ def main(): parser.add_argument( "--target-update-interval", type=int, - default=3 * 10 ** 4, + default=8 * 10 ** 3, help="Frequency (in timesteps) at which " + "the target network is updated.", ) parser.add_argument( From 0c46d761e7a2d6da4546ba4064159e79ee543dfe Mon Sep 17 00:00:00 2001 From: marioyc Date: Mon, 3 Aug 2020 20:46:12 +0900 Subject: [PATCH 08/16] fix log-sum-exp-trick calculation --- pfrl/agents/mdqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index 1effeb6f9..a8c37c48b 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -141,7 +141,7 @@ def _compute_target_values(self, exp_batch): # log-sum-exp-trick next_advantages = target_next_qout.q_values - next_q_max.unsqueeze(1) next_t_ln_pi = next_advantages - self.temperature * ( - advantages / self.temperature + next_advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) next_value = torch.sum(pi * (target_next_qout.q_values - next_t_ln_pi), dim=1) From 327d60ddb06f87d5dd92c36d3e9c06724e3cdb69 Mon Sep 17 00:00:00 2001 From: marioyc Date: Tue, 4 Aug 2020 05:08:49 +0900 Subject: [PATCH 09/16] fix next value calculation, add logs --- pfrl/agents/mdqn.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index a8c37c48b..e55c24c80 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -1,12 +1,19 @@ +import collections from logging import getLogger import torch +import numpy as np from pfrl.agents import dqn from pfrl.utils.batch_states import batch_states from pfrl.utils.recurrent import pack_and_forward +def _mean_or_nan(xs): + """Return its mean a non-empty sequence, numpy.nan for a empty one.""" + return np.mean(xs) if xs else np.nan + + class MDQN(dqn.DQN): """Munchausen Deep Q-Network algorithm. @@ -102,6 +109,10 @@ def __init__( self.scaling_term = scaling_term self.clip_l0 = clip_l0 + self.augmented_reward_record = collections.deque(maxlen=1000) + self.next_value_record = collections.deque(maxlen=1000) + self.next_entropy_record = collections.deque(maxlen=1000) + def _compute_target_values(self, exp_batch): # Compute Q-values for current states using the target network batch_state = exp_batch["state"] @@ -118,14 +129,14 @@ def _compute_target_values(self, exp_batch): t_ln_pi = advantages - self.temperature * ( advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) - pi = (t_ln_pi / self.temperature).exp() # add scaled log policy batch_actions = exp_batch["action"].long().unsqueeze(1) chosen_t_ln_pi = t_ln_pi.gather(dim=1, index=batch_actions).flatten() - exp_batch["reward"] += self.scaling_term * torch.max( + augmented_rewards = exp_batch["reward"] + self.scaling_term * torch.max( chosen_t_ln_pi, torch.tensor(self.clip_l0, device=self.device) ) + self.augmented_reward_record.extend(augmented_rewards.detach().cpu().numpy()) # value of next state (entropy-augmented) using the target network batch_next_state = exp_batch["next_state"] @@ -143,10 +154,21 @@ def _compute_target_values(self, exp_batch): next_t_ln_pi = next_advantages - self.temperature * ( next_advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) - next_value = torch.sum(pi * (target_next_qout.q_values - next_t_ln_pi), dim=1) + next_pi = (next_t_ln_pi / self.temperature).exp() + next_value = (next_pi * (target_next_qout.q_values - next_t_ln_pi)).sum(dim=1) + self.next_value_record.extend(next_value.detach().cpu().numpy()) + + next_entropy = -(next_pi * next_t_ln_pi).sum(dim=1) + self.next_entropy_record.extend(next_entropy.detach().cpu().numpy()) - batch_rewards = exp_batch["reward"] batch_terminal = exp_batch["is_state_terminal"] discount = exp_batch["discount"] - return batch_rewards + discount * (1.0 - batch_terminal) * next_value + return augmented_rewards + discount * (1.0 - batch_terminal) * next_value + + def get_statistics(self): + return super(MDQN, self).get_statistics() + [ + ("augmented_reward", _mean_or_nan(self.augmented_reward_record)), + ("next_value", _mean_or_nan(self.next_value_record)), + ("next_entropy", _mean_or_nan(self.next_entropy)), + ] From bfdcae047c52497aa2b41a9472d002920aa3ae10 Mon Sep 17 00:00:00 2001 From: marioyc Date: Tue, 4 Aug 2020 10:48:45 +0900 Subject: [PATCH 10/16] add additional statistics --- pfrl/agents/mdqn.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index e55c24c80..52e3f8db1 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -109,7 +109,10 @@ def __init__( self.scaling_term = scaling_term self.clip_l0 = clip_l0 + self.pi_sum_record = collections.deque(maxlen=1000) + self.bonus_reward_record = collections.deque(maxlen=1000) self.augmented_reward_record = collections.deque(maxlen=1000) + self.next_pi_sum_record = collections.deque(maxlen=1000) self.next_value_record = collections.deque(maxlen=1000) self.next_entropy_record = collections.deque(maxlen=1000) @@ -129,13 +132,17 @@ def _compute_target_values(self, exp_batch): t_ln_pi = advantages - self.temperature * ( advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) + pi = (t_ln_pi / self.temperature).exp() + self.pi_sum_record.extend(pi.detach().cpu().numpy()) # add scaled log policy batch_actions = exp_batch["action"].long().unsqueeze(1) chosen_t_ln_pi = t_ln_pi.gather(dim=1, index=batch_actions).flatten() - augmented_rewards = exp_batch["reward"] + self.scaling_term * torch.max( + bonus = self.scaling_term * torch.max( chosen_t_ln_pi, torch.tensor(self.clip_l0, device=self.device) ) + self.bonus_reward_record.extend(bonus.detach().cpu().numpy()) + augmented_rewards = exp_batch["reward"] + bonus self.augmented_reward_record.extend(augmented_rewards.detach().cpu().numpy()) # value of next state (entropy-augmented) using the target network @@ -155,6 +162,7 @@ def _compute_target_values(self, exp_batch): next_advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) next_pi = (next_t_ln_pi / self.temperature).exp() + self.next_pi_sum_record.extend(next_pi.detach().cpu().numpy()) next_value = (next_pi * (target_next_qout.q_values - next_t_ln_pi)).sum(dim=1) self.next_value_record.extend(next_value.detach().cpu().numpy()) @@ -168,7 +176,10 @@ def _compute_target_values(self, exp_batch): def get_statistics(self): return super(MDQN, self).get_statistics() + [ + ("pi_sum", _mean_or_nan(self.pi_sum_record)), + ("bonus", _mean_or_nan(self.bonus_reward_record)), ("augmented_reward", _mean_or_nan(self.augmented_reward_record)), + ("next_pi_sum", _mean_or_nan(self.next_pi_sum_record)), ("next_value", _mean_or_nan(self.next_value_record)), - ("next_entropy", _mean_or_nan(self.next_entropy)), + ("next_entropy", _mean_or_nan(self.next_entropy_record)), ] From eac093ce43a71e0bbb39f641a5f3ec9349155ba3 Mon Sep 17 00:00:00 2001 From: marioyc Date: Tue, 4 Aug 2020 13:16:49 +0900 Subject: [PATCH 11/16] fix debug of pi's sum --- pfrl/agents/mdqn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index 52e3f8db1..c2b6a9871 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -133,7 +133,7 @@ def _compute_target_values(self, exp_batch): advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) pi = (t_ln_pi / self.temperature).exp() - self.pi_sum_record.extend(pi.detach().cpu().numpy()) + self.pi_sum_record.extend(pi.sum(dim=1).detach().cpu().numpy()) # add scaled log policy batch_actions = exp_batch["action"].long().unsqueeze(1) @@ -162,7 +162,7 @@ def _compute_target_values(self, exp_batch): next_advantages / self.temperature ).exp().sum(dim=1).log().unsqueeze(1) next_pi = (next_t_ln_pi / self.temperature).exp() - self.next_pi_sum_record.extend(next_pi.detach().cpu().numpy()) + self.next_pi_sum_record.extend(next_pi.sum(dim=1).detach().cpu().numpy()) next_value = (next_pi * (target_next_qout.q_values - next_t_ln_pi)).sum(dim=1) self.next_value_record.extend(next_value.detach().cpu().numpy()) From a137895812061d279a43c2bb20f80571eeba62cb Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 18 Sep 2020 00:48:15 +0000 Subject: [PATCH 12/16] use logsumexp, clamp and softmax --- pfrl/agents/mdqn.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index c2b6a9871..24553561c 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -2,6 +2,7 @@ from logging import getLogger import torch +import torch.nn.functional as F import numpy as np from pfrl.agents import dqn @@ -110,6 +111,7 @@ def __init__( self.clip_l0 = clip_l0 self.pi_sum_record = collections.deque(maxlen=1000) + self.chosen_pi_record = collections.deque(maxlen=1000) self.bonus_reward_record = collections.deque(maxlen=1000) self.augmented_reward_record = collections.deque(maxlen=1000) self.next_pi_sum_record = collections.deque(maxlen=1000) @@ -131,16 +133,16 @@ def _compute_target_values(self, exp_batch): advantages = qout.q_values - qout.max.unsqueeze(1) t_ln_pi = advantages - self.temperature * ( advantages / self.temperature - ).exp().sum(dim=1).log().unsqueeze(1) + ).logsumexp(dim=1, keepdim=True) pi = (t_ln_pi / self.temperature).exp() self.pi_sum_record.extend(pi.sum(dim=1).detach().cpu().numpy()) # add scaled log policy batch_actions = exp_batch["action"].long().unsqueeze(1) chosen_t_ln_pi = t_ln_pi.gather(dim=1, index=batch_actions).flatten() - bonus = self.scaling_term * torch.max( - chosen_t_ln_pi, torch.tensor(self.clip_l0, device=self.device) - ) + chosen_pi = (chosen_t_ln_pi / self.temperature).exp() + self.chosen_pi_record.extend(chosen_pi.detach().cpu().numpy()) + bonus = self.scaling_term * torch.clamp(chosen_t_ln_pi, min=self.clip_l0, max=0) self.bonus_reward_record.extend(bonus.detach().cpu().numpy()) augmented_rewards = exp_batch["reward"] + bonus self.augmented_reward_record.extend(augmented_rewards.detach().cpu().numpy()) @@ -160,8 +162,10 @@ def _compute_target_values(self, exp_batch): next_advantages = target_next_qout.q_values - next_q_max.unsqueeze(1) next_t_ln_pi = next_advantages - self.temperature * ( next_advantages / self.temperature - ).exp().sum(dim=1).log().unsqueeze(1) - next_pi = (next_t_ln_pi / self.temperature).exp() + ).logsumexp(dim=1, keepdim=True) + #next_pi = (next_t_ln_pi / self.temperature).exp() + next_pi = F.softmax(target_next_qout.q_values / self.temperature, dim=1) + #next_pi = F.softmax(target_next_qout.q_values, dim=1) self.next_pi_sum_record.extend(next_pi.sum(dim=1).detach().cpu().numpy()) next_value = (next_pi * (target_next_qout.q_values - next_t_ln_pi)).sum(dim=1) self.next_value_record.extend(next_value.detach().cpu().numpy()) @@ -176,6 +180,7 @@ def _compute_target_values(self, exp_batch): def get_statistics(self): return super(MDQN, self).get_statistics() + [ + ("chosen_pi", _mean_or_nan(self.chosen_pi_record)), ("pi_sum", _mean_or_nan(self.pi_sum_record)), ("bonus", _mean_or_nan(self.bonus_reward_record)), ("augmented_reward", _mean_or_nan(self.augmented_reward_record)), From a80c86d51411bdccbe1c4c4f2bcffb565e2bb09b Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 18 Sep 2020 10:17:48 +0900 Subject: [PATCH 13/16] change optimizer hyperparameters --- examples/atari/train_mdqn_ale.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/atari/train_mdqn_ale.py b/examples/atari/train_mdqn_ale.py index 19f57ec8a..f88dac326 100644 --- a/examples/atari/train_mdqn_ale.py +++ b/examples/atari/train_mdqn_ale.py @@ -171,7 +171,7 @@ def main(): "Monitor env. Videos and additional information are saved as output files." ), ) - parser.add_argument("--lr", type=float, default=5e-4, help="Learning rate.") + parser.add_argument("--lr", type=float, default=5e-5, help="Learning rate.") parser.add_argument( "--prioritized", action="store_true", @@ -239,7 +239,7 @@ def make_env(test): ) # Use the Nature paper's hyperparameters - opt = optim.Adam(q_func.parameters(), lr=args.lr) + opt = optim.Adam(q_func.parameters(), lr=args.lr, eps=1e-2 / args.batch_size) # Select a replay buffer to use if args.prioritized: From 7106df55d6f62d44b1032bb6726afaad8b935c72 Mon Sep 17 00:00:00 2001 From: marioyc Date: Fri, 18 Sep 2020 11:24:27 +0900 Subject: [PATCH 14/16] add batch_size argument --- examples/atari/train_mdqn_ale.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/atari/train_mdqn_ale.py b/examples/atari/train_mdqn_ale.py index f88dac326..c135dc250 100644 --- a/examples/atari/train_mdqn_ale.py +++ b/examples/atari/train_mdqn_ale.py @@ -147,6 +147,7 @@ def main(): default=4, help="Frequency (in timesteps) of network updates.", ) + parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--eval-n-runs", type=int, default=10) parser.add_argument("--no-clip-delta", dest="clip_delta", action="store_false") parser.add_argument("--num-step-return", type=int, default=1) @@ -267,9 +268,10 @@ def phi(x): gamma=0.99, explorer=explorer, replay_start_size=args.replay_start_size, + minibatch_size=args.batch_size, + update_interval=args.update_interval, target_update_interval=args.target_update_interval, clip_delta=args.clip_delta, - update_interval=args.update_interval, batch_accumulator="sum", phi=phi, ) From ca4b9b016186a6e9761db9846d63330c943cb8b5 Mon Sep 17 00:00:00 2001 From: marioyc Date: Thu, 8 Oct 2020 22:40:08 +0900 Subject: [PATCH 15/16] simplify training script --- examples/atari/train_mdqn_ale.py | 90 +++++--------------------------- 1 file changed, 12 insertions(+), 78 deletions(-) diff --git a/examples/atari/train_mdqn_ale.py b/examples/atari/train_mdqn_ale.py index c135dc250..bf6697d92 100644 --- a/examples/atari/train_mdqn_ale.py +++ b/examples/atari/train_mdqn_ale.py @@ -1,6 +1,5 @@ import argparse -import torch import torch.nn as nn import torch.optim as optim import numpy as np @@ -12,56 +11,12 @@ from pfrl import explorers from pfrl import nn as pnn from pfrl import utils -from pfrl.q_functions import DuelingDQN from pfrl import replay_buffers from pfrl.wrappers import atari_wrappers from pfrl.initializers import init_chainer_default -class SingleSharedBias(nn.Module): - """Single shared bias used in the Double DQN paper. - - You can add this link after a Linear layer with nobias=True to implement a - Linear layer with a single shared bias parameter. - - See http://arxiv.org/abs/1509.06461. - """ - - def __init__(self): - super().__init__() - self.bias = nn.Parameter(torch.zeros([1], dtype=torch.float32)) - - def __call__(self, x): - return x + self.bias.expand_as(x) - - -def parse_arch(arch, n_actions): - if arch == "nature": - return nn.Sequential( - pnn.LargeAtariCNN(), - init_chainer_default(nn.Linear(512, n_actions)), - DiscreteActionValueHead(), - ) - elif arch == "doubledqn": - return nn.Sequential( - pnn.LargeAtariCNN(), - init_chainer_default(nn.Linear(512, n_actions, bias=False)), - SingleSharedBias(), - DiscreteActionValueHead(), - ) - elif arch == "nips": - return nn.Sequential( - pnn.SmallAtariCNN(), - init_chainer_default(nn.Linear(256, n_actions)), - DiscreteActionValueHead(), - ) - elif arch == "dueling": - return DuelingDQN(n_actions) - else: - raise RuntimeError("Not supported architecture: {}".format(arch)) - - def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -103,7 +58,6 @@ def main(): default=0.001, help="Exploration epsilon used during eval episodes.", ) - parser.add_argument("--noisy-net-sigma", type=float, default=None) parser.add_argument( "--arch", type=str, @@ -173,12 +127,6 @@ def main(): ), ) parser.add_argument("--lr", type=float, default=5e-5, help="Learning rate.") - parser.add_argument( - "--prioritized", - action="store_true", - default=False, - help="Use prioritized experience replay.", - ) parser.add_argument( "--checkpoint-frequency", type=int, @@ -225,36 +173,22 @@ def make_env(test): eval_env = make_env(test=True) n_actions = env.action_space.n - q_func = parse_arch(args.arch, n_actions) + q_func = nn.Sequential( + pnn.LargeAtariCNN(), + init_chainer_default(nn.Linear(512, n_actions)), + DiscreteActionValueHead(), + ) - if args.noisy_net_sigma is not None: - pnn.to_factorized_noisy(q_func, sigma_scale=args.noisy_net_sigma) - # Turn off explorer - explorer = explorers.Greedy() - else: - explorer = explorers.LinearDecayEpsilonGreedy( - 1.0, - args.final_epsilon, - args.final_exploration_frames, - lambda: np.random.randint(n_actions), - ) + explorer = explorers.LinearDecayEpsilonGreedy( + 1.0, + args.final_epsilon, + args.final_exploration_frames, + lambda: np.random.randint(n_actions), + ) - # Use the Nature paper's hyperparameters opt = optim.Adam(q_func.parameters(), lr=args.lr, eps=1e-2 / args.batch_size) - # Select a replay buffer to use - if args.prioritized: - # Anneal beta from beta0 to 1 throughout training - betasteps = args.steps / args.update_interval - rbuf = replay_buffers.PrioritizedReplayBuffer( - 10 ** 6, - alpha=0.6, - beta0=0.4, - betasteps=betasteps, - num_steps=args.num_step_return, - ) - else: - rbuf = replay_buffers.ReplayBuffer(10 ** 6, args.num_step_return) + rbuf = replay_buffers.ReplayBuffer(10 ** 6, args.num_step_return) def phi(x): # Feature extractor From 02e8779f7deceed721d112568a1c9a1660c32747 Mon Sep 17 00:00:00 2001 From: marioyc Date: Thu, 8 Oct 2020 22:52:35 +0900 Subject: [PATCH 16/16] remove debug statistics --- pfrl/agents/mdqn.py | 48 ++++++--------------------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/pfrl/agents/mdqn.py b/pfrl/agents/mdqn.py index 24553561c..062b4b318 100644 --- a/pfrl/agents/mdqn.py +++ b/pfrl/agents/mdqn.py @@ -1,20 +1,13 @@ -import collections from logging import getLogger import torch import torch.nn.functional as F -import numpy as np from pfrl.agents import dqn from pfrl.utils.batch_states import batch_states from pfrl.utils.recurrent import pack_and_forward -def _mean_or_nan(xs): - """Return its mean a non-empty sequence, numpy.nan for a empty one.""" - return np.mean(xs) if xs else np.nan - - class MDQN(dqn.DQN): """Munchausen Deep Q-Network algorithm. @@ -110,14 +103,6 @@ def __init__( self.scaling_term = scaling_term self.clip_l0 = clip_l0 - self.pi_sum_record = collections.deque(maxlen=1000) - self.chosen_pi_record = collections.deque(maxlen=1000) - self.bonus_reward_record = collections.deque(maxlen=1000) - self.augmented_reward_record = collections.deque(maxlen=1000) - self.next_pi_sum_record = collections.deque(maxlen=1000) - self.next_value_record = collections.deque(maxlen=1000) - self.next_entropy_record = collections.deque(maxlen=1000) - def _compute_target_values(self, exp_batch): # Compute Q-values for current states using the target network batch_state = exp_batch["state"] @@ -134,25 +119,22 @@ def _compute_target_values(self, exp_batch): t_ln_pi = advantages - self.temperature * ( advantages / self.temperature ).logsumexp(dim=1, keepdim=True) - pi = (t_ln_pi / self.temperature).exp() - self.pi_sum_record.extend(pi.sum(dim=1).detach().cpu().numpy()) # add scaled log policy batch_actions = exp_batch["action"].long().unsqueeze(1) chosen_t_ln_pi = t_ln_pi.gather(dim=1, index=batch_actions).flatten() - chosen_pi = (chosen_t_ln_pi / self.temperature).exp() - self.chosen_pi_record.extend(chosen_pi.detach().cpu().numpy()) - bonus = self.scaling_term * torch.clamp(chosen_t_ln_pi, min=self.clip_l0, max=0) - self.bonus_reward_record.extend(bonus.detach().cpu().numpy()) - augmented_rewards = exp_batch["reward"] + bonus - self.augmented_reward_record.extend(augmented_rewards.detach().cpu().numpy()) + augmented_rewards = exp_batch["reward"] + self.scaling_term * torch.clamp( + chosen_t_ln_pi, min=self.clip_l0, max=0 + ) # value of next state (entropy-augmented) using the target network batch_next_state = exp_batch["next_state"] if self.recurrent: target_next_qout, _ = pack_and_forward( - self.target_model, batch_next_state, exp_batch["next_recurrent_state"], + self.target_model, + batch_next_state, + exp_batch["next_recurrent_state"], ) else: target_next_qout = self.target_model(batch_next_state) @@ -163,28 +145,10 @@ def _compute_target_values(self, exp_batch): next_t_ln_pi = next_advantages - self.temperature * ( next_advantages / self.temperature ).logsumexp(dim=1, keepdim=True) - #next_pi = (next_t_ln_pi / self.temperature).exp() next_pi = F.softmax(target_next_qout.q_values / self.temperature, dim=1) - #next_pi = F.softmax(target_next_qout.q_values, dim=1) - self.next_pi_sum_record.extend(next_pi.sum(dim=1).detach().cpu().numpy()) next_value = (next_pi * (target_next_qout.q_values - next_t_ln_pi)).sum(dim=1) - self.next_value_record.extend(next_value.detach().cpu().numpy()) - - next_entropy = -(next_pi * next_t_ln_pi).sum(dim=1) - self.next_entropy_record.extend(next_entropy.detach().cpu().numpy()) batch_terminal = exp_batch["is_state_terminal"] discount = exp_batch["discount"] return augmented_rewards + discount * (1.0 - batch_terminal) * next_value - - def get_statistics(self): - return super(MDQN, self).get_statistics() + [ - ("chosen_pi", _mean_or_nan(self.chosen_pi_record)), - ("pi_sum", _mean_or_nan(self.pi_sum_record)), - ("bonus", _mean_or_nan(self.bonus_reward_record)), - ("augmented_reward", _mean_or_nan(self.augmented_reward_record)), - ("next_pi_sum", _mean_or_nan(self.next_pi_sum_record)), - ("next_value", _mean_or_nan(self.next_value_record)), - ("next_entropy", _mean_or_nan(self.next_entropy_record)), - ]