Adds naive REINFORCE algorithm

Author: ebetica

actor_critic/README.md

@@ -0,0 +1,9 @@
+# Reinforcement learning training example
+
+```bash
+pip install -r requirements.txt
+# For REINFORCE:
+python reinforce.py
+# For actor critic:
+python actor_critic.py
+```

reinforcement_learning/README.md

@@ -49,7 +49,7 @@ def forward(self, x):
 
 
 model = Policy()
-optimizer = optim.RMSprop(model.parameters(), lr=1e-3)
+optimizer = optim.Adam(model.parameters(), lr=1e-2)
 
 
 def select_action(state):
@@ -64,10 +64,15 @@ def finish_episode():
     R = 0
     saved_actions = model.saved_actions
     value_loss = 0
-    for (action, value), r in zip(saved_actions[::-1], model.rewards[::-1]):
+    rewards = []
+    for r in model.rewards[::-1]:
         R = r + args.gamma * R
-        action.reinforce(R - value.data.squeeze())
-        value_loss += F.smooth_l1_loss(value, Variable(torch.Tensor([R])))
+        rewards.insert(0, R)
+    rewards = torch.Tensor(rewards)
+    rewards = (rewards - rewards.mean()) / rewards.std()
+    for (action, value), r in zip(saved_actions, rewards):
+        action.reinforce(r)
+        value_loss += F.smooth_l1_loss(value, Variable(torch.Tensor([r])))
     optimizer.zero_grad()
     final_nodes = [value_loss] + list(map(lambda p: p.action, saved_actions))
     gradients = [torch.ones(1)] + [None] * len(saved_actions)
@@ -77,10 +82,10 @@ def finish_episode():
     del model.saved_actions[:]
 
 
-episode_durations = []
+running_reward = 10
 for i_episode in count(1):
     state = env.reset()
-    for t in count(1):
+    for t in range(10000): # Don't infinite loop while learning
         action = select_action(state)
         state, reward, done, _ = env.step(action[0,0])
         if args.render:
@@ -89,8 +94,12 @@ def finish_episode():
         if done:
             break
 
+    running_reward = running_reward * 0.99 + t * 0.01
     finish_episode()
-    episode_durations.append(t)
     if i_episode % args.log_interval == 0:
         print('Episode {}\tLast length: {:5d}\tAverage length: {:.2f}'.format(
-            i_episode, t, torch.Tensor(episode_durations[-100:]).mean()))
+            i_episode, t, running_reward))
+    if running_reward > 200:
+        print("Solved! Running reward is now {} and "
+              "the last episode runs to {} time steps!".format(running_reward, t))
+        break

actor_critic/main.py renamed to reinforcement_learning/actor_critic.py

@@ -0,0 +1,98 @@
+import argparse
+import gym
+import numpy as np
+from itertools import count
+from collections import namedtuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torch.autograd as autograd
+from torch.autograd import Variable
+import torchvision.transforms as T
+
+
+parser = argparse.ArgumentParser(description='PyTorch REINFORCE example')
+parser.add_argument('--gamma', type=int, default=0.99, metavar='G',
+                    help='discount factor (default: 0.99)')
+parser.add_argument('--seed', type=int, default=543, metavar='N',
+                    help='random seed (default: 1)')
+parser.add_argument('--render', action='store_true',
+                    help='render the environment')
+parser.add_argument('--log-interval', type=int, default=10, metavar='N',
+                    help='interval between training status logs (default: 10)')
+args = parser.parse_args()
+
+
+env = gym.make('CartPole-v0')
+env.seed(args.seed)
+torch.manual_seed(args.seed)
+
+
+class Policy(nn.Module):
+    def __init__(self):
+        super(Policy, self).__init__()
+        self.affine1 = nn.Linear(4, 128)
+        self.affine2 = nn.Linear(128, 2)
+
+        self.saved_actions = []
+        self.rewards = []
+
+    def forward(self, x):
+        x = F.relu(self.affine1(x))
+        action_scores = self.affine2(x)
+        return F.softmax(action_scores)
+
+
+model = Policy()
+optimizer = optim.Adam(model.parameters(), lr=1e-2)
+
+
+def select_action(state):
+    state = torch.from_numpy(state).float().unsqueeze(0)
+    probs = model(Variable(state))
+    action = probs.multinomial()
+    model.saved_actions.append(action)
+    return action.data
+
+
+def finish_episode():
+    R = 0
+    saved_actions = model.saved_actions
+    rewards = []
+    for r in model.rewards[::-1]:
+        R = r + args.gamma * R
+        rewards.insert(0, R)
+    rewards = torch.Tensor(rewards)
+    rewards = (rewards - rewards.mean()) / rewards.std()
+    for action, r in zip(model.saved_actions, rewards):
+        action.reinforce(r)
+    optimizer.zero_grad()
+    autograd.backward(model.saved_actions, [None for _ in model.saved_actions])
+    optimizer.step()
+    del model.rewards[:]
+    del model.saved_actions[:]
+
+
+running_reward = 10
+for i_episode in count(1):
+    state = env.reset()
+    for t in range(10000): # Don't infinite loop while learning
+        action = select_action(state)
+        state, reward, done, _ = env.step(action[0,0])
+        if args.render:
+            env.render()
+        model.rewards.append(reward)
+        if done:
+            break
+
+    running_reward = running_reward * 0.99 + t * 0.01
+    finish_episode()
+    if i_episode % args.log_interval == 0:
+        print('Episode {}\tLast length: {:5d}\tAverage length: {:.2f}'.format(
+            i_episode, t, running_reward))
+    if running_reward > 200:
+        print("Solved! Running reward is now {} and "
+              "the last episode runs to {} time steps!".format(running_reward, t))
+        break