Tennis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from shutil import copyfile\n",
    "from collections import namedtuple, deque\n",
    "import random\n",
    "import time\n",
    "import copy\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from unityagents import UnityEnvironment\n",
    "\n",
    "from IPython.display import display, clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Environment():\n",
    "    \n",
    "    def __init__(self,path):\n",
    "        self.env = UnityEnvironment(file_name=path);\n",
    "        self.brain_name = self.env.brain_names[0]\n",
    "        self.brain = self.env.brains[self.brain_name]\n",
    "        self.action_size = self.brain.vector_action_space_size\n",
    "        info = self.env.reset(train_mode=True)[self.brain_name]\n",
    "        self.state_size = len(info.vector_observations[0])\n",
    "        self.num_agents = len(info.agents)\n",
    "        self.print_info(info.vector_observations)\n",
    "        \n",
    "    def close(self):\n",
    "        self.env.close()\n",
    "    \n",
    "    def reset(self,train=True):\n",
    "        info = self.env.reset(train_mode=train)[self.brain_name]\n",
    "        return info.vector_observations\n",
    "    \n",
    "    def step(self,action):\n",
    "        info = self.env.step(action)[self.brain_name]\n",
    "        state = info.vector_observations\n",
    "        reward = info.rewards\n",
    "        done = info.local_done\n",
    "        return state, reward, done, info\n",
    "    \n",
    "    def print_info(self, state=None):\n",
    "        print('Num agents: \\t', self.num_agents)\n",
    "        print('State size: \\t', self.state_size)\n",
    "        print('Action size:\\t', self.action_size)\n",
    "        if state is not None:\n",
    "            print('Raw State shape:\\t', state.shape)\n",
    "            print('Raw State content:\\n', state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: TennisBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 8\n",
      "        Number of stacked Vector Observation: 3\n",
      "        Vector Action space type: continuous\n",
      "        Vector Action space size (per agent): 2\n",
      "        Vector Action descriptions: , \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num agents: \t 2\n",
      "State size: \t 24\n",
      "Action size:\t 2\n",
      "Raw State shape:\t (2, 24)\n",
      "Raw State content:\n",
      " [[ 0.          0.          0.          0.          0.          0.\n",
      "   0.          0.          0.          0.          0.          0.\n",
      "   0.          0.          0.          0.         -6.65278625 -1.5\n",
      "  -0.          0.          6.83172083  6.         -0.          0.        ]\n",
      " [ 0.          0.          0.          0.          0.          0.\n",
      "   0.          0.          0.          0.          0.          0.\n",
      "   0.          0.          0.          0.         -6.4669857  -1.5\n",
      "   0.          0.         -6.83172083  6.          0.          0.        ]]\n"
     ]
    }
   ],
   "source": [
    "env = Environment(\"Tennis.app\") \n",
    "# Place the Tennis.app file in the directory and pass \"Tennis.app\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hidden_init(layer):\n",
    "    fanin = layer.weight.data.size()[0]\n",
    "    limit = (1.0/ np.sqrt(fanin))\n",
    "    return (-limit, limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Actor(nn.Module):\n",
    "\n",
    "    def __init__(self, state_size, action_size, seed, fc1_units=256, fc2_units=128):\n",
    "        \"\"\" \n",
    "        PARAMETERS: \n",
    "            state_size (int): Dimension of a state\n",
    "            action_size (int): Dimension of an action\n",
    "            seed (int): Rand seed\n",
    "            fc1_units (int): No. of nodes in 1st hidden layer\n",
    "            fc2_units (int): No. of nodes in 2nd hidden layer\n",
    "        \"\"\"\n",
    "        super(Actor, self).__init__()\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.fc1 = nn.Linear(state_size, fc1_units)\n",
    "        self.fc2 = nn.Linear(fc1_units, fc2_units)\n",
    "        self.fc3 = nn.Linear(fc2_units, action_size)\n",
    "        self.reset_parameters()\n",
    "\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        self.fc1.weight.data.uniform_(*hidden_init(self.fc1))\n",
    "        self.fc2.weight.data.uniform_(*hidden_init(self.fc2))\n",
    "        self.fc3.weight.data.uniform_(-3e-3, 3e-3)\n",
    "\n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.fc1(state))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        return torch.tanh(self.fc3(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Critic(nn.Module):\n",
    "\n",
    "    def __init__(self, state_size, action_size, seed, fcs1_units=256, fc2_units=128):\n",
    "        \"\"\" \n",
    "        PARAMETERS: \n",
    "            state_size (int): Dimension of a state\n",
    "            action_size (int): Dimension of an action\n",
    "            seed (int): Rand seed\n",
    "            fcs1_units (int): No. of nodes in the first hidden layer\n",
    "            fc2_units (int): No. of nodes in the second hidden layer\n",
    "        \"\"\"\n",
    "        super(Critic, self).__init__()\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.fcs1 = nn.Linear(state_size, fcs1_units)\n",
    "        self.fc2 = nn.Linear(fcs1_units + action_size, fc2_units)\n",
    "        self.fc3 = nn.Linear(fc2_units, 1)\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        self.fcs1.weight.data.uniform_(*hidden_init(self.fcs1))\n",
    "        self.fc2.weight.data.uniform_(*hidden_init(self.fc2))\n",
    "        self.fc3.weight.data.uniform_(-3e-3, 3e-3)\n",
    "\n",
    "    def forward(self, state, action):\n",
    "        x = F.relu(self.fcs1(state))\n",
    "        x = torch.cat((x, action), dim=1)\n",
    "        x = F.relu(self.fc2(x))\n",
    "        return self.fc3(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "\n",
    "    def __init__(self, buffer_size, batch_size, seed):\n",
    "        \"\"\"\n",
    "        PARAMETERS:\n",
    "            buffer_size(int): Maximum buffer size\n",
    "            batch_size(int): Training batch size \n",
    "        \"\"\"\n",
    "        self.memory = deque(maxlen=buffer_size)\n",
    "        self.batch_size = batch_size\n",
    "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
    "        self.seed = random.seed(seed)\n",
    "    \n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        e = self.experience(state, action, reward, next_state, done)\n",
    "        self.memory.append(e)\n",
    "    \n",
    "    def sample(self):\n",
    "\n",
    "        exps = random.sample(self.memory, k=self.batch_size)\n",
    "\n",
    "        states = torch.from_numpy(np.vstack([e.state for e in exps if e is not None])).float().to(device)\n",
    "        actions = torch.from_numpy(np.vstack([e.action for e in exps if e is not None])).float().to(device)\n",
    "        rewards = torch.from_numpy(np.vstack([e.reward for e in exps if e is not None])).float().to(device)\n",
    "        next_states = torch.from_numpy(np.vstack([e.next_state for e in exps if e is not None])).float().to(device)\n",
    "        dones = torch.from_numpy(np.vstack([e.done for e in exps if e is not None]).astype(np.uint8)).float().to(device)\n",
    "\n",
    "        return (states, actions, rewards, next_states, dones)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBufferPrioritized:\n",
    "    def __init__(self, batch_size, buffer_size, seed, min_delta=1e-5):\n",
    "        self.seed = random.seed(seed)\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.min_delta = min_delta\n",
    "        \n",
    "        self.buffer = deque(maxlen=buffer_size)\n",
    "        self.deltas = deque(maxlen=buffer_size)\n",
    "        \n",
    "        self.exp_template = namedtuple(\"exp\", field_names=[\"state\",\"action\",\"reward\",\"new_state\",\"done\"])\n",
    "    \n",
    "    def add(self, state, action, reward, new_state, done):\n",
    "        exp = self.exp_template(state, action, reward, new_state, int(done))\n",
    "        self.buffer.append(exp)\n",
    "        self.deltas.append( max(self.deltas) if len(self.deltas) > 0 else self.min_delta )\n",
    "    \n",
    "    def sample(self,priority=0.5):\n",
    "        deltas = np.array(self.deltas)\n",
    "        probs = deltas**priority / np.sum(deltas**priority)\n",
    "        \n",
    "        exp_batch_idx = np.random.choice(np.arange(len(self.buffer)), size=self.batch_size, p=probs, replace=False)\n",
    "         \n",
    "        states = torch.from_numpy(np.vstack([self.buffer[idx].state for idx in exp_batch_idx])).float().to(device)\n",
    "        actions = torch.from_numpy(np.vstack([self.buffer[idx].action for idx in exp_batch_idx])).float().to(device)\n",
    "        rewards = torch.from_numpy(np.vstack([self.buffer[idx].reward for idx in exp_batch_idx])).float().to(device)\n",
    "        new_states = torch.from_numpy(np.vstack([self.buffer[idx].new_state for idx in exp_batch_idx])).float().to(device)\n",
    "        dones = torch.from_numpy(np.vstack([self.buffer[idx].done for idx in exp_batch_idx])).float().to(device)\n",
    "        probabilities = torch.from_numpy(np.vstack([probs[idx] for idx in exp_batch_idx])).float().to(device)\n",
    "        \n",
    "        return states, actions, rewards, new_states, dones, probabilities, exp_batch_idx\n",
    "    \n",
    "    def update_deltas(self,idxs,deltas):\n",
    "        for i,idx in enumerate(idxs):\n",
    "            self.deltas[idx] = deltas[i] + self.min_delta\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OUNoise: #Ornstein-Uhlenbeck process\n",
    "\n",
    "    def __init__(self, size, seed, mu=0., theta=0.15, sigma=0.2):\n",
    "        self.size = size\n",
    "        self.mu = mu * np.ones(size)\n",
    "        self.theta = theta\n",
    "        self.sigma = sigma\n",
    "        self.seed = random.seed(seed)\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.state = copy.copy(self.mu)\n",
    "\n",
    "    def sample(self):\n",
    "        x = self.state\n",
    "        rnd = np.random.standard_normal(self.size)\n",
    "\n",
    "        dx = self.theta * (self.mu - x) + self.sigma * rnd\n",
    "        self.state = x + dx\n",
    "        return self.state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent():\n",
    "    \n",
    "    def __init__(self, num_agents, state_size, action_size, buffer_size, batch_size, learn_every, update_every, random_seed, use_pre):\n",
    "        \"\"\"\n",
    "        PARAMETERS:\n",
    "            state_size (int): dimension of each state\n",
    "            action_size (int): dimension of each action\n",
    "            random_seed (int): random seed\n",
    "        \"\"\"\n",
    "        self.batch_size = batch_size\n",
    "        self.buffer_size = buffer_size\n",
    "        self.num_agents = num_agents\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.update_every = update_every\n",
    "        self.learn_every = learn_every\n",
    "        self.seed = random.seed(random_seed)\n",
    "        self.use_pre = use_pre\n",
    "        self.actor_local = Actor(state_size, action_size, random_seed).to(device)\n",
    "        self.actor_target = Actor(state_size, action_size, random_seed).to(device)\n",
    "        self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr = lr_actor)\n",
    "        self.critic_local = Critic(state_size, action_size, random_seed).to(device)\n",
    "        self.critic_target = Critic(state_size, action_size, random_seed).to(device)\n",
    "        self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr = lr_critic, weight_decay = weight_decay)\n",
    "        self.noise = OUNoise((num_agents,action_size), random_seed)\n",
    "        if self.use_pre:\n",
    "            self.memory = ReplayBufferPrioritized(self.buffer_size, self.batch_size, random_seed)\n",
    "        else:\n",
    "            self.memory = ReplayBuffer(self.buffer_size, self.batch_size, random_seed)\n",
    "        \n",
    "        print(self.actor_local)\n",
    "\n",
    "        print(self.critic_local)\n",
    "\n",
    "    def step(self, state, action, reward, next_state, done, priority=0.):\n",
    "        for i in range(self.num_agents):\n",
    "            self.memory.add(state[i,:], action[i,:], reward[i], next_state[i,:], done[i])\n",
    "        \n",
    "        self.i_step = self.i_step + 1            \n",
    "        to_learn = (self.i_step % self.learn_every ) == 0\n",
    "        to_update = ( self.i_step % self.update_every ) == 0\n",
    "        \n",
    "        if len(self.memory) > self.batch_size:\n",
    "            if to_learn:\n",
    "                exps = None\n",
    "                if self.use_pre:\n",
    "                    exps = self.memory.sample(priority)\n",
    "                else:\n",
    "                    exps = self.memory.sample()\n",
    "                self.learn(exps, gamma)\n",
    "        \n",
    "            if to_update:\n",
    "                self.soft_update(self.critic_local, self.critic_target, tau)\n",
    "                self.soft_update(self.actor_local, self.actor_target, tau) \n",
    "\n",
    "    def act(self, state, add_noise=True):\n",
    "        state = torch.from_numpy(state).float().to(device)\n",
    "        self.actor_local.eval()\n",
    "        with torch.no_grad():\n",
    "            action = self.actor_local(state).cpu().data.numpy()\n",
    "        self.actor_local.train()\n",
    "        \n",
    "        if add_noise:\n",
    "            action += self.noise.sample()\n",
    "        return np.clip(action, -1, 1)\n",
    "\n",
    "    def reset(self):\n",
    "        self.noise.reset()\n",
    "        self.i_step=0\n",
    "\n",
    "    def learn(self, exps, gamma):\n",
    "        if self.use_pre:\n",
    "            states, actions, rewards, next_states, dones, probs, batch_idx = exps\n",
    "        else:\n",
    "            states, actions, rewards, next_states, dones = exps\n",
    "            \n",
    "        actions_next = self.actor_target(next_states)\n",
    "        Qtargs_next = self.critic_target(next_states, actions_next)\n",
    "        Qtargs = rewards + (gamma * Qtargs_next * (1 - dones))\n",
    "        Q_expected = self.critic_local(states, actions)\n",
    "        critic_loss = F.mse_loss(Q_expected, Qtargs)\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        critic_loss.backward()\n",
    "        self.critic_optimizer.step()\n",
    "        actions_pred = self.actor_local(states)\n",
    "        actor_loss = -self.critic_local(states, actions_pred).mean()\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        actor_loss.backward()\n",
    "        self.actor_optimizer.step()    \n",
    "        \n",
    "        if self.use_pre:\n",
    "            Q_error = Q_expected - Qtargs\n",
    "            nd_ = torch.abs(td_error.detach().squeeze(1)).numpy()\n",
    "            self.memory.update_deltas(batch_idx,nd_)\n",
    "\n",
    "    def soft_update(self, local_model, target_model, tau):\n",
    "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
    "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actor(\n",
      "  (fc1): Linear(in_features=24, out_features=256, bias=True)\n",
      "  (fc2): Linear(in_features=256, out_features=128, bias=True)\n",
      "  (fc3): Linear(in_features=128, out_features=2, bias=True)\n",
      ")\n",
      "Critic(\n",
      "  (fcs1): Linear(in_features=24, out_features=256, bias=True)\n",
      "  (fc2): Linear(in_features=258, out_features=128, bias=True)\n",
      "  (fc3): Linear(in_features=128, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "use_pre = False\n",
    "PRE_PRIORITY_START = 0.5\n",
    "PRE_PRIORITY_END = 0.0\n",
    "PRE_PRIORITY_DECAY = 0.991\n",
    "\n",
    "batch_size = 128\n",
    "buffer_size = int(1e+5)\n",
    "gamma = 0.99\n",
    "tau = 1e-3\n",
    "weight_decay = 0\n",
    "lr_actor = 1e-4\n",
    "lr_critic = 1e-4\n",
    "update_step = 1\n",
    "learn_step  = 1\n",
    "rand_seed = 0\n",
    "\n",
    "num_episodes = 9000\n",
    "\n",
    "session = str(int(time.time()))\n",
    "\n",
    "agent_config = {\n",
    "    'num_agents':env.num_agents,\n",
    "    'state_size':env.state_size,\n",
    "    'action_size':env.action_size,\n",
    "    'buffer_size':buffer_size,\n",
    "    'batch_size':batch_size,\n",
    "    'learn_every':learn_step,\n",
    "    'update_every':update_step,\n",
    "    'random_seed':rand_seed,\n",
    "    'use_pre':use_pre\n",
    "    \n",
    "}\n",
    "\n",
    "agent = Agent(**agent_config)\n",
    "\n",
    "def DDPG(n_episodes=num_episodes):\n",
    "    priority = PRE_PRIORITY_START\n",
    "    print_every=100\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    max_score = 0\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        states = env.reset()\n",
    "        agent.reset()\n",
    "        score = np.zeros(env.num_agents)\n",
    "        priority = max(priority * PRE_PRIORITY_DECAY, PRE_PRIORITY_END)\n",
    "        while True:\n",
    "            actions = agent.act(states)\n",
    "            next_states, rewards, dones, _ = env.step(actions)\n",
    "            agent.step(states, actions, rewards, next_states, dones, priority)\n",
    "            score = score + rewards\n",
    "            states = next_states\n",
    "            if np.any(dones):\n",
    "                break\n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)\n",
    "        avgscore_ep = np.mean(score)\n",
    "        avgscore_100 = np.mean(scores_deque)\n",
    "        print('\\rEpisode {}/{}\\tScore: {:.2f} \\tAvgScore: {:.2f}'.format(i_episode, n_episodes, avgscore_ep, avgscore_100 ), end=\"\")\n",
    "        \n",
    "        if i_episode % print_every == 0:\n",
    "            print('')\n",
    "        \n",
    "        if avgscore_100 >= 0.5:\n",
    "            print(f\"\\n***Solved! Episodes taken: {i_episode:d} AvgScore: {avgscore_100:.3f}***\")\n",
    "            if avgscore_100 > max_score:\n",
    "                max_score = avgscore_100\n",
    "                #saving weights of networks\n",
    "                file_tpl = 'savedmodels/checkpoint_{}_{}_episode{}_score{:.3f}.pth'\n",
    "                file_tpl_best = 'savedmodels/checkpoint_{}_{}_best.pth'\n",
    "                fileactor = file_tpl.format(session,'actor', i_episode, avgscore_100)\n",
    "                filecritic = file_tpl.format(session,'critic', i_episode, avgscore_100)\n",
    "                torch.save(agent.actor_local.state_dict(), fileactor)\n",
    "                torch.save(agent.critic_local.state_dict(), filecritic)\n",
    "                copyfile(fileactor, file_tpl_best.format(session,'actor'))\n",
    "                copyfile(filecritic, file_tpl_best.format(session,'critic')) \n",
    "            \n",
    "                \n",
    "    return scores\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100/9000\tScore: -0.00 \tAvgScore: 0.01\n",
      "Episode 200/9000\tScore: -0.00 \tAvgScore: 0.01\n",
      "Episode 300/9000\tScore: -0.00 \tAvgScore: 0.02\n",
      "Episode 400/9000\tScore: -0.00 \tAvgScore: 0.01\n",
      "Episode 500/9000\tScore: -0.00 \tAvgScore: 0.01\n",
      "Episode 600/9000\tScore: -0.00 \tAvgScore: 0.01\n",
      "Episode 700/9000\tScore: 0.05 \tAvgScore: 0.011\n",
      "Episode 800/9000\tScore: -0.00 \tAvgScore: 0.02\n",
      "Episode 900/9000\tScore: 0.05 \tAvgScore: 0.022\n",
      "Episode 1000/9000\tScore: 0.05 \tAvgScore: 0.03\n",
      "Episode 1100/9000\tScore: -0.00 \tAvgScore: 0.03\n",
      "Episode 1200/9000\tScore: -0.00 \tAvgScore: 0.03\n",
      "Episode 1300/9000\tScore: 0.05 \tAvgScore: 0.044\n",
      "Episode 1400/9000\tScore: 0.05 \tAvgScore: 0.044\n",
      "Episode 1500/9000\tScore: 0.05 \tAvgScore: 0.065\n",
      "Episode 1600/9000\tScore: 0.05 \tAvgScore: 0.076\n",
      "Episode 1700/9000\tScore: 0.05 \tAvgScore: 0.100\n",
      "Episode 1800/9000\tScore: 0.20 \tAvgScore: 0.187\n",
      "Episode 1900/9000\tScore: 0.10 \tAvgScore: 0.200\n",
      "Episode 2000/9000\tScore: 0.15 \tAvgScore: 0.230\n",
      "Episode 2087/9000\tScore: 1.35 \tAvgScore: 0.507\n",
      "***Solved! Episodes taken: 2087 AvgScore: 0.503***\n",
      "Episode 2098/9000\tScore: 2.10 \tAvgScore: 0.519\n",
      "***Solved! Episodes taken: 2098 AvgScore: 0.512***\n",
      "Episode 2099/9000\tScore: 1.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2099 AvgScore: 0.519***\n",
      "Episode 2100/9000\tScore: 0.25 \tAvgScore: 0.52\n",
      "\n",
      "***Solved! Episodes taken: 2100 AvgScore: 0.520***\n",
      "Episode 2101/9000\tScore: 0.65 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2101 AvgScore: 0.526***\n",
      "Episode 2102/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2102 AvgScore: 0.525***\n",
      "Episode 2103/9000\tScore: 0.30 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2103 AvgScore: 0.527***\n",
      "Episode 2104/9000\tScore: 0.60 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2104 AvgScore: 0.532***\n",
      "Episode 2105/9000\tScore: 0.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2105 AvgScore: 0.523***\n",
      "Episode 2106/9000\tScore: 1.05 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2106 AvgScore: 0.532***\n",
      "Episode 2107/9000\tScore: 1.85 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2107 AvgScore: 0.545***\n",
      "Episode 2108/9000\tScore: 1.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2108 AvgScore: 0.556***\n",
      "Episode 2109/9000\tScore: 2.20 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2109 AvgScore: 0.574***\n",
      "Episode 2110/9000\tScore: 0.25 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2110 AvgScore: 0.575***\n",
      "Episode 2111/9000\tScore: 0.40 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2111 AvgScore: 0.578***\n",
      "Episode 2112/9000\tScore: 0.50 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2112 AvgScore: 0.580***\n",
      "Episode 2113/9000\tScore: 2.60 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2113 AvgScore: 0.605***\n",
      "Episode 2114/9000\tScore: 0.20 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2114 AvgScore: 0.594***\n",
      "Episode 2115/9000\tScore: -0.00 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2115 AvgScore: 0.592***\n",
      "Episode 2116/9000\tScore: 2.05 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2116 AvgScore: 0.605***\n",
      "Episode 2117/9000\tScore: -0.00 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 2117 AvgScore: 0.597***\n",
      "Episode 2118/9000\tScore: 0.25 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2118 AvgScore: 0.592***\n",
      "Episode 2119/9000\tScore: 0.30 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2119 AvgScore: 0.588***\n",
      "Episode 2120/9000\tScore: 0.20 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2120 AvgScore: 0.585***\n",
      "Episode 2121/9000\tScore: 0.50 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2121 AvgScore: 0.585***\n",
      "Episode 2122/9000\tScore: 0.05 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2122 AvgScore: 0.585***\n",
      "Episode 2123/9000\tScore: 0.10 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2123 AvgScore: 0.582***\n",
      "Episode 2124/9000\tScore: 0.10 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2124 AvgScore: 0.582***\n",
      "Episode 2125/9000\tScore: 0.05 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2125 AvgScore: 0.581***\n",
      "Episode 2126/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2126 AvgScore: 0.571***\n",
      "Episode 2127/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2127 AvgScore: 0.570***\n",
      "Episode 2128/9000\tScore: 0.95 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2128 AvgScore: 0.575***\n",
      "Episode 2129/9000\tScore: 0.45 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2129 AvgScore: 0.579***\n",
      "Episode 2130/9000\tScore: 0.15 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2130 AvgScore: 0.580***\n",
      "Episode 2131/9000\tScore: 1.45 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2131 AvgScore: 0.594***\n",
      "Episode 2132/9000\tScore: 0.75 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 2132 AvgScore: 0.597***\n",
      "Episode 2133/9000\tScore: 1.65 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2133 AvgScore: 0.607***\n",
      "Episode 2134/9000\tScore: 0.10 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2134 AvgScore: 0.606***\n",
      "Episode 2135/9000\tScore: 1.40 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2135 AvgScore: 0.614***\n",
      "Episode 2136/9000\tScore: 1.25 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 2136 AvgScore: 0.619***\n",
      "Episode 2137/9000\tScore: 0.80 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 2137 AvgScore: 0.620***\n",
      "Episode 2138/9000\tScore: 0.80 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2138 AvgScore: 0.625***\n",
      "Episode 2139/9000\tScore: 0.25 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2139 AvgScore: 0.627***\n",
      "Episode 2140/9000\tScore: 0.10 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2140 AvgScore: 0.628***\n",
      "Episode 2141/9000\tScore: 0.10 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2141 AvgScore: 0.627***\n",
      "Episode 2142/9000\tScore: 0.20 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2142 AvgScore: 0.627***\n",
      "Episode 2143/9000\tScore: 1.20 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2143 AvgScore: 0.634***\n",
      "Episode 2144/9000\tScore: 0.10 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2144 AvgScore: 0.633***\n",
      "Episode 2145/9000\tScore: -0.00 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2145 AvgScore: 0.633***\n",
      "Episode 2146/9000\tScore: 0.15 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2146 AvgScore: 0.626***\n",
      "Episode 2147/9000\tScore: 0.50 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2147 AvgScore: 0.630***\n",
      "Episode 2148/9000\tScore: 1.05 \tAvgScore: 0.64\n",
      "***Solved! Episodes taken: 2148 AvgScore: 0.641***\n",
      "Episode 2149/9000\tScore: 0.05 \tAvgScore: 0.64\n",
      "***Solved! Episodes taken: 2149 AvgScore: 0.641***\n",
      "Episode 2150/9000\tScore: 0.95 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2150 AvgScore: 0.633***\n",
      "Episode 2151/9000\tScore: 0.05 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2151 AvgScore: 0.631***\n",
      "Episode 2152/9000\tScore: 0.05 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2152 AvgScore: 0.612***\n",
      "Episode 2153/9000\tScore: 1.55 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 2153 AvgScore: 0.623***\n",
      "Episode 2154/9000\tScore: 0.65 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2154 AvgScore: 0.627***\n",
      "Episode 2155/9000\tScore: 0.05 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2155 AvgScore: 0.626***\n",
      "Episode 2156/9000\tScore: -0.00 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 2156 AvgScore: 0.618***\n",
      "Episode 2157/9000\tScore: 0.05 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2157 AvgScore: 0.611***\n",
      "Episode 2158/9000\tScore: 0.10 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 2158 AvgScore: 0.598***\n",
      "Episode 2159/9000\tScore: 0.10 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2159 AvgScore: 0.595***\n",
      "Episode 2160/9000\tScore: 0.05 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2160 AvgScore: 0.574***\n",
      "Episode 2161/9000\tScore: 0.60 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2161 AvgScore: 0.577***\n",
      "Episode 2162/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2162 AvgScore: 0.575***\n",
      "Episode 2163/9000\tScore: 0.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2163 AvgScore: 0.559***\n",
      "Episode 2164/9000\tScore: 0.65 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2164 AvgScore: 0.564***\n",
      "Episode 2165/9000\tScore: 0.70 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2165 AvgScore: 0.562***\n",
      "Episode 2166/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2166 AvgScore: 0.560***\n",
      "Episode 2167/9000\tScore: 2.25 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2167 AvgScore: 0.581***\n",
      "Episode 2168/9000\tScore: 0.05 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2168 AvgScore: 0.567***\n",
      "Episode 2169/9000\tScore: 0.10 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2169 AvgScore: 0.568***\n",
      "Episode 2170/9000\tScore: 1.70 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2170 AvgScore: 0.574***\n",
      "Episode 2171/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2171 AvgScore: 0.573***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 2172/9000\tScore: 0.40 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2172 AvgScore: 0.571***\n",
      "Episode 2173/9000\tScore: 1.55 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2173 AvgScore: 0.585***\n",
      "Episode 2174/9000\tScore: 0.20 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2174 AvgScore: 0.573***\n",
      "Episode 2175/9000\tScore: 0.54 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2175 AvgScore: 0.574***\n",
      "Episode 2176/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2176 AvgScore: 0.560***\n",
      "Episode 2177/9000\tScore: 0.05 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2177 AvgScore: 0.547***\n",
      "Episode 2178/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2178 AvgScore: 0.530***\n",
      "Episode 2179/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2179 AvgScore: 0.526***\n",
      "Episode 2180/9000\tScore: 1.05 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2180 AvgScore: 0.534***\n",
      "Episode 2181/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2181 AvgScore: 0.531***\n",
      "Episode 2182/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2182 AvgScore: 0.531***\n",
      "Episode 2183/9000\tScore: 0.90 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2183 AvgScore: 0.538***\n",
      "Episode 2184/9000\tScore: 0.25 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2184 AvgScore: 0.534***\n",
      "Episode 2185/9000\tScore: 0.70 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2185 AvgScore: 0.523***\n",
      "Episode 2186/9000\tScore: 1.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2186 AvgScore: 0.530***\n",
      "Episode 2187/9000\tScore: 1.65 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2187 AvgScore: 0.533***\n",
      "Episode 2188/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2188 AvgScore: 0.533***\n",
      "Episode 2189/9000\tScore: 0.30 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2189 AvgScore: 0.532***\n",
      "Episode 2190/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2190 AvgScore: 0.530***\n",
      "Episode 2191/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2191 AvgScore: 0.526***\n",
      "Episode 2192/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2192 AvgScore: 0.518***\n",
      "Episode 2193/9000\tScore: 0.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2193 AvgScore: 0.516***\n",
      "Episode 2194/9000\tScore: 0.40 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2194 AvgScore: 0.519***\n",
      "Episode 2195/9000\tScore: 1.70 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2195 AvgScore: 0.536***\n",
      "Episode 2196/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2196 AvgScore: 0.529***\n",
      "Episode 2197/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2197 AvgScore: 0.527***\n",
      "Episode 2198/9000\tScore: 0.15 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2198 AvgScore: 0.507***\n",
      "Episode 2200/9000\tScore: 0.65 \tAvgScore: 0.500\n",
      "\n",
      "***Solved! Episodes taken: 2200 AvgScore: 0.501***\n",
      "Episode 2203/9000\tScore: 1.20 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 2203 AvgScore: 0.504***\n",
      "Episode 2205/9000\tScore: 1.25 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2205 AvgScore: 0.512***\n",
      "Episode 2206/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 2206 AvgScore: 0.501***\n",
      "Episode 2300/9000\tScore: 0.05 \tAvgScore: 0.432\n",
      "Episode 2400/9000\tScore: -0.00 \tAvgScore: 0.44\n",
      "Episode 2417/9000\tScore: 1.00 \tAvgScore: 0.508\n",
      "***Solved! Episodes taken: 2417 AvgScore: 0.501***\n",
      "Episode 2418/9000\tScore: 2.60 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2418 AvgScore: 0.523***\n",
      "Episode 2419/9000\tScore: 1.15 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2419 AvgScore: 0.531***\n",
      "Episode 2420/9000\tScore: 0.90 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2420 AvgScore: 0.535***\n",
      "Episode 2421/9000\tScore: 1.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2421 AvgScore: 0.542***\n",
      "Episode 2422/9000\tScore: 0.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2422 AvgScore: 0.542***\n",
      "Episode 2423/9000\tScore: 0.75 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2423 AvgScore: 0.539***\n",
      "Episode 2424/9000\tScore: 0.75 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2424 AvgScore: 0.540***\n",
      "Episode 2425/9000\tScore: 0.15 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2425 AvgScore: 0.541***\n",
      "Episode 2426/9000\tScore: 1.85 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2426 AvgScore: 0.553***\n",
      "Episode 2427/9000\tScore: 1.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2427 AvgScore: 0.562***\n",
      "Episode 2428/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2428 AvgScore: 0.562***\n",
      "Episode 2429/9000\tScore: 0.20 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2429 AvgScore: 0.564***\n",
      "Episode 2430/9000\tScore: 0.15 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2430 AvgScore: 0.565***\n",
      "Episode 2431/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2431 AvgScore: 0.564***\n",
      "Episode 2432/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2432 AvgScore: 0.561***\n",
      "Episode 2433/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2433 AvgScore: 0.560***\n",
      "Episode 2434/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2434 AvgScore: 0.558***\n",
      "Episode 2435/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2435 AvgScore: 0.557***\n",
      "Episode 2436/9000\tScore: 0.35 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2436 AvgScore: 0.559***\n",
      "Episode 2437/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2437 AvgScore: 0.551***\n",
      "Episode 2438/9000\tScore: 1.55 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2438 AvgScore: 0.559***\n",
      "Episode 2439/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2439 AvgScore: 0.559***\n",
      "Episode 2440/9000\tScore: 0.15 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2440 AvgScore: 0.553***\n",
      "Episode 2441/9000\tScore: 0.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2441 AvgScore: 0.554***\n",
      "Episode 2442/9000\tScore: 0.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2442 AvgScore: 0.555***\n",
      "Episode 2443/9000\tScore: 0.05 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2443 AvgScore: 0.555***\n",
      "Episode 2444/9000\tScore: 1.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2444 AvgScore: 0.563***\n",
      "Episode 2445/9000\tScore: 1.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2445 AvgScore: 0.567***\n",
      "Episode 2446/9000\tScore: 2.55 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2446 AvgScore: 0.592***\n",
      "Episode 2447/9000\tScore: 0.85 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 2447 AvgScore: 0.600***\n",
      "Episode 2448/9000\tScore: 2.60 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2448 AvgScore: 0.626***\n",
      "Episode 2449/9000\tScore: 2.60 \tAvgScore: 0.65\n",
      "***Solved! Episodes taken: 2449 AvgScore: 0.652***\n",
      "Episode 2450/9000\tScore: 0.25 \tAvgScore: 0.65\n",
      "***Solved! Episodes taken: 2450 AvgScore: 0.651***\n",
      "Episode 2451/9000\tScore: 2.60 \tAvgScore: 0.68\n",
      "***Solved! Episodes taken: 2451 AvgScore: 0.676***\n",
      "Episode 2452/9000\tScore: 2.65 \tAvgScore: 0.70\n",
      "***Solved! Episodes taken: 2452 AvgScore: 0.703***\n",
      "Episode 2453/9000\tScore: 2.65 \tAvgScore: 0.73\n",
      "***Solved! Episodes taken: 2453 AvgScore: 0.729***\n",
      "Episode 2454/9000\tScore: 1.10 \tAvgScore: 0.74\n",
      "***Solved! Episodes taken: 2454 AvgScore: 0.740***\n",
      "Episode 2455/9000\tScore: 1.45 \tAvgScore: 0.75\n",
      "***Solved! Episodes taken: 2455 AvgScore: 0.749***\n",
      "Episode 2456/9000\tScore: 1.80 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2456 AvgScore: 0.767***\n",
      "Episode 2457/9000\tScore: 1.05 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2457 AvgScore: 0.773***\n",
      "Episode 2458/9000\tScore: 1.30 \tAvgScore: 0.79\n",
      "***Solved! Episodes taken: 2458 AvgScore: 0.786***\n",
      "Episode 2459/9000\tScore: 0.30 \tAvgScore: 0.78\n",
      "***Solved! Episodes taken: 2459 AvgScore: 0.778***\n",
      "Episode 2460/9000\tScore: 0.80 \tAvgScore: 0.76\n",
      "***Solved! Episodes taken: 2460 AvgScore: 0.761***\n",
      "Episode 2461/9000\tScore: 0.65 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2461 AvgScore: 0.767***\n",
      "Episode 2462/9000\tScore: 0.90 \tAvgScore: 0.78\n",
      "***Solved! Episodes taken: 2462 AvgScore: 0.775***\n",
      "Episode 2463/9000\tScore: 0.05 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2463 AvgScore: 0.769***\n",
      "Episode 2464/9000\tScore: 0.20 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2464 AvgScore: 0.771***\n",
      "Episode 2465/9000\tScore: 2.60 \tAvgScore: 0.78\n",
      "***Solved! Episodes taken: 2465 AvgScore: 0.783***\n",
      "Episode 2466/9000\tScore: 0.85 \tAvgScore: 0.79\n",
      "***Solved! Episodes taken: 2466 AvgScore: 0.790***\n",
      "Episode 2467/9000\tScore: 1.00 \tAvgScore: 0.78\n",
      "***Solved! Episodes taken: 2467 AvgScore: 0.783***\n",
      "Episode 2468/9000\tScore: 0.50 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2468 AvgScore: 0.775***\n",
      "Episode 2469/9000\tScore: 0.10 \tAvgScore: 0.76\n",
      "***Solved! Episodes taken: 2469 AvgScore: 0.755***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 2470/9000\tScore: 1.30 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2470 AvgScore: 0.768***\n",
      "Episode 2471/9000\tScore: 0.60 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2471 AvgScore: 0.765***\n",
      "Episode 2472/9000\tScore: 0.20 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2472 AvgScore: 0.766***\n",
      "Episode 2473/9000\tScore: 1.35 \tAvgScore: 0.77\n",
      "***Solved! Episodes taken: 2473 AvgScore: 0.774***\n",
      "Episode 2474/9000\tScore: 2.55 \tAvgScore: 0.80\n",
      "***Solved! Episodes taken: 2474 AvgScore: 0.799***\n",
      "Episode 2475/9000\tScore: 1.15 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2475 AvgScore: 0.809***\n",
      "Episode 2476/9000\tScore: 0.30 \tAvgScore: 0.80\n",
      "***Solved! Episodes taken: 2476 AvgScore: 0.804***\n",
      "Episode 2477/9000\tScore: 1.95 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2477 AvgScore: 0.822***\n",
      "Episode 2478/9000\tScore: 1.85 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2478 AvgScore: 0.838***\n",
      "Episode 2479/9000\tScore: 0.75 \tAvgScore: 0.83\n",
      "***Solved! Episodes taken: 2479 AvgScore: 0.833***\n",
      "Episode 2480/9000\tScore: 1.60 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2480 AvgScore: 0.849***\n",
      "Episode 2481/9000\tScore: 0.10 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2481 AvgScore: 0.850***\n",
      "Episode 2482/9000\tScore: -0.00 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2482 AvgScore: 0.840***\n",
      "Episode 2483/9000\tScore: 0.20 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2483 AvgScore: 0.841***\n",
      "Episode 2484/9000\tScore: 1.75 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2484 AvgScore: 0.851***\n",
      "Episode 2485/9000\tScore: 0.75 \tAvgScore: 0.86\n",
      "***Solved! Episodes taken: 2485 AvgScore: 0.856***\n",
      "Episode 2486/9000\tScore: 0.85 \tAvgScore: 0.86\n",
      "***Solved! Episodes taken: 2486 AvgScore: 0.858***\n",
      "Episode 2487/9000\tScore: -0.00 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2487 AvgScore: 0.846***\n",
      "Episode 2488/9000\tScore: 0.10 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2488 AvgScore: 0.836***\n",
      "Episode 2489/9000\tScore: 1.50 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2489 AvgScore: 0.851***\n",
      "Episode 2490/9000\tScore: -0.00 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2490 AvgScore: 0.846***\n",
      "Episode 2491/9000\tScore: -0.00 \tAvgScore: 0.83\n",
      "***Solved! Episodes taken: 2491 AvgScore: 0.827***\n",
      "Episode 2492/9000\tScore: 0.35 \tAvgScore: 0.83\n",
      "***Solved! Episodes taken: 2492 AvgScore: 0.831***\n",
      "Episode 2493/9000\tScore: 0.15 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2493 AvgScore: 0.813***\n",
      "Episode 2494/9000\tScore: 0.45 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2494 AvgScore: 0.816***\n",
      "Episode 2495/9000\tScore: 0.60 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2495 AvgScore: 0.820***\n",
      "Episode 2496/9000\tScore: 2.55 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2496 AvgScore: 0.845***\n",
      "Episode 2497/9000\tScore: 0.45 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2497 AvgScore: 0.844***\n",
      "Episode 2498/9000\tScore: 1.20 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2498 AvgScore: 0.842***\n",
      "Episode 2499/9000\tScore: -0.00 \tAvgScore: 0.83\n",
      "***Solved! Episodes taken: 2499 AvgScore: 0.829***\n",
      "Episode 2500/9000\tScore: 0.50 \tAvgScore: 0.83\n",
      "\n",
      "***Solved! Episodes taken: 2500 AvgScore: 0.834***\n",
      "Episode 2501/9000\tScore: 1.05 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2501 AvgScore: 0.845***\n",
      "Episode 2502/9000\tScore: 0.35 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2502 AvgScore: 0.836***\n",
      "Episode 2503/9000\tScore: 0.10 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2503 AvgScore: 0.836***\n",
      "Episode 2504/9000\tScore: 0.70 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2504 AvgScore: 0.842***\n",
      "Episode 2505/9000\tScore: 0.60 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2505 AvgScore: 0.841***\n",
      "Episode 2506/9000\tScore: -0.00 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2506 AvgScore: 0.839***\n",
      "Episode 2507/9000\tScore: 0.10 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2507 AvgScore: 0.819***\n",
      "Episode 2508/9000\tScore: 2.35 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2508 AvgScore: 0.840***\n",
      "Episode 2509/9000\tScore: 2.25 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2509 AvgScore: 0.855***\n",
      "Episode 2510/9000\tScore: 1.35 \tAvgScore: 0.86\n",
      "***Solved! Episodes taken: 2510 AvgScore: 0.864***\n",
      "Episode 2511/9000\tScore: 1.00 \tAvgScore: 0.86\n",
      "***Solved! Episodes taken: 2511 AvgScore: 0.863***\n",
      "Episode 2512/9000\tScore: 1.65 \tAvgScore: 0.88\n",
      "***Solved! Episodes taken: 2512 AvgScore: 0.878***\n",
      "Episode 2513/9000\tScore: 0.10 \tAvgScore: 0.88\n",
      "***Solved! Episodes taken: 2513 AvgScore: 0.879***\n",
      "Episode 2514/9000\tScore: 0.55 \tAvgScore: 0.86\n",
      "***Solved! Episodes taken: 2514 AvgScore: 0.864***\n",
      "Episode 2515/9000\tScore: 0.50 \tAvgScore: 0.86\n",
      "***Solved! Episodes taken: 2515 AvgScore: 0.858***\n",
      "Episode 2516/9000\tScore: 0.15 \tAvgScore: 0.86\n",
      "***Solved! Episodes taken: 2516 AvgScore: 0.855***\n",
      "Episode 2517/9000\tScore: 0.35 \tAvgScore: 0.85\n",
      "***Solved! Episodes taken: 2517 AvgScore: 0.849***\n",
      "Episode 2518/9000\tScore: 0.10 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2518 AvgScore: 0.824***\n",
      "Episode 2519/9000\tScore: 1.05 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2519 AvgScore: 0.823***\n",
      "Episode 2520/9000\tScore: 0.40 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2520 AvgScore: 0.818***\n",
      "Episode 2521/9000\tScore: 0.60 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2521 AvgScore: 0.814***\n",
      "Episode 2522/9000\tScore: -0.00 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2522 AvgScore: 0.813***\n",
      "Episode 2523/9000\tScore: 0.10 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2523 AvgScore: 0.806***\n",
      "Episode 2524/9000\tScore: 0.10 \tAvgScore: 0.80\n",
      "***Solved! Episodes taken: 2524 AvgScore: 0.800***\n",
      "Episode 2525/9000\tScore: 0.35 \tAvgScore: 0.80\n",
      "***Solved! Episodes taken: 2525 AvgScore: 0.802***\n",
      "Episode 2526/9000\tScore: 0.55 \tAvgScore: 0.79\n",
      "***Solved! Episodes taken: 2526 AvgScore: 0.789***\n",
      "Episode 2527/9000\tScore: 0.30 \tAvgScore: 0.78\n",
      "***Solved! Episodes taken: 2527 AvgScore: 0.780***\n",
      "Episode 2528/9000\tScore: 0.94 \tAvgScore: 0.79\n",
      "***Solved! Episodes taken: 2528 AvgScore: 0.790***\n",
      "Episode 2529/9000\tScore: 1.65 \tAvgScore: 0.80\n",
      "***Solved! Episodes taken: 2529 AvgScore: 0.804***\n",
      "Episode 2530/9000\tScore: 0.55 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2530 AvgScore: 0.808***\n",
      "Episode 2531/9000\tScore: 0.30 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2531 AvgScore: 0.810***\n",
      "Episode 2532/9000\tScore: 0.60 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2532 AvgScore: 0.815***\n",
      "Episode 2533/9000\tScore: 0.05 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2533 AvgScore: 0.816***\n",
      "Episode 2534/9000\tScore: 0.05 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2534 AvgScore: 0.815***\n",
      "Episode 2535/9000\tScore: 0.30 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2535 AvgScore: 0.818***\n",
      "Episode 2536/9000\tScore: 0.10 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2536 AvgScore: 0.816***\n",
      "Episode 2537/9000\tScore: 0.60 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2537 AvgScore: 0.822***\n",
      "Episode 2538/9000\tScore: 0.05 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2538 AvgScore: 0.807***\n",
      "Episode 2539/9000\tScore: 1.70 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2539 AvgScore: 0.823***\n",
      "Episode 2540/9000\tScore: 0.10 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2540 AvgScore: 0.822***\n",
      "Episode 2541/9000\tScore: 0.10 \tAvgScore: 0.82\n",
      "***Solved! Episodes taken: 2541 AvgScore: 0.821***\n",
      "Episode 2542/9000\tScore: 1.65 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2542 AvgScore: 0.836***\n",
      "Episode 2543/9000\tScore: 0.35 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2543 AvgScore: 0.839***\n",
      "Episode 2544/9000\tScore: 1.25 \tAvgScore: 0.84\n",
      "***Solved! Episodes taken: 2544 AvgScore: 0.841***\n",
      "Episode 2545/9000\tScore: 0.10 \tAvgScore: 0.83\n",
      "***Solved! Episodes taken: 2545 AvgScore: 0.832***\n",
      "Episode 2546/9000\tScore: 0.45 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2546 AvgScore: 0.811***\n",
      "Episode 2547/9000\tScore: 0.45 \tAvgScore: 0.81\n",
      "***Solved! Episodes taken: 2547 AvgScore: 0.807***\n",
      "Episode 2548/9000\tScore: 0.40 \tAvgScore: 0.78\n",
      "***Solved! Episodes taken: 2548 AvgScore: 0.785***\n",
      "Episode 2549/9000\tScore: 0.05 \tAvgScore: 0.76\n",
      "***Solved! Episodes taken: 2549 AvgScore: 0.759***\n",
      "Episode 2550/9000\tScore: 0.20 \tAvgScore: 0.76\n",
      "***Solved! Episodes taken: 2550 AvgScore: 0.759***\n",
      "Episode 2551/9000\tScore: 0.30 \tAvgScore: 0.74\n",
      "***Solved! Episodes taken: 2551 AvgScore: 0.736***\n",
      "Episode 2552/9000\tScore: 1.25 \tAvgScore: 0.72\n",
      "***Solved! Episodes taken: 2552 AvgScore: 0.722***\n",
      "Episode 2553/9000\tScore: 0.10 \tAvgScore: 0.70\n",
      "***Solved! Episodes taken: 2553 AvgScore: 0.696***\n",
      "Episode 2554/9000\tScore: 0.50 \tAvgScore: 0.69\n",
      "***Solved! Episodes taken: 2554 AvgScore: 0.690***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 2555/9000\tScore: 0.15 \tAvgScore: 0.68\n",
      "***Solved! Episodes taken: 2555 AvgScore: 0.677***\n",
      "Episode 2556/9000\tScore: 0.70 \tAvgScore: 0.67\n",
      "***Solved! Episodes taken: 2556 AvgScore: 0.666***\n",
      "Episode 2557/9000\tScore: 0.75 \tAvgScore: 0.66\n",
      "***Solved! Episodes taken: 2557 AvgScore: 0.663***\n",
      "Episode 2558/9000\tScore: 1.05 \tAvgScore: 0.66\n",
      "***Solved! Episodes taken: 2558 AvgScore: 0.661***\n",
      "Episode 2559/9000\tScore: 0.50 \tAvgScore: 0.66\n",
      "***Solved! Episodes taken: 2559 AvgScore: 0.663***\n",
      "Episode 2560/9000\tScore: 1.05 \tAvgScore: 0.67\n",
      "***Solved! Episodes taken: 2560 AvgScore: 0.665***\n",
      "Episode 2561/9000\tScore: 0.15 \tAvgScore: 0.66\n",
      "***Solved! Episodes taken: 2561 AvgScore: 0.660***\n",
      "Episode 2562/9000\tScore: 0.45 \tAvgScore: 0.66\n",
      "***Solved! Episodes taken: 2562 AvgScore: 0.656***\n",
      "Episode 2563/9000\tScore: 0.15 \tAvgScore: 0.66\n",
      "***Solved! Episodes taken: 2563 AvgScore: 0.657***\n",
      "Episode 2564/9000\tScore: 0.55 \tAvgScore: 0.66\n",
      "***Solved! Episodes taken: 2564 AvgScore: 0.660***\n",
      "Episode 2565/9000\tScore: 0.25 \tAvgScore: 0.64\n",
      "***Solved! Episodes taken: 2565 AvgScore: 0.637***\n",
      "Episode 2566/9000\tScore: 0.70 \tAvgScore: 0.64\n",
      "***Solved! Episodes taken: 2566 AvgScore: 0.635***\n",
      "Episode 2567/9000\tScore: 0.50 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2567 AvgScore: 0.630***\n",
      "Episode 2568/9000\tScore: 0.80 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2568 AvgScore: 0.633***\n",
      "Episode 2569/9000\tScore: 0.30 \tAvgScore: 0.64\n",
      "***Solved! Episodes taken: 2569 AvgScore: 0.635***\n",
      "Episode 2570/9000\tScore: 0.10 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 2570 AvgScore: 0.623***\n",
      "Episode 2571/9000\tScore: 0.35 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 2571 AvgScore: 0.621***\n",
      "Episode 2572/9000\tScore: 0.70 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 2572 AvgScore: 0.626***\n",
      "Episode 2573/9000\tScore: 0.05 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 2573 AvgScore: 0.613***\n",
      "Episode 2574/9000\tScore: 0.10 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 2574 AvgScore: 0.588***\n",
      "Episode 2575/9000\tScore: 0.75 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2575 AvgScore: 0.584***\n",
      "Episode 2576/9000\tScore: 0.05 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2576 AvgScore: 0.581***\n",
      "Episode 2577/9000\tScore: 0.60 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2577 AvgScore: 0.568***\n",
      "Episode 2578/9000\tScore: 0.90 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2578 AvgScore: 0.558***\n",
      "Episode 2579/9000\tScore: 0.90 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2579 AvgScore: 0.560***\n",
      "Episode 2580/9000\tScore: 0.60 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2580 AvgScore: 0.550***\n",
      "Episode 2581/9000\tScore: 2.55 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2581 AvgScore: 0.575***\n",
      "Episode 2582/9000\tScore: 0.80 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2582 AvgScore: 0.583***\n",
      "Episode 2583/9000\tScore: 0.35 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 2583 AvgScore: 0.584***\n",
      "Episode 2584/9000\tScore: 0.10 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2584 AvgScore: 0.568***\n",
      "Episode 2585/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2585 AvgScore: 0.561***\n",
      "Episode 2586/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2586 AvgScore: 0.553***\n",
      "Episode 2587/9000\tScore: 0.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2587 AvgScore: 0.554***\n",
      "Episode 2588/9000\tScore: 0.65 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2588 AvgScore: 0.559***\n",
      "Episode 2589/9000\tScore: 0.40 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2589 AvgScore: 0.548***\n",
      "Episode 2590/9000\tScore: 0.05 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2590 AvgScore: 0.549***\n",
      "Episode 2591/9000\tScore: 0.15 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2591 AvgScore: 0.550***\n",
      "Episode 2592/9000\tScore: 0.35 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2592 AvgScore: 0.550***\n",
      "Episode 2593/9000\tScore: 0.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2593 AvgScore: 0.551***\n",
      "Episode 2594/9000\tScore: 0.95 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 2594 AvgScore: 0.556***\n",
      "Episode 2595/9000\tScore: 0.30 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2595 AvgScore: 0.553***\n",
      "Episode 2596/9000\tScore: 1.30 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2596 AvgScore: 0.540***\n",
      "Episode 2597/9000\tScore: 0.90 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2597 AvgScore: 0.545***\n",
      "Episode 2598/9000\tScore: 1.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2598 AvgScore: 0.544***\n",
      "Episode 2599/9000\tScore: 0.75 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2599 AvgScore: 0.551***\n",
      "Episode 2600/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "\n",
      "***Solved! Episodes taken: 2600 AvgScore: 0.546***\n",
      "Episode 2601/9000\tScore: 1.55 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2601 AvgScore: 0.551***\n",
      "Episode 2602/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2602 AvgScore: 0.548***\n",
      "Episode 2603/9000\tScore: 0.05 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2603 AvgScore: 0.547***\n",
      "Episode 2604/9000\tScore: 1.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2604 AvgScore: 0.552***\n",
      "Episode 2605/9000\tScore: 1.95 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2605 AvgScore: 0.566***\n",
      "Episode 2606/9000\tScore: 0.25 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2606 AvgScore: 0.568***\n",
      "Episode 2607/9000\tScore: 0.50 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 2607 AvgScore: 0.572***\n",
      "Episode 2608/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 2608 AvgScore: 0.549***\n",
      "Episode 2609/9000\tScore: 1.50 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2609 AvgScore: 0.541***\n",
      "Episode 2610/9000\tScore: 1.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2610 AvgScore: 0.538***\n",
      "Episode 2611/9000\tScore: 1.30 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 2611 AvgScore: 0.541***\n",
      "Episode 2612/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2612 AvgScore: 0.525***\n",
      "Episode 2613/9000\tScore: 0.90 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2613 AvgScore: 0.533***\n",
      "Episode 2614/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2614 AvgScore: 0.528***\n",
      "Episode 2615/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2615 AvgScore: 0.524***\n",
      "Episode 2616/9000\tScore: 0.15 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2616 AvgScore: 0.524***\n",
      "Episode 2617/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2617 AvgScore: 0.521***\n",
      "Episode 2618/9000\tScore: 0.45 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2618 AvgScore: 0.525***\n",
      "Episode 2619/9000\tScore: 0.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2619 AvgScore: 0.517***\n",
      "Episode 2620/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2620 AvgScore: 0.514***\n",
      "Episode 2621/9000\tScore: 0.60 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2621 AvgScore: 0.514***\n",
      "Episode 2622/9000\tScore: 0.90 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2622 AvgScore: 0.523***\n",
      "Episode 2623/9000\tScore: 0.30 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2623 AvgScore: 0.525***\n",
      "Episode 2624/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2624 AvgScore: 0.525***\n",
      "Episode 2625/9000\tScore: 0.15 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2625 AvgScore: 0.523***\n",
      "Episode 2626/9000\tScore: 0.75 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2626 AvgScore: 0.525***\n",
      "Episode 2627/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2627 AvgScore: 0.523***\n",
      "Episode 2628/9000\tScore: 1.40 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 2628 AvgScore: 0.528***\n",
      "Episode 2629/9000\tScore: 0.65 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2629 AvgScore: 0.518***\n",
      "Episode 2630/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2630 AvgScore: 0.513***\n",
      "Episode 2631/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2631 AvgScore: 0.511***\n",
      "Episode 2632/9000\tScore: 0.75 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2632 AvgScore: 0.513***\n",
      "Episode 2633/9000\tScore: 0.05 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2633 AvgScore: 0.513***\n",
      "Episode 2634/9000\tScore: 0.35 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2634 AvgScore: 0.516***\n",
      "Episode 2635/9000\tScore: 0.70 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2635 AvgScore: 0.520***\n",
      "Episode 2636/9000\tScore: 0.35 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2636 AvgScore: 0.522***\n",
      "Episode 2637/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2637 AvgScore: 0.516***\n",
      "Episode 2638/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2638 AvgScore: 0.517***\n",
      "Episode 2641/9000\tScore: 0.30 \tAvgScore: 0.500\n",
      "***Solved! Episodes taken: 2641 AvgScore: 0.501***\n",
      "Episode 2643/9000\tScore: 1.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 2643 AvgScore: 0.502***\n",
      "Episode 2653/9000\tScore: 1.75 \tAvgScore: 0.500\n",
      "***Solved! Episodes taken: 2653 AvgScore: 0.500***\n",
      "Episode 2655/9000\tScore: 1.35 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2655 AvgScore: 0.508***\n",
      "Episode 2656/9000\tScore: 0.85 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2656 AvgScore: 0.509***\n",
      "Episode 2657/9000\tScore: 0.95 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2657 AvgScore: 0.511***\n",
      "Episode 2658/9000\tScore: 1.25 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2658 AvgScore: 0.513***\n",
      "Episode 2659/9000\tScore: 0.15 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2659 AvgScore: 0.510***\n",
      "Episode 2661/9000\tScore: 1.40 \tAvgScore: 0.510\n",
      "***Solved! Episodes taken: 2661 AvgScore: 0.512***\n",
      "Episode 2662/9000\tScore: 0.15 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2662 AvgScore: 0.509***\n",
      "Episode 2663/9000\tScore: 0.05 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2663 AvgScore: 0.508***\n",
      "Episode 2664/9000\tScore: 0.15 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 2664 AvgScore: 0.504***\n",
      "Episode 2665/9000\tScore: 0.45 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2665 AvgScore: 0.506***\n",
      "Episode 2666/9000\tScore: 2.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 2666 AvgScore: 0.519***\n",
      "Episode 2667/9000\tScore: 0.05 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2667 AvgScore: 0.515***\n",
      "Episode 2668/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2668 AvgScore: 0.507***\n",
      "Episode 2669/9000\tScore: 0.30 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2669 AvgScore: 0.507***\n",
      "Episode 2670/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2670 AvgScore: 0.508***\n",
      "Episode 2671/9000\tScore: 0.70 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2671 AvgScore: 0.511***\n",
      "Episode 2672/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 2672 AvgScore: 0.504***\n",
      "Episode 2673/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 2673 AvgScore: 0.506***\n",
      "Episode 2674/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 2674 AvgScore: 0.505***\n",
      "Episode 2679/9000\tScore: 2.50 \tAvgScore: 0.500\n",
      "***Solved! Episodes taken: 2679 AvgScore: 0.501***\n",
      "Episode 2700/9000\tScore: 0.10 \tAvgScore: 0.479\n",
      "Episode 2800/9000\tScore: 0.65 \tAvgScore: 0.266\n",
      "Episode 2900/9000\tScore: 0.45 \tAvgScore: 0.287\n",
      "Episode 3000/9000\tScore: 0.15 \tAvgScore: 0.211\n",
      "Episode 3100/9000\tScore: -0.00 \tAvgScore: 0.19\n",
      "Episode 3200/9000\tScore: 0.60 \tAvgScore: 0.264\n",
      "Episode 3300/9000\tScore: 0.40 \tAvgScore: 0.223\n",
      "Episode 3400/9000\tScore: 0.35 \tAvgScore: 0.144\n",
      "Episode 3500/9000\tScore: 0.60 \tAvgScore: 0.200\n",
      "Episode 3600/9000\tScore: -0.00 \tAvgScore: 0.14\n",
      "Episode 3700/9000\tScore: 0.35 \tAvgScore: 0.186\n",
      "Episode 3800/9000\tScore: 0.40 \tAvgScore: 0.177\n",
      "Episode 3900/9000\tScore: 0.15 \tAvgScore: 0.199\n",
      "Episode 4000/9000\tScore: 0.10 \tAvgScore: 0.155\n",
      "Episode 4100/9000\tScore: 0.05 \tAvgScore: 0.144\n",
      "Episode 4200/9000\tScore: 0.30 \tAvgScore: 0.121\n",
      "Episode 4300/9000\tScore: 0.25 \tAvgScore: 0.144\n",
      "Episode 4400/9000\tScore: 0.15 \tAvgScore: 0.167\n",
      "Episode 4500/9000\tScore: -0.00 \tAvgScore: 0.20\n",
      "Episode 4600/9000\tScore: 0.20 \tAvgScore: 0.277\n",
      "Episode 4700/9000\tScore: 0.05 \tAvgScore: 0.167\n",
      "Episode 4800/9000\tScore: 0.05 \tAvgScore: 0.242\n",
      "Episode 4900/9000\tScore: 0.45 \tAvgScore: 0.323\n",
      "Episode 5000/9000\tScore: -0.00 \tAvgScore: 0.24\n",
      "Episode 5100/9000\tScore: 0.20 \tAvgScore: 0.288\n",
      "Episode 5200/9000\tScore: -0.00 \tAvgScore: 0.20\n",
      "Episode 5300/9000\tScore: 0.35 \tAvgScore: 0.232\n",
      "Episode 5400/9000\tScore: 0.15 \tAvgScore: 0.219\n",
      "Episode 5500/9000\tScore: 0.25 \tAvgScore: 0.254\n",
      "Episode 5600/9000\tScore: -0.00 \tAvgScore: 0.28\n",
      "Episode 5700/9000\tScore: -0.00 \tAvgScore: 0.26\n",
      "Episode 5800/9000\tScore: -0.00 \tAvgScore: 0.33\n",
      "Episode 5900/9000\tScore: 0.35 \tAvgScore: 0.413\n",
      "Episode 5947/9000\tScore: 2.60 \tAvgScore: 0.508\n",
      "***Solved! Episodes taken: 5947 AvgScore: 0.505***\n",
      "Episode 5948/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 5948 AvgScore: 0.505***\n",
      "Episode 5949/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 5949 AvgScore: 0.502***\n",
      "Episode 5950/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 5950 AvgScore: 0.502***\n",
      "Episode 5962/9000\tScore: 1.70 \tAvgScore: 0.509\n",
      "***Solved! Episodes taken: 5962 AvgScore: 0.503***\n",
      "Episode 5963/9000\tScore: 1.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 5963 AvgScore: 0.515***\n",
      "Episode 5964/9000\tScore: 0.20 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 5964 AvgScore: 0.517***\n",
      "Episode 5965/9000\tScore: 0.75 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 5965 AvgScore: 0.524***\n",
      "Episode 5966/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 5966 AvgScore: 0.517***\n",
      "Episode 5967/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 5967 AvgScore: 0.510***\n",
      "Episode 6000/9000\tScore: 0.05 \tAvgScore: 0.455\n",
      "Episode 6100/9000\tScore: 0.40 \tAvgScore: 0.399\n",
      "Episode 6200/9000\tScore: 0.20 \tAvgScore: 0.412\n",
      "Episode 6300/9000\tScore: 0.10 \tAvgScore: 0.388\n",
      "Episode 6400/9000\tScore: 0.70 \tAvgScore: 0.416\n",
      "Episode 6500/9000\tScore: -0.00 \tAvgScore: 0.41\n",
      "Episode 6600/9000\tScore: -0.00 \tAvgScore: 0.42\n",
      "Episode 6653/9000\tScore: 2.60 \tAvgScore: 0.519\n",
      "***Solved! Episodes taken: 6653 AvgScore: 0.512***\n",
      "Episode 6654/9000\tScore: 0.85 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6654 AvgScore: 0.517***\n",
      "Episode 6655/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6655 AvgScore: 0.507***\n",
      "Episode 6656/9000\tScore: 0.40 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6656 AvgScore: 0.511***\n",
      "Episode 6657/9000\tScore: 0.85 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6657 AvgScore: 0.505***\n",
      "Episode 6660/9000\tScore: 0.10 \tAvgScore: 0.500\n",
      "***Solved! Episodes taken: 6660 AvgScore: 0.500***\n",
      "Episode 6661/9000\tScore: 2.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6661 AvgScore: 0.520***\n",
      "Episode 6662/9000\tScore: 1.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6662 AvgScore: 0.521***\n",
      "Episode 6663/9000\tScore: 0.25 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6663 AvgScore: 0.520***\n",
      "Episode 6664/9000\tScore: 2.60 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6664 AvgScore: 0.541***\n",
      "Episode 6665/9000\tScore: 0.25 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6665 AvgScore: 0.543***\n",
      "Episode 6666/9000\tScore: 0.45 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 6666 AvgScore: 0.545***\n",
      "Episode 6667/9000\tScore: 1.55 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 6667 AvgScore: 0.557***\n",
      "Episode 6668/9000\tScore: 0.30 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 6668 AvgScore: 0.559***\n",
      "Episode 6669/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 6669 AvgScore: 0.553***\n",
      "Episode 6670/9000\tScore: 0.80 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 6670 AvgScore: 0.559***\n",
      "Episode 6671/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 6671 AvgScore: 0.559***\n",
      "Episode 6672/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 6672 AvgScore: 0.557***\n",
      "Episode 6673/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6673 AvgScore: 0.543***\n",
      "Episode 6674/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6674 AvgScore: 0.542***\n",
      "Episode 6675/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6675 AvgScore: 0.542***\n",
      "Episode 6676/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6676 AvgScore: 0.540***\n",
      "Episode 6677/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6677 AvgScore: 0.538***\n",
      "Episode 6678/9000\tScore: 0.55 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6678 AvgScore: 0.520***\n",
      "Episode 6679/9000\tScore: 0.45 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6679 AvgScore: 0.525***\n",
      "Episode 6680/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6680 AvgScore: 0.513***\n",
      "Episode 6681/9000\tScore: 0.35 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6681 AvgScore: 0.516***\n",
      "Episode 6682/9000\tScore: 0.20 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6682 AvgScore: 0.518***\n",
      "Episode 6683/9000\tScore: 1.50 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6683 AvgScore: 0.531***\n",
      "Episode 6684/9000\tScore: 0.35 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6684 AvgScore: 0.531***\n",
      "Episode 6685/9000\tScore: 1.25 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6685 AvgScore: 0.528***\n",
      "Episode 6686/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6686 AvgScore: 0.527***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 6687/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6687 AvgScore: 0.526***\n",
      "Episode 6688/9000\tScore: 1.20 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6688 AvgScore: 0.537***\n",
      "Episode 6689/9000\tScore: 0.40 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6689 AvgScore: 0.515***\n",
      "Episode 6690/9000\tScore: 0.45 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6690 AvgScore: 0.518***\n",
      "Episode 6691/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6691 AvgScore: 0.512***\n",
      "Episode 6692/9000\tScore: 0.95 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6692 AvgScore: 0.518***\n",
      "Episode 6693/9000\tScore: 0.55 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6693 AvgScore: 0.523***\n",
      "Episode 6694/9000\tScore: 0.20 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6694 AvgScore: 0.523***\n",
      "Episode 6695/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6695 AvgScore: 0.519***\n",
      "Episode 6696/9000\tScore: 0.70 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6696 AvgScore: 0.514***\n",
      "Episode 6697/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6697 AvgScore: 0.505***\n",
      "Episode 6698/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 6698 AvgScore: 0.504***\n",
      "Episode 6699/9000\tScore: 0.10 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 6699 AvgScore: 0.505***\n",
      "Episode 6700/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "\n",
      "***Solved! Episodes taken: 6700 AvgScore: 0.505***\n",
      "Episode 6701/9000\tScore: 0.15 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 6701 AvgScore: 0.504***\n",
      "Episode 6702/9000\tScore: 1.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6702 AvgScore: 0.515***\n",
      "Episode 6703/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6703 AvgScore: 0.506***\n",
      "Episode 6704/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6704 AvgScore: 0.506***\n",
      "Episode 6705/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6705 AvgScore: 0.506***\n",
      "Episode 6706/9000\tScore: 0.90 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6706 AvgScore: 0.505***\n",
      "Episode 6708/9000\tScore: 0.90 \tAvgScore: 0.500\n",
      "***Solved! Episodes taken: 6708 AvgScore: 0.504***\n",
      "Episode 6709/9000\tScore: 0.85 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6709 AvgScore: 0.512***\n",
      "Episode 6710/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6710 AvgScore: 0.514***\n",
      "Episode 6711/9000\tScore: 0.65 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6711 AvgScore: 0.518***\n",
      "Episode 6712/9000\tScore: 1.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6712 AvgScore: 0.528***\n",
      "Episode 6713/9000\tScore: 0.30 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6713 AvgScore: 0.530***\n",
      "Episode 6714/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6714 AvgScore: 0.530***\n",
      "Episode 6715/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6715 AvgScore: 0.514***\n",
      "Episode 6716/9000\tScore: 0.20 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6716 AvgScore: 0.516***\n",
      "Episode 6717/9000\tScore: 0.85 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6717 AvgScore: 0.519***\n",
      "Episode 6718/9000\tScore: 0.75 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6718 AvgScore: 0.508***\n",
      "Episode 6719/9000\tScore: 0.70 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6719 AvgScore: 0.515***\n",
      "Episode 6720/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6720 AvgScore: 0.513***\n",
      "Episode 6721/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6721 AvgScore: 0.512***\n",
      "Episode 6722/9000\tScore: 2.60 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6722 AvgScore: 0.538***\n",
      "Episode 6723/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6723 AvgScore: 0.531***\n",
      "Episode 6724/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 6724 AvgScore: 0.527***\n",
      "Episode 6725/9000\tScore: 2.20 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 6725 AvgScore: 0.545***\n",
      "Episode 6726/9000\tScore: 0.60 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 6726 AvgScore: 0.551***\n",
      "Episode 6727/9000\tScore: 0.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 6727 AvgScore: 0.546***\n",
      "Episode 6728/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 6728 AvgScore: 0.546***\n",
      "Episode 6729/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 6729 AvgScore: 0.546***\n",
      "Episode 6730/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 6730 AvgScore: 0.520***\n",
      "Episode 6731/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6731 AvgScore: 0.512***\n",
      "Episode 6732/9000\tScore: 0.45 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6732 AvgScore: 0.510***\n",
      "Episode 6733/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6733 AvgScore: 0.510***\n",
      "Episode 6734/9000\tScore: 0.15 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 6734 AvgScore: 0.505***\n",
      "Episode 6735/9000\tScore: 0.25 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 6735 AvgScore: 0.503***\n",
      "Episode 6736/9000\tScore: 1.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6736 AvgScore: 0.513***\n",
      "Episode 6739/9000\tScore: 0.45 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 6739 AvgScore: 0.502***\n",
      "Episode 6740/9000\tScore: 1.05 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 6740 AvgScore: 0.512***\n",
      "Episode 6800/9000\tScore: 1.95 \tAvgScore: 0.432\n",
      "Episode 6900/9000\tScore: 0.35 \tAvgScore: 0.398\n",
      "Episode 7000/9000\tScore: -0.00 \tAvgScore: 0.30\n",
      "Episode 7100/9000\tScore: -0.00 \tAvgScore: 0.37\n",
      "Episode 7190/9000\tScore: 2.60 \tAvgScore: 0.529\n",
      "***Solved! Episodes taken: 7190 AvgScore: 0.524***\n",
      "Episode 7191/9000\tScore: 0.15 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7191 AvgScore: 0.524***\n",
      "Episode 7192/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7192 AvgScore: 0.519***\n",
      "Episode 7193/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7193 AvgScore: 0.509***\n",
      "Episode 7194/9000\tScore: 0.70 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7194 AvgScore: 0.510***\n",
      "Episode 7195/9000\tScore: 0.55 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7195 AvgScore: 0.514***\n",
      "Episode 7196/9000\tScore: 2.60 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7196 AvgScore: 0.534***\n",
      "Episode 7197/9000\tScore: 0.55 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7197 AvgScore: 0.536***\n",
      "Episode 7198/9000\tScore: 0.20 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7198 AvgScore: 0.526***\n",
      "Episode 7199/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7199 AvgScore: 0.524***\n",
      "Episode 7200/9000\tScore: 0.45 \tAvgScore: 0.53\n",
      "\n",
      "***Solved! Episodes taken: 7200 AvgScore: 0.528***\n",
      "Episode 7201/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7201 AvgScore: 0.507***\n",
      "Episode 7202/9000\tScore: 0.45 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7202 AvgScore: 0.512***\n",
      "Episode 7203/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7203 AvgScore: 0.511***\n",
      "Episode 7204/9000\tScore: 0.60 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7204 AvgScore: 0.514***\n",
      "Episode 7205/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7205 AvgScore: 0.514***\n",
      "Episode 7208/9000\tScore: 0.60 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7208 AvgScore: 0.503***\n",
      "Episode 7209/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7209 AvgScore: 0.501***\n",
      "Episode 7212/9000\tScore: 1.00 \tAvgScore: 0.510\n",
      "***Solved! Episodes taken: 7212 AvgScore: 0.506***\n",
      "Episode 7213/9000\tScore: 0.05 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7213 AvgScore: 0.506***\n",
      "Episode 7214/9000\tScore: 0.95 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7214 AvgScore: 0.512***\n",
      "Episode 7215/9000\tScore: 0.15 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7215 AvgScore: 0.514***\n",
      "Episode 7216/9000\tScore: 0.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7216 AvgScore: 0.517***\n",
      "Episode 7217/9000\tScore: 1.20 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7217 AvgScore: 0.529***\n",
      "Episode 7218/9000\tScore: 0.09 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7218 AvgScore: 0.530***\n",
      "Episode 7219/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7219 AvgScore: 0.530***\n",
      "Episode 7220/9000\tScore: 0.05 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7220 AvgScore: 0.512***\n",
      "Episode 7221/9000\tScore: 0.80 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7221 AvgScore: 0.517***\n",
      "Episode 7222/9000\tScore: 0.10 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7222 AvgScore: 0.517***\n",
      "Episode 7223/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7223 AvgScore: 0.517***\n",
      "Episode 7224/9000\tScore: 0.35 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7224 AvgScore: 0.521***\n",
      "Episode 7225/9000\tScore: 0.60 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7225 AvgScore: 0.525***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 7226/9000\tScore: 0.70 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7226 AvgScore: 0.532***\n",
      "Episode 7227/9000\tScore: 0.20 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7227 AvgScore: 0.531***\n",
      "Episode 7228/9000\tScore: 0.65 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7228 AvgScore: 0.526***\n",
      "Episode 7229/9000\tScore: 1.35 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7229 AvgScore: 0.539***\n",
      "Episode 7230/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7230 AvgScore: 0.533***\n",
      "Episode 7231/9000\tScore: 0.50 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7231 AvgScore: 0.538***\n",
      "Episode 7232/9000\tScore: 1.25 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7232 AvgScore: 0.550***\n",
      "Episode 7233/9000\tScore: 0.95 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7233 AvgScore: 0.557***\n",
      "Episode 7234/9000\tScore: 0.60 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7234 AvgScore: 0.556***\n",
      "Episode 7235/9000\tScore: 1.40 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7235 AvgScore: 0.569***\n",
      "Episode 7236/9000\tScore: 1.05 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7236 AvgScore: 0.578***\n",
      "Episode 7237/9000\tScore: 0.20 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7237 AvgScore: 0.575***\n",
      "Episode 7238/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7238 AvgScore: 0.568***\n",
      "Episode 7239/9000\tScore: 0.40 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7239 AvgScore: 0.571***\n",
      "Episode 7240/9000\tScore: 0.05 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7240 AvgScore: 0.571***\n",
      "Episode 7241/9000\tScore: 0.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7241 AvgScore: 0.550***\n",
      "Episode 7242/9000\tScore: 2.60 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7242 AvgScore: 0.573***\n",
      "Episode 7243/9000\tScore: 0.45 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7243 AvgScore: 0.573***\n",
      "Episode 7244/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7244 AvgScore: 0.563***\n",
      "Episode 7245/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7245 AvgScore: 0.554***\n",
      "Episode 7246/9000\tScore: 2.20 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7246 AvgScore: 0.561***\n",
      "Episode 7247/9000\tScore: 1.10 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7247 AvgScore: 0.570***\n",
      "Episode 7248/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7248 AvgScore: 0.553***\n",
      "Episode 7249/9000\tScore: 0.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7249 AvgScore: 0.553***\n",
      "Episode 7250/9000\tScore: 0.40 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7250 AvgScore: 0.548***\n",
      "Episode 7251/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7251 AvgScore: 0.548***\n",
      "Episode 7252/9000\tScore: 0.80 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7252 AvgScore: 0.552***\n",
      "Episode 7253/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7253 AvgScore: 0.550***\n",
      "Episode 7254/9000\tScore: 0.35 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7254 AvgScore: 0.546***\n",
      "Episode 7255/9000\tScore: 0.45 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7255 AvgScore: 0.549***\n",
      "Episode 7256/9000\tScore: 1.45 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7256 AvgScore: 0.564***\n",
      "Episode 7257/9000\tScore: 1.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7257 AvgScore: 0.574***\n",
      "Episode 7258/9000\tScore: 0.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7258 AvgScore: 0.564***\n",
      "Episode 7259/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7259 AvgScore: 0.563***\n",
      "Episode 7260/9000\tScore: 0.30 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7260 AvgScore: 0.557***\n",
      "Episode 7261/9000\tScore: 0.25 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7261 AvgScore: 0.558***\n",
      "Episode 7262/9000\tScore: 0.70 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7262 AvgScore: 0.565***\n",
      "Episode 7263/9000\tScore: 0.05 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7263 AvgScore: 0.548***\n",
      "Episode 7264/9000\tScore: 0.85 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7264 AvgScore: 0.549***\n",
      "Episode 7265/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7265 AvgScore: 0.548***\n",
      "Episode 7266/9000\tScore: 0.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7266 AvgScore: 0.548***\n",
      "Episode 7267/9000\tScore: 0.30 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7267 AvgScore: 0.549***\n",
      "Episode 7268/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7268 AvgScore: 0.548***\n",
      "Episode 7269/9000\tScore: 0.45 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7269 AvgScore: 0.553***\n",
      "Episode 7270/9000\tScore: 1.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7270 AvgScore: 0.561***\n",
      "Episode 7271/9000\tScore: 0.20 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7271 AvgScore: 0.562***\n",
      "Episode 7272/9000\tScore: 0.70 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7272 AvgScore: 0.568***\n",
      "Episode 7273/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7273 AvgScore: 0.564***\n",
      "Episode 7274/9000\tScore: 1.25 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7274 AvgScore: 0.572***\n",
      "Episode 7275/9000\tScore: 0.05 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7275 AvgScore: 0.568***\n",
      "Episode 7276/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7276 AvgScore: 0.560***\n",
      "Episode 7277/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7277 AvgScore: 0.557***\n",
      "Episode 7278/9000\tScore: 0.35 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7278 AvgScore: 0.535***\n",
      "Episode 7279/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7279 AvgScore: 0.522***\n",
      "Episode 7280/9000\tScore: 0.25 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7280 AvgScore: 0.500***\n",
      "Episode 7281/9000\tScore: 2.60 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7281 AvgScore: 0.508***\n",
      "Episode 7282/9000\tScore: 0.60 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7282 AvgScore: 0.513***\n",
      "Episode 7283/9000\tScore: 0.50 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7283 AvgScore: 0.512***\n",
      "Episode 7284/9000\tScore: 1.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7284 AvgScore: 0.512***\n",
      "Episode 7300/9000\tScore: -0.00 \tAvgScore: 0.43\n",
      "Episode 7355/9000\tScore: 2.60 \tAvgScore: 0.518\n",
      "***Solved! Episodes taken: 7355 AvgScore: 0.506***\n",
      "Episode 7359/9000\tScore: 2.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7359 AvgScore: 0.510***\n",
      "Episode 7360/9000\tScore: 0.80 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7360 AvgScore: 0.515***\n",
      "Episode 7361/9000\tScore: 1.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7361 AvgScore: 0.522***\n",
      "Episode 7362/9000\tScore: 1.75 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7362 AvgScore: 0.533***\n",
      "Episode 7363/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7363 AvgScore: 0.533***\n",
      "Episode 7364/9000\tScore: 0.54 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7364 AvgScore: 0.530***\n",
      "Episode 7365/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7365 AvgScore: 0.530***\n",
      "Episode 7366/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7366 AvgScore: 0.529***\n",
      "Episode 7367/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7367 AvgScore: 0.526***\n",
      "Episode 7368/9000\tScore: 2.40 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7368 AvgScore: 0.550***\n",
      "Episode 7369/9000\tScore: 0.60 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7369 AvgScore: 0.552***\n",
      "Episode 7370/9000\tScore: 0.25 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7370 AvgScore: 0.544***\n",
      "Episode 7371/9000\tScore: 0.15 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7371 AvgScore: 0.544***\n",
      "Episode 7372/9000\tScore: 0.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7372 AvgScore: 0.538***\n",
      "Episode 7373/9000\tScore: 0.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7373 AvgScore: 0.538***\n",
      "Episode 7374/9000\tScore: 0.40 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7374 AvgScore: 0.529***\n",
      "Episode 7375/9000\tScore: 0.55 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7375 AvgScore: 0.534***\n",
      "Episode 7376/9000\tScore: 2.45 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7376 AvgScore: 0.559***\n",
      "Episode 7377/9000\tScore: 0.45 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7377 AvgScore: 0.563***\n",
      "Episode 7378/9000\tScore: 0.20 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7378 AvgScore: 0.562***\n",
      "Episode 7379/9000\tScore: 0.05 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7379 AvgScore: 0.562***\n",
      "Episode 7380/9000\tScore: 0.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7380 AvgScore: 0.561***\n",
      "Episode 7381/9000\tScore: 0.65 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7381 AvgScore: 0.541***\n",
      "Episode 7382/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7382 AvgScore: 0.535***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 7383/9000\tScore: 0.60 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7383 AvgScore: 0.536***\n",
      "Episode 7384/9000\tScore: 0.25 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7384 AvgScore: 0.528***\n",
      "Episode 7385/9000\tScore: 1.15 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7385 AvgScore: 0.539***\n",
      "Episode 7386/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7386 AvgScore: 0.539***\n",
      "Episode 7387/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7387 AvgScore: 0.539***\n",
      "Episode 7388/9000\tScore: 0.30 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7388 AvgScore: 0.542***\n",
      "Episode 7389/9000\tScore: 0.30 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7389 AvgScore: 0.543***\n",
      "Episode 7390/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7390 AvgScore: 0.542***\n",
      "Episode 7391/9000\tScore: 0.85 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7391 AvgScore: 0.546***\n",
      "Episode 7392/9000\tScore: 1.05 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7392 AvgScore: 0.557***\n",
      "Episode 7393/9000\tScore: 0.95 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7393 AvgScore: 0.562***\n",
      "Episode 7394/9000\tScore: 1.35 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7394 AvgScore: 0.565***\n",
      "Episode 7395/9000\tScore: 1.15 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7395 AvgScore: 0.574***\n",
      "Episode 7396/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7396 AvgScore: 0.571***\n",
      "Episode 7397/9000\tScore: 0.10 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7397 AvgScore: 0.569***\n",
      "Episode 7398/9000\tScore: 0.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7398 AvgScore: 0.563***\n",
      "Episode 7399/9000\tScore: 2.10 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7399 AvgScore: 0.584***\n",
      "Episode 7400/9000\tScore: 0.40 \tAvgScore: 0.59\n",
      "\n",
      "***Solved! Episodes taken: 7400 AvgScore: 0.588***\n",
      "Episode 7401/9000\tScore: -0.00 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7401 AvgScore: 0.588***\n",
      "Episode 7402/9000\tScore: -0.00 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7402 AvgScore: 0.585***\n",
      "Episode 7403/9000\tScore: 0.15 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7403 AvgScore: 0.566***\n",
      "Episode 7404/9000\tScore: 0.50 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7404 AvgScore: 0.568***\n",
      "Episode 7405/9000\tScore: 0.10 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7405 AvgScore: 0.568***\n",
      "Episode 7406/9000\tScore: 0.20 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7406 AvgScore: 0.567***\n",
      "Episode 7407/9000\tScore: 0.05 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7407 AvgScore: 0.553***\n",
      "Episode 7408/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7408 AvgScore: 0.548***\n",
      "Episode 7409/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7409 AvgScore: 0.534***\n",
      "Episode 7410/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7410 AvgScore: 0.531***\n",
      "Episode 7411/9000\tScore: 0.50 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7411 AvgScore: 0.536***\n",
      "Episode 7412/9000\tScore: 0.70 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7412 AvgScore: 0.519***\n",
      "Episode 7413/9000\tScore: 1.35 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7413 AvgScore: 0.533***\n",
      "Episode 7414/9000\tScore: 1.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7414 AvgScore: 0.544***\n",
      "Episode 7415/9000\tScore: 0.55 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7415 AvgScore: 0.549***\n",
      "Episode 7416/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7416 AvgScore: 0.545***\n",
      "Episode 7417/9000\tScore: 0.50 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7417 AvgScore: 0.549***\n",
      "Episode 7418/9000\tScore: 0.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7418 AvgScore: 0.549***\n",
      "Episode 7419/9000\tScore: 0.25 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7419 AvgScore: 0.547***\n",
      "Episode 7420/9000\tScore: 1.75 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7420 AvgScore: 0.565***\n",
      "Episode 7421/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7421 AvgScore: 0.564***\n",
      "Episode 7422/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7422 AvgScore: 0.563***\n",
      "Episode 7423/9000\tScore: 0.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7423 AvgScore: 0.558***\n",
      "Episode 7424/9000\tScore: 0.60 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7424 AvgScore: 0.560***\n",
      "Episode 7425/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7425 AvgScore: 0.560***\n",
      "Episode 7426/9000\tScore: 0.05 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7426 AvgScore: 0.558***\n",
      "Episode 7427/9000\tScore: 0.75 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7427 AvgScore: 0.563***\n",
      "Episode 7428/9000\tScore: 1.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7428 AvgScore: 0.562***\n",
      "Episode 7429/9000\tScore: 0.35 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7429 AvgScore: 0.545***\n",
      "Episode 7430/9000\tScore: 0.75 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7430 AvgScore: 0.552***\n",
      "Episode 7431/9000\tScore: 0.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7431 AvgScore: 0.539***\n",
      "Episode 7432/9000\tScore: 1.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7432 AvgScore: 0.550***\n",
      "Episode 7433/9000\tScore: 0.15 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7433 AvgScore: 0.544***\n",
      "Episode 7434/9000\tScore: 0.30 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7434 AvgScore: 0.542***\n",
      "Episode 7435/9000\tScore: 1.55 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7435 AvgScore: 0.556***\n",
      "Episode 7436/9000\tScore: 0.50 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7436 AvgScore: 0.560***\n",
      "Episode 7437/9000\tScore: 1.70 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7437 AvgScore: 0.573***\n",
      "Episode 7438/9000\tScore: 0.15 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7438 AvgScore: 0.557***\n",
      "Episode 7439/9000\tScore: 0.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7439 AvgScore: 0.545***\n",
      "Episode 7440/9000\tScore: 0.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7440 AvgScore: 0.547***\n",
      "Episode 7441/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7441 AvgScore: 0.541***\n",
      "Episode 7442/9000\tScore: 1.30 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7442 AvgScore: 0.554***\n",
      "Episode 7443/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7443 AvgScore: 0.554***\n",
      "Episode 7444/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7444 AvgScore: 0.552***\n",
      "Episode 7445/9000\tScore: 0.30 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7445 AvgScore: 0.545***\n",
      "Episode 7446/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7446 AvgScore: 0.538***\n",
      "Episode 7447/9000\tScore: 0.05 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7447 AvgScore: 0.512***\n",
      "Episode 7448/9000\tScore: 0.30 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7448 AvgScore: 0.514***\n",
      "Episode 7449/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7449 AvgScore: 0.514***\n",
      "Episode 7450/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7450 AvgScore: 0.501***\n",
      "Episode 7453/9000\tScore: 0.25 \tAvgScore: 0.509\n",
      "***Solved! Episodes taken: 7453 AvgScore: 0.501***\n",
      "Episode 7500/9000\tScore: -0.00 \tAvgScore: 0.36\n",
      "Episode 7584/9000\tScore: 1.00 \tAvgScore: 0.509\n",
      "***Solved! Episodes taken: 7584 AvgScore: 0.504***\n",
      "Episode 7585/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7585 AvgScore: 0.503***\n",
      "Episode 7586/9000\tScore: 1.55 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7586 AvgScore: 0.518***\n",
      "Episode 7587/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7587 AvgScore: 0.511***\n",
      "Episode 7588/9000\tScore: 0.95 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7588 AvgScore: 0.519***\n",
      "Episode 7589/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7589 AvgScore: 0.511***\n",
      "Episode 7590/9000\tScore: 1.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7590 AvgScore: 0.523***\n",
      "Episode 7591/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7591 AvgScore: 0.522***\n",
      "Episode 7592/9000\tScore: 0.50 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7592 AvgScore: 0.521***\n",
      "Episode 7593/9000\tScore: 0.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7593 AvgScore: 0.521***\n",
      "Episode 7594/9000\tScore: 0.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7594 AvgScore: 0.519***\n",
      "Episode 7595/9000\tScore: 0.45 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7595 AvgScore: 0.520***\n",
      "Episode 7596/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7596 AvgScore: 0.502***\n",
      "Episode 7597/9000\tScore: 0.65 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7597 AvgScore: 0.505***\n",
      "Episode 7598/9000\tScore: 0.95 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7598 AvgScore: 0.504***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 7600/9000\tScore: 1.10 \tAvgScore: 0.51\n",
      "\n",
      "***Solved! Episodes taken: 7600 AvgScore: 0.509***\n",
      "Episode 7601/9000\tScore: 0.25 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7601 AvgScore: 0.510***\n",
      "Episode 7602/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7602 AvgScore: 0.509***\n",
      "Episode 7603/9000\tScore: 0.10 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7603 AvgScore: 0.504***\n",
      "Episode 7604/9000\tScore: 0.25 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7604 AvgScore: 0.505***\n",
      "Episode 7606/9000\tScore: 2.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7606 AvgScore: 0.517***\n",
      "Episode 7607/9000\tScore: 0.80 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7607 AvgScore: 0.522***\n",
      "Episode 7608/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7608 AvgScore: 0.516***\n",
      "Episode 7609/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7609 AvgScore: 0.511***\n",
      "Episode 7610/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7610 AvgScore: 0.506***\n",
      "Episode 7611/9000\tScore: 0.05 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7611 AvgScore: 0.505***\n",
      "Episode 7614/9000\tScore: 1.05 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7614 AvgScore: 0.504***\n",
      "Episode 7615/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7615 AvgScore: 0.504***\n",
      "Episode 7616/9000\tScore: 0.20 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7616 AvgScore: 0.505***\n",
      "Episode 7617/9000\tScore: 0.65 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7617 AvgScore: 0.509***\n",
      "Episode 7618/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7618 AvgScore: 0.509***\n",
      "Episode 7619/9000\tScore: 2.60 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7619 AvgScore: 0.531***\n",
      "Episode 7620/9000\tScore: 0.20 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7620 AvgScore: 0.533***\n",
      "Episode 7621/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7621 AvgScore: 0.529***\n",
      "Episode 7622/9000\tScore: 0.60 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7622 AvgScore: 0.518***\n",
      "Episode 7623/9000\tScore: 0.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7623 AvgScore: 0.516***\n",
      "Episode 7624/9000\tScore: 2.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7624 AvgScore: 0.536***\n",
      "Episode 7625/9000\tScore: 1.30 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7625 AvgScore: 0.548***\n",
      "Episode 7626/9000\tScore: 1.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7626 AvgScore: 0.551***\n",
      "Episode 7627/9000\tScore: 0.75 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7627 AvgScore: 0.558***\n",
      "Episode 7628/9000\tScore: 0.30 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7628 AvgScore: 0.559***\n",
      "Episode 7629/9000\tScore: 1.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7629 AvgScore: 0.547***\n",
      "Episode 7630/9000\tScore: 1.45 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7630 AvgScore: 0.559***\n",
      "Episode 7631/9000\tScore: 0.30 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7631 AvgScore: 0.562***\n",
      "Episode 7632/9000\tScore: 0.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7632 AvgScore: 0.555***\n",
      "Episode 7633/9000\tScore: 0.95 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7633 AvgScore: 0.558***\n",
      "Episode 7634/9000\tScore: 1.60 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7634 AvgScore: 0.573***\n",
      "Episode 7635/9000\tScore: 0.25 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7635 AvgScore: 0.565***\n",
      "Episode 7636/9000\tScore: 0.20 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7636 AvgScore: 0.567***\n",
      "Episode 7637/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7637 AvgScore: 0.563***\n",
      "Episode 7638/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7638 AvgScore: 0.561***\n",
      "Episode 7639/9000\tScore: 0.85 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7639 AvgScore: 0.566***\n",
      "Episode 7640/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7640 AvgScore: 0.557***\n",
      "Episode 7641/9000\tScore: 0.35 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7641 AvgScore: 0.544***\n",
      "Episode 7642/9000\tScore: 0.70 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7642 AvgScore: 0.551***\n",
      "Episode 7643/9000\tScore: 0.90 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7643 AvgScore: 0.559***\n",
      "Episode 7644/9000\tScore: 2.60 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7644 AvgScore: 0.580***\n",
      "Episode 7645/9000\tScore: 0.15 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7645 AvgScore: 0.581***\n",
      "Episode 7646/9000\tScore: 1.20 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7646 AvgScore: 0.593***\n",
      "Episode 7647/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7647 AvgScore: 0.567***\n",
      "Episode 7648/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7648 AvgScore: 0.557***\n",
      "Episode 7649/9000\tScore: 0.20 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7649 AvgScore: 0.558***\n",
      "Episode 7650/9000\tScore: 0.60 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7650 AvgScore: 0.556***\n",
      "Episode 7651/9000\tScore: 0.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7651 AvgScore: 0.551***\n",
      "Episode 7652/9000\tScore: 1.95 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7652 AvgScore: 0.564***\n",
      "Episode 7653/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7653 AvgScore: 0.552***\n",
      "Episode 7654/9000\tScore: 1.55 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7654 AvgScore: 0.558***\n",
      "Episode 7655/9000\tScore: 0.55 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7655 AvgScore: 0.542***\n",
      "Episode 7656/9000\tScore: 1.80 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7656 AvgScore: 0.560***\n",
      "Episode 7657/9000\tScore: 0.95 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7657 AvgScore: 0.568***\n",
      "Episode 7658/9000\tScore: 0.30 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7658 AvgScore: 0.545***\n",
      "Episode 7659/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7659 AvgScore: 0.545***\n",
      "Episode 7660/9000\tScore: 2.25 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7660 AvgScore: 0.554***\n",
      "Episode 7661/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7661 AvgScore: 0.551***\n",
      "Episode 7662/9000\tScore: 0.35 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7662 AvgScore: 0.555***\n",
      "Episode 7663/9000\tScore: 0.95 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7663 AvgScore: 0.564***\n",
      "Episode 7664/9000\tScore: 0.20 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7664 AvgScore: 0.564***\n",
      "Episode 7665/9000\tScore: 0.50 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7665 AvgScore: 0.563***\n",
      "Episode 7666/9000\tScore: 0.30 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7666 AvgScore: 0.566***\n",
      "Episode 7667/9000\tScore: 1.30 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7667 AvgScore: 0.578***\n",
      "Episode 7668/9000\tScore: 0.40 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7668 AvgScore: 0.582***\n",
      "Episode 7669/9000\tScore: -0.00 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7669 AvgScore: 0.576***\n",
      "Episode 7670/9000\tScore: 0.60 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7670 AvgScore: 0.572***\n",
      "Episode 7671/9000\tScore: 0.40 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7671 AvgScore: 0.572***\n",
      "Episode 7672/9000\tScore: -0.00 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7672 AvgScore: 0.564***\n",
      "Episode 7673/9000\tScore: 0.30 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7673 AvgScore: 0.565***\n",
      "Episode 7674/9000\tScore: 0.15 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7674 AvgScore: 0.566***\n",
      "Episode 7675/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7675 AvgScore: 0.566***\n",
      "Episode 7676/9000\tScore: 0.55 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7676 AvgScore: 0.572***\n",
      "Episode 7677/9000\tScore: 0.40 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7677 AvgScore: 0.575***\n",
      "Episode 7678/9000\tScore: 0.05 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7678 AvgScore: 0.566***\n",
      "Episode 7679/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7679 AvgScore: 0.550***\n",
      "Episode 7680/9000\tScore: 0.25 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7680 AvgScore: 0.553***\n",
      "Episode 7681/9000\tScore: 0.15 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7681 AvgScore: 0.542***\n",
      "Episode 7682/9000\tScore: 0.05 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7682 AvgScore: 0.539***\n",
      "Episode 7683/9000\tScore: 0.45 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7683 AvgScore: 0.542***\n",
      "Episode 7684/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7684 AvgScore: 0.532***\n",
      "Episode 7685/9000\tScore: 0.25 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7685 AvgScore: 0.535***\n",
      "Episode 7686/9000\tScore: 0.65 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7686 AvgScore: 0.526***\n",
      "Episode 7687/9000\tScore: 0.40 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7687 AvgScore: 0.530***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 7688/9000\tScore: 0.90 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7688 AvgScore: 0.529***\n",
      "Episode 7689/9000\tScore: 1.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7689 AvgScore: 0.539***\n",
      "Episode 7690/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7690 AvgScore: 0.526***\n",
      "Episode 7691/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7691 AvgScore: 0.526***\n",
      "Episode 7692/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7692 AvgScore: 0.521***\n",
      "Episode 7693/9000\tScore: 0.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7693 AvgScore: 0.524***\n",
      "Episode 7694/9000\tScore: 1.15 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7694 AvgScore: 0.535***\n",
      "Episode 7695/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7695 AvgScore: 0.531***\n",
      "Episode 7696/9000\tScore: 2.05 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7696 AvgScore: 0.552***\n",
      "Episode 7697/9000\tScore: 0.80 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7697 AvgScore: 0.553***\n",
      "Episode 7698/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7698 AvgScore: 0.544***\n",
      "Episode 7699/9000\tScore: 0.35 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7699 AvgScore: 0.544***\n",
      "Episode 7700/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "\n",
      "***Solved! Episodes taken: 7700 AvgScore: 0.533***\n",
      "Episode 7701/9000\tScore: 0.65 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7701 AvgScore: 0.537***\n",
      "Episode 7702/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7702 AvgScore: 0.537***\n",
      "Episode 7703/9000\tScore: 0.15 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7703 AvgScore: 0.538***\n",
      "Episode 7704/9000\tScore: 0.20 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7704 AvgScore: 0.537***\n",
      "Episode 7705/9000\tScore: 0.75 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7705 AvgScore: 0.544***\n",
      "Episode 7706/9000\tScore: 0.40 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7706 AvgScore: 0.525***\n",
      "Episode 7707/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7707 AvgScore: 0.517***\n",
      "Episode 7708/9000\tScore: 0.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7708 AvgScore: 0.517***\n",
      "Episode 7709/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7709 AvgScore: 0.516***\n",
      "Episode 7710/9000\tScore: 0.15 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7710 AvgScore: 0.518***\n",
      "Episode 7711/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7711 AvgScore: 0.517***\n",
      "Episode 7712/9000\tScore: 1.05 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7712 AvgScore: 0.527***\n",
      "Episode 7713/9000\tScore: 0.30 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7713 AvgScore: 0.526***\n",
      "Episode 7714/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7714 AvgScore: 0.515***\n",
      "Episode 7715/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7715 AvgScore: 0.515***\n",
      "Episode 7716/9000\tScore: 0.45 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7716 AvgScore: 0.518***\n",
      "Episode 7717/9000\tScore: 0.80 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7717 AvgScore: 0.519***\n",
      "Episode 7718/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7718 AvgScore: 0.519***\n",
      "Episode 7720/9000\tScore: 0.95 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7720 AvgScore: 0.507***\n",
      "Episode 7721/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7721 AvgScore: 0.507***\n",
      "Episode 7722/9000\tScore: 0.90 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7722 AvgScore: 0.510***\n",
      "Episode 7723/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7723 AvgScore: 0.509***\n",
      "Episode 7724/9000\tScore: 1.50 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7724 AvgScore: 0.503***\n",
      "Episode 7725/9000\tScore: 1.75 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7725 AvgScore: 0.508***\n",
      "Episode 7726/9000\tScore: 0.50 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7726 AvgScore: 0.501***\n",
      "Episode 7727/9000\tScore: 1.90 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7727 AvgScore: 0.512***\n",
      "Episode 7728/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7728 AvgScore: 0.510***\n",
      "Episode 7738/9000\tScore: 1.05 \tAvgScore: 0.508\n",
      "***Solved! Episodes taken: 7738 AvgScore: 0.503***\n",
      "Episode 7739/9000\tScore: 0.90 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7739 AvgScore: 0.504***\n",
      "Episode 7740/9000\tScore: 0.35 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7740 AvgScore: 0.507***\n",
      "Episode 7741/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7741 AvgScore: 0.506***\n",
      "Episode 7743/9000\tScore: 2.15 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7743 AvgScore: 0.512***\n",
      "Episode 7800/9000\tScore: 0.05 \tAvgScore: 0.444\n",
      "Episode 7900/9000\tScore: 1.70 \tAvgScore: 0.393\n",
      "Episode 7961/9000\tScore: 2.60 \tAvgScore: 0.513\n",
      "***Solved! Episodes taken: 7961 AvgScore: 0.512***\n",
      "Episode 7962/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7962 AvgScore: 0.509***\n",
      "Episode 7963/9000\tScore: 0.60 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7963 AvgScore: 0.514***\n",
      "Episode 7964/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7964 AvgScore: 0.513***\n",
      "Episode 7965/9000\tScore: 0.65 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7965 AvgScore: 0.508***\n",
      "Episode 7966/9000\tScore: 0.10 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7966 AvgScore: 0.505***\n",
      "Episode 7967/9000\tScore: 0.15 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 7967 AvgScore: 0.500***\n",
      "Episode 7970/9000\tScore: 1.40 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 7970 AvgScore: 0.506***\n",
      "Episode 7971/9000\tScore: 1.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7971 AvgScore: 0.516***\n",
      "Episode 7972/9000\tScore: 0.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7972 AvgScore: 0.517***\n",
      "Episode 7973/9000\tScore: 1.20 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 7973 AvgScore: 0.516***\n",
      "Episode 7974/9000\tScore: 1.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 7974 AvgScore: 0.526***\n",
      "Episode 7975/9000\tScore: 1.45 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7975 AvgScore: 0.541***\n",
      "Episode 7976/9000\tScore: 0.55 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7976 AvgScore: 0.544***\n",
      "Episode 7977/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 7977 AvgScore: 0.544***\n",
      "Episode 7978/9000\tScore: 0.40 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7978 AvgScore: 0.547***\n",
      "Episode 7979/9000\tScore: 0.60 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7979 AvgScore: 0.553***\n",
      "Episode 7980/9000\tScore: 0.05 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7980 AvgScore: 0.549***\n",
      "Episode 7981/9000\tScore: 0.25 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7981 AvgScore: 0.548***\n",
      "Episode 7982/9000\tScore: 1.10 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 7982 AvgScore: 0.551***\n",
      "Episode 7983/9000\tScore: 1.35 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 7983 AvgScore: 0.565***\n",
      "Episode 7984/9000\tScore: 1.80 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7984 AvgScore: 0.578***\n",
      "Episode 7985/9000\tScore: 0.10 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7985 AvgScore: 0.576***\n",
      "Episode 7986/9000\tScore: 0.05 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 7986 AvgScore: 0.577***\n",
      "Episode 7987/9000\tScore: 1.15 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7987 AvgScore: 0.587***\n",
      "Episode 7988/9000\tScore: 0.25 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7988 AvgScore: 0.589***\n",
      "Episode 7989/9000\tScore: 1.60 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 7989 AvgScore: 0.604***\n",
      "Episode 7990/9000\tScore: 0.70 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 7990 AvgScore: 0.609***\n",
      "Episode 7991/9000\tScore: -0.00 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 7991 AvgScore: 0.609***\n",
      "Episode 7992/9000\tScore: 0.20 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 7992 AvgScore: 0.611***\n",
      "Episode 7993/9000\tScore: 0.25 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 7993 AvgScore: 0.601***\n",
      "Episode 7994/9000\tScore: 0.60 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 7994 AvgScore: 0.605***\n",
      "Episode 7995/9000\tScore: 0.35 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 7995 AvgScore: 0.596***\n",
      "Episode 7996/9000\tScore: -0.00 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7996 AvgScore: 0.591***\n",
      "Episode 7997/9000\tScore: -0.00 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7997 AvgScore: 0.590***\n",
      "Episode 7998/9000\tScore: -0.00 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 7998 AvgScore: 0.585***\n",
      "Episode 7999/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 7999 AvgScore: 0.566***\n",
      "Episode 8000/9000\tScore: 0.55 \tAvgScore: 0.55\n",
      "\n",
      "***Solved! Episodes taken: 8000 AvgScore: 0.554***\n",
      "Episode 8001/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8001 AvgScore: 0.554***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 8002/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8002 AvgScore: 0.546***\n",
      "Episode 8003/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8003 AvgScore: 0.545***\n",
      "Episode 8004/9000\tScore: 0.30 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8004 AvgScore: 0.544***\n",
      "Episode 8005/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8005 AvgScore: 0.541***\n",
      "Episode 8006/9000\tScore: 0.75 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8006 AvgScore: 0.549***\n",
      "Episode 8007/9000\tScore: 0.55 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8007 AvgScore: 0.553***\n",
      "Episode 8008/9000\tScore: 0.35 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8008 AvgScore: 0.546***\n",
      "Episode 8009/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8009 AvgScore: 0.543***\n",
      "Episode 8010/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8010 AvgScore: 0.542***\n",
      "Episode 8011/9000\tScore: 0.05 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8011 AvgScore: 0.538***\n",
      "Episode 8012/9000\tScore: 0.85 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8012 AvgScore: 0.545***\n",
      "Episode 8013/9000\tScore: 0.40 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8013 AvgScore: 0.547***\n",
      "Episode 8014/9000\tScore: 0.25 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8014 AvgScore: 0.549***\n",
      "Episode 8015/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8015 AvgScore: 0.548***\n",
      "Episode 8016/9000\tScore: 0.60 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8016 AvgScore: 0.540***\n",
      "Episode 8017/9000\tScore: 0.30 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8017 AvgScore: 0.540***\n",
      "Episode 8018/9000\tScore: 0.05 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8018 AvgScore: 0.535***\n",
      "Episode 8019/9000\tScore: 0.65 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8019 AvgScore: 0.535***\n",
      "Episode 8020/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8020 AvgScore: 0.529***\n",
      "Episode 8021/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8021 AvgScore: 0.522***\n",
      "Episode 8022/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8022 AvgScore: 0.517***\n",
      "Episode 8023/9000\tScore: 1.45 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8023 AvgScore: 0.527***\n",
      "Episode 8024/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8024 AvgScore: 0.526***\n",
      "Episode 8025/9000\tScore: 0.15 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8025 AvgScore: 0.527***\n",
      "Episode 8026/9000\tScore: 0.20 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8026 AvgScore: 0.529***\n",
      "Episode 8027/9000\tScore: 0.10 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8027 AvgScore: 0.504***\n",
      "Episode 8029/9000\tScore: 0.65 \tAvgScore: 0.510\n",
      "***Solved! Episodes taken: 8029 AvgScore: 0.506***\n",
      "Episode 8030/9000\tScore: 0.05 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8030 AvgScore: 0.503***\n",
      "Episode 8031/9000\tScore: 0.30 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8031 AvgScore: 0.502***\n",
      "Episode 8032/9000\tScore: 0.45 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8032 AvgScore: 0.505***\n",
      "Episode 8033/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8033 AvgScore: 0.505***\n",
      "Episode 8034/9000\tScore: 0.40 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8034 AvgScore: 0.508***\n",
      "Episode 8035/9000\tScore: 0.85 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8035 AvgScore: 0.514***\n",
      "Episode 8036/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8036 AvgScore: 0.503***\n",
      "Episode 8037/9000\tScore: 0.30 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8037 AvgScore: 0.504***\n",
      "Episode 8100/9000\tScore: 0.10 \tAvgScore: 0.433\n",
      "Episode 8200/9000\tScore: 0.20 \tAvgScore: 0.378\n",
      "Episode 8300/9000\tScore: -0.00 \tAvgScore: 0.28\n",
      "Episode 8380/9000\tScore: 1.90 \tAvgScore: 0.508\n",
      "***Solved! Episodes taken: 8380 AvgScore: 0.502***\n",
      "Episode 8382/9000\tScore: 0.40 \tAvgScore: 0.500\n",
      "***Solved! Episodes taken: 8382 AvgScore: 0.503***\n",
      "Episode 8383/9000\tScore: -0.00 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8383 AvgScore: 0.500***\n",
      "Episode 8384/9000\tScore: 0.65 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8384 AvgScore: 0.505***\n",
      "Episode 8385/9000\tScore: 1.05 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8385 AvgScore: 0.515***\n",
      "Episode 8386/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8386 AvgScore: 0.515***\n",
      "Episode 8387/9000\tScore: 1.20 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8387 AvgScore: 0.523***\n",
      "Episode 8388/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8388 AvgScore: 0.520***\n",
      "Episode 8389/9000\tScore: 1.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8389 AvgScore: 0.530***\n",
      "Episode 8390/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8390 AvgScore: 0.522***\n",
      "Episode 8391/9000\tScore: 0.45 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8391 AvgScore: 0.526***\n",
      "Episode 8392/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8392 AvgScore: 0.526***\n",
      "Episode 8393/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8393 AvgScore: 0.523***\n",
      "Episode 8394/9000\tScore: 2.50 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8394 AvgScore: 0.542***\n",
      "Episode 8395/9000\tScore: 0.40 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8395 AvgScore: 0.545***\n",
      "Episode 8396/9000\tScore: 0.30 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8396 AvgScore: 0.548***\n",
      "Episode 8397/9000\tScore: 0.65 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8397 AvgScore: 0.547***\n",
      "Episode 8398/9000\tScore: 1.50 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 8398 AvgScore: 0.560***\n",
      "Episode 8399/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 8399 AvgScore: 0.561***\n",
      "Episode 8400/9000\tScore: 2.35 \tAvgScore: 0.58\n",
      "\n",
      "***Solved! Episodes taken: 8400 AvgScore: 0.585***\n",
      "Episode 8401/9000\tScore: 0.10 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 8401 AvgScore: 0.585***\n",
      "Episode 8402/9000\tScore: 1.60 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8402 AvgScore: 0.596***\n",
      "Episode 8403/9000\tScore: 0.30 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8403 AvgScore: 0.599***\n",
      "Episode 8404/9000\tScore: 0.25 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8404 AvgScore: 0.596***\n",
      "Episode 8405/9000\tScore: 0.40 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8405 AvgScore: 0.598***\n",
      "Episode 8406/9000\tScore: 0.20 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8406 AvgScore: 0.600***\n",
      "Episode 8407/9000\tScore: 0.05 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8407 AvgScore: 0.598***\n",
      "Episode 8408/9000\tScore: 0.05 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 8408 AvgScore: 0.592***\n",
      "Episode 8409/9000\tScore: 2.25 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8409 AvgScore: 0.610***\n",
      "Episode 8410/9000\tScore: 0.05 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8410 AvgScore: 0.610***\n",
      "Episode 8411/9000\tScore: 0.55 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8411 AvgScore: 0.613***\n",
      "Episode 8412/9000\tScore: 0.05 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8412 AvgScore: 0.613***\n",
      "Episode 8413/9000\tScore: 1.00 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 8413 AvgScore: 0.623***\n",
      "Episode 8414/9000\tScore: 0.25 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8414 AvgScore: 0.599***\n",
      "Episode 8415/9000\tScore: 0.20 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8415 AvgScore: 0.600***\n",
      "Episode 8416/9000\tScore: 0.65 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8416 AvgScore: 0.605***\n",
      "Episode 8417/9000\tScore: -0.00 \tAvgScore: 0.59\n",
      "***Solved! Episodes taken: 8417 AvgScore: 0.590***\n",
      "Episode 8418/9000\tScore: -0.00 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 8418 AvgScore: 0.581***\n",
      "Episode 8419/9000\tScore: 2.05 \tAvgScore: 0.60\n",
      "***Solved! Episodes taken: 8419 AvgScore: 0.598***\n",
      "Episode 8420/9000\tScore: 2.35 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8420 AvgScore: 0.605***\n",
      "Episode 8421/9000\tScore: 0.20 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8421 AvgScore: 0.607***\n",
      "Episode 8422/9000\tScore: 0.10 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8422 AvgScore: 0.608***\n",
      "Episode 8423/9000\tScore: 0.55 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8423 AvgScore: 0.614***\n",
      "Episode 8424/9000\tScore: 0.20 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8424 AvgScore: 0.615***\n",
      "Episode 8425/9000\tScore: 1.40 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 8425 AvgScore: 0.627***\n",
      "Episode 8426/9000\tScore: 0.50 \tAvgScore: 0.63\n",
      "***Solved! Episodes taken: 8426 AvgScore: 0.632***\n",
      "Episode 8427/9000\tScore: -0.00 \tAvgScore: 0.62\n",
      "***Solved! Episodes taken: 8427 AvgScore: 0.621***\n",
      "Episode 8428/9000\tScore: 0.50 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8428 AvgScore: 0.608***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 8429/9000\tScore: 0.20 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8429 AvgScore: 0.610***\n",
      "Episode 8430/9000\tScore: 0.25 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8430 AvgScore: 0.610***\n",
      "Episode 8431/9000\tScore: 0.30 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8431 AvgScore: 0.608***\n",
      "Episode 8432/9000\tScore: 0.25 \tAvgScore: 0.61\n",
      "***Solved! Episodes taken: 8432 AvgScore: 0.609***\n",
      "Episode 8433/9000\tScore: 0.10 \tAvgScore: 0.58\n",
      "***Solved! Episodes taken: 8433 AvgScore: 0.584***\n",
      "Episode 8434/9000\tScore: 0.10 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 8434 AvgScore: 0.567***\n",
      "Episode 8435/9000\tScore: -0.00 \tAvgScore: 0.57\n",
      "***Solved! Episodes taken: 8435 AvgScore: 0.567***\n",
      "Episode 8436/9000\tScore: 0.10 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 8436 AvgScore: 0.557***\n",
      "Episode 8437/9000\tScore: 0.40 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8437 AvgScore: 0.548***\n",
      "Episode 8438/9000\tScore: 0.75 \tAvgScore: 0.56\n",
      "***Solved! Episodes taken: 8438 AvgScore: 0.556***\n",
      "Episode 8439/9000\tScore: -0.00 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8439 AvgScore: 0.553***\n",
      "Episode 8440/9000\tScore: 0.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8440 AvgScore: 0.538***\n",
      "Episode 8441/9000\tScore: 0.55 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8441 AvgScore: 0.538***\n",
      "Episode 8442/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8442 AvgScore: 0.537***\n",
      "Episode 8443/9000\tScore: 0.10 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8443 AvgScore: 0.536***\n",
      "Episode 8444/9000\tScore: 0.05 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8444 AvgScore: 0.532***\n",
      "Episode 8445/9000\tScore: 0.15 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8445 AvgScore: 0.534***\n",
      "Episode 8446/9000\tScore: -0.00 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8446 AvgScore: 0.533***\n",
      "Episode 8447/9000\tScore: 0.24 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8447 AvgScore: 0.535***\n",
      "Episode 8448/9000\tScore: 1.20 \tAvgScore: 0.55\n",
      "***Solved! Episodes taken: 8448 AvgScore: 0.547***\n",
      "Episode 8449/9000\tScore: -0.00 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8449 AvgScore: 0.537***\n",
      "Episode 8450/9000\tScore: 0.05 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8450 AvgScore: 0.537***\n",
      "Episode 8451/9000\tScore: 0.50 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8451 AvgScore: 0.536***\n",
      "Episode 8452/9000\tScore: 0.40 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8452 AvgScore: 0.540***\n",
      "Episode 8453/9000\tScore: 0.20 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8453 AvgScore: 0.542***\n",
      "Episode 8454/9000\tScore: 0.05 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8454 AvgScore: 0.529***\n",
      "Episode 8455/9000\tScore: 0.15 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8455 AvgScore: 0.530***\n",
      "Episode 8456/9000\tScore: 0.70 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8456 AvgScore: 0.516***\n",
      "Episode 8457/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8457 AvgScore: 0.516***\n",
      "Episode 8458/9000\tScore: 0.30 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8458 AvgScore: 0.518***\n",
      "Episode 8459/9000\tScore: 0.10 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8459 AvgScore: 0.503***\n",
      "Episode 8460/9000\tScore: 1.40 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8460 AvgScore: 0.517***\n",
      "Episode 8461/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8461 AvgScore: 0.513***\n",
      "Episode 8472/9000\tScore: 0.80 \tAvgScore: 0.510\n",
      "***Solved! Episodes taken: 8472 AvgScore: 0.506***\n",
      "Episode 8500/9000\tScore: 0.15 \tAvgScore: 0.446\n",
      "Episode 8583/9000\tScore: 1.75 \tAvgScore: 0.500\n",
      "***Solved! Episodes taken: 8583 AvgScore: 0.503***\n",
      "Episode 8600/9000\tScore: 1.45 \tAvgScore: 0.486\n",
      "Episode 8612/9000\tScore: 1.55 \tAvgScore: 0.509\n",
      "***Solved! Episodes taken: 8612 AvgScore: 0.502***\n",
      "Episode 8613/9000\tScore: 0.35 \tAvgScore: 0.50\n",
      "***Solved! Episodes taken: 8613 AvgScore: 0.502***\n",
      "Episode 8629/9000\tScore: 1.55 \tAvgScore: 0.519\n",
      "***Solved! Episodes taken: 8629 AvgScore: 0.507***\n",
      "Episode 8630/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8630 AvgScore: 0.506***\n",
      "Episode 8631/9000\tScore: 0.90 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8631 AvgScore: 0.512***\n",
      "Episode 8632/9000\tScore: 0.55 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8632 AvgScore: 0.515***\n",
      "Episode 8633/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8633 AvgScore: 0.515***\n",
      "Episode 8634/9000\tScore: 0.10 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8634 AvgScore: 0.515***\n",
      "Episode 8635/9000\tScore: 1.30 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8635 AvgScore: 0.528***\n",
      "Episode 8636/9000\tScore: 0.15 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8636 AvgScore: 0.529***\n",
      "Episode 8637/9000\tScore: 0.85 \tAvgScore: 0.54\n",
      "***Solved! Episodes taken: 8637 AvgScore: 0.538***\n",
      "Episode 8638/9000\tScore: 0.10 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8638 AvgScore: 0.526***\n",
      "Episode 8639/9000\tScore: 1.20 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8639 AvgScore: 0.528***\n",
      "Episode 8640/9000\tScore: 0.05 \tAvgScore: 0.53\n",
      "***Solved! Episodes taken: 8640 AvgScore: 0.528***\n",
      "Episode 8641/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8641 AvgScore: 0.521***\n",
      "Episode 8642/9000\tScore: 0.25 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8642 AvgScore: 0.521***\n",
      "Episode 8643/9000\tScore: -0.00 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8643 AvgScore: 0.516***\n",
      "Episode 8644/9000\tScore: -0.00 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8644 AvgScore: 0.507***\n",
      "Episode 8645/9000\tScore: 0.20 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8645 AvgScore: 0.509***\n",
      "Episode 8700/9000\tScore: 0.90 \tAvgScore: 0.473\n",
      "Episode 8706/9000\tScore: 2.15 \tAvgScore: 0.508\n",
      "***Solved! Episodes taken: 8706 AvgScore: 0.504***\n",
      "Episode 8707/9000\tScore: 1.20 \tAvgScore: 0.52\n",
      "***Solved! Episodes taken: 8707 AvgScore: 0.516***\n",
      "Episode 8708/9000\tScore: 0.25 \tAvgScore: 0.51\n",
      "***Solved! Episodes taken: 8708 AvgScore: 0.515***\n",
      "Episode 8800/9000\tScore: 0.10 \tAvgScore: 0.389\n",
      "Episode 8900/9000\tScore: 1.65 \tAvgScore: 0.331\n",
      "Episode 9000/9000\tScore: 0.05 \tAvgScore: 0.314\n"
     ]
    }
   ],
   "source": [
    "scores = DDPG()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XecHXW9//HXZzebQgstIk2CggpYACPFiooF9YL1gngFbFwLivfHVUGlCCii2EEhSo20K83QO0KAEDYhpBLYJJtkU3dTdrO9fX5/nNmzZ3dP27Mzp76fj0eyZ2a+M/M9c875fma+853v19wdERERgKpCZ0BERIqHgoKIiMQpKIiISJyCgoiIxCkoiIhInIKCiIjEKSiIiEicgoKIiMQpKIiISNy4QmdgtPbcc0+fOnVqobMhIlJS5s6d2+TuUzKlK7mgMHXqVGprawudDRGRkmJmq7JJp+ojERGJU1AQEZE4BQUREYlTUBARkTgFBRERiVNQEBGROAUFERGJU1CQUWvZspEXH7i+0NkQkQgoKMiorZ/+Rd495wesXPFaobMiIiFTUJBRm9y9AYDuro4C50REwqagIKPWjwFQZQXOiIiETkFBcmDB/17gfIhI2BQUZNQGQoG5goJIuVFQkFHzgSsFU1AQKTcKCgLA1sb1dLRtj0+vW/kKjevqBxP0dEJrI53trezR1wTAuMalsfkALeuhrzePOS5dHd19NLW0s2ntyoLlwfv72bA6aD3Wtpmm1a/QULeIV2ofB6B5SyOtzVugeS0t2zbTsm0zbN9Ab2cbC195Jek2m9atoqe7CyC+7eatTWzZtJamDWtYvuA5ljz/IF2d7TStX013V2fS7XS2t7Jl09qUee/o7qP2ucfpaNtO47p6NjYsp7+vjw1r6gbf0zCrls6leUsjbdu3sXbFYtq2b8t4jBrqFlH38iz6+/qGzN+wpi7tOk0b1rBh9Wt4f/+I5QP527D6NTo72mjasCZjPvLNvMSqAKZNm+YaTyECF01mZdVUDrzgZWrvm8602h8C0Py9V5m8x15ww6eh/hnqqt/EQX3LB9c74H1w8gz49YFw9LfghMsL9AZKx0d++xSf3XodZ437Fxu/MY+99ntT3vMw5+4/c9TLP6PhyB+x37xfD1k296g/8K45P0i7/p3Hz+Lz73t7fLq9tZkdrngDc3b7FOPe8jGOnH02r73rAg6ee/GIdTeyB3uxmdqdj2faOXeOWL7s0qN4S+8yuKg56b4/c94fuWfCBUPmzd7v6xzTcG1sYth66+qXsc8NRwHQwo7sQhtb2YXdLkpdIL/0yD844rnvDm77G78DYMnshzj0oZN58fBf8u7PfHfIOnUvz+Kguz8Vn37h0J9y9H/+KD698Ol/8fYnTmP2607mmE23D66Y4n2Gzczmuvu0TOl0pSBxB/bXA9C7ak58XlvL1tiL+mcAhgYEgFWzoCNI89ojUWexLCxvbOODVS8D0NK0rjCZWD0bYERAAOhZ/WLG1V+uWz1kurO9FYCDtz5D16rY+skCAsBebAbgbS3/Trr8Lb3L0u774KqGEfN22/BsyvQtGwfHltmFtlh6WtLuo3PV4DFI3Pb21QsA6F8zZ8Q629YsGTJdvWb2kOm21S8BcOime9Puu9AUFEREImCMrD4qBQoKIiIRMFdQGMLM9jezJ81sqZktNrOzk6Q5zsyazWx+8O+CZNsSESk1pfocz7gIt90LnOPu88xsZ2CumT3q7kuGpXvG3T8dYT5EpOwVXwGsK4Vh3H29u88LXm8HlgL7RrU/kZJUoNZ/0Z3FFl/hXDileSzyck/BzKYCRwAvJFl8rJm9bGYPmtlh+ciPhKuzJ9aOu7Ur9pxCz02fp+cfJxcyS0VtEp3sZq2FzsaYtL36NN+59on49Nf+8jAAu7GdY9f/I9J9T6Q7fYLutiGTb3zquykSRsu2r6enr/SuFiIPCma2E3An8AN3H94ObB5wgLu/E/gzcE+KbZxpZrVmVtvY2BhthmXUGltjDyxta+8BoGbFY9TUPVTILBW1+8b/lP2sqaB5cBtbb4a/HX81f1nz2fj0PfzPWLOUtUtqbkif4G8fHjI5vmNTZHlJ551VK2hvLb3gH2lQMLMaYgHhZne/a/hyd29x99bg9QNAjZntmSTddHef5u7TpkyZEmWWRSL3pqr1hc5CeWtM/sR1uLKsGurrijYbEYiy9ZEB1wJL3f13KdK8PkiHmR0V5GdzVHmSaJVqawuR7JV/f/FRtj56L/AVYKGZzQ/m/QR4A4C7Xw18Afi2mfUCHcApXmr9bpS7rKoZyv+HUm6KoYfbsvvWjLFKrlhEFhTcfRYZPnd3vxK4Mqo8iBQ7nQMVqwgK+BIJGnqiWUZQNZBI5VJQEJGS197dlzlRjnwMVw3XzVoxYt4utI8lO5FTUBCRkndEVeoxDgrp+ufqC52FUVNQkBB4wv9SCtz005fk9M2QMbMkr6S4FUPro0pQir8IBQUZYSx1qDJaKpyluCgoiIiMVhlfaSkoSIjK94ciApTMswZjoaAgQy1/ggn9o20yl90PZfaKzWxty9DDpYSndVN8LObi5LBhIUsWv8z65o787z3t2f7gsiHP7aRZx5L8Dva05lyyVlAKCjLUjM9yxOb7Q99sb18/p0yfzVeuS9Z7ukRi+nFw3ccLnYv0rn4fh/7zA5zz66vyvusHF23IfeUsrxgen/DDxJVy318eKSjImGVTaTSQ5pX126PMiiRqWZt6WZGVT7fUXJL3fTa1puvBtMgOUB4pKEhoKvdnVIJ0+yd0nvGglsZBV1CQ0JTGV16Kh04jipGCgoyZftoi5UNBQUQkjUp7mFNBocJ19fbR2RNdD5OSP/39TmtXb5apw6vs681xcPpxpM9rx/CeT7u2Q3dbTvtKqb8PunIYR9kHj7W709LZkzYtncOHpx/U3t1LT5Jj2NfvbG7toqs3v79PBYUKd9xvnuKd58/My76OrVrMrqj1UVSuemge3/n5r2luT1NAReBH1z+c03rVNiww1c8aMnnqhX8enOjtgsv2g1/uk9O+knKHB34Il+0LfVkes6Ap6vJNrbztwodp7erlzjkr+X8XX8bG5hStmZ77E/xqf3bs2pR08aEXPMzp180ZMf+Xdz7LVZedw5cvvzm7vIVEQaHCrW/uZBLhDC6ednCe/l5uHf8Lbhz3y1D2VS68P7ez7GSOnPtjbhp/Oc2b6jMnDvHJ3PPXfCOcDd3wqSGTd0+4MP66vT3kKwTgDRseg/lBgduf7RXWUK2dvezy3GX8ffxv2WVDigcFl94LwI6dqZ+LeG75yKHpP7Pwe1xQM4M7er6XU95ypaAgY5dNARM8CfpmWxNxZirXAf0NAFhvZ173u5vlUP0ySr394bdtm9g9siDOxZ696wHYsT91FVEu3mKrQ91ethQUZOzKuHMwkUqjoCChjclcaa00Slm6z9wroNO3wiiN46qgICIyXAUHRgUFGbPsfj+qYpIipirQOAWFCjeeHo6vnlfobFSsfBZFffNuoe+qY0PdZv8Vbw51e6lUtW0cVXpf+UzGNPcvXE/fwAcw/TiYe0Pa9Ks2t/Hk/NeAWPXbTmTXxXymz3gSnYPPbKyZA785GDq2ZbXtKCgoVLinJ/yA39RMD2+Da5MHGPfKvRxPL39hoXrmt6luXBJMhfN5VLWOrrDOVePcf40q/cZ7zs8qXXdv0CS4aRnce3batC8+fgfn1twGwJfGPcmiid/AujO3vNrUEmsNluoh0aUTv8aMml/FJp66DNo2wdrarPIfBQWFCvd62zrmbQy58t6wYMzbExluY8vomtlubQ9xMKcgfu7bMn/ksu7BZqipwvu24GHCdD0HHFu9JOWyfFNQkDxRnW1x0ecRluiugQvzGUUWFMxsfzN70syWmtliMxtxbWYxfzKzOjNbYGZHRpUfiV42TVtVFEkuCtvcubKqPsdFuO1e4Bx3n2dmOwNzzexRd0+8TjoBODj4dzTw1+CvlJIsmh9VcAs/KROZTnrK5Sse2ZWCu69393nB6+3AUmDfYclOAm7ymNnArma2d1R5ksIrlx9OaIrw0skqqHlmyiuQxGOQ5fEol4c383JPwcymAkcAw0dt3xdI7AyngZGBQwqoui15z47J9PY5Xb2pOngrzA/m+mdXUt8Ufmdq4RlZ4Dy8eAPP1TUVIC/Fa/Ha5rzs54lXNvL0K+s4du31oW3T439LI2hEHhTMbCfgTuAH7j68x6hkR2nEr8TMzjSzWjOrbWxsjCKbksKO952ZMU1i1dBfn1qeIlX+zz7bOrr40MMf529/vSLv+x6L/54xl1P/Pvz8KX+KsZuLb3RcF/o2L6m5gR1saA/BX7uhlvtn/C5p+nwU6s0dsZZKbVmPixG+SIOCmdUQCwg3u/tdSZI0APsnTO8HrBueyN2nu/s0d582ZcqUaDIrSfV0jG78g0wD9uQzNHhPO1OrNvKT3r/kca+lofiK/XCNpT+v8SQfWyHZNocPCTFWq7Z0ALC8MfqeZ1OJsvWRAdcCS909eeiFmcBpQSukY4Bmd18fVZ5EpDLk6+TDi/Gm0BhF2frovcBXgIVmNvDUx0+ANwC4+9XAA8AngTqgHfhqhPkRkUD5FWXRy3y/OXmCsHohzpfIgoK7zyLDVaq7O/DdqPIgeTKK1irlXm0xemEekdIqfMpOht9BqQQHPdEs+VFBzRxHJ7zjooBbnEql1dEABQVJK6siK1VrlfUj+0Gq5NDQ0tnD4nXRNa1MVfgsWttM67DWLCXzLMKGRQXtMRTIPtqOtdVWkXwmUd5TkDIwmq/5AVWbmJDYcuOa98NF+WlfXgrOu+ZOdt/0PJfUpE93UtUs2pkIfCp9wizd8Zef0bzvcfw+lK3l2dXvpWuPQ5iQ6/qLkjV6TO1DVS/xwaqXky4bHlgHbG1tB4OW5m1QPeocxvUsTMhrAQOEgoKE6gt99xc6C3FeJGdeA67Y8j0m1WTuvfOP4wea0F4Yyn4vqrmJzRvvGRrhi/BZhFQmbF6a+8p3jK7tyvXjf5NyWXeKBzM/bHMB+FB18mCSrZcWLGBCEVQ1qfpIxswSvsg1JD+bKrYCuhAmWYjdOY/SjnQUbN+jEtr3JMzCdWzbGrjBnOneQr97UdyMVlCQMSv2ttqldqMvchkL3tI/XtEUrmPbpo3q6qxwn4GCgkghFeUVVOkHhbw9vJbFjkrtpERBQcJVjGVcxRjNwdcHlb2BYxVx4T4swhQqlCgoSKiKsagphnra/CqtM9NSUZQXdRFQUJC03D0+8Hg2+lL8cgpRMI+uDjezza1dTD33fm56vj51osd+DhdNhr7knapFafDdjjzWJVOFEVLJ29rVy9a2cG7sv7VnKSub2sb8Hd7UnH0X7oW8T6egIGkZcNQvHw95i6WpYWusBc8dcxtSJ3rh6tjfvvy3NMq64HenWH/6YRaG65uzP5nJuK0FT7J7b/ZjiyQaCCafqJqTVepC03MKEoKEc1S3TD1eRZ6b+J4q5XpfIjehs5FqT3amH+53rBi+scV5uiBFoxi+pGWtYIFLn2wYsqmhzKXqrpDXCwoKMmaFv+BNr2Tq00WKgIKC5InOTItHps8i1Tjb0fP+cL4n+ToNiPJCr1C/GAUFybMKOWvX/YyyMpaGbKXWJFpBQdLa3Vr5r+pH06apmTHYm6eFPWht0fHkBf7CO+Cu/y5wVVXqYz+agunlNYXs2Tac708V/dTP/EUo24Kx5aq9O/245an2V6hvklofSUaX1lyfdvkU3xJ/vZ81RZ2dgqnu2Ez9xC9zTee3gPcPXXjn1wHoqZrEeKCnv58MPWRHxpIUJxNt+HMTqYucb427N+QcZa/q0j1D2c6RVXWwoS6UbQF5q8tJ96xJvuhKQfKjDKpTxrfGnk/4cFfq5zZ6+vqH/M0kioeUstlmqVVpFK/M5/Ol1vmIgoLkRSFiQjH8wPJrNA+vyWikvqdQfsdSQUFEROIUFESyNKpzQp2Nl51cx7UujVEUBikoiISqGH7WmSlkjVLehy/VIDtSprbEe6qMFUPjrRd6M3cW5+5c/+xK2lIMlp6NUiie91h+TwH2qpAwWp09yRsOjGtaEto+3t5wC1UEzVcL+BEpKEikzr7tpRHzWp74bcb1nnp5OUc+/Dmm3/VQFNnKSRRB5nWv3oL3hNebZzbUUeDozZv7Ap7kamHPf30l47pHVGXXNHbH7ibe1jkvmHJqbPTPN4RBQUEitbV95FXB4pVpup4O7LL2Kd5ZtYIPrP1bFNkqKt1ZNl/NLFbYZxO8SuEqqpicxW2FzkLeKCiIlB0V+WHrd8v5RnOpUVCQslUZP+GRKvV9SzgiCwpmdp2ZbTKzRSmWH2dmzWY2P/h3QVR5kSJQIWdZuQm7Z9BM29NnUfQKeLEXZd9HNwBXAjelSfOMu386wjyIFL2w4uXoOuNTYChq5dj6yN2fBrZkTChlLecCzwb+jOHXUeJXJ/VNbXT25NYCRS2MwlVlTo13FTobeZF1UDCz95nZV4PXU8zswBD2f6yZvWxmD5rZYWn2faaZ1ZpZbWNjYwi7lXzZpT/XbpjDu34uRPEYRpl83BVP8Z2b52VOmMSMp5eOPQMyxKGdI5tXR6WQHRZmFRTM7ELgx8B5wawa4B9j3Pc84AB3fyfwZyDlUzzuPt3dp7n7tClTpoxxt5JP/9t6RZK5ldC+YRQ/6jQR5OlXczsJmjr/Nxn2pxZKkly2v87PAicCbQDuvg7YeSw7dvcWd28NXj8A1JhZOJ2pS9GY7C3Bq9KvzsjmHQzU6+e9V4RhdujbnnKZapYknWyDQrfHKikdwMx2HOuOzez1ZrGfjpkdFeRl81i3KzJc5ZwTZ//wmkgq2bY++j8zuwbY1cy+CXwNSPuoqZndChwH7GlmDcCFxKqdcPergS8A3zazXqADOMV1d6xsDR34JfsneMfyhSjs0JiFlLlGWoPsSCpZBQV3v8LMPgq0AG8BLnD3tAP3uvuXMiy/kliTVZERBoaUVOEllSiKEfmylTEomFk18LC7Hw+kH8FdJJVRfsfD+FEUul4/W+kukHM5CpmPnQKtpJbxnoK79wHtZjY5D/mRPFm0tpmp594f+X6Sn+nnadiRkGojn1y2iann3k/T9szt1HeyDgB+9Ze/w0WTYdXzGdc54pJH6Wnbyhk//xM/+sfT8flTbT0T+9s5/rxrErogT8dSvB6pblNrFtuTYnH+PYvy8nuF7O8pdAILzexRghZIAO7+/UhyJZF7ZMnGAu49v6fwYw0N/6xdA8BrG7fz3izXubjjF7EXi++CA45Nm9Yxqv96DDf4Bppf2yF+eJ6acE48zcIVR7P72w8fbdbT7lNKx4zZq/K2r2yDwv3BP5G8qpSKjqrWDQBMtvaky2s6m0Ldn+7VSCrZ3mi+0czGA28OZi1z957osiWVziroTDbvBbQa+UkaWQUFMzsOuBGoJ3Zxu7+ZnR70byQlKP9Fbq6DnqsAC5uOqKSTbfXRb4GPufsyADN7M3Ar8K6oMib5ULzFQxh13sX77qKih9fKRSEv5rJ9orlmICAAuPurBA+iSWkqleaaoSjkey3Ir1vXV5K7bINCrZldGwyMc5yZ/Q2YG2XGJDovrNjMHx57LS/76uzpY9P2kAemd4cHfgjr5oe73Uy7Df62d/exoGFbaNvNpgjv74dz71xA3aaRfRr19TtTz72f3r7sQsFlD74y6jxKfg0/aZtEyL+hNLINCt8GFgPfB84GlgDfiipTEq2Xrjub+omnUj/xy3nZ368fWpY5UQpJT7Q7t8Gc6XDTiblnahQOa5tN/cRTmdy9IT7v5GtmZ7XuxpbMP+alE7+WMc2Gxo38auH7mXn95SOWLV+zlvqJpzK1KtbMOFNo+Mfser44TrcDi9qwD3HBhG9SP/HUvOw623sK44A/uvvvIP6U84TIciWR+ta4e/O/01FWoxRT9daxLQ8BsHfr4lGvW7+5jb1CyMOOHbGA9NmumcClQ5bVtK4dllpdY5ebGsttsKVcZHul8DgwKWF6EvBY+NkRGUpFW650V0Fyk21QmDgw9gFA8HqHaLIkUlySXeQU05WMSJiyDQptZnbkwISZTSPW3bVIVkbbCCer9NluM6ST5sLGgdG+idS5Vduk4lfITyjbewo/AP5pZuuI5Xcf4OTIciVlY7Boyu1rnnytPBfPRXRZoAJdopb2SsHM3m1mr3f3F4G3ArcDvcBDwMo85E9KXNpCbPNyWPnMkFlL17cwf802dt22MMxMlIHYm5jav3ro7M5mdl4xtFuyTS2drNmavA8lKQ2rNxfu88t0pXANcHzw+ljgJ8D3gMOB6cRGTxNJ6ZCq1ezc0wTsOzhzoJD+c1AjeVFzfNEJf4wFifqJ16bcpgfPO/e5Ux1udrPyzqoVvL2/Lr87TXW1cve32XPZ0KBw9503c2nNM8nTk10TWCmsSQtnUJAvN5nvKVS7+5bg9cnAdHe/093PBw6KNmtSLj65Ke3IrWklKwpbOmLN89q6e3Pe7lhdbxcXbN9DNK8eMevSmusLkBEJ00er5xVs3xmDgpkNXE18BHgiYVm29yNECiPkLiZUny+VIFPBfivwbzNrItba6BkAMzsIaE63osgQiQV0vnuKDummQiFDQiHH7JXKkjYouPsvzOxxYG/gER8cTLaK2L0FkQpSFnesRdLKWAXk7iM6eQl6SRXJUshnuSqbE+hgSLiyfXhNSlxbVy/NHcUxWF5vv7OtPZuB6GPVJqk6lTMc1r0EfbEbzhuaO+nvj7KaZdi2m4f3OZRsldg6YR97d2d9cweRvl2pSAoKFeK0y27gqxdfWehsANC+bilnXvKnpMsOsVUcboPNPdtatvGLX12ctN32znTA9OPouelz1De18cvLL+bvTyQ83xAUmGO9QTxwTyLxnHwH64LfH4pn2X33O3/+yJjyMNwtzy7jN5dfxPYNeW4aK2VPLYgqxJ2cE/Rr+z8FysFgwfzx6lo+Xl2bNNWDE84bMv2B6oV8oHohC5d/FPb4UGzmsDb7Nav+TXPdbP40/iqemfsKHH9X0nRRWLLoJQ7b5/DUCULLw9Dt7FN7OV8ef1dI2xYZpCsFiZyF0DS0qjd9V1vVPbH+Gnfr3zzmfaWSdGiHnnx1aTx075N7m/K0X6k0CgoiGVj8ryrwpfxFFhTM7Doz22Rmi1IsNzP7k5nVmdmCxF5Ypbx4CFUoWW8hsdwO6eG1+HMOOWwvrOcLTK2MJE+ivFK4AfhEmuUnAAcH/84E/hphXqTQxlhAZwosSc/mB8rykArUXLYSRtVZbEPD964gIdGILCi4+9PAljRJTgJu8pjZwK5mtndU+ZESl1C4JisOw7gaSWUg0ES5j0zcFQQkPwp5T2FfYE3CdANDutKUcvHubQ9hHZlvAK/blvpmcqhFYl8PbN+Yw4ojc3H9s/UsWpu6x5ejN98DwB5j7hWmf3Cb586gaXvyZzdExqqQQSHpCV/ShGZnmlmtmdU2NjZGnC2JwoRrjs2YZuGcp1IvzOUsPVXNzb/Ogt++GXqze4Au0wb/Wbsm6fxEcyd+e5T7Su2FiWfxMXshtO2JJCpkUGgA9k+Y3g9Ylyyhu09392nuPm3KlCl5yZzk3+S2FSFsJYs6/KUzY3/7s3vKONM9CbVJknJSyKAwEzgtaIV0DNDs7usLmB8pEVZEw2Pmi1ofSb5E9kSzmd0KHAfsaWYNwIVADYC7Xw08AHwSqAPaga9GlRcpL+nPzPNfeIY8bEPyfSgoSJ5EFhTc/UsZljvw3aj2L6UnXdkaapGYYykeWvPSnKiSSvJDTzRLmRlN4ZldqMm0xfwMgKOgIPmhoCDFI82ZeOIzAuPWJelMbxT3GfpzLGBzLZYXNGzLcc1BO3RuGvM2RLKhoCBFI9u+hXa4/Ytj2k9Xb6zNf3tPb1bp04WbK8f/mX3aX0m7/oN//XG2WUvpHXV64F/yQ0FBioZ5doV09lIEmZBrYo7Y8mDa5T+uuS3cHYpESEFBJIOBGJLyikHV/VJGFBSkjGXqRE/NPEWGU1AQGTNdKkj5UFCQEhHeWb0GyxFJTUGh3DQug6bBwdyXbdjOqs1t8WlvT9ebeWE8uWwTDy5cT3eaoS1H07PFS6u3suW1OayufzU+b2FDM+ubY72wDmxq/muraG7Ppv+jDH0fKcZInnR0Rz/8a2RPNEuBXHVU7O9Fsa6ar/7TL9jGTlw/PjZ7+18+zC4Fyloqm2Z8k52tnYf7D+P9NWPdmnPvNedzQc0Mdo/PM/7jylmYwb//90O8wWKB4Nh73svUzlt4+cKPMXlS7jve0LgJqseab5HMfnL3Qn5/8uGR7kNBocz9fvzQ9u27tK4sUE5SO3ncUwDsYS05b8MSLnovqJmRNI07fPOmWh4eNv/3j77KRScelvO+P1c9K+d1RUZjeWNr5PtQ9ZEUjaqEgWSi0tM3ch99/ar/kdKQj6pKBQWpACr0pTzko58tBQUpCdk+U6AnD0TGRkFBSkJY50e6ZhBJT0GhTHV095VVXXlVb6xZbVtX8v6RBpbv0rs1/YaGVcruQOeQ5q49ff109vTR1tWLj6jALZ/jKaXJPPr7bmp9VKa+cuEfeMthR/KLQmdkFN5d9WrKZW959Awaupp43yP7UD9x5PJDnvwmAPv1NyRd/x22nEbfldN77h0yf8nEr3Hrtl8CbwPgpCufpW79Zs4bdwtVH/05p3/wkHhaPfQmhfbr5h8CcyLdh4JCmbpjwsWsfHWvsroW7F/2IPD1HNY0Zk44H4A1PfuMWPrG7YPjMyxZ38Lt4y/j6KpXqH/mVfjg/CFjOYgU0iF9yyLfRxkVGTLcgVUbC52FknS4LQdgam/wTIcP/FFwkPKnoCAVJZsKoFRpFBKkEigoSAkZe7E8ti3onoKUPwUFkQxUbSSVREFBSka+Wv8MDwJqdSSVRK2Pysm1Hy90DiK1/8bH+Xr160Pf7tFNd3Hq705iQ/+uTKCbSdYdXzZn5RaamrcHvaDqikHKn64Uysma2YXOQeTOr7k5ku3e0nI6K5raWDbxjCHzL3/oFT5ZHW27cJFioqAgkjVVI0n5U1AQSUMVRlJpIg0KZvYJM1tmZnVmdm6S5WeYWaOZzQ/+fSPo7FRNAAANx0lEQVTK/IiM9lx/6MPMulKQ8hfZjWYzqwauAj4KNAAvmtlMd18yLOnt7n5WVPkQSTTaM/9su+wWKRdRXikcBdS5+wp37wZuA06KcH8ikVJ4kEoQZVDYF1iTMN0QzBvu82a2wMzuMLP9I8xPWbj5hVVDxmlt2NrOdbOKb9zlYpLYBfYuvr2AOREpflEGhWQnVsMrZe8Fprr7O4DHgBuTbsjsTDOrNbPaxsbGkLNZWg6//z+Y/ufBDrFPu3YOF9+3hKbWrgLmqnRMJnVQqJ946oh5h3QvjL/WHQWpBFEGhQYg8cx/P2BdYgJ33+zuA6XZ34B3JduQu09392nuPm3KlCmRZLZUHFa1isur/hKfbu7oAfIzoHclOnX79QlTqkCS8hdlUHgRONjMDjSz8cApwMzEBGa2d8LkicDSCPNTlgZigbr8j0ZirFV3F1IJImt95O69ZnYW8DCxTgKuc/fFZnYxUOvuM4Hvm9mJQC+wBTgjqvxI5RpTh3YJ0VYhQSpBpH0fufsDwAPD5l2Q8Po84Lwo8yAyFuohVSqNnmgucQMta1R0RUVHViqLekktUe6OmfEmX021baNq20GFzlLR2tXacl63ubM3furU3N6dPrFIGdCVQom68bl6AO7wc7h9wiXs9vejCpuhMnVM1WDbh+O3/6uAORHJDwWFEpL4ENbyxtzPfkVEUlFQKFFqgioiUVBQKCGJD6gpJohIFBQUREQkTkGhhOjhKRGJmoKCiIjEKSiUilcfofriXeOTBnTdc3bh8iMiZUkPr5WKpUPbyB/d+E8mNNxQmLyISNnSlUKpGHZDYY/O1YXJh4iUNQWFUqUHFUQkAgoKIiISp6AgIiJxCgoiIhKnoJCovx9uPRVWPgNAR3cfp0x/ntc2ph7sPZPv3foSjy/dyN0vNfCzu+bDzf8Jq56HuTfAwz9lyboW/uvvL9DZ0zdkvS1t3Xzhr8/RcesZ8OrDDL/TfFTjnTnnSUQkFTVJTdS5DZbdD6tmwbmrmb1yM7NXbOGS+5dy09dy65r6z8uOY8bi49nKTlw67p7YzA0LYPt6AM5f8SnmrtrKorXNTJu6e2z5RZNp2O9LXLj+eSZtrIdld7P+wM+xd/JdiIiERkEhKUv4f+y+Mu4xWn3iiO0nFfR6946GW4dcxzVs7VBQEJHIqfoomWHNPRPHMQhn+2kOe4p9hZ0FEZFkFBQSDSt5bazPAqQqydNt1/tTLRhbXkREsqCgkMj7MqcZ1fZSFfCDQWHEVUjYeRARGQUFhUT9AwVySHcT+lMU8LlcKaj+SETywEKvL4/YtGnTvLa2Nqd1/1m7hivueIr3VC1myZRPclzTLZxVcw870wHAad0/5qbxlwNwVvf3uK//2Pi6b7R1fGnC88zqPoh/978TgBOrnmM3284z/W/n89VP891xM/l1z8ms8Sks8QM4e9xdnFj9fNb5q+/fi6lVG3N6byJSIS5qzmk1M5vr7tMypquYoNDXC5fsEX6GRETyKeKgUDnVRy0Nhc6BiEjRq5ygICIiGSkoiIhIXKRBwcw+YWbLzKzOzM5NsnyCmd0eLH/BzKZGmR8REUkvsqBgZtXAVcAJwKHAl8zs0GHJvg5sdfeDgN8Dl0eVHxERySzKK4WjgDp3X+Hu3cBtwEnD0pwE3Bi8vgP4iI35MeLkmlq7otisiEhZiTIo7AusSZhuCOYlTePuvUAzMKLdqJmdaWa1Zlbb2NiYU2aWzv13TuuJiBSL/jzcBo5yD8nO+Ic/FJFNGtx9urtPc/dpU6ZMySkzh+27W07rSXLP9Q2vCRSRqLUf8p+R7yPKrrMbgP0TpvcD1qVI02Bm44DJwJYoMrP7u78I7/5iFJuuSO8pdAYko61t3Ux/ZgWH7bMLO4yv5sNv3avQWSpqNz5Xz7sO2I237Tu50FlJaac87CPKoPAicLCZHQisBU4BTh2WZiZwOvA88AXgCS+1R6xFitRuO47nx594a6GzUTJOf8/UQmehKEQWFNy918zOAh4GqoHr3H2xmV0M1Lr7TOBaYIaZ1RG7QjglqvyIiEhmkY685u4PAA8Mm3dBwutOQHU6IiJFQk80i4hInIKCiIjEKSiIiEicgoKIiMQpKIiISJyCgoiIxJXccJxm1gisynH1PYGmELNT6nQ8htLxGKRjMVQ5HI8D3D1jP0ElFxTGwsxqsxmjtFLoeAyl4zFIx2KoSjoeqj4SEZE4BQUREYmrtKAwvdAZKDI6HkPpeAzSsRiqYo5HRd1TEBGR9CrtSkFERNKomKBgZp8ws2VmVmdm5xY6P1Ews/3N7EkzW2pmi83s7GD+7mb2qJm9FvzdLZhvZvan4JgsMLMjE7Z1epD+NTM7vVDvKQxmVm1mL5nZfcH0gWb2QvDebjez8cH8CcF0XbB8asI2zgvmLzOzjxfmnYydme1qZneY2SvB9+TYSv1+mNn/BL+TRWZ2q5lNrOTvRpy7l/0/YuM5LAfeCIwHXgYOLXS+InifewNHBq93Bl4FDgV+DZwbzD8XuDx4/UngQWLDoh4DvBDM3x1YEfzdLXi9W6Hf3xiOy/8DbgHuC6b/DzgleH018O3g9XeAq4PXpwC3B68PDb4zE4ADg+9SdaHfV47H4kbgG8Hr8cCulfj9IDY+/EpgUsJ34oxK/m4M/KuUK4WjgDp3X+Hu3cBtwEkFzlPo3H29u88LXm8HlhL78p9ErDAg+PuZ4PVJwE0eMxvY1cz2Bj4OPOruW9x9K/Ao8Ik8vpXQmNl+wKeAvwfTBnwYuCNIMvx4DBynO4CPBOlPAm5z9y53XwnUEftOlRQz2wX4ALHBrXD3bnffRuV+P8YBk4KhgHcA1lOh341ElRIU9gXWJEw3BPPKVnB5ewTwArCXu6+HWOAAXhckS3Vcyul4/QH4EdAfTO8BbHP33mA68b3F33ewvDlIXy7H441AI3B9UJ32dzPbkQr8frj7WuAKYDWxYNAMzKVyvxtxlRIULMm8sm12ZWY7AXcCP3D3lnRJk8zzNPNLipl9Gtjk7nMTZydJ6hmWlcXxIHZmfCTwV3c/AmgjVl2UStkej+C+yUnEqnz2AXYETkiStFK+G3GVEhQagP0TpvcD1hUoL5EysxpiAeFmd78rmL0xuOwn+LspmJ/quJTL8XovcKKZ1ROrMvwwsSuHXYMqAxj63uLvO1g+mdjY4eVyPBqABnd/IZi+g1iQqMTvx/HASndvdPce4C7gPVTudyOuUoLCi8DBQcuC8cRuFM0scJ5CF9RxXgssdfffJSyaCQy0EDkd+FfC/NOCVibHAM1B9cHDwMfMbLfgjOpjwbyS4u7nuft+7j6V2Gf+hLt/GXgS+EKQbPjxGDhOXwjSezD/lKAFyoHAwcCcPL2N0Lj7BmCNmb0lmPURYAmV+f1YDRxjZjsEv5uBY1GR340hCn2nO1//iLWkeJVY64CfFjo/Eb3H9xG7dF0AzA/+fZJY3efjwGvB392D9AZcFRyThcC0hG19jdhNszrgq4V+byEcm+MYbH30RmI/3Drgn8CEYP7EYLouWP7GhPV/GhynZcAJhX4/YzgOhwO1wXfkHmKthyry+wH8HHgFWATMINaCqGK/GwP/9ESziIjEVUr1kYiIZEFBQURE4hQUREQkTkFBRETiFBRERCROQUFKhpn1mdn8hH9pe7s1s2+Z2Wkh7LfezPYcRfqnzKw2YXqamT011nwE2zrDzK4MY1siyYzLnESkaHS4++HZJnb3q6PMTAavM7MT3P3BAuZhBDOrdve+QudDipeuFKTkBWfyl5vZnODfQcH8i8zsf4PX3zezJcG4ALcF83Y3s3uCebPN7B3B/D3M7JGg07hrSOjfxsz+K9jHfDO7xsyqU2TrN8DPkuR1yJm+md1nZscFr1uD9zHXzB4zs6OCq44VZnZiwmb2N7OHgv77L8yUt2C7F5vZC8CxuRxjqRwKClJKJg2rPjo5YVmLux8FXEmsf6PhzgWOcPd3AN8K5v0ceCmY9xPgpmD+hcAsj3UaNxN4A4CZHQKcDLw3uGLpA76cIq/PA11m9qFRvL8dgafc/V3AduBS4KPAZ4GLE9IdFez3cOCLQfVUurztCCxy96PdfdYo8iMVSNVHUkrSVR/dmvD390mWLwBuNrN7iHXvALFuQT4P4O5PBFcIk4mNOfC5YP79ZrY1SP8R4F3Ai7HucpjEYOdxyVxK7Grhx1m8N4Bu4KHg9UKgy917zGwhMDUh3aPuvhnAzO4K3kdvmrz1EeskUSQjBQUpF57i9YBPESvsTwTON7PDSN/tcbJtGHCju5+XVYZigeYSYqOWDehl6BX6xITXPT7Y70w/0BVspz+h585keRvowjlV3jp1H0GypeojKRcnJ/x9PnGBmVUB+7v7k8QG3NkV2Al4mqCKJajXb/LY+BOJ808g1mkcxDqL+4KZvS5YtruZHZAhX78I9jmgHjjczKrMbH9yG6Xro8G+JxEbGezZHPMmMoKuFKSUTDKz+QnTD7n7QLPUCcGN1CrgS8PWqwb+EVQNGfB7d99mZhcRG4VsAdDOYNfIPwduNbN5wL+JdbOMuy8xs58BjwSBpgf4LrAqVYbd/QEza0yY9SyxsYEXEuudc96ojkDMLGK9eh4E3OLutQCjzZtIMuolVUqexQbRmebuTYXOi0ipU/WRiIjE6UpBRETidKUgIiJxCgoiIhKnoCAiInEKCiIiEqegICIicQoKIiIS9/8BmeFp1y23dFgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f36926c1400>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "perf = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode Number')\n",
    "plt.rcParams[\"figure.figsize\"] = [18.0, 12.0]\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint_1545538436_actor_best.pth\r\n",
      "checkpoint_1545538436_actor_episode2087_score0.503.pth\r\n",
      "checkpoint_1545538436_actor_episode2098_score0.512.pth\r\n",
      "checkpoint_1545538436_actor_episode2099_score0.519.pth\r\n",
      "checkpoint_1545538436_actor_episode2100_score0.520.pth\r\n",
      "checkpoint_1545538436_actor_episode2101_score0.526.pth\r\n",
      "checkpoint_1545538436_actor_episode2103_score0.527.pth\r\n",
      "checkpoint_1545538436_actor_episode2104_score0.532.pth\r\n",
      "checkpoint_1545538436_actor_episode2107_score0.545.pth\r\n",
      "checkpoint_1545538436_actor_episode2108_score0.556.pth\r\n",
      "checkpoint_1545538436_actor_episode2109_score0.574.pth\r\n",
      "checkpoint_1545538436_actor_episode2110_score0.575.pth\r\n",
      "checkpoint_1545538436_actor_episode2111_score0.578.pth\r\n",
      "checkpoint_1545538436_actor_episode2112_score0.580.pth\r\n",
      "checkpoint_1545538436_actor_episode2113_score0.605.pth\r\n",
      "checkpoint_1545538436_actor_episode2133_score0.607.pth\r\n",
      "checkpoint_1545538436_actor_episode2135_score0.614.pth\r\n",
      "checkpoint_1545538436_actor_episode2136_score0.619.pth\r\n",
      "checkpoint_1545538436_actor_episode2137_score0.620.pth\r\n",
      "checkpoint_1545538436_actor_episode2138_score0.625.pth\r\n",
      "checkpoint_1545538436_actor_episode2139_score0.627.pth\r\n",
      "checkpoint_1545538436_actor_episode2140_score0.628.pth\r\n",
      "checkpoint_1545538436_actor_episode2143_score0.634.pth\r\n",
      "checkpoint_1545538436_actor_episode2148_score0.641.pth\r\n",
      "checkpoint_1545538436_actor_episode2449_score0.652.pth\r\n",
      "checkpoint_1545538436_actor_episode2451_score0.676.pth\r\n",
      "checkpoint_1545538436_actor_episode2452_score0.703.pth\r\n",
      "checkpoint_1545538436_actor_episode2453_score0.729.pth\r\n",
      "checkpoint_1545538436_actor_episode2454_score0.740.pth\r\n",
      "checkpoint_1545538436_actor_episode2455_score0.749.pth\r\n",
      "checkpoint_1545538436_actor_episode2456_score0.767.pth\r\n",
      "checkpoint_1545538436_actor_episode2457_score0.773.pth\r\n",
      "checkpoint_1545538436_actor_episode2458_score0.786.pth\r\n",
      "checkpoint_1545538436_actor_episode2466_score0.790.pth\r\n",
      "checkpoint_1545538436_actor_episode2474_score0.799.pth\r\n",
      "checkpoint_1545538436_actor_episode2475_score0.809.pth\r\n",
      "checkpoint_1545538436_actor_episode2477_score0.822.pth\r\n",
      "checkpoint_1545538436_actor_episode2478_score0.838.pth\r\n",
      "checkpoint_1545538436_actor_episode2480_score0.849.pth\r\n",
      "checkpoint_1545538436_actor_episode2481_score0.850.pth\r\n",
      "checkpoint_1545538436_actor_episode2484_score0.851.pth\r\n",
      "checkpoint_1545538436_actor_episode2485_score0.856.pth\r\n",
      "checkpoint_1545538436_actor_episode2486_score0.858.pth\r\n",
      "checkpoint_1545538436_actor_episode2510_score0.864.pth\r\n",
      "checkpoint_1545538436_actor_episode2512_score0.878.pth\r\n",
      "checkpoint_1545538436_actor_episode2513_score0.879.pth\r\n",
      "checkpoint_1545538436_critic_best.pth\r\n",
      "checkpoint_1545538436_critic_episode2087_score0.503.pth\r\n",
      "checkpoint_1545538436_critic_episode2098_score0.512.pth\r\n",
      "checkpoint_1545538436_critic_episode2099_score0.519.pth\r\n",
      "checkpoint_1545538436_critic_episode2100_score0.520.pth\r\n",
      "checkpoint_1545538436_critic_episode2101_score0.526.pth\r\n",
      "checkpoint_1545538436_critic_episode2103_score0.527.pth\r\n",
      "checkpoint_1545538436_critic_episode2104_score0.532.pth\r\n",
      "checkpoint_1545538436_critic_episode2107_score0.545.pth\r\n",
      "checkpoint_1545538436_critic_episode2108_score0.556.pth\r\n",
      "checkpoint_1545538436_critic_episode2109_score0.574.pth\r\n",
      "checkpoint_1545538436_critic_episode2110_score0.575.pth\r\n",
      "checkpoint_1545538436_critic_episode2111_score0.578.pth\r\n",
      "checkpoint_1545538436_critic_episode2112_score0.580.pth\r\n",
      "checkpoint_1545538436_critic_episode2113_score0.605.pth\r\n",
      "checkpoint_1545538436_critic_episode2133_score0.607.pth\r\n",
      "checkpoint_1545538436_critic_episode2135_score0.614.pth\r\n",
      "checkpoint_1545538436_critic_episode2136_score0.619.pth\r\n",
      "checkpoint_1545538436_critic_episode2137_score0.620.pth\r\n",
      "checkpoint_1545538436_critic_episode2138_score0.625.pth\r\n",
      "checkpoint_1545538436_critic_episode2139_score0.627.pth\r\n",
      "checkpoint_1545538436_critic_episode2140_score0.628.pth\r\n",
      "checkpoint_1545538436_critic_episode2143_score0.634.pth\r\n",
      "checkpoint_1545538436_critic_episode2148_score0.641.pth\r\n",
      "checkpoint_1545538436_critic_episode2449_score0.652.pth\r\n",
      "checkpoint_1545538436_critic_episode2451_score0.676.pth\r\n",
      "checkpoint_1545538436_critic_episode2452_score0.703.pth\r\n",
      "checkpoint_1545538436_critic_episode2453_score0.729.pth\r\n",
      "checkpoint_1545538436_critic_episode2454_score0.740.pth\r\n",
      "checkpoint_1545538436_critic_episode2455_score0.749.pth\r\n",
      "checkpoint_1545538436_critic_episode2456_score0.767.pth\r\n",
      "checkpoint_1545538436_critic_episode2457_score0.773.pth\r\n",
      "checkpoint_1545538436_critic_episode2458_score0.786.pth\r\n",
      "checkpoint_1545538436_critic_episode2466_score0.790.pth\r\n",
      "checkpoint_1545538436_critic_episode2474_score0.799.pth\r\n",
      "checkpoint_1545538436_critic_episode2475_score0.809.pth\r\n",
      "checkpoint_1545538436_critic_episode2477_score0.822.pth\r\n",
      "checkpoint_1545538436_critic_episode2478_score0.838.pth\r\n",
      "checkpoint_1545538436_critic_episode2480_score0.849.pth\r\n",
      "checkpoint_1545538436_critic_episode2481_score0.850.pth\r\n",
      "checkpoint_1545538436_critic_episode2484_score0.851.pth\r\n",
      "checkpoint_1545538436_critic_episode2485_score0.856.pth\r\n",
      "checkpoint_1545538436_critic_episode2486_score0.858.pth\r\n",
      "checkpoint_1545538436_critic_episode2510_score0.864.pth\r\n",
      "checkpoint_1545538436_critic_episode2512_score0.878.pth\r\n",
      "checkpoint_1545538436_critic_episode2513_score0.879.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls savedmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total score: 2.600000038743019\n"
     ]
    }
   ],
   "source": [
    "session_name = '1545538436'\n",
    "file_tpl_best = 'savedmodels/checkpoint_{}_{}_best.pth'\n",
    "\n",
    "checkpoint_actor = file_tpl_best.format(session_name,'actor')\n",
    "checkpoint_critic = file_tpl_best.format(session_name,'critic')\n",
    "\n",
    "# torch.load(file, map_location='cpu')\n",
    "agent.actor_local.load_state_dict(torch.load(checkpoint_actor, map_location='cpu'))\n",
    "agent.critic_local.load_state_dict(torch.load(checkpoint_critic, map_location='cpu'))\n",
    "        \n",
    "state = env.reset(train=False)     \n",
    "agent.reset()\n",
    "scores = np.zeros(env.num_agents)                          \n",
    "while True:\n",
    "    action = agent.act(state, add_noise=False)                 \n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    scores += reward                         \n",
    "    state = next_state                              \n",
    "    if np.any(done):                              \n",
    "        \n",
    "print('Total score:',np.mean(scores))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "drlnd",
   "language": "python",
   "name": "drlnd"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}