HW3_Q1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ordinary-coordinate",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from model import NSFrozenLake\n",
    "from amalearn.agent import AgentBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "immune-start",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% allowed actions\n",
    "LEFT = 0\n",
    "DOWN = 1\n",
    "RIGHT = 2\n",
    "UP = 3\n",
    "\n",
    "ACTIONS = [LEFT, DOWN, RIGHT, UP]\n",
    "\n",
    "#%% hyperparameters\n",
    "REPS = 20\n",
    "EPISODES = 100\n",
    "EPSILON = 0.1\n",
    "LEARNING_RATE = 0.1\n",
    "DISCOUNT = 0.9\n",
    "STUDENT_NUM = 810896059\n",
    "THETA = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "union-houston",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fewer-chase",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "environment = NSFrozenLake(studentNum=STUDENT_NUM)\n",
    "environment.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "capital-steering",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "you can see the environment in each step by render command :\n",
      "\n",
      "------------------------------\n",
      "| \u001b[44m0.000\u001b[0m | 0.001 | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | 0.001 | 0.001 | 0.001 | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | 0.001 | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | 0.000 | \n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "print(\"you can see the environment in each step by render command :\")\n",
    "environment.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "external-argentina",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "and this is the bare map for debugging :\n",
      "[[0.         0.001      0.31141408 0.24825232]\n",
      " [0.51044688 0.001      0.001      0.001     ]\n",
      " [0.63032472 0.6521101  0.68109729 0.001     ]\n",
      " [0.67382383 0.60654236 0.64488509 0.        ]]\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\\nand this is the bare map for debugging :\")\n",
    "print(environment.map)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "static-valuable",
   "metadata": {},
   "source": [
    "### get environment informations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "weekly-processor",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = []\n",
    "env_info = []\n",
    "for s0 in range(4):\n",
    "    for s1 in range(4):\n",
    "        dict = {}\n",
    "        for action in ACTIONS:\n",
    "            states, probs, fail_probs, dones = environment.possible_consequences(action=action ,state_now=(s0,s1))\n",
    "            dict[\"states\"] = states\n",
    "            dict[\"probs\"] = probs\n",
    "            dict[\"fail_probs\"] = fail_probs\n",
    "            dict[\"dones\"] = dones\n",
    "            temp.append(dict)\n",
    "    env_info.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "vulnerable-zoning",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'states': [(0, 0), (0, 1), (1, 0)],\n",
       " 'probs': array([0.95 , 0.025, 0.025]),\n",
       " 'fail_probs': array([0.        , 0.001     , 0.51044688]),\n",
       " 'dones': array([False, False, False])}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env_info[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cosmetic-franchise",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Q1Agent(AgentBase):\n",
    "    def __init__(self, id, environment):\n",
    "        super(Q1Agent, self).__init__(id, environment)\n",
    "        self.V = np.zeros((4, 4))\n",
    "        self.policy = np.zeros((4, 4))\n",
    "\n",
    "    def Q_value(self, s0, s1):\n",
    "        q_values = []\n",
    "        for a in ACTIONS:\n",
    "            next_states, probs, fail_probs, dones = self.environment.possible_consequences(action=a, state_now=(s0, s1))\n",
    "            q_value = 0\n",
    "            for i,s in enumerate(next_states):\n",
    "                r = -1\n",
    "                if dones[i]:\n",
    "                    r += 50\n",
    "                q_value +=  probs[i] * (fail_probs[i]*(r-10) + ((1-fail_probs[i])*r +  DISCOUNT*(self.V[s])))\n",
    "            q_values.append(q_value)\n",
    "        return q_values\n",
    "\n",
    "\n",
    "    def optimal_policy(self):\n",
    "        for s0 in range(4):\n",
    "            for s1 in range(4):\n",
    "                max_q = 0\n",
    "                max_a = -1\n",
    "                for a in ACTIONS:\n",
    "                    next_states, probs, fail_probs, dones = self.environment.possible_consequences(action=a, state_now=(s0, s1))\n",
    "                    q_value = 0\n",
    "                    for i, s in enumerate(next_states):\n",
    "                        r = -1\n",
    "                        if dones[i]:\n",
    "                            r += 50\n",
    "                        q_value +=  probs[i] * (fail_probs[i]*(r-10) + ((1-fail_probs[i])*r +  DISCOUNT*(self.V[s])))\n",
    "                    if q_value > max_q:\n",
    "                        max_q = q_value\n",
    "                        max_a = a\n",
    "                self.policy[s0][s1] = max_a\n",
    "\n",
    "                \n",
    "    def value_iteration(self):\n",
    "        iter=0\n",
    "        while True:\n",
    "            iter+=1\n",
    "#             print(iter)\n",
    "            delta = 0\n",
    "            for s0 in range(4):\n",
    "                for s1 in range(4):\n",
    "                    v = self.V[s0][s1]\n",
    "                    self.V[s0][s1] = max(self.Q_value(s0, s1))\n",
    "                    delta = max(delta, abs(v - self.V[s0][s1]))\n",
    "            if delta < THETA:\n",
    "                break\n",
    "        self.optimal_policy()\n",
    "        print(self.policy)\n",
    "\n",
    "    def select_action(self, s0, s1):\n",
    "        selected_action = self.policy[s0][s1]\n",
    "        return selected_action\n",
    "\n",
    "    def take_action(self, s0, s1) -> (object, float, bool, object):\n",
    "        action = self.select_action(s0, s1)\n",
    "        obs, r, d, i = self.environment.step(action)\n",
    "        self.environment.render()\n",
    "        return obs, r, d, i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "dated-equilibrium",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2. 1. 1. 1.]\n",
      " [2. 2. 2. 1.]\n",
      " [2. 2. 2. 1.]\n",
      " [2. 2. 2. 1.]]\n",
      "\n",
      "------------------------------\n",
      "| 0.000 | \u001b[44m0.001\u001b[0m | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | 0.001 | 0.001 | 0.001 | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | 0.001 | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | 0.000 | \n",
      "------------------------------\n",
      "\n",
      "------------------------------\n",
      "| 0.000 | 0.001 | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | \u001b[44m0.001\u001b[0m | 0.001 | 0.001 | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | 0.001 | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | 0.000 | \n",
      "------------------------------\n",
      "\n",
      "------------------------------\n",
      "| 0.000 | 0.001 | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | 0.001 | \u001b[44m0.001\u001b[0m | 0.001 | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | 0.001 | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | 0.000 | \n",
      "------------------------------\n",
      "\n",
      "------------------------------\n",
      "| 0.000 | 0.001 | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | 0.001 | 0.001 | \u001b[44m0.001\u001b[0m | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | 0.001 | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | 0.000 | \n",
      "------------------------------\n",
      "\n",
      "------------------------------\n",
      "| 0.000 | 0.001 | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | 0.001 | 0.001 | \u001b[44m0.001\u001b[0m | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | 0.001 | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | 0.000 | \n",
      "------------------------------\n",
      "\n",
      "------------------------------\n",
      "| 0.000 | 0.001 | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | 0.001 | 0.001 | 0.001 | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | \u001b[44m0.001\u001b[0m | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | 0.000 | \n",
      "------------------------------\n",
      "\n",
      "------------------------------\n",
      "| 0.000 | 0.001 | 0.311 | 0.248 | \n",
      "------------------------------\n",
      "| 0.510 | 0.001 | 0.001 | 0.001 | \n",
      "------------------------------\n",
      "| 0.630 | 0.652 | 0.681 | 0.001 | \n",
      "------------------------------\n",
      "| 0.674 | 0.607 | 0.645 | \u001b[44m0.000\u001b[0m | \n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "environment = NSFrozenLake(studentNum=STUDENT_NUM)\n",
    "agent = Q1Agent(\"1\", environment)\n",
    "current_state = environment.reset()\n",
    "agent.value_iteration()\n",
    "done = 0\n",
    "while done == 0:\n",
    "    current_state, reward, done, information = agent.take_action(current_state[0], current_state[1])\n",
    "#     print(agent.environment.render())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amino-alexandria",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(observation, reward, done, info)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}