Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the following in Terminal to create a virtual environment for this tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Make virtual environment. Tested on python3.6+. Run in terminal, not in Jupyter.\n",
    "python3.7 -m venv tutorial-env\n",
    "source tutorial-env/bin/activate\n",
    "\n",
    "# Install COBS requirements\n",
    "pip3 install -r requirements.txt\n",
    "\n",
    "# point jupyter to virtualenv \n",
    "pip3 install ipykernel\n",
    "python -m ipykernel install --user --name=tutorial_env\n",
    "\n",
    "# Now, restart jupyter notebook and change the Kernel by selecting Kernel > Change kernel > tutorial_env"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the first tutorial, we choose to run an experiment where we increase the size of the dataset to see how sample efficient the methods are. The environment is the Graph environment with a horizon of 4 and a tabular function class for the Q functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import json\n",
    "from copy import deepcopy\n",
    "\n",
    "from ope.envs.graph import Graph\n",
    "from ope.policies.basics import BasicPolicy\n",
    "\n",
    "from ope.experiment_tools.experiment import ExperimentRunner, analysis\n",
    "from ope.experiment_tools.config import Config\n",
    "from ope.experiment_tools.factory import setup_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get configuration\n",
    "configuration_filename = \"tabular_example_cfg.json\"\n",
    "with open('cfgs/{0}'.format(configuration_filename), 'r') as f:\n",
    "    param = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = setup_params(param) # Setup parameters\n",
    "runner = ExperimentRunner() # Instantiate a runner for an experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run 5 experiments, each with a varying number of trajectories\n",
    "for N in range(5):\n",
    "    \n",
    "    configuration = deepcopy(param['experiment']) # Make sure to deepcopy as to never change original\n",
    "    configuration['num_traj'] = 8*2**N # Increase dataset size\n",
    "\n",
    "    # store these credentials in an object\n",
    "    cfg = Config(configuration)\n",
    "\n",
    "    # initialize environment with the parameters from the config file.\n",
    "    # If you'd like to use a different environment, swap this line\n",
    "    env = Graph(make_pomdp=cfg.is_pomdp,\n",
    "                number_of_pomdp_states=cfg.pomdp_horizon,\n",
    "                transitions_deterministic=not cfg.stochastic_env,\n",
    "                max_length=cfg.horizon,\n",
    "                sparse_rewards=cfg.sparse_rewards,\n",
    "                stochastic_rewards=cfg.stochastic_rewards)\n",
    "\n",
    "    # set seed for the experiment\n",
    "    np.random.seed(cfg.seed)\n",
    "\n",
    "    # processor processes the state for storage,  {(processor(x), a, r, processor(x'), done)}\n",
    "    processor = lambda x: x\n",
    "\n",
    "    # absorbing state for padding if episode ends before horizon is reached. This is environment dependent.\n",
    "    absorbing_state = processor(np.array([env.n_dim - 1]))\n",
    "\n",
    "    # Setup policies. BasicPolicy takes the form [P(a=0), P(a=1), ..., P(a=n)]\n",
    "    # For different policies, swap in here\n",
    "    actions = [0, 1]\n",
    "    pi_e = BasicPolicy(\n",
    "        actions, [max(.001, cfg.eval_policy), 1 - max(.001, cfg.eval_policy)])\n",
    "    pi_b = BasicPolicy(\n",
    "        actions, [max(.001, cfg.base_policy), 1 - max(.001, cfg.base_policy)])\n",
    "\n",
    "    # add env, policies, absorbing state and processor\n",
    "    cfg.add({\n",
    "        'env': env,\n",
    "        'pi_e': pi_e,\n",
    "        'pi_b': pi_b,\n",
    "        'processor': processor,\n",
    "        'absorbing_state': absorbing_state\n",
    "    })\n",
    "    cfg.add({'models': param['models']})\n",
    "\n",
    "\n",
    "    # Add the configuration\n",
    "    runner.add(cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Run the experiments\n",
    "results = runner.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analyze the results\n",
    "# Each row in the result is (OPE estimator, V(pi_e), MSE Error from on-policy: (V(pi_e) - True)**2)\n",
    "for num, result in enumerate(results):\n",
    "    print('Result Experiment %s' % (num+1))\n",
    "    analysis(result)\n",
    "    print('*'*20)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the second tutorial, we choose to run the same experiment as before but with a different environment and different Q function class. The environment is the Pixel-Gridworld (Pix-GW) environment with a horizon of 5 and a NN function class for the Q functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ope.envs.gridworld import Gridworld\n",
    "from ope.policies.epsilon_greedy_policy import EGreedyPolicy\n",
    "from ope.policies.tabular_model import TabularPolicy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get configuration\n",
    "configuration_filename = \"nn_example_cfg.json\"\n",
    "with open('cfgs/{0}'.format(configuration_filename), 'r') as f:\n",
    "    param = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = setup_params(param)\n",
    "runner = ExperimentRunner() # make new runner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for N in range(5):\n",
    "    configuration = deepcopy(param['experiment']) # Make sure to deepcopy as to never change original\n",
    "    configuration['num_traj'] = 8*2**N # Increase dataset size\n",
    "\n",
    "    # store these credentials in an object\n",
    "    cfg = Config(configuration)\n",
    "\n",
    "    # initialize environment with the parameters from the config file.\n",
    "    env = Gridworld(slippage=.2*cfg.stochastic_env)\n",
    "\n",
    "    # Set seed and \n",
    "    np.random.seed(cfg.seed)\n",
    "    eval_policy = cfg.eval_policy\n",
    "    base_policy = cfg.base_policy\n",
    "\n",
    "    # to_grid and from_grid are particular to Gridworld\n",
    "    # These functions are special to convert an index in a grid to an 'image'\n",
    "    def to_grid(x, gridsize=[8, 8]):\n",
    "        x = x.reshape(-1)\n",
    "        x = x[0]\n",
    "        out = np.zeros(gridsize)\n",
    "        if x >= 64:\n",
    "            return out\n",
    "        else:\n",
    "            out[x//gridsize[0], x%gridsize[1]] = 1.\n",
    "        return out\n",
    "\n",
    "    # This function takes an 'image' and returns the position in the grid\n",
    "    def from_grid(x, gridsize=[8, 8]):\n",
    "        if len(x.shape) == 3:\n",
    "            if np.sum(x) == 0:\n",
    "                x = np.array([gridsize[0] * gridsize[1]])\n",
    "            else:\n",
    "                x = np.array([np.argmax(x.reshape(-1))])\n",
    "        return x\n",
    "\n",
    "    # processor processes the state for storage,  {(processor(x), a, r, processor(x'), done)}\n",
    "    processor = lambda x: x\n",
    "    \n",
    "    # Set up e-greedy policy using epsilon-optimal\n",
    "    policy = env.best_policy()\n",
    "    \n",
    "    # absorbing state for padding if episode ends before horizon is reached. This is environment dependent.\n",
    "    absorbing_state = processor(np.array([len(policy)]))\n",
    "\n",
    "    # Setup policies.\n",
    "    pi_e = EGreedyPolicy(model=TabularPolicy(policy, absorbing=absorbing_state), processor=from_grid, prob_deviation=eval_policy, action_space_dim=env.n_actions)\n",
    "    pi_b = EGreedyPolicy(model=TabularPolicy(policy, absorbing=absorbing_state), processor=from_grid, prob_deviation=base_policy, action_space_dim=env.n_actions)\n",
    "\n",
    "    cfg.add({\n",
    "        'env': env,\n",
    "        'pi_e': pi_e,\n",
    "        'pi_b': pi_b,\n",
    "        'processor': processor,\n",
    "        'absorbing_state': absorbing_state,\n",
    "        'convert_from_int_to_img': to_grid, # if environment state is an int, can convert to image through this function\n",
    "    })\n",
    "    cfg.add({'models': param['models']})\n",
    "\n",
    "    runner.add(cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run the experiments\n",
    "results = runner.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analyze the results\n",
    "# Each row in the result is (OPE estimator, V(pi_e), MSE Error from on-policy: (V(pi_e) - True)**2)\n",
    "for num, result in enumerate(results):\n",
    "    print('Result Experiment %s' % (num+1))\n",
    "    analysis(result)\n",
    "    print('*'*20)\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tutorial_env",
   "language": "python",
   "name": "tutorial_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}