CoolNotz.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configururation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Enviornment Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENV_USE_FRAME_STACK = False\n",
    "ENV_FRAME_STACK_COUNT = 4\n",
    "ENV_USE_GRAYSCALE_OBSERVATION = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_USE_CNN = True\n",
    "MODEL_DEVICE = \"auto\" # \"auto\", \"cuda\", \"cpu\", or \"mps\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Traning Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAINING_NAME = \"TRAINING_NAME\"                     # Name for the current training (Used for checkpoint-name and log-name)\n",
    "TRAINING_EVAL_FREQUENCY = 5000                      # How often the agent should be evaluated\n",
    "TRAINING_EVAL_TIMES = 5                             # How many simulations to perform per evaluation\n",
    "\n",
    "TRAINING_TOTAL_TIMESTEPS = 600000                   # Total number timesteps before training is complete\n",
    "\n",
    "TRAINING_LEARNING_RATE = 1e-5                       # Learning rate\n",
    "TRAINING_LEARNING_STARTS_AT = 10000                 # Timesteps to perform before learing begins\n",
    "\n",
    "TRAINING_REPLAY_BUFFER_SIZE = 100000                # Number of episode-replays to save in the replay buffer\n",
    "TRAINING_BATCH_SIZE = 32                            # Number of replays to pick each gradient\n",
    "TRAINING_TARGET_UPDATE_INTERVAL = 1                 # How often the target network will be updated by online network\n",
    "TRAINING_GRADIENT_STEPS = -1                        # How many gradient steps shall be performed per training\n",
    "\n",
    "TRAINING_USE_PRIORITIZED_REPLAY = False             # Wether or not to use Prioritized Replay\n",
    "TRAINING_PRIORITIZED_REPLAY_EPS = 1e-5              # Epsilon of prioritized replay\n",
    "TRAINING_PRIORITIZED_REPLAY_BETA_START = 1.0        # Start beta of prioritized replay, beta controls the importance of prioritized replays\n",
    "TRAINING_PRIORITIZED_REPLAY_BETA_END = 0.1          # End beta-value\n",
    "TRAINING_PRIORITIZED_REPLAY_BETA_FRACTION = 0.55    # Fraction of total training time for beta-decrease\n",
    "\n",
    "TRAINING_EPS_START = 0.9                            # Start value for epsilon, epsilon controls exploration rate\n",
    "TRAINING_EPS_END = 0.0095                           # End value for epsilon\n",
    "TRAINING_EPS_FRACTION = 0.5                         # Fraction of total training time for epsilon-decrease\n",
    "\n",
    "TRAINING_GAMMA = 0.99                               # Static gamma, controls how importance uncertain rewards in far future (1.0) and the ones in the near future (0.0)\n",
    "TRAINING_TAU = 0.975                                # Static tau, controls how much of the online network shall be copied to the target network at each copy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup Minedojo Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Environments import SkyRunner, MultithreadGymWrapper\n",
    "\n",
    "# Multithreaded environment wrapper\n",
    "env = MultithreadGymWrapper.MultithreadGymWrapper(thread_int=1, env_int=1,\n",
    "    frame_stack=ENV_USE_FRAME_STACK,\n",
    "    frames_int=ENV_FRAME_STACK_COUNT,\n",
    "    use_grayscale=ENV_USE_GRAYSCALE_OBSERVATION\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load EVAL-Enviornment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluation environment\n",
    "eval_env = SkyRunner.SkyRunnerEnvironment(\n",
    "    frame_stack=ENV_USE_FRAME_STACK,\n",
    "    frames_int=ENV_FRAME_STACK_COUNT,\n",
    "    use_grayscale=ENV_USE_GRAYSCALE_OBSERVATION)\n",
    "eval_env.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Begin training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import Train\n",
    "\n",
    "Train.train(\n",
    "    env=eval_env,\n",
    "    eval_env=eval_env,\n",
    "    name=TRAINING_NAME,\n",
    "    eval_freq=TRAINING_EVAL_FREQUENCY,                  \n",
    "    n_eval_episodes=TRAINING_EVAL_TIMES,                           \n",
    "    total_timesteps=TRAINING_TOTAL_TIMESTEPS,                 \n",
    "    learning_rate=TRAINING_LEARNING_RATE,                     \n",
    "    learning_starts=0,\n",
    "    buffer_size=TRAINING_REPLAY_BUFFER_SIZE,              \n",
    "    batch_size=TRAINING_BATCH_SIZE,                          \n",
    "    target_update_interval=TRAINING_TARGET_UPDATE_INTERVAL,              \n",
    "    gradient_steps=TRAINING_GRADIENT_STEPS,                       \n",
    "    use_prioritized_replay=TRAINING_USE_PRIORITIZED_REPLAY,           \n",
    "    prioritized_replay_eps=TRAINING_PRIORITIZED_REPLAY_EPS,           \n",
    "    prioritized_replay_initial_beta=TRAINING_PRIORITIZED_REPLAY_BETA_START,      \n",
    "    prioritized_replay_final_beta=TRAINING_PRIORITIZED_REPLAY_BETA_END,        \n",
    "    prioritized_replay_beta_fraction=TRAINING_PRIORITIZED_REPLAY_BETA_FRACTION,  \n",
    "    exploration_initial_eps=TRAINING_EPS_START,                          \n",
    "    exploration_final_eps=TRAINING_EPS_END,                         \n",
    "    exploration_fraction=TRAINING_EPS_FRACTION ,                       \n",
    "    gamma=TRAINING_GAMMA,                            \n",
    "    tau=TRAINING_TAU,\n",
    "    use_cnn=MODEL_USE_CNN,\n",
    "    device=MODEL_DEVICE                               \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preview Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3 import DQN\n",
    "from CustomBaselines3.DoubleDQN import DoubleDQN\n",
    "\n",
    "model = DoubleDQN.load(\"./dDQN-checkpoints/\" + TRAINING_NAME + \"/final_model.zip\")\n",
    "\n",
    "obs = env.reset()\n",
    "acc_r = 0\n",
    "while True:\n",
    "    act, st = model.predict(obs, deterministic=True)\n",
    "    obs, reward, done, info = env.step(act)\n",
    "\n",
    "    acc_r += reward\n",
    "\n",
    "    env.render()\n",
    "\n",
    "    if done:\n",
    "        obs = env.reset()\n",
    "        print(\"Finished with reward %d\" % acc_r)\n",
    "        acc_r = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shutdown Environments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "env.close()\n",
    "eval_env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.6 ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "9100f79a7cf9690ce7cf79f8b2e62e57361e326a7400229bc21352d33c935312"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}