InteraccionLeyendoPantalla_No_LSTM.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit ('Master': conda)",
   "metadata": {
    "interpreter": {
     "hash": "57d46a1f3f975f92cc34d815bf69a7d3644582cc16f1cedc66cb95f17202c91e"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyautogui\n",
    "import numpy as np\n",
    "import time\n",
    "import pydirectinput\n",
    "import cv2\n",
    "import mss\n",
    "import time\n",
    "import pywinauto\n",
    "import win32gui\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "from KeyPress import PressKey, ReleaseKey\n",
    "from collections import deque"
   ]
  },
  {
   "source": [
    "# Cargamos el modelo"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_dict = {\"up\": \"x\", \"down\": \"b\", \"left\": \"y\", \"right\": \"a\", \"none\": \"\"}\n",
    "keys_dict_di = {\"up\": 0x2D, \"down\": 0x30, \"left\": 0x15, \"right\": 0x1E, \"none\": 0x2D}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['down', 'none', 'right']"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "classes = ['down', 'none', 'right']\n",
    "classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Primero hay que definir el modelo\n",
    "model_custom = nn.Sequential(*[\n",
    "    nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding = 1, stride = 1),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding = 1, stride = 2),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding = 1, stride = 2),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding = 1, stride = 2),\n",
    "    nn.ReLU(),\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(132608, 3)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "device = device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\vila_\\anaconda3\\envs\\Master\\lib\\site-packages\\torch\\nn\\modules\\rnn.py:58: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "TemporalModel(\n",
       "  (convolutions): Sequential(\n",
       "    (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (3): ReLU()\n",
       "    (4): MaxPool2d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
       "    (5): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (6): ReLU()\n",
       "    (7): MaxPool2d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
       "    (8): Flatten(start_dim=1, end_dim=-1)\n",
       "    (9): Linear(in_features=16128, out_features=100, bias=True)\n",
       "    (10): ReLU()\n",
       "  )\n",
       "  (lstm): LSTM(100, 50, batch_first=True, dropout=0.5)\n",
       "  (final_linear): Linear(in_features=50, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "model_custom = TemporalModel()\n",
    "#Cargamos los pesos del modelo entrenado\n",
    "model_custom.load_state_dict(torch.load(\"ModeloCustomEtiquetadoFrames_DELAY.pth\"))\n",
    "model_custom.eval()\n",
    "model_custom.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_action(model, img, classes):\n",
    "    \"\"\"\n",
    "    Predecimos la clase correspondiente para una imagen dada con el modelo deseado.\n",
    "    El preprocesado de la imagen se hace dentro de la función, no fuera.\n",
    "\n",
    "    Args:\n",
    "    model -> Modelo que queremos usar para hacer la predicción\n",
    "    img -> Imagen cuya acción queremos predecir\n",
    "    classes -> Lista de las posibles clases\n",
    "\n",
    "    Returns:\n",
    "    Acción en formato str\n",
    "    \"\"\"\n",
    "    #La pasamos a RGB\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGB)\n",
    "    #Recortamos solo la sección que nos interesa\n",
    "    img = img[49:-12,20:-20]\n",
    "    #Ajustamos la imagen al tamaño que espera la red\n",
    "    img = cv2.resize(img, dsize = (293,224))\n",
    "    #Transformamos la imagen a Tensor\n",
    "    img = torchvision.transforms.ToTensor()(img)\n",
    "    img = img.to(device)\n",
    "    with torch.no_grad():\n",
    "        pred = model(torch.unsqueeze(img, dim = 0))\n",
    "    return classes[torch.softmax(pred, dim = 1).argmax(dim = 1)]"
   ]
  },
  {
   "source": [
    "# Interacción con la pantalla"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(956, 269, 1499, 776)"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "#Left, Top, Right, Bottom\n",
    "window_coords = win32gui.GetWindowRect(pywinauto.findwindows.find_window(title_re = \"Super Mario\"))\n",
    "window_coords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "left, top, right, bottom = window_coords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "width = right - left\n",
    "height = bottom - top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# font \n",
    "font = cv2.FONT_HERSHEY_SIMPLEX \n",
    "  \n",
    "# org \n",
    "org = (50, 50) \n",
    "  \n",
    "# fontScale \n",
    "fontScale = 1\n",
    "   \n",
    "# Blue color in BGR \n",
    "color = (255, 0, 0) \n",
    "  \n",
    "# Line thickness of 2 px \n",
    "thickness = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_dict_di = {\"up\": 0x2D, \"down\": 0x30, \"left\": 0x15, \"right\": 0x1E, \"none\": 0x2D}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "time.sleep(2)\n",
    "\n",
    "with mss.mss() as  sct:\n",
    "    monitor = {\"top\": top, \"left\": left, \"width\": width, \"height\": height}\n",
    "\n",
    "    while \"Screen Recording\": #Aquí da igual lo que pongas porque una string es como si fuera un True\n",
    "        last_time = time.time()\n",
    "        #Esto coge las imágenes en BGRA\n",
    "        sct_img = sct.grab(monitor)\n",
    "        img = np.array(sct_img)\n",
    "        \n",
    "        #Predecimos la acción con nuestro modelo\n",
    "        action = predict_action(model_custom, img, classes)\n",
    "\n",
    "        # pydirectinput.press(keys_dict[action], _pause = False)\n",
    "\n",
    "        text = \"Action: {} fps: {}\".format(action, 1 / (time.time() - last_time))\n",
    "        tries = 0\n",
    "        if action != \"none\":\n",
    "            tries = 4\n",
    "            for i in range(tries):\n",
    "                PressKey(keys_dict_di[action])\n",
    "                time.sleep(0.001)\n",
    "                ReleaseKey(keys_dict_di[action])\n",
    "\n",
    "        img = cv2.putText(img, text, org, font,  \n",
    "                   fontScale, color, thickness, cv2.LINE_AA) \n",
    "\n",
    "        cv2.imshow(\"Cuac\", img)\n",
    "\n",
    "        # Press \"q\" to quit\n",
    "        if cv2.waitKey(25) & 0xFF == ord(\"q\"):\n",
    "            cv2.destroyAllWindows()\n",
    "            break"
   ]
  }
 ]
}