Frame-Level-Speech-Recognition.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F9ERgBpbcMmB"
      },
      "source": [
        "# Frame-Level Speech Recognition"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CLkH6GMGcWcE"
      },
      "source": [
        "We will be working with MFCC data consisting of 27 features at each time step/frame. Our model should be able to recognize the phoneme occured in that frame."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z4vZbDmJvMp1"
      },
      "source": [
        "# Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rwYu9sSUnSho"
      },
      "outputs": [],
      "source": [
        "!pip install torchsummaryX wandb --quiet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qI4qfx7tiBZt",
        "outputId": "c418076e-db6d-4ab1-9ff0-d75fa8906b6a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Device:  cuda\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "from torchsummaryX import summary\n",
        "import sklearn\n",
        "import gc\n",
        "import zipfile\n",
        "import pandas as pd\n",
        "from tqdm.auto import tqdm\n",
        "import os\n",
        "import datetime\n",
        "import wandb\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "print(\"Device: \", device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8yBgXjKV1O0Z"
      },
      "outputs": [],
      "source": [
        "### If you are using colab, you can import google drive to save model checkpoints in a folder\n",
        "#from google.colab import drive\n",
        "#drive.mount('/content/idl/models/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N-9qE20hmCgQ"
      },
      "outputs": [],
      "source": [
        "### PHONEME LIST\n",
        "PHONEMES = [\n",
        "            '[SIL]',   'AA',    'AE',    'AH',    'AO',    'AW',    'AY',\n",
        "            'B',     'CH',    'D',     'DH',    'EH',    'ER',    'EY',\n",
        "            'F',     'G',     'HH',    'IH',    'IY',    'JH',    'K',\n",
        "            'L',     'M',     'N',     'NG',    'OW',    'OY',    'P',\n",
        "            'R',     'S',     'SH',    'T',     'TH',    'UH',    'UW',\n",
        "            'V',     'W',     'Y',     'Z',     'ZH',    '[SOS]', '[EOS]']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZIi0Big7vPa9"
      },
      "source": [
        "# Kaggle"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBCbeRhixGM7"
      },
      "source": [
        "This section contains code that helps you install kaggle's API, creating kaggle.json with you username and API key details. Make sure to input those in the given code to ensure you can download data from the competition successfully."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TPBUd7Cnl-Rx",
        "outputId": "35216198-04f0-49aa-c46a-a9f0d7b4bb08"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting kaggle==1.5.8\n",
            "  Downloading kaggle-1.5.8.tar.gz (59 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/59.2 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.2/59.2 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: kaggle\n",
            "  Building wheel for kaggle (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kaggle: filename=kaggle-1.5.8-py3-none-any.whl size=73248 sha256=a2704ddce65997212cbca09daac32d59d52a8dded8698b9ca1b754a3385fc83a\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/76/ca/e58f8afa83166a0e68f0d5cd2e7f99d260bdc40e35da080eee\n",
            "Successfully built kaggle\n",
            "Installing collected packages: kaggle\n",
            "  Attempting uninstall: kaggle\n",
            "    Found existing installation: kaggle 1.5.16\n",
            "    Uninstalling kaggle-1.5.16:\n",
            "      Successfully uninstalled kaggle-1.5.16\n",
            "Successfully installed kaggle-1.5.8\n"
          ]
        }
      ],
      "source": [
        "!pip install --upgrade --force-reinstall --no-deps kaggle==1.5.8\n",
        "!mkdir /root/.kaggle\n",
        "\n",
        "with open(\"/root/.kaggle/kaggle.json\", \"w+\") as f:\n",
        "    f.write('{\"username\":\"\",\"key\":\"\"}')\n",
        "    # Put your kaggle username & key here\n",
        "\n",
        "!chmod 600 /root/.kaggle/kaggle.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "if2Somqfbje1",
        "outputId": "98d059d4-81cf-46ba-aa34-2fadb435bf59"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading 11785-hw1p2-s24.zip to /content\n",
            "100% 3.73G/3.75G [00:15<00:00, 269MB/s]\n",
            "100% 3.75G/3.75G [00:15<00:00, 266MB/s]\n"
          ]
        }
      ],
      "source": [
        "# commands to download data from kaggle\n",
        "!kaggle competitions download -c 11785-hw1p2-s24\n",
        "\n",
        "!unzip -qo /content/11785-hw1p2-s24.zip -d '/content'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vuzce0_TdcaR"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XTnK9VLlWDws",
        "outputId": "75bb8184-c4e3-480d-9688-693b8ba9f278"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2703\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "files = os.listdir('../content/11-785-s24-hw1p2/dev-clean/mfcc')\n",
        "print(f'{len(files)}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oDHs1rfjZXMt",
        "outputId": "6813c9d2-1a16-48ad-f69d-6215d74bc0d4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(135,)\n",
            "(915, 27)\n",
            "[33]\n"
          ]
        }
      ],
      "source": [
        "mfcc = '../content/11-785-s24-hw1p2/dev-clean/mfcc/' + files[1]\n",
        "tran = '../content/11-785-s24-hw1p2/dev-clean/transcript/' + files[1]\n",
        "vect = np.load(mfcc)\n",
        "dat_pad = np.pad(vect, ((5, 5), (0, 0)), mode='constant', constant_values=0)\n",
        "print(dat_pad[-5:].flatten().shape)\n",
        "print(vect.shape)\n",
        "trans = np.load(tran)\n",
        "print(np.where(np.array(PHONEMES) == 'UH')[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "AT717MyjleXC",
        "outputId": "bc966417-b614-4ea5-8be0-ce8dce47a3b2"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAA/CAYAAABAbqrOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzjUlEQVR4nO29a7BlxVk//Ote1307lznDmcMQhoxJ3kKFJMhlJFjReqESE14VtCxNESXRikUcIgQLJSrJBwuhzJdoTCVqleQDIIqvEaUSLN7BoJTclRASJReSP4jMhTlzztmXdevu5/3wdPfae2a4TAIzQ6Z/VbvOOWv1XqtX93P59fM8vY4gIkJAQEBAQEBAwFGCPNYdCAgICAgICDixEMhHQEBAQEBAwFFFIB8BAQEBAQEBRxWBfAQEBAQEBAQcVQTyERAQEBAQEHBUEchHQEBAQEBAwFFFIB8BAQEBAQEBRxWBfAQEBAQEBAQcVQTyERAQEBAQEHBUEchHQEBAQEBAwFHFEZGPG2+8Eeeeey4GgwGWl5dxySWX4Kmnnppp81M/9VMQQsx8rrjiile10wEBAQEBAQGvXxwR+bjvvvuwc+dOPPjgg7jnnnvQNA3e9a53YTwez7T70Ic+hOeff95//viP//hV7XRAQEBAQEDA6xfxkTS+++67Z/7+/Oc/j+XlZTz22GN45zvf6Y93u12srKy8Oj0MCAgICAgI+IHCEZGPg7G+vg4A2LRp08zxW2+9FbfccgtWVlbwMz/zM7j++uvR7XYPe42qqlBVlf/bGIPV1VUsLS1BCPH9dC8gICAgICDgKIGIMBwOsXXrVkj5MokV+h6htaaLL76YLrjggpnjf/7nf0533303PfHEE3TLLbfQKaecQpdeeumLXucTn/gEAQif8Amf8Amf8AmfH4DPs88++7IcQhAR4XvAhz/8YXzpS1/C/fffjze84Q0v2u7ee+/FhRdeiG9961t405vedMj5gyMf6+vr2LZtG/71/wb68bGJfLiIy/c4NC/6/Vdy3cNFe76X6xxJu1eK6esJIWZ+vtb3flm42xyByByu7y/2PC96z+/zft/vd49knI/6nJzACGP9/cn7D8L9vx98v31/Ofv8WmGkCO+8F1hbW8P8/PxLtv2e0i5XXnkl7rrrLvzrv/7rSxIPANixYwcAvCj5yLIMWZYdcrwfC8ylbdjGDeSL/Q0B74Beykgf3M41lVL4v4Vw7ae+A/7ewdee/lPKWYNDJCAEt+Fj5HcATfeJiGCMOzfbZ35O+SIEpBUwiLY9GWrbCEAIHkcydMgzHjKOUzD+Ou1zEhGkFBByanxIwNhrz4z11OAdfG76/i8F/0wvoz+HewT3/DjMd9v7Cj8/0/NyMNFy1zm4L9Njc/DvB48rj6dgOZm6nu/ri4BlAxBS+r4ZY/w883zavjsZNHTo/AonAwJkCDJq7zk9zi/2TIcdw8MQsIPneLq9kAJG01Tb2e/4dlOy8XJkfvrci9qIg3R4Vs+ndfYw/TqcrE7/epBMsI74Xs70edreHPxMTt/89QW3O6QPNDtG/lmkmPkumVm7crBsTI/Jwc8xPbdsG9s+Tz/S9LVoqv9CCoAAY9q20zo/oydT/WrbHjo+/vmm+tyePOiYmPpl6pmmr+vG7XCYuRcAKaX9zuFlYBqHnZtpezI1T9PXm27r2jhdN5pau3HQvQ++Bn9vehBm9fzgPh52rF/inLuP8wXT3xGCAJhXVDJxROSDiPCRj3wEX/jCF/DlL38Z27dvf9nvPP744wCAk08++Uhu5e8nrPLRlMFiQYcXBikFxJSTM4YQxfIQ5eMGgIwktNIwph08Ywja3kMKwNjBlhJoNBAJ/t0rshBTTsE6BG28AXFKp+zPyBqjNGkNidEsVE5RlSFE05xHzJIW//gHzavxTsh4YyUjCWOMF3Ct23MAwWiCIbIyTDAEJBH8uHnCRAAJIIqkVyaluE9uPA4mMHEScb+08d/x46zJz58hguMwxgDa9sFdU0pLagxBE4/hjLAfZtyFEIjilmwZTYclOq69lG17Jz9ObqYdxLTDdIrfGCsXVgZcG+fUPSGbdtAENJrHPXLjZwiaaOZvIYW/v5QCTWNA4Dbu/u4+045SKwNt3FixkTDawBD3U0aWEAt42XMy1j4vP3MUyfZ8y9X8vVoDe3iy75yJ1sbKJB8jw/IWx2LGyAshvKMzxkBYcq2VsXN9EEmbmleBVq/ctXgOmKQ5+6CV8Q7VGJphVoZYBqc4GaLIycWsQXXf9zZjal7EtLMWrj3ro7TyqWrNYxxHEIJlRUbttd0cONmrGkIsgShqbZmTL2eLooNIJuspC10cC9+HacfmZE012tssT5z8+PL1tCWsLpfv9CRy8zplM6DpsHPWkjyCMVZ3qCVZ2hlM4RgD2vnSNGNrZvV51j47/3CwfTLGzI6vI0eO6Emrj7YfTm610u3zuX4B/hmNJjSmJRCRaOWCDKDJ2lhpBQ1sJwWEXxS6BbAxBOH6Lsj2AdBNq8czsj8tl1Grl84ekaEp/jFrA/las+Pk5JFlm9tEifT6LK0vALUEi+3FyxOOaRwR+di5cyduu+023HnnnRgMBti9ezcAYH5+Hp1OB9/+9rdx22234b3vfS+WlpbwxBNP4KMf/Sje+c534q1vfesRdcwbN0PQ1olCtIbcOUMp5WFXpW7gpwfTQSuFJIthdd07SYDJggHaSZ5yMHESTU2smSE0xhg0ilBpbptI/p7XfwPEEl4InJANGxbUWPB9IVu9cw4eYAGKEunvrxUbwziJphwmMKkJiSTACqobjzQG4jQGLBGBId5nbYVYW2btDArAY+zuIyzBE4IQRa3hIyLUCoglEwQAGI8VBIBY8DNbbsWGKxKW+PBYN9xVJBJIPdGbVlxWnDSWaBoDbQjaGoxYkvfrTBinnIXrn+E2jug45Zz2YarRaDT3Q9t5FwC6MRtbY1pDCcFttAFGip9rEAMppoyPJUvGiquxDi23fXPjFFnCKVwbAFpbomX77mQ9jtt+AECSyJYgWePnDH8kmUR6WTHk5Toh8s61MQBpS3rApKi0RJsAiMZ4OXZO2REGrQyUJiRxS7SMAQpt+yfYCCdOfkHQql0lSwCjikAg5BE86XaGdHYhAEhDiLPYG3syVo7BRtmTBsFzo00btYkkoA3B+ZlOws9gFHlTHAl27IkU0Nq0823REiN26sqOWxTxca2nSI0m1BoQgjyZ1o3y19LKIIolmsqgUawrzuemEVBrthuxaP9uiB1sRIQ05jFwsgKQH+emMZ4ME+Dns6wIca1gwGOvCMiiVneiRCKRvChzixenG1XDBF4KoK4JBtq/o4FNCtuMRrfjbQiQVtaiuI2MlorvH8tWN2IBP5dRInmV74SUphaUkbRkmjxRlAKotL2WAqSg9pns4o1JG1+uVjwvbm6TWNjFKNvvSJAffwLQGIIUbNc1Ab0YyCT8eDQNeR3JUwmtmfwrK3+wcqSt/klysiZn7Li0SicjCaO5L2bKnzkdTRK+hx8Xa0Ngx7RoWCacjXHJg8b2oxMBSSohpYRqtLcZ2vtHJqrC6foUKdGOuFnCSiAfEdIN8QLp0HXIi+KIaj5eLJRy88034wMf+ACeffZZvP/978eTTz6J8XiMU089FZdeein+4A/+AHNzc6/oHhsbG5ifn8dX3hujF/HKwJEDR0iaWiFJY1bgWkFK6Y2SVjxAWhmUtUGhecCzhBUgzWJobdDUBnEsEKcxmkphXBMKzZPVjVtDZqYESVP7YhSD1lk5w7eYt06NqDWgpRVmZQWgZylfbdhIRxKYKOsMwNdKJF+30vy7E3Bp7w3wd9QUqe3GQC8BJg0r9FoDz7bzqFXSRLZGRxOwXgtkEXknM5e095EAXqiBWgvbL0I3Bvpx+/04EkjSCHWlsFbxsdT11638AAwVsFoLfmYBZBGhH3O/hRR+DoDZ6GJqDZUybf+ny4FiAeQpH5jUPOa9zK7OrHhXDV/bkQBtiY8zYm6cpAD6KZM61WiQYWOURjyPpW7bNpZQDhsBQ/w8qSVRecTkyTnCif3uIAHmOnJmlecMj9YGtWrH3clZqYFuBHQ7kSdVxhBWS57/SPA9M9nKEuw45fHsiq6syRPiLGqddBJxO6Vmo1FJKpFmMaIkglYG1aRGZQ1cHrOsKeI+CvBzO4MylwJJGqMslZe9UrP8l06uBc+Dg2u3mFq5sYSxagj7q1amDAGFFkglYVPK8+P0MZXcbxBQNwZrNZ9z86KJx8oFvGrjrtfqpotq9VKBWtFMRKQxfDyK5SHRztrw+MeSnx9gQu30wMkdrJ7vr1h28oj82EWizSKUuu3zsBHeqbj2yrS2YWDlqqr0jP7EtuOFdZQCs2RSKcJaDYwUt4slYRC34+x0L7EyVhoegzwC5nKJYWlwoBZYytgZurl1cyVhCbHkBVxdaewteawT2Y6/I72RmCXsemq8atMuVpxDzaPZKKKT/zxqbZSTq44ds8rwsyiry5klpE1tMFZ8rteNEaetzgFMxoxuFwZxEnn9qooGSRqDiDApNBRZX5JIHy2ajrjHVqe0YvLs5kxbWXR2xvECp88sd5YQWrJS1jSjR4klroaAseKxJvDvEwWclLdj5mQjEjwOcRLZSHkr3y4i7qKYwFT0EG0ka6QI5+7i2s2X8/nfc8HpawVHPv7jXQKDRKCsecXimP30KqGTs4FxRtxNoJTMLA8UfHyxwxpX1waVBrrJVO7LtIZlrKzhkkDaSaxwsDfUyviJkFIgyWK74iHoRkPaEHWax9CNxnhYI0mlD5UBfH2nAM6QpVnkUwdEhKybelbsQvnVpJ5Jm/QXOjZawSHlplYgAuqiAQSQd1MOpdYaRhvv1PK0VTDCrDFQBuhkEnVj/GpWKUKSSCR2xVlNai/kAqxYADsHF3prNJDGAlkn8eM2HT52zDyP2rmsTEsypg2FIWDJlgNFluBETtltqFkrDVVrNI1BFLUOAQC6g5xXqo2GVgZxEqGpFMp6KhGNNlTqVihu1elWcKWNigwSbjvRbaRGCmDY8LlIAGs1H48FP9diL4KMJOqi8SuqXsryoxUbsiSN7LO00YTYkmsAULXG+lhho+ExTyU7LU1AJyLMJy354vmQiBO+ZjlpkCQScdpG7ZpK8f14se6jKuWU0W6MJT3xlGGnNuoU2dV1ksUtoa8UDoy1dwilJfNpzE5hug9aGYwtUcymiDEADLI2HZOkLGQuxalq7c+pRmNsCbYjE4SWyDvH07EOrDHAQgpkeQzVaI5eiPZZ5zKOYowrg4niZ+ilAsqG1Psdlj3VaChFbdqIgLI22HB9sePXGB6/uYQNV62mIhox2nQcsQxlsnUYI9USoFS2xMKZa6M5+uJsoif5UUucX5iQ16vazJL4POLPAUvMejH/ncbCyo7x9yFD6C90oLXBxnrloxupXUE3teKojDVqzg4kNtIbxRJRHAFEaOp2te36CsFpjY3CoBMBeTdBdy6DqjXqUvm0Rlkqn3p1qSAhOWqQd1Nvn4mAYtJ4We1k1obZMSpGNc9TzrIlLBGQkhe2VdH4iLKU0uthFEvkvRRaG5SjGlXRoNPPfIRNNRrK2mIhOLJX2ohJbURLaGNCobkzp3TZsbv0JtdisQ2b1ISxYnnoxizHB6x9GbB5xQabfAzSNvWUZjFkJNFUCkVlkCV8L605NZJ1EshIohhVM2nDrJO0CyIpUFcKtbUJWSJQNEx4s24C1bDCuiioW0BtVAY/dg+9vsnH4++JsKkXoxjXSPPEOhZWqv3rNRYHic3NMTlw+etaMbvvDDKMNyp29p2YIxE2pFjWHB5KY/i8thACWTdBMay8c+r3EzbUmg2nqjUmyqc00Y/ZcEdxhKZW2F8CE8VRhJESPiqirXJvn+MJ3tiosd6wkdq63EGcRoABykmNclJDGw4HCsG5fuc8ophjh6pWqCuFUrVRjNgajfFEoTEslC6K4wxwfz4HAMSJhGoMlJUsF8ZjRZaeQLl6mbybcjjUrQAMMBmWPq7dnc+hGyZCddlMEQ5AaQ5Z1nZVupAC3V7iaxHqip1V1k2sgRBIOzGSNEZTq5mwK5MpvodT0ijmftVFAyEFuoMMaSdBNWmwb7X0zi/J+JnYGEa+1kNI4dNLbukRRRzaLMc1iAhpnswQn2JUo6kUsk5ia1IMophXC3tXKwxSjgh8dyw4GiGApYywNGA5bCqFOI3Y4EXSGz0iIM1jlOPa909GswZvMqqRJDxH7lpFZfBCDZRaYKw4SnJKl7Bgl97lpEatgNW6jaospEAvl17+AWA0agCwIR0pYD4F+oPU1mAwaSka8s4qSSUaG13sxfAGdMyXQSqB+YUcSRpBRFy39cLeMQptU3J2dQYBlMqudDPpjaeMpDXoTDiyTjKbv9ecvqhLhXJSY9y0kY9eJv2CwJG7OOEoalNrm0IFGkvQ3eo1zdkA8wqXFxwykqgq7e1IbHXBOSnnnNxcpp3E18+U4xp1qdDUCo3m0HycRlwjUCm/YMpy6xytTjldK0qDiW5X/4sp0JvLoJXB7rUGpQHmk5aouEVNP+OFy7AB5nPhbQfQ1ggZQ+gOMshYoikVsm5i7Qvrvktda6XRVMrLybg0nrARgIUE6PdiQAhEES/IVK2gFCHLY39dIQSGqxPsLth5ugiMiwZ3+imqovFpK4CJ/lwC9AapXfBJv+hyqRq3KErjNj3m6l7KmnzUJbKppCiJmCjZRZ9bnLj0x7AwqA0wtjbcRWFqG8GcKIFuTMhkm3pdyYGTMkK/n7A8EHFqP2G/Eac8/nXRoCoaH8HfV7SRsH7M9+rFfK+5DusCEZjgabJETvqFb5JGXk+qCftK90ysL2jrcgielKyPFbKojYrlvZR1qWh81GO2kJUO6cvBJNIYwtgIvPWu5vVPPvoRryiWTuoi7SSoiwZEQFU00Ep7YXIK6yahrjQ6PWZ3QghMNkpsNMB8xs7JRQ20MigmDfI8RprHGCz1MF4rUBUNUrvajxIJo3gy0m6CKEkg8w6oqTBZHUFE7NjiPIOIYtTDod2VAJSjGjLm39NOCqM0JsMKTaU8ey8nNSLrRDv9zPfLKUKcRlCV5vRSxELtCvCIgCRtizuzTorupj7qcQEZSWSblkBKgYyBGg+x+vyQC84EG/Iolj7sVxUNoliiP9/BaL1AXWn05jIUowpPD1lAV3JOuUSxRH+hA9VolOMaxpBn00Rs0JwBc6tsp5DjYc2h95SdQt5LIWPpc0lpJ0aUpjCqYWc3rLCxUaPQvGLT1EZJXPQhSwSPnTIoJzUAWAXl/iRpBNUYlKVCp5sg7yaQUiLJIwgprUI2XFvRydFMChTDGpNhyRElRZibS5F1mASkORuV4f4xilHtw5pRIjEuDbqpQNZNMX9SH8JWEcq8AyEldDEBqQbFkLeXk10N1kWD7hyTQ1UrFOMak6atH1pY6qK32ENTVtDKYHDyFsi8g2rfHjRlhSTPIOMIEBKTAxsYrk7Q1Aa9QYr+YocLG6MY2UlbEHX7EJ0B9MZ+mLJAM1yHGo85elEyMUqymJ36uEbeS73DSjspICSK9Qm0MujO5z7SFMXS53zzzcsQSYJ6/z6QMYDRXk+drBER6kKhLhsv202tMSo0hk1rExyhGCuBSLLRd2m9LAJ6/RRZJ0GSx76YXFUaxbhGXTQzRXoAkOZMDtI8RpLH3jHrRqOaNMh6fL4pNepKoSpqH510EY+sm3qyDQB5L0N/IYeMIzRlw8a6m6Mal6gmDfJegqzXgaoqTDZ4DnsLdr4r7SMCjvh2+uwMojRlexLHUJMJ1vYMMRorpDGw5bRFxN0eRBSjGa7zfI0q1KVC2knQm8+RdHKosgQAlOPG16040jUcNphoYCGDj1YW4walJYiDQYKl05Y5nbMx5HRBJ0fU7UOXBYZ7DvDCoZODVIN4MA8RxzD29QnNaIjh/rEnf925zOq38vPtIgcujN+UCnESIe0mSPoDqPEQWhkknRz1pMBkvUSURD6S1pTK10alHZ7PclRzjZzk81XReFcBV5itNHYPOQocCxul6idIsxj9xY4noG5RIKTE+r7RTCH2aK3wfsiR1+ldQarWvn4minnR4NMuaYS4NwAZjWY0Qjlu2LbYRZHTKyElZBwj3bQZk+f/F8WwQpLH6C70oQpeBCaDOcBo6GLC0Ru7AjCG7JjImYLyjf0T7N/gNoMUNoIjUI4rVA3XFbm6mySN/QJmcVPOfmtc+TqcOOX0ERnCeqlx9v/3Ok+7PHIhb7eVEuj0Ug5DJ8zQaxsfZgPJih5PrUqjhOOLw5JzaYOkjUD0e7FfLRfjhkOQKQv9Rt3m77sxYdgI5BHhlEG7OgTYYQwLzg3WRqAyLLxLGRc7KQJyyYy20hwOXq/ZcG7tgEOYSkNrwoFhg6ENp8+lVkAzFlDVaJQVC2ASsXKNK4NvDwVK09aApBHn+BZTJgdrdZsrdauhfKruxdXvutzo8qZW8NI88VELY/N7RhsMSxtWBrP8vJv4EGPeS1EMKx/aXq2BtZpXDZtSIJXk+7A4lyBJY7uycsrFY1sXDdYKwyHhtE2VxZHARkWYzzkd0emlfiUdZxF0bXhXgw3FqkZjMqrx3bGABDBICCd1BdZL8kVXUrT1GwdqgURy0W8WtemfSHD0qzPILOt3+c22ENYV6DqCaozB2gsTSAEsnNRnma0UNvZPUGtgfiFjh0Xtriu3Q6eptTdKqmFybQyhqjTyPEbeS3mVVyguPFMG++0rclyePJa2ZieyhaGNQa1d0SW3FQK+vsmllQCen42mLarLkja9uLFRY6iYACgC+jGPxWotsCllud+wNQl5xOe3zMWoSl7dE3HNTyp5Jd4dWMfbaE/AAY7CjRtOQ0jRkoxeDHStQ3YpN1WzLViruR8uOrAlJ2wexGxQbYGxqnn17lZoxahC3k2QdVJfeNdf7HCaUhkfhStHNYpx5W1LrTmdlkfA3FzqQ+TFuMZ+9vG+TsL93hiBoeKail5E6Nl6ilO6bdpEU1vH8N1xG8FyxbiRIEy0wKix45sQEsH2Zy5h3XDy7eZSGx7zyvC1XP1AHrE9mTScvjtQA8s5YSFtawpcSsjV0BlbjOyeKRZs33oJpzeNNti/XrOdybhOYqM0PlIM8FwudXmhtTFWmOvFltgKNLXGcL3EWPEiUUbC1xqB2gL3umj87/sKrvuZSwhLPYnBQhfGGKjGILb6oRqN0aiB9bO+6Ns9i7OPcxlHzQHg2QmP81wCpBHbBUVsa7sxz5EAsClrayhcTYWrGTp4S2yax20FtGh3gu3eUN5WziXkI5MGgDICpQbWrE8aN8C2PmEQEzSxTOWSv6etHVutbYRDsA87pUM4qSf97kejCV87AF8LOZ9w/VAqgYUO6+D+icFCCr8jzUX2a8V1MK620tlD5zNUrbFvrHH+vxxj8vGZz3wGn/zkJ7F792687W1vw6c//Wmcd955L/s9Rz6e+H8SzGcSVdH4oj9FwEI/RpzGGK2XXDiWiHYXhzKIbDgVACbjBkkskPcyqEYjsQwtiqUvJnQ5q95cjiTnaxubypFJgnKjwMbq2IcwgbbWYbVqUwl53BacpbIt5HJ557WaDffIOvzYOsGtXfI1IUsndVFblu6e2XIPn8LJI1b4NGeG7hi/SxNUla1RscWQTJBYkE5ie4+1GthnawaWUsJSxvnaNI9RFY0vkEsTLiZ7oWKH43KZ/zPmARaWUJw+xxX2LhzPxXFM+gzx/aVgcrVWAU9tSO9c3HUXM2BzRnhjj9CbYzKU9xI0lcaevRM2mLbS24XoXV5fAj407dJhbo4McT6ztDn8tZoNbiL5fq5YVdmdD67ottRMHPsx1xbFSeRXhK5gMU8FVgvOg/YyabfD8j1KLRAJVup+zErrdvo0tcLegtv0YiZGcRIhTmPECcu8W6mB2Lh9ayiwuxToRkAnZsezlLHTyFPp5TqKpdUBgdF66Xe5aGJZqLTASofQ6yXo2kjbcHWCvaUlh10mv6rhGg6XvjPU1qoAHJ1pFCGO2noOF/nQDadJJoX2RZ2Fbon4SAGjRviiv4WUvE70Y1sPAZZrZzRdbYGLCLotqm41p21VuEvlfWcksLdkudVWBgF2wptzLqicaKCwdRiFFlir2UEvZYS3DLhP+0qBFyqB7X2D5ZzTuVme8O42S2gArrPKehw5IEMwilAVNWp7Pk4i9OZzFMMKew/UfidcbIngWAmv60spOwxXJzVRdvwsiRgk8Mc6lqAMG+C5gnV6EDPhjgVfd5BQuxvMErks57Qmr8YTTndmMeIsQjmufb2YjCR6CzmSNIaIYhTrE+zfO0KlWY+4vodluDvIEWeRX5EPDxSoywZNbTwpcjZzpS+xf2IwbAQKW1O10iGszMUw2mDv2GCiBNsuAJtSwuYOFx6/UHOqZ2khhdZMSMYKWK2EjYoSGsMLikTy80qwXKVZzDtb7A6p5wuWkW5EWOlw6ifr8FxWRYNq0nAa1G4xHk6U1/W9JRPLQgNvHhA2bx1Y3dC+wHRjdYxRRThQt4tUgMm5sy1pznJTlw2imOWEd5DwjsqmUj7F4mpgHIFJ7KIo6yZWbzWaimtPOoMMQloCPap8pNxlCPYONbII2LyYcRS8sTU2st3BxalK8oWyrgZLCPgidoBTWeW4xtgAZ91tjh35+Ju/+Rv86q/+Kj73uc9hx44d+NSnPoU77rgDTz31FJaXl1/yu9O7XeYyiW4/w2RU+RxdkkY+TJ3agqGmdnUfnHt3+TUXAxZSgozxP+tS+Xy6K85yueHhgQL7C0JtCwxdNf5aLbBqi7PGild7bxqQZ84bDfDcRGA5J5zUaY2i2347Ui3rns/a+ow9JSvwcs75wrpUqBVho2mrugWA/bWwhoe3TZa2H6WtpxCwTi7mrWTaKt9SBr81b9+E/HaxRDij29apbLYkRGt2xOt1W/iZR23xJOfkBcoJp7+e2zDYaFiJM0t49pb8ZoXlDhv6SLBxJALeNDA4eSD9lr44ibio1hYumaml40S113u+YAUmYqO7mBEqLRBLIJNkdyAJn0c+OSfkUVvcJdDu1pkoXqkbMFFzRa8C/KxbcsLWxcSnj6Z3MbnoRBRHqMsG6xON5ya8G+iknLBWsyOLJV8nEuw0FjfliGKJtRcm2GiATdaY7qtaOculIwoC+yomMCsdwmkLTEzSPEFdKTz3QoVu3M6967v77lDZLcA26hTL1vDlETC32Jl9140Qtr6pLcZ1leyNlbE9pfA7viaaCcRazTIkwPdwO6u2dghvtg58ur7XRZpWrQNJ7Lz1rYwcvNvFRTMKDewuBCaa59htqexaWXY7KwSASBK25Hyv/RWgyUVkCAtJS8rdzqZOP7PvcuDQcd5NIW0Rbl1w2sXtLgDaYvUoaesocluz5Mbtf/5niEpzJM3p2v6Ko58Du8tj0I1hjEFdcwRrtRbYaATvYrOLmOUOYS4h5L64me3FwB6LJdeASCmwsVb6gnpH0Ds5F07WNe/6mag2Apt12cFqA+wr2xV2HgGDmOVmU8bF96ktkC7GtS+cHeRcj+RqroalQTfhEH5TKfyfNYOTOzyBa3VbFBvZFEcSC4wq8iQsj4C8E2NtpPDdMROPSHD0tBezTa608DJDJPCGPjvCsuYttXEsMKwIe0qBQrGdXM6Y2ExHU/uDFHWpQETIexnyHtdv7d89wr7S7gaZStW5nTRpx9XFKF8v5lKUT69q5BHbzjzijQ6unjDNYxvxZVJfFQ32TchHqPeUbBsjwWSrFxO69pwj6Ut28bi/ZF2IZCv7jV3EfXvItt7tdnGLw83zCRPZQvmIpns3y/pYYWSjkou58NHFrMt1lVXReDstIzGzwcJFoly9UwGJ0//f4tiRjx07duDcc8/Fn/3ZnwHgd2Cceuqp+MhHPoLrrrvuJb87vdtlLuXdF5FgBYgi4QXGVSi7YlC3G8XlDdu3cMKnENwA+Xd+2Jyjy//KSGKwif8BnogTyDSDqSuQMZBpiijvcA0FEXQxQTUuYZTxua/UFlaVowrFyLJlm8uO4ghxxoWlUcK7YCB5xeRCyESwRUoJRCRh3J77LIeQEtXaAVsoynlStwulLhqoxvjCxziJIGMWZk5TNSgL5bctuiJGLvJMoBsF1RhkPb7PeHUI1bS7e8pJ46vvgbbWYthw2Pf/OjnDYKkHmWZIFjYhyjogrbhuY/0A1zkQkPQHnG8tJmiKit/PMPWOlSiS6MzlyDZvgUjY+1R7/hcb+zZQl8rWlQjIWKIu7Iqzx0Y/6nS5voIMICPAaO5D0wBEnBZSxuanM6QLmxD3WTlEJGGaBqapYeoKMIb/rkrILEdn66lAdx7QCmhK6PEQRjWQcYJmYw16MoZw9UcATNNA1Qrj9RLluEaccG2Le6toU3G6QCvjIzNStu9WcIWsTHx4W7jbeQJho145x3nTTuzHUSsD3WgvR1xcqXyKJ+smvtbH5f1dobCLAEa2Rsm93M0VorkXYsm842Uy7g0Q9ed5vKOYf1YT6PEGTFWiGa6jOLBhV4Lt9fM+6xKPORuvyUZlV2fs/J0jdzubqrLxRk9Gs4sJWEMJw7ZGK+NXcFEkOGdud6m47YlCChSjCnVt2pc/CV4ZuwiU0c7wcqRl+p7TLy90i5ko4XTDYFMH6dw8om4PpBV0MfG1P6auUI0mvlg4zWPESYRkMMe60+lCxCmgG+iyQPXCHjSjEaoJ67hW2velrviVA26lXAwr1JUC2RqsNE8gYwnSBGV3hblFlyvYdLsbujZF5Qqdp3fXjde59intxHxNK6fe3tqo6/TLrVyRqSsQdkW51YSLLbub+lBl5e1eUyp05zLE3S5EFPtx1pMRiiHLRZxG3uHzLja+/2itYNm0Kba0E/s2bp5cX9zYueMy4hV/VXA9zHij9DUdqtaIkojrMNy7QGoeX5e6Hw4brNbtzrc84ij4wuY+iIhr/GrlX3/Q7bTF7s4n+MLzSAKCbZGMI67Nsm8KFBHXbNUHXoBuFISUSAZziLLc21SZJIj7cyCjISJO8+jxCLosoMsCMBypi5II1bhGYQvbfe1MpaCsjhndvkHZ1Z+4GkpjjI+WuXotJwvrlcHbv6SPDfmo6xrdbhd/93d/h0suucQfv/zyy7G2toY777xzpv2L/W+Xx3+xj/lM+rw6gIPeOum2XLVvo2SHzaEqowhKc2olSWLI2KUK7EuEplZ4AgKGuAahv9iBzDo8eUSQaQKZpN7wQPAEQEYQUQTS2l+HVA2jGpDit+2IJAYpzeQljiGiCDLNACFBqoYaj3Hg2T3Qmos0s26CdNNJvhy93Q7MfY37A2/ESLFzhZRwr0/ntnw/IYQ9z4Ilonjm5RH8xkYJNRmi3LcXq7s3MKkMBt2YHZctEJs/dQXZljfYF5c0fE0h+T6Ngq5LHHj6GWwcKBBLge58jsHyIqI8h0xSNiRxAiEkTF3CqMYbFz/+UQwhI5DR0MUY9dp+qJJ39PQ3zzMZidttmmTTYqQVRBSzga8KO2aSiWOcgIydhyiGTBNARjBVBVKNHycQgbQdM1uUOf1iHdIa4+d3Y7ReclFjGnk5W9s7xP8ZcfTk5A6nWBaX+xiccrI3FiKK2zkkAzUeYbJvP7/UqdsFGY1ybYLJuOL0RSKxuG0F6fwiZJaBlIaIWN5IK5A2MHUF01TQRWHHWPoxFTICZAQZO+PP8wwJUKP8c5umQrW6isl6geF66V9IJwRwyvZFJPOLPG5Jytd0+WqwESeteBzdKyLtl50smqbBePcL2LN3hFHDK7NaC6QR4YeWc2zavpXlQka+3yKKIZJ45nowGkZpyDiCyLqgupzpC8jAKA3SGsK+SYplgPtMml+c4sewYVIspETU7XFfy7KVJTIgrWGayssYNQqkFVQx4R1TfBOWHVfz4973YUNQ/aUBZJqz3sfsCKR1CF6HtWKnIADSxus5v4iNSZOIIp6DOOFr2KJip/9GNdDFBOv/swfjdS40HyzwAirpJsgWF70cOplw9zZawVQVTFMj6nQQ9+ZAqmEbZ52eaSoYxXuIjVLeJgGASPiaajTExu5VrB0ofLRi6ylz6J684vXbP5ddIJimtk6TnSIRYbRvDVoZLG5bQTK3cJA8VahX98GoBqrWtji4RpanmNu+zdtobxecLgjhdQ9EqPbvxb5n9mN1wq9d2JS1rwYQUqI7l6GzvMUW+aZcxFmXXh5lkk3JHS9sTFWhGY+x/7l11JpTPd25HItv3IpkfpGf3co52YUN+5YUILszUBtQU9uFtYSpK6jJGDCa+2F1QcQJTF2hHo6x5383/HbuKIkwv9RD56TNXp6cD3I6wf1P/bVZfwm6bryPnX7FPRH8wtD7Ihf6dO5TYsZeDhvC2+8YvaL/7QJ6lfHcc88RAPr3f//3mePXXnstnXfeeYe0D//VNnzCJ3zCJ3zC5wfn80r+q+339I/lXk187GMfwzXXXOP/Xltbw2mnnYZnnnnm5ZlTwHGBjY0NnHrqqXj22Wdf8ZtsA44dwny9/hDm7PWFE3W+iAjD4RBbt2592bavOvnYvHkzoijCnj17Zo7v2bMHKysrh7R/sf9qOz8/f0JN2g8C5ubmwpy9jhDm6/WHMGevL5yI8/VKgwby5ZscGdI0xdlnn41du3b5Y8YY7Nq1C+eff/6rfbuAgICAgICA1xlek7TLNddcg8svvxznnHMOzjvvPHzqU5/CeDzGBz/4wdfidgEBAQEBAQGvI7wm5OOXfumXsG/fPnz84x/H7t278fa3vx133303tmzZ8rLfzbIMn/jEJw6bigk4PhHm7PWFMF+vP4Q5e30hzNfL47h7vXpAQEBAQEDADzZe9ZqPgICAgICAgICXQiAfAQEBAQEBAUcVgXwEBAQEBAQEHFUE8hEQEBAQEBBwVHHckY/PfOYzeOMb34g8z7Fjxw48/PDDx7pLJyRuvPFGnHvuuRgMBlheXsYll1yCp556aqZNWZbYuXMnlpaW0O/38Qu/8AuHvFzumWeewcUXX4xut4vl5WVce+21UEodzUc5IXHTTTdBCIGrr77aHwvzdfzhueeew/vf/34sLS2h0+ngzDPPxKOPPurPExE+/vGP4+STT0an08FFF12Eb37zmzPXWF1dxWWXXYa5uTksLCzg13/91zEajY72o/zAQ2uN66+/Htu3b0en08Gb3vQm/OEf/iGm92yE+ToCvAr/zuVVw+23305pmtJf/dVf0de+9jX60Ic+RAsLC7Rnz55j3bUTDu9+97vp5ptvpieffJIef/xxeu9730vbtm2j0Wjk21xxxRV06qmn0q5du+jRRx+lH//xH6d3vOMd/rxSis444wy66KKL6D//8z/pi1/8Im3evJk+9rGPHYtHOmHw8MMP0xvf+EZ661vfSldddZU/Hubr+MLq6iqddtpp9IEPfIAeeughevrpp+mf//mf6Vvf+pZvc9NNN9H8/Dz9wz/8A33lK1+hn/3Zn6Xt27dTURS+zU//9E/T2972NnrwwQfp3/7t3+jNb34zve997zsWj/QDjRtuuIGWlpborrvuou985zt0xx13UL/fpz/5kz/xbcJ8vXIcV+TjvPPOo507d/q/tda0detWuvHGG49hrwKIiPbu3UsA6L777iMiorW1NUqShO644w7f5r/+678IAD3wwANERPTFL36RpJS0e/du3+azn/0szc3NUVVVR/cBThAMh0N6y1veQvfccw/95E/+pCcfYb6OP/zu7/4u/cRP/MSLnjfG0MrKCn3yk5/0x9bW1ijLMvrrv/5rIiL6+te/TgDokUce8W2+9KUvkRCCnnvuudeu8ycgLr74Yvq1X/u1mWM///M/T5dddhkRhfk6Uhw3aZe6rvHYY4/hoosu8seklLjooovwwAMPHMOeBQDA+vo6AGDTpk0AgMceewxN08zM1+mnn45t27b5+XrggQdw5plnzrxc7t3vfjc2Njbwta997Sj2/sTBzp07cfHFF8/MCxDm63jEP/7jP+Kcc87BL/7iL2J5eRlnnXUW/vIv/9Kf/853voPdu3fPzNn8/Dx27NgxM2cLCws455xzfJuLLroIUko89NBDR+9hTgC84x3vwK5du/CNb3wDAPCVr3wF999/P97znvcACPN1pDjm/9XW4YUXXoDW+pC3oG7ZsgX//d//fYx6FQDw/+a5+uqrccEFF+CMM84AAOzevRtpmmJhYWGm7ZYtW7B7927f5nDz6c4FvLq4/fbb8R//8R945JFHDjkX5uv4w9NPP43PfvazuOaaa/B7v/d7eOSRR/Bbv/VbSNMUl19+uR/zw83J9JwtLy/PnI/jGJs2bQpz9irjuuuuw8bGBk4//XREUQStNW644QZcdtllABDm6whx3JCPgOMXO3fuxJNPPon777//WHcl4EXw7LPP4qqrrsI999yDPM+PdXcCXgGMMTjnnHPwR3/0RwCAs846C08++SQ+97nP4fLLLz/GvQs4GH/7t3+LW2+9Fbfddht+9Ed/FI8//jiuvvpqbN26NczX94DjJu2yefNmRFF0SPX9nj17sLKycox6FXDllVfirrvuwr/8y7/gDW94gz++srKCuq6xtrY20356vlZWVg47n+5cwKuHxx57DHv37sWP/diPIY5jxHGM++67D3/6p3+KOI6xZcuWMF/HGU4++WT8yI/8yMyxH/7hH8YzzzwDoB3zl7KJKysr2Lt378x5pRRWV1fDnL3KuPbaa3Hdddfhl3/5l3HmmWfiV37lV/DRj34UN954I4AwX0eK44Z8pGmKs88+G7t27fLHjDHYtWsXzj///GPYsxMTRIQrr7wSX/jCF3Dvvfdi+/btM+fPPvtsJEkyM19PPfUUnnnmGT9f559/Pr761a/OKNs999yDubm5Q4xuwPeHCy+8EF/96lfx+OOP+88555yDyy67zP8e5uv4wgUXXHDI9vVvfOMbOO200wAA27dvx8rKysycbWxs4KGHHpqZs7W1NTz22GO+zb333gtjDHbs2HEUnuLEwWQygZSzLjOKIhhjAIT5OmIc64rXadx+++2UZRl9/vOfp69//ev0G7/xG7SwsDBTfR9wdPDhD3+Y5ufn6ctf/jI9//zz/jOZTHybK664grZt20b33nsvPfroo3T++efT+eef78+7rZvvete76PHHH6e7776bTjrppLB18yhhercLUZiv4w0PP/wwxXFMN9xwA33zm9+kW2+9lbrdLt1yyy2+zU033UQLCwt055130hNPPEE/93M/d9itm2eddRY99NBDdP/999Nb3vKWE3Lr5muNyy+/nE455RS/1fbv//7vafPmzfQ7v/M7vk2Yr1eO44p8EBF9+tOfpm3btlGapnTeeefRgw8+eKy7dEICwGE/N998s29TFAX95m/+Ji0uLlK326VLL72Unn/++ZnrfPe736X3vOc91Ol0aPPmzfTbv/3b1DTNUX6aExMHk48wX8cf/umf/onOOOMMyrKMTj/9dPqLv/iLmfPGGLr++utpy5YtlGUZXXjhhfTUU0/NtNm/fz+9733vo36/T3Nzc/TBD36QhsPh0XyMEwIbGxt01VVX0bZt2yjPc/qhH/oh+v3f//2Zbehhvl45BNHU69kCAgICAgICAl5jHDc1HwEBAQEBAQEnBgL5CAgICAgICDiqCOQjICAgICAg4KgikI+AgICAgICAo4pAPgICAgICAgKOKgL5CAgICAgICDiqCOQjICAgICAg4KgikI+AgICAgICAo4pAPgICAgICAgKOKgL5CAgICAgICDiqCOQjICAgICAg4KgikI+AgICAgICAo4r/HwUnyvoK/Fe8AAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from matplotlib import cm\n",
        "mfcc_data= np.swapaxes(vect, 0 ,1)\n",
        "cax = plt.imshow(mfcc_data, interpolation='nearest', cmap=cm.Oranges_r, origin='lower')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t-yei9o4nYNC",
        "outputId": "056ee2f5-8e0d-4085-f118-fc413376cf60"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(915, 27)\n",
            "(915, 27)\n",
            "(24705,)\n"
          ]
        }
      ],
      "source": [
        "mfcc = vect\n",
        "print(vect.shape)\n",
        "mfcc = ((mfcc) - (np.mean(mfcc, axis=0))) / (np.sqrt(np.var(mfcc, axis=0)))\n",
        "print(mfcc.shape)\n",
        "concat = np.concatenate(mfcc, axis=0)\n",
        "print(concat.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YpLCvi3AJC5z"
      },
      "outputs": [],
      "source": [
        "# Dataset class to load train and validation data\n",
        "\n",
        "class AudioDataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, root, phonemes = PHONEMES, context=0, partition= \"train-clean-100\"): # Feel free to add more arguments\n",
        "\n",
        "        self.context    = context\n",
        "        self.phonemes   = phonemes\n",
        "\n",
        "        self.mfcc_dir       = os.path.join(root, partition, 'mfcc')\n",
        "\n",
        "        self.transcript_dir = os.path.join(root, partition, 'transcript')\n",
        "\n",
        "        mfcc_names          = sorted(os.listdir(self.mfcc_dir))\n",
        "\n",
        "        transcript_names    = sorted(os.listdir(self.transcript_dir))\n",
        "\n",
        "        # Making sure that we have the same no. of mfcc and transcripts\n",
        "        assert len(mfcc_names) == len(transcript_names)\n",
        "\n",
        "        self.mfccs, self.transcripts = [], []\n",
        "\n",
        "        for i in range(len(mfcc_names)):\n",
        "        #   Load a single mfcc\n",
        "            mfcc = np.load(os.path.join(self.mfcc_dir, mfcc_names[i]))\n",
        "        #   Do Cepstral Normalization of mfcc (explained in writeup)\n",
        "            mfcc = ((mfcc) - (np.mean(mfcc, axis=0))) / (np.std(mfcc, axis=0) + 1e-5)\n",
        "\n",
        "        #   Load the corresponding transcript\n",
        "            transcript  = np.load(os.path.join(self.transcript_dir, transcript_names[i]))\n",
        "        # Remove [SOS] and [EOS] from the transcript\n",
        "        # (Is there an efficient way to do this without traversing through the transcript?)\n",
        "        # Note that SOS will always be in the starting and EOS at end, as the name suggests.\n",
        "            transcript = transcript[1:-1]\n",
        "        #   Append each mfcc to self.mfcc, transcript to self.transcript\n",
        "            self.mfccs.append(mfcc)\n",
        "            self.transcripts.append(transcript)\n",
        "\n",
        "        # NOTE:\n",
        "        # Each mfcc is of shape T1 x 27, T2 x 27, ...\n",
        "        # Each transcript is of shape (T1+2) x 27, (T2+2) x 27 before removing [SOS] and [EOS]\n",
        "\n",
        "        # TODO: Concatenate all mfccs in self.mfccs such that\n",
        "        # the final shape is T x 27 (Where T = T1 + T2 + ...)\n",
        "        self.mfccs          = np.concatenate(self.mfccs, axis=0)\n",
        "        #self.mfccs          = np.vstack(self.mfccs)\n",
        "        print(f'MFCC shape {self.mfccs.shape} ')\n",
        "\n",
        "        # TODO: Concatenate all transcripts in self.transcripts such that\n",
        "        # the final shape is (T,) meaning, each time step has one phoneme output\n",
        "        self.transcripts    = np.concatenate(self.transcripts, axis=0)\n",
        "        #self.transcripts    = np.hstack(self.transcripts)\n",
        "        print(f'Transcripts shape {self.transcripts.shape}')\n",
        "        # Hint: Use numpy to concatenate\n",
        "\n",
        "        # Length of the dataset is now the length of concatenated mfccs/transcripts\n",
        "        self.length = len(self.mfccs)\n",
        "\n",
        "        # Take some time to think about what we have done.\n",
        "        # self.mfcc is an array of the format (Frames x Features).\n",
        "        # Our goal is to recognize phonemes of each frame\n",
        "        # From hw0, you will be knowing what context is.\n",
        "        # We can introduce context by padding zeros on top and bottom of self.mfcc\n",
        "        self.mfccs = np.pad(self.mfccs, ((context, context), (0, 0)), mode='constant', constant_values=0)\n",
        "\n",
        "        # The available phonemes in the transcript are of string data type\n",
        "        # But the neural network cannot predict strings as such.\n",
        "        # Hence, we map these phonemes to integers\n",
        "        print(f'Check {self.transcripts.shape} and sample {self.transcripts[100]}')\n",
        "        # TODO: Map the phonemes to their corresponding list indexes in self.phonemes\n",
        "        self.transcripts = np.array([np.where((np.array(self.phonemes) == trans))[0] for trans in self.transcripts])\n",
        "        # Now, if an element in self.transcript is 0, it means that it is 'SIL' (as per the above example)\n",
        "        print(f'Mfccs shape {self.mfccs.shape} and sample {self.mfccs[100]}')\n",
        "        print(f'Transcripts shape {self.transcripts.shape} and sample {self.transcripts[100]}')\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.length\n",
        "\n",
        "    def __getitem__(self, ind):\n",
        "\n",
        "        # TODO: Based on context and offset, return a frame at given index with context frames to the left, and right.\n",
        "        #frames = np.concatenate([self.mfccs[:self.context], self.mfccs[self.context + ind:self.context + ind + 1], self.mfccs[-self.context:]])\n",
        "        frames = self.mfccs[ind:ind + 2*self.context + 1]\n",
        "        # After slicing, you get an array of shape ontext2*c+1 x 27. But our MLP needs 1d data and not 2d.\n",
        "        frames = frames.flatten()\n",
        "\n",
        "        frames      = torch.FloatTensor(frames) # Convert to tensors\n",
        "        phonemes    = torch.squeeze(torch.tensor(np.squeeze(self.transcripts[ind])))\n",
        "\n",
        "        return frames, phonemes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e8KfVP39S6o7"
      },
      "outputs": [],
      "source": [
        "class AudioTestDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, root, context=0, partition= \"test-clean\"):\n",
        "\n",
        "        self.context    = context\n",
        "        self.mfcc_dir       = os.path.join(root, partition, 'mfcc')\n",
        "        mfcc_names          = sorted(os.listdir(self.mfcc_dir))\n",
        "\n",
        "        self.mfccs = []\n",
        "\n",
        "        for i in range(len(mfcc_names)):\n",
        "        #   Load a single mfcc\n",
        "            mfcc = np.load(os.path.join(self.mfcc_dir, mfcc_names[i]))\n",
        "        #   Do Cepstral Normalization of mfcc (explained in writeup)\n",
        "            mfcc = ((mfcc) - (np.mean(mfcc, axis=0))) / (np.std(mfcc, axis=0) + 1e-5)\n",
        "            self.mfccs.append(mfcc)\n",
        "\n",
        "        self.mfccs = np.concatenate(self.mfccs, axis=0)\n",
        "        self.length = len(self.mfccs)\n",
        "\n",
        "        # We can introduce context by padding zeros on top and bottom of self.mfcc\n",
        "        self.mfccs = np.pad(self.mfccs, ((context, context), (0, 0)), mode='constant', constant_values=0)\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.length\n",
        "\n",
        "    def __getitem__(self, ind):\n",
        "\n",
        "        # TODO: Based on context and offset, return a frame at given index with context frames to the left, and right.\n",
        "        frames = self.mfccs[ind:ind + 2*self.context + 1]\n",
        "        # After slicing, you get an array of shape 2*context+1 x 27. But our MLP needs 1d data and not 2d.\n",
        "        frames = frames.flatten()\n",
        "        frames = torch.FloatTensor(frames) # Convert to tensors\n",
        "\n",
        "        return frames\n",
        "\n",
        "    # TODO: Create a test dataset class similar to the previous class but you dont have transcripts for this\n",
        "    # Imp: Read the mfccs in sorted order, do NOT shuffle the data here or in your dataloader."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qNacQ8bpt9nw"
      },
      "source": [
        "# Parameters Configuration"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WE7tsinAuLNy"
      },
      "source": [
        "Storing your parameters and hyperparameters in a single configuration dictionary makes it easier to keep track of them during each experiment. It can also be used with weights and biases to log your parameters for each experiment and keep track of them across multiple experiments."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PmKwlFqgt_Zq"
      },
      "outputs": [],
      "source": [
        "config = {\n",
        "    'epochs'        : 80,\n",
        "    'batch_size'    : 2048,\n",
        "    'context'       : 20,\n",
        "    'init_lr'       : 1e-3,\n",
        "    'architecture'  : 'high-cutoff'\n",
        "    # Add more as you need them - e.g dropout values, weight decay, scheduler parameters\n",
        "}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2mlwaKlDt_2c"
      },
      "source": [
        "# Create Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uyPB1jGA8uM3",
        "outputId": "eb635277-24cd-4ac8-e6ae-df11b5cff87d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "MFCC shape (36091157, 27) \n",
            "Transcripts shape (36091157,)\n",
            "Check (36091157,) and sample AH\n",
            "Mfccs shape (36091197, 27) and sample [ 1.7638558e+00  7.6890886e-02 -1.5383435e+00 -1.2271409e-02\n",
            " -6.7100900e-01 -3.2965618e-01 -5.9578854e-01 -1.0991193e+00\n",
            " -1.8357016e+00  2.9777536e-01 -6.2612665e-01 -1.0256163e-01\n",
            " -9.7565281e-01 -7.8316338e-02 -7.6276708e-01 -4.4900739e-01\n",
            " -2.8761163e-01 -1.6419492e+00 -1.1264700e+00 -7.9027420e-01\n",
            "  1.4396261e-01  1.0080085e+00 -1.5419359e+00 -1.7541655e+00\n",
            " -2.7218571e-02  5.2518483e-09  2.0897222e-01]\n",
            "Transcripts shape (36091157, 1) and sample [3]\n",
            "Result Shapes - Data -> (36091197, 27) and Labels -> (36091157, 1) \n"
          ]
        }
      ],
      "source": [
        "train_data = AudioDataset(root='../content/11-785-s24-hw1p2/', context=config['context'], partition= \"train-clean-100\")\n",
        "print(f'Result Shapes - Data -> {train_data.mfccs.shape} and Labels -> {train_data.transcripts.shape} ')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7xi7V8x8W9z4",
        "outputId": "0d360b17-6f82-4790-f755-0aaece15ff81"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "MFCC shape (1928204, 27) \n",
            "Transcripts shape (1928204,)\n",
            "Check (1928204,) and sample L\n",
            "Mfccs shape (1928244, 27) and sample [ 1.58368915e-01 -1.21859744e-01  5.12083292e-01  1.15812123e+00\n",
            "  4.58461791e-01 -3.01561415e-01  2.08318830e-01  7.19309568e-01\n",
            "  7.96862960e-01 -1.48222816e+00  9.07102302e-02 -2.47009650e-01\n",
            "  4.21092987e-01 -8.97688940e-02  5.86933732e-01 -4.45923179e-01\n",
            "  7.59552836e-01  1.40139163e+00 -1.47331461e-01 -7.29686379e-01\n",
            "  6.72928095e-01 -3.87415558e-01  1.04793763e+00 -5.67102432e-01\n",
            "  3.70796591e-01 -7.24416482e-09 -8.84944618e-01]\n",
            "Transcripts shape (1928204, 1) and sample [21]\n",
            "Result Shapes - Data -> (1928244, 27) and Labels -> (1928204, 1) \n"
          ]
        }
      ],
      "source": [
        "val_data = AudioDataset('../content/11-785-s24-hw1p2/', context=config['context'], partition= \"dev-clean\")\n",
        "print(f'Result Shapes - Data -> {val_data.mfccs.shape} and Labels -> {val_data.transcripts.shape} ')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M2m-meVm80qV",
        "outputId": "b83bb29f-b5ff-4085-ed6e-5e1ddb22fd0b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Result Shapes - Data -> (1934178, 27)\n"
          ]
        }
      ],
      "source": [
        "test_data = AudioTestDataset('../content/11-785-s24-hw1p2/', context=config['context'], partition= \"test-clean\")\n",
        "print(f'Result Shapes - Data -> {test_data.mfccs.shape}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4mzoYfTKu14s",
        "outputId": "967f9e42-abf0-49df-92ee-e3c562c639a3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Batch size     :  2048\n",
            "Context        :  20\n",
            "Input size     :  1107\n",
            "Output symbols :  42\n",
            "Train dataset samples = 36091157, batches = 17623\n",
            "Validation dataset samples = 1928204, batches = 942\n",
            "Test dataset samples = 1934138, batches = 945\n"
          ]
        }
      ],
      "source": [
        "# Define dataloaders for train, val and test datasets\n",
        "# Dataloaders will yield a batch of frames and phonemes of given batch_size at every iteration\n",
        "# We shuffle train dataloader but not val & test dataloader. Why?\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "    dataset     = train_data,\n",
        "    num_workers = 2,\n",
        "    batch_size  = config['batch_size'],\n",
        "    pin_memory  = True,\n",
        "    shuffle     = True\n",
        ")\n",
        "\n",
        "val_loader = torch.utils.data.DataLoader(\n",
        "    dataset     = val_data,\n",
        "    num_workers = 2,\n",
        "    batch_size  = config['batch_size'],\n",
        "    pin_memory  = True,\n",
        "    shuffle     = False\n",
        ")\n",
        "\n",
        "test_loader = torch.utils.data.DataLoader(\n",
        "    dataset     = test_data,\n",
        "    num_workers = 2,\n",
        "    batch_size  = config['batch_size'],\n",
        "    pin_memory  = True,\n",
        "    shuffle     = False\n",
        ")\n",
        "\n",
        "\n",
        "print(\"Batch size     : \", config['batch_size'])\n",
        "print(\"Context        : \", config['context'])\n",
        "print(\"Input size     : \", (2*config['context']+1)*27)\n",
        "print(\"Output symbols : \", len(PHONEMES))\n",
        "\n",
        "print(\"Train dataset samples = {}, batches = {}\".format(train_data.__len__(), len(train_loader)))\n",
        "print(\"Validation dataset samples = {}, batches = {}\".format(val_data.__len__(), len(val_loader)))\n",
        "print(\"Test dataset samples = {}, batches = {}\".format(test_data.__len__(), len(test_loader)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n-GV3UvgLSoF",
        "outputId": "125ea6b8-b4a4-41fe-8f73-6dce524853af"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([2048, 1107]) torch.Size([2048])\n"
          ]
        }
      ],
      "source": [
        "# Testing code to check if your data loaders are working\n",
        "for i, data in enumerate(train_loader):\n",
        "    frames, phoneme = data\n",
        "    print(frames.shape, phoneme.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ShVNjHUk_wQ1",
        "outputId": "56d271ca-9859-40ce-d611-2fa492bbc2da"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([2048, 1107]) torch.Size([2048])\n"
          ]
        }
      ],
      "source": [
        "# Testing code to check if your data loaders are working\n",
        "for i, data in enumerate(val_loader):\n",
        "    frames, phoneme = data\n",
        "    print(frames.shape, phoneme.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3WuYgHGGHy33",
        "outputId": "89654f82-7af0-40ef-a4aa-b758a9537195"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([2048, 1107])\n"
          ]
        }
      ],
      "source": [
        "for i, data in enumerate(test_loader):\n",
        "    frames = data\n",
        "    print(frames.shape)\n",
        "    break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yY1lT24uMqQQ"
      },
      "outputs": [],
      "source": [
        "def init_xavier(m):\n",
        "  if type(m) == torch.nn.Linear:\n",
        "    fan_in = m.weight.size()[1]\n",
        "    fan_out = m.weight.size()[0]\n",
        "    std = np.sqrt(1.0/(fan_in + fan_out))\n",
        "    m.weight.data.normal_(0,std)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nxjwve20JRJ2"
      },
      "source": [
        "# Network Architecture\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3NJzT-mRw6iy"
      },
      "source": [
        "This section defines your network architecture for the homework. We have given you a sample architecture that can easily clear the very low cutoff for the early submission deadline."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OvcpontXQq9j"
      },
      "outputs": [],
      "source": [
        "# This architecture will make you cross the very low cutoff\n",
        "# However, you need to run a lot of experiments to cross the medium or high cutoff\n",
        "import torch.nn.init as init\n",
        "class Network(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, input_size, output_size):\n",
        "\n",
        "        super(Network, self).__init__()\n",
        "        layers = []\n",
        "        k = 15\n",
        "        size_list = [input_size, 2048, 4096, 2048, 1024, 1024, 512, 512, 256, output_size]\n",
        "        for i in range(len(size_list)-2):\n",
        "            linear = torch.nn.Linear(size_list[i],size_list[i+1])\n",
        "            layers.append(linear)\n",
        "            init.kaiming_normal_(linear.weight, mode='fan_in')\n",
        "            layers.append(torch.nn.BatchNorm1d(size_list[i+1]))\n",
        "            layers.append(torch.nn.GELU())\n",
        "            layers.append(torch.nn.Dropout(0.30))\n",
        "        layers.append(torch.nn.Linear(size_list[-2], size_list[-1]))\n",
        "        self.model = torch.nn.Sequential(*layers)\n",
        "\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = self.model(x)\n",
        "\n",
        "        return out"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HejoSXe3vMVU"
      },
      "source": [
        "# Define Model, Loss Function and Optimizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xAhGBH7-xxth"
      },
      "source": [
        "Here we define the model, loss function, optimizer and optionally a learning rate scheduler."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "_qtrEM1ZvLje",
        "outputId": "38db1738-e2c9-41d7-d28c-f5a4deeb871a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "=========================================================================\n",
            "                         Kernel Shape  Output Shape     Params  Mult-Adds\n",
            "Layer                                                                    \n",
            "0_model.Linear_0         [1107, 2048]  [2048, 2048]  2.269184M  2.267136M\n",
            "1_model.BatchNorm1d_1          [2048]  [2048, 2048]     4.096k     2.048k\n",
            "2_model.GELU_2                      -  [2048, 2048]          -          -\n",
            "3_model.Dropout_3                   -  [2048, 2048]          -          -\n",
            "4_model.Linear_4         [2048, 4096]  [2048, 4096]  8.392704M  8.388608M\n",
            "5_model.BatchNorm1d_5          [4096]  [2048, 4096]     8.192k     4.096k\n",
            "6_model.GELU_6                      -  [2048, 4096]          -          -\n",
            "7_model.Dropout_7                   -  [2048, 4096]          -          -\n",
            "8_model.Linear_8         [4096, 2048]  [2048, 2048]  8.390656M  8.388608M\n",
            "9_model.BatchNorm1d_9          [2048]  [2048, 2048]     4.096k     2.048k\n",
            "10_model.GELU_10                    -  [2048, 2048]          -          -\n",
            "11_model.Dropout_11                 -  [2048, 2048]          -          -\n",
            "12_model.Linear_12       [2048, 1024]  [2048, 1024]  2.098176M  2.097152M\n",
            "13_model.BatchNorm1d_13        [1024]  [2048, 1024]     2.048k     1.024k\n",
            "14_model.GELU_14                    -  [2048, 1024]          -          -\n",
            "15_model.Dropout_15                 -  [2048, 1024]          -          -\n",
            "16_model.Linear_16       [1024, 1024]  [2048, 1024]    1.0496M  1.048576M\n",
            "17_model.BatchNorm1d_17        [1024]  [2048, 1024]     2.048k     1.024k\n",
            "18_model.GELU_18                    -  [2048, 1024]          -          -\n",
            "19_model.Dropout_19                 -  [2048, 1024]          -          -\n",
            "20_model.Linear_20        [1024, 512]   [2048, 512]     524.8k   524.288k\n",
            "21_model.BatchNorm1d_21         [512]   [2048, 512]     1.024k      512.0\n",
            "22_model.GELU_22                    -   [2048, 512]          -          -\n",
            "23_model.Dropout_23                 -   [2048, 512]          -          -\n",
            "24_model.Linear_24         [512, 512]   [2048, 512]   262.656k   262.144k\n",
            "25_model.BatchNorm1d_25         [512]   [2048, 512]     1.024k      512.0\n",
            "26_model.GELU_26                    -   [2048, 512]          -          -\n",
            "27_model.Dropout_27                 -   [2048, 512]          -          -\n",
            "28_model.Linear_28         [512, 256]   [2048, 256]   131.328k   131.072k\n",
            "29_model.BatchNorm1d_29         [256]   [2048, 256]      512.0      256.0\n",
            "30_model.GELU_30                    -   [2048, 256]          -          -\n",
            "31_model.Dropout_31                 -   [2048, 256]          -          -\n",
            "32_model.Linear_32          [256, 42]    [2048, 42]    10.794k    10.752k\n",
            "-------------------------------------------------------------------------\n",
            "                          Totals\n",
            "Total params          23.152938M\n",
            "Trainable params      23.152938M\n",
            "Non-trainable params         0.0\n",
            "Mult-Adds             23.129856M\n",
            "=========================================================================\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torchsummaryX/torchsummaryX.py:101: FutureWarning: The default value of numeric_only in DataFrame.sum is deprecated. In a future version, it will default to False. In addition, specifying 'numeric_only=None' is deprecated. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
            "  df_sum = df.sum()\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-dfe31dd7-e975-424c-bf92-ce5b9e45abbd\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Kernel Shape</th>\n",
              "      <th>Output Shape</th>\n",
              "      <th>Params</th>\n",
              "      <th>Mult-Adds</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Layer</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0_model.Linear_0</th>\n",
              "      <td>[1107, 2048]</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>2269184.0</td>\n",
              "      <td>2267136.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1_model.BatchNorm1d_1</th>\n",
              "      <td>[2048]</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>4096.0</td>\n",
              "      <td>2048.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2_model.GELU_2</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3_model.Dropout_3</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4_model.Linear_4</th>\n",
              "      <td>[2048, 4096]</td>\n",
              "      <td>[2048, 4096]</td>\n",
              "      <td>8392704.0</td>\n",
              "      <td>8388608.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5_model.BatchNorm1d_5</th>\n",
              "      <td>[4096]</td>\n",
              "      <td>[2048, 4096]</td>\n",
              "      <td>8192.0</td>\n",
              "      <td>4096.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6_model.GELU_6</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 4096]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7_model.Dropout_7</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 4096]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8_model.Linear_8</th>\n",
              "      <td>[4096, 2048]</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>8390656.0</td>\n",
              "      <td>8388608.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9_model.BatchNorm1d_9</th>\n",
              "      <td>[2048]</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>4096.0</td>\n",
              "      <td>2048.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10_model.GELU_10</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11_model.Dropout_11</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 2048]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12_model.Linear_12</th>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>2098176.0</td>\n",
              "      <td>2097152.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13_model.BatchNorm1d_13</th>\n",
              "      <td>[1024]</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>2048.0</td>\n",
              "      <td>1024.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14_model.GELU_14</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15_model.Dropout_15</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16_model.Linear_16</th>\n",
              "      <td>[1024, 1024]</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>1049600.0</td>\n",
              "      <td>1048576.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17_model.BatchNorm1d_17</th>\n",
              "      <td>[1024]</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>2048.0</td>\n",
              "      <td>1024.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18_model.GELU_18</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19_model.Dropout_19</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 1024]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20_model.Linear_20</th>\n",
              "      <td>[1024, 512]</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>524800.0</td>\n",
              "      <td>524288.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21_model.BatchNorm1d_21</th>\n",
              "      <td>[512]</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>1024.0</td>\n",
              "      <td>512.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22_model.GELU_22</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23_model.Dropout_23</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24_model.Linear_24</th>\n",
              "      <td>[512, 512]</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>262656.0</td>\n",
              "      <td>262144.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25_model.BatchNorm1d_25</th>\n",
              "      <td>[512]</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>1024.0</td>\n",
              "      <td>512.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26_model.GELU_26</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27_model.Dropout_27</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 512]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28_model.Linear_28</th>\n",
              "      <td>[512, 256]</td>\n",
              "      <td>[2048, 256]</td>\n",
              "      <td>131328.0</td>\n",
              "      <td>131072.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29_model.BatchNorm1d_29</th>\n",
              "      <td>[256]</td>\n",
              "      <td>[2048, 256]</td>\n",
              "      <td>512.0</td>\n",
              "      <td>256.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30_model.GELU_30</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 256]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31_model.Dropout_31</th>\n",
              "      <td>-</td>\n",
              "      <td>[2048, 256]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32_model.Linear_32</th>\n",
              "      <td>[256, 42]</td>\n",
              "      <td>[2048, 42]</td>\n",
              "      <td>10794.0</td>\n",
              "      <td>10752.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dfe31dd7-e975-424c-bf92-ce5b9e45abbd')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-dfe31dd7-e975-424c-bf92-ce5b9e45abbd button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-dfe31dd7-e975-424c-bf92-ce5b9e45abbd');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-04807f9f-bbb8-4b1e-9acd-c51d10e25555\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-04807f9f-bbb8-4b1e-9acd-c51d10e25555')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-04807f9f-bbb8-4b1e-9acd-c51d10e25555 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                         Kernel Shape  Output Shape     Params  Mult-Adds\n",
              "Layer                                                                    \n",
              "0_model.Linear_0         [1107, 2048]  [2048, 2048]  2269184.0  2267136.0\n",
              "1_model.BatchNorm1d_1          [2048]  [2048, 2048]     4096.0     2048.0\n",
              "2_model.GELU_2                      -  [2048, 2048]        NaN        NaN\n",
              "3_model.Dropout_3                   -  [2048, 2048]        NaN        NaN\n",
              "4_model.Linear_4         [2048, 4096]  [2048, 4096]  8392704.0  8388608.0\n",
              "5_model.BatchNorm1d_5          [4096]  [2048, 4096]     8192.0     4096.0\n",
              "6_model.GELU_6                      -  [2048, 4096]        NaN        NaN\n",
              "7_model.Dropout_7                   -  [2048, 4096]        NaN        NaN\n",
              "8_model.Linear_8         [4096, 2048]  [2048, 2048]  8390656.0  8388608.0\n",
              "9_model.BatchNorm1d_9          [2048]  [2048, 2048]     4096.0     2048.0\n",
              "10_model.GELU_10                    -  [2048, 2048]        NaN        NaN\n",
              "11_model.Dropout_11                 -  [2048, 2048]        NaN        NaN\n",
              "12_model.Linear_12       [2048, 1024]  [2048, 1024]  2098176.0  2097152.0\n",
              "13_model.BatchNorm1d_13        [1024]  [2048, 1024]     2048.0     1024.0\n",
              "14_model.GELU_14                    -  [2048, 1024]        NaN        NaN\n",
              "15_model.Dropout_15                 -  [2048, 1024]        NaN        NaN\n",
              "16_model.Linear_16       [1024, 1024]  [2048, 1024]  1049600.0  1048576.0\n",
              "17_model.BatchNorm1d_17        [1024]  [2048, 1024]     2048.0     1024.0\n",
              "18_model.GELU_18                    -  [2048, 1024]        NaN        NaN\n",
              "19_model.Dropout_19                 -  [2048, 1024]        NaN        NaN\n",
              "20_model.Linear_20        [1024, 512]   [2048, 512]   524800.0   524288.0\n",
              "21_model.BatchNorm1d_21         [512]   [2048, 512]     1024.0      512.0\n",
              "22_model.GELU_22                    -   [2048, 512]        NaN        NaN\n",
              "23_model.Dropout_23                 -   [2048, 512]        NaN        NaN\n",
              "24_model.Linear_24         [512, 512]   [2048, 512]   262656.0   262144.0\n",
              "25_model.BatchNorm1d_25         [512]   [2048, 512]     1024.0      512.0\n",
              "26_model.GELU_26                    -   [2048, 512]        NaN        NaN\n",
              "27_model.Dropout_27                 -   [2048, 512]        NaN        NaN\n",
              "28_model.Linear_28         [512, 256]   [2048, 256]   131328.0   131072.0\n",
              "29_model.BatchNorm1d_29         [256]   [2048, 256]      512.0      256.0\n",
              "30_model.GELU_30                    -   [2048, 256]        NaN        NaN\n",
              "31_model.Dropout_31                 -   [2048, 256]        NaN        NaN\n",
              "32_model.Linear_32          [256, 42]    [2048, 42]    10794.0    10752.0"
            ]
          },
          "execution_count": 85,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "INPUT_SIZE  = (2*config['context'] + 1) * 27 # Why is this the case?\n",
        "model       = Network(INPUT_SIZE, len(val_data.phonemes)).to(device)\n",
        "summary(model, frames.to(device))\n",
        "# Check number of parameters of your network\n",
        "# Remember, you are limited to 24 million parameters for HW1 (including ensembles)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UROGEVJevKD-"
      },
      "outputs": [],
      "source": [
        "criterion = torch.nn.CrossEntropyLoss() # Defining Loss function.\n",
        "# We use CE because the task is multi-class classification\n",
        "#model.apply(init_xavier)\n",
        "#optimizer = torch.optim.Adam(model.parameters(), lr= config['init_lr']) #Defining Optimizer\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=0.000001)\n",
        "#optimizer = torch.optim.Adam(model.parameters())\n",
        "#scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max',\n",
        " #   factor=0.1, patience=2, verbose=True)\n",
        "\n",
        "epochs = config['epochs']\n",
        "total_steps = epochs * len(train_loader)\n",
        "new_epochs = 10\n",
        "min_lr = 3.855e-7\n",
        "#scheduler =torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=total_steps, eta_min=0)\n",
        "#lr_lambda = lambda epoch: lr + (max_lr - lr) * epoch / 20\n",
        "\n",
        "# Create the learning rate scheduler\n",
        "#scheduler = LambdaLR(optimizer, lr_lambda)\n",
        "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=new_epochs)\n",
        "scheduler.load_state_dict(torch.load(\"model_checkpoint_high_99.pth\"))\n",
        "\n",
        "\n",
        "# Recommended : Define Scheduler for Learning Rate,\n",
        "# including but not limited to StepLR, MultiStepLR, CosineAnnealingLR, ReduceLROnPlateau, etc.\n",
        "# You can refer to Pytorch documentation for more information on how to use them.\n",
        "scaler = torch.cuda.amp.GradScaler()\n",
        "# Is your training time very high?\n",
        "# Look into mixed precision training if your GPU (Tesla T4, V100, etc) can make use of it\n",
        "# Refer - https://pytorch.org/docs/stable/notes/amp_examples.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bOSPuNkt8_S6",
        "outputId": "d3bb7281-0308-4ad1-b5b5-996e29acf33f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Current learning rate: [1e-06]\n"
          ]
        }
      ],
      "source": [
        "current_learning_rate = scheduler.get_last_lr()\n",
        "print(f\"Current learning rate: {current_learning_rate}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pMh3IYgq8-W7"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IBwunYpyugFg"
      },
      "source": [
        "# Training and Validation Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1JgeNhx4x2-P"
      },
      "source": [
        "This section covers the training, and validation functions for each epoch of running your experiment with a given model architecture. The code has been provided to you, but we recommend going through the comments to understand the workflow to enable you to write these loops for future HWs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XblOHEVtKab2",
        "outputId": "86b42ef2-5d08-4909-e8df-94603a50f65f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "9"
            ]
          },
          "execution_count": 151,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "torch.cuda.empty_cache()\n",
        "gc.collect()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8wjPz7DHqKcL"
      },
      "outputs": [],
      "source": [
        "def train(model, dataloader, optimizer, criterion):\n",
        "\n",
        "    model.train()\n",
        "    tloss, tacc = 0, 0 # Monitoring loss and accuracy\n",
        "    batch_bar   = tqdm(total=len(train_loader), dynamic_ncols=True, leave=False, position=0, desc='Train')\n",
        "\n",
        "    for i, (frames, phonemes) in enumerate(dataloader):\n",
        "\n",
        "        ### Initialize Gradients\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        with torch.autocast(device_type='cuda', dtype=torch.float16):\n",
        "          ### Move Data to Device (Ideally GPU)\n",
        "          frames      = frames.to(device)\n",
        "          phonemes    = phonemes.to(device)\n",
        "\n",
        "          ### Forward Propagation\n",
        "          logits  = model(frames)\n",
        "\n",
        "          ### Loss Calculation\n",
        "          loss    = criterion(logits, phonemes)\n",
        "\n",
        "        ### Backward Propagation\n",
        "        #loss.backward()\n",
        "        scaler.scale(loss).backward()\n",
        "        # scaler.step(optimizer)\n",
        "        #scaler.update()\n",
        "\n",
        "        ### Gradient Descent\n",
        "        optimizer.step()\n",
        "\n",
        "\n",
        "        tloss   += loss.item()\n",
        "        tacc    += torch.sum(torch.argmax(logits, dim= 1) == phonemes).item()/logits.shape[0]\n",
        "\n",
        "        batch_bar.set_postfix(loss=\"{:.04f}\".format(float(tloss / (i + 1))),\n",
        "                              acc=\"{:.04f}%\".format(float(tacc*100 / (i + 1))))\n",
        "        batch_bar.update()\n",
        "\n",
        "\n",
        "        ### Release memory\n",
        "        del frames, phonemes, logits\n",
        "        torch.cuda.empty_cache()\n",
        "\n",
        "    scheduler.step()\n",
        "    batch_bar.close()\n",
        "    tloss   /= len(train_loader)\n",
        "    tacc    /= len(train_loader)\n",
        "\n",
        "    return tloss, tacc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q5npQNFH315V"
      },
      "outputs": [],
      "source": [
        "def eval(model, dataloader):\n",
        "\n",
        "    model.eval() # set model in evaluation mode\n",
        "    vloss, vacc = 0, 0 # Monitoring loss and accuracy\n",
        "    batch_bar   = tqdm(total=len(val_loader), dynamic_ncols=True, position=0, leave=False, desc='Val')\n",
        "\n",
        "    for i, (frames, phonemes) in enumerate(dataloader):\n",
        "\n",
        "        ### Move data to device (ideally GPU)\n",
        "        frames      = frames.to(device)\n",
        "        phonemes    = phonemes.to(device)\n",
        "\n",
        "        # makes sure that there are no gradients computed as we are not training the model now\n",
        "        with torch.inference_mode():\n",
        "            ### Forward Propagation\n",
        "            logits  = model(frames)\n",
        "            ### Loss Calculation\n",
        "            loss    = criterion(logits, phonemes)\n",
        "\n",
        "        vloss   += loss.item()\n",
        "        vacc    += torch.sum(torch.argmax(logits, dim= 1) == phonemes).item()/logits.shape[0]\n",
        "\n",
        "        # Do you think we need loss.backward() and optimizer.step() here?\n",
        "\n",
        "        batch_bar.set_postfix(loss=\"{:.04f}\".format(float(vloss / (i + 1))),\n",
        "                              acc=\"{:.04f}%\".format(float(vacc*100 / (i + 1))))\n",
        "        batch_bar.update()\n",
        "\n",
        "        ### Release memory\n",
        "        del frames, phonemes, logits\n",
        "        torch.cuda.empty_cache()\n",
        "\n",
        "    batch_bar.close()\n",
        "    vloss   /= len(val_loader)\n",
        "    vacc    /= len(val_loader)\n",
        "\n",
        "    return vloss, vacc"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yMd_XxPku5qp"
      },
      "source": [
        "# Weights and Biases Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tjIbhR1wwbgI"
      },
      "source": [
        "This section is to enable logging metrics and files with Weights and Biases. Please refer to wandb documentationa and recitation 0 that covers the use of weights and biases for logging, hyperparameter tuning and monitoring your runs for your homeworks. Using this tool makes it very easy to show results when submitting your code and models for homeworks, and also extremely useful for study groups to organize and run ablations under a single team in wandb.\n",
        "\n",
        "We have written code for you to make use of it out of the box, so that you start using wandb for all your HWs from the beginning."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SCDYx5VEu6qI",
        "outputId": "60bd9e35-6c66-46f0-fa12-1cf241aa4d48"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Calling wandb.login() after wandb.init() has no effect.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 153,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "wandb.login(key=\"\") #API Key is in your wandb account, under settings (wandb.ai/settings)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "xvUnYd3Bw2up",
        "outputId": "246c9c78-e23e-494e-953a-64fae264632b"
      },
      "outputs": [],
      "source": [
        "# Create your wandb run\n",
        "run = wandb.init(\n",
        "    name    = \"fcnn-high-cutoff\", ### Wandb creates random run names if you skip this field, we recommend you give useful names\n",
        "    reinit  = True, ### Allows reinitalizing runs when you re-run this cell\n",
        "    #id     = \"\", ### Insert specific run id here if you want to resume a previous run\n",
        "    #resume = \"must\", ### You need this to resume previous runs, but comment out reinit = True when using this\n",
        "    project = \"\", ### Project should be created in your wandb account\n",
        "    config  = config ### Wandb Config for your run\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wft15E_IxYFi",
        "outputId": "0496a927-2804-45d2-f4c0-eb9429b812d1"
      },
      "outputs": [],
      "source": [
        "### Save your model architecture as a string with str(model)\n",
        "model_arch  = str(model)\n",
        "\n",
        "### Save it in a txt file\n",
        "arch_file   = open(\"model_high_cutoff_extra_epochs.txt\", \"w\")\n",
        "file_write  = arch_file.write(model_arch)\n",
        "arch_file.close()\n",
        "\n",
        "### log it in your wandb run with wandb.save()\n",
        "wandb.save('model_high_cutoff_extra_epochs.txt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cw4lcpXOoGhw"
      },
      "outputs": [],
      "source": [
        "for epoch in range(150):\n",
        "  name = f'model_checkpoint_mp_{epoch}.pth'\n",
        "  wandb.save(name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nclx_04fu7Dd"
      },
      "source": [
        "# Experiment"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MdLMWfEpyGOB"
      },
      "source": [
        "Now, it is time to finally run your ablations! Have fun!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 967,
          "referenced_widgets": [
            "06b78d90e45e4df7a1a1a1db7b5319ed",
            "336f44d2efec4d6e9cb7268628952083",
            "6abf9c3d91b74c83b55dba788b18b6fe",
            "78ac608a02d74c0fa6fcd70f16ad6fd9",
            "fef2295782ba43418ea14464bc086217",
            "8ec674315d074c7cbf81364c8059929b",
            "e25a6a9bd7a340f690f9ac3f73f6038a",
            "e8c59bc847524e33822584a587241ca5"
          ]
        },
        "id": "MG4F77Nm0Am9",
        "outputId": "2b8110cc-7ec9-40e2-dd78-a2d409106ee8"
      },
      "outputs": [],
      "source": [
        "# Iterate over number of epochs to train and evaluate your model\n",
        "torch.cuda.empty_cache()\n",
        "gc.collect()\n",
        "wandb.watch(model, log=\"all\")\n",
        "model.load_state_dict(torch.load(\"model_checkpoint_high_99.pth\"))\n",
        "for epoch in range(new_epochs):\n",
        "\n",
        "    print(\"\\nEpoch {}/{}\".format(epoch+1, new_epochs))\n",
        "\n",
        "    curr_lr                 = float(optimizer.param_groups[0]['lr'])\n",
        "    train_loss, train_acc   = train(model, train_loader, optimizer, criterion)\n",
        "    val_loss, val_acc       = eval(model, val_loader)\n",
        "\n",
        "    print(\"\\tTrain Acc {:.04f}%\\tTrain Loss {:.04f}\\t Learning Rate {:.07f}\".format(train_acc*100, train_loss, curr_lr))\n",
        "    print(\"\\tVal Acc {:.04f}%\\tVal Loss {:.04f}\".format(val_acc*100, val_loss))\n",
        "\n",
        "    ### Log metrics at each epoch in your run\n",
        "    # Optionally, you can log at each batch inside train/eval functions\n",
        "    # (explore wandb documentation/wandb recitation)\n",
        "    wandb.log({'train_acc': train_acc*100, 'train_loss': train_loss,\n",
        "               'val_acc': val_acc*100, 'valid_loss': val_loss, 'lr': curr_lr})\n",
        "    name = f'additional_model_checkpoint_high_{epoch}.pth'\n",
        "    torch.save(model.state_dict(), name)\n",
        "    ### Highly Recommended: Save checkpoint in drive and/or wandb if accuracy is better than your current best\n",
        "\n",
        "### Finish your wandb run\n",
        "run.finish()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QbxvEbd7C3ql"
      },
      "outputs": [],
      "source": [
        "torch.save(model.state_dict(), 'model_high.pth')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_kXwf5YUo_4A"
      },
      "source": [
        "# Testing and submission to Kaggle"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WI1hSFYLpJvH"
      },
      "source": [
        "Before we get to the following code, make sure to see the format of submission given in *sample_submission.csv*. Once you have done so, it is time to fill the following function to complete your inference on test data. Refer the eval function from previous cells to get an idea of how to go about completing this function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R-SU9fZ3xHtk"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "from operator import itemgetter\n",
        "\n",
        "def test(model, test_loader):\n",
        "    ### What you call for model to perform inference?\n",
        "    model.eval() # TODO train or eval?\n",
        "\n",
        "    ### List to store predicted phonemes of test data\n",
        "    test_predictions = []\n",
        "    res = np.array([])\n",
        "    ### Which mode do you need to avoid gradients?\n",
        "    with torch.inference_mode():\n",
        "\n",
        "        for i, mfccs in enumerate(tqdm(test_loader)):\n",
        "\n",
        "            mfccs   = mfccs.to(device)\n",
        "\n",
        "            logits  = model(mfccs)\n",
        "\n",
        "            ### Get most likely predicted phoneme with argmax\n",
        "            predicted_phonemes = torch.argmax(logits, dim=1)\n",
        "            get_elements = itemgetter(*predicted_phonemes)\n",
        "            ### How do you store predicted_phonemes with test_predictions? Hint, look at eval\n",
        "            test_predictions.extend(get_elements(PHONEMES))\n",
        "\n",
        "    return test_predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wG9v6Xmxu7wp",
        "outputId": "fb2a815c-1bee-466c-b46c-c8aef2e6e5a3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/945 [00:00<?, ?it/s]\u001b[A\n",
            "  0%|          | 1/945 [00:00<04:45,  3.31it/s]\u001b[A\n",
            "  0%|          | 3/945 [00:00<01:58,  7.98it/s]\u001b[A\n",
            "  1%|          | 5/945 [00:00<01:26, 10.84it/s]\u001b[A\n",
            "  1%|          | 7/945 [00:00<01:10, 13.30it/s]\u001b[A\n",
            "  1%|          | 9/945 [00:00<01:01, 15.13it/s]\u001b[A\n",
            "  1%|▏         | 12/945 [00:01<02:17,  6.78it/s]\u001b[A\n",
            "  2%|▏         | 15/945 [00:01<01:39,  9.35it/s]\u001b[A\n",
            "  2%|▏         | 18/945 [00:01<01:17, 11.93it/s]\u001b[A\n",
            "  2%|▏         | 21/945 [00:01<01:04, 14.26it/s]\u001b[A\n",
            "  3%|▎         | 24/945 [00:02<00:56, 16.32it/s]\u001b[A\n",
            "  3%|▎         | 27/945 [00:02<00:50, 18.24it/s]\u001b[A\n",
            "  3%|▎         | 30/945 [00:02<00:47, 19.16it/s]\u001b[A\n",
            "  3%|▎         | 33/945 [00:02<00:44, 20.55it/s]\u001b[A\n",
            "  4%|▍         | 36/945 [00:02<00:43, 21.11it/s]\u001b[A\n",
            "  4%|▍         | 39/945 [00:02<00:41, 21.94it/s]\u001b[A\n",
            "  4%|▍         | 42/945 [00:02<00:39, 22.63it/s]\u001b[A\n",
            "  5%|▍         | 45/945 [00:02<00:39, 23.06it/s]\u001b[A\n",
            "  5%|▌         | 48/945 [00:03<00:38, 23.50it/s]\u001b[A\n",
            "  5%|▌         | 51/945 [00:03<00:37, 23.74it/s]\u001b[A\n",
            "  6%|▌         | 54/945 [00:03<00:37, 23.82it/s]\u001b[A\n",
            "  6%|▌         | 57/945 [00:03<01:12, 12.19it/s]\u001b[A\n",
            "  6%|▋         | 60/945 [00:03<01:01, 14.46it/s]\u001b[A\n",
            "  7%|▋         | 63/945 [00:04<00:53, 16.49it/s]\u001b[A\n",
            "  7%|▋         | 66/945 [00:04<00:48, 18.21it/s]\u001b[A\n",
            "  7%|▋         | 69/945 [00:04<00:45, 19.42it/s]\u001b[A\n",
            "  8%|▊         | 72/945 [00:04<00:42, 20.35it/s]\u001b[A\n",
            "  8%|▊         | 75/945 [00:04<00:40, 21.29it/s]\u001b[A\n",
            "  8%|▊         | 78/945 [00:04<00:39, 21.75it/s]\u001b[A\n",
            "  9%|▊         | 81/945 [00:04<00:38, 22.39it/s]\u001b[A\n",
            "  9%|▉         | 84/945 [00:04<00:37, 23.12it/s]\u001b[A\n",
            "  9%|▉         | 87/945 [00:05<00:36, 23.38it/s]\u001b[A\n",
            " 10%|▉         | 90/945 [00:05<00:36, 23.65it/s]\u001b[A\n",
            " 10%|▉         | 93/945 [00:05<00:36, 23.59it/s]\u001b[A\n",
            " 10%|█         | 96/945 [00:05<00:36, 23.44it/s]\u001b[A\n",
            " 10%|█         | 99/945 [00:05<00:35, 23.80it/s]\u001b[A\n",
            " 11%|█         | 102/945 [00:05<00:35, 23.63it/s]\u001b[A\n",
            " 11%|█         | 105/945 [00:05<00:35, 23.71it/s]\u001b[A\n",
            " 11%|█▏        | 108/945 [00:05<00:35, 23.90it/s]\u001b[A\n",
            " 12%|█▏        | 111/945 [00:06<00:35, 23.59it/s]\u001b[A\n",
            " 12%|█▏        | 114/945 [00:06<00:34, 23.94it/s]\u001b[A\n",
            " 12%|█▏        | 117/945 [00:06<00:34, 24.22it/s]\u001b[A\n",
            " 13%|█▎        | 120/945 [00:06<00:34, 24.04it/s]\u001b[A\n",
            " 13%|█▎        | 123/945 [00:06<00:34, 24.01it/s]\u001b[A\n",
            " 13%|█▎        | 126/945 [00:06<00:34, 23.69it/s]\u001b[A\n",
            " 14%|█▎        | 129/945 [00:06<00:34, 23.90it/s]\u001b[A\n",
            " 14%|█▍        | 132/945 [00:06<00:33, 24.14it/s]\u001b[A\n",
            " 14%|█▍        | 135/945 [00:07<01:05, 12.44it/s]\u001b[A\n",
            " 15%|█▍        | 138/945 [00:07<00:56, 14.40it/s]\u001b[A\n",
            " 15%|█▍        | 141/945 [00:07<00:49, 16.34it/s]\u001b[A\n",
            " 15%|█▌        | 144/945 [00:07<00:44, 18.00it/s]\u001b[A\n",
            " 16%|█▌        | 147/945 [00:07<00:41, 19.27it/s]\u001b[A\n",
            " 16%|█▌        | 150/945 [00:08<00:39, 20.10it/s]\u001b[A\n",
            " 16%|█▌        | 153/945 [00:08<00:37, 20.89it/s]\u001b[A\n",
            " 17%|█▋        | 156/945 [00:08<00:36, 21.81it/s]\u001b[A\n",
            " 17%|█▋        | 159/945 [00:08<00:35, 22.02it/s]\u001b[A\n",
            " 17%|█▋        | 162/945 [00:08<00:34, 22.69it/s]\u001b[A\n",
            " 17%|█▋        | 165/945 [00:08<00:33, 23.13it/s]\u001b[A\n",
            " 18%|█▊        | 168/945 [00:08<00:33, 23.53it/s]\u001b[A\n",
            " 18%|█▊        | 171/945 [00:08<00:32, 23.68it/s]\u001b[A\n",
            " 18%|█▊        | 174/945 [00:09<00:32, 23.94it/s]\u001b[A\n",
            " 19%|█▊        | 177/945 [00:09<00:32, 23.73it/s]\u001b[A\n",
            " 19%|█▉        | 180/945 [00:09<00:32, 23.70it/s]\u001b[A\n",
            " 19%|█▉        | 183/945 [00:09<00:31, 23.88it/s]\u001b[A\n",
            " 20%|█▉        | 186/945 [00:09<00:31, 23.84it/s]\u001b[A\n",
            " 20%|██        | 189/945 [00:09<00:31, 23.88it/s]\u001b[A\n",
            " 20%|██        | 192/945 [00:09<00:31, 24.04it/s]\u001b[A\n",
            " 21%|██        | 195/945 [00:09<00:31, 23.96it/s]\u001b[A\n",
            " 21%|██        | 198/945 [00:10<00:31, 24.01it/s]\u001b[A\n",
            " 21%|██▏       | 201/945 [00:10<00:31, 23.77it/s]\u001b[A\n",
            " 22%|██▏       | 204/945 [00:10<00:31, 23.87it/s]\u001b[A\n",
            " 22%|██▏       | 207/945 [00:10<00:54, 13.61it/s]\u001b[A\n",
            " 22%|██▏       | 210/945 [00:10<00:46, 15.79it/s]\u001b[A\n",
            " 23%|██▎       | 213/945 [00:11<00:41, 17.71it/s]\u001b[A\n",
            " 23%|██▎       | 216/945 [00:11<00:37, 19.23it/s]\u001b[A\n",
            " 23%|██▎       | 219/945 [00:11<00:35, 20.45it/s]\u001b[A\n",
            " 23%|██▎       | 222/945 [00:11<00:33, 21.36it/s]\u001b[A\n",
            " 24%|██▍       | 225/945 [00:11<00:32, 21.96it/s]\u001b[A\n",
            " 24%|██▍       | 228/945 [00:11<00:31, 22.43it/s]\u001b[A\n",
            " 24%|██▍       | 231/945 [00:11<00:30, 23.16it/s]\u001b[A\n",
            " 25%|██▍       | 234/945 [00:11<00:30, 22.95it/s]\u001b[A\n",
            " 25%|██▌       | 237/945 [00:12<00:30, 23.24it/s]\u001b[A\n",
            " 25%|██▌       | 240/945 [00:12<00:30, 23.32it/s]\u001b[A\n",
            " 26%|██▌       | 243/945 [00:12<00:29, 23.63it/s]\u001b[A\n",
            " 26%|██▌       | 246/945 [00:12<00:29, 23.66it/s]\u001b[A\n",
            " 26%|██▋       | 249/945 [00:12<00:29, 23.73it/s]\u001b[A\n",
            " 27%|██▋       | 252/945 [00:12<00:28, 23.92it/s]\u001b[A\n",
            " 27%|██▋       | 255/945 [00:12<00:29, 23.57it/s]\u001b[A\n",
            " 27%|██▋       | 258/945 [00:12<00:28, 23.74it/s]\u001b[A\n",
            " 28%|██▊       | 261/945 [00:13<00:29, 23.51it/s]\u001b[A\n",
            " 28%|██▊       | 264/945 [00:13<00:28, 23.60it/s]\u001b[A\n",
            " 28%|██▊       | 267/945 [00:13<00:28, 23.39it/s]\u001b[A\n",
            " 29%|██▊       | 270/945 [00:13<00:29, 23.22it/s]\u001b[A\n",
            " 29%|██▉       | 273/945 [00:13<00:28, 23.46it/s]\u001b[A\n",
            " 29%|██▉       | 276/945 [00:13<00:28, 23.70it/s]\u001b[A\n",
            " 30%|██▉       | 279/945 [00:13<00:28, 23.35it/s]\u001b[A\n",
            " 30%|██▉       | 282/945 [00:13<00:27, 23.77it/s]\u001b[A\n",
            " 30%|███       | 285/945 [00:14<00:27, 23.95it/s]\u001b[A\n",
            " 30%|███       | 288/945 [00:14<00:27, 24.18it/s]\u001b[A\n",
            " 31%|███       | 291/945 [00:14<00:26, 24.25it/s]\u001b[A\n",
            " 31%|███       | 294/945 [00:14<00:27, 24.10it/s]\u001b[A\n",
            " 31%|███▏      | 297/945 [00:14<00:26, 24.08it/s]\u001b[A\n",
            " 32%|███▏      | 300/945 [00:14<00:26, 24.04it/s]\u001b[A\n",
            " 32%|███▏      | 303/945 [00:14<00:26, 24.12it/s]\u001b[A\n",
            " 32%|███▏      | 306/945 [00:14<00:26, 24.07it/s]\u001b[A\n",
            " 33%|███▎      | 309/945 [00:15<00:26, 24.10it/s]\u001b[A\n",
            " 33%|███▎      | 312/945 [00:15<00:26, 24.04it/s]\u001b[A\n",
            " 33%|███▎      | 315/945 [00:15<00:26, 23.92it/s]\u001b[A\n",
            " 34%|███▎      | 318/945 [00:15<00:26, 23.72it/s]\u001b[A\n",
            " 34%|███▍      | 321/945 [00:15<00:26, 23.91it/s]\u001b[A\n",
            " 34%|███▍      | 324/945 [00:15<00:25, 23.96it/s]\u001b[A\n",
            " 35%|███▍      | 327/945 [00:15<00:25, 24.27it/s]\u001b[A\n",
            " 35%|███▍      | 330/945 [00:15<00:25, 24.18it/s]\u001b[A\n",
            " 35%|███▌      | 333/945 [00:16<00:25, 23.79it/s]\u001b[A\n",
            " 36%|███▌      | 336/945 [00:16<00:26, 23.37it/s]\u001b[A\n",
            " 36%|███▌      | 339/945 [00:16<00:26, 23.23it/s]\u001b[A\n",
            " 36%|███▌      | 342/945 [00:16<00:25, 23.41it/s]\u001b[A\n",
            " 37%|███▋      | 345/945 [00:16<00:25, 23.50it/s]\u001b[A\n",
            " 37%|███▋      | 348/945 [00:16<00:25, 23.81it/s]\u001b[A\n",
            " 37%|███▋      | 351/945 [00:16<00:24, 23.81it/s]\u001b[A\n",
            " 37%|███▋      | 354/945 [00:16<00:25, 23.42it/s]\u001b[A\n",
            " 38%|███▊      | 357/945 [00:17<00:24, 23.74it/s]\u001b[A\n",
            " 38%|███▊      | 360/945 [00:17<00:24, 23.55it/s]\u001b[A\n",
            " 38%|███▊      | 363/945 [00:17<00:24, 23.64it/s]\u001b[A\n",
            " 39%|███▊      | 366/945 [00:17<00:24, 23.75it/s]\u001b[A\n",
            " 39%|███▉      | 369/945 [00:17<00:24, 23.87it/s]\u001b[A\n",
            " 39%|███▉      | 372/945 [00:17<00:23, 23.98it/s]\u001b[A\n",
            " 40%|███▉      | 375/945 [00:17<00:23, 24.18it/s]\u001b[A\n",
            " 40%|████      | 378/945 [00:17<00:23, 24.06it/s]\u001b[A\n",
            " 40%|████      | 381/945 [00:18<00:23, 23.98it/s]\u001b[A\n",
            " 41%|████      | 384/945 [00:18<00:23, 23.39it/s]\u001b[A\n",
            " 41%|████      | 387/945 [00:18<00:23, 23.32it/s]\u001b[A\n",
            " 41%|████▏     | 390/945 [00:18<00:23, 23.39it/s]\u001b[A\n",
            " 42%|████▏     | 393/945 [00:18<00:23, 23.75it/s]\u001b[A\n",
            " 42%|████▏     | 396/945 [00:18<00:22, 24.11it/s]\u001b[A\n",
            " 42%|████▏     | 399/945 [00:18<00:22, 24.31it/s]\u001b[A\n",
            " 43%|████▎     | 402/945 [00:18<00:22, 24.54it/s]\u001b[A\n",
            " 43%|████▎     | 405/945 [00:19<00:22, 24.26it/s]\u001b[A\n",
            " 43%|████▎     | 408/945 [00:19<00:22, 23.56it/s]\u001b[A\n",
            " 43%|████▎     | 411/945 [00:19<00:22, 23.81it/s]\u001b[A\n",
            " 44%|████▍     | 414/945 [00:19<00:22, 24.06it/s]\u001b[A\n",
            " 44%|████▍     | 417/945 [00:19<00:21, 24.32it/s]\u001b[A\n",
            " 44%|████▍     | 420/945 [00:19<00:21, 24.13it/s]\u001b[A\n",
            " 45%|████▍     | 423/945 [00:19<00:21, 24.25it/s]\u001b[A\n",
            " 45%|████▌     | 426/945 [00:19<00:21, 24.29it/s]\u001b[A\n",
            " 45%|████▌     | 429/945 [00:20<00:21, 24.44it/s]\u001b[A\n",
            " 46%|████▌     | 432/945 [00:20<00:21, 24.41it/s]\u001b[A\n",
            " 46%|████▌     | 435/945 [00:20<00:20, 24.48it/s]\u001b[A\n",
            " 46%|████▋     | 438/945 [00:20<00:20, 24.30it/s]\u001b[A\n",
            " 47%|████▋     | 441/945 [00:20<00:20, 24.15it/s]\u001b[A\n",
            " 47%|████▋     | 444/945 [00:20<00:20, 24.14it/s]\u001b[A\n",
            " 47%|████▋     | 447/945 [00:20<00:20, 24.36it/s]\u001b[A\n",
            " 48%|████▊     | 450/945 [00:20<00:20, 24.46it/s]\u001b[A\n",
            " 48%|████▊     | 453/945 [00:21<00:20, 23.72it/s]\u001b[A\n",
            " 48%|████▊     | 456/945 [00:21<00:20, 23.78it/s]\u001b[A\n",
            " 49%|████▊     | 459/945 [00:21<00:39, 12.27it/s]\u001b[A\n",
            " 49%|████▉     | 462/945 [00:21<00:33, 14.34it/s]\u001b[A\n",
            " 49%|████▉     | 465/945 [00:21<00:29, 16.27it/s]\u001b[A\n",
            " 50%|████▉     | 468/945 [00:22<00:26, 18.07it/s]\u001b[A\n",
            " 50%|████▉     | 471/945 [00:22<00:33, 14.18it/s]\u001b[A\n",
            " 50%|█████     | 474/945 [00:22<00:28, 16.27it/s]\u001b[A\n",
            " 50%|█████     | 477/945 [00:22<00:26, 17.88it/s]\u001b[A\n",
            " 51%|█████     | 480/945 [00:22<00:23, 19.42it/s]\u001b[A\n",
            " 51%|█████     | 483/945 [00:22<00:22, 20.50it/s]\u001b[A\n",
            " 51%|█████▏    | 486/945 [00:23<00:21, 21.38it/s]\u001b[A\n",
            " 52%|█████▏    | 489/945 [00:23<00:20, 22.19it/s]\u001b[A\n",
            " 52%|█████▏    | 492/945 [00:23<00:19, 22.69it/s]\u001b[A\n",
            " 52%|█████▏    | 495/945 [00:23<00:19, 22.83it/s]\u001b[A\n",
            " 53%|█████▎    | 498/945 [00:23<00:19, 22.94it/s]\u001b[A\n",
            " 53%|█████▎    | 501/945 [00:23<00:19, 23.35it/s]\u001b[A\n",
            " 53%|█████▎    | 504/945 [00:23<00:18, 23.53it/s]\u001b[A\n",
            " 54%|█████▎    | 507/945 [00:23<00:18, 23.70it/s]\u001b[A\n",
            " 54%|█████▍    | 510/945 [00:24<00:18, 23.51it/s]\u001b[A\n",
            " 54%|█████▍    | 513/945 [00:24<00:18, 23.64it/s]\u001b[A\n",
            " 55%|█████▍    | 516/945 [00:24<00:17, 23.86it/s]\u001b[A\n",
            " 55%|█████▍    | 519/945 [00:24<00:17, 24.00it/s]\u001b[A\n",
            " 55%|█████▌    | 522/945 [00:24<00:17, 24.15it/s]\u001b[A\n",
            " 56%|█████▌    | 525/945 [00:24<00:17, 23.81it/s]\u001b[A\n",
            " 56%|█████▌    | 528/945 [00:24<00:17, 23.49it/s]\u001b[A\n",
            " 56%|█████▌    | 531/945 [00:24<00:17, 23.84it/s]\u001b[A\n",
            " 57%|█████▋    | 534/945 [00:25<00:17, 23.73it/s]\u001b[A\n",
            " 57%|█████▋    | 537/945 [00:25<00:17, 23.73it/s]\u001b[A\n",
            " 57%|█████▋    | 540/945 [00:25<00:17, 23.74it/s]\u001b[A\n",
            " 57%|█████▋    | 543/945 [00:25<00:16, 23.96it/s]\u001b[A\n",
            " 58%|█████▊    | 546/945 [00:25<00:16, 24.15it/s]\u001b[A\n",
            " 58%|█████▊    | 549/945 [00:25<00:16, 23.89it/s]\u001b[A\n",
            " 58%|█████▊    | 552/945 [00:25<00:16, 23.58it/s]\u001b[A\n",
            " 59%|█████▊    | 555/945 [00:25<00:16, 23.62it/s]\u001b[A\n",
            " 59%|█████▉    | 558/945 [00:26<00:16, 23.95it/s]\u001b[A\n",
            " 59%|█████▉    | 561/945 [00:26<00:15, 24.22it/s]\u001b[A\n",
            " 60%|█████▉    | 564/945 [00:26<00:15, 23.92it/s]\u001b[A\n",
            " 60%|██████    | 567/945 [00:26<00:15, 23.73it/s]\u001b[A\n",
            " 60%|██████    | 570/945 [00:26<00:16, 23.37it/s]\u001b[A\n",
            " 61%|██████    | 573/945 [00:26<00:15, 23.43it/s]\u001b[A\n",
            " 61%|██████    | 576/945 [00:26<00:16, 22.96it/s]\u001b[A\n",
            " 61%|██████▏   | 579/945 [00:26<00:16, 22.55it/s]\u001b[A\n",
            " 62%|██████▏   | 582/945 [00:27<00:17, 21.06it/s]\u001b[A\n",
            " 62%|██████▏   | 585/945 [00:27<00:17, 20.35it/s]\u001b[A\n",
            " 62%|██████▏   | 588/945 [00:27<00:17, 20.07it/s]\u001b[A\n",
            " 63%|██████▎   | 591/945 [00:27<00:17, 20.50it/s]\u001b[A\n",
            " 63%|██████▎   | 594/945 [00:27<00:17, 20.11it/s]\u001b[A\n",
            " 63%|██████▎   | 597/945 [00:27<00:17, 20.19it/s]\u001b[A\n",
            " 63%|██████▎   | 600/945 [00:28<00:34,  9.86it/s]\u001b[A\n",
            " 64%|██████▎   | 602/945 [00:28<00:31, 11.06it/s]\u001b[A\n",
            " 64%|██████▍   | 604/945 [00:28<00:27, 12.33it/s]\u001b[A\n",
            " 64%|██████▍   | 607/945 [00:28<00:23, 14.22it/s]\u001b[A\n",
            " 65%|██████▍   | 610/945 [00:29<00:20, 15.99it/s]\u001b[A\n",
            " 65%|██████▍   | 613/945 [00:29<00:18, 17.52it/s]\u001b[A\n",
            " 65%|██████▌   | 616/945 [00:29<00:17, 18.67it/s]\u001b[A\n",
            " 66%|██████▌   | 619/945 [00:29<00:16, 19.34it/s]\u001b[A\n",
            " 66%|██████▌   | 622/945 [00:29<00:15, 20.44it/s]\u001b[A\n",
            " 66%|██████▌   | 625/945 [00:29<00:15, 20.69it/s]\u001b[A\n",
            " 66%|██████▋   | 628/945 [00:29<00:15, 20.84it/s]\u001b[A\n",
            " 67%|██████▋   | 631/945 [00:30<00:14, 20.97it/s]\u001b[A\n",
            " 67%|██████▋   | 634/945 [00:30<00:14, 21.77it/s]\u001b[A\n",
            " 67%|██████▋   | 637/945 [00:30<00:13, 22.20it/s]\u001b[A\n",
            " 68%|██████▊   | 640/945 [00:30<00:13, 22.74it/s]\u001b[A\n",
            " 68%|██████▊   | 643/945 [00:30<00:13, 23.08it/s]\u001b[A\n",
            " 68%|██████▊   | 646/945 [00:30<00:13, 22.61it/s]\u001b[A\n",
            " 69%|██████▊   | 649/945 [00:30<00:12, 22.97it/s]\u001b[A\n",
            " 69%|██████▉   | 652/945 [00:30<00:12, 23.28it/s]\u001b[A\n",
            " 69%|██████▉   | 655/945 [00:31<00:12, 23.48it/s]\u001b[A\n",
            " 70%|██████▉   | 658/945 [00:31<00:23, 12.13it/s]\u001b[A\n",
            " 70%|██████▉   | 661/945 [00:31<00:19, 14.20it/s]\u001b[A\n",
            " 70%|███████   | 664/945 [00:31<00:17, 16.22it/s]\u001b[A\n",
            " 71%|███████   | 667/945 [00:31<00:15, 18.10it/s]\u001b[A\n",
            " 71%|███████   | 670/945 [00:32<00:14, 19.50it/s]\u001b[A\n",
            " 71%|███████   | 673/945 [00:32<00:13, 20.65it/s]\u001b[A\n",
            " 72%|███████▏  | 676/945 [00:32<00:12, 21.29it/s]\u001b[A\n",
            " 72%|███████▏  | 679/945 [00:32<00:12, 22.13it/s]\u001b[A\n",
            " 72%|███████▏  | 682/945 [00:32<00:11, 22.63it/s]\u001b[A\n",
            " 72%|███████▏  | 685/945 [00:32<00:11, 23.13it/s]\u001b[A\n",
            " 73%|███████▎  | 688/945 [00:32<00:11, 23.30it/s]\u001b[A\n",
            " 73%|███████▎  | 691/945 [00:33<00:16, 15.86it/s]\u001b[A\n",
            " 73%|███████▎  | 694/945 [00:33<00:14, 17.73it/s]\u001b[A\n",
            " 74%|███████▍  | 697/945 [00:33<00:12, 19.24it/s]\u001b[A\n",
            " 74%|███████▍  | 700/945 [00:33<00:12, 20.31it/s]\u001b[A\n",
            " 74%|███████▍  | 703/945 [00:33<00:11, 21.23it/s]\u001b[A\n",
            " 75%|███████▍  | 706/945 [00:33<00:10, 21.92it/s]\u001b[A\n",
            " 75%|███████▌  | 709/945 [00:33<00:10, 22.68it/s]\u001b[A\n",
            " 75%|███████▌  | 712/945 [00:34<00:10, 23.29it/s]\u001b[A\n",
            " 76%|███████▌  | 715/945 [00:34<00:09, 23.63it/s]\u001b[A\n",
            " 76%|███████▌  | 718/945 [00:34<00:09, 23.92it/s]\u001b[A\n",
            " 76%|███████▋  | 721/945 [00:34<00:09, 24.10it/s]\u001b[A\n",
            " 77%|███████▋  | 724/945 [00:34<00:09, 24.06it/s]\u001b[A\n",
            " 77%|███████▋  | 727/945 [00:34<00:09, 23.15it/s]\u001b[A\n",
            " 77%|███████▋  | 730/945 [00:34<00:09, 23.49it/s]\u001b[A\n",
            " 78%|███████▊  | 733/945 [00:34<00:09, 23.16it/s]\u001b[A\n",
            " 78%|███████▊  | 736/945 [00:35<00:08, 23.54it/s]\u001b[A\n",
            " 78%|███████▊  | 739/945 [00:35<00:08, 23.70it/s]\u001b[A\n",
            " 79%|███████▊  | 742/945 [00:35<00:08, 23.97it/s]\u001b[A\n",
            " 79%|███████▉  | 745/945 [00:35<00:08, 24.24it/s]\u001b[A\n",
            " 79%|███████▉  | 748/945 [00:35<00:15, 12.36it/s]\u001b[A\n",
            " 79%|███████▉  | 751/945 [00:36<00:13, 14.42it/s]\u001b[A\n",
            " 80%|███████▉  | 754/945 [00:36<00:11, 16.19it/s]\u001b[A\n",
            " 80%|████████  | 757/945 [00:36<00:10, 17.97it/s]\u001b[A\n",
            " 80%|████████  | 760/945 [00:36<00:09, 19.41it/s]\u001b[A\n",
            " 81%|████████  | 763/945 [00:36<00:08, 20.41it/s]\u001b[A\n",
            " 81%|████████  | 766/945 [00:36<00:08, 21.15it/s]\u001b[A\n",
            " 81%|████████▏ | 769/945 [00:36<00:08, 21.82it/s]\u001b[A\n",
            " 82%|████████▏ | 772/945 [00:36<00:07, 22.20it/s]\u001b[A\n",
            " 82%|████████▏ | 775/945 [00:37<00:07, 22.40it/s]\u001b[A\n",
            " 82%|████████▏ | 778/945 [00:37<00:07, 22.96it/s]\u001b[A\n",
            " 83%|████████▎ | 781/945 [00:37<00:06, 23.43it/s]\u001b[A\n",
            " 83%|████████▎ | 784/945 [00:37<00:06, 23.39it/s]\u001b[A\n",
            " 83%|████████▎ | 787/945 [00:37<00:06, 22.88it/s]\u001b[A\n",
            " 84%|████████▎ | 790/945 [00:37<00:06, 22.85it/s]\u001b[A\n",
            " 84%|████████▍ | 793/945 [00:37<00:06, 22.93it/s]\u001b[A\n",
            " 84%|████████▍ | 796/945 [00:37<00:06, 23.17it/s]\u001b[A\n",
            " 85%|████████▍ | 799/945 [00:38<00:06, 23.11it/s]\u001b[A\n",
            " 85%|████████▍ | 802/945 [00:38<00:06, 23.26it/s]\u001b[A\n",
            " 85%|████████▌ | 805/945 [00:38<00:05, 23.54it/s]\u001b[A\n",
            " 86%|████████▌ | 808/945 [00:38<00:05, 23.85it/s]\u001b[A\n",
            " 86%|████████▌ | 811/945 [00:38<00:05, 24.05it/s]\u001b[A\n",
            " 86%|████████▌ | 814/945 [00:38<00:05, 24.09it/s]\u001b[A\n",
            " 86%|████████▋ | 817/945 [00:38<00:05, 23.97it/s]\u001b[A\n",
            " 87%|████████▋ | 820/945 [00:39<00:05, 22.85it/s]\u001b[A\n",
            " 87%|████████▋ | 823/945 [00:39<00:05, 23.10it/s]\u001b[A\n",
            " 87%|████████▋ | 826/945 [00:39<00:05, 23.19it/s]\u001b[A\n",
            " 88%|████████▊ | 829/945 [00:39<00:05, 23.14it/s]\u001b[A\n",
            " 88%|████████▊ | 832/945 [00:39<00:04, 23.07it/s]\u001b[A\n",
            " 88%|████████▊ | 835/945 [00:39<00:04, 23.15it/s]\u001b[A\n",
            " 89%|████████▊ | 838/945 [00:39<00:04, 23.37it/s]\u001b[A\n",
            " 89%|████████▉ | 841/945 [00:39<00:04, 23.27it/s]\u001b[A\n",
            " 89%|████████▉ | 844/945 [00:40<00:04, 23.18it/s]\u001b[A\n",
            " 90%|████████▉ | 847/945 [00:40<00:04, 23.42it/s]\u001b[A\n",
            " 90%|████████▉ | 850/945 [00:40<00:04, 23.42it/s]\u001b[A\n",
            " 90%|█████████ | 853/945 [00:40<00:03, 23.65it/s]\u001b[A\n",
            " 91%|█████████ | 856/945 [00:40<00:03, 23.86it/s]\u001b[A\n",
            " 91%|█████████ | 859/945 [00:40<00:03, 23.94it/s]\u001b[A\n",
            " 91%|█████████ | 862/945 [00:40<00:03, 23.98it/s]\u001b[A\n",
            " 92%|█████████▏| 865/945 [00:40<00:03, 23.31it/s]\u001b[A\n",
            " 92%|█████████▏| 868/945 [00:41<00:03, 23.65it/s]\u001b[A\n",
            " 92%|█████████▏| 871/945 [00:41<00:03, 23.20it/s]\u001b[A\n",
            " 92%|█████████▏| 874/945 [00:41<00:03, 22.98it/s]\u001b[A\n",
            " 93%|█████████▎| 877/945 [00:41<00:02, 22.81it/s]\u001b[A\n",
            " 93%|█████████▎| 880/945 [00:41<00:02, 22.72it/s]\u001b[A\n",
            " 93%|█████████▎| 883/945 [00:41<00:02, 22.81it/s]\u001b[A\n",
            " 94%|█████████▍| 886/945 [00:41<00:02, 22.87it/s]\u001b[A\n",
            " 94%|█████████▍| 889/945 [00:41<00:02, 22.85it/s]\u001b[A\n",
            " 94%|█████████▍| 892/945 [00:42<00:02, 23.23it/s]\u001b[A\n",
            " 95%|█████████▍| 895/945 [00:42<00:02, 23.40it/s]\u001b[A\n",
            " 95%|█████████▌| 898/945 [00:42<00:02, 23.45it/s]\u001b[A\n",
            " 95%|█████████▌| 901/945 [00:42<00:01, 23.57it/s]\u001b[A\n",
            " 96%|█████████▌| 904/945 [00:42<00:01, 23.82it/s]\u001b[A\n",
            " 96%|█████████▌| 907/945 [00:42<00:01, 23.92it/s]\u001b[A\n",
            " 96%|█████████▋| 910/945 [00:42<00:01, 23.81it/s]\u001b[A\n",
            " 97%|█████████▋| 913/945 [00:42<00:01, 23.85it/s]\u001b[A\n",
            " 97%|█████████▋| 916/945 [00:43<00:01, 23.47it/s]\u001b[A\n",
            " 97%|█████████▋| 919/945 [00:43<00:01, 23.81it/s]\u001b[A\n",
            " 98%|█████████▊| 922/945 [00:43<00:01, 22.70it/s]\u001b[A\n",
            " 98%|█████████▊| 925/945 [00:43<00:00, 22.69it/s]\u001b[A\n",
            " 98%|█████████▊| 928/945 [00:43<00:00, 22.76it/s]\u001b[A\n",
            " 99%|█████████▊| 931/945 [00:43<00:00, 22.77it/s]\u001b[A\n",
            " 99%|█████████▉| 934/945 [00:43<00:00, 22.90it/s]\u001b[A\n",
            " 99%|█████████▉| 937/945 [00:44<00:00, 22.82it/s]\u001b[A\n",
            " 99%|█████████▉| 940/945 [00:44<00:00, 23.02it/s]\u001b[A\n",
            "100%|██████████| 945/945 [00:44<00:00, 21.24it/s]\n"
          ]
        }
      ],
      "source": [
        "predictions = test(model, test_loader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZE1hRnvf0bFz"
      },
      "outputs": [],
      "source": [
        "### Create CSV file with predictions\n",
        "with open(\"./submission.csv\", \"w+\") as f:\n",
        "    f.write(\"id,label\\n\")\n",
        "    for i in range(len(predictions)):\n",
        "        f.write(\"{},{}\\n\".format(i, predictions[i]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LjcammuCxMKN",
        "outputId": "34fc1dc2-3cbb-4376-9a92-bd539c0498aa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.6.5 / client 1.5.8)\n",
            "100% 19.3M/19.3M [00:03<00:00, 6.43MB/s]\n",
            "Successfully submitted to 11785-HW1P2-S24"
          ]
        }
      ],
      "source": [
        "### Submit to kaggle competition using kaggle API (Uncomment below to use)\n",
        "!kaggle competitions submit -c 11785-hw1p2-s24 -f ./submission.csv -m \"HW1P2 Submission\"\n",
        "\n",
        "### However, its always safer to download the csv file and then upload to kaggle"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "No6OT0NhGpXJ",
        "outputId": "6cd6bbbc-6f29-4252-ea3c-0003d6d1cb65"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4f1e4ff0-cdc7-41c4-aae2-74a37a66e1d9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>[SIL]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>[SIL]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>[SIL]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>[SIL]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>[SIL]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>95</td>\n",
              "      <td>T</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>96</td>\n",
              "      <td>T</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>97</td>\n",
              "      <td>T</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>98</td>\n",
              "      <td>DH</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>99</td>\n",
              "      <td>DH</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>100 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4f1e4ff0-cdc7-41c4-aae2-74a37a66e1d9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4f1e4ff0-cdc7-41c4-aae2-74a37a66e1d9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4f1e4ff0-cdc7-41c4-aae2-74a37a66e1d9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-65d93d03-8f27-4393-ac44-83bd5b0cc626\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-65d93d03-8f27-4393-ac44-83bd5b0cc626')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-65d93d03-8f27-4393-ac44-83bd5b0cc626 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "    id  label\n",
              "0    0  [SIL]\n",
              "1    1  [SIL]\n",
              "2    2  [SIL]\n",
              "3    3  [SIL]\n",
              "4    4  [SIL]\n",
              "..  ..    ...\n",
              "95  95      T\n",
              "96  96      T\n",
              "97  97      T\n",
              "98  98     DH\n",
              "99  99     DH\n",
              "\n",
              "[100 rows x 2 columns]"
            ]
          },
          "execution_count": 69,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv('submission.csv')\n",
        "df.head(100)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.5"
    },
    "vscode": {
      "interpreter": {
        "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "06b78d90e45e4df7a1a1a1db7b5319ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_336f44d2efec4d6e9cb7268628952083",
              "IPY_MODEL_6abf9c3d91b74c83b55dba788b18b6fe"
            ],
            "layout": "IPY_MODEL_78ac608a02d74c0fa6fcd70f16ad6fd9"
          }
        },
        "336f44d2efec4d6e9cb7268628952083": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "LabelModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fef2295782ba43418ea14464bc086217",
            "placeholder": "​",
            "style": "IPY_MODEL_8ec674315d074c7cbf81364c8059929b",
            "value": "0.920 MB of 0.920 MB uploaded\r"
          }
        },
        "6abf9c3d91b74c83b55dba788b18b6fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e25a6a9bd7a340f690f9ac3f73f6038a",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e8c59bc847524e33822584a587241ca5",
            "value": 1
          }
        },
        "78ac608a02d74c0fa6fcd70f16ad6fd9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ec674315d074c7cbf81364c8059929b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e25a6a9bd7a340f690f9ac3f73f6038a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8c59bc847524e33822584a587241ca5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fef2295782ba43418ea14464bc086217": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}