2. Train NER Model using Pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train NER Model using Pytorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use Flair which is a pytorch based popular NLP framework to train the model. This is a custom library which can be replaced by any other library. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Upload the training files to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "import time\n",
    "import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_session = sagemaker.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = sagemaker_session.upload_data(path='data/', key_prefix='data/ner-dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training files uploaded here : s3://sagemaker-us-east-1-275443674968/data/ner-dataset\n"
     ]
    }
   ],
   "source": [
    "print(\"Training files uploaded here : {}\".format(inputs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model using Sagemaker Pytorch Estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import IPython\n",
    "install_needed = False  # should only be True once\n",
    "if install_needed:\n",
    "    print(\"installing deps and restarting kernel\")\n",
    "    !{sys.executable} -m pip install -U sagemaker\n",
    "    !{sys.executable} -m pip install -U smdebug\n",
    "    IPython.Application.instance().kernel.do_shutdown(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.pytorch import PyTorch\n",
    "role = sagemaker.get_execution_role()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train_instance_count has been renamed in sagemaker>=2.\n",
      "See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.\n",
      "train_instance_type has been renamed in sagemaker>=2.\n",
      "See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.\n",
      "train_use_spot_instances has been renamed in sagemaker>=2.\n",
      "See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.\n"
     ]
    }
   ],
   "source": [
    "estimator = PyTorch(entry_point='train.py',\n",
    "                    source_dir='code',\n",
    "                    role=role,\n",
    "                    framework_version='1.4.0',\n",
    "                    py_version=\"py3\",\n",
    "                    train_instance_count=1,\n",
    "                    train_instance_type='ml.g5.xlarge',\n",
    "                    hyperparameters={\n",
    "                        'epochs': 25,\n",
    "                        'learning_rate': 0.1,\n",
    "                        'hidden_size':128},\n",
    "                    train_use_spot_instances=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-28 17:28:17 Starting - Starting the training job...\n",
      "2022-06-28 17:28:34 Starting - Preparing the instances for trainingProfilerReport-1656437297: InProgress\n",
      ".........\n",
      "2022-06-28 17:30:16 Downloading - Downloading input data......\n",
      "2022-06-28 17:31:17 Training - Downloading the training image.......\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2022-06-28 17:32:26,213 sagemaker-containers INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2022-06-28 17:32:26,239 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2022-06-28 17:32:26,243 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2022-06-28 17:32:26,552 sagemaker-containers INFO     Module default_user_module_name does not provide a setup.py. \u001b[0m\n",
      "\u001b[34mGenerating setup.py\u001b[0m\n",
      "\u001b[34m2022-06-28 17:32:26,552 sagemaker-containers INFO     Generating setup.cfg\u001b[0m\n",
      "\u001b[34m2022-06-28 17:32:26,552 sagemaker-containers INFO     Generating MANIFEST.in\u001b[0m\n",
      "\u001b[34m2022-06-28 17:32:26,552 sagemaker-containers INFO     Installing module with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 -m pip install . -r requirements.txt\u001b[0m\n",
      "\u001b[34mProcessing /tmp/tmp7pwdo30d/module_dir\u001b[0m\n",
      "\u001b[34mCollecting flair==0.7\n",
      "  Downloading flair-0.7-py3-none-any.whl (448 kB)\u001b[0m\n",
      "\u001b[34mCollecting mpld3==0.3\n",
      "  Downloading mpld3-0.3.tar.gz (788 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.26.0 in /opt/conda/lib/python3.6/site-packages (from flair==0.7->-r requirements.txt (line 1)) (4.56.0)\u001b[0m\n",
      "\u001b[34mCollecting sentencepiece<=0.1.91\n",
      "  Downloading sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1 MB)\u001b[0m\n",
      "\u001b[34mCollecting tabulate\n",
      "  Downloading tabulate-0.8.10-py3-none-any.whl (29 kB)\u001b[0m\n",
      "\u001b[34mCollecting ftfy\n",
      "  Downloading ftfy-6.0.3.tar.gz (64 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib>=2.2.3 in /opt/conda/lib/python3.6/site-packages (from flair==0.7->-r requirements.txt (line 1)) (3.3.4)\u001b[0m\n",
      "\u001b[34mCollecting deprecated>=1.2.4\n",
      "  Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting gdown\n",
      "  Downloading gdown-4.5.1.tar.gz (14 kB)\n",
      "  Installing build dependencies: started\u001b[0m\n",
      "\u001b[34m  Installing build dependencies: finished with status 'done'\n",
      "  Getting requirements to build wheel: started\u001b[0m\n",
      "\u001b[34m  Getting requirements to build wheel: finished with status 'done'\n",
      "    Preparing wheel metadata: started\n",
      "    Preparing wheel metadata: finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting janome\n",
      "  Downloading Janome-0.4.2-py2.py3-none-any.whl (19.7 MB)\u001b[0m\n",
      "\u001b[34mCollecting segtok>=1.5.7\n",
      "  Downloading segtok-1.5.11-py3-none-any.whl (24 kB)\u001b[0m\n",
      "\u001b[34mCollecting gensim<=3.8.3,>=3.4.0\n",
      "  Downloading gensim-3.8.3-cp36-cp36m-manylinux1_x86_64.whl (24.2 MB)\u001b[0m\n",
      "\u001b[34mCollecting scikit-learn>=0.21.3\n",
      "  Downloading scikit_learn-0.24.2-cp36-cp36m-manylinux2010_x86_64.whl (22.2 MB)\u001b[0m\n",
      "\u001b[34mCollecting hyperopt>=0.1.1\n",
      "  Downloading hyperopt-0.2.7-py2.py3-none-any.whl (1.6 MB)\u001b[0m\n",
      "\u001b[34mCollecting konoha<5.0.0,>=4.0.0\n",
      "  Downloading konoha-4.6.5-py3-none-any.whl (20 kB)\u001b[0m\n",
      "\u001b[34mCollecting transformers<=3.5.1,>=3.5.0\n",
      "  Downloading transformers-3.5.1-py3-none-any.whl (1.3 MB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.6.1 in /opt/conda/lib/python3.6/site-packages (from flair==0.7->-r requirements.txt (line 1)) (2.8.1)\u001b[0m\n",
      "\u001b[34mCollecting lxml\n",
      "  Downloading lxml-4.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (6.4 MB)\u001b[0m\n",
      "\u001b[34mCollecting bpemb>=0.3.2\n",
      "  Downloading bpemb-0.3.3-py3-none-any.whl (19 kB)\u001b[0m\n",
      "\u001b[34mCollecting langdetect\n",
      "  Downloading langdetect-1.0.9.tar.gz (981 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch>=1.1.0 in /opt/conda/lib/python3.6/site-packages (from flair==0.7->-r requirements.txt (line 1)) (1.4.0)\u001b[0m\n",
      "\u001b[34mCollecting sqlitedict>=1.6.0\n",
      "  Downloading sqlitedict-2.0.0.tar.gz (46 kB)\u001b[0m\n",
      "\n",
      "2022-06-28 17:32:37 Training - Training image download completed. Training in progress.\u001b[34mCollecting regex\n",
      "  Downloading regex-2022.6.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (749 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from bpemb>=0.3.2->flair==0.7->-r requirements.txt (line 1)) (1.16.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from bpemb>=0.3.2->flair==0.7->-r requirements.txt (line 1)) (2.22.0)\u001b[0m\n",
      "\u001b[34mCollecting wrapt<2,>=1.10\n",
      "  Downloading wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (74 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six>=1.5.0 in /opt/conda/lib/python3.6/site-packages (from gensim<=3.8.3,>=3.4.0->flair==0.7->-r requirements.txt (line 1)) (1.15.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: smart-open>=1.8.1 in /opt/conda/lib/python3.6/site-packages (from gensim<=3.8.3,>=3.4.0->flair==0.7->-r requirements.txt (line 1)) (3.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: scipy>=0.18.1 in /opt/conda/lib/python3.6/site-packages (from gensim<=3.8.3,>=3.4.0->flair==0.7->-r requirements.txt (line 1)) (1.2.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cloudpickle in /opt/conda/lib/python3.6/site-packages (from hyperopt>=0.1.1->flair==0.7->-r requirements.txt (line 1)) (1.6.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: future in /opt/conda/lib/python3.6/site-packages (from hyperopt>=0.1.1->flair==0.7->-r requirements.txt (line 1)) (0.17.1)\u001b[0m\n",
      "\u001b[34mCollecting py4j\n",
      "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: networkx>=2.2 in /opt/conda/lib/python3.6/site-packages (from hyperopt>=0.1.1->flair==0.7->-r requirements.txt (line 1)) (2.5)\u001b[0m\n",
      "\u001b[34mCollecting overrides<4.0.0,>=3.0.0\n",
      "  Downloading overrides-3.1.0.tar.gz (11 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: importlib-metadata<4.0.0,>=3.7.0 in /opt/conda/lib/python3.6/site-packages (from konoha<5.0.0,>=4.0.0->flair==0.7->-r requirements.txt (line 1)) (3.7.2)\u001b[0m\n",
      "\u001b[34mCollecting requests\n",
      "  Downloading requests-2.27.1-py2.py3-none-any.whl (63 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata<4.0.0,>=3.7.0->konoha<5.0.0,>=4.0.0->flair==0.7->-r requirements.txt (line 1)) (3.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata<4.0.0,>=3.7.0->konoha<5.0.0,>=4.0.0->flair==0.7->-r requirements.txt (line 1)) (3.7.4.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /opt/conda/lib/python3.6/site-packages (from matplotlib>=2.2.3->flair==0.7->-r requirements.txt (line 1)) (2.4.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.6/site-packages (from matplotlib>=2.2.3->flair==0.7->-r requirements.txt (line 1)) (8.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib>=2.2.3->flair==0.7->-r requirements.txt (line 1)) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.6/site-packages (from matplotlib>=2.2.3->flair==0.7->-r requirements.txt (line 1)) (0.10.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: decorator>=4.3.0 in /opt/conda/lib/python3.6/site-packages (from networkx>=2.2->hyperopt>=0.1.1->flair==0.7->-r requirements.txt (line 1)) (4.4.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->bpemb>=0.3.2->flair==0.7->-r requirements.txt (line 1)) (2020.12.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->bpemb>=0.3.2->flair==0.7->-r requirements.txt (line 1)) (1.25.11)\u001b[0m\n",
      "\u001b[34mCollecting charset-normalizer~=2.0.0\n",
      "  Downloading charset_normalizer-2.0.12-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->bpemb>=0.3.2->flair==0.7->-r requirements.txt (line 1)) (2.8)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn>=0.21.3->flair==0.7->-r requirements.txt (line 1)) (1.0.1)\u001b[0m\n",
      "\u001b[34mCollecting threadpoolctl>=2.0.0\n",
      "  Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging in /opt/conda/lib/python3.6/site-packages (from transformers<=3.5.1,>=3.5.0->flair==0.7->-r requirements.txt (line 1)) (20.9)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: protobuf in /opt/conda/lib/python3.6/site-packages (from transformers<=3.5.1,>=3.5.0->flair==0.7->-r requirements.txt (line 1)) (3.15.5)\u001b[0m\n",
      "\u001b[34mCollecting sacremoses\n",
      "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\u001b[0m\n",
      "\u001b[34mCollecting tokenizers==0.9.3\n",
      "  Downloading tokenizers-0.9.3-cp36-cp36m-manylinux1_x86_64.whl (2.9 MB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from transformers<=3.5.1,>=3.5.0->flair==0.7->-r requirements.txt (line 1)) (0.8)\u001b[0m\n",
      "\u001b[34mCollecting filelock\n",
      "  Downloading filelock-3.4.1-py3-none-any.whl (9.9 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: wcwidth in /opt/conda/lib/python3.6/site-packages (from ftfy->flair==0.7->-r requirements.txt (line 1)) (0.2.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.6/site-packages (from gdown->flair==0.7->-r requirements.txt (line 1)) (4.9.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.6/site-packages (from beautifulsoup4->gdown->flair==0.7->-r requirements.txt (line 1)) (2.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /opt/conda/lib/python3.6/site-packages (from requests->bpemb>=0.3.2->flair==0.7->-r requirements.txt (line 1)) (1.7.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers<=3.5.1,>=3.5.0->flair==0.7->-r requirements.txt (line 1)) (7.1.2)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: default-user-module-name, mpld3, overrides, sqlitedict, ftfy, gdown, langdetect, sacremoses\n",
      "  Building wheel for default-user-module-name (setup.py): started\n",
      "  Building wheel for default-user-module-name (setup.py): finished with status 'done'\n",
      "  Created wheel for default-user-module-name: filename=default_user_module_name-1.0.0-py2.py3-none-any.whl size=9653 sha256=65479304bd959edae772de11a163bfda56e46fa24c83e1ac29114bfbd8e8baca\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-izlg9k4u/wheels/e8/f1/59/d354a3bd2006f29cd01b2d1bdd06aa303f58e8a28f064ceafb\n",
      "  Building wheel for mpld3 (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for mpld3 (setup.py): finished with status 'done'\n",
      "  Created wheel for mpld3: filename=mpld3-0.3-py3-none-any.whl size=116678 sha256=9041b09cdffff25454b52bf74bbb1e0762a225ce8a84df4858fddad8de0446aa\n",
      "  Stored in directory: /root/.cache/pip/wheels/c0/74/c9/ac92f0c4c9eb137d440e86c2822aba0b96b63e608dd5737164\n",
      "  Building wheel for overrides (setup.py): started\n",
      "  Building wheel for overrides (setup.py): finished with status 'done'\n",
      "  Created wheel for overrides: filename=overrides-3.1.0-py3-none-any.whl size=10175 sha256=f88ebe4caf47d7aa3146e7a39fe712b1e2335c8ee1e4f7154d4e118dbae07eb5\n",
      "  Stored in directory: /root/.cache/pip/wheels/e6/3b/34/ae59fc8d35c37f01099425ab73599e45e9b9b599a7ccc2c45f\n",
      "  Building wheel for sqlitedict (setup.py): started\n",
      "  Building wheel for sqlitedict (setup.py): finished with status 'done'\n",
      "  Created wheel for sqlitedict: filename=sqlitedict-2.0.0-py3-none-any.whl size=15718 sha256=d2b801ca7f24dddf416e60a66c84d99e5f746d32edbb9ca0b1f6b52bc93bf07b\n",
      "  Stored in directory: /root/.cache/pip/wheels/c3/ea/90/1812907e81ceb6c0393eb6802f7aba97d4ee157b0e15bc8101\n",
      "  Building wheel for ftfy (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for ftfy (setup.py): finished with status 'done'\n",
      "  Created wheel for ftfy: filename=ftfy-6.0.3-py3-none-any.whl size=41913 sha256=9ea6f8c491e7d3dcf3f56cc8f436a1332af396e526a2f16b0db1e7798441b0be\n",
      "  Stored in directory: /root/.cache/pip/wheels/ff/2a/24/75041425faf3347ab146a4a3d0484f723b2c44a7966a06e3f0\n",
      "  Building wheel for gdown (PEP 517): started\n",
      "  Building wheel for gdown (PEP 517): finished with status 'done'\n",
      "  Created wheel for gdown: filename=gdown-4.5.1-py3-none-any.whl size=14951 sha256=f3a079e41a25f8bde4087d9976371b8bda563242878c0750d5ddb8c532946120\n",
      "  Stored in directory: /root/.cache/pip/wheels/9b/0f/b5/18491a0b635e27a4ca24e38909932443118ce3874d682514a8\n",
      "  Building wheel for langdetect (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for langdetect (setup.py): finished with status 'done'\n",
      "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993221 sha256=ce7e988aad1f3103c26c7a104c81b94234069fd70144850031a674e71dee797f\n",
      "  Stored in directory: /root/.cache/pip/wheels/22/e8/62/ef79403841bab16f1c4260b967bee7fa579d78552a66c7f6e0\n",
      "  Building wheel for sacremoses (setup.py): started\n",
      "  Building wheel for sacremoses (setup.py): finished with status 'done'\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895254 sha256=4bcf934904537b2812a62278beda0ba24979eb0612ffb8f9ccd3126f22e9c863\n",
      "  Stored in directory: /root/.cache/pip/wheels/4c/64/31/e9900a234b23fb3e9dc565d6114a9d6ff84a72dbdd356502b4\u001b[0m\n",
      "\u001b[34mSuccessfully built default-user-module-name mpld3 overrides sqlitedict ftfy gdown langdetect sacremoses\u001b[0m\n",
      "\u001b[34mInstalling collected packages: charset-normalizer, requests, regex, wrapt, tokenizers, threadpoolctl, sentencepiece, sacremoses, py4j, overrides, gensim, filelock, transformers, tabulate, sqlitedict, segtok, scikit-learn, mpld3, lxml, langdetect, konoha, janome, hyperopt, gdown, ftfy, deprecated, bpemb, flair, default-user-module-name\n",
      "  Attempting uninstall: requests\n",
      "    Found existing installation: requests 2.22.0\n",
      "    Uninstalling requests-2.22.0:\n",
      "      Successfully uninstalled requests-2.22.0\u001b[0m\n",
      "\u001b[34m  Attempting uninstall: scikit-learn\n",
      "    Found existing installation: scikit-learn 0.21.2\n",
      "    Uninstalling scikit-learn-0.21.2:\n",
      "      Successfully uninstalled scikit-learn-0.21.2\u001b[0m\n",
      "\u001b[34mSuccessfully installed bpemb-0.3.3 charset-normalizer-2.0.12 default-user-module-name-1.0.0 deprecated-1.2.13 filelock-3.4.1 flair-0.7 ftfy-6.0.3 gdown-4.5.1 gensim-3.8.3 hyperopt-0.2.7 janome-0.4.2 konoha-4.6.5 langdetect-1.0.9 lxml-4.9.0 mpld3-0.3 overrides-3.1.0 py4j-0.10.9.5 regex-2022.6.2 requests-2.27.1 sacremoses-0.0.53 scikit-learn-0.24.2 segtok-1.5.11 sentencepiece-0.1.91 sqlitedict-2.0.0 tabulate-0.8.10 threadpoolctl-3.1.0 tokenizers-0.9.3 transformers-3.5.1 wrapt-1.14.1\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:04,519 sagemaker-containers INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"eval\": \"/opt/ml/input/data/eval\",\n",
      "        \"train\": \"/opt/ml/input/data/train\",\n",
      "        \"validation\": \"/opt/ml/input/data/validation\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"epochs\": 25,\n",
      "        \"hidden_size\": 128,\n",
      "        \"learning_rate\": 0.1\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"eval\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"train\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"validation\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"pytorch-training-2022-06-28-17-28-17-450\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-east-1-275443674968/pytorch-training-2022-06-28-17-28-17-450/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"train\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 1,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.g5.xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.g5.xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"train.py\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"epochs\":25,\"hidden_size\":128,\"learning_rate\":0.1}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=train.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g5.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g5.xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"eval\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"validation\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"eval\",\"train\",\"validation\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=train\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-east-1-275443674968/pytorch-training-2022-06-28-17-28-17-450/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"eval\":\"/opt/ml/input/data/eval\",\"train\":\"/opt/ml/input/data/train\",\"validation\":\"/opt/ml/input/data/validation\"},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"epochs\":25,\"hidden_size\":128,\"learning_rate\":0.1},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"eval\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"validation\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"pytorch-training-2022-06-28-17-28-17-450\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-east-1-275443674968/pytorch-training-2022-06-28-17-28-17-450/source/sourcedir.tar.gz\",\"module_name\":\"train\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":1,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g5.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g5.xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--epochs\",\"25\",\"--hidden_size\",\"128\",\"--learning_rate\",\"0.1\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_EVAL=/opt/ml/input/data/eval\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TRAIN=/opt/ml/input/data/train\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_VALIDATION=/opt/ml/input/data/validation\u001b[0m\n",
      "\u001b[34mSM_HP_EPOCHS=25\u001b[0m\n",
      "\u001b[34mSM_HP_HIDDEN_SIZE=128\u001b[0m\n",
      "\u001b[34mSM_HP_LEARNING_RATE=0.1\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python36.zip:/opt/conda/lib/python3.6:/opt/conda/lib/python3.6/lib-dynload:/opt/conda/lib/python3.6/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 train.py --epochs 25 --hidden_size 128 --learning_rate 0.1\u001b[0m\n",
      "\u001b[34mstarting to train flair model\u001b[0m\n",
      "\u001b[34mNamespace(batch_size=32, current_host='algo-1', epochs=25, eval='/opt/ml/input/data/eval', hidden_size=128, hosts=['algo-1'], learning_rate=0.1, model_dir='/opt/ml/model', train='/opt/ml/input/data/train', validation='/opt/ml/input/data/validation')\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:07,883 Reading data from .\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:07,883 Train: /opt/ml/input/data/train/train.txt\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:07,883 Dev: /opt/ml/input/data/validation/train.txt\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:07,883 Test: /opt/ml/input/data/eval/train.txt\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:08,547 https://flair.informatik.hu-berlin.de/resources/embeddings/token/glove.gensim.vectors.npy not found in cache, downloading to /tmp/tmp8ejmirq8\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:15,675 copying /tmp/tmp8ejmirq8 to cache at /root/.flair/embeddings/glove.gensim.vectors.npy\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:15,781 removing temp file /tmp/tmp8ejmirq8\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:16,142 https://flair.informatik.hu-berlin.de/resources/embeddings/token/glove.gensim not found in cache, downloading to /tmp/tmphzjp9067\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:18,372 copying /tmp/tmphzjp9067 to cache at /root/.flair/embeddings/glove.gensim\u001b[0m\n",
      "\u001b[34m2022-06-28 17:33:18,386 removing temp file /tmp/tmphzjp9067\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,870 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,870 Model: \"SequenceTagger(\n",
      "  (embeddings): StackedEmbeddings(\n",
      "    (list_embedding_0): WordEmbeddings('glove')\n",
      "  )\n",
      "  (word_dropout): WordDropout(p=0.05)\n",
      "  (locked_dropout): LockedDropout(p=0.5)\n",
      "  (embedding2nn): Linear(in_features=100, out_features=100, bias=True)\n",
      "  (rnn): LSTM(100, 128, batch_first=True, bidirectional=True)\n",
      "  (linear): Linear(in_features=256, out_features=8, bias=True)\n",
      "  (beta): 1.0\n",
      "  (weights): None\n",
      "  (weight_tensor) None\u001b[0m\n",
      "\u001b[34m)\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,870 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,870 Corpus: \"Corpus: 199 train + 199 dev + 199 test sentences\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,870 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,870 Parameters:\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - learning_rate: \"0.1\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - mini_batch_size: \"32\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - patience: \"3\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - anneal_factor: \"0.5\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - max_epochs: \"25\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - shuffle: \"True\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - train_with_dev: \"False\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871  - batch_growth_annealing: \"False\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871 Model training base path: \"/opt/ml/model\"\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871 Device: cuda:0\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,871 Embeddings storage mode: cpu\u001b[0m\n",
      "\u001b[34m2022-06-28 18:14:30,872 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m[2022-06-28 18:14:31.071 algo-1:96 INFO json_config.py:90] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\u001b[0m\n",
      "\u001b[34m[2022-06-28 18:14:31.071 algo-1:96 INFO hook.py:192] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\u001b[0m\n",
      "\u001b[34m[2022-06-28 18:14:31.071 algo-1:96 INFO hook.py:237] Saving to /opt/ml/output/tensors\u001b[0m\n",
      "\u001b[34m[2022-06-28 18:14:31.071 algo-1:96 INFO state_store.py:67] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\u001b[0m\n",
      "\u001b[34m[2022-06-28 18:14:31.093 algo-1:96 INFO hook.py:382] Monitoring the collections: losses\u001b[0m\n",
      "\u001b[34m[2022-06-28 18:14:31.093 algo-1:96 INFO hook.py:443] Hook is writing from the hook with pid: 96\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,241 epoch 1 - iter 1/7 - loss 13.16397953 - samples/sec: 0.13 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,304 epoch 1 - iter 2/7 - loss 11.68481159 - samples/sec: 506.29 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,367 epoch 1 - iter 3/7 - loss 10.85065460 - samples/sec: 508.70 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,429 epoch 1 - iter 4/7 - loss 9.98411310 - samples/sec: 521.35 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,493 epoch 1 - iter 5/7 - loss 9.53369551 - samples/sec: 502.72 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,558 epoch 1 - iter 6/7 - loss 9.07665936 - samples/sec: 496.58 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,597 epoch 1 - iter 7/7 - loss 8.93875544 - samples/sec: 808.47 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,598 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,598 EPOCH 1 done: loss 8.9388 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,790 DEV : loss 6.052187442779541 - score 0.3512\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:29,793 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,029 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,087 epoch 2 - iter 1/7 - loss 5.05638552 - samples/sec: 554.83 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,139 epoch 2 - iter 2/7 - loss 5.61766744 - samples/sec: 615.15 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,199 epoch 2 - iter 3/7 - loss 5.45380449 - samples/sec: 539.23 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,254 epoch 2 - iter 4/7 - loss 4.98482561 - samples/sec: 587.00 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,308 epoch 2 - iter 5/7 - loss 5.14062996 - samples/sec: 597.01 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,362 epoch 2 - iter 6/7 - loss 5.35288843 - samples/sec: 598.73 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,392 epoch 2 - iter 7/7 - loss 5.30194303 - samples/sec: 1095.28 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,392 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,392 EPOCH 2 done: loss 5.3019 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,544 DEV : loss 4.4806365966796875 - score 0.0755\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,547 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,548 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,606 epoch 3 - iter 1/7 - loss 5.00885010 - samples/sec: 555.43 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,672 epoch 3 - iter 2/7 - loss 5.30223703 - samples/sec: 485.71 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,727 epoch 3 - iter 3/7 - loss 4.93704224 - samples/sec: 589.01 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,776 epoch 3 - iter 4/7 - loss 4.72419167 - samples/sec: 654.92 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,833 epoch 3 - iter 5/7 - loss 4.53119025 - samples/sec: 565.05 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,891 epoch 3 - iter 6/7 - loss 4.50414503 - samples/sec: 556.82 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,912 epoch 3 - iter 7/7 - loss 4.27878690 - samples/sec: 1511.75 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,913 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:33,913 EPOCH 3 done: loss 4.2788 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:34,073 DEV : loss 3.4062459468841553 - score 0.5263\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:34,076 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,418 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,467 epoch 4 - iter 1/7 - loss 3.26213455 - samples/sec: 663.53 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,517 epoch 4 - iter 2/7 - loss 3.56871986 - samples/sec: 653.80 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,566 epoch 4 - iter 3/7 - loss 3.70029720 - samples/sec: 648.29 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,621 epoch 4 - iter 4/7 - loss 3.87176955 - samples/sec: 587.63 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,681 epoch 4 - iter 5/7 - loss 3.76182027 - samples/sec: 537.41 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,739 epoch 4 - iter 6/7 - loss 4.06870671 - samples/sec: 558.08 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,765 epoch 4 - iter 7/7 - loss 4.11649605 - samples/sec: 1234.10 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,766 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,766 EPOCH 4 done: loss 4.1165 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,935 DEV : loss 3.2397453784942627 - score 0.5367\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:37,938 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,268 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,319 epoch 5 - iter 1/7 - loss 5.15881634 - samples/sec: 639.59 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,374 epoch 5 - iter 2/7 - loss 5.09056592 - samples/sec: 587.88 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,429 epoch 5 - iter 3/7 - loss 4.46545219 - samples/sec: 588.27 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,495 epoch 5 - iter 4/7 - loss 4.34336728 - samples/sec: 489.75 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,543 epoch 5 - iter 5/7 - loss 4.01650476 - samples/sec: 657.94 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,610 epoch 5 - iter 6/7 - loss 3.77292609 - samples/sec: 486.61 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,639 epoch 5 - iter 7/7 - loss 3.56739746 - samples/sec: 1096.83 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,640 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,640 EPOCH 5 done: loss 3.5674 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,812 DEV : loss 2.894995927810669 - score 0.5482\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:41,815 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,112 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,162 epoch 6 - iter 1/7 - loss 2.60198975 - samples/sec: 641.03 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,214 epoch 6 - iter 2/7 - loss 2.77336526 - samples/sec: 630.97 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,273 epoch 6 - iter 3/7 - loss 2.92760166 - samples/sec: 544.77 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,328 epoch 6 - iter 4/7 - loss 3.20417905 - samples/sec: 585.57 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,376 epoch 6 - iter 5/7 - loss 3.02805452 - samples/sec: 664.61 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,441 epoch 6 - iter 6/7 - loss 3.31541324 - samples/sec: 497.99 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,466 epoch 6 - iter 7/7 - loss 3.19787179 - samples/sec: 1298.89 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,467 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,467 EPOCH 6 done: loss 3.1979 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,635 DEV : loss 3.5331759452819824 - score 0.5099\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,638 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,639 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,701 epoch 7 - iter 1/7 - loss 4.74983788 - samples/sec: 513.36 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,760 epoch 7 - iter 2/7 - loss 4.15642762 - samples/sec: 551.94 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,820 epoch 7 - iter 3/7 - loss 3.49368056 - samples/sec: 538.10 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,870 epoch 7 - iter 4/7 - loss 3.24647027 - samples/sec: 645.15 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,919 epoch 7 - iter 5/7 - loss 3.13549266 - samples/sec: 656.72 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:45,971 epoch 7 - iter 6/7 - loss 3.04109879 - samples/sec: 627.18 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:46,006 epoch 7 - iter 7/7 - loss 3.06880726 - samples/sec: 912.77 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:46,007 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:46,007 EPOCH 7 done: loss 3.0688 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:46,173 DEV : loss 2.659468173980713 - score 0.5615\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:46,176 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,519 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,572 epoch 8 - iter 1/7 - loss 2.99166107 - samples/sec: 616.93 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,629 epoch 8 - iter 2/7 - loss 3.65788221 - samples/sec: 560.79 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,684 epoch 8 - iter 3/7 - loss 3.31456145 - samples/sec: 593.70 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,736 epoch 8 - iter 4/7 - loss 3.21386606 - samples/sec: 611.80 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,799 epoch 8 - iter 5/7 - loss 3.11061826 - samples/sec: 512.40 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,849 epoch 8 - iter 6/7 - loss 3.12305522 - samples/sec: 654.57 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,877 epoch 8 - iter 7/7 - loss 3.11596404 - samples/sec: 1155.15 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,878 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:49,878 EPOCH 8 done: loss 3.1160 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,059 DEV : loss 3.515068292617798 - score 0.5099\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,062 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,062 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,112 epoch 9 - iter 1/7 - loss 3.83696175 - samples/sec: 658.49 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,166 epoch 9 - iter 2/7 - loss 3.94690228 - samples/sec: 596.71 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,217 epoch 9 - iter 3/7 - loss 3.57081119 - samples/sec: 627.48 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,280 epoch 9 - iter 4/7 - loss 3.29047811 - samples/sec: 516.83 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,335 epoch 9 - iter 5/7 - loss 3.06595454 - samples/sec: 578.26 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,400 epoch 9 - iter 6/7 - loss 3.04425597 - samples/sec: 501.88 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,431 epoch 9 - iter 7/7 - loss 3.24712672 - samples/sec: 1041.47 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,431 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,432 EPOCH 9 done: loss 3.2471 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,607 DEV : loss 2.421847343444824 - score 0.5916\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:50,610 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:53,896 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:53,955 epoch 10 - iter 1/7 - loss 2.19082499 - samples/sec: 554.44 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,011 epoch 10 - iter 2/7 - loss 2.75566351 - samples/sec: 578.90 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,064 epoch 10 - iter 3/7 - loss 2.58789547 - samples/sec: 608.18 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,132 epoch 10 - iter 4/7 - loss 2.99753219 - samples/sec: 470.69 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,197 epoch 10 - iter 5/7 - loss 3.07960830 - samples/sec: 498.25 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,253 epoch 10 - iter 6/7 - loss 2.98008668 - samples/sec: 576.31 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,274 epoch 10 - iter 7/7 - loss 2.91574536 - samples/sec: 1545.97 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,274 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,274 EPOCH 10 done: loss 2.9157 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,443 DEV : loss 3.217219591140747 - score 0.5099\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,446 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,447 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,503 epoch 11 - iter 1/7 - loss 3.90112114 - samples/sec: 568.70 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,566 epoch 11 - iter 2/7 - loss 3.52206993 - samples/sec: 510.09 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,627 epoch 11 - iter 3/7 - loss 3.05470173 - samples/sec: 534.08 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,682 epoch 11 - iter 4/7 - loss 2.69704819 - samples/sec: 591.26 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,740 epoch 11 - iter 5/7 - loss 2.95973091 - samples/sec: 550.44 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,785 epoch 11 - iter 6/7 - loss 2.91038100 - samples/sec: 726.15 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,821 epoch 11 - iter 7/7 - loss 3.02139166 - samples/sec: 887.65 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,822 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,822 EPOCH 11 done: loss 3.0214 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,982 DEV : loss 2.2430577278137207 - score 0.6082\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:54,985 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,256 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,319 epoch 12 - iter 1/7 - loss 2.67794132 - samples/sec: 517.70 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,371 epoch 12 - iter 2/7 - loss 2.99835670 - samples/sec: 621.80 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,415 epoch 12 - iter 3/7 - loss 2.90656296 - samples/sec: 737.92 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,464 epoch 12 - iter 4/7 - loss 2.76233375 - samples/sec: 654.30 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,529 epoch 12 - iter 5/7 - loss 2.67566280 - samples/sec: 494.43 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,587 epoch 12 - iter 6/7 - loss 2.56373934 - samples/sec: 563.37 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,613 epoch 12 - iter 7/7 - loss 2.57048280 - samples/sec: 1231.80 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,613 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,613 EPOCH 12 done: loss 2.5705 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,767 DEV : loss 2.6178269386291504 - score 0.5337\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,770 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,771 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,825 epoch 13 - iter 1/7 - loss 2.29441118 - samples/sec: 595.72 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,876 epoch 13 - iter 2/7 - loss 2.10566783 - samples/sec: 625.92 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,927 epoch 13 - iter 3/7 - loss 2.02750254 - samples/sec: 636.17 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:58,973 epoch 13 - iter 4/7 - loss 2.07281035 - samples/sec: 699.71 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,031 epoch 13 - iter 5/7 - loss 2.48557734 - samples/sec: 562.07 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,094 epoch 13 - iter 6/7 - loss 2.52493143 - samples/sec: 510.28 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,123 epoch 13 - iter 7/7 - loss 2.48845679 - samples/sec: 1134.42 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,123 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,123 EPOCH 13 done: loss 2.4885 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,300 DEV : loss 2.4052460193634033 - score 0.5714\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,303 BAD EPOCHS (no improvement): 2\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,303 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,357 epoch 14 - iter 1/7 - loss 3.70778847 - samples/sec: 595.83 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,421 epoch 14 - iter 2/7 - loss 3.11312413 - samples/sec: 504.18 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,477 epoch 14 - iter 3/7 - loss 2.99051619 - samples/sec: 573.01 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,536 epoch 14 - iter 4/7 - loss 2.86122352 - samples/sec: 551.60 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,581 epoch 14 - iter 5/7 - loss 2.67002406 - samples/sec: 709.20 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,637 epoch 14 - iter 6/7 - loss 2.66200443 - samples/sec: 583.22 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,672 epoch 14 - iter 7/7 - loss 2.51897281 - samples/sec: 915.67 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,673 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,673 EPOCH 14 done: loss 2.5190 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,856 DEV : loss 2.1022002696990967 - score 0.6138\u001b[0m\n",
      "\u001b[34m2022-06-28 18:18:59,859 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,317 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,380 epoch 15 - iter 1/7 - loss 1.74423385 - samples/sec: 519.84 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,432 epoch 15 - iter 2/7 - loss 1.94077790 - samples/sec: 619.27 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,487 epoch 15 - iter 3/7 - loss 2.21446919 - samples/sec: 589.89 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,537 epoch 15 - iter 4/7 - loss 2.23079246 - samples/sec: 638.35 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,590 epoch 15 - iter 5/7 - loss 2.28424001 - samples/sec: 615.94 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,651 epoch 15 - iter 6/7 - loss 2.47228205 - samples/sec: 529.02 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,685 epoch 15 - iter 7/7 - loss 2.49099278 - samples/sec: 957.78 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,685 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,685 EPOCH 15 done: loss 2.4910 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,852 DEV : loss 2.2368223667144775 - score 0.5914\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,857 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,857 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,916 epoch 16 - iter 1/7 - loss 2.34147000 - samples/sec: 544.18 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:03,976 epoch 16 - iter 2/7 - loss 2.27862036 - samples/sec: 542.35 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,041 epoch 16 - iter 3/7 - loss 2.57837923 - samples/sec: 491.79 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,100 epoch 16 - iter 4/7 - loss 2.74000555 - samples/sec: 545.43 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,163 epoch 16 - iter 5/7 - loss 2.64248781 - samples/sec: 510.82 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,221 epoch 16 - iter 6/7 - loss 2.51923909 - samples/sec: 557.53 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,254 epoch 16 - iter 7/7 - loss 2.55795079 - samples/sec: 1000.53 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,254 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,254 EPOCH 16 done: loss 2.5580 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,453 DEV : loss 1.9898333549499512 - score 0.6194\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:04,456 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:07,817 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:07,867 epoch 17 - iter 1/7 - loss 1.53464699 - samples/sec: 642.52 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:07,928 epoch 17 - iter 2/7 - loss 1.66404378 - samples/sec: 527.89 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:07,985 epoch 17 - iter 3/7 - loss 2.07825700 - samples/sec: 568.50 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,049 epoch 17 - iter 4/7 - loss 2.03670493 - samples/sec: 508.43 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,117 epoch 17 - iter 5/7 - loss 2.40646756 - samples/sec: 475.20 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,171 epoch 17 - iter 6/7 - loss 2.36865379 - samples/sec: 588.12 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,207 epoch 17 - iter 7/7 - loss 2.65715676 - samples/sec: 921.36 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,207 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,207 EPOCH 17 done: loss 2.6572 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,382 DEV : loss 1.9680185317993164 - score 0.6384\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:08,385 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:11,684 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:11,738 epoch 18 - iter 1/7 - loss 1.23171711 - samples/sec: 601.09 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:11,792 epoch 18 - iter 2/7 - loss 1.86769879 - samples/sec: 590.51 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:11,856 epoch 18 - iter 3/7 - loss 2.12424930 - samples/sec: 504.94 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:11,910 epoch 18 - iter 4/7 - loss 2.12385476 - samples/sec: 597.80 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:11,970 epoch 18 - iter 5/7 - loss 2.12986622 - samples/sec: 539.66 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:12,031 epoch 18 - iter 6/7 - loss 2.26927574 - samples/sec: 528.01 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:12,058 epoch 18 - iter 7/7 - loss 2.11771253 - samples/sec: 1232.86 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:12,058 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:12,058 EPOCH 18 done: loss 2.1177 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:12,232 DEV : loss 1.8237061500549316 - score 0.6445\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:12,235 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,539 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,594 epoch 19 - iter 1/7 - loss 1.80059659 - samples/sec: 598.68 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,647 epoch 19 - iter 2/7 - loss 2.13506705 - samples/sec: 606.40 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,720 epoch 19 - iter 3/7 - loss 2.05078447 - samples/sec: 440.61 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,777 epoch 19 - iter 4/7 - loss 2.12986436 - samples/sec: 568.88 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,837 epoch 19 - iter 5/7 - loss 2.22110751 - samples/sec: 530.65 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,895 epoch 19 - iter 6/7 - loss 2.20383098 - samples/sec: 560.07 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,920 epoch 19 - iter 7/7 - loss 2.21202106 - samples/sec: 1331.90 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,920 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:15,920 EPOCH 19 done: loss 2.2120 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,092 DEV : loss 1.8031885623931885 - score 0.6292\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,095 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,096 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,153 epoch 20 - iter 1/7 - loss 2.34496689 - samples/sec: 562.39 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,211 epoch 20 - iter 2/7 - loss 2.06156826 - samples/sec: 559.87 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,269 epoch 20 - iter 3/7 - loss 1.83932149 - samples/sec: 549.56 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,332 epoch 20 - iter 4/7 - loss 1.89005902 - samples/sec: 517.17 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,393 epoch 20 - iter 5/7 - loss 2.07312796 - samples/sec: 528.99 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,457 epoch 20 - iter 6/7 - loss 2.10587734 - samples/sec: 499.65 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,482 epoch 20 - iter 7/7 - loss 2.35720101 - samples/sec: 1302.09 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,482 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,483 EPOCH 20 done: loss 2.3572 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,639 DEV : loss 1.9530328512191772 - score 0.6384\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,642 BAD EPOCHS (no improvement): 2\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,642 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,696 epoch 21 - iter 1/7 - loss 2.47787094 - samples/sec: 604.32 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,749 epoch 21 - iter 2/7 - loss 2.36704242 - samples/sec: 608.56 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,804 epoch 21 - iter 3/7 - loss 2.50350738 - samples/sec: 581.91 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,869 epoch 21 - iter 4/7 - loss 2.55670577 - samples/sec: 497.95 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,932 epoch 21 - iter 5/7 - loss 2.43224411 - samples/sec: 511.43 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:16,996 epoch 21 - iter 6/7 - loss 2.32103928 - samples/sec: 507.75 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,018 epoch 21 - iter 7/7 - loss 2.21200965 - samples/sec: 1415.14 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,019 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,019 EPOCH 21 done: loss 2.2120 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,183 DEV : loss 1.7803841829299927 - score 0.6406\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,186 BAD EPOCHS (no improvement): 3\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,186 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,233 epoch 22 - iter 1/7 - loss 2.57958150 - samples/sec: 689.30 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,283 epoch 22 - iter 2/7 - loss 2.39531493 - samples/sec: 648.79 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,346 epoch 22 - iter 3/7 - loss 2.43385617 - samples/sec: 510.87 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,403 epoch 22 - iter 4/7 - loss 2.59482980 - samples/sec: 564.90 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,450 epoch 22 - iter 5/7 - loss 2.72061338 - samples/sec: 688.01 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,503 epoch 22 - iter 6/7 - loss 2.52191806 - samples/sec: 615.89 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,526 epoch 22 - iter 7/7 - loss 2.43197833 - samples/sec: 1418.51 - lr: 0.100000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,526 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,526 EPOCH 22 done: loss 2.4320 - lr 0.1000000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,693 DEV : loss 1.7761889696121216 - score 0.6234\u001b[0m\n",
      "\u001b[34mEpoch    22: reducing learning rate of group 0 to 5.0000e-02.\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,696 BAD EPOCHS (no improvement): 4\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,696 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,756 epoch 23 - iter 1/7 - loss 2.32448530 - samples/sec: 537.93 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,810 epoch 23 - iter 2/7 - loss 2.17781401 - samples/sec: 603.71 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,869 epoch 23 - iter 3/7 - loss 2.12200856 - samples/sec: 547.73 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,931 epoch 23 - iter 4/7 - loss 2.24592841 - samples/sec: 516.40 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:17,996 epoch 23 - iter 5/7 - loss 2.23146687 - samples/sec: 499.39 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:18,057 epoch 23 - iter 6/7 - loss 2.19300048 - samples/sec: 527.37 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:18,087 epoch 23 - iter 7/7 - loss 2.01770456 - samples/sec: 1095.47 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:18,087 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:18,087 EPOCH 23 done: loss 2.0177 - lr 0.0500000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:18,245 DEV : loss 1.706174373626709 - score 0.6533\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:18,248 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,573 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,625 epoch 24 - iter 1/7 - loss 1.51811433 - samples/sec: 623.25 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,682 epoch 24 - iter 2/7 - loss 1.55931807 - samples/sec: 568.50 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,741 epoch 24 - iter 3/7 - loss 2.00294431 - samples/sec: 546.82 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,799 epoch 24 - iter 4/7 - loss 2.25460738 - samples/sec: 553.09 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,852 epoch 24 - iter 5/7 - loss 2.39128013 - samples/sec: 605.74 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,907 epoch 24 - iter 6/7 - loss 2.37812157 - samples/sec: 592.05 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,942 epoch 24 - iter 7/7 - loss 2.38082293 - samples/sec: 935.20 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,942 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:21,942 EPOCH 24 done: loss 2.3808 - lr 0.0500000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,105 DEV : loss 1.8620479106903076 - score 0.6455\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,109 BAD EPOCHS (no improvement): 1\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,109 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,165 epoch 25 - iter 1/7 - loss 1.84290028 - samples/sec: 573.92 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,226 epoch 25 - iter 2/7 - loss 2.07397103 - samples/sec: 530.24 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,278 epoch 25 - iter 3/7 - loss 1.82417206 - samples/sec: 617.47 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,339 epoch 25 - iter 4/7 - loss 2.25318995 - samples/sec: 530.71 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,399 epoch 25 - iter 5/7 - loss 2.11663716 - samples/sec: 540.36 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,457 epoch 25 - iter 6/7 - loss 2.06630963 - samples/sec: 559.01 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,481 epoch 25 - iter 7/7 - loss 2.30813617 - samples/sec: 1310.20 - lr: 0.050000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,482 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,482 EPOCH 25 done: loss 2.3081 - lr 0.0500000\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,647 DEV : loss 1.6936701536178589 - score 0.6616\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:22,650 BAD EPOCHS (no improvement): 0\u001b[0m\n",
      "\u001b[34msaving best model\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:29,197 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:29,197 Testing using best model ...\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:29,197 loading file /opt/ml/model/best-model.pt\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:30,261 0.7005#0110.6268#0110.6616\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:30,261 \u001b[0m\n",
      "\u001b[34mResults:\u001b[0m\n",
      "\u001b[34m- F1-score (micro) 0.6616\u001b[0m\n",
      "\u001b[34m- F1-score (macro) 0.5053\u001b[0m\n",
      "\u001b[34mBy class:\u001b[0m\n",
      "\u001b[34mjob_role   tp: 128 - fp: 55 - fn: 67 - precision: 0.6995 - recall: 0.6564 - f1-score: 0.6772\u001b[0m\n",
      "\u001b[34mlocation   tp: 3 - fp: 1 - fn: 11 - precision: 0.7500 - recall: 0.2143 - f1-score: 0.3333\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:30,262 ----------------------------------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34mtraining completed\u001b[0m\n",
      "\u001b[34m2022-06-28 18:19:31,296 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2022-06-28 18:20:36 Uploading - Uploading generated training model\n",
      "2022-06-28 18:21:31 Completed - Training job completed\n",
      "Training seconds: 3090\n",
      "Billable seconds: 3090\n"
     ]
    }
   ],
   "source": [
    "estimator.fit({'train': inputs + \"/train.txt\",\"validation\":inputs + \"/train.txt\",\"eval\":inputs  + \"/train.txt\"})\n",
    "\n",
    "#estimator.fit({'train': inputs,\"validation\":inputs,\"eval\":inputs})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deploy Model using Sagemaker Model class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Download the output files from the training job\n",
    "2. Archieve inference code and dependencies along with the model object.\n",
    "3. Upload the archieve to s3\n",
    "4. Create a pytorch Model and deploy.\n",
    "5. Invoke the model\n",
    "6. Cleanup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Download and unzip output files from training job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = estimator.latest_training_job.describe()[\"ModelArtifacts\"][\"S3ModelArtifacts\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bucket name is sagemaker-us-east-1-275443674968\n",
      "File path is pytorch-training-2022-06-28-17-28-17-450/output/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "bucket_name = model_path.replace(\"s3://\",\"\").split(\"/\")[0]\n",
    "prefix = \"/\".join(model_path.replace(\"s3://\",\"\").split(\"/\")[1:])\n",
    "\n",
    "print(\"bucket name is {}\".format(bucket_name))\n",
    "print(\"File path is {}\".format(prefix))\n",
    "\n",
    "sagemaker_session.download_data(path=\"output/\",bucket=bucket_name,key_prefix=prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tarfile\n",
    "model_tar = tarfile.open('output/model.tar.gz')\n",
    "model_tar.extractall('output/') # specify which folder to extract to\n",
    "model_tar.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Create an tar containing the model file and inference code\n",
    "\n",
    "We need to create a tar.gz file in the below format \n",
    "\n",
    "model.tar.gz/\n",
    "    \n",
    "        |- model.pt\n",
    "    \n",
    "        |- code/\n",
    "    \n",
    "          |- inference.py\n",
    "      \n",
    "          |- requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the required folder structure and copy the necessary contents\n",
    "\n",
    "! mkdir deployment\n",
    "! cp output/final-model.pt deployment/model.pt\n",
    "! mkdir deployment/code\n",
    "! cp code/inference.py deployment/code/inference.py\n",
    "! cp code/requirements.txt deployment/code/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the tar file \n",
    "\n",
    "import tarfile\n",
    "with tarfile.open('model.tar.gz', mode='w:gz') as archive:\n",
    "    archive.add('deployment',arcname='.')\n",
    "archive.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Upload Archieve/tar file to s3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = sagemaker_session.upload_data(path='model.tar.gz', key_prefix='data/ner-dataset/model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model stored in this location s3://sagemaker-us-east-1-275443674968/data/ner-dataset/model/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "print(\"Model stored in this location {}\".format(model_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. Create a pytorch Model and Deploy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.pytorch import PyTorchModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker import get_execution_role\n",
    "role = get_execution_role()\n",
    "\n",
    "pytorch_model = PyTorchModel(model_data=model_path, role=role,\n",
    "                             entry_point='inference.py', framework_version='1.4.0',\n",
    "                              py_version=\"py3\", source_dir='code')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------!"
     ]
    }
   ],
   "source": [
    "predictor = pytorch_model.deploy(instance_type='ml.t2.large', initial_instance_count=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5. Invoke Model endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'entities': [{'text': 'Class A CDL Truck Driver', 'start_pos': 0, 'end_pos': 24, 'label': 'job_role', 'confidence': 0.671200567483902}]}\n"
     ]
    }
   ],
   "source": [
    "predictor.serializer = sagemaker.serializers.JSONSerializer()\n",
    "predictor.deserializer = sagemaker.deserializers.JSONDeserializer()\n",
    "\n",
    "response = predictor.predict({\"text\":\"Class A CDL Truck Driver\"})\n",
    "\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor.delete_endpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}