SASRec_interactive.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "85d18154-197a-415c-8bef-313d5ada67c6",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Pytorch/lightning implementation of SASRec model\n",
    "Implementation of **Self-attentive sequential recommendation** paper:\n",
    "```\n",
    "@inproceedings{kang2018self,\n",
    "  title={Self-attentive sequential recommendation},\n",
    "  author={Kang, Wang-Cheng and McAuley, Julian},\n",
    "  booktitle={2018 IEEE International Conference on Data Mining (ICDM)},\n",
    "  pages={197--206},\n",
    "  year={2018},\n",
    "  organization={IEEE}\n",
    "}\n",
    "```\n",
    "Originally taken [this code](https://github.com/pmixer/SASRec.pytorchhttps://github.com/pmixer/SASRec.pytorch) and rewritten model class plus used lightning.  \n",
    "This notebook serves the purpose of interactive code execution/debugging.  \n",
    "Main code for multiple GPU training is [here](./SASRecMain.py) \n",
    "\n",
    "Author: Sergei Bazhin  \n",
    "Date: 2021-DEC - JAN-2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32eadb38-f592-4973-a4d7-ffa3836de872",
   "metadata": {},
   "outputs": [],
   "source": [
    "%config Completer.use_jedi = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a01efcee-7815-4263-938f-60f29ff0ad4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import pytorch_lightning as pl\n",
    "import argparse\n",
    "from importlib import reload\n",
    "# module with datasets definition = train, validation and test\n",
    "import DataHelper as DH\n",
    "import SASRecModel as SASRec\n",
    "SASRec = reload(SASRec)\n",
    "DH = reload(DH)\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from pytorch_lightning.callbacks import StochasticWeightAveraging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7c7c5a3a-5fe2-43a8-8ecc-102af3e74fe9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('dataset', 'ml-1m')\n",
      "('maxlen', 200)\n",
      "('hidden_units', 50)\n",
      "('d_model', 50)\n",
      "('num_blocks', 2)\n",
      "('num_heads', 1)\n",
      "('dropout_rate', 0.2)\n",
      "('ndcg_samples', 100)\n",
      "('top_k', 10)\n",
      "('opt', 'Adam')\n",
      "('lr', 0.001)\n",
      "('weight_decay', 0.001)\n",
      "('batch_size', 128)\n",
      "('warmup_proportion', 0.2)\n",
      "('max_iters', 10000)\n",
      "('num_epochs', 201)\n",
      "('use_swa', False)\n",
      "('swa_epoch_start', 0.8)\n",
      "('swa_annealing_epochs', 10)\n",
      "('xavier_init', True)\n",
      "('inference_only', False)\n",
      "('checkpoint_path', None)\n",
      "('strategy', 'ddp_spawn')\n",
      "('precision', 16)\n",
      "('accelerator', 'auto')\n",
      "('devices', 'auto')\n"
     ]
    }
   ],
   "source": [
    "# setup command line arguments\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--dataset', default='ml-1m', \n",
    "                    required=True, \n",
    "                    help=\"dataset to use : Beauty, ml-1m(default), Steam or Video\")\n",
    "\n",
    "parser.add_argument('--maxlen', default=50, type=int, \n",
    "                    help=\"truncate input sequence to last maxlen items, default 50\")\n",
    "parser.add_argument('--hidden_units', default=50, type=int, help=\"synonym for d_model\") # synonym for d_model\n",
    "parser.add_argument('--d_model', default=50, type=int, \n",
    "                    help=\"Transformer internal dimention\") # same as hidden_units   \n",
    "parser.add_argument('--num_blocks', default=2, type=int, help=\"Number of blocks in Transformer\")\n",
    "parser.add_argument('--num_heads', default=1, type=int, help=\"Number of heads in self-attention\")\n",
    "parser.add_argument('--dropout_rate', default=0.5, type=float, help=\"Dropout rate for Transformer\")\n",
    "\n",
    "\n",
    "parser.add_argument('--ndcg_samples', default=100, type=int, \n",
    "                    help=\"How many random items to pick up in hit-rate and ndcg calculation, default 100\")\n",
    "parser.add_argument('--top_k', default=10, type=int, \n",
    "                    help=\"How many items with high scores to pick for hit-rate and ndcg calculation, default 10\")\n",
    "parser.add_argument('--opt', default='Adam', type=str, help=\"Oplimizer to use: Adam(default), AdmaW, FusedAdam(requires apex library)\")\n",
    "parser.add_argument('--lr', default=0.001, type=float, \n",
    "                    help=\"learning rate, default 0.001\")\n",
    "parser.add_argument('--weight_decay', default=0.001, type=float, help=\"Weight decay for AdmaW\")\n",
    "parser.add_argument('--batch_size', default=128, type=int)\n",
    "parser.add_argument('--warmup_proportion', default=0.2, type=float, help=\"Fraction of total optimization steps to increase learning rate from zero to max value\")\n",
    "# for different optimizers - regular Adam uses num_epochs and LAMB uses max_iters\n",
    "parser.add_argument('--max_iters', default=10000, type=int, help=\"Optimization budget in update iterations\")\n",
    "parser.add_argument('--num_epochs', default=201, type=int, help=\"Number of epochs to train\")\n",
    "# swa parameters\n",
    "parser.add_argument('--use_swa', default=False, type=bool, help=\"Use Stochastic Weights Ageraging algorythm\")\n",
    "parser.add_argument('--swa_epoch_start', default=0.8, type=float, help=\"Start SWA after that part of total epochs\")\n",
    "parser.add_argument('--swa_annealing_epochs', default=10, type=int, help=\"Number of epochs in the annealing phase of SWA\")\n",
    "\n",
    "# xavier init\n",
    "parser.add_argument('--xavier_init', default=True, type=bool, help=\"Use xavier normal to init the model\")\n",
    "\n",
    "parser.add_argument('--inference_only', default=False, type=bool)\n",
    "parser.add_argument('--checkpoint_path', default=None, type=str, help=\"Path to lightning checkpoint file\")\n",
    "\n",
    "# Torch Lightning settings\n",
    "# https://pytorch-lightning.readthedocs.io/en/stable/advanced/multi_gpu.html\n",
    "# Data Parallel (strategy='dp') (multiple-gpus, 1 machine)\n",
    "# DistributedDataParallel (strategy='ddp') (multiple-gpus across many machines (python script based)).\n",
    "# DistributedDataParallel (strategy='ddp_spawn') (multiple-gpus across many machines (spawn based)).\n",
    "# DistributedDataParallel 2 (strategy='ddp2') (DP in a machine, DDP across machines).\n",
    "# Horovod (strategy='horovod') (multi-machine, multi-gpu, configured at runtime)\n",
    "# TPUs (tpu_cores=8|x) (tpu or TPU pod)\n",
    "parser.add_argument('--strategy', default='ddp_spawn', type=str, help=\"Lightning parallel training strategy dp, ddp, ddp_spawn(default), ddp2, etc \")\n",
    "parser.add_argument('--precision', default=16, type=int, help=\"Lightning precision for model data during trining 16(default) or 32\")\n",
    "parser.add_argument('--accelerator', default=\"auto\", type=str, help=\"Lightning accelerator auto(defaut), cpu, gpu, tpu\")\n",
    "parser.add_argument('--devices', default=\"auto\", type=str, \n",
    "                    help=\"Lightning devices to use - see https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html#devices\")\n",
    "\n",
    "args = parser.parse_args( ['--dataset=ml-1m', '--maxlen=200', '--dropout_rate=0.2'])\n",
    "args = vars(args)\n",
    "print(*[(k,v) for (k,v) in args.items()], sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97f1a18c-368c-4e42-ba43-229867df5b6b",
   "metadata": {},
   "source": [
    "## Perepare the data  \n",
    "We have 3 datasets:  \n",
    " - for training\n",
    " - for validation\n",
    " - for testing  \n",
    " They all contain all users and the last two items in sequence are split between validation (penultimate item) and test (last item)  \n",
    " Training has all user items but without last two that falls into validation and test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e6655b5-40d8-4994-a586-db80988f7497",
   "metadata": {},
   "source": [
    "**user_train** - dict with key = *userid* and value = list of all items selected in respected time order  \n",
    "**user_valid** - dict with the same structure as above but with penulitimate item (just one item)  \n",
    "**user_test** - same as above but with ultimate item selected  \n",
    "i.e. you have user 5 with items 1, 29, 34, 15, 8 in his sequence of items there will be the below data in vars:  \n",
    "```\n",
    "user_train[5] = [1,29,34]  \n",
    "user_valid[5] = [15]  \n",
    "user_test[5] = [8]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "738fa43a-ad2e-4032-b501-c0c3b3d19a27",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read dataset\n",
    "dataset = DH.data_partition('ml-1m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a21b317e-d7ba-4059-9e80-32e61c9a1604",
   "metadata": {},
   "outputs": [],
   "source": [
    "[user_train, user_valid, user_test, usernum, itemnum] = dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1f4e6ee1-26e8-4202-be8c-0e80562e3836",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average sequence length: 163.5\n"
     ]
    }
   ],
   "source": [
    "# batches got sliced by users, i.e. batch accumulate BATCH_SIZE user sequences of items selected/bought\n",
    "BATCH_SIZE = args['batch_size']\n",
    "num_batch = len(user_train) // BATCH_SIZE  # number of batches\n",
    "\n",
    "user_train_lens = list(map(len,[v for k,v in user_train.items()]))\n",
    "print(f'average sequence length: {sum(user_train_lens)/len(user_train):.1f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f174a3f0-1cf6-4442-8075-b04fbe22c0e4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "579115274ad447918aa6471f92af530a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=6040.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# dataset for validation\n",
    "valid_data = DH.SequenceDataValidation(user_train, user_valid, usernum, itemnum, args['maxlen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "854eea48-8968-44e2-b9de-ecadd01eb243",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2289563e26444aa989dc41d81d28049",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=6040.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# dataset for test\n",
    "test_data = DH.SequenceDataTest(user_train, user_valid, user_test, usernum, itemnum, args['maxlen'], args['ndcg_samples'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a30c07fb-1414-4e99-8233-9a7e0b92205f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce5bcdd8b0d843b982e504fcf4de503a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=6040.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#dataset for training\n",
    "train_data = DH.SequenceData(user_train, usernum, itemnum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "94ae2beb-f60d-4964-9dfc-c3f0d18630bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=4,\n",
    "                          shuffle=False, collate_fn = lambda x: DH.tokenize_batch(x, max_len=args['maxlen']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad1fba4c-3f02-4103-9cb4-ee2e4cda8295",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Unit-test training loader "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "03f4f500-1dc7-437f-9588-715b40699147",
   "metadata": {},
   "outputs": [],
   "source": [
    "u, seq, pos, neg = next(iter(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "02db37e5-1e1e-422a-8297-059859ef8dc2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 200])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a465df18-908c-4fbd-82bc-88d510974a98",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(seq[0]) == args['maxlen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2bef9895-0471-45fb-b51b-8a808407d44f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3df854b8-5104-4f79-b072-397b8d903a03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ed7a6fa3-2244-469e-816e-f33dd0d4aa2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# random user from batch\n",
    "_u = np.random.randint(0,len(u))\n",
    "_u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "549890b0-9b1c-421f-acc6-98f4802cbb25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[255 256 179 167 172 157 257  39 199 258]\n"
     ]
    }
   ],
   "source": [
    "# train sequnce\n",
    "print(seq[_u].numpy()[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d8ea8ae4-20b8-467c-9ca9-69bb2f2e295e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[256 179 167 172 157 257  39 199 258  29]\n"
     ]
    }
   ],
   "source": [
    "# train shifted one item ahead\n",
    "print(pos[_u].numpy()[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "fdc0d829-b18b-4a6a-aa18-f51327da00d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2377  992 1485 2751  470 3337  547 1605 1347 1024]\n"
     ]
    }
   ],
   "source": [
    "# negative sequnce\n",
    "print(neg[_u].numpy()[-10:])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "484d4a74-aaf8-4b48-bcf4-a037fde45224",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Unit-test validation and test data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a1a02b6b-1d6d-47b2-91e4-53ca50b73f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "ii = np.random.randint(1,usernum+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fa64a8cc-0c89-48e6-9807-93239638cd52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                       Validation data \n",
      "\n",
      "Main sequence                  : 1100 45 985 1862 370 65 642 66 48 501\n",
      "Validation sequene             : 639 2288 2136 2025 874 3413 2600 2114 2417 260\n"
     ]
    }
   ],
   "source": [
    "print(\"{0:}{1:>40}\".format(\"\\n\",\"Validation data \\n\"))\n",
    "print(\"{0:<30}\".format(\"Main sequence \"),\":\",*valid_data.seq[ii,-10:].numpy()) \n",
    "print(\"{0:<30}\".format(\"Validation sequene \"),\":\", *valid_data.valid[ii,:10].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e5c6b7ec-dcea-4554-96ea-d7ec0beadbcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                             Test data \n",
      "\n",
      "Main sequence                  : 1100 45 985 1862 370 65 642 66 48 501 639\n",
      "Validation sequene             : 710 3359 420 35 2368 3280 205 2602 1460 2534\n"
     ]
    }
   ],
   "source": [
    "print(\"{0:}{1:>40}\".format(\"\\n\",\"Test data \\n\"))\n",
    "print(\"{0:<30}\".format(\"Main sequence \"),\":\",*test_data.seq[ii, -11:].numpy()) \n",
    "print(\"{0:<30}\".format(\"Validation sequene \"),\":\", *test_data.valid[ii,:10].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "0348a620-b6fb-4625-8e70-e205acfeb13d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2529 645 812 592 963 1035 1038 837 816 1044 1047 3018\n"
     ]
    }
   ],
   "source": [
    "print(*user_train[ii+1][-10:], *user_valid[ii+1], *user_test[ii+1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "650c4f82-c618-471b-911b-2683275f60f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=4, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9acd950-b14b-48fb-8313-8e6d69946d3a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Unit-test metrics calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "5ceee561-e6f3-4e67-8bc7-8154ee299c17",
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "[train, valid, test, usernum, itemnum] = deepcopy(dataset)\n",
    "\n",
    "NDCG = 0.0\n",
    "HT = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "f6f32a0e-867a-4813-ab12-2c63516d5900",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of users in batch\n",
    "u = [122, 144]\n",
    "\n",
    "# get validation items\n",
    "valid_seq = torch.as_tensor([valid[_u] for _u in u], dtype=torch.int) # (batch x 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "4e6159c1-b672-44d1-bbde-73311e22d9e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a matrix from train sequence (batch, args['maxlen'] - 1)\n",
    "final_seq = torch.zeros((len(u),args['maxlen'] - 1), dtype=torch.int)\n",
    "for ii,_u in enumerate(u):\n",
    "    idx = min(args['maxlen'] - 1, len(train[_u]))\n",
    "    final_seq[ii, -idx:] = torch.as_tensor(train[_u][-idx:])\n",
    "\n",
    "# final seq (batch, args['maxlen'])\n",
    "final_seq = torch.cat((final_seq, valid_seq), dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "91fc1614-823f-48d6-8c99-ed0d5db032b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# making a test sequence with one element from test and the rest random\n",
    "test_seq = torch.zeros((len(u), 101), dtype=torch.int)\n",
    "# all elements that are in train plus padding zero\n",
    "for ii, (_u, seq) in enumerate(zip(u, final_seq)):\n",
    "    items_not_in_seq = np.array(list(set(range(1,itemnum+1)) - set(seq.numpy()))) # random stuff not in seq\n",
    "    test_seq[ii,0] = test[_u][0] # get true next element from test set\n",
    "    test_seq[ii,1:] = torch.from_numpy(items_not_in_seq[np.random.randint(0, len(items_not_in_seq), 100)]) # fill the rest with random stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "082c53a8-7ce4-473e-b5de-59bb9d251f1d",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 919, 2871, 2688, 3360,  480, 2031, 2828,  524,  598, 1611, 2785, 3042,\n",
       "         1532, 1063, 1467, 2040, 2763, 2916, 1823,  668, 1124, 1976, 1673,  525,\n",
       "         1518, 2615, 2776, 1437, 1079, 1978,  406, 2954,  897, 2514, 3390,  417,\n",
       "         1519, 2713, 2332, 1517,  321, 2499, 1283, 2517, 2688,  185, 2233,  453,\n",
       "         1756, 2679,  456,  669, 2025, 1473, 2226,  812, 2107, 2198,  643, 2980,\n",
       "         2694,    8, 1778, 2027,  322, 1815, 1284, 2881, 1716,  243, 2384, 2340,\n",
       "          617,  906, 2236, 2315, 1637, 3184, 2395,  610, 2118, 2334, 3355, 1912,\n",
       "         1521,  604, 2003, 1173, 1519, 2207, 2051, 2369, 1029,  109, 2086,  228,\n",
       "          106,  865,  520,  725, 2287],\n",
       "        [ 672,  770, 3298,  668,   94, 3290,   20, 2532,  766, 3186, 1625, 2100,\n",
       "         2251,  709,  391, 3336, 1080,  137, 2104, 1768,  476,  381, 1283, 1948,\n",
       "         2409,  414, 2309, 1415, 1231, 2427, 1998, 2464,  899, 3323,  971,  134,\n",
       "         1400, 2559,  527, 2520, 3211,  926,  428,  734, 3321, 1288, 1761, 1507,\n",
       "          616, 2259,  134, 2538,  525, 1321, 2767, 1187,  849,  557, 1781, 1065,\n",
       "         2819, 2200, 1652, 2251, 2424, 1955, 2593,   76,  939, 3395, 2944, 1001,\n",
       "         1806,  426, 2064, 3162, 1696, 2528, 2098,  721,  523,  837, 3385,  367,\n",
       "         2841,  442, 1019, 1371, 3160, 2711, 2782, 3187,  771,  106, 2270,  959,\n",
       "         1966,  670,  983, 3208, 1337]], dtype=torch.int32)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "93c8bc3a-b2fa-4b52-a23f-116b33cfdf58",
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    log_feats = model.log2feats(final_seq) # shape (batch, seq_len, hidden_dim) = (1x200x50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "d894dcfe-4d5d-4e3b-ab93-b6a21e1a33ff",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 0.6619,  0.6596, -0.1745, -0.2133, -0.7123,  0.4204,  0.4625, -0.6209,\n",
       "          -0.4138,  0.0817, -0.1496,  0.6792,  0.5925,  0.7006, -0.6502, -0.5306,\n",
       "           0.2231, -0.6313,  0.7336,  0.5250,  0.6301,  0.3757,  0.5017, -0.7404,\n",
       "           0.4616,  0.5239, -0.6687, -0.6023, -0.6875, -0.5923,  0.6337, -0.1855,\n",
       "           0.6248,  0.4989, -0.4843, -0.3989,  0.7503, -0.4861,  0.3672, -0.7682,\n",
       "          -0.7489,  0.0185,  0.2974,  0.7162,  0.3279, -0.6246, -0.8353,  0.4458,\n",
       "          -0.9165,  0.6383],\n",
       "         [ 0.6621,  0.6496, -0.1759, -0.2299, -0.7259,  0.4308,  0.4675, -0.5906,\n",
       "          -0.4444,  0.0792, -0.1448,  0.6820,  0.5929,  0.7078, -0.6550, -0.5333,\n",
       "           0.2334, -0.6315,  0.7423,  0.5466,  0.6331,  0.3593,  0.5242, -0.7519,\n",
       "           0.4644,  0.5121, -0.6704, -0.5962, -0.6760, -0.6022,  0.6266, -0.1775,\n",
       "           0.6117,  0.4997, -0.4859, -0.4087,  0.7321, -0.4957,  0.3658, -0.7753,\n",
       "          -0.7522,  0.0150,  0.3030,  0.7147,  0.3252, -0.6060, -0.8337,  0.4408,\n",
       "          -0.8919,  0.6464]]),\n",
       " torch.Size([2, 50]))"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_feat = log_feats[:, -1, :] # last hidden state/embedding\n",
    "final_feat, final_feat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "8f068b57-8c3a-4099-abc7-786501021776",
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    item_embs = model.item_emb(test_seq) # shape torch.Size([1, 101, 50]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "3f385f81-4058-42a0-9712-8eac127ab326",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([2, 101, 50]), torch.Size([2, 50, 1]))"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_embs.shape, final_feat.unsqueeze(-1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "128e1f04-ccec-4302-8542-dff55eac0c3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 101, 1])"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = torch.bmm(item_embs, final_feat.unsqueeze(-1))\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "fbed01a1-2f10-436e-9ccf-2587f9a458db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 101])"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = -logits.squeeze()\n",
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "id": "b90017c4-d985-42a2-b636-83fd28049bb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[43, 98, 22, 40, 38, 92, 21, 75, 48, 68,  0,  8, 57, 89, 35],\n",
       "        [30, 36, 78,  7, 51, 97, 31, 72, 64, 24,  0, 28, 99, 22, 86]])"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_, indices = torch.topk(predictions,15,dim=1, largest=False)\n",
    "indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "id": "3810315e-638f-4e37-8aa8-f92611c82638",
   "metadata": {},
   "outputs": [],
   "source": [
    "_, indices = torch.where(indices == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "id": "49fba730-962b-42fb-8dd5-6fb1537cca46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10, 10])"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "id": "d28536b3-1770-4e87-8489-bb03b23d8ac5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 1], dtype=torch.int32)"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hits = torch.as_tensor(indices < 11, dtype=torch.int)\n",
    "hits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "ceca46b7-1d42-48a3-a601-42b26d227946",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.2789, 0.2789])"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hits/torch.log2(indices+2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c53acbff-a67d-4002-847d-b2b24d502029",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Assemble a model  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d0f4a62-802c-4efe-9ac6-d8ae8ac11901",
   "metadata": {},
   "source": [
    "**From `Attention is all you need` paper:**  \n",
    "We apply dropout to the **output of each sub-layer**, before it is added to the sub-layer input and normalized. \n",
    "In addition, we apply dropout to the **sums of the embeddings and the positional encodings** in both the encoder and decoder stacks. \n",
    "For the base model, we use a rate of P drop = 0.1.  \n",
    "\n",
    "The encoder is composed of a stack of N = 6 identical layers. Each layer has two `sub-layers`.  \n",
    "The first is a multi-head self-attention mechanism, and the second is a simple, position-wise fully connected feed-forward network.   \n",
    "We employ a residual connection around each of the two sub-layers, followed by layer normalization.  \n",
    "That is, the output of each sub-layer is `LayerNorm(x + Sublayer(x))`, where Sublayer(x) is the function implemented by the sub-layer itself.  \n",
    "To facilitate these residual connections, all sub-layers in the model, as well as the embedding layers, produce outputs of dimension dmodel = 512.\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c684ea74-d1b1-4260-bbdd-530bd18f2652",
   "metadata": {},
   "source": [
    "However, when the network goes deeper, several problems become exacerbated: \n",
    "1) the increased model capacity leads to overfitting; \n",
    "2) the training process becomes unstable (due to vanishing gradients etc.); and \n",
    "3) models with more parameters often require more training time.  \n",
    "\n",
    "We perform the following operations to alleviate these problems: `g(x) = x + Dropout(g(LayerNorm(x)))`,  \n",
    "where `g(x)` represents the self attention layer or the feed-forward network.  \n",
    "That is to say, for layer `g` in each block, we apply **layer normalization**on the input `x` before feeding into `g`,   \n",
    "apply **dropout** on `g’s` output, and add the input `x` to the final output.   \n",
    "The **dropout** rate of turning off neurons is 0.2 for **MovieLens-1m** and 0.5 for the other three datasets due to their sparsity.  \n",
    "We also apply a **dropout** layer on the embedding `E`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "273a518d-04cb-4fe2-bdc7-3381f27234a3",
   "metadata": {},
   "source": [
    "The optimizer is the Adam optimizer, the `learning rate` is set to `0.001`, and the `batch size` is `128`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a237dd27-a182-40ac-afba-185045c540c5",
   "metadata": {},
   "source": [
    "**Shared Item Embedding**  \n",
    "To reduce the model size and alleviate overfitting, we consider a single item embedding M:  \n",
    "`ri,t=Ft Mi`. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f004e72f-5513-4f6b-a592-6db8260c1c65",
   "metadata": {},
   "source": [
    "<img src=\"./sasrec-loss-function.png\" alt=\"Loss function\" style=\"width: 600px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c87c96d-c714-4760-a4c2-a24ec809ed9d",
   "metadata": {},
   "source": [
    "Note that we ignore the terms where `ot = <pad>`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d3529c7-4d53-42d3-b3e4-7c4ec05b4be7",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Ablation study\n",
    "Remove PE (Positional Embedding): Without the positional embedding P, the attention weight on each item depends only on item embeddings. That is to say, the model makes recommendations based on users’ past actions, but their order doesn’t matter. This variant might be suitable for sparse datasets, where user sequences are typically short. This variant performs better then the default model on the sparsest dataset (Beauty), but worse on other denser datasets."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38ce6a91-3f2f-4b36-b8d1-13f602f7dd1c",
   "metadata": {},
   "source": [
    "## Model assembly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "58246255-5d5b-4248-ada9-d37959069371",
   "metadata": {},
   "outputs": [],
   "source": [
    "SASRec = reload(SASRec)\n",
    "from torch.nn import MultiheadAttention, LayerNorm, Dropout, Conv1d, Embedding, BCEWithLogitsLoss\n",
    "from SASRecModel import PointWiseFF, SASRecEncoderLayer, PositinalEncoder, SASRecEncoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cadc492-c8b3-42d3-98c7-fb25557fca4c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Building blocks for Encoder:\n",
    "- **encoder** layer \n",
    "- **positional encoder**\n",
    "- **point-wise** feed-forward"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9ff784d-37ba-4dc6-ae9b-0d77de2a5b47",
   "metadata": {},
   "source": [
    "### Unit-test building blocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a89e55e9-cc3a-4196-935b-68013747bfa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_x = torch.randn(BATCH_SIZE, args['maxlen'], args['d_model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bfb1a710-7329-47ac-8d53-40e59c07f1fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_test = Embedding(args['maxlen'], embedding_dim=args['hidden_units'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "71b9468f-5fdd-4806-bf5e-d537defa7093",
   "metadata": {},
   "outputs": [],
   "source": [
    "pe_test = Embedding(args['maxlen'], embedding_dim=args['d_model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "da45b892-0b37-4ecd-99a9-815e939b0f95",
   "metadata": {},
   "outputs": [],
   "source": [
    "pe_for_one_sequence = pe_test(torch.arange(0,input_x.shape[1], dtype=torch.int)) # get a single positional embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dc4f0dc5-78b3-40a6-855d-d60a12c34e97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 200, 50])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.tile(pe_for_one_sequence, (BATCH_SIZE,1,1)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b5418ec3-3338-4462-b9b6-396f53142695",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([200, 50]), torch.Size([128, 200, 50]))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pe_for_one_sequence.shape, input_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "36e8da56-e4d1-4770-8bc5-fd04eda0a8ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 200, 50])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pe_test = PositinalEncoder(args['maxlen'], args['d_model'])\n",
    "pe_test(input_x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7a76f388-a107-4531-83ba-3fc3b0bbdbb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_test = SASRecEncoderLayer(itemnum, **args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "998e4fdb-acb1-4acc-9ed8-5607926ee570",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 200, 50])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_test(input_x).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48fb32f0-728b-4fb6-b946-5d2ffc739201",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Unit-test final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b7aa6039-0203-4bd0-87f7-ea2981e442fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sas_rec_test = SASRecEncoder(itemnum, **args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5d089e7b-6fb5-4a6c-a31b-a43854475bcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "minib = next(iter(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ceb3354b-ebc8-4be7-90df-9bee7cbd71f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/core/lightning.py:416: UserWarning: You are trying to `self.log()` but the `self.trainer` reference is not registered on the model yet. This is most likely because the model hasn't been passed to the `Trainer`\n",
      "  \"You are trying to `self.log()` but the `self.trainer` reference is not registered on the model yet.\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'loss': tensor(6.1220, grad_fn=<AddBackward0>)}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# need to comment out logging loss to run this cell\n",
    "sas_rec_test.training_step(minib, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "id": "9bb9f29f-beba-4a99-ba97-c4db770996c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "u, seq, pos, neg = next(iter(train_loader))\n",
    "item_emb = sas_rec_test.forward(seq) # get embeddings from transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7ad60681-3084-4ad2-ac86-539ca2108ef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([4, 200, 50]), torch.Size([4, 200]))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sas_rec_test(seq).shape, seq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "id": "1520a5c5-f694-4eaa-88b8-5bcb0c511ed9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pos_scores = sas_rec_test.compute_relevance_scores(item_emb, pos) # scores for positive sequence\n",
    "# neg_scores = sas_rec_test.compute_relevance_scores(item_emb, neg) # scores for negative sequence\n",
    "\n",
    "# pos_labels = torch.ones(pos_scores.shape)\n",
    "# neg_labels = torch.zeros(neg_scores.shape)\n",
    "\n",
    "# indices = torch.where(pos!=0) # exclude padding from loss computation\n",
    "\n",
    "# loss = sas_rec_test.loss(pos_scores[indices], pos_labels[indices]) # loss for positive sequence\n",
    "# loss += self.loss(neg_scores[indices], neg_labels[indices]) # loss for negative sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f152bd13-a3cb-4c6a-b928-10a3ac902d72",
   "metadata": {},
   "source": [
    "## Declare data loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6ecb94db-d42d-44e1-add3-03bcfa534519",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Batch size is - 128\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"\\nBatch size is - {args['batch_size']}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d3621d80-57e1-4d06-bfd6-6e9fc50743eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26865b106bde419cb6fcd2c0e9b272fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=6040.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "val_loader = torch.utils.data.DataLoader(dataset=DH.SequenceDataValidation(user_train, user_valid, usernum, itemnum, args['maxlen']), \n",
    "                                         batch_size=args['batch_size'], shuffle=True, \n",
    "                                         drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "965d7892-98db-48c7-b7e2-9dd0734e94c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc038d08680643a792b934a5affc8a9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=6040.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_loader = torch.utils.data.DataLoader(dataset=DH.SequenceDataTest(user_train, user_valid, user_test, usernum, itemnum, args['maxlen'], args['ndcg_samples']), \n",
    "                                          batch_size=args['batch_size'], shuffle=True, \n",
    "                                          drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c7ffb092-1535-4d81-bf4b-ba11e0102e6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88e54fc1dcd0424ab1bc647c6a40525e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=6040.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset=DH.SequenceData(user_train, usernum, itemnum), \n",
    "                                           batch_size=args['batch_size'],\n",
    "                                           shuffle=True, \n",
    "                                           collate_fn=DH.tokenize_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "a97ba9b6-57d9-4d9f-a474-217436614215",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "          0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  2,  3,\n",
       "          4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,\n",
       "         22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,\n",
       "         40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,\n",
       "         58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75,\n",
       "         76, 77], dtype=torch.int32),\n",
       " tensor([  78,  282, 2760, 1584,  642, 3178, 1761, 2186,  593, 2509, 1610, 2178,\n",
       "          335, 2391,  801,  425,  844, 1628, 1004, 2066, 2450, 2805, 1155,  762,\n",
       "          962, 2505, 2022, 2761, 2762, 3073, 2106, 1429, 1458, 1599, 1495, 1064,\n",
       "          847, 1009,  444,  107, 3006, 2802, 2884, 2656, 1293,  349, 3309,  100,\n",
       "         2906,  997, 2200, 1130,  201,  716,  781, 1352, 2118, 2058, 2092, 1734,\n",
       "         2371, 1454, 1881, 1322, 3302, 3366, 3233, 3333, 1082, 1570, 1365, 2094,\n",
       "         2925, 2698, 3396,  668, 2357,  467, 1340, 2636,  379, 3086,  918, 2790,\n",
       "          314,  943,  585, 2626, 2038, 1323,  859,  930, 1768, 1683, 3050, 3361,\n",
       "          558, 1979,  571,  247, 1597], dtype=torch.int32))"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# seq, val = next(iter(val_loader))\n",
    "# seq[0], val[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ca0f7a4a-14e1-4e4a-a2a8-4865d3939a95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dataset': 'ml-1m',\n",
       " 'maxlen': 200,\n",
       " 'hidden_units': 50,\n",
       " 'd_model': 50,\n",
       " 'num_blocks': 2,\n",
       " 'num_heads': 1,\n",
       " 'dropout_rate': 0.2,\n",
       " 'ndcg_samples': 100,\n",
       " 'top_k': 10,\n",
       " 'opt': 'Adam',\n",
       " 'lr': 0.001,\n",
       " 'weight_decay': 0.001,\n",
       " 'batch_size': 128,\n",
       " 'warmup_proportion': 0.2,\n",
       " 'max_iters': 10000,\n",
       " 'num_epochs': 201,\n",
       " 'use_swa': False,\n",
       " 'swa_epoch_start': 0.8,\n",
       " 'swa_annealing_epochs': 10,\n",
       " 'xavier_init': True,\n",
       " 'inference_only': False,\n",
       " 'checkpoint_path': None,\n",
       " 'strategy': 'ddp_spawn',\n",
       " 'precision': 16,\n",
       " 'accelerator': 'auto',\n",
       " 'devices': 'auto'}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3f1c8490-34f1-4987-897b-cb5449d31a88",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SASRecEncoder(itemnum, **args)\n",
    "# model.load_state_dict(torch.load(\"bazman_sasrec.pt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9ea426e4-d69c-449d-8c29-e582cb404b7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ie.weight                                sucess\n",
      "pe.pe.weight                             sucess\n",
      "enc_stack.0.norm_1.weight                failure\n",
      "enc_stack.0.norm_1.bias                  failure\n",
      "enc_stack.0.norm_2.weight                failure\n",
      "enc_stack.0.norm_2.bias                  failure\n",
      "enc_stack.0.attn.in_proj_weight          sucess\n",
      "enc_stack.0.attn.in_proj_bias            failure\n",
      "enc_stack.0.attn.out_proj.weight         sucess\n",
      "enc_stack.0.attn.out_proj.bias           failure\n",
      "enc_stack.0.ff.conv1.weight              sucess\n",
      "enc_stack.0.ff.conv1.bias                failure\n",
      "enc_stack.0.ff.conv2.weight              sucess\n",
      "enc_stack.0.ff.conv2.bias                failure\n",
      "enc_stack.1.norm_1.weight                failure\n",
      "enc_stack.1.norm_1.bias                  failure\n",
      "enc_stack.1.norm_2.weight                failure\n",
      "enc_stack.1.norm_2.bias                  failure\n",
      "enc_stack.1.attn.in_proj_weight          sucess\n",
      "enc_stack.1.attn.in_proj_bias            failure\n",
      "enc_stack.1.attn.out_proj.weight         sucess\n",
      "enc_stack.1.attn.out_proj.bias           failure\n",
      "enc_stack.1.ff.conv1.weight              sucess\n",
      "enc_stack.1.ff.conv1.bias                failure\n",
      "enc_stack.1.ff.conv2.weight              sucess\n",
      "enc_stack.1.ff.conv2.bias                failure\n",
      "final_norm.weight                        failure\n",
      "final_norm.bias                          failure\n"
     ]
    }
   ],
   "source": [
    "# weight initialization\n",
    "for name, param in model.named_parameters():\n",
    "    try:\n",
    "        torch.nn.init.xavier_normal_(param.data)\n",
    "        print(f\"{name:<40} sucess\")\n",
    "    except:\n",
    "        print(f\"{name:<40} failure\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "9bb21382-ee75-4912-abb3-0e2f47666c34",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save checkpoints\n",
    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
    "checkpoint_callback = ModelCheckpoint(monitor=\"hr_val\", mode='max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "e9de0c50-dd7c-4f23-9f74-622e4ff3b14f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using 16bit native Automatic Mixed Precision (AMP)\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n"
     ]
    }
   ],
   "source": [
    "# run tensorboard before the script launch\n",
    "# tensorboard --logdir ./lightning_logs/ --host 0.0.0.0\n",
    "\n",
    "# https://pytorch-lightning.readthedocs.io/en/stable/advanced/multi_gpu.html\n",
    "# https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html\n",
    "# strategy, accelerator, devices\n",
    "trainer = pl.Trainer(gpus=[0], \n",
    "                     auto_select_gpus=False, \n",
    "                     max_epochs=300,\n",
    "                     reload_dataloaders_every_n_epochs=1,\n",
    "                     val_check_interval=1.0,\n",
    "                     callbacks=[checkpoint_callback],\n",
    "                     log_every_n_steps= int(len(train_data)/args['batch_size']/3), # log 4 times per epoch\n",
    "                     # limit_val_batches=0, How much of validation dataset to check. Useful when debugging or testing something that happens at the end of an epoch.\n",
    "                     num_sanity_val_steps=10, \n",
    "                     precision=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "9b1ceac9-674c-489d-8751-4f3f64c480f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name        | Type              | Params\n",
      "--------------------------------------------------\n",
      "0 | ie          | Embedding         | 170 K \n",
      "1 | pe          | PositinalEncoder  | 10.0 K\n",
      "2 | emb_dropout | Dropout           | 0     \n",
      "3 | enc_stack   | Sequential        | 31.0 K\n",
      "4 | final_norm  | LayerNorm         | 100   \n",
      "5 | loss        | BCEWithLogitsLoss | 0     \n",
      "--------------------------------------------------\n",
      "211 K     Trainable params\n",
      "0         Non-trainable params\n",
      "211 K     Total params\n",
      "0.424     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=1.0, bar_style='info', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:662: UserWarning: Your `val_dataloader` has `shuffle=True`, it is strongly recommended that you turn this off for val/test/predict dataloaders.\n",
      "  category=UserWarning,\n",
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:133: UserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:133: UserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "febdb8cf2cba474a994b926f62ea3b15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/callbacks/progress/base.py:211: UserWarning: The progress bar already tracks a metric with the name(s) 'loss' and `self.log('loss', ..., prog_bar=True)` will overwrite this value.  If this is undesired, change the name or override `get_metrics()` in the progress bar callback.\n",
      "  UserWarning,\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3cb711cf87547cdb8d47c8f6d77b07f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py:688: UserWarning: Detected KeyboardInterrupt, attempting graceful shutdown...\n",
      "  rank_zero_warn(\"Detected KeyboardInterrupt, attempting graceful shutdown...\")\n"
     ]
    }
   ],
   "source": [
    "trainer.fit(model, train_loader, val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "13f166fd-97ac-4946-ba46-245134ce2196",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), f\"bazman_sasrec_{trainer.logger.version}.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "51e91fd3-9db3-46b9-b4ec-ae9c27e2c954",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:662: UserWarning: Your `test_dataloader` has `shuffle=True`, it is strongly recommended that you turn this off for val/test/predict dataloaders.\n",
      "  category=UserWarning,\n",
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:133: UserWarning: The dataloader, test_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b43f189c81104149be6c628a8c3259cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Testing'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'hr_test': 0.6969747543334961, 'ndcg_test': 0.43651801347732544}\n",
      "--------------------------------------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'ndcg_test': 0.43651801347732544, 'hr_test': 0.6969747543334961}]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.test(model, test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92de6c51-b0c1-4047-bafc-074a06d89cd8",
   "metadata": {},
   "source": [
    "## Playground zone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "273e460d-4f7f-42b8-8ce8-ee5fdbc306c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0]], dtype=torch.int32)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_test = torch.zeros((1,1), dtype=torch.int)\n",
    "emb_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "b27680cc-8f31-40c1-8af5-2030c0cdbb1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.1040,  0.7982, -0.4536,  0.3354,  0.7432,  0.4915,  1.7926,\n",
       "          -0.3678, -1.4826, -0.1286,  0.2442, -0.8088, -0.1810, -2.6418,\n",
       "           0.4637, -0.8777,  2.1836, -1.3642, -1.2097,  2.2583, -1.5101,\n",
       "          -0.7968, -0.3392, -0.3674, -1.6548, -0.1962, -0.8084, -1.4440,\n",
       "           1.4914,  1.4214, -0.9880, -1.0189,  1.4612, -1.5223,  1.2919,\n",
       "          -0.0792,  0.4893,  1.8704,  0.7675, -0.6249,  0.4509, -0.3465,\n",
       "          -0.5978, -1.1212,  0.7557, -1.6298,  1.0226, -0.5319, -1.1509,\n",
       "          -2.3767]]], grad_fn=<EmbeddingBackward>)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.pe(emb_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "a1b76636-2d88-4011-a976-8f0713d6dc74",
   "metadata": {},
   "outputs": [],
   "source": [
    "dcg = lambda x: 1/np.log2(x+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "c3e8e5ae-a123-4856-9dc4-0e53aae17e83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.2890648263178879,\n",
       " 0.15019048322368797,\n",
       " 0.13070098600339125,\n",
       " 0.12145326590959868)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dcg(10), dcg(100), dcg(200), dcg(300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "c0733636-6098-46c5-ae5a-5d97f51c85c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model = SASRecEncoder.load_from_checkpoint(\"sasrec.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "ba7db160-331a-4c28-b8a1-ba114fe922ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:662: UserWarning: Your `val_dataloader` has `shuffle=True`, it is strongly recommended that you turn this off for val/test/predict dataloaders.\n",
      "  category=UserWarning,\n",
      "/home/testuser/.local/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:133: UserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "DATALOADER:0 VALIDATE RESULTS\n",
      "{'hr_val': 0.827293872833252, 'ndcg_val': 0.5855227708816528}\n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'ndcg_val': 0.5855227708816528, 'hr_val': 0.827293872833252}]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.validate(best_model, dataloaders=val_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b587f98-b27d-4a11-8441-c6a25f5a1b44",
   "metadata": {},
   "source": [
    "### Export to ONNX "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "id": "07efd5e4-6918-418b-9120-6a21e281591b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=1,\n",
    "                          shuffle=False, collate_fn = lambda x: DH.tokenize_batch(x, max_len=args['maxlen']))\n",
    "u, seq, pos, neg = next(iter(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "id": "a562027b-a3ea-423c-89b9-5fd7afbc8194",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.1139, -1.0945, -0.3163,  ...,  0.3335, -0.7704,  0.3022],\n",
       "         [-0.7676, -1.0056, -0.2909,  ..., -0.0067, -1.3996,  0.6735],\n",
       "         [-0.7053, -1.0908, -0.8257,  ...,  0.0653, -0.6168,  0.6413],\n",
       "         ...,\n",
       "         [-0.8415,  2.4567,  1.3969,  ..., -0.2030, -1.7757, -0.4156],\n",
       "         [-1.2199,  1.9336,  0.7832,  ...,  0.3152, -1.1556, -0.4826],\n",
       "         [-1.2295,  2.9491,  1.5308,  ..., -0.2035, -0.4542,  0.2170]]],\n",
       "       grad_fn=<NativeLayerNormBackward>)"
      ]
     },
     "execution_count": 416,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model.forward(seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "id": "381d6826-0882-40df-8550-f1399f1398c4",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/onnx/symbolic_opset9.py:1929: UserWarning: ONNX export unsqueeze with negative axis -1 might cause the onnx model to be incorrect. Negative axis is not supported in ONNX. Axis is converted to 2 based on input shape at export time. Passing an tensor of different rank in execution will be incorrect.\n",
      "  \"Passing an tensor of different rank in execution will be incorrect.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "graph(%input.1 : Int(1, 200, strides=[200, 1], requires_grad=0, device=cpu),\n",
      "      %ie.weight : Float(3417, 50, strides=[50, 1], requires_grad=1, device=cpu),\n",
      "      %pe.pe.weight : Float(200, 50, strides=[50, 1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.norm_1.weight : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.norm_1.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.norm_2.weight : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.norm_2.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.attn.in_proj_weight : Float(150, 50, strides=[50, 1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.attn.in_proj_bias : Float(150, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.attn.out_proj.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.ff.conv1.weight : Float(50, 50, 1, strides=[50, 1, 1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.ff.conv1.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.ff.conv2.weight : Float(50, 50, 1, strides=[50, 1, 1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.0.ff.conv2.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.norm_1.weight : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.norm_1.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.norm_2.weight : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.norm_2.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.attn.in_proj_weight : Float(150, 50, strides=[50, 1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.attn.in_proj_bias : Float(150, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.attn.out_proj.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.ff.conv1.weight : Float(50, 50, 1, strides=[50, 1, 1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.ff.conv1.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.ff.conv2.weight : Float(50, 50, 1, strides=[50, 1, 1], requires_grad=1, device=cpu),\n",
      "      %enc_stack.1.ff.conv2.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %final_norm.weight : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %final_norm.bias : Float(50, strides=[1], requires_grad=1, device=cpu),\n",
      "      %289 : Long(1, strides=[1], requires_grad=0, device=cpu),\n",
      "      %290 : Long(1, strides=[1], requires_grad=0, device=cpu),\n",
      "      %291 : Long(1, strides=[1], requires_grad=0, device=cpu),\n",
      "      %292 : Long(1, strides=[1], requires_grad=0, device=cpu),\n",
      "      %293 : Long(3, strides=[1], requires_grad=0, device=cpu),\n",
      "      %294 : Bool(1, 200, 200, strides=[40000, 200, 1], requires_grad=0, device=cpu),\n",
      "      %295 : Float(50, 50, strides=[1, 50], requires_grad=0, device=cpu),\n",
      "      %296 : Long(1, strides=[1], requires_grad=0, device=cpu),\n",
      "      %297 : Long(1, strides=[1], requires_grad=0, device=cpu),\n",
      "      %298 : Long(3, strides=[1], requires_grad=0, device=cpu),\n",
      "      %299 : Bool(1, 200, 200, strides=[40000, 200, 1], requires_grad=0, device=cpu),\n",
      "      %300 : Float(50, 50, strides=[1, 50], requires_grad=0, device=cpu)):\n",
      "  %29 : Long(1, 200, strides=[200, 1], requires_grad=0, device=cpu) = onnx::Cast[to=7](%input.1)\n",
      "  %30 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={0}]()\n",
      "  %31 : Bool(1, 200, strides=[200, 1], device=cpu) = onnx::Equal(%29, %30)\n",
      "  %32 : Bool(1, 200, strides=[200, 1], requires_grad=0, device=cpu) = onnx::Not(%31) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:153:0\n",
      "  %33 : Bool(1, 200, strides=[200, 1], requires_grad=0, device=cpu) = onnx::Cast[to=9](%32) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:153:0\n",
      "  %34 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Gather(%ie.weight, %input.1) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2043:0\n",
      "  %35 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={7.07107}]()\n",
      "  %36 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Mul(%34, %35)\n",
      "  %37 : Long(2, strides=[1], device=cpu) = onnx::Shape(%input.1)\n",
      "  %38 : Long(device=cpu) = onnx::Constant[value={1}]()\n",
      "  %39 : Long(device=cpu) = onnx::Gather[axis=0](%37, %38) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:104:0\n",
      "  %43 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%39)\n",
      "  %45 : Long(1, strides=[1], device=cpu) = onnx::Sub(%43, %290)\n",
      "  %46 : Long(1, strides=[1], device=cpu) = onnx::Div(%45, %289)\n",
      "  %47 : Int(200, device=cpu) = onnx::ConstantOfShape[value={1}](%46)\n",
      "  %48 : Long(1, 200, device=cpu) = onnx::NonZero(%47)\n",
      "  %49 : Long(200, 1, strides=[1, 1], device=cpu) = onnx::Transpose[perm=[1, 0]](%48)\n",
      "  %50 : Long(200, strides=[1], device=cpu) = onnx::Squeeze[axes=[1]](%49)\n",
      "  %51 : Long(200, strides=[1], device=cpu) = onnx::Mul(%50, %289)\n",
      "  %52 : Long(200, strides=[1], device=cpu) = onnx::Add(%51, %290)\n",
      "  %53 : Int(200, strides=[1], requires_grad=0, device=cpu) = onnx::Cast[to=6](%52) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:104:0\n",
      "  %54 : Int(200, strides=[1], requires_grad=0, device=cpu) = onnx::Cast[to=6](%53) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:104:0\n",
      "  %55 : Float(200, 50, strides=[50, 1], requires_grad=1, device=cpu) = onnx::Gather(%pe.pe.weight, %54) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2043:0\n",
      "  %56 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Unsqueeze[axes=[0]](%55) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:106:0\n",
      "  %57 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Concat[axis=0](%56) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:106:0\n",
      "  %58 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%36, %57) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1168:0\n",
      "  %59 : Bool(1, 200, 1, strides=[200, 1, 1], requires_grad=0, device=cpu) = onnx::Unsqueeze[axes=[2]](%33) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:158:0\n",
      "  %60 : Float(1, 200, 1, strides=[200, 1, 1], requires_grad=0, device=cpu) = onnx::Cast[to=1](%59)\n",
      "  %61 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Mul(%58, %60) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:158:0\n",
      "  %62 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%61)\n",
      "  %63 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Sub(%61, %62)\n",
      "  %64 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={2}]()\n",
      "  %65 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Pow(%63, %64)\n",
      "  %66 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%65)\n",
      "  %67 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={1e-08}]()\n",
      "  %68 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Add(%66, %67)\n",
      "  %69 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Sqrt(%68)\n",
      "  %70 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Div(%63, %69)\n",
      "  %71 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Mul(%70, %enc_stack.0.norm_1.weight)\n",
      "  %72 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%71, %enc_stack.0.norm_1.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2346:0\n",
      "  %73 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%72) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1017:0\n",
      "  %74 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%61) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1017:0\n",
      "  %75 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%61) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1017:0\n",
      "  %76 : Long(3, strides=[1], device=cpu) = onnx::Shape(%73)\n",
      "  %77 : Long(device=cpu) = onnx::Constant[value={0}]()\n",
      "  %78 : Long(device=cpu) = onnx::Gather[axis=0](%76, %77) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4948:0\n",
      "  %79 : Long(3, strides=[1], device=cpu) = onnx::Shape(%73)\n",
      "  %80 : Long(device=cpu) = onnx::Constant[value={1}]()\n",
      "  %81 : Long(device=cpu) = onnx::Gather[axis=0](%79, %80) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4948:0\n",
      "  %82 : Long(3, strides=[1], device=cpu) = onnx::Shape(%73)\n",
      "  %83 : Long(device=cpu) = onnx::Constant[value={2}]()\n",
      "  %84 : Long(device=cpu) = onnx::Gather[axis=0](%82, %83) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4948:0\n",
      "  %85 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %86 : Long(device=cpu) = onnx::Div(%84, %85)\n",
      "  %87 : Long(device=cpu) = onnx::Cast[to=7](%86)\n",
      "  %88 : Long(requires_grad=0, device=cpu) = onnx::Cast[to=7](%87) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4954:0\n",
      "  %89 : Float(50, 50, strides=[50, 1], requires_grad=1, device=cpu), %90 : Float(50, 50, strides=[50, 1], requires_grad=1, device=cpu), %91 : Float(50, 50, strides=[50, 1], requires_grad=1, device=cpu) = onnx::Split[axis=0, split=[50, 50, 50]](%enc_stack.0.attn.in_proj_weight)\n",
      "  %92 : Float(50, strides=[1], requires_grad=1, device=cpu), %93 : Float(50, strides=[1], requires_grad=1, device=cpu), %94 : Float(50, strides=[1], requires_grad=1, device=cpu) = onnx::Split[axis=0, split=[50, 50, 50]](%enc_stack.0.attn.in_proj_bias)\n",
      "  %95 : Float(50, 50, strides=[50, 1], device=cpu) = onnx::Transpose[perm=[1, 0]](%89)\n",
      "  %96 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%73, %95)\n",
      "  %97 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%92, %96) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5023:0\n",
      "  %98 : Float(50, 50, strides=[50, 1], device=cpu) = onnx::Transpose[perm=[1, 0]](%90)\n",
      "  %99 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%74, %98)\n",
      "  %100 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%93, %99) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5025:0\n",
      "  %101 : Float(50, 50, strides=[50, 1], device=cpu) = onnx::Transpose[perm=[1, 0]](%91)\n",
      "  %102 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%75, %101)\n",
      "  %103 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%94, %102) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5034:0\n",
      "  %105 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %106 : Long(requires_grad=0, device=cpu) = onnx::Mul(%81, %105)\n",
      "  %107 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%78)\n",
      "  %108 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%106)\n",
      "  %109 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%88)\n",
      "  %110 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%107, %108, %109)\n",
      "  %111 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%97, %110) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5023:0\n",
      "  %112 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%111) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5023:0\n",
      "  %113 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %114 : Long(requires_grad=0, device=cpu) = onnx::Mul(%81, %113)\n",
      "  %117 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%114)\n",
      "  %118 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%88)\n",
      "  %119 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%291, %117, %118)\n",
      "  %120 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%100, %119) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5025:0\n",
      "  %121 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %122 : Long(requires_grad=0, device=cpu) = onnx::Mul(%81, %121)\n",
      "  %125 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%122)\n",
      "  %126 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%88)\n",
      "  %127 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%292, %125, %126)\n",
      "  %128 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%103, %127) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5034:0\n",
      "  %129 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%128) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5034:0\n",
      "  %131 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=0, device=cpu) = onnx::ConstantOfShape[value={0}](%293) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5071:0\n",
      "  %133 : Float(device=cpu) = onnx::Constant[value={-inf}]()\n",
      "  %134 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=0, device=cpu) = onnx::Where(%294, %133, %131)\n",
      "  %135 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={7.07107}]()\n",
      "  %136 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Div(%112, %135)\n",
      "  %137 : Float(1, 50, 200, strides=[50, 1, 50], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 2, 0]](%120) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4825:0\n",
      "  %138 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=1, device=cpu) = onnx::MatMul(%136, %137) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4825:0\n",
      "  %139 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=1, device=cpu) = onnx::Add(%138, %134) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4827:0\n",
      "  %140 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=1, device=cpu) = onnx::Softmax[axis=2](%139) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1679:0\n",
      "  %141 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::MatMul(%140, %129) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4832:0\n",
      "  %142 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%141) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5083:0\n",
      "  %143 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%78)\n",
      "  %144 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%81)\n",
      "  %145 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%84)\n",
      "  %146 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%143, %144, %145)\n",
      "  %147 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%142, %146) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5083:0\n",
      "  %149 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%147, %295)\n",
      "  %150 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%enc_stack.0.attn.out_proj.bias, %149) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1847:0\n",
      "  %151 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%150) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1040:0\n",
      "  %152 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%61, %151) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:87:0\n",
      "  %153 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%152)\n",
      "  %154 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Sub(%152, %153)\n",
      "  %155 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={2}]()\n",
      "  %156 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Pow(%154, %155)\n",
      "  %157 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%156)\n",
      "  %158 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={1e-08}]()\n",
      "  %159 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Add(%157, %158)\n",
      "  %160 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Sqrt(%159)\n",
      "  %161 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Div(%154, %160)\n",
      "  %162 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Mul(%161, %enc_stack.0.norm_2.weight)\n",
      "  %163 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%162, %enc_stack.0.norm_2.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2346:0\n",
      "  %164 : Float(1, 50, 200, strides=[10000, 1, 50], requires_grad=1, device=cpu) = onnx::Transpose[perm=[0, 2, 1]](%163) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:34:0\n",
      "  %165 : Float(1, 50, 200, strides=[10000, 200, 1], requires_grad=1, device=cpu) = onnx::Conv[dilations=[1], group=1, kernel_shape=[1], pads=[0, 0], strides=[1]](%164, %enc_stack.0.ff.conv1.weight, %enc_stack.0.ff.conv1.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py:295:0\n",
      "  %166 : Float(1, 50, 200, strides=[10000, 200, 1], requires_grad=1, device=cpu) = onnx::Relu(%165) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1298:0\n",
      "  %167 : Float(1, 50, 200, strides=[10000, 200, 1], requires_grad=1, device=cpu) = onnx::Conv[dilations=[1], group=1, kernel_shape=[1], pads=[0, 0], strides=[1]](%166, %enc_stack.0.ff.conv2.weight, %enc_stack.0.ff.conv2.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py:295:0\n",
      "  %168 : Float(1, 200, 50, strides=[10000, 1, 200], requires_grad=1, device=cpu) = onnx::Transpose[perm=[0, 2, 1]](%167) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1168:0\n",
      "  %169 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%152, %168) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:91:0\n",
      "  %170 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%169)\n",
      "  %171 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Sub(%169, %170)\n",
      "  %172 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={2}]()\n",
      "  %173 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Pow(%171, %172)\n",
      "  %174 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%173)\n",
      "  %175 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={1e-08}]()\n",
      "  %176 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Add(%174, %175)\n",
      "  %177 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Sqrt(%176)\n",
      "  %178 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Div(%171, %177)\n",
      "  %179 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Mul(%178, %enc_stack.1.norm_1.weight)\n",
      "  %180 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%179, %enc_stack.1.norm_1.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2346:0\n",
      "  %181 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%180) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1017:0\n",
      "  %182 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%169) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1017:0\n",
      "  %183 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%169) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1017:0\n",
      "  %184 : Long(3, strides=[1], device=cpu) = onnx::Shape(%181)\n",
      "  %185 : Long(device=cpu) = onnx::Constant[value={0}]()\n",
      "  %186 : Long(device=cpu) = onnx::Gather[axis=0](%184, %185) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4948:0\n",
      "  %187 : Long(3, strides=[1], device=cpu) = onnx::Shape(%181)\n",
      "  %188 : Long(device=cpu) = onnx::Constant[value={1}]()\n",
      "  %189 : Long(device=cpu) = onnx::Gather[axis=0](%187, %188) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4948:0\n",
      "  %190 : Long(3, strides=[1], device=cpu) = onnx::Shape(%181)\n",
      "  %191 : Long(device=cpu) = onnx::Constant[value={2}]()\n",
      "  %192 : Long(device=cpu) = onnx::Gather[axis=0](%190, %191) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4948:0\n",
      "  %193 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %194 : Long(device=cpu) = onnx::Div(%192, %193)\n",
      "  %195 : Long(device=cpu) = onnx::Cast[to=7](%194)\n",
      "  %196 : Long(requires_grad=0, device=cpu) = onnx::Cast[to=7](%195) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4954:0\n",
      "  %197 : Float(50, 50, strides=[50, 1], requires_grad=1, device=cpu), %198 : Float(50, 50, strides=[50, 1], requires_grad=1, device=cpu), %199 : Float(50, 50, strides=[50, 1], requires_grad=1, device=cpu) = onnx::Split[axis=0, split=[50, 50, 50]](%enc_stack.1.attn.in_proj_weight)\n",
      "  %200 : Float(50, strides=[1], requires_grad=1, device=cpu), %201 : Float(50, strides=[1], requires_grad=1, device=cpu), %202 : Float(50, strides=[1], requires_grad=1, device=cpu) = onnx::Split[axis=0, split=[50, 50, 50]](%enc_stack.1.attn.in_proj_bias)\n",
      "  %203 : Float(50, 50, strides=[50, 1], device=cpu) = onnx::Transpose[perm=[1, 0]](%197)\n",
      "  %204 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%181, %203)\n",
      "  %205 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%200, %204) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5023:0\n",
      "  %206 : Float(50, 50, strides=[50, 1], device=cpu) = onnx::Transpose[perm=[1, 0]](%198)\n",
      "  %207 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%182, %206)\n",
      "  %208 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%201, %207) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5025:0\n",
      "  %209 : Float(50, 50, strides=[50, 1], device=cpu) = onnx::Transpose[perm=[1, 0]](%199)\n",
      "  %210 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%183, %209)\n",
      "  %211 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%202, %210) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5034:0\n",
      "  %213 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %214 : Long(requires_grad=0, device=cpu) = onnx::Mul(%189, %213)\n",
      "  %215 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%186)\n",
      "  %216 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%214)\n",
      "  %217 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%196)\n",
      "  %218 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%215, %216, %217)\n",
      "  %219 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%205, %218) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5023:0\n",
      "  %220 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%219) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5023:0\n",
      "  %221 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %222 : Long(requires_grad=0, device=cpu) = onnx::Mul(%189, %221)\n",
      "  %225 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%222)\n",
      "  %226 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%196)\n",
      "  %227 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%296, %225, %226)\n",
      "  %228 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%208, %227) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5025:0\n",
      "  %229 : Long(requires_grad=0, device=cpu) = onnx::Constant[value={1}]()\n",
      "  %230 : Long(requires_grad=0, device=cpu) = onnx::Mul(%189, %229)\n",
      "  %233 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%230)\n",
      "  %234 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%196)\n",
      "  %235 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%297, %233, %234)\n",
      "  %236 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%211, %235) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5034:0\n",
      "  %237 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%236) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5034:0\n",
      "  %239 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=0, device=cpu) = onnx::ConstantOfShape[value={0}](%298) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5071:0\n",
      "  %241 : Float(device=cpu) = onnx::Constant[value={-inf}]()\n",
      "  %242 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=0, device=cpu) = onnx::Where(%299, %241, %239)\n",
      "  %243 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={7.07107}]()\n",
      "  %244 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Div(%220, %243)\n",
      "  %245 : Float(1, 50, 200, strides=[50, 1, 50], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 2, 0]](%228) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4825:0\n",
      "  %246 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=1, device=cpu) = onnx::MatMul(%244, %245) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4825:0\n",
      "  %247 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=1, device=cpu) = onnx::Add(%246, %242) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4827:0\n",
      "  %248 : Float(1, 200, 200, strides=[40000, 200, 1], requires_grad=1, device=cpu) = onnx::Softmax[axis=2](%247) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1679:0\n",
      "  %249 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::MatMul(%248, %237) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:4832:0\n",
      "  %250 : Float(200, 1, 50, strides=[50, 10000, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%249) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5083:0\n",
      "  %251 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%186)\n",
      "  %252 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%189)\n",
      "  %253 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze[axes=[0]](%192)\n",
      "  %254 : Long(3, strides=[1], device=cpu) = onnx::Concat[axis=0](%251, %252, %253)\n",
      "  %255 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Reshape(%250, %254) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:5083:0\n",
      "  %257 : Float(200, 1, 50, strides=[50, 50, 1], device=cpu) = onnx::MatMul(%255, %300)\n",
      "  %258 : Float(200, 1, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%enc_stack.1.attn.out_proj.bias, %257) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1847:0\n",
      "  %259 : Float(1, 200, 50, strides=[50, 50, 1], requires_grad=1, device=cpu) = onnx::Transpose[perm=[1, 0, 2]](%258) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/activation.py:1040:0\n",
      "  %260 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%169, %259) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:87:0\n",
      "  %261 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%260)\n",
      "  %262 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Sub(%260, %261)\n",
      "  %263 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={2}]()\n",
      "  %264 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Pow(%262, %263)\n",
      "  %265 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%264)\n",
      "  %266 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={1e-08}]()\n",
      "  %267 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Add(%265, %266)\n",
      "  %268 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Sqrt(%267)\n",
      "  %269 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Div(%262, %268)\n",
      "  %270 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Mul(%269, %enc_stack.1.norm_2.weight)\n",
      "  %271 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%270, %enc_stack.1.norm_2.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2346:0\n",
      "  %272 : Float(1, 50, 200, strides=[10000, 1, 50], requires_grad=1, device=cpu) = onnx::Transpose[perm=[0, 2, 1]](%271) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:34:0\n",
      "  %273 : Float(1, 50, 200, strides=[10000, 200, 1], requires_grad=1, device=cpu) = onnx::Conv[dilations=[1], group=1, kernel_shape=[1], pads=[0, 0], strides=[1]](%272, %enc_stack.1.ff.conv1.weight, %enc_stack.1.ff.conv1.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py:295:0\n",
      "  %274 : Float(1, 50, 200, strides=[10000, 200, 1], requires_grad=1, device=cpu) = onnx::Relu(%273) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1298:0\n",
      "  %275 : Float(1, 50, 200, strides=[10000, 200, 1], requires_grad=1, device=cpu) = onnx::Conv[dilations=[1], group=1, kernel_shape=[1], pads=[0, 0], strides=[1]](%274, %enc_stack.1.ff.conv2.weight, %enc_stack.1.ff.conv2.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py:295:0\n",
      "  %276 : Float(1, 200, 50, strides=[10000, 1, 200], requires_grad=1, device=cpu) = onnx::Transpose[perm=[0, 2, 1]](%275) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:1168:0\n",
      "  %277 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%260, %276) # /home/testuser/backup_folder/SASRec.pytorch/SASRecModel.py:91:0\n",
      "  %278 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%277)\n",
      "  %279 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Sub(%277, %278)\n",
      "  %280 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={2}]()\n",
      "  %281 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Pow(%279, %280)\n",
      "  %282 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::ReduceMean[axes=[-1]](%281)\n",
      "  %283 : Float(requires_grad=0, device=cpu) = onnx::Constant[value={1e-08}]()\n",
      "  %284 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Add(%282, %283)\n",
      "  %285 : Float(1, 200, 1, strides=[200, 1, 1], device=cpu) = onnx::Sqrt(%284)\n",
      "  %286 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Div(%279, %285)\n",
      "  %287 : Float(1, 200, 50, strides=[10000, 50, 1], device=cpu) = onnx::Mul(%286, %final_norm.weight)\n",
      "  %288 : Float(1, 200, 50, strides=[10000, 50, 1], requires_grad=1, device=cpu) = onnx::Add(%287, %final_norm.bias) # /opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:2346:0\n",
      "  return (%288)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "torch.onnx.export(best_model, seq, \"sasrec.onnx\", verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "ac5e5845-f0c5-4325-ac87-9a3e93288537",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_x = torch.randn(BATCH_SIZE, args['maxlen'], args['d_model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "83a1fa4a-b4fb-4490-b255-0121d59805f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "pe_test = Embedding(args['maxlen'], embedding_dim=args['d_model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "e3656158-7b7c-4ab8-bebb-d555dfd2fbc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "pe_for_one_sequence = pe_test(torch.arange(0,input_x.shape[1], dtype=torch.int)) # get a single positional embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "015db547-da7e-42e0-9946-b41312d52044",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(22.0447)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.linalg.matrix_norm(((next(best_model.pe.parameters())).data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "d16360e9-e560-4afe-a24d-b7835c7410c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "v.data[0,0] = 0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "f791949b-148f-435e-8fd8-39e46ae9abc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(v.data<-1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "a1949d5c-2b0e-42ba-92ea-fcdc46a73157",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(22.0447)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.linalg.matrix_norm(v.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "6b9b8358-d4e7-45e0-90ef-1653e7ca2952",
   "metadata": {},
   "outputs": [],
   "source": [
    "_u = 45\n",
    "all_items_set = set(range(itemnum))\n",
    "user_items_set = set(user_train[_u])  - set(user_valid[_u]) # union of what user has selected up to valid sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbb4060b-7f91-4f37-87ac-5b7bc15f9ee1",
   "metadata": {},
   "source": [
    "### Reading raw data from movie lens 1M\n",
    "https://grouplens.org/datasets/movielens/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "id": "8ca5eb4d-e210-43b4-a9e6-40a99ff76e94",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "from datetime import datetime\n",
    "display_settings = {\n",
    "    'display.max_columns': 100,\n",
    "    'display.max_rows':100,\n",
    "    'display.expand_frame_repr': True,  # Wrap to multiple pages\n",
    "    'display.precision': 2,\n",
    "    'display.show_dimensions': True,\n",
    "    'display.float_format': '{:,.2f}'.format,\n",
    "    'io.hdf.default_format':'table' # appendable hp5 table\n",
    "}\n",
    "\n",
    "for op, value in display_settings.items():\n",
    "    pd.set_option(\"{}\".format(op), value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "id": "eb78ce88-3c6b-46b8-b1e5-287ca9093da1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose wich dataset to read - 20M or 1M\n",
    "ratings = pd.read_csv('data/ratings-20m.csv', dtype={'userId':np.int32, 'movieId':np.int32, 'rating':np.float32})\n",
    "# ratings = pd.read_csv('data/ratings-1m.csv', delimiter=\"::\", engine='python',\n",
    "#                       names=['userId','movieId','rating','timestamp'], \n",
    "#                       dtype={'userId':np.int32, 'movieId':np.int32, 'rating':np.float32})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "id": "6abc7b24-eafd-41ec-b5bc-eef938cd61f7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ratings['timestamp'] = ratings['timestamp'].apply(lambda x: datetime.fromtimestamp(x))\n",
    "ratings = ratings.sort_values(by=['userId','timestamp'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "id": "ab2f034c-8419-4bff-b110-aeb871893a80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>924</td>\n",
       "      <td>3.50</td>\n",
       "      <td>2004-09-10 03:06:38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>919</td>\n",
       "      <td>3.50</td>\n",
       "      <td>2004-09-10 03:07:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>1</td>\n",
       "      <td>2683</td>\n",
       "      <td>3.50</td>\n",
       "      <td>2004-09-10 03:07:30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>1</td>\n",
       "      <td>1584</td>\n",
       "      <td>3.50</td>\n",
       "      <td>2004-09-10 03:07:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>1079</td>\n",
       "      <td>4.00</td>\n",
       "      <td>2004-09-10 03:07:45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    userId  movieId  rating           timestamp\n",
       "20       1      924    3.50 2004-09-10 03:06:38\n",
       "19       1      919    3.50 2004-09-10 03:07:01\n",
       "86       1     2683    3.50 2004-09-10 03:07:30\n",
       "61       1     1584    3.50 2004-09-10 03:07:36\n",
       "23       1     1079    4.00 2004-09-10 03:07:45\n",
       "\n",
       "[5 rows x 4 columns]"
      ]
     },
     "execution_count": 403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "id": "7a461339-dcba-4969-b40f-27b4e67da263",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter out movies with less than 5 ragings\n",
    "movie_cnt = ratings.movieId.value_counts()\n",
    "ratings = ratings[ratings.movieId.isin(movie_cnt[movie_cnt>=5].index)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "id": "78119240-0516-47d5-ba72-475c5d7d8320",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18345"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(ratings.movieId.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "id": "fdab4aa0-51e6-4880-96ae-5f2a67609d75",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a mapping so movie ids are sequential\n",
    "movieId2seq = dict() # original movieId -> sequential id\n",
    "seq2movieId = np.array(list(set(ratings.movieId.values))) # sequential id to original movie mapping\n",
    "for i,movie_id in enumerate(seq2movieId):\n",
    "    movieId2seq[movie_id] = i+1 \n",
    "# change ids to sequential\n",
    "ratings['movieId'] = ratings['movieId'].map(movieId2seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "id": "e75ac6f8-187b-4ccf-a57a-f786c5416bd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings[['userId','movieId']].to_csv(\"data/ml-20m.txt\",sep=\" \", header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "id": "72cb82d6-5727-472f-a47f-38270fa484e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r-- 1 testuser testuser 9,1M янв 28 10:02 data/ml-1m_manual.txt\n",
      "-rw-r--r-- 1 testuser testuser 8,7M дек 15 17:12 data/ml-1m.txt\n",
      "-rw-r--r-- 1 testuser testuser 209M янв 28 10:10 data/ml-20m.txt\n"
     ]
    }
   ],
   "source": [
    "!ls -lh data/ml*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "id": "e3140520-fe46-4697-a77a-1a2b11bff608",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 924\n",
      "1 919\n",
      "1 2683\n",
      "1 1584\n",
      "1 1079\n",
      "1 653\n",
      "1 2959\n",
      "1 337\n",
      "1 1304\n",
      "1 3996\n"
     ]
    }
   ],
   "source": [
    "!head data/ml-20m.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94d67039-0446-4e33-b1b1-ccec8698151a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}