step-by-step.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b197111f3dd1d21d",
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:23:38.643505Z",
     "start_time": "2024-02-09T04:23:10.263131Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import tiktoken\n",
    "import torch\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "'\\nThis notebook is to illustrate the Transformer architecture.\\nSteps are followed by my article post: \\nhttps://waylandzhang.github.io/en/let-s-code-llm.html\\n'"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "This notebook is to illustrate the Transformer architecture.\n",
    "Steps are followed by my article post: \n",
    "https://waylandzhang.github.io/en/let-s-code-llm.html\n",
    "\"\"\""
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:23:49.123240Z",
     "start_time": "2024-02-09T04:23:49.078938Z"
    }
   },
   "id": "220167a759cf381c",
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "<torch._C.Generator at 0x7f8aadb9d710>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Hyperparameters\n",
    "batch_size = 4  # How many batches per training step\n",
    "context_length = 16  # Length of the token chunk each batch\n",
    "d_model = 64  # The vector size of the token embeddings\n",
    "num_layers = 8  # Number of transformer blocks\n",
    "num_heads = 4  # Number of heads in Multi-head attention # 我们的代码中通过 d_model / num_heads = 来获取 head_size\n",
    "learning_rate = 1e-3  # 0.001\n",
    "dropout = 0.1 # Dropout rate\n",
    "max_iters = 500  # Total of training iterations\n",
    "eval_interval = 50  # How often to evaluate the model \n",
    "eval_iters = 20  # How many iterations to average the loss over when evaluating the model\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'  # Instead of using the cpu, we'll use the GPU if it's available.\n",
    "\n",
    "TORCH_SEED = 1337\n",
    "torch.manual_seed(TORCH_SEED)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:23:51.966821Z",
     "start_time": "2024-02-09T04:23:51.948834Z"
    }
   },
   "id": "adec81e8b065abf4",
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# download a sample txt file from https://huggingface.co/datasets/goendalf666/sales-textbook_for_convincing_and_selling/raw/main/sales_textbook.txt\n",
    "if not os.path.exists('data/sales_textbook.txt'):\n",
    "    url = 'https://huggingface.co/datasets/goendalf666/sales-textbook_for_convincing_and_selling/raw/main/sales_textbook.txt'\n",
    "    with open('sales_textbook.txt', 'w') as f:\n",
    "        f.write(requests.get(url).text)\n",
    "\n",
    "with open('data/sales_textbook.txt', 'r', encoding='utf-8') as f:\n",
    "    text = f.read()\n",
    "    "
   ],
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:23:53.539657Z",
     "start_time": "2024-02-09T04:23:53.516333Z"
    }
   },
   "id": "initial_id",
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenized text size: 77919\n",
      "The maximum value in the tokenized text is: 100069\n"
     ]
    }
   ],
   "source": [
    "# Using TikToken to tokenize the source text\n",
    "encoding = tiktoken.get_encoding(\"cl100k_base\")\n",
    "tokenized_text = encoding.encode(text)\n",
    "tokenized_text = torch.tensor(tokenized_text, dtype=torch.long, device=device) # Convert tokens into a tensor\n",
    "max_token_value = tokenized_text.max().item() # the maximum index value in our vocabulary\n",
    "\n",
    "print(f\"Tokenized text size: {len(tokenized_text)}\")\n",
    "print(f\"The maximum value in the tokenized text is: {max_token_value}\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:24:44.270382Z",
     "start_time": "2024-02-09T04:24:44.140668Z"
    }
   },
   "id": "d8eb984bfcb7a06b",
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26072, 220, 16, 25, 17283, 23097, 403, 323, 17013, 1711]\n",
      "Chapter 1: Building Rapport and Capturing\n",
      ".\n",
      "By mastering the art of identifying underlying motivations and desires, we equip ourselves with\n"
     ]
    }
   ],
   "source": [
    "# Illustration purpose\n",
    "print(encoding.encode('Chapter 1: Building Rapport and Capturing'))\n",
    "print(encoding.decode([26072, 220, 16, 25, 17283, 23097, 403, 323, 17013, 1711])) # \"Rapport\" is tokenized as two tokens: \"Rap\"[23097] and \"port\"[403]\n",
    "print(encoding.decode([627, 1383, 88861, 279,1989, 315, 25607, 16940, 65931, 323, 32097, 11, 584, 26458, 13520, 449]))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:24:50.051282Z",
     "start_time": "2024-02-09T04:24:50.032966Z"
    }
   },
   "id": "a1453076ae7f2e3",
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Split train and validation\n",
    "split_idx = int(len(tokenized_text) * 0.9)\n",
    "train_data = tokenized_text[:split_idx]\n",
    "val_data = tokenized_text[split_idx:]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:24:52.889469Z",
     "start_time": "2024-02-09T04:24:52.878327Z"
    }
   },
   "id": "1bd965d977d210d8",
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 16]) torch.Size([4, 16])\n"
     ]
    }
   ],
   "source": [
    "# Prepare data for training batch\n",
    "data = train_data\n",
    "idxs = torch.randint(low=0, high=len(data) - context_length, size=(batch_size,))\n",
    "x_batch = torch.stack([data[idx:idx + context_length] for idx in idxs])\n",
    "y_batch = torch.stack([data[idx + 1:idx + context_length + 1] for idx in idxs])\n",
    "print(x_batch.shape, x_batch.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:25:08.155569Z",
     "start_time": "2024-02-09T04:25:08.138702Z"
    }
   },
   "id": "dd4d699c1f25bbe6",
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our batches:\n",
      "       0      1     2      3     4      5      6      7      8      9    10     11     12     13     14     15\n",
      "0    279   6763  1920     13   578   5845    311  13750  19570    279  907    323   7720    315   1057   3956\n",
      "1   3495  14955    11    477  5064  23146    430   9788    279  66732  315    701  10209     13   3296  32644\n",
      "2  38769  10742    11  20958   264   6928  19451     11  11125  64784   11    323  56501  54111    439   6975\n",
      "3  43496    872  8830    719  1101   3727    279   6130   2733   6755  323  16365    627  29831  19682   5900\n"
     ]
    }
   ],
   "source": [
    "# Illustration purpose\n",
    "pd.set_option('display.expand_frame_repr', False)\n",
    "print(\"Our batches:\\n\", pd.DataFrame(x_batch.data.detach().cpu().numpy()))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:25:43.766159Z",
     "start_time": "2024-02-09T04:25:43.748077Z"
    }
   },
   "id": "e96ee8f724daac71",
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token Embedding Look-up table:  Embedding(100070, 64)\n"
     ]
    }
   ],
   "source": [
    "# Define Token Embedding look-up table\n",
    "token_embedding_lookup_table = nn.Embedding(max_token_value+1, d_model)\n",
    "print(\"Token Embedding Look-up table: \", token_embedding_lookup_table)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:25:48.830907Z",
     "start_time": "2024-02-09T04:25:48.749815Z"
    }
   },
   "id": "5ade5a79e8c689ef",
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "(torch.Size([4, 16, 64]), torch.Size([4, 16, 64]))"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get x and y embedding\n",
    "x_batch_embedding = token_embedding_lookup_table(x_batch.data) # [4, 16, 64] [batch_size, context_length, d_model]\n",
    "y_batch_embedding = token_embedding_lookup_table(y_batch.data)\n",
    "\n",
    "x_batch_embedding.shape, y_batch_embedding.shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:25:54.090804Z",
     "start_time": "2024-02-09T04:25:54.057773Z"
    }
   },
   "id": "86aa314dc5f6a76e",
   "execution_count": 13
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Position Encoding Look-up Table:  torch.Size([4, 16, 64])\n"
     ]
    },
    {
     "data": {
      "text/plain": "          0         1         2         3         4         5         6         7         8         9   ...        54        55        56        57        58        59        60        61        62        63\n0   0.000000  1.000000  0.000000  1.000000  0.000000  1.000000  0.000000  1.000000  0.000000  1.000000  ...  0.000000  1.000000  0.000000  1.000000  0.000000  1.000000  0.000000  1.000000  0.000000  1.000000\n1   0.841471  0.540302  0.681561  0.731761  0.533168  0.846009  0.409309  0.912396  0.310984  0.950415  ...  0.000422  1.000000  0.000316  1.000000  0.000237  1.000000  0.000178  1.000000  0.000133  1.000000\n2   0.909297 -0.416147  0.997480  0.070948  0.902131  0.431463  0.746904  0.664932  0.591127  0.806578  ...  0.000843  1.000000  0.000632  1.000000  0.000474  1.000000  0.000356  1.000000  0.000267  1.000000\n3   0.141120 -0.989992  0.778273 -0.627927  0.993253 -0.115966  0.953635  0.300967  0.812649  0.582754  ...  0.001265  0.999999  0.000949  1.000000  0.000711  1.000000  0.000533  1.000000  0.000400  1.000000\n4  -0.756802 -0.653644  0.141539 -0.989933  0.778472 -0.627680  0.993281 -0.115730  0.953581  0.301137  ...  0.001687  0.999999  0.001265  0.999999  0.000949  1.000000  0.000711  1.000000  0.000533  1.000000\n5  -0.958924  0.283662 -0.571127 -0.820862  0.323935 -0.946079  0.858896 -0.512150  0.999947 -0.010342  ...  0.002108  0.999998  0.001581  0.999999  0.001186  0.999999  0.000889  1.000000  0.000667  1.000000\n6  -0.279415  0.960170 -0.977396 -0.211416 -0.230368 -0.973104  0.574026 -0.818837  0.947148 -0.320796  ...  0.002530  0.999997  0.001897  0.999998  0.001423  0.999999  0.001067  0.999999  0.000800  1.000000\n7   0.656987  0.753902 -0.859313  0.511449 -0.713721 -0.700430  0.188581 -0.982058  0.800422 -0.599437  ...  0.002952  0.999996  0.002214  0.999998  0.001660  0.999999  0.001245  0.999999  0.000933  1.000000\n8   0.989358 -0.145500 -0.280228  0.959933 -0.977262 -0.212036 -0.229904 -0.973213  0.574318 -0.818632  ...  0.003374  0.999994  0.002530  0.999997  0.001897  0.999998  0.001423  0.999999  0.001067  0.999999\n9   0.412118 -0.911130  0.449194  0.893434 -0.939824  0.341660 -0.608108 -0.793854  0.291259 -0.956644  ...  0.003795  0.999993  0.002846  0.999996  0.002134  0.999998  0.001600  0.999999  0.001200  0.999999\n10 -0.544021 -0.839072  0.937633  0.347628 -0.612937  0.790132 -0.879767 -0.475405 -0.020684 -0.999786  ...  0.004217  0.999991  0.003162  0.999995  0.002371  0.999997  0.001778  0.999998  0.001334  0.999999\n11 -0.999990  0.004426  0.923052 -0.384674 -0.097276  0.995257 -0.997283 -0.073661 -0.330575 -0.943780  ...  0.004639  0.999989  0.003478  0.999994  0.002609  0.999997  0.001956  0.999998  0.001467  0.999999\n12 -0.536573  0.843854  0.413275 -0.910606  0.448343  0.893862 -0.940067  0.340989 -0.607683 -0.794179  ...  0.005060  0.999987  0.003795  0.999993  0.002846  0.999996  0.002134  0.999998  0.001600  0.999999\n13  0.420167  0.907447 -0.318216 -0.948018  0.855881  0.517173 -0.718144  0.695895 -0.824528 -0.565821  ...  0.005482  0.999985  0.004111  0.999992  0.003083  0.999995  0.002312  0.999997  0.001734  0.999998\n14  0.990607  0.136737 -0.878990 -0.476839  0.999823 -0.018796 -0.370395  0.928874 -0.959605 -0.281349  ...  0.005904  0.999983  0.004427  0.999990  0.003320  0.999995  0.002490  0.999997  0.001867  0.999998\n15  0.650288 -0.759688 -0.968206  0.250154  0.835838 -0.548975  0.042249  0.999107 -0.999519  0.031022  ...  0.006325  0.999980  0.004743  0.999989  0.003557  0.999994  0.002667  0.999996  0.002000  0.999998\n\n[16 rows x 64 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>54</th>\n      <th>55</th>\n      <th>56</th>\n      <th>57</th>\n      <th>58</th>\n      <th>59</th>\n      <th>60</th>\n      <th>61</th>\n      <th>62</th>\n      <th>63</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>...</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.841471</td>\n      <td>0.540302</td>\n      <td>0.681561</td>\n      <td>0.731761</td>\n      <td>0.533168</td>\n      <td>0.846009</td>\n      <td>0.409309</td>\n      <td>0.912396</td>\n      <td>0.310984</td>\n      <td>0.950415</td>\n      <td>...</td>\n      <td>0.000422</td>\n      <td>1.000000</td>\n      <td>0.000316</td>\n      <td>1.000000</td>\n      <td>0.000237</td>\n      <td>1.000000</td>\n      <td>0.000178</td>\n      <td>1.000000</td>\n      <td>0.000133</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.909297</td>\n      <td>-0.416147</td>\n      <td>0.997480</td>\n      <td>0.070948</td>\n      <td>0.902131</td>\n      <td>0.431463</td>\n      <td>0.746904</td>\n      <td>0.664932</td>\n      <td>0.591127</td>\n      <td>0.806578</td>\n      <td>...</td>\n      <td>0.000843</td>\n      <td>1.000000</td>\n      <td>0.000632</td>\n      <td>1.000000</td>\n      <td>0.000474</td>\n      <td>1.000000</td>\n      <td>0.000356</td>\n      <td>1.000000</td>\n      <td>0.000267</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.141120</td>\n      <td>-0.989992</td>\n      <td>0.778273</td>\n      <td>-0.627927</td>\n      <td>0.993253</td>\n      <td>-0.115966</td>\n      <td>0.953635</td>\n      <td>0.300967</td>\n      <td>0.812649</td>\n      <td>0.582754</td>\n      <td>...</td>\n      <td>0.001265</td>\n      <td>0.999999</td>\n      <td>0.000949</td>\n      <td>1.000000</td>\n      <td>0.000711</td>\n      <td>1.000000</td>\n      <td>0.000533</td>\n      <td>1.000000</td>\n      <td>0.000400</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-0.756802</td>\n      <td>-0.653644</td>\n      <td>0.141539</td>\n      <td>-0.989933</td>\n      <td>0.778472</td>\n      <td>-0.627680</td>\n      <td>0.993281</td>\n      <td>-0.115730</td>\n      <td>0.953581</td>\n      <td>0.301137</td>\n      <td>...</td>\n      <td>0.001687</td>\n      <td>0.999999</td>\n      <td>0.001265</td>\n      <td>0.999999</td>\n      <td>0.000949</td>\n      <td>1.000000</td>\n      <td>0.000711</td>\n      <td>1.000000</td>\n      <td>0.000533</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-0.958924</td>\n      <td>0.283662</td>\n      <td>-0.571127</td>\n      <td>-0.820862</td>\n      <td>0.323935</td>\n      <td>-0.946079</td>\n      <td>0.858896</td>\n      <td>-0.512150</td>\n      <td>0.999947</td>\n      <td>-0.010342</td>\n      <td>...</td>\n      <td>0.002108</td>\n      <td>0.999998</td>\n      <td>0.001581</td>\n      <td>0.999999</td>\n      <td>0.001186</td>\n      <td>0.999999</td>\n      <td>0.000889</td>\n      <td>1.000000</td>\n      <td>0.000667</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>-0.279415</td>\n      <td>0.960170</td>\n      <td>-0.977396</td>\n      <td>-0.211416</td>\n      <td>-0.230368</td>\n      <td>-0.973104</td>\n      <td>0.574026</td>\n      <td>-0.818837</td>\n      <td>0.947148</td>\n      <td>-0.320796</td>\n      <td>...</td>\n      <td>0.002530</td>\n      <td>0.999997</td>\n      <td>0.001897</td>\n      <td>0.999998</td>\n      <td>0.001423</td>\n      <td>0.999999</td>\n      <td>0.001067</td>\n      <td>0.999999</td>\n      <td>0.000800</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.656987</td>\n      <td>0.753902</td>\n      <td>-0.859313</td>\n      <td>0.511449</td>\n      <td>-0.713721</td>\n      <td>-0.700430</td>\n      <td>0.188581</td>\n      <td>-0.982058</td>\n      <td>0.800422</td>\n      <td>-0.599437</td>\n      <td>...</td>\n      <td>0.002952</td>\n      <td>0.999996</td>\n      <td>0.002214</td>\n      <td>0.999998</td>\n      <td>0.001660</td>\n      <td>0.999999</td>\n      <td>0.001245</td>\n      <td>0.999999</td>\n      <td>0.000933</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.989358</td>\n      <td>-0.145500</td>\n      <td>-0.280228</td>\n      <td>0.959933</td>\n      <td>-0.977262</td>\n      <td>-0.212036</td>\n      <td>-0.229904</td>\n      <td>-0.973213</td>\n      <td>0.574318</td>\n      <td>-0.818632</td>\n      <td>...</td>\n      <td>0.003374</td>\n      <td>0.999994</td>\n      <td>0.002530</td>\n      <td>0.999997</td>\n      <td>0.001897</td>\n      <td>0.999998</td>\n      <td>0.001423</td>\n      <td>0.999999</td>\n      <td>0.001067</td>\n      <td>0.999999</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>0.412118</td>\n      <td>-0.911130</td>\n      <td>0.449194</td>\n      <td>0.893434</td>\n      <td>-0.939824</td>\n      <td>0.341660</td>\n      <td>-0.608108</td>\n      <td>-0.793854</td>\n      <td>0.291259</td>\n      <td>-0.956644</td>\n      <td>...</td>\n      <td>0.003795</td>\n      <td>0.999993</td>\n      <td>0.002846</td>\n      <td>0.999996</td>\n      <td>0.002134</td>\n      <td>0.999998</td>\n      <td>0.001600</td>\n      <td>0.999999</td>\n      <td>0.001200</td>\n      <td>0.999999</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>-0.544021</td>\n      <td>-0.839072</td>\n      <td>0.937633</td>\n      <td>0.347628</td>\n      <td>-0.612937</td>\n      <td>0.790132</td>\n      <td>-0.879767</td>\n      <td>-0.475405</td>\n      <td>-0.020684</td>\n      <td>-0.999786</td>\n      <td>...</td>\n      <td>0.004217</td>\n      <td>0.999991</td>\n      <td>0.003162</td>\n      <td>0.999995</td>\n      <td>0.002371</td>\n      <td>0.999997</td>\n      <td>0.001778</td>\n      <td>0.999998</td>\n      <td>0.001334</td>\n      <td>0.999999</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>-0.999990</td>\n      <td>0.004426</td>\n      <td>0.923052</td>\n      <td>-0.384674</td>\n      <td>-0.097276</td>\n      <td>0.995257</td>\n      <td>-0.997283</td>\n      <td>-0.073661</td>\n      <td>-0.330575</td>\n      <td>-0.943780</td>\n      <td>...</td>\n      <td>0.004639</td>\n      <td>0.999989</td>\n      <td>0.003478</td>\n      <td>0.999994</td>\n      <td>0.002609</td>\n      <td>0.999997</td>\n      <td>0.001956</td>\n      <td>0.999998</td>\n      <td>0.001467</td>\n      <td>0.999999</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>-0.536573</td>\n      <td>0.843854</td>\n      <td>0.413275</td>\n      <td>-0.910606</td>\n      <td>0.448343</td>\n      <td>0.893862</td>\n      <td>-0.940067</td>\n      <td>0.340989</td>\n      <td>-0.607683</td>\n      <td>-0.794179</td>\n      <td>...</td>\n      <td>0.005060</td>\n      <td>0.999987</td>\n      <td>0.003795</td>\n      <td>0.999993</td>\n      <td>0.002846</td>\n      <td>0.999996</td>\n      <td>0.002134</td>\n      <td>0.999998</td>\n      <td>0.001600</td>\n      <td>0.999999</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>0.420167</td>\n      <td>0.907447</td>\n      <td>-0.318216</td>\n      <td>-0.948018</td>\n      <td>0.855881</td>\n      <td>0.517173</td>\n      <td>-0.718144</td>\n      <td>0.695895</td>\n      <td>-0.824528</td>\n      <td>-0.565821</td>\n      <td>...</td>\n      <td>0.005482</td>\n      <td>0.999985</td>\n      <td>0.004111</td>\n      <td>0.999992</td>\n      <td>0.003083</td>\n      <td>0.999995</td>\n      <td>0.002312</td>\n      <td>0.999997</td>\n      <td>0.001734</td>\n      <td>0.999998</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>0.990607</td>\n      <td>0.136737</td>\n      <td>-0.878990</td>\n      <td>-0.476839</td>\n      <td>0.999823</td>\n      <td>-0.018796</td>\n      <td>-0.370395</td>\n      <td>0.928874</td>\n      <td>-0.959605</td>\n      <td>-0.281349</td>\n      <td>...</td>\n      <td>0.005904</td>\n      <td>0.999983</td>\n      <td>0.004427</td>\n      <td>0.999990</td>\n      <td>0.003320</td>\n      <td>0.999995</td>\n      <td>0.002490</td>\n      <td>0.999997</td>\n      <td>0.001867</td>\n      <td>0.999998</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>0.650288</td>\n      <td>-0.759688</td>\n      <td>-0.968206</td>\n      <td>0.250154</td>\n      <td>0.835838</td>\n      <td>-0.548975</td>\n      <td>0.042249</td>\n      <td>0.999107</td>\n      <td>-0.999519</td>\n      <td>0.031022</td>\n      <td>...</td>\n      <td>0.006325</td>\n      <td>0.999980</td>\n      <td>0.004743</td>\n      <td>0.999989</td>\n      <td>0.003557</td>\n      <td>0.999994</td>\n      <td>0.002667</td>\n      <td>0.999996</td>\n      <td>0.002000</td>\n      <td>0.999998</td>\n    </tr>\n  </tbody>\n</table>\n<p>16 rows × 64 columns</p>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define Position Encoding look-up table\n",
    "position_encoding_lookup_table = torch.zeros(context_length, d_model)\n",
    "position = torch.arange(0, context_length, dtype=torch.float).unsqueeze(1)\n",
    "div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "position_encoding_lookup_table[:, 0::2] = torch.sin(position * div_term)\n",
    "position_encoding_lookup_table[:, 1::2] = torch.cos(position * div_term)\n",
    "position_encoding_lookup_table = position_encoding_lookup_table.unsqueeze(0).expand(batch_size, -1, -1) #add batch dimension\n",
    "\n",
    "print(\"Position Encoding Look-up Table: \", position_encoding_lookup_table.shape) # [4, 16, 64] [batch_size, context_length, d_model]\n",
    "pd.DataFrame(position_encoding_lookup_table[0].detach().cpu().numpy())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:26:57.181192Z",
     "start_time": "2024-02-09T04:26:57.135243Z"
    }
   },
   "id": "17f145bd89a13b23",
   "execution_count": 16
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 640x480 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAHHCAYAAACY6dMIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABeuElEQVR4nO3deVxUVf8H8M/MAMM6bMqWKLik4oa5EFppSYKaaU+bZS5Y+rigGWZqT4ppira45qNtLj1Z2qZpJUoatrmCpqmRGiqpgKaAgGwz5/eHP26O3EFmgZlhPu/X6750vnPOPecOiF/OOfdchRBCgIiIiMgOKK3dASIiIqLaYuJCREREdoOJCxEREdkNJi5ERERkN5i4EBERkd1g4kJERER2g4kLERER2Q0mLkRERGQ3mLgQERGR3WDiQmQkhUKB2bNn16psWFgYRo4cWaf9qa3Zs2dDoVBYuxt1ZuTIkQgLC9OLGfO1IiL7wMSF7NratWuhUCikw9XVFXfeeScSEhKQm5tbL3345ZdfMHv2bOTn59dLe3Vt5MiRep/prZ8vEZE1OVm7A0SWMGfOHISHh6O0tBQ//fQTVq5ciW+//Ra//fYb3N3dLdrW9evX4eT0zz+dX375Ba+++ipGjhwJHx8fvbKZmZlQKu3v9wO1Wo3333+/WlylUlmhN6a79WtFRPaP/6KpQejXrx+6du0KAHjuuefg7++PRYsW4auvvsJTTz1l0baMGXVQq9UWbbu+ODk54ZlnnrF2N8zGESKihsf+fhUkqoUHHngAAJCVlQUAqKysxNy5c9GiRQuo1WqEhYXh5ZdfRllZmV69gwcPIjY2Fo0aNYKbmxvCw8MxatQovTI3r5uYPXs2pk6dCgAIDw+XplTOnDkDQH6Ny59//onHH38cfn5+cHd3x913341vvvlGr0xaWhoUCgU+/fRTzJs3D02aNIGrqyv69OmDU6dO6ZX98ccf8fjjj6Np06ZQq9UIDQ3FCy+8gOvXr5v8+dVG1TTdzz//jMTERDRu3BgeHh545JFHcOnSpWrlt23bhl69esHLywsajQbdunXDxx9/rFfms88+Q5cuXeDm5oZGjRrhmWeewfnz56uda/PmzWjfvj1cXV3Rvn17bNq0SbaPt65xqVrnc+rUKWmEzNvbG/Hx8SgpKdGre/36dUyaNAmNGjWCl5cXHn74YZw/f57rZoisjCMu1CCdPn0aAODv7w/gxijMunXr8Nhjj2HKlCnYt28fkpOTceLECek/vby8PPTt2xeNGzfG9OnT4ePjgzNnzuDLL7802M6//vUv/PHHH/jkk0+wePFiNGrUCADQuHFj2fK5ubno0aMHSkpKMGnSJPj7+2PdunV4+OGH8fnnn+ORRx7RK79gwQIolUq8+OKLKCgowOuvv46hQ4di3759UpnPPvsMJSUlGDduHPz9/bF//34sX74cf/31Fz777DOTP8PLly9Xi7m4uECj0ejFJk6cCF9fXyQlJeHMmTNYsmQJEhISsHHjRqnM2rVrMWrUKLRr1w4zZsyAj48PDh06hJSUFDz99NNSmfj4eHTr1g3JycnIzc3F0qVL8fPPP+PQoUPSNNyOHTvw6KOPIiIiAsnJyfj7778RHx+PJk2a1PrannjiCYSHhyM5ORkZGRl4//33ERAQgIULF0plRo4ciU8//RTDhg3D3Xffjd27d2PAgAHGfIREVBcEkR1bs2aNACC+++47cenSJZGdnS02bNgg/P39hZubm/jrr7/E4cOHBQDx3HPP6dV98cUXBQCxa9cuIYQQmzZtEgDEgQMHamwTgEhKSpJev/HGGwKAyMrKqla2WbNmYsSIEdLryZMnCwDixx9/lGLXrl0T4eHhIiwsTGi1WiGEEN9//70AINq2bSvKysqkskuXLhUAxNGjR6VYSUlJtXaTk5OFQqEQZ8+elWJJSUmiNv/kR4wYIQDIHrGxsVK5qs8+JiZG6HQ6Kf7CCy8IlUol8vPzhRBC5OfnCy8vLxEVFSWuX7+u11ZVvfLychEQECDat2+vV+brr78WAMSsWbOkWGRkpAgODpbOL4QQO3bsEABEs2bN9M5/69eq6jMYNWqUXrlHHnlE+Pv7S6/T09MFADF58mS9ciNHjqx2TiKqX5wqogYhJiYGjRs3RmhoKIYMGQJPT09s2rQJd9xxB7799lsAQGJiol6dKVOmAIA0TVP1G/3XX3+NioqKOunnt99+i+7du+Oee+6RYp6enhgzZgzOnDmD48eP65WPj4+Hi4uL9Pree+8FcGO6qYqbm5v09+LiYly+fBk9evSAEAKHDh0yqZ+urq5ITU2tdixYsKBa2TFjxujdZn3vvfdCq9Xi7NmzAIDU1FRcu3YN06dPr7bmpKrewYMHkZeXh/Hjx+uVGTBgANq0aSN9jS5evIjDhw9jxIgR8Pb2lso9+OCDiIiIqPX1jR07Vu/1vffei7///huFhYUAgJSUFADA+PHj9cpNnDix1m0QUd3gVBE1CCtWrMCdd94JJycnBAYGonXr1tLdPGfPnoVSqUTLli316gQFBcHHx0f6D7ZXr1549NFH8eqrr2Lx4sXo3bs3Bg8ejKefftpii2zPnj2LqKioavG2bdtK77dv316KN23aVK+cr68vAODq1atS7Ny5c5g1axa2bNmiFweAgoICk/qpUqkQExNTq7K362PVtN3N13Wrqq9B69atq73Xpk0b/PTTT3rlWrVqVa1c69atkZGRYXafNRqN9D0THh6uV+7W7yEiqn9MXKhB6N69u3RXkSG323xNoVDg888/x969e7F161Zs374do0aNwltvvYW9e/fC09PTkl2uFUO3HwshAABarRYPPvggrly5gmnTpqFNmzbw8PDA+fPnMXLkSOh0Oqv30RbZY5+J6AZOFVGD16xZM+h0Opw8eVIvnpubi/z8fDRr1kwvfvfdd2PevHk4ePAg1q9fj2PHjmHDhg0Gz2/MbrTNmjVDZmZmtfjvv/8uvW+Mo0eP4o8//sBbb72FadOmYdCgQYiJiUFISIhR56lLLVq0AAD89ttvBstUXbfcZ5OZmSm9X/XnrV9LQ3VNVfU9U3VXWpVb7+giovrHxIUavP79+wMAlixZohdftGgRAEh3ily9erXab9yRkZEAUO226Zt5eHgAQK12zu3fvz/279+PPXv2SLHi4mK8++67CAsLM2qdBvDPyMHN/RZCYOnSpUadpy717dsXXl5eSE5ORmlpqd57Vf3u2rUrAgICsGrVKr3Petu2bThx4oT0NQoODkZkZCTWrVunNw2WmppabX2QOWJjYwEA//3vf/Xiy5cvt1gbRGQaThVRg9epUyeMGDEC7777LvLz89GrVy/s378f69atw+DBg3H//fcDANatW4f//ve/eOSRR9CiRQtcu3YN7733HjQajZT8yOnSpQsA4D//+Q+GDBkCZ2dnDBw4UEpobjZ9+nR88skn6NevHyZNmgQ/Pz+sW7cOWVlZ+OKLL4zeZbdNmzZo0aIFXnzxRZw/fx4ajQZffPFFtbUuxqqsrMRHH30k+94jjzwie22GaDQaLF68GM899xy6deuGp59+Gr6+vvj1119RUlKCdevWwdnZGQsXLkR8fDx69eqFp556SrodOiwsDC+88IJ0vuTkZAwYMAD33HMPRo0ahStXrmD58uVo164dioqKzLruKl26dMGjjz6KJUuW4O+//5Zuh/7jjz8AGDfKRkSWxcSFHML777+P5s2bY+3atdi0aROCgoIwY8YMJCUlSWWqEpoNGzYgNzcX3t7e6N69O9avX19tkebNunXrhrlz52LVqlVISUmRphjk/nMPDAzEL7/8gmnTpmH58uUoLS1Fx44dsXXrVpP2CHF2dsbWrVsxadIkJCcnw9XVFY888ggSEhLQqVMno89XpaysDMOGDZN9z9C11eTZZ59FQEAAFixYgLlz58LZ2Rlt2rTRS0hGjhwJd3d3LFiwANOmTZM2s1u4cKHeoxTi4uLw2Wef4ZVXXsGMGTPQokULrFmzBl999RXS0tJMuVxZH374IYKCgvDJJ59g06ZNiImJwcaNG9G6dWvuyEtkRQrB1WhERLVy+PBhdO7cGR999BGGDh1q7e4QOSSucSEikiH3yIQlS5ZAqVTivvvus0KPiAjgVBERkazXX38d6enpuP/+++Hk5IRt27Zh27ZtGDNmDEJDQ63dPSKHxakiIiIZqampePXVV3H8+HEUFRWhadOmGDZsGP7zn//AyYm/8xFZC6eKiIhkPPjgg/jpp59w5coVlJeX49SpU0hKSmLSQnbrhx9+wMCBAxESEgKFQoHNmzfftk5aWhruuusuqNVqtGzZEmvXrq1WZsWKFQgLC4OrqyuioqKwf/9+y3f+JkxciIiIHEBxcTE6deqEFStW1Kp8VlYWBgwYgPvvvx+HDx/G5MmT8dxzz2H79u1SmY0bNyIxMRFJSUnIyMhAp06dEBsbi7y8vLq6DE4VERERORqFQoFNmzZh8ODBBstMmzYN33zzjd6u10OGDEF+fr70INKoqCh069YNb7/9NgBAp9MhNDQUEydOxPTp0+uk7w1+zFOn0+HChQvw8vLiplFERFQjIQSuXbuGkJAQozeENEZpaSnKy8vNPo8Qotr/bWq12iIPht2zZ0+1h63GxsZi8uTJAIDy8nKkp6djxowZ0vtKpRIxMTF6u4NbWoNPXC5cuMA7AIiIyCjZ2dlo0qRJnZy7tLQU4c08kZOnNftcnp6e1XaMTkpKwuzZs80+d05ODgIDA/VigYGBKCwsxPXr13H16lVotVrZMlXPX6sLDT5x8fLyAgA0SXoFylt2u/z10dWydTp9Mcrg+YytY6ny9dEGr7v+23DU666PNhz1s+V1m1ensEiHZnedkf7vqAvl5eXIydPibHoYNF6mj+oUXtOhWZczyM7OhkajkeKWGG2xZQ0+cakaQlO6ulZLXAx9w9xazpw6lipfH23wuuu/DUe97vpow1E/W163ZerUx9ICTy8FPL1Mb0eHG3U1Go1e4mIpQUFByM3N1Yvl5uZCo9HAzc0NKpUKKpVKtkxQUJDF+1OFdxURERFZgVbozD7qUnR0NHbu3KkXS01NRXR0NADAxcUFXbp00Suj0+mwc+dOqUxdaPAjLkRERLZIBwEdTL+x19i6RUVFOHXqlPQ6KysLhw8fhp+fH5o2bYoZM2bg/Pnz+PDDDwEAY8eOxdtvv42XXnoJo0aNwq5du/Dpp5/im2++kc6RmJiIESNGoGvXrujevTuWLFmC4uJixMfHm3xdt8PEhYiIyAEcPHgQ999/v/Q6MTERADBixAisXbsWFy9exLlz56T3w8PD8c033+CFF17A0qVL0aRJE7z//vuIjY2Vyjz55JO4dOkSZs2ahZycHERGRiIlJaXagl1LYuJCRERkBTroYM5kj7G1e/fujZq2bpPbFbd37944dOhQjedNSEhAQkKCUX0xh12scanv7YSJiIjqmlYIsw9HZPOJizW2EyYiIiLbZPOJy6JFizB69GjEx8cjIiICq1atgru7O1avNnzPPhERka2rWpxrzuGIbDpxqdpO+OYth+tjO2EiIqK6poOA1ozDURMXm16ce/nyZaO3Ey4rK0NZWZn0urCwsE77SERERPXHpkdcTJGcnAxvb2/p4HOKiIjIFnGqyDQ2nbg0atTI6O2EZ8yYgYKCAunIzs6uj64SEREZhXcVmcamExdTthNWq9XScxvq6vkNREREZB02vcYFsM52wkRERHVN9/+HOfUdkc0nLtbYTpiIiKiuVd0dZE59R2TziQtQ/9sJExER1TWtuHGYU98R2fQaFyIiIqKb2cWICxERUUPDNS6mYeJCRERkBToooIXCrPqOiFNFREREZDc44kJERGQFOnHjMKe+I3KYxGXtQ6vg6aU/wJRw/l7ZslPjtho8z4ZrvrLx2HsPy8YP3/TcpJu1ipTf0fevyiKDbfu0uiIbL9Bdl427hBbLxstEhXwDAfJ9BYAKoZWN63zlz6UV8rOvOk/589RE527cTK7O1bjywsX4f/3C2bg6wsmENlRGVjB2/NSU8VZjR6ZNGcluCG3YYp/qow1b7JOpdeqB1sypInPq2jNOFREREZHdcJgRFyIiIlvCERfTMHEhIiKyAp1QQCfMuKvIjLr2jFNFREREZDc44kJERGQFnCoyDRMXIiIiK9BCCa0ZEx/G36PZMDBxISIisgJh5hoXwTUuRERERLaNIy5ERERWwDUupmHiQkREZAVaoYRWmLHGxUG3/OdUEREREdkNjrgQERFZgQ4K6MwYP9DBMYdcmLgQERFZAde4mIZTRURERGQ3OOJCRERkBeYvzuVUEREREdWTG2tczHjIIqeKiIiIiGwbR1yIiIisQGfms4oc9a4ijrgQERFZQdUaF3MOU6xYsQJhYWFwdXVFVFQU9u/fb7Bs7969oVAoqh0DBgyQyowcObLa+3FxcSb1rTY44kJERGQFOijrfR+XjRs3IjExEatWrUJUVBSWLFmC2NhYZGZmIiAgoFr5L7/8EuXl5dLrv//+G506dcLjjz+uVy4uLg5r1qyRXqvVaqP7VlsccSEiInIQixYtwujRoxEfH4+IiAisWrUK7u7uWL16tWx5Pz8/BAUFSUdqairc3d2rJS5qtVqvnK+vb51dAxMXIiIiK9AKhdkHABQWFuodZWVlsu2Vl5cjPT0dMTExUkypVCImJgZ79uypVZ8/+OADDBkyBB4eHnrxtLQ0BAQEoHXr1hg3bhz+/vtvEz+V23OYqaIAVTm8VPp52sElnWXLvv3GPoPnCd8yRja+u/8i2XhC1mOy8dFNfpSNb7rWzmDbsU1+l43/Wu4mG28XdFE2/lel/Dd1UKMCg20X6Epl4x7e8vEyUSkbd/KskI1XCK3BthVu8ufSCp18BRcDcQOEk/HDrUbXURndBKA0rg1hbHlT7qQ0tk59tEFkp7RmLs7V/v9UUWhoqF48KSkJs2fPrlb+8uXL0Gq1CAwM1IsHBgbi99/l/3+52f79+/Hbb7/hgw8+0IvHxcXhX//6F8LDw3H69Gm8/PLL6NevH/bs2QOVypQffjVzmMSFiIioIcrOzoZGo5Fe19X6kg8++AAdOnRA9+7d9eJDhgyR/t6hQwd07NgRLVq0QFpaGvr06WPxfnCqiIiIyAp0Qmn2AQAajUbvMJS4NGrUCCqVCrm5uXrx3NxcBAUF1djX4uJibNiwAc8+++xtr6t58+Zo1KgRTp06VctPwjhMXIiIiKygaqrInMMYLi4u6NKlC3bu3CnFdDoddu7ciejo6BrrfvbZZygrK8Mzzzxz23b++usv/P333wgODjaqf7XFxIWIiMhBJCYm4r333sO6detw4sQJjBs3DsXFxYiPjwcADB8+HDNmzKhW74MPPsDgwYPh7++vFy8qKsLUqVOxd+9enDlzBjt37sSgQYPQsmVLxMbG1sk1cI0LERGRFegA6c4gU+sb68knn8SlS5cwa9Ys5OTkIDIyEikpKdKC3XPnzkGp1B/TyMzMxE8//YQdO3ZUO59KpcKRI0ewbt065OfnIyQkBH379sXcuXPrbK0NExciIiIrMH8DOtPqJiQkICEhQfa9tLS0arHWrVtDGHgStZubG7Zv325SP0zFqSIiIiKyGzaduCQnJ6Nbt27w8vJCQEAABg8ejMzMTGt3i4iIyGzWelaRvbPpq969ezcmTJiAvXv3IjU1FRUVFejbty+Ki4ut3TUiIiKz6KAw+3BENr3GJSUlRe/12rVrERAQgPT0dNx3331W6hUREZH5zB01cdQRF5tOXG5VUHBjS3o/Pz+DZcrKyvSe01BYWFjn/SIiIqL6YTfpmk6nw+TJk9GzZ0+0b9/eYLnk5GR4e3tLx63PcCAiIrIF9b0BXUNhN1c9YcIE/Pbbb9iwYUON5WbMmIGCggLpyM7OrqceEhER1Z5OKMw+HJFdTBUlJCTg66+/xg8//IAmTZrUWFatVtfZpjdERERkXTaduAghMHHiRGzatAlpaWkIDw+3dpeIiIgsQmfmdI85m9fZM5tOXCZMmICPP/4YX331Fby8vJCTkwMA8Pb2hpubm5V7R0REZLqbn/Bsan1HZNNXvXLlShQUFKB3794IDg6Wjo0bN1q7a0RERGQFNj3iYujZCERERPZOCwW0ZmwiZ05de2bTiQsREVFDxaki0zjmVRMREZFd4ogLERGRFWhh3nSP1nJdsStMXIiIiKyAU0WmcZjEpd8P46B0c9WLtfp4n2zZlFcNb2DXcn2FbLzpw56y8T/SmsvGY5/dLBu/L6OfwbaXt/9ENr6l4C7ZeA/fP2XjR8uDZONtfPIMtp2rlf8HEqi5Jhsv0JXLxj09SmXjZUL+cwUAZ9dK2Xilgd83lGr5uFbo5BtwMRCvqY7KuIXjwsjyAIyfyDW2vML4Phm7UWe9bOzpmOsTqQHgQxZN45hXTURERHbJYUZciIiIbImAAjozhgyFgw43MnEhIiKyAk4VmcYxr5qIiIjsEkdciIiIrEAnFNCZsYLdnLr2jIkLERGRFWjNfDq0OXXtmWNeNREREdkljrgQERFZAaeKTMPEhYiIyAp0UEJnxsSHOXXtmWNeNREREdkljrgQERFZgVYooDVjusecuvaMiQsREZEVcI2LaZi4EBERWYEw8+nQgjvnEhEREdk2jrgQERFZgRYKaM14UKI5de0ZExciIiIr0Anz1qnohAU7Y0c4VURERER2gyMuREREVqAzc3GuOXXtmWNeNRERkZXpoDD7MMWKFSsQFhYGV1dXREVFYf/+/QbLrl27FgqFQu9wdXXVKyOEwKxZsxAcHAw3NzfExMTg5MmTJvWtNpi4EBEROYiNGzciMTERSUlJyMjIQKdOnRAbG4u8vDyDdTQaDS5evCgdZ8+e1Xv/9ddfx7Jly7Bq1Srs27cPHh4eiI2NRWlpaZ1cAxMXIiIiK6jaOdecw1iLFi3C6NGjER8fj4iICKxatQru7u5YvXq1wToKhQJBQUHSERgYKL0nhMCSJUvwyiuvYNCgQejYsSM+/PBDXLhwAZs3bzblY7ktJi5ERERWULXGxZzDGOXl5UhPT0dMTIwUUyqViImJwZ49ewzWKyoqQrNmzRAaGopBgwbh2LFj0ntZWVnIycnRO6e3tzeioqJqPKc5HGZx7p1LiuCkqtCLVUZ3lC07/qe7DJ6n1U8ZsvGUErVsPPS767Jxz9GusvGSQ/4G2+50V7lsfGJ2a9n48vafyMa3FMhfXzvPCwbbPlnRWDYe5nlFNn5Fp5KN+3mUyMav6SoNtu3uKn/dFUIrG3dykY9XQj6udJaP18hJ/j5ErdDJl1caf9+iUBlZx9hfvkz5tUVR9/df1ssu5o65/QU1UIWFhXqv1Wo11Orq/yddvnwZWq1Wb8QEAAIDA/H777/Lnrt169ZYvXo1OnbsiIKCArz55pvo0aMHjh07hiZNmiAnJ0c6x63nrHrP0jjiQkREZAU6KKTnFZl0/H8GHhoaCm9vb+lITk62WB+jo6MxfPhwREZGolevXvjyyy/RuHFjvPPOOxZrw1gOM+JCRERkS4QZdwZV1QeA7OxsaDQaKS432gIAjRo1gkqlQm5url48NzcXQUFBtWrT2dkZnTt3xqlTpwBAqpebm4vg4GC9c0ZGRtb6WozBERciIiIrMGu05aYnS2s0Gr3DUOLi4uKCLl26YOfOnf/0QafDzp07ER0dXas+a7VaHD16VEpSwsPDERQUpHfOwsJC7Nu3r9bnNBZHXIiIiBxEYmIiRowYga5du6J79+5YsmQJiouLER8fDwAYPnw47rjjDmm6ac6cObj77rvRsmVL5Ofn44033sDZs2fx3HPPAbhxx9HkyZPx2muvoVWrVggPD8fMmTMREhKCwYMH18k1MHEhIiKyAmvsnPvkk0/i0qVLmDVrFnJychAZGYmUlBRpce25c+egVP5z3qtXr2L06NHIycmBr68vunTpgl9++QURERFSmZdeegnFxcUYM2YM8vPzcc899yAlJaXaRnWWwsSFiIjICm6e7jG1vikSEhKQkJAg+15aWpre68WLF2Px4sU1nk+hUGDOnDmYM2eOSf0xFte4EBERkd2wq8RlwYIF0nwaERGRPbPWs4rsnd1MFR04cADvvPMOOnaU3zSOiIjInlhrqsje2cWIS1FREYYOHYr33nsPvr6+1u4OERERWYldJC4TJkzAgAED9J6FYEhZWRkKCwv1DiIiIltjqX1cHI3NTxVt2LABGRkZOHDgQK3KJycn49VXX63jXhEREZmHU0WmsekRl+zsbDz//PNYv359re8HnzFjBgoKCqQjOzu7jntJRERE9cWmR1zS09ORl5eHu+7652nGWq0WP/zwA95++22UlZVBpdJ/CrGhp2ISERHZEo64mMamE5c+ffrg6NGjerH4+Hi0adMG06ZNq5a0EBER2QsBmPmQRcdk04mLl5cX2rdvrxfz8PCAv79/tTgREZE94YiLaWx6jQsRERHRzWx6xEXOrc9RICIiskcccTGN3SUuREREDQETF9NwqoiIiIjsBkdciIiIrIAjLqZh4kJERGQFQiggzEg+zKlrzzhVRERERHbDYUZcdFnZ0Cmc9WK5G5vLlg37r7NsHACcmjaRjc/KbCkb9z9wQjZ+pLxUNh6QUWmwbc/R8o89KDwp/8TsVp2vy8b3X24mG58WnmKw7YySMNl4uPtl2fj5So1sPNDtmmz8Wg2/OXi5lsnGS4VWNq5WV8jGKwyUVznpDLatM7DFk6KGOvKNmLBVlMK4OkJpZBum/LJmbB0jr8EU9fJLp2P+Ykt1TAeFWRvQmVPXnjlM4kJERGRLuMbFNJwqIiIiIrvBERciIiIr4OJc0zBxISIisgJOFZmGiQsREZEVcMTFNFzjQkRERHaDIy5ERERWIMycKnLUERcmLkRERFYgAAgztjqq+12SbBOnioiIiMhucMSFiIjICnRQQMGdc43GxIWIiMgKeFeRaThVRERERHaDIy5ERERWoBMKKLgBndGYuBAREVmBEGbeVeSgtxVxqoiIiIjsBkdciIiIrICLc03DxIWIiMgKmLiYhlNFREREVlD1dGhzDlOsWLECYWFhcHV1RVRUFPbv32+w7HvvvYd7770Xvr6+8PX1RUxMTLXyI0eOhEKh0Dvi4uJM6lttMHEhIiJyEBs3bkRiYiKSkpKQkZGBTp06ITY2Fnl5ebLl09LS8NRTT+H777/Hnj17EBoair59++L8+fN65eLi4nDx4kXp+OSTT+rsGpi4EBERWUHVXUXmHMZatGgRRo8ejfj4eERERGDVqlVwd3fH6tWrZcuvX78e48ePR2RkJNq0aYP3338fOp0OO3fu1CunVqsRFBQkHb6+vqZ8JLXCxIWIiMgKbiQfCjMO49orLy9Heno6YmJipJhSqURMTAz27NlTq3OUlJSgoqICfn5+evG0tDQEBASgdevWGDduHP7++2/jOmcEoxOX2bNnQ6fTVYsXFBTgqaeeskiniIiIqHYKCwv1jrKyMtlyly9fhlarRWBgoF48MDAQOTk5tWpr2rRpCAkJ0Ut+4uLi8OGHH2Lnzp1YuHAhdu/ejX79+kGr1Zp+UTUw+q6iDz74ADt27MBHH32E5s2bA7iRaQ0fPhxBQUEW76Cl5I6MhErtqhf7uesi2bKPftfT4HnOTYmSjSt2yZcXlVmy8f9diZaNe/1q+JunQHddNu79h/wCLX+lm2z87PlGsvHmd14x2PaH13rIxuP8j8rGsyv8ZePBrgWy8StaV9k4APi4yl93iYFfN9xdKmTjFaJ6wg0Azi6VBtuuEPL/8JRO8ufSGXjQvEIlH9ca6NONRgy/Jd+IccWFwoRxZmPXApqydtCUftkax7zZg4xkqbuKQkND9eJJSUmYPXu2OV2TtWDBAmzYsAFpaWlwdf3nZ/aQIUOkv3fo0AEdO3ZEixYtkJaWhj59+li8H0aPuBw5cgRNmjRBZGQk3nvvPUydOhV9+/bFsGHD8Msvv1i8g0RERA2RsMABANnZ2SgoKJCOGTNmyLbXqFEjqFQq5Obm6sVzc3NvO/Dw5ptvYsGCBdixYwc6duxYY9nmzZujUaNGOHXqVI3lTGX0iIuvry8+/fRTvPzyy/j3v/8NJycnbNu2rU6yKiIiIqqZRqOBRqO5bTkXFxd06dIFO3fuxODBgwFAWmibkJBgsN7rr7+OefPmYfv27ejatett2/nrr7/w999/Izg4uNbXYAyTFucuX74cS5cuxVNPPYXmzZtj0qRJ+PXXXy3dNyIiogbLvIW5pk0zJSYm4r333sO6detw4sQJjBs3DsXFxYiPjwcADB8+XG/EZuHChZg5cyZWr16NsLAw5OTkICcnB0VFRQCAoqIiTJ06FXv37sWZM2ewc+dODBo0CC1btkRsbKxlPqhbGD3iEhcXh4MHD2LdunV47LHHcP36dSQmJuLuu+/Gq6++ipdeeqku+klERNSw3DzfY2p9Iz355JO4dOkSZs2ahZycHERGRiIlJUVasHvu3Dkolf+MaaxcuRLl5eV47LHH9M5TtY5GpVLhyJEjWLduHfLz8xESEoK+ffti7ty5UKvVZlycYUYnLlqtFkeOHEFISAgAwM3NDStXrsRDDz2E5557jokLERFRbZi5OBcm1k1ISDA4NZSWlqb3+syZMzWey83NDdu3bzepH6YyOnFJTU2VjQ8YMABHj8rfYUJERERkCSatcfnxxx/xzDPPIDo6Wtr293//+x9+//13i3YOAM6fP49nnnkG/v7+cHNzQ4cOHXDw4EGLt0NERFSfrLFzbkNgdOLyxRdfIDY2Fm5ubjh06JC00U1BQQHmz59v0c5dvXoVPXv2hLOzM7Zt24bjx4/jrbfeqtOthImIiOqDNRbnNgRGTxW99tprWLVqFYYPH44NGzZI8Z49e+K1116zaOcWLlyI0NBQrFmzRoqFh4dbtA0iIiKyH0aPuGRmZuK+++6rFvf29kZ+fr4l+iTZsmULunbtiscffxwBAQHo3Lkz3nvvvRrrlJWVVdv+mIiIyOYIhfmHAzI6cQkKCpLdDe+nn36SHgFgKX/++SdWrlyJVq1aYfv27Rg3bhwmTZqEdevWGayTnJwMb29v6bh1K2QiIiJbwDUupjE6cRk9ejSef/557Nu3DwqFAhcuXMD69evx4osvYty4cRbtnE6nw1133YX58+ejc+fOGDNmDEaPHo1Vq1YZrDNjxgy9rY+zs7Mt2iciIiKyHqPXuEyfPh06nQ59+vRBSUkJ7rvvPqjVarz44ouYOHGiRTsXHByMiIgIvVjbtm3xxRdfGKyjVqvrbNMbIiIii7HCBnQNgdGJi0KhwH/+8x9MnToVp06dQlFRESIiIuDp6WnxzvXs2ROZmZl6sT/++APNmjWzeFtERET1yVJPh3Y0RicuVVxcXKqNhljaCy+8gB49emD+/Pl44oknsH//frz77rt4991367RdIiIisk21Slz+9a9/1fqEX375pcmduVW3bt2wadMmzJgxA3PmzEF4eDiWLFmCoUOHWqwNIiIiq3HQ6R5z1Cpx8fb2lv4uhMCmTZvg7e0tPd46PT0d+fn5RiU4tfXQQw/hoYcesvh5iYiIrIlTRaapVeJy8wZw06ZNwxNPPIFVq1ZBpVIBuPHgxfHjx0Oj0dRNL4mIiBoaLs41idG3Q69evRovvviilLQAgEqlQmJiIlavXm3RzhERERHdzOjEpbKyUvZhir///jt0Op1FOkVERNTwKSxwOB6j7yqKj4/Hs88+i9OnT6N79+4AgH379mHBggWIj4+3eAeJiIgaJE4VmcToxOXNN99EUFAQ3nrrLVy8eBHAjY3ipk6diilTpli8g0RERERVjE5clEolXnrpJbz00kvSAwztYVHu0Gd3wNVT/3K/u95ItqxTYGOD5+n+6BHZ+IWxTWXjolNr2fim351l4y3OHTXY9qEyD9m4z8ly2bhKIT8T6PyXi2w8UGV45vDkVfnPZEzQFdn4d9faycaD1AWy8Tytl8G2/dXFsvFrOpVs3MNF/vOoMPDriYuT1mDbOshPf6pU8nFD5RUGytdIaeSvUwpjyxtX3KQ6NjqS7aA3Y5Ct4YiLSUzegA6wj4SFiIjIJpn7hGcHzcCNXpybm5uLYcOGISQkBE5OTlCpVHoHERERUV0xesRl5MiROHfuHGbOnIng4GAoFI6Z8REREZlDiBuHOfUdkdGJy08//YQff/wRkZGRddAdIiIiB8E1LiYxeqooNDQUwlHTPCIiIrIqoxOXJUuWYPr06Thz5kwddIeIiMhBVC3ONedwQEZPFT355JMoKSlBixYt4O7uDmdn/dt6r1yRvz2WiIiI/qEQxu9icGt9R2R04rJkyZI66AYREZGD4RoXkxiduIwYMaIu+kFERER0W7VOXKp2yb0dbkpHRERUC9yAziS1Tlx8fHxq3LNFCAGFQgGt1vD26URERPT/OFVkklonLt9//31d9oOIiIjotmqduPTq1asu+0FERORYOOJiErMeskhEREQmYuJiEqM3oCMiIiKyFo64EBERWQPvKjIJExciIiIr4M65puFUEREREdkNoxOX4uJizJw5Ez169EDLli3RvHlzvYOIiIhqQVjgMMGKFSsQFhYGV1dXREVFYf/+/TWW/+yzz9CmTRu4urqiQ4cO+Pbbb/UvQwjMmjULwcHBcHNzQ0xMDE6ePGla52rB6Kmi5557Drt378awYcMQHBxc46Z0REREZDs2btyIxMRErFq1ClFRUViyZAliY2ORmZmJgICAauV/+eUXPPXUU0hOTsZDDz2Ejz/+GIMHD0ZGRgbat28PAHj99dexbNkyrFu3DuHh4Zg5cyZiY2Nx/PhxuLq6WvwajE5ctm3bhm+++QY9e/a0eGeIiIgchQJmrnExoc6iRYswevRoxMfHAwBWrVqFb775BqtXr8b06dOrlV+6dCni4uIwdepUAMDcuXORmpqKt99+G6tWrYIQAkuWLMErr7yCQYMGAQA+/PBDBAYGYvPmzRgyZIjJ12eI0VNFvr6+8PPzs3hHiIiIyHiFhYV6R1lZmWy58vJypKenIyYmRooplUrExMRgz549snX27NmjVx4AYmNjpfJZWVnIycnRK+Pt7Y2oqCiD5zSX0SMuc+fOxaxZs7Bu3Tq4u7vXRZ/qxL+9z0DjpZ+ndViVIFvW5THD59nYZJFs/NEj8iNQF6ZEycbd0+XPr1AazqG/KegkG3f985JsvEB3XTbumS1/fk+F2mDbf1/2ko2HOF2Tjf9Z3Eg2Hud/VDZ+qdLwwzn9nItl49d0LrJxD+dy2XipkP/VRu1cabDtCqGTjTs5yT+TS2ugDYVKPq6rYZLaUB2tgT4Z/WuICb+uifq4jcHYfjWUWys46+54LHQ7dGhoqF44KSkJs2fPrlb88uXL0Gq1CAwM1IsHBgbi999/l20iJydHtnxOTo70flXMUBlLMzpxeeutt3D69GkEBgYiLCwMzs7Oeu9nZGRYrHNEREQNloV2zs3OzoZG888vf2q14V9CGwKjE5fBgwfXQTeIiIjIFBqNRi9xMaRRo0ZQqVTIzc3Vi+fm5iIoKEi2TlBQUI3lq/7Mzc1FcHCwXpnIyEhjLqPWjE5ckpKS6qIfREREjqWen1Xk4uKCLl26YOfOndIghE6nw86dO5GQIL90Ijo6Gjt37sTkyZOlWGpqKqKjowEA4eHhCAoKws6dO6VEpbCwEPv27cO4ceOMvaJaMXnn3PT0dJw4cQIA0K5dO3Tu3NlinSIiImrorLFzbmJiIkaMGIGuXbuie/fuWLJkCYqLi6W7jIYPH4477rgDycnJAIDnn38evXr1wltvvYUBAwZgw4YNOHjwIN59990bfVAoMHnyZLz22mto1aqVdDt0SEhInc3QGJ245OXlYciQIUhLS4OPjw8AID8/H/fffz82bNiAxo0bW7qPREREZAFPPvkkLl26hFmzZiEnJweRkZFISUmRFteeO3cOSuU/K/179OiBjz/+GK+88gpefvlltGrVCps3b5b2cAGAl156CcXFxRgzZgzy8/Nxzz33ICUlpU72cAFMSFwmTpyIa9eu4dixY2jbti0A4Pjx4xgxYgQmTZqETz75xOKdJCIianDqeaqoSkJCgsGpobS0tGqxxx9/HI8//rjB8ykUCsyZMwdz5swxrUNGMnofl5SUFPz3v/+VkhYAiIiIwIoVK7Bt2zaLdk6r1WLmzJkIDw+Hm5sbWrRogblz50IYuOWUiIjIblhpy397Z/SIi06nq3YLNAA4OztDpzOwv4SJFi5ciJUrV2LdunVo164dDh48iPj4eHh7e2PSpEkWbYuIiIhsn9EjLg888ACef/55XLhwQYqdP38eL7zwAvr06WPRzv3yyy8YNGgQBgwYgLCwMDz22GPo27fvbR8IRUREZOuqFueaczgioxOXt99+G4WFhQgLC0OLFi3QokULhIeHo7CwEMuXL7do53r06IGdO3fijz/+AAD8+uuv+Omnn9CvXz+DdcrKyqptf0xERGRzqnbONedwQEZPFYWGhiIjIwPfffedtEVw27Ztqz3LwBKmT5+OwsJCtGnTBiqVClqtFvPmzcPQoUMN1klOTsarr75q8b4QERFZlJUW59o7k/ZxUSgUePDBB/Hggw9auj96Pv30U6xfvx4ff/wx2rVrh8OHD2Py5MkICQnBiBEjZOvMmDEDiYmJ0uvCwsJqz3EgIiIi+1SrxGXZsmUYM2YMXF1dsWzZshrLWnLR7NSpUzF9+nTpsdgdOnTA2bNnkZycbDBxUavVDf45DUREZP+ssQFdQ1CrxGXx4sUYOnQoXF1dsXjxYoPlFAqFRROXkpISvY1wAEClUln87iUiIqJ6x6kik9QqccnKypL9e10bOHAg5s2bh6ZNm6Jdu3Y4dOgQFi1ahFGjRtVbH4iIiMh2GH1X0Zw5c1BSUlItfv36dYvvmrd8+XI89thjGD9+PNq2bYsXX3wR//73vzF37lyLtkNERFTvzL0V2kFHXIxOXF599VUUFRVVi5eUlFj8bh4vLy8sWbIEZ8+exfXr13H69Gm89tprcHFxsWg7RERE9Y4755rE6MRFCAGFovq947/++iv8/Pws0ikiIiIiObW+HdrX1xcKhQIKhQJ33nmnXvKi1WpRVFSEsWPH1kkniYiIGhwuzjVJrROXJUuWQAiBUaNG4dVXX4W3t7f0nouLC8LCwhAdHV0nnSQiImpoeDu0aWqduFTtmxIeHo4ePXrIPmiRiIiIqC7VKnEpLCyERqMBAHTu3BnXr1/H9evXZctWlSMiIiKytFolLr6+vrh48SICAgLg4+Mjuzi3atGuVqu1eCct4fGTcXDy0N9RN2z5MdmyrlsM37V0SVspG1e6yu/W6/lArmzc43Vv2bgyzPDjCb7Llm8j8KL83jpZFfJrr73+kr8GlcLwWm3lJfnPxM9Alb+KfGTjQYEFsvH9JS0Mtu3rXP32ewDI17nLxjXOpbLxYp18Z12d5D8PAKgwMImsUslvgqiDfFypNGHTRKWR48DGljdlnNnYZ7o1kGfAOeiz7KiucY2LSWqVuOzatUu6Y+j777+v0w4RERE5Aq5xMU2tEpdevXrJ/p2IiIioPhm9j0tKSgp++ukn6fWKFSsQGRmJp59+GlevXrVo54iIiBo0bj5nNKMTl6lTp6KwsBAAcPToUSQmJqJ///7IyspCYmKixTtIRETUIHHnXJPU+nboKllZWYiIiAAAfPHFFxg4cCDmz5+PjIwM9O/f3+IdJCIiIqpi9IiLi4uL9JDF7777Dn379gUA+Pn5SSMxREREVDNzHrBo7sJee2b0iMs999yDxMRE9OzZE/v378fGjRsBAH/88QeaNGli8Q4SERE1SLwd2iRGj7i8/fbbcHJywueff46VK1fijjvuAABs27YNcXFxFu8gERERURWjR1yaNm2Kr7/+ulp88eLFFukQERGRI+A+LqYxOnEBbjwNevPmzThx4gQAoF27dnj44YehUqks2jkiIqIGi1NFJjE6cTl16hT69++P8+fPo3Xr1gCA5ORkhIaG4ptvvkGLFoa3biciIiIyh9FrXCZNmoQWLVogOzsbGRkZyMjIwLlz5xAeHo5JkybVRR+JiIgaHu7jYhKjR1x2796NvXv3Ss8uAgB/f38sWLAAPXv2tGjniIiIGiqucTGN0YmLWq3GtWvXqsWLiorg4mL4qcpERER0E65xMYnRU0UPPfQQxowZg3379kEIASEE9u7di7Fjx+Lhhx+uiz4SERERATAhcVm2bBlatGiB6OhouLq6wtXVFT179kTLli2xdOnSuugjERFRw8M1LiYxeqrIx8cHX331FU6dOiXdDt22bVu0bNnS4p0jIiJqqLjGxTS1Tlx0Oh3eeOMNbNmyBeXl5ejTpw+SkpLg5uZWl/0jIiIiktR6qmjevHl4+eWX4enpiTvuuANLly7FhAkT6rJvREREDZcNTxVduXIFQ4cOhUajgY+PD5599lkUFRXVWH7ixIlo3bo13Nzc0LRpU0yaNAkFBQV65RQKRbVjw4YNRvWt1iMuH374If773//i3//+N4AbT4YeMGAA3n//fSiVRi+VISIicmi2PFU0dOhQXLx4EampqaioqEB8fDzGjBmDjz/+WLb8hQsXcOHCBbz55puIiIjA2bNnMXbsWFy4cAGff/65Xtk1a9boPdvQx8fHqL7VOnE5d+4c+vfvL72OiYmBQqHAhQsX+FRoIiKiBuLEiRNISUnBgQMH0LVrVwDA8uXL0b9/f7z55psICQmpVqd9+/b44osvpNctWrTAvHnz8Mwzz6CyshJOTv+kGz4+PggKCjK5f7UeKqmsrISrq6tezNnZGRUVFSY3TkRE5LAsNFVUWFiod5SVlZnVrT179sDHx0dKWoAbgxVKpRL79u2r9XkKCgqg0Wj0khYAmDBhAho1aoTu3btj9erVEMK4oaNaj7gIITBy5Eio1WopVlpairFjx8LDw0OKffnll0Z1oL6ULguGk7N+4uXheUG27Afhnxk8z0PHhsvG3br7yMZfu3OdbPyt3x6QjRf2am6w7WsnFbLxAAPfpOmlzWTjbueLZeMVQmuwbbdL8m17KeU3HbxU6Ckb91fJt51T5m2w7fYef8nG87XusnGNc6lsvEzIPwTU1clw8l1u4B+Us0r+s9IamHRWqeTjOugMtq1QGqpj4B+5gfJaYaAN+S9pzYysI0wZyza2XyZdh4PejkG2xUIb0IWGhuqFk5KSMHv2bJNPm5OTg4CAAL2Yk5MT/Pz8kJOTU6tzXL58GXPnzsWYMWP04nPmzMEDDzwAd3d37NixA+PHj0dRUZFRjwyqdeIyYsSIarFnnnmm1g0RERGR5WVnZ0Oj0Uivbx5guNn06dOxcOHCGs9Vtc2JOQoLCzFgwABERERUS6Bmzpwp/b1z584oLi7GG2+8UTeJy5o1a2p9UiIiIqqZAqYNGN5cHwA0Go1e4mLIlClTMHLkyBrLNG/eHEFBQcjLy9OLV1ZW4sqVK7ddm3Lt2jXExcXBy8sLmzZtgrOzc43lo6KiMHfuXJSVlRlMuG5l9AZ0REREZAH1/Kyixo0bo3HjxrctFx0djfz8fKSnp6NLly4AgF27dkGn0yEqKspgvcLCQsTGxkKtVmPLli3V1sXKOXz4MHx9fWudtABMXIiIiKzCVm+Hbtu2LeLi4jB69GisWrUKFRUVSEhIwJAhQ6Q7is6fP48+ffrgww8/RPfu3VFYWIi+ffuipKQEH330kbRQGLiRMKlUKmzduhW5ubm4++674erqitTUVMyfPx8vvviiUf1j4kJERER61q9fj4SEBPTp0wdKpRKPPvooli1bJr1fUVGBzMxMlJSUAAAyMjKkO45ufQRQVlYWwsLC4OzsjBUrVuCFF16AEAItW7bEokWLMHr0aKP6xsSFiIjIGup5qsgYfn5+BjebA4CwsDC925h79+5929ua4+Li9DaeM5VVt7z94YcfMHDgQISEhEChUGDz5s167wshMGvWLAQHB8PNzQ0xMTE4efKkdTpLRERkaTa43b+ts2riUlxcjE6dOmHFihWy77/++utYtmwZVq1ahX379sHDwwOxsbEoLZXfp4OIiIgaNqtOFfXr1w/9+vWTfU8IgSVLluCVV17BoEGDANx4XlJgYCA2b96MIUOG1GdXiYiILMpWF+faOpt9OmJWVhZycnIQExMjxby9vREVFYU9e/ZYsWdEREQWYMNPh7ZlNrs4t2pb4cDAQL14YGBgjVsOl5WV6T2noep2LCIiIrJ/NjviYqrk5GR4e3tLx63PcCAiIrIFVVNF5hyOyGYTl6pthXNzc/Xiubm5NW45PGPGDBQUFEhHdnZ2nfaTiIjIJJwqMonNJi7h4eEICgrCzp07pVhhYSH27duH6Ohog/XUarX03IbaPr+BiIiI7INV17gUFRXh1KlT0uusrCwcPnwYfn5+aNq0KSZPnozXXnsNrVq1Qnh4OGbOnImQkBAMHjzYep0mIiKyAN5VZBqrJi4HDx7E/fffL71OTEwEAIwYMQJr167FSy+9hOLiYowZMwb5+fm45557kJKSUqsHNxEREdk0G94515ZZNXG53RbBCoUCc+bMwZw5c+qxV0RERPWAiYtJbHaNCxEREdGtbHYfFyIiooaMa1xMw8SFiIjIGjhVZBJOFREREZHd4IgLERGRFSiEgKKGG1RqU98ROUziot6RASeFs17s98V3y5atqGH8reKTQNn45T7y5fu4aWXjr1+6JBvPu6ulwba9/5CPK93dZeO/FMifS5nzt2z8qq7UYNtuefKfifqWz7RKab78Let+ynLZeE6pl8G27/Muko1fqpTfXNDLSf46ioV8X92d5PsEABUGvhVcnOS/rjoDP0hUKp1sXFvTXXVK434oKRRGFQeMPD/ZGGO/3mR7OFVkEk4VERERkd1wmBEXIiIiW8K7ikzDxIWIiMgaOFVkEk4VERERkd3giAsREZEVcKrINExciIiIrIFTRSZh4kJERGQFHHExDde4EBERkd3giAsREZE1cKrIJExciIiIrMRRp3vMwakiIiIishsccSEiIrIGIW4c5tR3QExciIiIrIB3FZmGU0VERERkNzjiQkREZA28q8gkTFyIiIisQKG7cZhT3xFxqoiIiIjsBkdciIiIrIFTRSbhiAsREZEVVN1VZM5RV65cuYKhQ4dCo9HAx8cHzz77LIqKimqs07t3bygUCr1j7NixemXOnTuHAQMGwN3dHQEBAZg6dSoqKyuN6htHXIiIiKzBhvdxGTp0KC5evIjU1FRUVFQgPj4eY8aMwccff1xjvdGjR2POnDnSa3d3d+nvWq0WAwYMQFBQEH755RdcvHgRw4cPh7OzM+bPn1/rvjFxISIiIsmJEyeQkpKCAwcOoGvXrgCA5cuXo3///njzzTcREhJisK67uzuCgoJk39uxYweOHz+O7777DoGBgYiMjMTcuXMxbdo0zJ49Gy4uLrXqH6eKiIiIrMBSU0WFhYV6R1lZmVn92rNnD3x8fKSkBQBiYmKgVCqxb9++GuuuX78ejRo1Qvv27TFjxgyUlJTonbdDhw4IDAyUYrGxsSgsLMSxY8dq3T+HGXEpi70LWmdXvdgXg5fKlh1+8gmD52n01XHZeJMt8pliVoX8nKDypuGzm/l3zjPYtkeqt2xcERIoG8/IVcvGA69mycYvVBr+dnC/ZNwcpCpf/lxeSoVsPK/Ey+C5fJQlsvE/KoPl21CVysav6dxk4+5OFQbbLhXyub1apZWNVxhYLadUyt+3qIPh+xkN1TFcwchhY/kvxW3q1EcbJtSxMaIBXAPVAwstzg0NDdULJyUlYfbs2SafNicnBwEBAXoxJycn+Pn5IScnx2C9p59+Gs2aNUNISAiOHDmCadOmITMzE19++aV03puTFgDS65rOeyuHSVyIiIgaouzsbGg0Gum1Wi3/S+v06dOxcOHCGs914sQJk/sxZswY6e8dOnRAcHAw+vTpg9OnT6NFixYmn/dWTFyIiIiswFLPKtJoNHqJiyFTpkzByJEjayzTvHlzBAUFIS9Pf/S/srISV65cMbh+RU5UVBQA4NSpU2jRogWCgoKwf/9+vTK5ubkAYNR5mbgQERFZQz3fVdS4cWM0btz4tuWio6ORn5+P9PR0dOnSBQCwa9cu6HQ6KRmpjcOHDwMAgoODpfPOmzcPeXl50lRUamoqNBoNIiIian1eLs4lIiIiSdu2bREXF4fRo0dj//79+Pnnn5GQkIAhQ4ZIdxSdP38ebdq0kUZQTp8+jblz5yI9PR1nzpzBli1bMHz4cNx3333o2LEjAKBv376IiIjAsGHD8Ouvv2L79u145ZVXMGHCBIPTW3KYuBAREVmBLW9At379erRp0wZ9+vRB//79cc899+Ddd9+V3q+oqEBmZqZ015CLiwu+++479O3bF23atMGUKVPw6KOPYuvWrVIdlUqFr7/+GiqVCtHR0XjmmWcwfPhwvX1faoNTRURERNZgw1v++/n51bjZXFhYGMRNU1WhoaHYvXv3bc/brFkzfPvtt2b1jSMuREREZDesmrj88MMPGDhwIEJCQqBQKLB582bpvYqKCkybNg0dOnSAh4cHQkJCMHz4cFy4cMF6HSYiIrIQW54qsmVWTVyKi4vRqVMnrFixotp7JSUlyMjIwMyZM5GRkYEvv/wSmZmZePjhh63QUyIiIgvTCfMPB2TVNS79+vVDv379ZN/z9vZGamqqXuztt99G9+7dce7cOTRt2rQ+ukhERFQ3bHiNiy2zq8W5BQUFUCgU8PHxMVimrKxM7zkNhYWF9dAzIiIiqg92szi3tLQU06ZNw1NPPVXjDoHJycnw9vaWjluf4UBERGQLFDBzjYu1L8BK7CJxqaiowBNPPAEhBFauXFlj2RkzZqCgoEA6srOz66mXRERERqjaOdecwwHZ/FRRVdJy9uxZ7Nq167bPY1Cr1UbtwEdERET2w6YTl6qk5eTJk/j+++/h7+9v7S4RERFZhKUesuhorJq4FBUV4dSpU9LrrKwsHD58GH5+fggODsZjjz2GjIwMfP3119BqtcjJyQFwY0c/FxcXa3WbiIjIfLyryCRWTVwOHjyI+++/X3qdmJgIABgxYgRmz56NLVu2AAAiIyP16n3//ffo3bt3fXWTiIiIbIRVE5fevXvrPevgVjW9R0REZM8UQkBhxv9z5tS1Zza9xoWIiKjB0v3/YU59B2QXt0MTERERARxxISIisgpOFZmGiQsREZE18K4ikzBxISIisgZzd7/liEvD5j7pApw89HfUbaSqkC2b96nhJ08Hlh6Sjc9uul02Pj8nVv5EdzaTDT8Xtstg21/+2U02XtImUDaef15+CVPATQ+hvNkfFQEG21ZfKpWNa4X86jCXfPmnaLgrnGXjV0vcDLatUcq3fbnCUzbewvWSbLxYJ7+jspuB7wMAqDCwDEytqpSNaw38IHFSyn9O2hp+ZVIq5d/TGViRpzBQ3qAaHnRi6Otq9MNRTHiYijB2Vy1HfWALkYNymMSFiIjIlnDnXNMwcSEiIrIGThWZhLdDExERkd3giAsREZEVKHQ3DnPqOyImLkRERNbAqSKTcKqIiIiI7AZHXIiIiKyBG9CZhIkLERGRFXDLf9NwqoiIiIjsBkdciIiIrIGLc03CxIWIiMgaBGDgCR61r++AmLgQERFZAde4mIZrXIiIiMhucMSFiIjIGgTMXONisZ7YFSYuRERE1sDFuSbhVBERERHZDY64EBERWYMOgMLM+g6IIy5ERERWUHVXkTlHXbly5QqGDh0KjUYDHx8fPPvssygqKjJY/syZM1AoFLLHZ5999s81y7y/YcMGo/rGERciIiLSM3ToUFy8eBGpqamoqKhAfHw8xowZg48//li2fGhoKC5evKgXe/fdd/HGG2+gX79+evE1a9YgLi5Oeu3j42NU35i4EBERWYONLs49ceIEUlJScODAAXTt2hUAsHz5cvTv3x9vvvkmQkJCqtVRqVQICgrSi23atAlPPPEEPD099eI+Pj7VyhqDU0VERETWUJW4mHPUgT179sDHx0dKWgAgJiYGSqUS+/btq9U50tPTcfjwYTz77LPV3pswYQIaNWqE7t27Y/Xq1RBGXgdHXIiIiOxYYWGh3mu1Wg21Wm3y+XJychAQEKAXc3Jygp+fH3Jycmp1jg8++ABt27ZFjx499OJz5szBAw88AHd3d+zYsQPjx49HUVERJk2aVOv+OUzisrHldmi89AeYWu54XrZs2y9OGTxP4YBOsvGOLntl49//2EE27ttZfin5QM/TBtv+LKeZbDx/YKhs3D3bwImUKtnwsetNDLatuiK/KOu6KJeNq/Plz6NWyH/LlRQZ/kfmrSyTjV8pd5eNR3pcl41f07nKxj2c5M8PAKVC/rNyVmll4xUGzuOkMn75v1Jp3G8hCgN3J+gM7VKlMOG3NXPugLAlxl6HKZ+VLWooX7+GwkJTRaGh+v8HJCUlYfbs2dWKT58+HQsXLqzxlCdOnDC9P//v+vXr+PjjjzFz5sxq790c69y5M4qLi/HGG28wcSEiIrJ5FrodOjs7GxqNRgobGm2ZMmUKRo4cWeMpmzdvjqCgIOTl5enFKysrceXKlVqtTfn8889RUlKC4cOH37ZsVFQU5s6di7KyslqPEjFxISIisgJLPWRRo9HoJS6GNG7cGI0bN75tuejoaOTn5yM9PR1dunQBAOzatQs6nQ5RUVG3rf/BBx/g4YcfrlVbhw8fhq+vr1FTW0xciIiISNK2bVvExcVh9OjRWLVqFSoqKpCQkIAhQ4ZIdxSdP38effr0wYcffoju3btLdU+dOoUffvgB3377bbXzbt26Fbm5ubj77rvh6uqK1NRUzJ8/Hy+++KJR/WPiQkREZA02ejs0AKxfvx4JCQno06cPlEolHn30USxbtkx6v6KiApmZmSgpKdGrt3r1ajRp0gR9+/atdk5nZ2esWLECL7zwAoQQaNmyJRYtWoTRo0cb1TcmLkRERNagE+Yt/NbVXeLi5+dncLM5AAgLC5O9jXn+/PmYP3++bJ24uDi9jedMxX1ciIiIyG5YNXH54YcfMHDgQISEhEChUGDz5s0Gy44dOxYKhQJLliypt/4RERHVGRvdgM7WWTVxKS4uRqdOnbBixYoay23atAl79+6V3WaYiIjIPpmbtDhm4mLVNS79+vWr9vClW50/fx4TJ07E9u3bMWDAgHrqGREREdkim16cq9PpMGzYMEydOhXt2rWrVZ2ysjKUlf2zE+qtWyETERHZBBu+q8iW2fTi3IULF8LJycmorYCTk5Ph7e0tHbduhUxERGQTdML8wwHZbOKSnp6OpUuXYu3atVAYegiLjBkzZqCgoEA6srMNPbCHiIiI7I3NJi4//vgj8vLy0LRpUzg5OcHJyQlnz57FlClTEBYWZrCeWq2Wtj+u7TbIRERE9U7ozD8ckM2ucRk2bBhiYmL0YrGxsRg2bBji4+Ot1CsiIiIL4RoXk1g1cSkqKsKpU6ek11lZWTh8+DD8/PzQtGlT+Pv765V3dnZGUFAQWrduXd9dJSIisiydmbc0O+gaF6smLgcPHsT9998vvU5MTAQAjBgxAmvXrrVSr4iIiMhWWTVx6d27t+yzDgw5c+ZM3XWGiIioPnGqyCQ2u8aFiIioQRMwM3GxWE/sis3eVURERER0K464EBERWQOnikzCxIWIiMgadDoAZuzFonPMfVw4VURERER2w2FGXFbmN4drpf7ltn1d/gGMuvwCg+e5+nSRbHx/WYVsvMlOrWz83AD5nDFA5WGwbXHTwyNvVnCnfBuND8i3ofKUb+NYoa/BtpEv/1nl6ypl4+p8+d8EVAr5PumKnQ027aGUP1d+ubts3EtZKhu/UCF/fe7KcoNtlwr5frmq5L/eFQZGbp1V8l+jihp2vlQq5U+mNTA8rFAYN2ysMHD+mivVw9B07Z/wYVp5IlvBqSKTOEziQkREZFOYuJiEU0VERERkNzjiQkREZA3c8t8kTFyIiIisQAgdhBlPeDanrj1j4kJERGQNQpg3asI1LkRERES2jSMuRERE1iDMXOPioCMuTFyIiIisQacDFGasU3HQNS6cKiIiIiK7wREXIiIia+BUkUmYuBAREVmB0OkgzJgqctTboTlVRERERHaDIy5ERETWwKkikzBxISIisgadMO+J6w6auHCqiIiIiOwGR1yIiIisQQgA5uzj4pgjLkxciIiIrEDoBIQZU0WCiQsRERHVG6GDeSMuvB2aiIiIyKZxxIWIiMgKOFVkGiYuRERE1sCpIpM0+MSlKiMtLaqs9l6ltky2jk5UGDyftkS+TvE1+W+gyopS+Tauy8/SFRo4DwBUGuiX7rp8G9py+TYqRblsvKLYYNOo1Mln9tcM9FdbLt8nQ9dn6BpqaqOiWP46Sq5pZeOlldW/BwCgrNzw17tYaVzbRYa+D4rlv28MXRtg+HvN4GdYYrnP3Ng6tthGTf+W6roNk6671Mg2jCxfH20YKl8fbVjquguLbsTqYzSjEhVm7T9XCcM/uxoyhWjgY01//fUXQkNDrd0NIiKyI9nZ2WjSpEmdnLu0tBTh4eHIyckx+1xBQUHIysqCq6urBXpmHxp84qLT6XDhwgV4eXnh2rVrCA0NRXZ2NjQajbW7Vm8KCwt53bzuBo/Xzeu2BCEErl27hpCQECiVdXf/SmlpKcrL5UdujeHi4uJQSQvgAFNFSqVSypoVCgUAQKPRONQ/8Cq8bsfC63YsvG7L8fb2tuj55Li6ujpcwmEpvB2aiIiI7AYTFyIiIrIbDpW4qNVqJCUlQa1WW7sr9YrXzet2BLxuXjc5hga/OJeIiIgaDocacSEiIiL7xsSFiIiI7AYTFyIiIrIbTFyIiIjIbjhU4rJixQqEhYXB1dUVUVFR2L9/v7W7ZFE//PADBg4ciJCQECgUCmzevFnvfSEEZs2aheDgYLi5uSEmJgYnT560TmctJDk5Gd26dYOXlxcCAgIwePBgZGZm6pUpLS3FhAkT4O/vD09PTzz66KPIzc21Uo8tY+XKlejYsaO0+VZ0dDS2bdsmvd8Qr1nOggULoFAoMHnyZCnWEK999uzZUCgUekebNm2k9xviNVc5f/48nnnmGfj7+8PNzQ0dOnTAwYMHpfcb4s81qpnDJC4bN25EYmIikpKSkJGRgU6dOiE2NhZ5eXnW7prFFBcXo1OnTlixYoXs+6+//jqWLVuGVatWYd++ffDw8EBsbCxKa3gwmq3bvXs3JkyYgL179yI1NRUVFRXo27cviov/eWLkCy+8gK1bt+Kzzz7D7t27ceHCBfzrX/+yYq/N16RJEyxYsADp6ek4ePAgHnjgAQwaNAjHjh0D0DCv+VYHDhzAO++8g44dO+rFG+q1t2vXDhcvXpSOn376SXqvoV7z1atX0bNnTzg7O2Pbtm04fvw43nrrLfj6+kplGuLPNboN4SC6d+8uJkyYIL3WarUiJCREJCcnW7FXdQeA2LRpk/Rap9OJoKAg8cYbb0ix/Px8oVarxSeffGKFHtaNvLw8AUDs3r1bCHHjGp2dncVnn30mlTlx4oQAIPbs2WOtbtYJX19f8f777zvENV+7dk20atVKpKamil69eonnn39eCNFwv95JSUmiU6dOsu811GsWQohp06aJe+65x+D7jvJzjfQ5xIhLeXk50tPTERMTI8WUSiViYmKwZ88eK/as/mRlZSEnJ0fvM/D29kZUVFSD+gwKCgoAAH5+fgCA9PR0VFRU6F13mzZt0LRp0wZz3VqtFhs2bEBxcTGio6Md4ponTJiAAQMG6F0j0LC/3idPnkRISAiaN2+OoUOH4ty5cwAa9jVv2bIFXbt2xeOPP46AgAB07twZ7733nvS+o/xcI30OkbhcvnwZWq0WgYGBevHAwECLPFbcHlRdZ0P+DHQ6HSZPnoyePXuiffv2AG5ct4uLC3x8fPTKNoTrPnr0KDw9PaFWqzF27Fhs2rQJERERDfqaAWDDhg3IyMhAcnJytfca6rVHRUVh7dq1SElJwcqVK5GVlYV7770X165da7DXDAB//vknVq5ciVatWmH79u0YN24cJk2ahHXr1gFwjJ9rVF2Dfzo0OY4JEybgt99+05v7b8hat26Nw4cPo6CgAJ9//jlGjBiB3bt3W7tbdSo7OxvPP/88UlNTHerJuv369ZP+3rFjR0RFRaFZs2b49NNP4ebmZsWe1S2dToeuXbti/vz5AIDOnTvjt99+w6pVqzBixAgr946sxSFGXBo1agSVSlVtlX1ubi6CgoKs1Kv6VXWdDfUzSEhIwNdff43vv/8eTZo0keJBQUEoLy9Hfn6+XvmGcN0uLi5o2bIlunTpguTkZHTq1AlLly5t0Necnp6OvLw83HXXXXBycoKTkxN2796NZcuWwcnJCYGBgQ322m/m4+ODO++8E6dOnWrQX+/g4GBEREToxdq2bStNkzX0n2skzyESFxcXF3Tp0gU7d+6UYjqdDjt37kR0dLQVe1Z/wsPDERQUpPcZFBYWYt++fXb9GQghkJCQgE2bNmHXrl0IDw/Xe79Lly5wdnbWu+7MzEycO3fOrq9bjk6nQ1lZWYO+5j59+uDo0aM4fPiwdHTt2hVDhw6V/t5Qr/1mRUVFOH36NIKDgxv017tnz57Vtjf4448/0KxZMwAN9+ca3Ya1VwfXlw0bNgi1Wi3Wrl0rjh8/LsaMGSN8fHxETk6OtbtmMdeuXROHDh0Shw4dEgDEokWLxKFDh8TZs2eFEEIsWLBA+Pj4iK+++kocOXJEDBo0SISHh4vr169bueemGzdunPD29hZpaWni4sWL0lFSUiKVGTt2rGjatKnYtWuXOHjwoIiOjhbR0dFW7LX5pk+fLnbv3i2ysrLEkSNHxPTp04VCoRA7duwQQjTMazbk5ruKhGiY1z5lyhSRlpYmsrKyxM8//yxiYmJEo0aNRF5enhCiYV6zEELs379fODk5iXnz5omTJ0+K9evXC3d3d/HRRx9JZRrizzWqmcMkLkIIsXz5ctG0aVPh4uIiunfvLvbu3WvtLlnU999/LwBUO0aMGCGEuHHr4MyZM0VgYKBQq9WiT58+IjMz07qdNpPc9QIQa9askcpcv35djB8/Xvj6+gp3d3fxyCOPiIsXL1qv0xYwatQo0axZM+Hi4iIaN24s+vTpIyUtQjTMazbk1sSlIV77k08+KYKDg4WLi4u44447xJNPPilOnTolvd8Qr7nK1q1bRfv27YVarRZt2rQR7777rt77DfHnGtVMIYQQ1hnrISIiIjKOQ6xxISIiooaBiQsRERHZDSYuREREZDeYuBAREZHdYOJCREREdoOJCxEREdkNJi5ERERkN5i4EFnJmTNnoFAocPjwYQBAWloaFApFtWfOWNPIkSMxePBga3fjtsLCwrBkyRJrd4OI6gETF2rwRo4cCYVCUe2Ii4uzdtf09OjRAxcvXoS3t3edtlOVICkUCiiVSnh7e6Nz58546aWXcPHiRb2yS5cuxdq1a+u0P5Zw4MABjBkzxtrdIKJ64GTtDhDVh7i4OKxZs0YvplarrdQbeS4uLvX6RNvMzExoNBoUFhYiIyMDr7/+Oj744AOkpaWhQ4cOAFDnSZSlNG7c2NpdIKJ6whEXcghqtRpBQUF6h6+vr/S+QqHA+++/j0ceeQTu7u5o1aoVtmzZoneOY8eO4aGHHoJGo4GXlxfuvfdenD59GsCNJzPPmTMHTZo0gVqtRmRkJFJSUvTq79+/H507d4arqyu6du2KQ4cO6b1/61TR2rVr4ePjg+3bt6Nt27bw9PREXFyc3qhIZWUlJk2aBB8fH/j7+2PatGkYMWJEraZ3AgICEBQUhDvvvBNDhgzBzz//jMaNG2PcuHFSmVuninr37o2JEydi8uTJ8PX1RWBgIN577z0UFxcjPj4eXl5eaNmyJbZt26bX1m+//YZ+/frB09MTgYGBGDZsGC5fvqx33kmTJuGll16Cn58fgoKCMHv2bOl9IQRmz56Npk2bQq1WIyQkBJMmTZLev3Wq6Ny5cxg0aBA8PT2h0WjwxBNPIDc3V3p/9uzZiIyMxP/+9z+EhYXB29sbQ4YMwbVr1277uRGRdTFxIfp/r776Kp544gkcOXIE/fv3x9ChQ3HlyhUAwPnz53HfffdBrVZj165dSE9Px6hRo1BZWQngxpTKW2+9hTfffBNHjhxBbGwsHn74YZw8eRIAUFRUhIceeggRERFIT0/H7Nmz8eKLL962TyUlJXjzzTfxv//9Dz/88APOnTunV2/hwoVYv3491qxZg59//hmFhYXYvHmzSdfv5uaGsWPH4ueff0ZeXp7BcuvWrUOjRo2wf/9+TJw4EePGjcPjjz+OHj16ICMjA3379sWwYcNQUlICAMjPz8cDDzyAzp074+DBg0hJSUFubi6eeOKJauf18PDAvn378Prrr2POnDlITU0FAHzxxRdYvHgx3nnnHZw8eRKbN2+WRoVupdPpMGjQIFy5cgW7d+9Gamoq/vzzTzz55JN65U6fPo3Nmzfj66+/xtdff43du3djwYIFJn12RFSPrPyQR6I6N2LECKFSqYSHh4feMW/ePKkMAPHKK69Ir4uKigQAsW3bNiGEEDNmzBDh4eGivLxcto2QkBC98wkhRLdu3cT48eOFEEK88847wt/fX1y/fl16f+XKlQKAOHTokBDin6d7X716VQghxJo1awQAvacAr1ixQgQGBkqvAwMDxRtvvCG9rqysFE2bNhWDBg0y+Hnc2s7Ntm3bJgCIffv2CSFufHY3n6tXr17innvu0WvPw8NDDBs2TIpdvHhRABB79uwRQggxd+5c0bdvX712srOzBQDpKb63nleIG5/ftGnThBBCvPXWW+LOO+80+Pk3a9ZMLF68WAghxI4dO4RKpRLnzp2T3j927JgAIPbv3y+EECIpKUm4u7uLwsJCqczUqVNFVFSU7PmJyHZwxIUcwv3334/Dhw/rHWPHjtUr07FjR+nvHh4e0Gg00sjD4cOHce+998LZ2bnauQsLC3HhwgX07NlTL96zZ0+cOHECAHDixAl07NgRrq6u0vvR0dG37be7uztatGghvQ4ODpb6VFBQgNzcXHTv3l16X6VSoUuXLrc9ryHi/x8Wr1AoDJa5+XNSqVTw9/fXG/0IDAwEAKmfv/76K77//nt4enpKR5s2bQBAmmq79byA/rU+/vjjuH79Opo3b47Ro0dj06ZN0mjXrU6cOIHQ0FCEhoZKsYiICPj4+EhfD+DG9JKXl5dse0Rku7g4lxyCh4cHWrZsWWOZW5MShUIBnU4H4MY0ijXI9akquagLVf+xh4WFGdWnm2NVSU/VZ1dUVISBAwdi4cKF1c4VHBxc43mrzhEaGorMzEx89913SE1Nxfjx4/HGG29g9+7dsslkbdTUHhHZLo64ENVCx44d8eOPP6KioqLaexqNBiEhIfj555/14j///DMiIiIAAG3btsWRI0dQWloqvb93716z+uTt7Y3AwEAcOHBAimm1WmRkZJh0vuvXr+Pdd9/FfffdZ9G7dO666y4cO3YMYWFhaNmypd7h4eFR6/O4ublh4MCBWLZsGdLS0rBnzx4cPXq0Wrm2bdsiOzsb2dnZUuz48ePIz8+Xvh5EZL+YuJBDKCsrQ05Ojt5x810tt5OQkIDCwkIMGTIEBw8exMmTJ/G///0PmZmZAICpU6di4cKF2LhxIzIzMzF9+nQcPnwYzz//PADg6aefhkKhwOjRo3H8+HF8++23ePPNN82+rokTJyI5ORlfffUVMjMz8fzzz+Pq1as1TvVUycvLQ05ODk6ePIkNGzagZ8+euHz5MlauXGl2v242YcIEXLlyBU899RQOHDiA06dPY/v27YiPj4dWq63VOdauXYsPPvgAv/32G/7880989NFHcHNzQ7NmzaqVjYmJQYcOHTB06FBkZGRg//79GD58OHr16oWuXbta9NqIqP5xqogcQkpKit60BAC0bt0av//+e63q+/v7Y9euXZg6dSp69eoFlUqFyMhIaV3LpEmTUFBQgClTpiAvLw8RERHYsmULWrVqBQDw9PTE1q1bMXbsWHTu3BkRERFYuHAhHn30UbOua9q0acjJycHw4cOhUqkwZswYxMbGQqVS3bZu69atoVAo4OnpiebNm6Nv375ITEy0+F4yVaNR06ZNQ9++fVFWVoZmzZohLi4OSmXtfnfy8fHBggULkJiYCK1Wiw4dOmDr1q3w9/evVlahUOCrr77CxIkTcd9990GpVCIuLg7Lly+36HURkXUoRF1OmBNRvdLpdGjbti2eeOIJzJ0719rdISKyOI64ENmxs2fPYseOHejVqxfKysrw9ttvIysrC08//bS1u0ZEVCe4xoXIjimVSqxduxbdunVDz549cfToUXz33Xdo27attbtGRFQnOFVEREREdoMjLkRERGQ3mLgQERGR3WDiQkRERHaDiQsRERHZDSYuREREZDeYuBAREZHdYOJCREREdoOJCxEREdkNJi5ERERkN/4P5W22K2+mxTsAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Illustration Purpose Only\n",
    "def visualize_pe(pe):\n",
    "    plt.imshow(pe, aspect=\"auto\")\n",
    "    plt.title(\"Positional Encoding\")\n",
    "    plt.xlabel(\"Encoding Dimension\")\n",
    "    plt.ylabel(\"Position Index\")\n",
    "    plt.colorbar()\n",
    "    plt.show()\n",
    "\n",
    "position_encoding_lookup_table2_np = position_encoding_lookup_table[0].cpu().numpy()\n",
    "visualize_pe(position_encoding_lookup_table2_np)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:27:19.636816Z",
     "start_time": "2024-02-09T04:27:18.851615Z"
    }
   },
   "id": "fa43476e35fdb265",
   "execution_count": 18
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2         3         4         5         6         7         8         9   ...        54        55        56        57        58        59        60        61        62        63\n0   1.399742 -0.206918  0.327990  0.748870  0.531874  0.751807  0.351938  0.610762  0.129977 -0.207580  ... -0.547752  2.515671  1.223743  1.833670 -0.404194  1.563555  0.591292  0.275255  1.670105  0.758279\n1   1.407957 -0.561974  2.393893  0.377252  1.083746  0.138066 -0.334590  1.670161  0.329176  2.342795  ...  0.873984  2.226714  0.794747  1.598629  0.884658  1.032520  1.353795  1.059697  1.172643  1.527427\n2   0.482819  1.301215  0.653670 -0.846175  0.628521  1.126829 -0.102938 -0.636202  0.428573  0.553769  ... -0.934116  2.145729 -1.913517  0.552654  0.597746  2.673483 -1.969119  1.397835 -0.438208  0.437077\n3   0.851059  0.379319  0.070685  0.910763 -1.117662  0.325378  0.947827  0.472564  0.516017  0.790074  ...  0.072798  0.264451  0.070916 -1.744750  1.088080  0.002188  0.715526 -0.357311  1.604357  1.920290\n4  -2.720048 -0.354717  0.272903 -0.906938  0.932237 -1.449321 -0.226828 -1.203768  2.488952  2.130765  ...  0.533600  0.432145 -2.389682  0.913403  0.066965  1.655225  0.625080  0.236625 -0.692241  0.992275\n5  -0.147775  0.718796  0.559903 -0.004127 -0.690035 -0.998508  0.331355 -1.222723  0.836060 -1.353497  ... -0.044886 -0.201054 -0.926252  1.322522  0.587324  1.108183 -1.652407  2.918813  0.942309  1.584330\n6   0.038134  3.066611 -1.069607  0.424900 -1.142844 -2.948736  0.505219 -0.617680  1.280667 -0.168857  ...  0.202646  1.051821  1.306703  1.517673  0.050767  1.044631  1.347860  0.678610 -0.477987  0.833080\n7  -1.368717  2.010294 -1.177933  1.943613  0.931115 -2.610583 -0.812628 -1.958096  2.302625  0.242536  ... -2.336618  1.190780 -0.052987  3.281737 -0.415515  0.198294 -1.392472  2.863094 -0.392633  0.868254\n8   3.896336 -0.053480 -1.065470  1.569054 -1.767442 -0.238040 -1.386770 -0.574759  0.118922 -1.069920  ...  3.366446  0.203255  1.759607  2.526687 -0.652322  2.660683  0.966853  1.618786  1.664013  2.768385\n9   1.811860 -2.118048  0.777184  0.642304 -0.407949  0.093468 -0.256171 -1.183092  0.421236 -2.164224  ... -0.543957  2.515664  1.226589  1.833666 -0.402059  1.563552  0.592893  0.275254  1.671305  0.758279\n10 -0.173195 -0.566551 -0.977713  0.379931 -0.595008 -0.661265 -2.512674 -1.641119  2.048560 -0.448827  ... -1.471520  1.216835  0.725749  2.340466 -0.706827  2.097339  0.233203  1.104480 -1.582538  2.481634\n11 -1.030607  0.309522  0.723369  0.637186 -0.141566  2.726367 -2.486894  0.854962 -0.917957 -2.444283  ...  0.308122  0.597250  0.045924  0.127549  1.359100  2.417307 -1.434337  0.254191 -0.240947  0.542596\n12 -1.463599  0.860966 -0.461636  0.274126  1.176311  0.329000 -2.790855  1.193104 -1.473961 -2.029624  ... -0.055479  1.198788  0.546739  0.594932  0.364360  2.217009 -0.566188  2.032797  0.175648  2.400138\n13  1.045454 -0.221277 -2.794333 -0.666396  0.902610  1.350903 -1.489786 -0.188839  0.085725  0.034213  ... -2.388670  0.201825 -0.965606  1.399124 -0.765167  1.607603 -1.889009  2.642380 -0.154336  0.069571\n14 -0.007462  1.160874 -0.672790 -2.764018  0.996131  0.188258 -0.232188 -0.063762 -0.468037  0.291926  ...  0.152741  1.273031  0.311204 -0.205802 -0.798966  0.182580 -0.989119  1.227733  0.583054  1.285420\n15  1.162378 -1.163164 -2.252026 -0.785099  1.221152 -0.297189  1.771679  2.388696 -0.672970 -0.417672  ...  1.367740  2.225356 -0.191733 -0.841328 -0.835119  0.627744  0.116415  0.363205  0.752731  1.664508\n\n[16 rows x 64 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>54</th>\n      <th>55</th>\n      <th>56</th>\n      <th>57</th>\n      <th>58</th>\n      <th>59</th>\n      <th>60</th>\n      <th>61</th>\n      <th>62</th>\n      <th>63</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.399742</td>\n      <td>-0.206918</td>\n      <td>0.327990</td>\n      <td>0.748870</td>\n      <td>0.531874</td>\n      <td>0.751807</td>\n      <td>0.351938</td>\n      <td>0.610762</td>\n      <td>0.129977</td>\n      <td>-0.207580</td>\n      <td>...</td>\n      <td>-0.547752</td>\n      <td>2.515671</td>\n      <td>1.223743</td>\n      <td>1.833670</td>\n      <td>-0.404194</td>\n      <td>1.563555</td>\n      <td>0.591292</td>\n      <td>0.275255</td>\n      <td>1.670105</td>\n      <td>0.758279</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.407957</td>\n      <td>-0.561974</td>\n      <td>2.393893</td>\n      <td>0.377252</td>\n      <td>1.083746</td>\n      <td>0.138066</td>\n      <td>-0.334590</td>\n      <td>1.670161</td>\n      <td>0.329176</td>\n      <td>2.342795</td>\n      <td>...</td>\n      <td>0.873984</td>\n      <td>2.226714</td>\n      <td>0.794747</td>\n      <td>1.598629</td>\n      <td>0.884658</td>\n      <td>1.032520</td>\n      <td>1.353795</td>\n      <td>1.059697</td>\n      <td>1.172643</td>\n      <td>1.527427</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.482819</td>\n      <td>1.301215</td>\n      <td>0.653670</td>\n      <td>-0.846175</td>\n      <td>0.628521</td>\n      <td>1.126829</td>\n      <td>-0.102938</td>\n      <td>-0.636202</td>\n      <td>0.428573</td>\n      <td>0.553769</td>\n      <td>...</td>\n      <td>-0.934116</td>\n      <td>2.145729</td>\n      <td>-1.913517</td>\n      <td>0.552654</td>\n      <td>0.597746</td>\n      <td>2.673483</td>\n      <td>-1.969119</td>\n      <td>1.397835</td>\n      <td>-0.438208</td>\n      <td>0.437077</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.851059</td>\n      <td>0.379319</td>\n      <td>0.070685</td>\n      <td>0.910763</td>\n      <td>-1.117662</td>\n      <td>0.325378</td>\n      <td>0.947827</td>\n      <td>0.472564</td>\n      <td>0.516017</td>\n      <td>0.790074</td>\n      <td>...</td>\n      <td>0.072798</td>\n      <td>0.264451</td>\n      <td>0.070916</td>\n      <td>-1.744750</td>\n      <td>1.088080</td>\n      <td>0.002188</td>\n      <td>0.715526</td>\n      <td>-0.357311</td>\n      <td>1.604357</td>\n      <td>1.920290</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-2.720048</td>\n      <td>-0.354717</td>\n      <td>0.272903</td>\n      <td>-0.906938</td>\n      <td>0.932237</td>\n      <td>-1.449321</td>\n      <td>-0.226828</td>\n      <td>-1.203768</td>\n      <td>2.488952</td>\n      <td>2.130765</td>\n      <td>...</td>\n      <td>0.533600</td>\n      <td>0.432145</td>\n      <td>-2.389682</td>\n      <td>0.913403</td>\n      <td>0.066965</td>\n      <td>1.655225</td>\n      <td>0.625080</td>\n      <td>0.236625</td>\n      <td>-0.692241</td>\n      <td>0.992275</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-0.147775</td>\n      <td>0.718796</td>\n      <td>0.559903</td>\n      <td>-0.004127</td>\n      <td>-0.690035</td>\n      <td>-0.998508</td>\n      <td>0.331355</td>\n      <td>-1.222723</td>\n      <td>0.836060</td>\n      <td>-1.353497</td>\n      <td>...</td>\n      <td>-0.044886</td>\n      <td>-0.201054</td>\n      <td>-0.926252</td>\n      <td>1.322522</td>\n      <td>0.587324</td>\n      <td>1.108183</td>\n      <td>-1.652407</td>\n      <td>2.918813</td>\n      <td>0.942309</td>\n      <td>1.584330</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.038134</td>\n      <td>3.066611</td>\n      <td>-1.069607</td>\n      <td>0.424900</td>\n      <td>-1.142844</td>\n      <td>-2.948736</td>\n      <td>0.505219</td>\n      <td>-0.617680</td>\n      <td>1.280667</td>\n      <td>-0.168857</td>\n      <td>...</td>\n      <td>0.202646</td>\n      <td>1.051821</td>\n      <td>1.306703</td>\n      <td>1.517673</td>\n      <td>0.050767</td>\n      <td>1.044631</td>\n      <td>1.347860</td>\n      <td>0.678610</td>\n      <td>-0.477987</td>\n      <td>0.833080</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>-1.368717</td>\n      <td>2.010294</td>\n      <td>-1.177933</td>\n      <td>1.943613</td>\n      <td>0.931115</td>\n      <td>-2.610583</td>\n      <td>-0.812628</td>\n      <td>-1.958096</td>\n      <td>2.302625</td>\n      <td>0.242536</td>\n      <td>...</td>\n      <td>-2.336618</td>\n      <td>1.190780</td>\n      <td>-0.052987</td>\n      <td>3.281737</td>\n      <td>-0.415515</td>\n      <td>0.198294</td>\n      <td>-1.392472</td>\n      <td>2.863094</td>\n      <td>-0.392633</td>\n      <td>0.868254</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>3.896336</td>\n      <td>-0.053480</td>\n      <td>-1.065470</td>\n      <td>1.569054</td>\n      <td>-1.767442</td>\n      <td>-0.238040</td>\n      <td>-1.386770</td>\n      <td>-0.574759</td>\n      <td>0.118922</td>\n      <td>-1.069920</td>\n      <td>...</td>\n      <td>3.366446</td>\n      <td>0.203255</td>\n      <td>1.759607</td>\n      <td>2.526687</td>\n      <td>-0.652322</td>\n      <td>2.660683</td>\n      <td>0.966853</td>\n      <td>1.618786</td>\n      <td>1.664013</td>\n      <td>2.768385</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>1.811860</td>\n      <td>-2.118048</td>\n      <td>0.777184</td>\n      <td>0.642304</td>\n      <td>-0.407949</td>\n      <td>0.093468</td>\n      <td>-0.256171</td>\n      <td>-1.183092</td>\n      <td>0.421236</td>\n      <td>-2.164224</td>\n      <td>...</td>\n      <td>-0.543957</td>\n      <td>2.515664</td>\n      <td>1.226589</td>\n      <td>1.833666</td>\n      <td>-0.402059</td>\n      <td>1.563552</td>\n      <td>0.592893</td>\n      <td>0.275254</td>\n      <td>1.671305</td>\n      <td>0.758279</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>-0.173195</td>\n      <td>-0.566551</td>\n      <td>-0.977713</td>\n      <td>0.379931</td>\n      <td>-0.595008</td>\n      <td>-0.661265</td>\n      <td>-2.512674</td>\n      <td>-1.641119</td>\n      <td>2.048560</td>\n      <td>-0.448827</td>\n      <td>...</td>\n      <td>-1.471520</td>\n      <td>1.216835</td>\n      <td>0.725749</td>\n      <td>2.340466</td>\n      <td>-0.706827</td>\n      <td>2.097339</td>\n      <td>0.233203</td>\n      <td>1.104480</td>\n      <td>-1.582538</td>\n      <td>2.481634</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>-1.030607</td>\n      <td>0.309522</td>\n      <td>0.723369</td>\n      <td>0.637186</td>\n      <td>-0.141566</td>\n      <td>2.726367</td>\n      <td>-2.486894</td>\n      <td>0.854962</td>\n      <td>-0.917957</td>\n      <td>-2.444283</td>\n      <td>...</td>\n      <td>0.308122</td>\n      <td>0.597250</td>\n      <td>0.045924</td>\n      <td>0.127549</td>\n      <td>1.359100</td>\n      <td>2.417307</td>\n      <td>-1.434337</td>\n      <td>0.254191</td>\n      <td>-0.240947</td>\n      <td>0.542596</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>-1.463599</td>\n      <td>0.860966</td>\n      <td>-0.461636</td>\n      <td>0.274126</td>\n      <td>1.176311</td>\n      <td>0.329000</td>\n      <td>-2.790855</td>\n      <td>1.193104</td>\n      <td>-1.473961</td>\n      <td>-2.029624</td>\n      <td>...</td>\n      <td>-0.055479</td>\n      <td>1.198788</td>\n      <td>0.546739</td>\n      <td>0.594932</td>\n      <td>0.364360</td>\n      <td>2.217009</td>\n      <td>-0.566188</td>\n      <td>2.032797</td>\n      <td>0.175648</td>\n      <td>2.400138</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>1.045454</td>\n      <td>-0.221277</td>\n      <td>-2.794333</td>\n      <td>-0.666396</td>\n      <td>0.902610</td>\n      <td>1.350903</td>\n      <td>-1.489786</td>\n      <td>-0.188839</td>\n      <td>0.085725</td>\n      <td>0.034213</td>\n      <td>...</td>\n      <td>-2.388670</td>\n      <td>0.201825</td>\n      <td>-0.965606</td>\n      <td>1.399124</td>\n      <td>-0.765167</td>\n      <td>1.607603</td>\n      <td>-1.889009</td>\n      <td>2.642380</td>\n      <td>-0.154336</td>\n      <td>0.069571</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>-0.007462</td>\n      <td>1.160874</td>\n      <td>-0.672790</td>\n      <td>-2.764018</td>\n      <td>0.996131</td>\n      <td>0.188258</td>\n      <td>-0.232188</td>\n      <td>-0.063762</td>\n      <td>-0.468037</td>\n      <td>0.291926</td>\n      <td>...</td>\n      <td>0.152741</td>\n      <td>1.273031</td>\n      <td>0.311204</td>\n      <td>-0.205802</td>\n      <td>-0.798966</td>\n      <td>0.182580</td>\n      <td>-0.989119</td>\n      <td>1.227733</td>\n      <td>0.583054</td>\n      <td>1.285420</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>1.162378</td>\n      <td>-1.163164</td>\n      <td>-2.252026</td>\n      <td>-0.785099</td>\n      <td>1.221152</td>\n      <td>-0.297189</td>\n      <td>1.771679</td>\n      <td>2.388696</td>\n      <td>-0.672970</td>\n      <td>-0.417672</td>\n      <td>...</td>\n      <td>1.367740</td>\n      <td>2.225356</td>\n      <td>-0.191733</td>\n      <td>-0.841328</td>\n      <td>-0.835119</td>\n      <td>0.627744</td>\n      <td>0.116415</td>\n      <td>0.363205</td>\n      <td>0.752731</td>\n      <td>1.664508</td>\n    </tr>\n  </tbody>\n</table>\n<p>16 rows × 64 columns</p>\n</div>"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Add positional encoding into the input embedding vector\n",
    "input_embedding_x = x_batch_embedding + position_encoding_lookup_table # [4, 16, 64] [batch_size, context_length, d_model]\n",
    "input_embedding_y = y_batch_embedding + position_encoding_lookup_table\n",
    "pd.DataFrame(input_embedding_x[0].detach().cpu().numpy())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:28:10.855919Z",
     "start_time": "2024-02-09T04:28:10.789425Z"
    }
   },
   "id": "8bc5355ba995b07b",
   "execution_count": 20
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([4, 16, 64])"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Prepare Query, Key, Value for Multi-head Attention\n",
    "X = input_embedding_x\n",
    "query = key = value = X # [4, 16, 64] [batch_size, context_length, d_model]\n",
    "query.shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:28:49.421348Z",
     "start_time": "2024-02-09T04:28:49.395576Z"
    }
   },
   "id": "b49bfcfcb646b81f",
   "execution_count": 21
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.843055 -0.550052 -0.127732 -0.844840 -0.819225  0.063522 -0.187879  1.182711 -1.542636  0.582238  0.123996 -0.023149  0.141312  0.812356 -1.628145  0.312126\n",
      "1 -0.350486  0.484634 -0.140094  0.746821  1.481563 -0.266805  0.312059 -0.710153 -0.557109  0.462987 -0.575465  0.824084 -0.910005 -0.060633 -0.851140 -0.696344\n",
      "2 -0.140998 -0.055692 -0.718412 -0.797249  0.366570  1.513673  0.577341 -0.707247 -0.275017  1.270264 -0.846118 -1.158900  1.652279  0.261800 -0.884520 -0.846511\n",
      "3 -0.244500  0.759894  0.067822  0.388658 -0.178537  1.044808 -0.276759 -0.887757 -0.158977 -0.619547 -0.298364 -0.604880 -1.250720  0.388842 -1.088377  0.081781\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.614938 -0.657467  0.080918 -0.701147 -0.332033 -0.185407 -1.033170  0.447317  0.462455  0.497301 -0.083056 -1.593413  0.454453  0.745469 -1.651096  0.451589\n",
      "1 -0.145782  0.062150  0.549227  0.992860  0.473911 -0.393052  0.089704  0.394661 -0.309076 -0.449691 -0.507969  0.825263 -0.800274 -0.936774 -1.369464  0.705076\n",
      "2  0.238615  1.076501  0.382592 -0.691137 -1.052643  0.575740  0.474749 -0.176139  0.206100  1.265228 -0.710662  0.764700  1.243479 -0.662066 -1.538389 -0.075418\n",
      "3  0.319884  0.242967 -0.233653 -0.966210 -0.693450  0.361394 -1.224569  0.128644 -0.085611  0.866901 -0.032113  0.048993 -0.604088 -1.324803 -1.287755  0.321337\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.477557 -0.665898 -0.062522 -0.844530 -0.971296 -0.579648 -0.250835  0.137787  0.971833  0.627744 -0.350412 -1.107923  1.009746  0.223489 -1.007623  0.238047\n",
      "1 -0.157736 -0.379647  0.224449  0.343057  0.675092 -0.883075  0.771069 -0.058893 -0.942641 -0.543024  0.037494 -0.085867 -1.234599  0.592343 -1.158115 -0.058505\n",
      "2 -0.451328  0.773131  0.400444 -0.951134 -0.099844  0.243306  0.205238 -0.197144  0.122609 -0.414965 -1.047807  2.015589  0.876089 -0.899640 -1.242613 -1.861980\n",
      "3  1.023614  0.791576 -0.105600 -0.760460 -0.164954 -1.103957 -1.514200  0.519381 -0.235128  0.016063 -0.164853 -0.578453  0.694984 -0.830543 -0.427421 -0.185568\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.306343  1.002409 -0.558756 -0.368392 -0.373141 -1.728940  0.253534  0.143261  0.430667  0.641169 -1.484428 -0.382139  0.807904  0.482248 -1.512201  1.232279\n",
      "1 -0.550624 -0.022481  0.110000  0.862089  0.557518  0.142511 -0.161860 -0.929796 -0.371089  0.245111  1.033634  0.591592  0.655191  0.273143 -1.534093 -0.017986\n",
      "2 -0.343869  0.463462  0.655731 -1.096173  0.000537  1.985811  0.916595  0.886440  0.575299  0.540631 -0.838614 -0.029521  1.687372 -0.517567 -0.002811 -0.565382\n",
      "3  0.490431  0.417849  0.480311  0.934108 -0.065339 -0.010034  0.094153 -0.461706  0.424854  0.309975 -0.281256  1.245372  0.686575 -1.852056 -0.234020 -0.428271\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.438333 -0.953410 -0.510782 -0.506377  0.031280 -0.456937  0.935150  0.421769 -0.444609  1.587090  1.122377  0.088356 -0.050521 -0.872715 -0.639425  0.572517\n",
      "1  0.295063 -0.300459 -0.087846 -0.854635 -0.029546 -0.896483  0.200802 -1.218089 -0.877742  0.419937 -0.423555  1.188569 -0.862476 -0.562507 -0.539659 -0.153115\n",
      "2  0.086299 -1.197778 -0.423674  0.552355  0.725277  0.213812  0.077010  0.029639  0.246063  1.194938 -0.297176  0.751607  0.912519  0.577433 -1.155185  0.272354\n",
      "3 -0.901111  0.514844 -0.300310  0.770331 -0.456820 -0.062889 -0.767173  0.156300  0.364622 -0.741814 -0.279991 -0.147554 -0.752332 -0.346314 -0.393168 -0.086922\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.098005 -1.362101 -0.880886 -0.048790 -0.831976 -0.919179  0.458893  0.475047 -0.364164  0.514087 -0.198763  0.427323 -0.647461  0.266048 -0.577928  1.007184\n",
      "1 -0.478354 -0.721252  0.784030  0.061834  0.492232 -0.712863 -0.290083 -0.502997  0.520079  0.264797  0.367906  0.900001 -0.784960 -0.213522  0.282126 -0.722758\n",
      "2 -0.036579 -0.290290  0.008043 -0.143929  0.665982  0.520363  0.824291  0.072430  0.679129  1.127481 -0.546705  0.412062  1.899873 -0.461428 -0.429431 -0.858572\n",
      "3 -1.332145 -0.286588 -0.214228  0.020612 -0.352273 -0.049678 -0.492795 -0.235711  0.168299 -0.326701  0.104917 -0.673956 -0.071659  0.481643  0.049477  0.066592\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.433669 -1.239478 -0.150002 -1.126047  0.151928 -0.831389  0.543729  0.171746 -0.161562  0.301515  0.752866  1.031278  0.243580  0.017311 -1.302813 -0.161804\n",
      "1  2.030139  0.848804 -1.016704  0.075851  0.832878 -0.595107 -0.152181 -0.398114  0.323685  0.558464  1.005251  1.297820 -1.275880 -0.522440 -0.065408 -0.632012\n",
      "2  0.335642 -0.023684 -1.246166 -0.892868  1.718963  1.081414  0.382986  0.042498  1.659014 -1.029562 -0.731215  0.100014 -0.983408  0.299675 -0.636578 -0.885547\n",
      "3 -0.060804  1.340595  0.476056  1.011678 -0.739153  0.736712 -0.095485  1.207922  0.523271 -0.370075  0.505099  1.276299  0.144184  1.645972 -0.691074 -0.852811\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.147221 -1.018187 -0.014174 -1.058182 -0.086709 -0.285635  0.605431  1.288928  0.728841  0.193770  0.235218  0.624430 -1.221081  1.286846 -0.811486 -0.250773\n",
      "1 -0.830173 -0.201015  0.375342  1.265234 -0.471125  0.340549 -0.578870 -0.874984 -1.361585 -0.547903  0.919332  0.737287 -0.343581  0.053766  1.055182 -0.393544\n",
      "2 -0.943106 -0.257088  0.703252 -1.483492  0.573838  1.792996  0.816352  0.868350  1.144592  0.029599  0.419578 -0.108692  0.989305  0.400592 -0.742554 -1.176377\n",
      "3 -0.643927 -0.588140 -0.749730 -0.096672 -0.239110 -0.649397  0.518488  0.144483  1.279443 -1.409347 -0.396471 -0.893814  0.444066  1.756735  0.131318 -1.130873\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.799027  0.856980  0.446163  0.499316 -0.239582  0.088171 -0.536068 -0.025151 -0.556649  0.104557  1.030410 -0.996602 -0.416170 -0.349448 -0.417678  0.926921\n",
      "1  0.276474  0.690336 -1.160764  0.339022  1.380023 -2.299778 -0.115020  0.680937  0.577072 -0.210091  0.209844 -0.105638 -0.505048  0.118918 -0.305055 -0.552583\n",
      "2  0.049127  0.937139 -0.036767 -1.256461  0.315426  1.488370  1.166160  1.563194  0.747862  0.093654 -0.987244 -1.068065  0.889559  0.637340  1.055367 -0.694108\n",
      "3 -0.127101  1.703753 -0.780233 -0.362808 -0.211616 -0.016047 -0.382914  0.765831 -0.267362 -0.231050  1.126518 -0.050696 -0.417823  0.072670  0.054113  0.375692\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.807956 -0.080249  0.028504 -0.205558 -0.510860 -0.623191 -0.371553  1.062170 -2.003340  0.103988  0.828822  0.067705  0.314004  0.260999 -1.272962  0.705449\n",
      "1  0.075933  0.159653 -0.375793  0.480969  1.502070 -0.806800  0.496836 -0.671475 -0.481707  0.587016 -0.592886  0.452936 -0.618359 -0.193267 -0.913532 -1.056499\n",
      "2 -0.166165 -0.111579 -0.476513 -0.971685  0.430499  1.906987  0.882759 -0.295891  0.490625  1.036930 -0.382306 -1.500847  1.169874  0.514438 -0.322263 -1.096552\n",
      "3 -0.605010  1.021955  0.607436  0.416600 -0.263974  1.275973 -0.368588 -0.490732  0.345144 -0.508597  0.101237 -0.505796 -1.383213  0.777961 -0.794481  0.200481\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.975871 -0.355146  0.559124  0.062120 -0.388668 -0.141609 -0.105870  1.215286 -0.553361 -0.017332  1.186850  0.127466 -0.455665 -0.147964 -0.403373 -0.231228\n",
      "1 -0.205101 -0.035606 -0.262179  0.460006 -0.181990 -0.047399  1.185728 -0.185939 -0.446449  0.147912 -0.173577  0.529385 -0.016233  0.513281 -0.778500 -0.917174\n",
      "2 -1.216011 -1.284461  1.441692 -0.500132  0.130558  1.589988  0.537305 -0.073081  0.368621 -0.432179  0.292155 -0.420973  0.415114  0.966622 -0.029067 -0.783478\n",
      "3 -1.276993  0.963070 -0.561037  0.083244 -0.130642 -0.090679 -0.371224 -0.047228  0.877646 -0.731854 -0.649757 -1.325310 -0.018269 -0.085761 -1.308902 -0.753023\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.609152  0.293588  0.681168  0.918692 -0.678428 -0.354196  0.642279  0.126239  0.576536 -0.103225  0.383504 -0.529539 -0.000528 -0.494850 -0.752642  0.999774\n",
      "1  0.014018 -0.073415 -0.514388  0.181423  0.368242 -0.733541  1.133217  0.010973  0.346221 -0.306160  0.369414  0.021123 -0.623327  0.922354 -0.711439  0.181036\n",
      "2 -1.416831  0.690725 -0.128975  0.545911  0.819613  0.555542  0.133320  0.751006 -0.056599 -0.231622 -1.088230 -0.624539 -0.307156  0.812401  0.886276 -0.646983\n",
      "3 -0.565820  0.253910 -0.284727  0.483336 -0.032595 -0.357083  0.346250  0.425751  0.231868 -0.779479  0.215294  0.004534 -0.338554 -0.836733 -1.012783 -1.264050\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.246534  0.311533 -0.668713  0.257104  0.410948 -0.281124  0.246150  0.809893 -0.196731  0.002324  0.671286 -0.184460 -1.098380 -0.228604  0.200689  0.926620\n",
      "1  1.258551  0.512766 -0.600787 -0.730703  0.199514 -1.407976  0.178042  0.112705 -1.211821 -1.171478  0.350218  1.014140 -0.667218  0.248370  0.630296 -0.835714\n",
      "2 -0.033908  0.473868 -0.262070  0.831669  0.498441 -0.002046  0.850105 -0.393790  0.640280 -0.812363 -1.168075  0.823002  0.174674  0.574397 -0.715130 -1.762390\n",
      "3  0.124571 -0.550699 -1.080804 -1.263693 -0.748561 -0.488424 -0.637535  1.093251 -0.212879 -1.900913  0.290618 -0.548380 -0.378159  0.075068 -0.142519 -0.604316\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.045093 -0.017782 -0.036026  0.738201  0.535909  0.540001 -0.284811  0.817065  0.161400 -0.418709  0.505471 -0.807120 -1.493107 -0.354721 -0.677752 -1.364141\n",
      "1 -0.553766 -0.643495 -1.064137  0.517800 -0.125831 -0.866458  0.517786 -0.161137  0.205678 -0.741630  0.237892  0.478060  0.579788  0.911892  0.221268 -0.473353\n",
      "2 -0.975538  0.020212  0.270379 -0.424409 -0.191103  2.134469  0.342933  0.477962  0.237305 -0.928211 -0.706870 -0.156639 -0.178393  1.178581 -0.449019 -1.222883\n",
      "3  0.464261 -0.061849  0.318785  0.414360  1.113342  0.312915 -1.301479  0.484767  0.295944 -1.226489 -0.421380 -1.187196 -1.161356 -0.462495 -0.462336 -0.033009\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.450824 -0.462299  0.040806  0.125190 -0.824344  0.226403 -0.032060  0.119213 -0.317265 -0.176777  0.032324 -0.008321 -1.653131  0.684965 -1.229449 -0.189300\n",
      "1  0.128412  0.410048 -0.352405 -0.017320  0.521036 -0.909860  1.139977 -1.093351 -0.397575  0.403441  0.459461  0.752749  0.028293  0.395985  0.351194 -0.202403\n",
      "2 -0.208809  0.352684 -1.591495 -0.094741  0.688446  0.350263  0.301609 -0.364587  0.342892 -0.051749 -1.301603 -0.089152 -0.648856  0.080372 -1.173354 -0.958096\n",
      "3 -0.127549  0.289107 -0.347003  0.595761  0.312818  0.595343  0.013879  0.676382  0.473617 -0.335544 -0.053533  0.074190 -0.820578  0.144357 -0.719646 -0.173205\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.162300  0.810238 -0.950388  0.148364  0.149366  0.110746  0.224680 -1.256393  0.587622  0.771594 -0.404516 -0.575203 -0.644574  0.833663 -0.296932  0.524873\n",
      "1  0.247992  0.218081 -0.906489  0.290906  0.408525  0.238430 -0.589142  0.493921  0.276924 -0.795749  0.255871 -0.493924 -0.131710  1.504040 -0.143763 -1.431793\n",
      "2 -0.738435  1.368045 -0.962342  0.105499  0.967354 -0.287253  0.924607 -0.144282 -0.480593  0.760399 -0.268804 -0.655696  0.737238  0.279763 -0.736466 -0.631314\n",
      "3  1.751726  0.910328 -0.860379  0.358043 -0.686785 -0.679672 -0.331689 -0.186889 -0.187969  0.584522  1.299885 -0.126923 -0.778028 -0.455524  0.194849  0.312603\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.040592 -1.732430  0.535824 -0.954973 -2.603649  0.957650 -0.246723  0.535815  0.109470  0.249254  0.016516 -0.833666  1.394057  0.508778 -0.962780  0.256198\n",
      "1 -0.393435 -1.428576  0.527408  0.083288  0.718811 -1.246769 -0.440604  0.529822 -0.705668 -0.320694  0.520680 -0.457919 -1.183793 -0.457522 -1.093361  1.068064\n",
      "2 -0.241522  1.396123  0.328996 -0.781264  0.504538  0.107321  0.134646 -0.644498 -0.511961  0.427737 -1.987920 -0.009629  0.166767 -0.461718 -0.606908 -0.372021\n",
      "3  0.210581 -0.098174 -0.696049 -0.555822 -0.827056 -0.590123 -1.126034 -1.177567 -0.730200  0.104929 -0.356334 -0.220717  0.136896 -0.590295 -2.865559 -0.424233\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.411827 -2.483540  0.053123 -0.925263  0.360051  0.623676 -0.124993  0.159468  0.198072  0.594068 -0.474884 -1.280501  0.303108 -0.850220 -0.892376 -0.939363\n",
      "1 -0.044534 -0.204753  0.233314  0.749154 -0.218644 -0.693456 -0.457773  1.001401  1.039590 -0.014641 -0.307821  1.190629 -1.070807 -0.807371 -0.881589 -0.527952\n",
      "2 -0.699358 -1.333792 -0.699475 -0.129278  0.059782  0.764894 -0.056004 -1.113135 -0.499442  0.918923 -1.280703 -0.491481  1.078909 -0.093876 -0.641482 -0.118734\n",
      "3 -0.564800  0.499971 -0.642852  0.397720  0.169384  0.624886 -0.979860 -0.546058 -0.446781 -1.158767 -0.091098 -0.327687 -1.669778 -0.101686 -1.274172 -0.141335\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.068883 -1.115615 -0.357018 -0.015936  0.008345  0.085775 -0.354136 -0.656835  0.423316  0.345923  0.836919 -1.179749 -0.554443 -1.097682  0.860269 -0.158142\n",
      "1  0.009750 -0.282828 -0.829414  0.071886  0.169630 -0.810468 -0.203141  0.086831 -0.448596 -0.323520 -0.166213  0.891237  0.018219 -0.639605 -0.662989 -0.733895\n",
      "2 -0.472119 -0.585412 -0.768696  0.632276  0.115810  0.220249  0.413234 -0.142372  0.506677  0.865187 -1.043125  0.052915  1.086173  0.775785 -0.687548  0.561888\n",
      "3 -0.400620  0.729620 -0.715408  0.108888  0.510725 -0.097342 -0.594547  0.713915 -0.543335  0.048470  0.142754  0.394833 -1.054841  0.061452 -0.944729 -0.458728\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.902081 -1.200709 -0.257382  0.646042 -0.182794 -0.184262 -0.027675  0.217132  0.142002  2.054800 -0.394128 -0.465937  0.745164  0.564093 -1.616877  0.484326\n",
      "1 -0.663733 -0.980381  1.089543  1.456469  0.975607 -0.855795  0.499958 -0.500753  0.047210  0.749712 -0.699079  0.198319 -0.639294 -1.052600 -0.938783  0.392807\n",
      "2 -0.306831  0.724243  0.561730 -0.751851  0.319922  0.967821  0.163208  0.383438  0.316170  1.546890 -0.366850  0.070543  2.103587 -0.546666 -0.496985 -0.860911\n",
      "3 -0.743987 -0.726349  0.046075 -0.221491  0.149525  1.097855 -0.537350 -0.875409  0.714005 -0.008700  0.344593 -0.926214 -1.185889 -1.015363 -1.575312 -0.704233\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.848914 -1.111739  0.006205 -0.039756 -0.933432  0.043748  0.154334  0.911096 -0.581574  0.443663 -0.595020 -0.262369 -0.380455  1.255284 -0.666683  0.853622\n",
      "1  0.815517 -0.364623  0.975614 -0.373926 -0.307815 -0.307191 -0.302004 -1.346682 -0.586013  0.702760  0.458446  0.274101 -0.601604  0.523892 -0.964500 -0.109025\n",
      "2  0.973533  0.140913  0.455739 -0.230147  0.263246  0.627085  1.259960 -0.178967  1.052730 -0.874989 -0.313048  0.577313  0.932426  0.900780 -1.945323 -1.009919\n",
      "3 -0.103244  0.544170 -1.326607 -0.637684  0.144137 -0.298935 -0.438170  0.646421  0.243075 -0.110419 -0.587781 -0.322077 -0.514877 -0.507403 -1.168004  0.435127\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.510558 -0.052268 -1.116502 -0.161902 -0.586372  0.105670  0.028299  0.774327 -0.053025  1.081806 -0.170327 -0.089421  0.502616  0.287767 -1.521247  0.794785\n",
      "1 -0.311080 -1.562692  0.837166 -0.634363  0.307649 -1.948408 -0.372313 -0.605613 -0.936221 -0.192276 -0.056637  0.101307 -0.792975 -0.105986 -0.236257  0.015216\n",
      "2  0.017085 -0.507896  0.899571 -0.455941 -0.362668  1.391112 -0.107061  0.238046 -0.315525  0.932545 -0.898798  0.875802  0.983320  0.222729 -0.074483 -0.998997\n",
      "3 -0.385237 -0.218126 -0.137004 -0.461211 -1.078457 -0.409342 -1.257615 -0.549745 -0.321394 -0.854914  0.355592 -0.779808 -0.950687  0.051155 -0.046016  0.221253\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.889797 -1.164731 -0.700482  0.033665  0.390086  0.031297 -0.490637 -0.085371 -0.487951  0.966926 -0.260996  0.527974 -0.205336  0.379985 -0.240390 -0.151295\n",
      "1  0.280330 -0.805784 -0.540771 -0.075365  0.600227 -0.736507  0.075909 -1.635359 -0.503545 -0.522346  0.632771  0.485401 -0.870348 -0.356595 -0.837994 -0.012191\n",
      "2 -0.942717  0.034204  0.100116 -0.430781  0.131741  1.203927  0.480484 -0.452285 -0.342294 -0.279016 -0.535711 -0.099177  0.673115  0.186452 -1.986973 -1.628827\n",
      "3 -0.553349  1.407444  0.062491  0.135272  0.325641  0.257729 -1.722388  0.102674  0.471423 -0.801807 -0.128275 -1.109145 -0.602943  0.500599 -0.271009 -0.016864\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.093964  0.205285 -0.285254 -1.113492 -1.152514  0.298618  0.117928 -0.059249 -0.927987  0.459642  1.681641 -1.200673 -0.799749  0.008045  0.030887 -0.257713\n",
      "1 -0.932471 -0.084573  0.361824  0.234218  0.319488 -1.438632 -0.029714 -0.626279 -0.282511  0.028903  1.048054  0.950885 -0.160759  0.389284  0.627767 -1.276038\n",
      "2 -0.475502  0.312370  0.484566 -0.332308  0.955019  1.655013  0.669675  0.759300  1.482135  0.827720 -0.718860 -0.228196  1.390810  0.481911  0.028049 -0.151780\n",
      "3 -0.218880  1.172321 -1.219250 -0.444534 -0.235476  0.236778 -0.081136  0.352131 -0.074666 -0.132513  0.780349  0.006259 -0.204236  0.361812 -0.265033 -0.688900\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.830415 -0.279750  0.046583 -0.391788 -0.563857 -0.386821 -0.225548  1.137196 -1.963818  0.181007  0.755685  0.229591  0.377930  0.351241 -1.333537  0.596582\n",
      "1  0.185397  0.247017 -0.415759  0.552577  1.578399 -0.754548  0.363617 -0.764385 -0.492507  0.617592 -0.529309  0.495332 -0.758185 -0.161221 -0.736383 -1.138794\n",
      "2 -0.105419 -0.304523 -0.553586 -1.196757  0.522040  2.052510  0.764300 -0.362917  0.456191  1.061301 -0.505191 -1.449561  1.084323  0.408596 -0.402934 -1.207800\n",
      "3 -0.449628  1.182088  0.634406  0.548405 -0.409534  1.220581 -0.390373 -0.595042  0.331115 -0.611581  0.080876 -0.430741 -1.242464  0.818053 -0.714036  0.302046\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.129658 -0.045261  0.570697 -0.255184  0.144431 -0.933445  0.781100  0.685583 -0.498323 -0.430914  0.485012 -0.771712 -0.414206 -0.440226 -0.081936  0.399950\n",
      "1  0.609679 -0.131552  0.411169 -0.882083 -0.518400 -1.112237 -0.255672 -0.659279  0.042839 -0.307857  0.214738  0.506229  0.242895  0.102200 -0.515571  0.586174\n",
      "2  1.080462 -0.214256 -1.082755  0.432201 -0.256774  1.216463  1.270056 -0.025288  0.998846  0.798230 -0.634019  0.690348  1.320518  0.368305 -1.035058 -1.378777\n",
      "3  0.020668 -0.425955 -0.029217 -0.564785 -0.019478  0.283089 -0.785113  0.626727 -0.789826 -0.741861 -0.760041 -0.227344 -0.559743 -0.552307  0.521503 -0.307743\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.017067 -0.132201  0.005626  0.503344  0.283302 -0.019817 -0.375682  0.887630 -0.013207 -0.690582  1.040520 -1.066494 -0.977409 -0.785577 -0.662742 -0.741300\n",
      "1 -0.899417 -1.241653 -0.731693  0.271986 -0.212329 -1.088669  0.407414 -0.006261  0.089814 -0.797364  0.367399  0.209349  0.372228  0.605199 -0.200268 -0.365062\n",
      "2 -0.846173  0.287940  0.647478 -0.366624 -0.408595  2.297337  0.545167  0.798921  0.669671 -0.767625 -0.397918 -0.263753 -0.029016  1.440027  0.004349 -1.003745\n",
      "3  0.165065 -0.109351  0.492917  0.181230  1.139297  0.234374 -1.204153  0.233503  0.183980 -0.872556 -0.557667 -1.474802 -1.108856 -0.114233 -0.701452  0.033226\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  2.639729  0.432699  0.228840  0.328165 -1.619762 -0.143244 -0.148332  1.446185 -0.136884 -0.225079  1.478134 -0.445745 -0.352616  0.028643 -1.034523  0.454969\n",
      "1 -1.186099 -0.324240 -0.791502 -1.016192  1.248316 -0.972243  1.534065 -0.547292 -0.946302  0.431607  0.267453  0.309295 -0.433166  0.201256 -0.104921 -0.838163\n",
      "2  0.106800  0.043553  0.900919  0.428170 -0.247493  0.183408  1.895423  0.936508  0.222471 -0.774004 -0.092559  0.208208  0.731011  0.268178 -0.025636 -0.487947\n",
      "3  0.084314 -0.803575 -0.570023 -0.064708 -1.060818  0.203365 -0.065024  0.202545  0.112469 -0.145246 -0.513070 -0.679315 -0.260318  0.085924 -1.460547 -1.566682\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.557170 -0.069228 -0.252429  1.265160 -0.494375  0.057511 -0.535007  0.295485  0.095640  0.343272  0.357061 -1.167645 -1.233099  0.834950 -1.384201 -0.601127\n",
      "1  0.260277  0.225782 -0.290534  0.660958 -0.063076 -0.393807  0.589620 -0.713500  0.398319 -0.530572 -0.241501  1.107055  0.356750  1.185921 -0.153039  0.215772\n",
      "2 -0.330623  0.452588  0.956771 -0.085573  0.729110  1.394057  0.861967  0.413198  0.140418 -1.303610 -0.348097 -0.386850  0.105175  0.568483 -0.466887 -1.195699\n",
      "3  0.410844  0.677328 -0.645257  0.401230  0.496405 -0.010768  0.305602  1.510772  0.423297  0.059943  0.245373 -0.200643 -0.773368 -0.625001 -0.626335 -0.814548\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.400794  1.674622 -0.415682 -0.043126 -0.117706 -1.618550  0.164571  0.309570  0.279495  0.378610 -1.406251 -0.275353 -0.007750  0.573838 -1.323963  0.907701\n",
      "1 -0.215844  0.564049 -0.409821  0.822482  0.802106 -0.206013  0.197558 -0.683868 -0.381642  0.043122  1.011164  0.687814  1.032969  0.730056 -1.051225 -0.444363\n",
      "2 -0.378914  0.294372  0.494542 -1.077895  0.473817  2.164143  1.176582  0.797308  0.450885 -0.162525 -0.898714 -0.042541  0.997564 -0.456949  0.199126 -0.993319\n",
      "3  0.643288  0.502156  0.548274  1.268327 -0.210520  0.171023  0.186575  0.253788  0.681656 -0.329138 -0.144567  1.465396  0.942959 -1.870250  0.069844 -0.950228\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.124051  0.371224 -0.305690  1.633266  1.128819 -0.565459  0.768883 -0.629083  0.272960  1.084742 -0.375039 -0.911658 -0.139525  0.008518 -0.125098 -0.617350\n",
      "1  0.061303 -0.101059  0.229803  0.550959  0.169033 -0.269751 -0.631112 -0.705359 -0.034888 -0.472547  0.160753  0.327375  0.070442  1.157730 -0.530561 -0.087982\n",
      "2  0.609980  0.256873 -0.059367 -0.192184  0.232327  1.274882  1.126307  0.743813 -0.913144  1.074244 -0.373726 -0.221155  2.028465  0.966211 -0.987181 -0.694455\n",
      "3  1.318778  0.719581  0.533894  0.456643  0.172207  1.013823 -1.328375  0.407472  0.906544 -0.761099  0.555193  0.784876 -1.695383 -1.376250  0.846673  0.197437\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.346843  0.402914 -0.150677  0.686670 -0.074748 -0.799165 -1.057026 -0.457870  0.001222 -0.029633  0.191230 -1.684285  0.309775 -0.031503 -0.056870  0.114892\n",
      "1  0.501270 -0.394868 -0.925444 -0.100001  0.339064 -1.044209  0.320324  0.616051 -0.516770 -0.791340 -0.093174 -0.221809  0.761773  0.443387 -1.108984  0.406432\n",
      "2 -0.031257  0.680584  0.657833 -0.708211 -0.352712  1.367602  1.522030  0.917488 -0.668881 -0.727117 -0.187182 -0.723699  0.793422  0.353857  0.172010 -0.420275\n",
      "3  0.659493  0.942593  0.302045  0.603299 -0.180087  0.333502 -1.532934  0.614952  0.680199  0.657408  0.167041  0.711187 -0.971582 -0.351090 -1.314679 -0.199633\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.089579 -0.973977  0.785348  0.142430 -0.214122  0.011994 -0.505738  0.275072 -0.760688  0.252204  0.822373 -0.967037 -0.197454  0.037943 -0.570538 -0.011652\n",
      "1  0.334160 -0.007269  0.605975  0.523686  0.846121 -0.731225  0.078269  0.136376 -0.141776  0.233214 -0.746309 -0.068632 -0.161884 -0.851085 -1.308490  0.871895\n",
      "2  0.362325  0.289150 -0.313092 -0.715971  0.189924  1.530835 -0.059266 -1.019142 -0.295478  1.257605 -0.598340 -1.377333  1.552617  0.794436 -1.117288 -0.343388\n",
      "3 -0.733659  0.890256 -0.073768  0.246744 -0.061131  0.390820 -0.930806  0.110049  0.317988 -0.263796  0.176278 -1.281914 -0.835195 -0.446496 -1.792402  0.635732\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.257294 -0.695780  0.121141 -0.177667 -0.508855 -0.030935  0.275189  0.017120 -0.159492  0.637873  0.150516 -0.303546  0.428513 -0.441419  0.376588 -0.236788\n",
      "1 -0.910946 -0.198898  0.122654 -0.736171 -0.036079 -0.905127 -0.353397 -0.145026 -0.461541  0.065215  0.192835  0.078686 -0.315833 -1.149468 -1.861958  1.334020\n",
      "2 -0.168921  0.189825  0.158124 -0.004286 -0.458063  0.967991  0.559503 -0.480577 -0.439033  0.417869 -1.184282  0.057145  1.950150  0.222633 -1.324048 -0.784906\n",
      "3 -0.112746 -0.499071 -0.856497  0.014403 -0.251019 -0.796245 -1.281106 -0.721319 -0.473868 -0.149497 -0.647904 -0.931394 -0.289107 -0.584801 -0.534413 -1.095125\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.068883 -1.115615 -0.357018 -0.015936  0.008345  0.085775 -0.354136 -0.656835  0.423316  0.345923  0.836919 -1.179749 -0.554443 -1.097682  0.860269 -0.158142\n",
      "1  0.009750 -0.282828 -0.829414  0.071886  0.169630 -0.810468 -0.203141  0.086831 -0.448596 -0.323520 -0.166213  0.891237  0.018219 -0.639605 -0.662989 -0.733895\n",
      "2 -0.472119 -0.585412 -0.768696  0.632276  0.115810  0.220249  0.413234 -0.142372  0.506677  0.865187 -1.043125  0.052915  1.086173  0.775785 -0.687548  0.561888\n",
      "3 -0.400620  0.729620 -0.715408  0.108888  0.510725 -0.097342 -0.594547  0.713915 -0.543335  0.048470  0.142754  0.394833 -1.054841  0.061452 -0.944729 -0.458728\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.176404  0.531872  0.409214  0.192468 -1.084885  0.238997 -0.798379  0.188346  0.059492 -0.024330  0.924521 -1.091804 -0.326156  0.291749 -0.732185 -0.054935\n",
      "1 -0.615884  0.863978 -0.437723 -0.152925  1.116533 -0.681281  0.415692 -0.764032 -0.687655  0.304295 -0.105089  0.731977  0.626084  0.591653 -0.146139  0.494095\n",
      "2 -0.189808  0.788168  0.899602 -1.132650 -0.081587  0.835441  0.785523  0.663442  0.063898 -0.166113 -0.850259  0.151737  1.317178 -0.699426 -0.841369 -0.484020\n",
      "3  0.683071  1.052087 -1.006277 -0.075180  0.912621 -0.327234 -1.111313  0.549032  0.110626 -0.140059 -0.477161 -0.342185 -0.293553 -0.850333 -0.414416  0.550237\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.073625  0.150127 -0.907866  0.334299 -0.202801 -0.097351  0.876653 -0.385648  0.045448  2.448797  0.102841  1.521410  0.074046  0.981755 -1.376029 -0.471205\n",
      "1  0.400723  0.219497  0.124874 -0.358635 -0.443369 -0.247101 -0.808554 -0.685117  0.247060  0.634416 -0.134543  0.128437 -0.833682  0.816896 -0.164843  0.732479\n",
      "2  0.263923  0.820781 -0.206039  0.285594  0.476259  0.501493 -0.342558 -0.732048 -0.340279  1.199124 -0.888102  0.372415  1.133199 -0.030477 -1.117813 -1.242966\n",
      "3  0.707586  1.043648 -0.785705 -0.182822 -1.052228 -1.017795 -0.886804 -1.194169  0.279976  0.415883 -0.114251 -1.453649 -0.459333 -0.811049  0.461218  0.328624\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.565246 -0.903284  0.252414  0.429646 -0.713902  1.917740  0.362854  0.713402 -0.210875  0.616729  1.185414  0.169722  0.619079  0.519919 -0.158226 -1.496089\n",
      "1  0.888702 -0.035670 -0.126169 -0.364905  0.810807 -0.035866 -0.105930  0.211207 -0.261469 -0.215912 -1.080510  0.501321 -0.848003  0.389684  0.131101 -0.584177\n",
      "2 -0.815836  0.537262  0.411533  0.070017  0.632414 -0.549836  0.097125 -0.331563  0.368747 -1.247963 -0.250107  0.091148  0.837426  0.622526 -0.783870 -0.812426\n",
      "3  0.127784 -0.015729 -1.144995 -0.323159 -0.314418 -0.153439 -1.000394  0.642610  0.306637  0.099248  0.058967 -0.746932 -0.273949  0.180222 -1.573224 -0.010142\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.255211 -0.157632 -0.574581 -0.166197  0.775349 -0.140706  0.687652  0.807035 -0.184444  0.656217  0.474147 -0.447196  0.060639 -0.188234 -1.242352 -0.329322\n",
      "1  0.488382 -0.108822 -0.275919 -0.090759 -0.832167  0.028082 -0.041036 -1.339602  0.134018  0.438956  0.426242  0.013138 -0.717898 -0.010739  0.128523  0.400315\n",
      "2  0.567573 -0.723301  0.511999 -0.188116 -0.571319  0.991571  0.793728  0.696589 -0.254946 -1.087185 -0.457050  0.546511  0.099871  0.519373 -1.122454 -1.564487\n",
      "3  0.835880 -0.165621  0.295777 -0.285468  0.093285  0.132677 -0.379293  0.561905  0.621888 -0.311707 -0.840513 -0.725356 -0.189057  0.202919  0.051076 -0.613800\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.236179 -1.009632 -0.326470 -0.297060 -0.118641  0.074772  0.023244 -0.203720  0.170052  0.070126  1.334358 -0.704639 -0.528730 -1.149122  0.729896  0.029149\n",
      "1  0.322406 -0.267601 -0.963115 -0.098632  0.339756 -1.158210 -0.381949  0.085271 -0.557520 -0.320726  0.213669  0.647582 -0.255545 -0.395816 -0.170296 -1.198567\n",
      "2 -0.267999 -0.983763 -0.655108  0.191769  0.576468  0.802856  0.599542 -0.029581  0.807375  0.281877 -0.961682  0.093886  0.208873  0.895718 -0.360856 -0.000831\n",
      "3 -0.354376  1.055429 -0.356498  0.454247 -0.083929 -0.238622 -0.499768  0.774745 -0.215797 -0.343473  0.161658  0.368649 -0.487320  0.695373 -0.789305 -0.547663\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.038222 -0.280386 -0.843706 -0.616376  0.294691  0.285665 -0.341028  0.934008 -0.646730 -1.280372  0.236532  0.280084 -0.841593  0.712234 -1.591673  0.064760\n",
      "1  0.129691  0.112961 -0.562535  0.463562  0.526540 -0.433082 -1.213341 -0.475484  0.278862 -0.680838  0.434830  1.604740  0.338845  0.895621  0.145595 -0.223971\n",
      "2 -0.329862  0.591885 -1.080422 -1.282320  0.325834  1.647089  1.472171 -0.050331  0.198912 -0.014550 -0.356193 -0.371771  0.288919  0.205028  0.371042 -0.932187\n",
      "3  0.546775 -0.186697  0.209333  0.978451  0.718924  0.990332  0.238876  0.500508 -0.173352 -1.233035 -0.107134 -0.610185 -0.074153  0.489221  0.113718  0.280216\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.306279 -0.150934  0.773144  0.595103 -0.325866 -0.395447 -0.256868  0.796103 -0.280283 -0.303306  1.391957 -1.493501 -0.704256 -0.555533 -1.038594 -0.122333\n",
      "1 -0.061770 -0.297525 -0.457170 -0.168390  0.302301 -1.931828  0.410539  0.788897  0.600575 -0.890034 -0.427509 -0.468240  0.590850  1.146330 -0.039833 -0.277839\n",
      "2 -0.456495  1.298474  0.484908 -0.122877  0.398217  1.352437  1.198192  0.608493  0.803741  0.279618 -0.096278  0.157943 -0.848379  0.831901  0.467420  0.236334\n",
      "3 -0.263925  1.021767  0.642150  0.543245  0.314341 -0.524004 -0.234829  1.024676  0.541816 -1.119100  0.150054  0.552932 -0.928476 -0.283212 -0.447764 -0.602209\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.169938 -0.497667 -0.325872  0.179360  0.028744 -0.469345 -0.364128 -0.387989  0.060916 -0.149403  1.493312 -1.122818 -0.762886 -1.368208  0.920178  0.245208\n",
      "1  0.088814 -0.410940 -0.922853 -0.260886  0.199503 -1.318330 -0.001653  0.331418 -0.532446 -0.429147  0.038133  0.582173  0.127124 -0.449546 -0.571451 -1.034238\n",
      "2 -0.412180 -0.528039 -0.503837  0.738685  0.370084  0.470210  0.903698  0.125189  0.880732  0.189189 -0.687172 -0.031385  0.387594  1.141662 -0.148143  0.251607\n",
      "3 -0.714183  0.683091 -0.422763  0.152439  0.290620 -0.064837 -0.431068  1.105868 -0.195865 -0.142729  0.210017  0.234452 -0.808928  0.553445 -0.942682 -0.842847\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.609152  0.293588  0.681168  0.918692 -0.678428 -0.354196  0.642279  0.126239  0.576536 -0.103225  0.383504 -0.529539 -0.000528 -0.494850 -0.752642  0.999774\n",
      "1  0.014018 -0.073415 -0.514388  0.181423  0.368242 -0.733541  1.133217  0.010973  0.346221 -0.306160  0.369414  0.021123 -0.623327  0.922354 -0.711439  0.181036\n",
      "2 -1.416831  0.690725 -0.128975  0.545911  0.819613  0.555542  0.133320  0.751006 -0.056599 -0.231622 -1.088230 -0.624539 -0.307156  0.812401  0.886276 -0.646983\n",
      "3 -0.565820  0.253910 -0.284727  0.483336 -0.032595 -0.357083  0.346250  0.425751  0.231868 -0.779479  0.215294  0.004534 -0.338554 -0.836733 -1.012783 -1.264050\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.089952  0.067533  0.085976 -0.181909 -0.242819 -0.559610 -0.495977 -0.046075 -0.098181 -0.457995  0.522625 -1.265378 -1.320826 -0.209618  0.640950  0.411471\n",
      "1  1.201720  0.347223 -0.412666 -0.487666 -0.788717 -0.983542  0.471513 -0.094740 -1.259613 -0.879070  0.363518  0.810040  1.047161  0.236667 -0.032963 -0.061229\n",
      "2  0.425411 -0.076583  0.659215  0.843612  0.921594  1.298184  0.342010  0.924739  1.574549 -0.853615 -0.380778  0.607227 -0.442868  1.298039  0.039827  0.212155\n",
      "3 -0.287385  0.249837 -0.788417 -0.484166  0.454116 -0.390626 -0.147466  1.692854 -0.124048 -0.319251 -0.351036  0.625752  0.184101 -0.575827 -0.779026 -0.565858\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.329653  0.529794 -0.331515  0.677400 -0.131367 -0.555976  0.278185 -0.807618 -0.061015  1.134797 -0.002193  0.735074 -0.201729 -0.046618 -0.306439  0.227947\n",
      "1  1.058406  0.809351 -1.344075 -0.246845  0.175689 -0.245918  0.350410 -0.492653 -0.494349 -0.492613  0.239605  0.506383  0.320137  0.692919 -0.615415  0.188230\n",
      "2  0.185587  1.260174  0.250561 -0.299587  1.242047 -0.277770  0.612673  0.279593 -0.020054 -0.857408 -0.524378 -0.238635  0.308315  0.027083  0.397161 -0.839262\n",
      "3  1.383054  0.896572 -0.590994  0.132179 -0.037870 -0.235835 -0.011826  0.426327  0.076197  0.507105 -0.080371 -0.052626  0.483993 -0.217879  0.242060 -0.172769\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.584967  0.036679 -0.297522  0.097387  0.485003 -0.238667  0.350071 -0.361201 -0.267589  0.658243  0.040019 -0.734725 -0.959120 -0.350231  0.195392  0.406447\n",
      "1  0.852087 -0.366939 -0.282480  0.265795 -0.002569 -0.844916 -0.104893 -0.690379 -0.224831 -0.746555  1.011040  1.010993 -0.554457  0.348798  0.947264 -0.397369\n",
      "2 -0.200723 -0.304258 -0.723998  0.231694  0.869864  1.757347  0.229796  0.930889  0.621773  0.163052 -0.344292 -0.835321 -0.158639  0.378007 -0.990519 -1.116215\n",
      "3 -0.100365  0.223015  0.344705 -0.073661 -0.639542  0.149555 -0.734439  0.478714 -0.072013 -0.749258  1.757566  0.081617 -1.066923 -0.352199  0.226350 -0.415477\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.431029 -0.389901 -0.247779  1.057589 -0.214162 -0.377291 -0.831226 -0.075418 -0.452601  0.135655  0.525034 -0.443614 -1.325111  0.056101  0.228472 -0.178676\n",
      "1  1.723565  0.278511 -0.016063 -1.201930  0.733769 -1.793392 -0.558585  0.130216 -0.274069  0.038803 -0.863897  0.813440  0.919791  0.494655  0.573374  0.027652\n",
      "2  0.879813 -0.209875  0.213537  0.656168  0.473445  0.980684 -0.236595 -0.546739  0.569917  1.428659 -0.737947  0.451540  0.750646  0.683225 -0.826336  0.000961\n",
      "3  0.036668  0.604981 -0.078725 -0.471874  0.906546  0.596679 -0.927565  0.293336 -0.211209 -0.397249  0.382137  0.309855 -1.381001 -0.803773 -0.138390  0.917402\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.362992 -0.324788 -0.199897 -0.665557 -0.714832  0.774445  0.666566  0.487251  0.055709  1.294657 -0.629741 -0.634790 -0.727247  0.827464 -0.410294  0.111627\n",
      "1 -0.726221 -1.016504  0.375666 -0.016473 -0.293093 -0.453114  0.620295 -0.496051 -0.858718 -0.680479  0.327625  0.234066 -0.865707  0.594118 -1.148260  0.229093\n",
      "2  0.585169  0.930731 -0.295074  0.362957  0.147258  0.483375  1.154191 -0.452694 -1.047820 -0.006229 -0.810283 -0.388122  0.672026  0.566222 -1.201632 -0.011228\n",
      "3  0.786654  0.380319 -0.605672  0.383441 -0.101051  0.076134 -0.676884 -0.484011  0.056946 -0.020415 -0.457473 -0.753593 -0.350079 -0.780334 -1.821025 -0.593597\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.176822 -2.470879 -0.147898 -0.274015  0.651451  0.796163 -0.673211  0.159244  0.584264  0.825909  0.290970 -0.608795 -0.352897 -0.225098 -0.972970 -0.860101\n",
      "1  1.448978 -0.258498 -1.089631  0.271407  0.727970 -0.072184  0.364827  0.583580 -0.054674  0.277727 -0.891726  1.263470 -0.313823  0.171239 -0.670323 -0.375938\n",
      "2 -0.203743  0.770183  0.648911 -0.115609 -0.092174  1.593996  0.051193 -0.894547  0.432105 -0.469846 -0.743074  0.249704  0.485517  0.563877 -0.916904 -0.435618\n",
      "3  0.720368  0.267263  0.131167  0.399877  0.984353  0.196040 -0.693219  0.515869  0.182996 -0.152200 -0.281154 -0.016874 -0.803759 -0.564829 -0.649378  0.401524\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.308425 -0.539138  0.259385  0.296139 -0.440926  0.288650 -0.458033 -0.505212  0.679706  0.771421  0.926825 -0.857782 -0.079759 -0.741500 -1.539279 -0.015836\n",
      "1 -0.062496  0.018222 -0.693836  0.729388  1.335306 -0.897933 -0.791998 -0.767458 -0.063126  0.372156 -0.413532  0.537594  0.143167 -0.032629 -0.408854  0.525611\n",
      "2 -0.563814  0.754812  0.150471 -0.752292 -0.196089  0.811713  0.596193 -0.068530  0.573009  1.598698 -0.534681  0.204389  1.164141  0.366169 -0.638497  0.336121\n",
      "3  0.018690  0.668563  0.362883 -0.175717  0.777268  0.265388  0.305292  0.003245  0.814834  0.119982  0.105408  0.698817 -0.878941 -1.288623 -0.465074 -0.137698\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.633797 -0.141075  0.139497  0.358901 -0.697078 -0.262024  0.213127  0.320573 -0.346077  1.143742  0.120111 -0.583446  0.061218  0.039754  0.213620 -0.518166\n",
      "1  0.244168 -0.542625  0.169464 -0.562253 -0.065921 -1.249915  0.501507 -0.605107 -0.280301  0.933891 -0.708358  0.235843 -0.427313  0.163050 -0.979141  0.109747\n",
      "2  0.581454  0.309372 -0.450554  0.661134 -0.098451  0.071689  0.438015 -0.642685  0.419394  1.336107 -0.757630  0.287456  2.246995 -0.247059 -0.576627 -0.452991\n",
      "3  0.024407  0.043233 -1.071366 -0.464482 -0.056488  0.196217 -1.077691 -0.171896 -0.610767 -0.282540  0.311062 -1.076181 -0.833077 -0.622530 -0.472522 -0.361623\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.471197  0.088107 -0.383323 -0.223166 -0.469665 -2.372031  0.272508  0.037552  1.050081  1.027021 -0.132061 -0.094782  1.077780  0.878793 -0.239834 -0.230372\n",
      "1  0.029179 -0.375825  0.352634  0.852334  0.187841 -0.277456  0.070253  0.059308 -0.135633  0.185454 -0.571119  0.115326 -0.333283 -0.260872 -0.540465  0.015298\n",
      "2 -0.107907  0.848008 -0.207849  0.075016 -0.292608 -0.287172  0.107562  1.009907  1.871122  0.219246  0.006427  1.198960  1.336066  0.160271 -1.012163 -1.876254\n",
      "3  0.513124  0.250150 -0.362078 -0.456548 -0.820738 -0.722619 -0.163761  0.326078 -0.196407  0.575446  0.202406 -0.401170  0.376073 -0.906170  0.381597 -0.071292\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.467125 -0.984563 -0.257064 -2.412939 -0.639123 -0.345935  0.859396  1.475097 -0.012862  0.375938  0.619530 -0.325560  0.118446  0.458732 -1.207592  0.147781\n",
      "1  1.088158  0.810464  0.807675  0.654824  0.703983 -0.603323  0.043721 -0.270394 -0.900159  0.019439 -0.293724  1.303474 -0.680663  0.589663  0.252987 -0.994002\n",
      "2  0.292755 -0.045952 -0.128179 -0.426169  1.002135  1.233764  0.672043 -0.698936  0.949019 -0.219085  0.124727  0.918181  1.181413 -0.125518 -0.416058 -1.771814\n",
      "3 -0.242158  0.389674 -0.669892 -0.280750 -0.688672  0.363253  0.630064  0.664664  0.136880 -1.421761 -0.212463 -0.382077  0.715042  0.261851 -0.037906  0.047845\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.059226 -0.558208 -0.249727 -0.602460 -0.761082 -0.116131  0.261581  1.301103 -1.791569  0.495411  0.351794  0.675023  0.378143  0.823263 -1.746921  0.506247\n",
      "1  0.274707  0.351110 -0.267733  0.665270  1.453197 -0.418145  0.193962 -1.205970 -0.500388  0.785514 -0.261972  0.550507 -1.111312 -0.008571 -0.504501 -0.966200\n",
      "2 -0.096769 -0.546517 -0.722104 -1.146778  0.559480  1.752102  0.669178 -0.494542  0.156485  0.845991 -0.670795 -1.131252  0.897950  0.253173 -1.034849 -1.355884\n",
      "3 -0.536298  1.018879  0.436880  0.469634 -0.633801  0.954112 -0.318205 -0.859970  0.359073 -0.614701 -0.042932 -0.458950 -1.168430  0.884767 -0.871933  0.298743\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.610654 -1.370669 -0.277214 -0.888282 -0.160174  0.249848  0.726394 -0.309036  0.245068 -0.293404  0.884923 -1.118040 -0.858950 -0.627054 -1.436908 -0.067506\n",
      "1  0.547155 -0.337403 -0.414298 -0.212033 -0.145155  0.207373  0.381346  0.100462  0.709092  0.317103 -0.289169  0.348626 -0.340906  1.027436  0.504305 -0.969035\n",
      "2 -0.273024 -0.651273 -0.300382 -0.134794 -0.397924  1.016577  0.910840 -0.153315  0.340498 -0.124381 -0.059416 -0.479783  0.712562  0.236867 -1.093192 -0.390130\n",
      "3  0.162120  0.305357  0.109529 -0.471994  0.701465 -0.630411  0.013097  0.355996  0.202033  0.387094 -0.159123 -0.428632 -1.473287  0.255073 -0.325924  0.773818\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.858824 -0.177800 -0.352326 -0.744648 -0.964368 -0.167893 -0.971616  1.283839  0.319020  0.239411  0.495465 -0.076289  0.106109  0.246583 -0.913282 -0.194536\n",
      "1  0.298962  0.576147  0.073836 -0.247572  0.533223 -1.038875  0.314859 -0.377404 -0.484762 -0.297289 -0.050916  1.660669  0.520465 -0.142441 -0.137322 -1.251222\n",
      "2 -0.033523 -0.829220 -0.304602 -0.110416 -0.150082  1.594558  0.274587  0.676428  1.243627  0.277391 -0.565361 -0.421763 -0.036186 -0.461395  0.704446 -0.372949\n",
      "3  0.148561  0.996812 -0.108721  0.710904 -1.185102  1.112566 -0.759846  0.024740 -0.183061 -0.134591 -0.500687 -0.171026 -0.172277 -0.232857 -0.280900  0.070325\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.488703 -0.725229  0.173900 -0.830414  0.213070 -0.718127 -0.444598  0.672499 -1.035795  0.015452  0.295015 -0.910450 -0.307357  0.305210 -0.243458  0.940930\n",
      "1  1.042728 -0.129670  0.313524 -1.872151 -0.012815 -0.607496  0.371844 -0.861372 -0.658208 -0.640721  0.625612  0.016250 -0.624924 -0.071195  0.438702 -0.223898\n",
      "2  0.565551  0.743168  0.227358  0.633667  0.500689  1.503283  0.044266  0.352887  0.546992  0.214653  0.130992 -1.011248 -0.438571 -0.165587 -0.744111 -0.907086\n",
      "3  0.286262  0.421306  0.353233 -0.335494 -0.862758 -0.010216 -1.786810  0.390620  0.424009 -0.530579  0.090104 -1.364448 -0.198621 -0.126026  0.187003 -0.110236\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.683051  0.255799  0.749018  0.855960 -0.666782 -0.468342  0.535272  0.089472  0.529222 -0.168698  0.622150 -0.635523  0.240684 -0.716005 -0.665860  1.135421\n",
      "1 -0.067731 -0.270443 -0.465293  0.120904  0.368781 -0.819422  1.016821  0.149845  0.338049 -0.305068  0.320750 -0.081365 -0.627034  0.785185 -0.829301  0.136954\n",
      "2 -1.385310  0.756915  0.017838  0.404852  0.797507  0.749105  0.096549  0.821897  0.128030 -0.020867 -0.998237 -0.762568 -0.179816  0.874281  1.155172 -0.573110\n",
      "3 -0.496411  0.383706 -0.159284  0.536126 -0.037892 -0.341835  0.271312  0.318583  0.227475 -0.699325  0.258262 -0.061990 -0.343047 -0.732064 -0.997765 -1.098356\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.590556 -0.126418 -0.008332 -0.129837 -0.218032  0.165845 -0.376837  0.704664 -0.395220 -0.282819 -0.006145 -0.243167 -1.332027  1.710084 -0.399444  0.197928\n",
      "1  0.804329  0.749667  0.707012 -1.197384  0.487677 -1.247303  0.620496 -0.566080 -0.653210  0.731915 -0.119785 -0.021137  0.388589  0.057226 -0.883791 -0.808471\n",
      "2  0.709311  0.215274 -0.333312  1.049628 -0.046226 -0.605354  1.996749  0.004057  0.490174  0.132027 -0.014141  1.074840  0.392585 -0.311970 -1.730481 -0.528037\n",
      "3 -0.374562 -0.796446 -1.307556 -0.275224 -0.090963  1.311741  0.028888  1.471485  0.982600 -0.028525 -1.000560 -0.926102 -0.157814 -0.307164  0.156203 -0.537111\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  0.401719  0.425182 -0.292916 -0.701397 -0.326698 -0.279441 -0.798718  0.513086  0.320014  0.617820 -0.086840 -0.417280 -1.574415 -0.074147  0.181661 -0.622625\n",
      "1  0.125515  0.108487 -0.512970 -0.430991 -0.051895 -2.067563  1.478227 -0.140694 -0.479147 -0.896122  0.062240  0.174992 -0.011013 -0.770407 -0.188411 -0.085206\n",
      "2  0.193214  0.273297 -0.967667  0.689172  0.477827  0.713157  0.815838  0.416669  0.088426  0.044879 -0.690438 -0.935450  0.204611  0.335932  0.014221  0.127856\n",
      "3 -0.343215  0.089352 -0.411378 -0.590583 -0.172385  0.425989 -1.009057  0.996905 -0.125823 -0.892946  0.006623  0.012984 -1.091260 -0.097457 -0.360760 -0.812595\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0  1.215814 -0.422020  0.019028  0.565088 -0.494545  0.823563 -0.480298  1.053373 -0.271921  0.653941  0.552066 -0.897783 -1.794641  0.332980 -1.265209  1.206126\n",
      "1  0.164707 -0.261773  0.114364  0.238532  0.831699 -0.905116  0.107201 -0.855621 -0.653122 -0.311885 -0.033229  0.752663 -0.481584  0.115888 -0.548276 -0.652191\n",
      "2  0.416088  0.351545  0.118157 -0.563940  0.611703  0.765792  1.143817 -0.260738  0.213900  0.278640 -0.615235  0.565911  0.454787 -0.064289 -0.472896  0.166102\n",
      "3 -0.358502  0.241573 -0.537251 -0.294055  0.697770 -0.024773 -0.013605  1.368784  0.674618 -0.099743 -0.501064 -0.418571  0.113558 -0.607647 -1.413939 -0.550027\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.030005  0.560489  0.692207  0.802440 -1.049383 -1.130618 -0.233082 -0.138613  0.441630  0.025435 -0.203886 -0.245355 -0.877590  0.185147  0.144022 -0.151590\n",
      "1  0.886263  0.015150 -0.640962 -0.522613  0.534007 -2.012024  0.963597 -0.149933 -0.269668  0.051612 -0.241905  0.215622  0.793468  0.193036 -0.299267 -0.301879\n",
      "2 -0.777996 -1.183819 -0.050823  0.102093  0.829035  1.152996  0.128101  1.013316  1.131896  0.165107 -0.910276 -0.281369  0.560279  0.269381 -0.152854 -1.395591\n",
      "3 -0.647185  1.049913 -0.477628  0.617693 -0.168791  0.307628 -0.439353  1.122450 -0.519324 -0.404857  0.109275  1.548271 -1.384961 -0.262460  0.355068  0.332892\n",
      "         0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n",
      "0 -0.070381 -0.865200 -1.882663 -0.408366 -0.598919  0.541644  0.638400  0.586413  0.963336  0.899848 -0.811803 -0.428355  0.283824  0.715828 -0.689767 -1.133214\n",
      "1  0.163508  0.148884 -0.098002  0.161132 -0.572744 -0.657556 -0.488363 -0.237395 -0.239877  0.452628 -0.079612 -0.221905 -0.779132  0.257819 -0.168416 -1.584344\n",
      "2  0.159060 -0.572794 -0.154949  0.059427  0.620815 -0.265221 -0.102914 -0.490620 -1.777335  0.243808 -1.639493  0.448959  0.437344  0.562454 -0.772508  0.010445\n",
      "3  0.485404 -0.012398 -1.315031  0.238375 -1.477421 -0.787531 -1.435814 -0.116352  0.708661 -0.874855  0.670505 -0.339437 -0.194216 -0.594283 -0.826917 -0.267774\n",
      "torch.Size([4, 16, 4, 16])\n"
     ]
    }
   ],
   "source": [
    "# Define Query, Key, Value weight matrices\n",
    "Wq = nn.Linear(d_model, d_model)\n",
    "Wk = nn.Linear(d_model, d_model)\n",
    "Wv = nn.Linear(d_model, d_model)\n",
    "\n",
    "Q = Wq(query) #[4, 16, 64]\n",
    "Q = Q.view(batch_size, -1, num_heads, d_model // num_heads)  #[4, 16, 4, 16]\n",
    "\n",
    "K = Wk(key) #[4, 16, 64]\n",
    "K = K.view(batch_size, -1, num_heads, d_model // num_heads)  #[4, 16, 4, 16]\n",
    "\n",
    "V = Wv(value) #[4, 16, 64]\n",
    "V = V.view(batch_size, -1, num_heads, d_model // num_heads)  #[4, 16, 4, 16]\n",
    "\n",
    "# print(torch.round(Q[0] * 100) / 100)\n",
    "qqq = Q.detach().cpu().numpy()\n",
    "for qs in qqq:\n",
    "    for qss in qs:\n",
    "        print(pd.DataFrame(qss))\n",
    "\n",
    "print(Q.shape) # [4, 16, 4, 16] [batch_size, context_length, num_heads, head_size]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:28:52.125568Z",
     "start_time": "2024-02-09T04:28:51.769196Z"
    }
   },
   "id": "37ba89375b2f44ca",
   "execution_count": 22
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Transpose q,k,v from [batch_size, context_length, num_heads, head_size] to [batch_size, num_heads, context_length, head_size]\n",
    "# The reason is that treat each batch with \"num_heads\" as its first dimension.\n",
    "Q = Q.transpose(1, 2) # [4, 4, 16, 16]\n",
    "K = K.transpose(1, 2) # [4, 4, 16, 16]\n",
    "V = V.transpose(1, 2) # [4, 4, 16, 16]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:29:30.525962Z",
     "start_time": "2024-02-09T04:29:30.508654Z"
    }
   },
   "id": "a6b2187b2d310a2e",
   "execution_count": 23
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n0   0.827912  0.872087  0.805252  0.638937  0.403635 -0.220579  0.074281  0.066307  0.161097  0.776864 -0.015753  0.059171 -0.167397 -0.519080 -0.113654  0.370162\n1   1.123528  0.464093  0.514921 -0.184883  0.292600 -0.127225  0.205937  0.207778  0.602376  1.016719  0.158707  0.074483  0.526718  0.460966  0.633765  0.534267\n2   1.040623  0.605709  0.119165 -0.133348 -0.169015 -0.146091  0.444743  0.050206  0.774002  0.830742 -0.216136 -0.111672  0.751038  0.496729  0.747770  0.609336\n3   0.900045  0.596818  0.359724  0.836026  0.074438  0.652813  1.057635 -0.026944  1.124048  0.971823 -0.467323 -0.247706  0.002623 -0.360194  0.017813  0.965252\n4   1.061119  1.158659  0.633724 -0.320907  0.383561 -0.448183 -0.238576 -0.078154 -0.033891  0.852194  0.056048  0.533376  0.968322  0.675573  0.506570  0.651593\n5   0.071058  0.273445  0.044782  0.250272 -0.114290  0.121239  0.472959  0.086112 -0.077087 -0.158122 -0.416886  0.212225  0.220158 -0.492304  0.175022  0.505993\n6   0.897961  1.322748  0.553525  0.079147  0.502846 -0.506932  0.223247 -0.400829  0.203914  0.658943 -0.018270  0.620021  0.600703  0.066074  0.316328  0.332746\n7   0.025299  0.779429  0.627483 -0.354743  0.218848 -0.055096  0.297431 -0.119230 -0.025749  0.031660  0.079259  0.274235  0.095197 -0.625557 -0.000797  0.226548\n8   0.529123 -0.196815  0.232964  0.031045  0.214278 -0.328799 -0.685732  0.060671  0.237945  0.642808  0.532076  0.191681 -0.066689  0.383821 -0.080174  0.330105\n9   0.935247  0.690427  0.511019  1.063499  0.504674 -0.227777 -0.035515 -0.006277  0.174739  0.899135  0.184269  0.238904 -0.084467 -0.414828 -0.173580  0.431657\n10  0.467166  0.532110  0.238954  0.085494  0.207894 -0.739952 -0.371635 -0.087244 -0.217038  0.353716  0.276079  0.202890  0.124148 -0.118437 -0.063674  0.186508\n11  0.016316 -0.238568 -0.610145 -0.418894 -0.609519 -0.072247 -0.179781 -0.085679  0.529486  0.309438  0.002346 -0.154983  0.340535  0.199168 -0.240800  0.511496\n12  0.210837  0.166079  0.430884  0.002882  0.447536  0.287130 -0.069169  0.463648 -0.251514  0.225306  0.302280  0.252891  0.182381 -0.017385 -0.036945  0.483606\n13 -0.131177  0.249064  0.690767 -0.918174  0.273945 -0.226493 -0.467489  0.787192 -0.527138 -0.107953  0.469084  0.095217  0.443432  0.514454  0.195352  0.647130\n14 -0.268342  0.041401  0.530629 -0.301306  0.030230 -0.376372 -0.326081  0.092524 -0.112594 -0.274940  0.107115  0.171046 -0.394140 -0.418468 -0.190215  0.442426\n15 -0.219957 -0.423727  0.599078 -0.366324  0.452232  0.868501 -0.051142  0.200568  0.335000  0.050063  0.305444  0.225228 -0.176068  0.332432  0.156352  0.471814",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n      <th>11</th>\n      <th>12</th>\n      <th>13</th>\n      <th>14</th>\n      <th>15</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.827912</td>\n      <td>0.872087</td>\n      <td>0.805252</td>\n      <td>0.638937</td>\n      <td>0.403635</td>\n      <td>-0.220579</td>\n      <td>0.074281</td>\n      <td>0.066307</td>\n      <td>0.161097</td>\n      <td>0.776864</td>\n      <td>-0.015753</td>\n      <td>0.059171</td>\n      <td>-0.167397</td>\n      <td>-0.519080</td>\n      <td>-0.113654</td>\n      <td>0.370162</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.123528</td>\n      <td>0.464093</td>\n      <td>0.514921</td>\n      <td>-0.184883</td>\n      <td>0.292600</td>\n      <td>-0.127225</td>\n      <td>0.205937</td>\n      <td>0.207778</td>\n      <td>0.602376</td>\n      <td>1.016719</td>\n      <td>0.158707</td>\n      <td>0.074483</td>\n      <td>0.526718</td>\n      <td>0.460966</td>\n      <td>0.633765</td>\n      <td>0.534267</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.040623</td>\n      <td>0.605709</td>\n      <td>0.119165</td>\n      <td>-0.133348</td>\n      <td>-0.169015</td>\n      <td>-0.146091</td>\n      <td>0.444743</td>\n      <td>0.050206</td>\n      <td>0.774002</td>\n      <td>0.830742</td>\n      <td>-0.216136</td>\n      <td>-0.111672</td>\n      <td>0.751038</td>\n      <td>0.496729</td>\n      <td>0.747770</td>\n      <td>0.609336</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.900045</td>\n      <td>0.596818</td>\n      <td>0.359724</td>\n      <td>0.836026</td>\n      <td>0.074438</td>\n      <td>0.652813</td>\n      <td>1.057635</td>\n      <td>-0.026944</td>\n      <td>1.124048</td>\n      <td>0.971823</td>\n      <td>-0.467323</td>\n      <td>-0.247706</td>\n      <td>0.002623</td>\n      <td>-0.360194</td>\n      <td>0.017813</td>\n      <td>0.965252</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.061119</td>\n      <td>1.158659</td>\n      <td>0.633724</td>\n      <td>-0.320907</td>\n      <td>0.383561</td>\n      <td>-0.448183</td>\n      <td>-0.238576</td>\n      <td>-0.078154</td>\n      <td>-0.033891</td>\n      <td>0.852194</td>\n      <td>0.056048</td>\n      <td>0.533376</td>\n      <td>0.968322</td>\n      <td>0.675573</td>\n      <td>0.506570</td>\n      <td>0.651593</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.071058</td>\n      <td>0.273445</td>\n      <td>0.044782</td>\n      <td>0.250272</td>\n      <td>-0.114290</td>\n      <td>0.121239</td>\n      <td>0.472959</td>\n      <td>0.086112</td>\n      <td>-0.077087</td>\n      <td>-0.158122</td>\n      <td>-0.416886</td>\n      <td>0.212225</td>\n      <td>0.220158</td>\n      <td>-0.492304</td>\n      <td>0.175022</td>\n      <td>0.505993</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.897961</td>\n      <td>1.322748</td>\n      <td>0.553525</td>\n      <td>0.079147</td>\n      <td>0.502846</td>\n      <td>-0.506932</td>\n      <td>0.223247</td>\n      <td>-0.400829</td>\n      <td>0.203914</td>\n      <td>0.658943</td>\n      <td>-0.018270</td>\n      <td>0.620021</td>\n      <td>0.600703</td>\n      <td>0.066074</td>\n      <td>0.316328</td>\n      <td>0.332746</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.025299</td>\n      <td>0.779429</td>\n      <td>0.627483</td>\n      <td>-0.354743</td>\n      <td>0.218848</td>\n      <td>-0.055096</td>\n      <td>0.297431</td>\n      <td>-0.119230</td>\n      <td>-0.025749</td>\n      <td>0.031660</td>\n      <td>0.079259</td>\n      <td>0.274235</td>\n      <td>0.095197</td>\n      <td>-0.625557</td>\n      <td>-0.000797</td>\n      <td>0.226548</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.529123</td>\n      <td>-0.196815</td>\n      <td>0.232964</td>\n      <td>0.031045</td>\n      <td>0.214278</td>\n      <td>-0.328799</td>\n      <td>-0.685732</td>\n      <td>0.060671</td>\n      <td>0.237945</td>\n      <td>0.642808</td>\n      <td>0.532076</td>\n      <td>0.191681</td>\n      <td>-0.066689</td>\n      <td>0.383821</td>\n      <td>-0.080174</td>\n      <td>0.330105</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>0.935247</td>\n      <td>0.690427</td>\n      <td>0.511019</td>\n      <td>1.063499</td>\n      <td>0.504674</td>\n      <td>-0.227777</td>\n      <td>-0.035515</td>\n      <td>-0.006277</td>\n      <td>0.174739</td>\n      <td>0.899135</td>\n      <td>0.184269</td>\n      <td>0.238904</td>\n      <td>-0.084467</td>\n      <td>-0.414828</td>\n      <td>-0.173580</td>\n      <td>0.431657</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>0.467166</td>\n      <td>0.532110</td>\n      <td>0.238954</td>\n      <td>0.085494</td>\n      <td>0.207894</td>\n      <td>-0.739952</td>\n      <td>-0.371635</td>\n      <td>-0.087244</td>\n      <td>-0.217038</td>\n      <td>0.353716</td>\n      <td>0.276079</td>\n      <td>0.202890</td>\n      <td>0.124148</td>\n      <td>-0.118437</td>\n      <td>-0.063674</td>\n      <td>0.186508</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>0.016316</td>\n      <td>-0.238568</td>\n      <td>-0.610145</td>\n      <td>-0.418894</td>\n      <td>-0.609519</td>\n      <td>-0.072247</td>\n      <td>-0.179781</td>\n      <td>-0.085679</td>\n      <td>0.529486</td>\n      <td>0.309438</td>\n      <td>0.002346</td>\n      <td>-0.154983</td>\n      <td>0.340535</td>\n      <td>0.199168</td>\n      <td>-0.240800</td>\n      <td>0.511496</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>0.210837</td>\n      <td>0.166079</td>\n      <td>0.430884</td>\n      <td>0.002882</td>\n      <td>0.447536</td>\n      <td>0.287130</td>\n      <td>-0.069169</td>\n      <td>0.463648</td>\n      <td>-0.251514</td>\n      <td>0.225306</td>\n      <td>0.302280</td>\n      <td>0.252891</td>\n      <td>0.182381</td>\n      <td>-0.017385</td>\n      <td>-0.036945</td>\n      <td>0.483606</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>-0.131177</td>\n      <td>0.249064</td>\n      <td>0.690767</td>\n      <td>-0.918174</td>\n      <td>0.273945</td>\n      <td>-0.226493</td>\n      <td>-0.467489</td>\n      <td>0.787192</td>\n      <td>-0.527138</td>\n      <td>-0.107953</td>\n      <td>0.469084</td>\n      <td>0.095217</td>\n      <td>0.443432</td>\n      <td>0.514454</td>\n      <td>0.195352</td>\n      <td>0.647130</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>-0.268342</td>\n      <td>0.041401</td>\n      <td>0.530629</td>\n      <td>-0.301306</td>\n      <td>0.030230</td>\n      <td>-0.376372</td>\n      <td>-0.326081</td>\n      <td>0.092524</td>\n      <td>-0.112594</td>\n      <td>-0.274940</td>\n      <td>0.107115</td>\n      <td>0.171046</td>\n      <td>-0.394140</td>\n      <td>-0.418468</td>\n      <td>-0.190215</td>\n      <td>0.442426</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>-0.219957</td>\n      <td>-0.423727</td>\n      <td>0.599078</td>\n      <td>-0.366324</td>\n      <td>0.452232</td>\n      <td>0.868501</td>\n      <td>-0.051142</td>\n      <td>0.200568</td>\n      <td>0.335000</td>\n      <td>0.050063</td>\n      <td>0.305444</td>\n      <td>0.225228</td>\n      <td>-0.176068</td>\n      <td>0.332432</td>\n      <td>0.156352</td>\n      <td>0.471814</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 640x480 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAHHCAYAAACLPpP8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB2iElEQVR4nO3deXxM1/8/8NckkX1HNkIItUQQYtdGSMUuVYpaYm2pPYqmH8RSgtZOpSiqtaRqLW1Q+xJLRCwNQRoSJLaQSEISmfP7wy/zNbKYm8yVMK/n4zGPh3vuvee878zcydu9556jEEIISKCvr4/ExETY2dmplT969Ah2dnbIycmRUh0RERGRip7UHQrKYzIzM2FoaFjsgIiIiEh3GWi64ZIlSwAACoUCq1evhrm5uWpdTk4Ojh49ipo1a2o/QiIiItIZCk1v5VSpUgUAcOvWLVSsWBH6+vqqdYaGhnBxccGMGTPQpEkTeSIlIiKi957GiUkub29vbNu2DTY2NnLFRERERDpKcmJCREREJBeN+pgEBARg5syZMDMzQ0BAQKHbLliwQCuBERERke7RKDE5f/48srOzAQCRkZFQKBT5bldQOREREZEmNLqVc/HiRdSpUwd6epKfLiYiIiLSmEaZhoeHBx4+fAgAqFq1Kh49eiRrUERERKSbNEpMrK2tERcXBwC4efMmlEqlrEERUeEUCgWmTZsmeztKpRJ16tTBrFmzZG+rJISEhKBSpUrIzMws6VCI6P/TKDH59NNP4eXlhSpVqkChUMDT0xNVq1bN90VU2v34449QKBQFjrkTHR2NadOm4ebNm/nuu27dOnkD/P/++uuvt5J8FGbTpk1ISEjAyJEj86z7999/0bdvX1SoUAFGRkZwcnJC3759ER0dXex209PTizy9xeHDh6FQKPDHH3+olWdlZaFTp07Q09PDmjVrAAADBgxAVlYWfvrpp2LHTETaofHjwmFhYbhx4wZGjx6NGTNmwMLCIt/txowZo9UAibStRYsWuHv3Lm7evInr16+jWrVqauv/+OMP9OjRA4cOHUKrVq3U1tWpUwflypXD4cOHZY9z5MiRWL58eb7TQDx//hwGBgYwMNB48OYiqV+/Ppo0aZLnD/e2bdvQu3dv2NraYvDgwahSpQpu3ryJn3/+GcnJyQgNDUXXrl0ltbV//36EhITg4MGDePLkCfT19VGlShV0794dY8aMgYODg0b1HD58GN7e3tiyZQu6d+8OAMjOzsann36K3bt3Y9WqVRg8eLBq+0mTJiE0NBRxcXHswE9UGgiJBgwYIFJTU6XuRlQq/PfffwKA2LZtmyhfvryYNm1anm22bNkiAIhDhw7lWefm5ia8vLzkD1QIMWLECFGEU1RrIiMjBQDxzz//qJXfuHFDmJqaipo1a4r79++rrXvw4IGoWbOmMDc3F//9959G7aSlpYlPP/1UKBQK0b59e7F06VKxe/du8fvvv4upU6eK6tWrC2tra/HHH39oVN+hQ4cEALFlyxYhhBBZWVnCz89PKBQKsXLlyjzbR0RECADiwIEDGtVPRPIq8q/e9evXRVhYmMjIyBBCCKFUKrUWFJFcZs6cKWxsbERmZqYYPny4qF69utr6tWvXCgB5XocOHRKVK1fOU/5qkvL48WMxZswYUbFiRWFoaChcXV3FnDlzRE5OjmqbuLg4AUB8//334qeffhJVq1YVhoaGwtPTU5w5c0a1nb+/f75x5AIggoKC1GKPjIwU7dq1ExYWFsLMzEy0bt1ahIeH53t8x48fF+PGjRPlypUTpqamws/PL0+SMXXqVGFoaCiysrLUyr/88ksBQBw9ejTf9/jIkSMCgBg+fHjBH8T/l52dLVq1aiUqVaqkdvyvbzN37lxhaGgodu/e/cY6X01MsrOzRbdu3YRCoRAhISEF7mNraytGjx79xrqJSH6SE5NHjx6J1q1bC4VCIfT09ERsbKwQQoiBAweKgIAArQdIpE01a9YUgwcPFkIIcfToUQFA7Q9ibGysGD16tAAgvv32W/Hrr7+KX3/9VSQlJYnt27eLihUripo1a6rK9+3bJ4QQIj09XdStW1eULVtWfPvttyIkJET0799fKBQKMWbMGFX9uYmJh4eHqFatmpg7d66YN2+eKFeunKhYsaIqCTh58qT4+OOPBQBVW7/++quqntcTk8uXLwszMzPh6OgoZs6cKebMmSOqVKkijIyMxKlTp1Tb5SYmHh4eonXr1mLp0qVi/PjxQl9fX3z22Wdq75WPj49o0KBBnvfQyclJuLi4FPo+u7i4iIoVK77h0xBixowZwtHRUdy9e1dVlpOTI9LS0lT/fvDggRBCiB9//FHY2dm98YptbmKyadMm0b17d6FQKMSPP/5Y6D4+Pj6iYcOGb4yXiOQnOTHp16+f8PX1FQkJCcLc3FyVmISFhYnatWtrPUAibcm9ZL9//34hxMurfBUrVlRLHIQo2q2cmTNnCjMzM3Ht2jW18m+++Ubo6+uL+Ph4IcT/JSZly5YVycnJqu127twpAIg///xTVVbYrZzXExM/Pz9haGioOh+FEOLu3bvCwsJCfPTRR6qy3MTEx8dH7SrnuHHjhL6+vnjy5ImqrGLFiuLTTz9Va/fJkycCgOjatWu+ceXq0qWLAFBoEpGSkiIsLS3Fjh07VGUrV64UNjY2AoBwc3MTW7duVXsPGjRokO/tmFflJia5V7iWL19e6PZCCPHFF18IExOTN25HRPKTPGLavn37MHfuXFSsWFGtvHr16rh165bU6ojemg0bNsDe3h7e3t4AXj5y27NnT2zevLnIT4Dk2rJlCz788EPY2Njg4cOHqpePjw9ycnJw9OhRte179uypNhHmhx9+CAD477//JLedk5ODffv2wc/PT+3JOEdHR3z++ec4fvw4UlNT1fb54osv1Dp6fvjhh8jJyVE7hx89epRnss6nT58CQIGd33Plrs/dPj/79u2Dra0tunTpAuDlqNJffvklPv30U2zfvh09e/bE0KFD1fbp2rWrxh2P7927BwMDA9XM6IWxsbHBs2fPkJGRoVHdRCQfyYlJeno6TE1N85QnJyfDyMhIK0ERaVtOTg42b94Mb29vxMXF4caNG7hx4waaNGmCe/fu4cCBA8Wq//r16wgLC0P58uXVXj4+PgCA+/fvq21fqVIlteXcBODx48eS237w4AEyMjJQo0aNPOtq1aoFpVKJhISEIrUvXnsiSJOEI3e9QqFAuXLlCtzm3Llz8PLyUiVIq1evRqtWrbBq1Sr4+flhypQpGDVqlNo+9vb2ePDgQaFt55o3bx4qVaqE7t2748SJE4Vum3ucfCqHqORJftbwww8/xPr16zFz5kwAL09kpVKJefPmqf4nSlTaHDx4EImJidi8eTM2b96cZ/2GDRvQtm3bItevVCrx8ccfY+LEifmu/+CDD9SW9fX1893u9URALpq0X7Zs2TyJipWVFZycnHDx4sVC67948SIqVqwIQ0PDArd59OgRnJycVMs3b95Eo0aN1LZp3Lix2nJCQgLKli1baNu5HB0dsX//frRs2RIdO3bEkSNHUK9evXy3ffz4MUxNTWFiYqJR3UQkH8mJybx589CmTRtEREQgKysLEydOxL///ovk5OQ3/q+EqKRs2LABdnZ2WL58eZ5127Ztw/bt2xESEgITE5NC/9dc0DpXV1ekpaWprpBog6b/ey9fvjxMTU0RExOTZ93Vq1ehp6cHZ2dnye3XrFlTNeLzqzp37oyffvoJx48fR8uWLfOsP3bsGG7evPnGmcgtLS2RkpKiWnZwcEBsbKzaNq/e2nr+/Dl+/fVXTJ06VeNjqFq1Kvbu3QsvLy/4+vri2LFjqF69ep7t4uLiUKtWLY3rJSL5SL6VU6dOHVy7dg0tW7ZE165dkZ6ejm7duuH8+fNwdXWVI0aiYnn27Bm2bduGTp06oXv37nleI0eOxNOnT7Fr1y4AgJmZGQDgyZMneeoyMzPLt/yzzz5DeHg49u7dm2fdkydP8OLFC8lxFxbHq/T19dG2bVvs3LlTbbTae/fuYePGjWjZsiUsLS0lt9+sWTNcvnw5z3DtX3/9NUxNTfHll1/mmTcrOTkZw4YNg6WlZb6jxb6qVq1aOH36tGr5k08+wfbt27F8+XLcunULf/31F2bPng3gZbLTtm1b2NjYoG/fvpKOw93dHXv27EFaWho+/vhj3LlzJ882kZGRaN68uaR6iUgmJdv3lkh+mzdvFgDUnv54VU5Ojihfvrzo3LmzEEKIxMREoa+vL5o2bSrWrVsnNm3aJO7duyeEEOKrr74SCoVCzJw5U2zatEk1KFd6erpo0KCBMDAwEEOGDBErVqwQP/zwg/D39xdmZmaqR15fHcfkdXjtSZvff/9dABD9+vUTv/32m9i0aVOB2+Y+LlyhQgUxa9YsMXfuXFG1atUCHxc+e/asWtu5T7K8+iRS7lNMe/fuzRPrH3/8IcqUKSMcHR3F5MmTxc8//yymTJkinJychImJidi5c2e+7/Wrbt++LQwMDERkZKSqbPjw4aoxW0xNTcX3338vAAg9PT3x2Wefqd7Hwrw+wFquvXv3CkNDQ1GrVi3x8OHDPMf5+kByRFQyipSYPH78WOzdu1f8+uuv4pdfflF7EZU2nTt3FsbGxiI9Pb3AbQYMGCDKlCmj+oO1atUqUbVqVaGvr6/2BzspKUl07NhRWFhY5Blg7enTpyIwMFBUq1ZNGBoainLlyonmzZuLH374QTU+iZTE5MWLF2LUqFGifPnyQqFQaDTAmq+vrzA3NxempqbC29tbnDx5Um0bKYmJEELUrVtXNe7L6y5duiQ+//xz4eDgIPT09AQAYWxsLP799998t8+Pv7+/aNKkicjMzFSVxcbGimPHjonHjx+LZ8+eifDwcLXHmN+koMRECCFCQ0OFnp6eaNSokepR5kmTJolKlSpxkEiiUkLjuXJy/fnnn+jTpw/S0tJgaWmpdh9coVAgOTm56JdviKhU+fXXXzFixAjEx8fD2tq60G3Xr1+PAQMGoG/fvli/fr1G9T98+BANGzZEnTp1sGnTpnxvOeXk5GD79u2qeW+0KTMzEy4uLvjmm284zxdRKSE5Mfnggw/QoUMHzJ49O9/Hhono/aFUKlG3bl307t0b//vf/964/dy5c/HNN98gMDBQ1T/kTa5du4aOHTsiNTUVI0eOxMcffwwnJyekpqbi+PHjWLZsGZKSkhAZGZnnMefiCgkJwezZs3H9+nUOd0BUSkhOTMzMzHDp0iW1gZyIiIrj6dOn+P7777F69WokJiaqyi0sLNCnTx9MnToVjo6OJRghEb0tkhOTbt26oVevXvjss8/kiomIdJQQAjdu3EBSUhIsLS1Rq1atQsdCIaL3j+RxTDp27IgJEyYgOjoa7u7uKFOmjNr63OGliYikUigUqF69er5jjRCRbpB8xURPr+ChTxQKRbHnHCEiIiLdJTkxISIiIpKL5Fs5r3r+/DmMjY21FYsslEol7t69CwsLC07QRUREhRJC4OnTp3Bycir0DkFxPX/+HFlZWcWux9DQsNT/HZZKcmKSk5OD2bNnIyQkBPfu3cO1a9dQtWpVTJkyBS4uLhg8eLAccRbZ3bt3izRPCBER6a6EhARUrFhRlrqfP38OW1tbPHv2rNh1OTg4IC4u7r1KTiQnJrNmzcIvv/yCefPmYejQoaryOnXqYNGiRaUuMcmdpv3WlkmwNJVvnIKnnYqf+b5Jt6F1Za1/26rCZ4zVBqsnDrK3oTwu/yB/ei1tZW/jfZBinSR7G2/jO3X0z6ayt/FR51OytyH3ufHxzpqy1v825GRl4Py6Iaq/HXLIysrCs2fP8PnnnxfrqbOsrCxs3LgRWVlZup2YrF+/HitXrkSbNm0wbNgwVXm9evVw9epVrQanDbm3byxNjWBpJt8H9zZuExkYyjugnaVC/gGmLC3ln1ZeKePnnEvvLRzH+0C8J98pM1Nz2dt4H84NuX+j3qa38ZtuaGjIx+HzIfkG2p07d1CtWrU85UqlEtnZ2VoJioiIiHST5MSkdu3aOHbsWJ7yP/74Ax4eHloJioiIiLQrODgYjRo1goWFBezs7ODn54eYmJg37rdlyxbUrFkTxsbGcHd3x19//SVrnJJv5UydOhX+/v64c+cOlEoltm3bhpiYGKxfvx67d++WI0YiIiIqpiNHjmDEiBFo1KgRXrx4gW+//RZt27ZFdHQ0zMzM8t3n5MmT6N27N4KDg9GpUyds3LgRfn5+iIyMRJ06dWSJU/IVk65du+LPP//EP//8AzMzM0ydOhVXrlzBn3/+iY8//liOGImIiKiYwsLCMGDAALi5uaFevXpYt24d4uPjce7cuQL3Wbx4Mdq1a4cJEyagVq1amDlzJho0aIBly5bJFmeRHtL+8MMPsX//fty/fx8ZGRk4fvw42rZtq+3YVJYvXw4XFxcYGxujSZMmOHPmjGxtERER6YKUlBQAgK1twU8ZhoeHw8fHR63M19cX4eHhssVV5AHWIiIicOXKFQAv+500bNhQa0G9KjQ0FAEBAQgJCUGTJk2waNEi+Pr6IiYmBnZ2drK0SURE9K5ITU1VWzYyMoKRUeFPxCmVSowdOxYtWrQo9JZMUlIS7O3t1crs7e2RlCTfUACSr5jcvn0bH374IRo3bowxY8ZgzJgxaNSoEVq2bInbt29rPcAFCxZg6NChGDhwIGrXro2QkBCYmppizZo1Wm+LiIjoXePs7AwrKyvVKzg4+I37jBgxApcvX8bmzZvfQoTSSE5MhgwZguzsbFy5cgXJyclITk7GlStXoFQqMWTIEK0Gl5WVhXPnzqldRtLT04OPj4+sl5GIiIjeFQkJCUhJSVG9AgMDC91+5MiR2L17Nw4dOvTG0W0dHBxw7949tbJ79+7BwUG+gQ0lJyZHjhzBihUrUKNGDVVZjRo1sHTpUhw9elSrwT18+BA5OTmSLiNlZmYiNTVV7UVERPS+srS0VHsVdBtHCIGRI0di+/btOHjwIKpUqfLGups1a4YDBw6ole3fvx/NmjXTSuz5kZyYODs75zuQWk5ODpycnLQSVHEEBwerXdLiPDlEREQvb9/89ttv2LhxIywsLJCUlISkpCS1OXv69++vdsVlzJgxCAsLw/z583H16lVMmzYNERERGDlypGxxSk5Mvv/+e4waNQoRERGqsoiICIwZMwY//PCDVoMrV64c9PX1JV1GCgwMVLuklZCQoNWYiIiI3kUrVqxASkoKWrVqBUdHR9UrNDRUtU18fDwSExNVy82bN8fGjRuxcuVK1KtXD3/88Qd27Ngh2xgmgIZP5djY2KjNG5Ceno4mTZrAwODl7i9evICBgQEGDRoEPz8/rQVnaGiIhg0b4sCBA6p6lUolDhw4UGC2pklvZCIiIl0jhHjjNocPH85T1qNHD/To0UOGiPKnUWKyaNEimcMoWEBAAPz9/eHp6YnGjRtj0aJFSE9Px8CBA0ssJiIiIpKHRomJv7+/3HEUqGfPnnjw4AGmTp2KpKQk1K9fH2FhYXk6xBIREdG7r8gDrL1NI0eOlLWjDREREZUORRqSnoiIiEgOTEyIiIio1GBiQkRERKUGExMiIiIqNTTq/NqtWzeNK9y2bVuRgyEiIiLdplFiYmVlpfq3EALbt2+HlZUVPD09AQDnzp3DkydPJCUw75t2X3nI3kb40ntv3qgYDj9eIWv9ANAKM2Vv40hyF9nb+HbUDdnbONE9Wtb6W/xRW9b6AQBfyf9YfzjkPS8AwMt2l+xtAGVlb0HucyPsxy2y1g8AVjnyTn2SmvoMVitlbYLeQKPEZO3atap/T5o0CZ999hlCQkKgr68P4OU8OV999RUsLS3liZKIiIh0guQ+JmvWrMHXX3+tSkoAQF9fHwEBAVizZo1WgyMiIiLdIjkxefHiBa5evZqn/OrVq1AqlVoJioiIiHST5JFfBw4ciMGDByM2NhaNGzcGAJw+fRpz5szh/DVERERULJITkx9++AEODg6YP3++ampkR0dHTJgwAePHj9d6gERERKQ7JCcmenp6mDhxIiZOnIjU1FQAYKdXIiIi0opiTeLHhISIiIi0SXLn13v37qFfv35wcnKCgYEB9PX11V5ERERERSX5ismAAQMQHx+PKVOmwNHREQqFQo64iIiISAdJTkyOHz+OY8eOoX79+jKEQ0RERLpM8q0cZ2dnCCHkiIWIiIh0nOTEZNGiRfjmm29w8+ZNGcIhIiIiXSb5Vk7Pnj2RkZEBV1dXmJqaokyZMmrrk5OTtRYcERER6RbJicmiRYtkCIOIiIioCImJv7+/HHEQERERFW+AtefPnyMrK0utjIOuERERUVFJ7vyanp6OkSNHws7ODmZmZrCxsVF7ERERERWV5MRk4sSJOHjwIFasWAEjIyOsXr0a06dPh5OTE9avXy9HjERERKQjJN/K+fPPP7F+/Xq0atUKAwcOxIcffohq1aqhcuXK2LBhA/r06SNHnERERKQDJF8xSU5ORtWqVQG87E+S+3hwy5YtcfToUe1GR0RERDpFcmJStWpVxMXFAQBq1qyJ33//HcDLKynW1tZaDY6IiIh0i+TEZODAgbhw4QIA4JtvvsHy5cthbGyMcePGYcKECVoPkIiIiHSH5D4m48aNU/3bx8cHV69exblz51CtWjXUrVtXq8GROuWRR7LW3+q3mbLW/7Z42e6SvY3wpWVlb0N5RN76w5fek7cBkkTu8/ttsMpxKukQ3indM/bALFvy9QGV9Gwl1mkvnFKjWOOYAEDlypVRuXJlbcRCREREOq7oqRoRERGRljExISIiolKDiQkRERGVGkxMiIiIqNQoUmISGxuLyZMno3fv3rh//z4A4O+//8a///6r1eCIiIhIt0hOTI4cOQJ3d3ecPn0a27ZtQ1paGgDgwoULCAoK0nqAREREpDskJybffPMNvvvuO+zfvx+Ghoaq8tatW+PUqVNaDS44OBiNGjWChYUF7Ozs4Ofnh5iYGK22QURERKWH5MTk0qVL+OSTT/KU29nZ4eHDh1oJKteRI0cwYsQInDp1Cvv370d2djbatm2L9PR0rbZDREREpYPkAdasra2RmJiIKlWqqJWfP38eFSpU0FpgABAWFqa2vG7dOtjZ2eHcuXP46KOPtNoWERERlTzJV0x69eqFSZMmISkpCQqFAkqlEidOnMDXX3+N/v37yxGjSkpKCgDA1ta2wG0yMzORmpqq9iIiIqJ3g+TEZPbs2ahZsyacnZ2RlpaG2rVr46OPPkLz5s0xefJkOWIEACiVSowdOxYtWrRAnTp1CtwuODgYVlZWqpezs7NsMREREZF2Sb6VY2hoiFWrVmHKlCm4fPky0tLS4OHhgerVq8sRn8qIESNw+fJlHD9+vNDtAgMDERAQoFpOTU1lckJERPSOKPIkfpUqVUKlSpW0GUuBRo4cid27d+Po0aOoWLFiodsaGRnByMjorcRFRERE2iU5MRk0aFCh69esWVPkYF4nhMCoUaOwfft2HD58OE+HWyIiInq/SE5MHj9+rLacnZ2Ny5cv48mTJ2jdurXWAgNe3r7ZuHEjdu7cCQsLCyQlJQEArKysYGJiotW2iIiIqORJTky2b9+ep0ypVGL48OFwdXXVSlC5VqxYAQBo1aqVWvnatWsxYMAArbZFREREJa/IfUxepaenh4CAALRq1QoTJ07URpUAXt7KISIiIt2htdmFY2Nj8eLFC21VR0RERDpI8hWTVx/FBV5e1UhMTMSePXvg7++vtcCIiIhI90i+YnL+/Hm118WLFwEA8+fPx6JFi7QdHxEREWnB0aNH0blzZzg5OUGhUGDHjh2Fbn/48GEoFIo8r9wHUeQi+YrJoUOH5IiDiIiIZJSeno569eph0KBB6Natm8b7xcTEwNLSUrVsZ2cnR3gqWun8+i44/rgDzDLNZas/+LF2H5XOT4s/Vspaf92Nf8laPwD89FND2dvQ8yorexvNRtnL3gYgbxvhXvdkrR8ADvedInsbgTbynhcvyf95hy+V//PwOrJL1vq//PKmrPUDwLzVjrLWnyoyZa2/JLVv3x7t27eXvJ+dnR2sra21H1ABNEpMPDw8oFAoNKowMjKyWAERERGR5l6frFbbI6DXr18fmZmZqFOnDqZNm4YWLVpore78aJSY+Pn5yRoEERERFc3r88EFBQVh2rRpxa7X0dERISEh8PT0RGZmJlavXo1WrVrh9OnTaNCgQbHrL4hGiUlQUJBsARAREVHRJSQkqPUB0dbVkho1aqBGjRqq5ebNmyM2NhYLFy7Er7/+qpU28qMzfUyIiIjeR5aWlmqJiZwaN26M48ePy9qG5MQkJycHCxcuxO+//474+HhkZWWprU9OTtZacERERFR6REVFwdFR3g7IkscxmT59OhYsWICePXsiJSUFAQEB6NatG/T09LRyT4uIiIi0Ly0tDVFRUYiKigIAxMXFISoqCvHx8QCAwMBA9O/fX7X9okWLsHPnTty4cQOXL1/G2LFjcfDgQYwYMULWOCVfMdmwYQNWrVqFjh07Ytq0aejduzdcXV1Rt25dnDp1CqNHj5YjTiIiIiqGiIgIeHt7q5ZzR3L39/fHunXrkJiYqEpSACArKwvjx4/HnTt3YGpqirp16+Kff/5Rq0MOkhOTpKQkuLu7AwDMzc2RkpICAOjUqROmTJF/zAIiIiKSrlWrVoVOjrtu3Tq15YkTJ2p1Yl5NSb6VU7FiRSQmJgIAXF1dsW/fPgDA2bNntfrcNBEREekeyYnJJ598ggMHDgAARo0ahSlTpqB69ero378/Bg0apPUAiYiISHdIvpUzZ84c1b979uyJypUr4+TJk6hevTo6d+6s1eCIiIhIt0hOTJ4/fw5jY2PVctOmTdG0aVOtBkVERES6SfKtHDs7O/j7+2P//v1QKpVyxEREREQ6SnJi8ssvvyAjIwNdu3ZFhQoVMHbsWERERMgRGxEREemYInV+3bJlC+7du4fZs2cjOjoaTZs2xQcffIAZM2bIESMRERHpCMmJSS4LCwsMHDgQ+/btw8WLF2FmZobp06drMzYiIiLSMUVOTJ4/f47ff/8dfn5+aNCgAZKTkzFhwgRtxkZEREQ6RvJTOXv37sXGjRuxY8cOGBgYoHv37ti3bx8++ugjOeIjIiIiHSI5Mfnkk0/QqVMnrF+/Hh06dECZMmXkiIuIiIh0kOTE5N69e7CwsJAjFiIiItJxkvuYMCkhIiIiuRS58ysRERGRtkm+lfOumn70PxgYmspWf/hvM2WrO9fs7dVkrd/7k4ay1v+2NBtlL3sb4Uvvyd6G3N7K+/QWzosTR6Jlb0PPq6zsbUw7c1f+NrycZK3f0SRT1voBwOoneY9BkfoMsJK1CXoDnUlMiIiISpO6WwbDUmFU5P1TRSaAOW/c7l1T5Fs5N27cwN69e/Hs2TMAgBBCa0ERERGRbpKcmDx69Ag+Pj744IMP0KFDByQmJgIABg8ejPHjx2s9QCIiItIdkhOTcePGwcDAAPHx8TA1/b8+Gz179kRYWJhWgyMiIiLdIrmPyb59+7B3715UrFhRrbx69eq4deuW1gIjIiIi3SP5ikl6erralZJcycnJMDIqeiceIiIiIsmJyYcffoj169erlhUKBZRKJebNmwdvb2+tBkdERES6RXJiMm/ePKxcuRLt27dHVlYWJk6ciDp16uDo0aOYO3euHDGqzJkzBwqFAmPHjpW1HSIiIioZkhOTOnXq4Nq1a2jZsiW6du2K9PR0dOvWDefPn4erq6scMQIAzp49i59++gl169aVrQ0iIiIqWUUaYM3Kygr/+9//tB1LgdLS0tCnTx+sWrUK33333Vtrl4iIiN6uIiUmT548wZkzZ3D//n0olUq1df3799dKYK8aMWIEOnbsCB8fnzcmJpmZmcjM/L9hkVNTU7UeDxEREclDcmLy559/ok+fPkhLS4OlpSUUCoVqnUKh0HpisnnzZkRGRuLs2bMabR8cHIzp06drNQYiIiJ6OyT3MRk/fjwGDRqEtLQ0PHnyBI8fP1a9kpOTtRpcQkICxowZgw0bNsDY2FijfQIDA5GSkqJ6JSQkaDUmIiIiko/kKyZ37tzB6NGj8x3LRNvOnTuH+/fvo0GDBqqynJwcHD16FMuWLUNmZib09fXV9jEyMuJ4KkRERO8oyYmJr68vIiIiULVqVTniUdOmTRtcunRJrWzgwIGoWbMmJk2alCcpISIionebRonJrl27VP/u2LEjJkyYgOjoaLi7u6NMmTJq23bp0kVrwVlYWKBOnTpqZWZmZihbtmyeciIiInr3aZSY+Pn55SmbMWNGnjKFQoGcnJxiB0VERES6SaPE5PVHgkvS4cOHSzoEIiIikonkp3LWr1+vNk5IrqysLLU5dIiIiIikkpyYDBw4ECkpKXnKnz59ioEDB2olKCIiItJNkhMTIYTaoGq5bt++DSsrK60ERURERLpJ48eFPTw8oFAooFAo0KZNGxgY/N+uOTk5iIuLQ7t27WQJkoiIiHSDxolJ7pM5UVFR8PX1hbm5uWqdoaEhXFxc8Omnn2o9QCIiItIdGicmQUFBAAAXFxf07NlT4yHidUWK/l3Z2/DOOSFr/c1G2ctaPwCEL70nextvw9t4r94Hb+d9egvfWy/5v7dTn8k/YrXyyCNZ65/m5SRr/YD836kXWRmy1k9vJnnkV39/fzniICIiIpLe+ZWIiIhILkxMiIiIqNRgYkJERESlRpETk6ysLMTExODFixfajIeIiIh0mOTEJCMjA4MHD4apqSnc3NwQHx8PABg1ahTmzJmj9QCJiIhId0hOTAIDA3HhwgUcPnxY7ZFhHx8fhIaGajU4IiIi0i2SHxfesWMHQkND0bRpU7Wh6d3c3BAbG6vV4IiIiEi3SL5i8uDBA9jZ2eUpT09Pz3cOHSIiIiJNSU5MPD09sWfPHtVybjKyevVqNGvWTHuRERERkc6RfCtn9uzZaN++PaKjo/HixQssXrwY0dHROHnyJI4cOSJHjERERKQjJF8xadmyJaKiovDixQu4u7tj3759sLOzQ3h4OBo2bChHjERERKQjijSOiaurK1atWoUzZ84gOjoav/32G9zd3bUdGxEREWnJ0aNH0blzZzg5OUGhUGDHjh1v3Ofw4cNo0KABjIyMUK1aNaxbt072OCUnJvr6+rh//36e8kePHkFfX18rQREREZF2paeno169eli+fLlG28fFxaFjx47w9vZGVFQUxo4diyFDhmDv3r2yxim5j4kQIt/yzMxMGBoaFjsgIiIi0r727dujffv2Gm8fEhKCKlWqYP78+QCAWrVq4fjx41i4cCF8fX3lClPzxGTJkiUAXj6Fs3r1apibm6vW5eTk4OjRo6hZs6b2IyQiIqICpaamqi0bGRnByMio2PWGh4fDx8dHrczX1xdjx44tdt2F0TgxWbhwIYCXV0xCQkLUbtsYGhrCxcUFISEh2o+QiIiICuTs7Ky2HBQUhGnTphW73qSkJNjb26uV2dvbIzU1Fc+ePYOJiUmx28iPxolJXFwcAMDb2xvbtm2DjY2NLAERERGR5hISEmBpaala1sbVkpIkuY/JoUOH5IiDiIiIisDS0lItMdEWBwcH3Lt3T63s3r17sLS0lO1qCVCExAQAbt++jV27diE+Ph5ZWVlq6xYsWKCVwIiIiKjkNGvWDH/99Zda2f79+2Uf5V1yYnLgwAF06dIFVatWxdWrV1GnTh3cvHkTQgg0aNBAjhiJiIiomNLS0nDjxg3VclxcHKKiomBra4tKlSohMDAQd+7cwfr16wEAw4YNw7JlyzBx4kQMGjQIBw8exO+//642LY0cJI9jEhgYiK+//hqXLl2CsbExtm7dioSEBHh5eaFHjx5yxEhERETFFBERAQ8PD3h4eAAAAgIC4OHhgalTpwIAEhMTER8fr9q+SpUq2LNnD/bv34969eph/vz5WL16tayPCgOAQhQ0MEkBLCwsEBUVBVdXV9jY2OD48eNwc3PDhQsX0LVrV9y8eVOmUIsmNTUVVlZWSElZBEtL+e6JNRtl/+aNiulE92hZ69fzKitr/aR7UvTvyt6GVY6T7G28jfM7fOm9N29Esn8WL7IyELHyc6SkpMjSbwP4v79Lnl9shIGhaZHreRuxlgTJV0zMzMxU/UocHR0RGxurWvfw4UPtRUZEREQ6R3Ifk6ZNm+L48eOoVasWOnTogPHjx+PSpUvYtm0bmjZtKkeMREREpCMkJyYLFixAWloaAGD69OlIS0tDaGgoqlevzidyiIiIqFgkJyZVq1ZV/dvMzIyjvRIREZHWSO5jcvbsWZw+fTpP+enTpxEREaGVoIiIiEg3SU5MRowYgYSEhDzld+7cwYgRI7QSFBEREekmyYlJdHR0vgOpeXh4IDpa+4+z3rlzB3379kXZsmVhYmICd3d3XpkhIiJ6T0nuY2JkZIR79+6p9TUBXg7MYmBQpBHuC/T48WO0aNEC3t7e+Pvvv1G+fHlcv36dEwgSERG9pyRnEm3btkVgYCB27twJKysrAMCTJ0/w7bff4uOPP9ZqcHPnzoWzszPWrl2rKqtSpYpW2yAiIqLSQ/KtnB9++AEJCQmoXLkyvL294e3tjSpVqiApKQnz58/XanC7du2Cp6cnevToATs7O3h4eGDVqlWF7pOZmYnU1FS1FxEREb0bJCcmFSpUwMWLFzFv3jzUrl0bDRs2xOLFi3Hp0iU4OztrNbj//vsPK1asQPXq1bF3714MHz4co0ePxi+//FLgPsHBwbCyslK9tB0TERERyadInULMzMzwxRdfaDuWPJRKJTw9PTF79mwALzvYXr58GSEhIfD39893n8DAQAQEBKiWU1NTmZwQERG9IzRKTHbt2oX27dujTJky2LVrV6HbdunSRSuBAS/n4qldu7ZaWa1atbB169YC9zEyMoKRkZHWYiAiIqK3R6PExM/PD0lJSbCzs4Ofn1+B2ykUCuTk5GgrNrRo0QIxMTFqZdeuXUPlypW11gYRERGVHholJkqlMt9/y23cuHFo3rw5Zs+ejc8++wxnzpzBypUrsXLlyrcWAxEREb09kju/vk2NGjXC9u3bsWnTJtSpUwczZ87EokWL0KdPn5IOjYiIiGRQpM6vBw4cwIEDB3D//v08V1DWrFmjlcByderUCZ06ddJqnURERFQ6SU5Mpk+fjhkzZsDT0xOOjo5QKBRyxEVEREQ6SHJiEhISgnXr1qFfv35yxENEREQ6THIfk6ysLDRv3lyOWIiIiEjHSU5MhgwZgo0bN8oRCxEREek4ybdynj9/jpUrV+Kff/5B3bp1UaZMGbX1CxYs0FpwREREpFskJyYXL15E/fr1AQCXL19WW1eaO8KmWCdBKGQcEfYre/nq/v+OJGtvVN38fDvqhqz1vy3Bj+WfLqHVbzNlb6PZKHm/U2E/npe1fgBo95WH7G1glPxNhC+9J3sbyiOPZG9DbnpeZWVv40T3aFnrT01/DhsOlVWiJCcmhw4dkiMOIiIiotI9wBoRERHpFslXTLy9vQu9ZXPw4MFiBURERES6S3Jiktu/JFd2djaioqJw+fJl+Pv7aysuIiIi0kGSE5OFCxfmWz5t2jSkpaUVOyAiIiLSXVrrY9K3b1+tz5NDREREukVriUl4eDiMjY21VR0RERHpIMm3crp166a2LIRAYmIiIiIiMGXKFK0FRkRERLpHcmJiZWWltqynp4caNWpgxowZaNu2rdYCIyIiIt0jOTFZu3atHHEQERERSU9MXpWWlgalUqlWZmlpWayAiIiISHdJ7vwaFxeHjh07wszMDFZWVrCxsYGNjQ2sra1hY2MjR4xERESkIyRfMenbty+EEFizZg3s7e1L9cR9RERE9G6RnJhcuHAB586dQ40aNeSIh4iIiHSY5Fs5jRo1QkJCghyxEBERkY6TfMVk9erVGDZsGO7cuYM6deqgTJkyauvr1q2rteCIiIhIt0hOTB48eIDY2FgMHDhQVaZQKCCEgEKhQE5OjlYDJCIiIt0hOTEZNGgQPDw8sGnTJnZ+JSIiIq2SnJjcunULu3btQrVq1eSIh4iIiHSY5M6vrVu3xoULF+SIhYiIiHSc5CsmnTt3xrhx43Dp0iW4u7vn6fzapUsXrQVHREREukVyYjJs2DAAwIwZM/KsY+dXIiIiKg7Jicnrc+O8K7oNrQsDQ1PZ6g/78bxsdeey6B4ta/1hP2bKWj8AtPvKQ/Y2Am1Wyt4GRsnfxNv4Tr0Pwpfek72NZqPsZW8jfKnsTUB55JGs9afo35W1fkD+35AXWRmy1v+qA9/fh6WlSZH3T019Bqu38HP3tknuY0JEREQklyLNLnz27FkcOnQI9+/fz3MFZcGCBVoJjIiIiHSP5MRk9uzZmDx5MmrUqJFnHBOOaUJERETFITkxWbx4MdasWYMBAwbIEA4RERHpMsl9TPT09NCiRQs5YiEiIiIdJzkxGTduHJYvXy5HLERERCSz5cuXw8XFBcbGxmjSpAnOnDlT4Lbr1q2DQqFQexkbG8san+RbOV9//TU6duwIV1dX1K5dO88Aa9u2bdNacDk5OZg2bRp+++03JCUlwcnJCQMGDMDkyZPZn4WIiEii0NBQBAQEICQkBE2aNMGiRYvg6+uLmJgY2NnZ5buPpaUlYmJiVMty//2VnJiMHj0ahw4dgre3N8qWLStrgHPnzsWKFSvwyy+/wM3NDRERERg4cCCsrKwwevRo2dolIiJ6Hy1YsABDhw7FwIEDAQAhISHYs2cP1qxZg2+++SbffRQKBRwcHN5ajJITk19++QVbt25Fx44d5YhHzcmTJ9G1a1dVWy4uLti0aVOhl52IiIgor6ysLJw7dw6BgYGqMj09Pfj4+CA8PLzA/dLS0lC5cmUolUo0aNAAs2fPhpubm2xxSu5jYmtrC1dXVzliyaN58+Y4cOAArl27BgC4cOECjh8/jvbt2xe4T2ZmJlJTU9VeRERE76vX/+ZlZuY/ivfDhw+Rk5MDe3v1kYzt7e2RlJSU7z41atTAmjVrsHPnTvz2229QKpVo3rw5bt++rfXjyCU5MZk2bRqCgoKQkSH/sL3ffPMNevXqhZo1a6JMmTLw8PDA2LFj0adPnwL3CQ4OhpWVlerl7Owse5xEREQlxdnZWe3vXnBwsNbqbtasGfr374/69evDy8sL27ZtQ/ny5fHTTz9prY3XSb6Vs2TJEsTGxsLe3h4uLi55Or9GRkZqLbjff/8dGzZswMaNG+Hm5oaoqCiMHTsWTk5O8Pf3z3efwMBABAQEqJZTU1OZnBAR0XsrISEBlpaWqmUjI6N8tytXrhz09fVx7576/FL37t3TuA9J7kWCGzduFD3gN5CcmPj5+ckQRv4mTJigumoCAO7u7rh16xaCg4MLTEyMjIwK/FCIiIjeN5aWlmqJSUEMDQ3RsGFDHDhwQPW3XKlU4sCBAxg5cqRGbeXk5ODSpUvo0KFDcUIulOTEJCgoSI448pWRkQE9PfW7Tfr6+u/sDMdEREQlKSAgAP7+/vD09ETjxo2xaNEipKenq57S6d+/PypUqKC6HTRjxgw0bdoU1apVw5MnT/D999/j1q1bGDJkiGwxFmkSPwA4d+4crly5AgBwc3ODh4f2p6Lu3LkzZs2ahUqVKsHNzQ3nz5/HggULMGjQIK23RURE9L7r2bMnHjx4gKlTpyIpKQn169dHWFiYqkNsfHy82gWBx48fY+jQoUhKSoKNjQ0aNmyIkydPonbt2rLFKDkxuX//Pnr16oXDhw/D2toaAPDkyRN4e3tj8+bNKF++vNaCW7p0KaZMmYKvvvoK9+/fh5OTE7788ktMnTpVa20QERHpkpEjRxZ46+bw4cNqywsXLsTChQvfQlT/R/JTOaNGjcLTp0/x77//Ijk5GcnJybh8+TJSU1O1PuiZhYUFFi1ahFu3buHZs2eIjY3Fd999B0NDQ622Q0RERKWD5CsmYWFh+Oeff1CrVi1VWe3atbF8+XK0bdtWq8ERERGRbpF8xUSpVOZ5RBh4+QgRO6USERFRcUhOTFq3bo0xY8bg7t27qrI7d+5g3LhxaNOmjVaDIyIiIt0iOTFZtmwZUlNT4eLiAldXV7i6uqJKlSpITU3F0qVL5YiRiIiIdITkPibOzs6IjIzEP//8g6tXrwIAatWqBR8fH60HR0RERLpFIYQQJR2EnFJTU2FlZYXHe6bC0sxYtnb0vMrKVncu5ZFHstZ/JLmLrPUDgJftLtnbeF+OgzTzPpx7ADDDJP+J17Rp6jN5R8V+H8691PTnsOk4AykpKRqNplqkNv7/36WUlEWwtDQpRj3PYGU1VtZYS4LkWzmjR4/GkiVL8pQvW7YMY8eO1UZMREREpKMkJyZbt25FixYt8pQ3b94cf/zxh1aCIiIiIt0kOTF59OgRrKys8pRbWlri4cOHWgmKiIiIdJPkxKRatWoICwvLU/7333+jatWqWgmKiIiIdJPkp3ICAgIwcuRIPHjwAK1btwYAHDhwAPPnz8eiRYu0HR8RERHpEMmJyaBBg5CZmYlZs2Zh5syZAAAXFxesWLEC/fv313qAREREpDskJyYAMHz4cAwfPhwPHjyAiYkJzM3NtR0XERER6aAiJSa5ypcvr604iIiIiKR3fiUiIiKSCxMTIiIiKjWYmBAREVGpITkx+e+//+SIg4iIiKhoA6x5e3vjt99+w/Pnz+WIiYiIiHSU5MQkMjISdevWRUBAABwcHPDll1/izJkzcsRGREREOkZyYlK/fn0sXrwYd+/exZo1a5CYmIiWLVuiTp06WLBgAR48eCBHnERERKQDitz51cDAAN26dcOWLVswd+5c3LhxA19//TWcnZ3Rv39/JCYmajNOIiIi0gFFTkwiIiLw1VdfwdHREQsWLMDXX3+N2NhY7N+/H3fv3kXXrl21GScRERHpAMkjvy5YsABr165FTEwMOnTogPXr16NDhw7Q03uZ41SpUgXr1q2Di4uLtmMlIiKi95zkxGTFihUYNGgQBgwYAEdHx3y3sbOzw88//1zs4IiIiEi3SE5Mrl+//sZtDA0N4e/vX6SAiIiISHdplJhcvHhR4wrr1q1b5GCIiIhIt2mUmNSvXx8KhQJCiHzX565TKBTIycnRaoBERESkOzRKTOLi4uSO453XbJS97G2EL5W3fm+ckLcBAM1G1Za9jfCl8h+H8ojsTUDPq6ys9SuPPJK1fgBo8Yf8nzf+kL+J2a1byN7GVNtdsrcxwyRT1vr3Hrwha/0AcKK77E28NcrjyVCaGRd9//T3c/R1jRKTypUryx0HERERUdHGMfn111/RokULODk54datWwCARYsWYefOnVoNjoiIiHSL5MRkxYoVCAgIQIcOHfDkyRNVnxJra2ssWrRI2/ERERGRDpGcmCxduhSrVq3C//73P+jr66vKPT09cenSJa0GR0RERLpFcmISFxcHDw+PPOVGRkZIT0/XSlBERESkmyQnJlWqVEFUVFSe8rCwMNSqVUsbMREREZGOkpyYBAQEYMSIEQgNDYUQAmfOnMGsWbMQGBiIiRMnSqrr6NGj6Ny5M5ycnKBQKLBjxw619UIITJ06FY6OjjAxMYGPj49GI88SERHRu0lyYjJkyBDMnTsXkydPRkZGBj7//HOsWLECixcvRq9evSTVlZ6ejnr16mH58uX5rp83bx6WLFmCkJAQnD59GmZmZvD19cXz5+/ns9tERES6TvJcOQDQp08f9OnTBxkZGUhLS4OdnV2RGm/fvj3at2+f7zohBBYtWoTJkyeja9euAID169fD3t4eO3bskJwEERERUekn+YrJd999pxoJ1tTUtMhJyZvExcUhKSkJPj4+qjIrKys0adIE4eHhsrRJREREJUtyYrJlyxZUq1YNzZs3x48//oiHDx/KEReSkpIAAPb26kO929vbq9blJzMzE6mpqWovIiIiejdITkwuXLiAixcvolWrVvjhhx/g5OSEjh07YuPGjcjIyJAjRkmCg4NhZWWlejk7O5d0SERERKShIg1J7+bmhtmzZ+O///7DoUOH4OLigrFjx8LBwUFrgeXWde/ePbXye/fuFdpOYGAgUlJSVK+EhAStxURERETyKlJi8iozMzOYmJjA0NAQ2dnZ2ogJwMvxUhwcHHDgwAFVWWpqKk6fPo1mzZoVuJ+RkREsLS3VXkRERPRuKFJiEhcXh1mzZsHNzQ2enp44f/48pk+fXmjfj/ykpaUhKipKNWBbXFwcoqKiEB8fD4VCgbFjx+K7777Drl27cOnSJfTv3x9OTk7w8/MrSthERERUykl+XLhp06Y4e/Ys6tati4EDB6J3796oUKFCkRqPiIiAt7e3ajkgIAAA4O/vj3Xr1mHixIlIT0/HF198gSdPnqBly5YICwuDsbFxkdojIiKi0k1yYtKmTRusWbMGtWvXLnbjrVq1ghCiwPUKhQIzZszAjBkzit0WERERlX6SE5NZs2bJEQcRERGRZolJQEAAZs6cCTMzM9XtloIsWLBAK4ERERGR7tEoMTl//rzqiZvz58/LGhARERHpLo0Sk0OHDuX7byIiIiJtkvy48KBBg/D06dM85enp6Rg0aJBWgiIiIiLdpBCFPRaTD319fSQmJuaZvO/hw4dwcHDAixcvtBpgcaWmpsLKygrxim9gqTAq6XCKJfKPHrLW72W7S9b6AUDPq6zsbTQbZf/mjYppdutqsrfh/ckJ2duQ29v4LMKX3nvzRu8A5ZFHJR1Csb2N81tuqanPYGU1FikpKbIN0Jn7d+nxnqmwNCv68Bep6c9h03GG5FiXL1+O77//HklJSahXrx6WLl2Kxo0bF7j9li1bMGXKFNy8eRPVq1fH3Llz0aFDhyLH/SYaXzFJTU1FSkoKhBB4+vSp2iR5jx8/xl9//SXbTMNERERUfKGhoQgICEBQUBAiIyNRr149+Pr64v79+/luf/LkSfTu3RuDBw/G+fPn4efnBz8/P1y+fFm2GDVOTKytrWFrawuFQoEPPvgANjY2qle5cuUwaNAgjBgxQrZAiYiIqHgWLFiAoUOHYuDAgahduzZCQkJgamqKNWvW5Lv94sWL0a5dO0yYMAG1atXCzJkz0aBBAyxbtky2GDUex+TQoUMQQqB169bYunUrbG1tVesMDQ1RuXJlODk5yRIkERER5S81NVVt2cjICEZGebsuZGVl4dy5cwgMDFSV6enpwcfHB+Hh4fnWHR4enmeYEF9fX+zYsaP4gRdA48TEy8sLwMv5bCpVqgSFQiFbUERERKQZZ2dnteWgoCBMmzYtz3YPHz5ETk4O7O3V+37Z29vj6tWr+dadlJSU7/ZS58aTQvLIrwcPHoS5uTl69FDviLllyxZkZGTA399fa8ERERFR4RISEtQ6v+Z3teRdIvlx4eDgYJQrVy5PuZ2dHWbPnq2VoIiIiEgzlpaWaq+CEpNy5cpBX18f9+6pP8l27949ODg45LuPg4ODpO21QXJiEh8fjypVquQpr1y5MuLj47USFBEREWmXoaEhGjZsiAMHDqjKlEolDhw4gGbNmuW7T7NmzdS2B4D9+/cXuL02SE5M7OzscPHixTzlFy5cQNmy7/4z7ERERO+rgIAArFq1Cr/88guuXLmC4cOHIz09HQMHDgQA9O/fX61z7JgxYxAWFob58+fj6tWrmDZtGiIiIjBy5EjZYpTcx6R3794YPXo0LCws8NFHHwEAjhw5gjFjxqBXr15aD5CIiIi0o2fPnnjw4AGmTp2KpKQk1K9fH2FhYaoOrvHx8dDT+79rFs2bN8fGjRsxefJkfPvtt6hevTp27NiBOnXqyBaj5MRk5syZuHnzJtq0aQMDg5e7K5VK9O/fn31MiIiISrmRI0cWeMXj8OHDecp69OiR54EXOUlOTAwNDREaGoqZM2fiwoULMDExgbu7OypXrixHfERERKRDJCcmuVxcXCCEgKurq+rKCREREVFxSO78mpGRgcGDB8PU1BRubm6qJ3FGjRqFOXPmaD1AIiIi0h2SE5PAwEBcuHABhw8fhrHx/82K6OPjg9DQUK0GR0RERLpF8j2YHTt2IDQ0FE2bNlUblt7NzQ2xsbFaDY6IiIh0i+QrJg8ePICdnV2e8vT0dM6fQ0RERMUiOTHx9PTEnj17VMu5ycjq1atlHQmOiIiI3n+Sb+XMnj0b7du3R3R0NF68eIHFixcjOjoaJ0+exJEjR+SIkYiIiHSE5CsmLVu2RFRUFF68eAF3d3fs27cPdnZ2CA8PR8OGDeWIkYiIiHSERldMAgICMHPmTJiZmeHo0aNo3rw5Vq1aJXdsREREpGM0umKydOlSpKWlAQC8vb2RnJwsa1BERESkmzS6YuLi4oIlS5agbdu2EEIgPDwcNjY2+W6bO7EfERERkVQaJSbff/89hg0bhuDgYCgUCnzyySf5bqdQKJCTk6PVALXF6okDLC1NZKtfeeSRbHXn8vY6IWv9yrfQd7nZKHvZ2zjRPVr2NvS87snehtzexnc2fKnsTbyV71T40nf/8waAI8ldZK3fG/L+RgHyf2+V6c9lrf9Vxx93gFmmeZH3T89IAzBDewGVEholJn5+fvDz80NaWhosLS0RExOT71gmRERERMWhUR+TgIAApKenw9zcHIcOHUKVKlVgZWWV74uIiIioqCR3fm3dujU7vxIREZEs2PmViIiISg2d6fxKREREpR87vxIREVGpIWmunFc7vxoYSJ5mh4iIiKhQkufK8fLywq1btzB58mT07t0b9+/fBwD8/fff+PfffyXVdfToUXTu3BlOTk5QKBTYsWOHal12djYmTZoEd3d3mJmZwcnJCf3798fdu3elhkxERETvCMmJyZEjR+Du7o7Tp09j27Ztqqd1Lly4gKCgIEl1paeno169eli+fHmedRkZGYiMjMSUKVMQGRmJbdu2ISYmBl26yDtAEBEREZUcyfdjvvnmG3z33XcICAiAhYWFqrx169ZYtmyZpLrat2+P9u3b57vOysoK+/fvVytbtmwZGjdujPj4eFSqVElq6ERERFTKSU5MLl26hI0bN+Ypt7Ozw8OHD7USVEFSUlKgUChgbW1d4DaZmZnIzMxULaempsoaExEREWmP5Fs51tbWSExMzFN+/vx5VKhQQStB5ef58+eYNGkSevfuDUtLywK3Cw4OVhuJ1tnZWbaYiIiISLskJya9evXCpEmTkJSUBIVCAaVSiRMnTuDrr79G//795YgR2dnZ+OyzzyCEwIoVKwrdNjAwECkpKapXQkKCLDERERGR9km+lTN79myMGDECzs7OyMnJQe3atZGTk4PPP/8ckydP1nqAuUnJrVu3cPDgwUKvlgCAkZERjIyMtB4HERERyU9yYmJoaIhVq1Zh6tSpuHTpEtLS0uDh4YHq1atrPbjcpOT69es4dOgQypYtq/U2iIiIqPQo8ihpzs7Oxe6/kZaWhhs3bqiW4+LiEBUVBVtbWzg6OqJ79+6IjIzE7t27kZOTg6SkJACAra0tDA0Ni9U2ERERlT4lOnxrREQEvL29VcsBAQEAAH9/f0ybNg27du0CANSvX19tv0OHDqFVq1ZvK0wiIiJ6S0o0MWnVqhWEEAWuL2wdERERvX8kP5VDREREJBcmJkRERFRqFOlWzpMnT/Dzzz/jypUrAAA3NzcMGjQIVlZWWg2OiIiIdIvkKyYRERFwdXXFwoULkZycjOTkZCxYsACurq6IjIyUI0YiIiLSEZKvmIwbNw5dunTBqlWrYGDwcvcXL15gyJAhGDt2LI4ePar1IImIiEg3SE5MIiIi1JISADAwMMDEiRPh6emp1eDeJS3+qC17G7OTq8la/7cHb7x5o2IKX3pP9jYA+QfiO7S9hexteNnukr2N98Hb+E4pjzySvY0jyV1kb0P+7xQHwaTik3wrx9LSEvHx8XnKExISYGFhoZWgiIiISDdJTkx69uyJwYMHIzQ0FAkJCUhISMDmzZsxZMgQ9O7dW44YiYiISEdIvpXzww8/QKFQoH///njx4gUAoEyZMhg+fDjmzJmj9QCJiIhIdxRpEr/FixcjODgYsbGxAABXV1eYmppqPTgiIiLSLZJv5QwaNAhPnz6Fqakp3N3d4e7uDlNTU6Snp2PQoEFyxEhEREQ6QnJi8ssvv+DZs2d5yp89e4b169drJSgiIiLSTRrfyklNTYUQAkIIPH36FMbGxqp1OTk5+Ouvv2BnZydLkERERKQbNE5MrK2toVAooFAo8MEHH+RZr1AoMH36dK0GR0RERLpF48Tk0KFDEEKgdevW2Lp1K2xtbVXrDA0NUblyZTg5OckSJBEREekGjRMTLy8vAEBcXBwqVaoEhUIhW1BERESkmyQ/Lly5cmU54iAiIiKS/lQOERERkVyYmBAREVGpwcSEiIiISg3JiUlQUBBu3bolRyxERERUCiQnJ6NPnz6wtLSEtbU1Bg8ejLS0tEL3adWqlWpYkdzXsGHDJLctOTHZuXMnXF1d0aZNG2zcuBGZmZmSGyUiIqLSq0+fPvj333+xf/9+7N69G0ePHsUXX3zxxv2GDh2KxMRE1WvevHmS25acmERFReHs2bNwc3PDmDFj4ODggOHDh+Ps2bOSGyciIqLS5cqVKwgLC8Pq1avRpEkTtGzZEkuXLsXmzZtx9+7dQvc1NTWFg4OD6mVpaSm5/SL1MfHw8MCSJUtw9+5d/Pzzz7h9+zZatGiBunXrYvHixUhJSSlKtURERCRRamqq2qu4dzLCw8NhbW0NT09PVZmPjw/09PRw+vTpQvfdsGEDypUrhzp16iAwMBAZGRmS2y9W51chBLKzs5GVlQUhBGxsbLBs2TI4OzsjNDS0OFUTERGRBpydnWFlZaV6BQcHF6u+pKSkPHPfGRgYwNbWFklJSQXu9/nnn+O3337DoUOHEBgYiF9//RV9+/aV3L7kAdYA4Ny5c1i7di02bdoEIyMj9O/fH8uXL0e1atUAAEuXLsXo0aPRs2fPolRPREREGkpISFC7ZWJkZJTvdt988w3mzp1baF1Xrlwpchyv9kFxd3eHo6Mj2rRpg9jYWLi6umpcj+TExN3dHVevXkXbtm3x888/o3PnztDX11fbpnfv3hgzZozUqomIiEgiS0tLjfpyjB8/HgMGDCh0m6pVq8LBwQH3799XK3/x4gWSk5Ph4OCgcVxNmjQBANy4cUPexOSzzz7DoEGDUKFChQK3KVeuHJRKpdSqiYiISCbly5dH+fLl37hds2bN8OTJE5w7dw4NGzYEABw8eBBKpVKVbGgiKioKAODo6CgpTsmJSW5fktc9e/YM33//PaZOnSq1yvfCie7Rb6EVedsIX1pW1vrfJ96fnHgLrfDzKC30vN7CZ7Fd/ibkpjzyqKRDeKeccNwBI/P8b7toIjNNnuE6atWqhXbt2mHo0KEICQlBdnY2Ro4ciV69esHJyQkAcOfOHbRp0wbr169H48aNERsbi40bN6JDhw4oW7YsLl68iHHjxuGjjz5C3bp1JbUvufPr9OnT8x1kJSMjA9OnT5daHREREZUyGzZsQM2aNdGmTRt06NABLVu2xMqVK1Xrs7OzERMTo3rqxtDQEP/88w/atm2LmjVrYvz48fj000/x559/Sm67SFdMFApFnvILFy7A1tZWcgBERERUutja2mLjxo0FrndxcYEQQrXs7OyMI0eOaKVtjRMTGxsb1RCzH3zwgVpykpOTg7S0tCINPUtERESUS+PEZNGiRRBCYNCgQZg+fTqsrKxU6wwNDeHi4oJmzZrJEiQRERHpBo0TE39/fwBAlSpV0Lx5c5QpU0a2oIiIiEg3aZSYpKamqp6R9vDwwLNnz/Ds2bN8ty3KuPhEREREgIZP5djY2KgGW7G2toaNjU2eV265FEePHkXnzp3h5OQEhUKBHTt2FLjtsGHDoFAosGjRIkltEBER0btDoysmBw8eVD1xc+jQIa01np6ejnr16mHQoEHo1q1bgdtt374dp06dUj0/TURERO8njRITLy+vfP9dXO3bt0f79u0L3ebOnTsYNWoU9u7di44dO2qtbSIiIip9NEpMLl68qHGFUkd4K4xSqUS/fv0wYcIEuLm5abRPZmam2pTPqampWouHiIiI5KVRYlK/fn0oFAq1wVTyo1AokJOTo5XAAGDu3LkwMDDA6NGjNd4nODiYI9ASERG9ozRKTOLi4uSOI49z585h8eLFiIyMzHek2YIEBgYiICBAtZyamgpnZ2c5QiQiIiIt0ygxqVy5stxx5HHs2DHcv38flSpVUpXl5ORg/PjxWLRoEW7evJnvfkZGRjAyKvqkSERERFRyNEpMdu3ahfbt26NMmTLYtWtXodt26dJFK4H169cPPj4+amW+vr7o168fBg4cqJU2iIiIqHTRKDHx8/NDUlIS7Ozs4OfnV+B2UvuYpKWl4caNG6rluLg4REVFwdbWFpUqVULZsupTjZcpUwYODg6oUaOGxm0QERHRu0OjxESpVOb77+KKiIiAt7e3ajm3b4i/vz/WrVuntXaIiIjo3aDxXDlyaNWq1Ruf9HlVQf1KiIiI6P2g0ZD0rztw4AA6deoEV1dXuLq6olOnTvjnn3+0HRsRERHpGMmJyY8//oh27drBwsICY8aMwZgxY2BpaYkOHTpg+fLlcsRIREREOkLyrZzZs2dj4cKFGDlypKps9OjRaNGiBWbPno0RI0ZoNUAiIiLSHZKvmDx58gTt2rXLU962bVukpKRoJSgiIiLSTZITky5dumD79u15ynfu3IlOnTppJSgiIiLSTRrdylmyZInq37Vr18asWbNw+PBhNGvWDABw6tQpnDhxAuPHj5cnSi1QHk+G0sxYtvr1vMq+eaNSTnnkkextvI336dD2FrK34WVb+ECD74IjydoZDLEw3p+ckL2Nt+FtfKfeBrk/87fxeb+N3ykqWRolJgsXLlRbtrGxQXR0NKKjo1Vl1tbWWLNmDSZPnqzdCImIiEhnlNpJ/IiIiEj3FGkcEyIiIiI5FGnk19u3b2PXrl2Ij49HVlaW2roFCxZoJTAiIiLSPZITkwMHDqBLly6oWrUqrl69ijp16uDmzZsQQqBBgwZyxEhEREQ6QvKtnMDAQHz99de4dOkSjI2NsXXrViQkJMDLyws9evSQI0YiIiLSEZITkytXrqB///4AAAMDAzx79gzm5uaYMWMG5s6dq/UAiYiISHdITkzMzMxU/UocHR0RGxurWvfw4UPtRUZEREQ6R3Ifk6ZNm+L48eOoVasWOnTogPHjx+PSpUvYtm0bmjZtKkeMREREpCMkJyYLFixAWloaAGD69OlIS0tDaGgoqlevzidyiIiIqFgkJyZVq1ZV/dvMzAwhISFaDYiIiIh0FwdYIyIiolKDiQkRERGVGkxMiIiIqNRgYkJERESlRpETk6ysLMTExODFixfajIeIiIh0mOTEJCMjA4MHD4apqSnc3NwQHx8PABg1ahTmzJmj9QCJiIhIdxRprpwLFy7g8OHDMDY2VpX7+PggNDRUq8ERERGRbpE8jsmOHTsQGhqKpk2bQqFQqMrd3NzUhqcnIiIikkryFZMHDx7Azs4uT3l6erpaokJEREQkleTExNPTE3v27FEt5yYjq1evRrNmzbQXGREREekcybdyZs+ejfbt2yM6OhovXrzA4sWLER0djZMnT+LIkSNyxFgsQggAQGpGpqzt6KU+k7X+t0GZ/lz2Nt7G+5SekSZ7G6lG8r9Xcnsr79N7cF4Ab+e9eh+8jc9b7t+p3L8VuX875JSZXry/S8Xdv7RSiCK8+7GxsZgzZw4uXLiAtLQ0NGjQAJMmTYK7u7scMRbL7du34ezsXNJhEBHROyQhIQEVK1aUpe7nz5+jSpUqSEpKKnZdDg4OiIuLU3sY5V1XpMTkXaJUKnH37l1YWFho1AcmNTUVzs7OSEhIgKWl5VuIUB48jtLjfTgG4P04jvfhGAAeh5yEEHj69CmcnJygpyffGKTPnz9HVlZWsesxNDR8r5ISoAi3coCXf+xv3LiB+/fvQ6lUqq376KOPtBKYtujp6RUp67W0tCw1J0px8DhKj/fhGID34zjeh2MAeBxysbKykr0NY2Pj9y6h0BbJicmpU6fw+eef49atW3nuwSkUCuTk5GgtOCIiItItkhOTYcOGqZ7McXR05CPCREREpDWSE5Pr16/jjz/+QLVq1eSIp8QZGRkhKCgIRkZGJR1KsfA4So/34RiA9+M43odjAHgc9H6T3Pm1devWmDhxItq1aydXTERERKSjNLpicvHiRdW/R40ahfHjxyMpKQnu7u4oU6aM2rZ169bVboRERESkMzS6YqKnpweFQlHggDO569j5lYiIiIpDoysmcXFxcsdBREREpHkfk0GDBmHx4sWwsLCQOyYiIiLSURoPa/fLL7/g2bP3Y96LwixfvhwuLi4wNjZGkyZNcObMmZIOSZLg4GA0atQIFhYWsLOzg5+fH2JiYko6rGKZM2cOFAoFxo4dW9KhSHbnzh307dsXZcuWhYmJCdzd3REREVHSYWksJycHU6ZMQZUqVWBiYgJXV1fMnDnzrcwjUhxHjx5F586d4eTkBIVCgR07dqitF0Jg6tSpcHR0hImJCXx8fHD9+vWSCbYQhR1Hdna2aioQMzMzODk5oX///rh7927JBZyPN30Wrxo2bBgUCgUWLVr01uKj0kfjxKS0/xBpQ2hoKAICAhAUFITIyEjUq1cPvr6+uH//fkmHprEjR45gxIgROHXqFPbv34/s7Gy0bdsW6enpJR1akZw9exY//fTTO9mp+vHjx2jRogXKlCmDv//+G9HR0Zg/fz5sbGxKOjSNzZ07FytWrMCyZctw5coVzJ07F/PmzcPSpUtLOrRCpaeno169eli+fHm+6+fNm4clS5YgJCQEp0+fhpmZGXx9ffH8eemanLGw48jIyEBkZCSmTJmCyMhIbNu2DTExMejSpUsJRFqwN30WubZv345Tp07BycnpLUVGpZbQkEKhEDdu3BApKSmFvt5ljRs3FiNGjFAt5+TkCCcnJxEcHFyCURXP/fv3BQBx5MiRkg5FsqdPn4rq1auL/fv3Cy8vLzFmzJiSDkmSSZMmiZYtW5Z0GMXSsWNHMWjQILWybt26iT59+pRQRNIBENu3b1ctK5VK4eDgIL7//ntV2ZMnT4SRkZHYtGlTCUSomdePIz9nzpwRAMStW7feTlASFXQMt2/fFhUqVBCXL18WlStXFgsXLnzrsVHpIWmGog8++AA2Njb5vqytrd+p/wm+LisrC+fOnYOPj4+qTE9PDz4+PggPDy/ByIonJSUFAGBra1vCkUg3YsQIdOzYUe0zeZfs2rULnp6e6NGjB+zs7ODh4YFVq1aVdFiSNG/eHAcOHMC1a9cAABcuXMDx48fRvn37Eo6s6OLi4pCUlKT2vbKyskKTJk3e6XMdeHm+KxQKWFtbl3QoGlMqlejXrx8mTJgANze3kg6HSgFJI7/+8ccf7+QfOE08fPgQOTk5sLe3Vyu3t7fH1atXSyiq4lEqlRg7dixatGiBOnXqlHQ4kmzevBmRkZE4e/ZsSYdSZP/99x9WrFiBgIAAfPvttzh79ixGjx4NQ0ND+Pv7l3R4Gvnmm2+QmpqKmjVrQl9fHzk5OZg1axb69OlT0qEVWe5U8/md69qYhr6kPH/+HJMmTULv3r1L1YR4bzJ37lwYGBhg9OjRJR0KlRKSEpMWLVrAzs5OrlhIy0aMGIHLly/j+PHjJR2KJAkJCRgzZgz279//Ts++qVQq4enpidmzZwMAPDw8cPnyZYSEhLwzicnvv/+ODRs2YOPGjXBzc0NUVBTGjh0LJyend+YYdEF2djY+++wzCCGwYsWKkg5HY+fOncPixYsRGRnJeddIRdKtnPdZuXLloK+vj3v37qmV37t3Dw4ODiUUVdGNHDkSu3fvxqFDh1CxYsWSDkeSc+fO4f79+2jQoAEMDAxgYGCAI0eOYMmSJTAwMHhnBvFzdHRE7dq11cpq1aqF+Pj4EopIugkTJuCbb75Br1694O7ujn79+mHcuHEIDg4u6dCKLPd8fl/O9dyk5NatW9i/f/87dbXk2LFjuH//PipVqqQ612/duoXx48fDxcWlpMOjEqJxYlK5cmXo6+vLGUuJMjQ0RMOGDXHgwAFVmVKpxIEDB9CsWbMSjEwaIQRGjhyJ7du34+DBg6hSpUpJhyRZmzZtcOnSJURFRalenp6e6NOnD6Kiot6Z72GLFi3yPKp97do1VK5cuYQiki4jIwN6euo/E/r6+lAqlSUUUfFVqVIFDg4Oaud6amoqTp8+/U6d68D/JSXXr1/HP//8g7Jly5Z0SJL069cPFy9eVDvXnZycMGHCBOzdu7ekw6MSovGtHF0Y/TUgIAD+/v7w9PRE48aNsWjRIqSnp2PgwIElHZrGRowYgY0bN2Lnzp2wsLBQ3TO3srKCiYlJCUenGQsLizx9YszMzFC2bNl3qq/MuHHj0Lx5c8yePRufffYZzpw5g5UrV2LlypUlHZrGOnfujFmzZqFSpUpwc3PD+fPnsWDBAgwaNKikQytUWloabty4oVqOi4tDVFQUbG1tUalSJYwdOxbfffcdqlevjipVqmDKlClwcnKCn59fyQWdj8KOw9HREd27d0dkZCR2796NnJwc1flua2sLQ0PDkgpbzZs+i9eTqTJlysDBwQE1atR426FSaVHSjwWVNkuXLhWVKlUShoaGonHjxuLUqVMlHZIkAPJ9rV27tqRDK5Z38XFhIYT4888/RZ06dYSRkZGoWbOmWLlyZUmHJElqaqoYM2aMqFSpkjA2NhZVq1YV//vf/0RmZmZJh1aoQ4cO5Xse+Pv7CyFePjI8ZcoUYW9vL4yMjESbNm1ETExMyQadj8KOIy4ursDz/dChQyUdusqbPovX8XFh0nhIeiIiIiK5sfMrERERlRpMTIiIiKjU0FpiEhERgaNHj2qrOiIiItJBWutjUqtWLVy7du2dGWOCiIiISh+tJSZ3795Fdnb2OzVGAxEREZUufCqHiIiISg3JfUwiIyNx6dIl1fLOnTvh5+eHb7/9FllZWVoNjoiIiHSL5MTkyy+/VE2B/t9//6FXr14wNTXFli1bMHHiRK0HKIebN29CoVAgKirqrbft4uKCRYsWFbqNQqHAjh07AOSN9fDhw1AoFHjy5Imsccptx44dqFatGvT19TF27NgCy+TQqlUrrdZfkt+nd9mr3/N3QWmNd9q0aahfv35Jh1FkAwYMKHTE3XXr1sHa2vqtxVMYIQS++OIL2Nraqs55bf+eaOpt/V6WBMmJybVr11QnwZYtW/DRRx9h48aNWLduHbZu3art+CRzcXGBQqEo8DVgwICSDvGNEhMT0b59+3zXNW/eHImJibCysgKgvZP2bSc8X375Jbp3746EhATMnDmzwLLiKOiYtm3bppX6qXgK+54XhSZJvybe9T/0JJ+wsDCsW7cOu3fvRmJiIurUqZPn90Rb38M30fbvZWmSZ66crKysQudYEEKoJvD6559/0KlTJwCAs7MzHj58KFOYmjt79qzqyaCTJ0/i008/RUxMjGrGTRMTEzx+/LgkQ3yjwmY4NTQ0fCdnQH1VWloa7t+/D19fXzg5ORVYJhdbW1tZ6yfNvOvfY9I9sbGxcHR0RPPmzVVlJfF78jZ/L1/3phxBK3V5eXmJESNGiDFjxoiyZcuKVq1aCSGEOHz4sGjUqJEwNDQUDg4OYtKkSSI7O1t4e3uL/v37i3Xr1gl9fX3VvDLly5cX1tbWqrHu4+PjRY8ePYSVlZWwsbERXbp0EXFxcar1hw4dEo0aNRKmpqbCyspKNG/eXNy8eVMIIURUVJRo1aqVMDc3FxYWFqJBgwbi7Nmzksfbz52j4fHjx2rluXNMbN26VbRq1UqYmJiIunXripMnT6ptd+zYMdGyZUthbGwsKlasKEaNGiXS0tIKbO/GjRuiS5cuws7OTpiZmQlPT0+xf/9+tW0qV64sZsyYIXr16iVMTU2Fk5OTWLZsmdo2AMT27dvVYj1//nyeY8pvDoqgoCAxffp04ebmlie+evXqicmTJ+cpz2/Ojdx5LJ4/fy5GjRolypcvL4yMjESLFi3EmTNnCnwPcvcZP368cHJyEqampqJx48aquTvyi7mgMk0+g+fPn4uJEyeKihUrCkNDQ+Hq6ipWr15d6DG9Ou9OYGCgaNy4cZ5jqFu3rpg+fbpqedWqVaJmzZrCyMhI1KhRQyxfvjzP+3f+/HmhVCqFq6ur+P7779XqO3/+vAAgrl+/XuD79vPPP4vatWurzrkRI0ao1t26dUt06dJFmJmZCQsLC9GjRw+RlJSkWh8UFCTq1asnfv75Z+Hs7CzMzMzE8OHDxYsXL8TcuXOFvb29KF++vPjuu+/U2gQgQkJCRMeOHYWJiYmoWbOmOHnypLh+/brw8vISpqamolmzZuLGjRuqffz9/UXXrl3V6hkzZozw8vJSLXt5eYlRo0aJCRMmCBsbG2Fvby+CgoLytJ37PRdCiISEBNGrVy9hY2MjTE1NRcOGDVVzVb3p3PLy8srzeeeSch6vXbu2wHmmAIhVq1YJPz8/YWJiIqpVqyZ27typtv+lS5dEu3bthJmZmbCzsxN9+/YVDx48yLetlJQUYWxsLP766y+18m3btglzc3ORnp4uhBBi4sSJonr16sLExERUqVJFTJ48WWRlZam2z/3sX30vXp9XqmvXrmpz0xR2jhZk/vz5ok6dOsLU1FRUrFhRDB8+XDx9+lTtvbOyshJhYWGiZs2awszMTPj6+oq7d++qtnnx4oUYN26csLKyEra2tmLChAmif//+eb5Pr8qtd/v27aJatWrCyMhItG3bVsTHx6ttt2PHDuHh4SGMjIxElSpVxLRp00R2drZqfXE/P39/f7XvReXKlfO83wV9D2/evCk6deokrK2thampqahdu7bYs2dPgcecnJws+vXrJ6ytrYWJiYlo166duHbtmhCi4N/Q/Lzpd0PTczm/HOF1uXVNmzZNlCtXTlhYWIgvv/xSbV4tTfMNeHl5CXNzczFhwgRx9epVcfXqVXH79m1hamoqvvrqK3HlyhWxfft2Ua5cOREUFCQuXLgg6tSpIwwNDYWxsbFYt26duHHjhujWrZvqBz4rK0vUqlVLDBo0SFy8eFFER0eLzz//XNSoUUNkZmaK7OxsYWVlJb7++mtx48YNER0dLdatWydu3bolhBDCzc1N9O3bV1y5ckVcu3ZN/P777yIqKqrAD7Egb0pMatasKXbv3i1iYmJE9+7dReXKlVVf5Bs3bggzMzOxcOFCce3aNXHixAnh4eEhBgwYUGB7UVFRIiQkRFy6dElcu3ZNTJ48WRgbG6uOS4iXiYmFhYUIDg4WMTExYsmSJUJfX1/s27dPtY2miUlmZqZYtGiRsLS0FImJiSIxMVE8ffpUJCQkCD09PbUEIjIyUigUChEbG5sn7hcvXoitW7cKACImJkYkJiaKJ0+eCCGEGD16tHBychJ//fWX+Pfff4W/v7+wsbERjx49KvB9GDJkiGjevLk4evSouHHjhvj++++FkZGRuHbtmsjMzBQxMTGqxDAxMbHAMk0+g88++0w4OzuLbdu2idjYWPHPP/+IzZs3F3pMr/6QXL58WQBQ+8ObW5abRPz222/C0dFRbN26Vfz3339i69atwtbWVqxbty7fz2jWrFmidu3aau/J6NGjxUcffVTge/bjjz8KY2NjsWjRIhETEyPOnDmjmsgsJydH1K9fX7Rs2VJERESIU6dOiYYNG6r9eAQFBQlzc3PRvXt38e+//4pdu3YJQ0ND4evrK0aNGiWuXr0q1qxZIwCoTUwJQFSoUEGEhoaKmJgY4efnJ1xcXETr1q1FWFiYiI6OFk2bNhXt2rVT7aPpj5mlpaWYNm2auHbtmvjll1+EQqEo8Hv+9OlTUbVqVfHhhx+KY8eOievXr4vQ0FDVfxbedG49evRIVKxYUcyYMUN1Lggh/TzOyMgQ48ePF25ubqp6MjIyVPFWrFhRbNy4UVy/fl2MHj1amJubq86Fx48fi/Lly4vAwEBx5coVERkZKT7++GPh7e1d4OfevXt30bdvX7WyTz/9VK1s5syZ4sSJEyIuLk7s2rVL2Nvbi7lz56p99lITk8LO0YIsXLhQHDx4UMTFxYkDBw6IGjVqiOHDh6vWr127VpQpU0b4+PiIs2fPinPnzolatWqJzz//XLXN3LlzhY2Njdi6dauIjo4WgwcPFhYWFm9MTMqUKSM8PT3FyZMnRUREhGjcuLFo3ry5apujR48KS0tLsW7dOhEbGyv27dsnXFxcxLRp01TbFPfze/LkiZgxY4aoWLGiSExMFPfv38/zfhf0PezYsaP4+OOPxcWLF0VsbKz4888/xZEjRwo85i5duohatWqJo0ePiqioKOHr6yuqVasmsrKyCvy9fJ0mvxuansuv5wj58ff3F+bm5qJnz57i8uXLYvfu3aJ8+fLi22+/LbSu/PINeHl5CQ8PD7UGvv32W1GjRg2hVCpVZcuXLxfm5uYiJydHpKamCiMjI7Fq1SrV+mfPnqmy+F9//TXP/pmZmcLExETs3btXPHr0SAAQhw8fzvcALSwsVD/6xfGmxGT16tWqsn///VcAEFeuXBFCCDF48GDxxRdfqO137NgxoaenJ549e6ZxDG5ubmLp0qWq5cqVK6v9yAshRM+ePUX79u1Vy5omJkL83/8mXte+fXu1H41Ro0YVmOnmV68QQqSlpYkyZcqIDRs2qMqysrKEk5OTmDdvXr713Lp1S+jr64s7d+6olbdp00YEBgYKIV7+ALye5edX9qbPIPfkfP2qVGHHJETeH+569eqJGTNmqJYDAwNFkyZNVMuurq5i48aNanXMnDlTNGvWTAiR9zO6c+eO0NfXF6dPnxZCvHzPypUrV+h32snJSfzvf//Ld92+ffuEvr6+2v8Qc7+vuclnUFCQMDU1FampqaptfH19hYuLi8jJyVGV1ahRQwQHB6uWAahdRQsPDxcAxM8//6wq27RpkzA2NlYta/pj1rJlS7VtGjVqJCZNmqTWdu73/KeffhIWFhaFJryvy+/cen1W2qKcx6//oX813lffq7S0NAFA/P3330KIl9+Jtm3bqu2TkJCgSo7zs337drWrI7lXUXLrzM/3338vGjZsWGC8b0pMNDlHNbFlyxZRtmxZ1XLu1aZXk/zly5cLe3t71bKjo6Pab0d2draoWLHiGxOT1xPqK1euCACqc6xNmzZi9uzZavv9+uuvwtHRUbWsjc9v4cKFqisluV5/v/P7Hrq7u6slSYW5du2aACBOnDihKnv48KEwMTERv//+uxAi/9/L12nyu6Hpufx6jpAff39/YWtrq/ouCyHEihUrVHlDQXXll2/oAUDDhg3Vbu9cuXIFzZo1g0KhUJW1aNECaWlpOHPmDA4fPozMzEy0adMGZ86cwdixY7F+/XqUKVMGAHDhwgXcuHEDFhYWMDc3h7m5OWxtbfH8+XPExsbC1tYWAwYMgK+vLzp37ozFixcjMTFR1VZAQACGDBkCHx8fzJkzB7GxsZBD3bp1Vf92dHQEANy/f191DOvWrVPFb25uDl9fXyiVSsTFxeVbX1paGr7++mvUqlUL1tbWMDc3x5UrVxAfH6+2XbNmzfIsX7lyRZuHhqFDh2LTpk14/vw5srKysHHjRgwaNEhSHbGxscjOzkaLFi1UZWXKlEHjxo0LjPfSpUvIycnBBx98oPbeHTlyRPLn+KbPICoqCvr6+vDy8pJU7+v69OmDjRs3AnjZh2rTpk3o06cPACA9PR2xsbEYPHiwWhzfffddgcfj5OSEjh07Ys2aNQCAP//8E5mZmejRo0e+29+/fx93795FmzZt8l1/5coVODs7w9nZWVVWu3ZtWFtbq30OLi4usLCwUC3b29ujdu3a0NPTUyvL/Y7nevU8sLe3BwC4u7urlT1//hypqan5xleQV+sFXp5jr7edKyoqCh4eHgXer9f03HpdUc5jTY/JzMwMlpaWar8Zhw4dUmurZs2aAFDgd6VDhw4oU6YMdu3aBQDYunUrLC0t4ePjo9omNDQULVq0gIODA8zNzTF58uQ3HndhinqO/vPPP2jTpg0qVKgACwsL9OvXD48ePUJGRoZqG1NTU7i6uqqWX/3MU1JSkJiYiCZNmqjWGxgYwNPT840xGxgYoFGjRqrlmjVrqn3/L1y4gBkzZqgdz9ChQ5GYmKgWn7Y/P02NHj0a3333HVq0aIGgoCBcvHixwG2vXLkCAwMDtfepbNmyqFGjhqS/E5r+bmji9RyhIPXq1YOpqalquVmzZkhLS0NCQkKBdeWXbxgALz8gTY0ePVr1aNeDBw/w8ccfw83NDRs2bEBSUhKmTp2KtLQ0NGzYEBs2bMizf/ny5QEAa9euxejRoxEWFobQ0FBMnjwZ+/fvR9OmTTFt2jR8/vnn2LNnD/7++28EBQVh8+bN+OSTTzSOUxO5iRQA1ZuS27E3LS0NX375JUaPHp1nv0qVKuVb39dff439+/fjhx9+QLVq1WBiYoLu3buXyPgunTt3hpGREbZv3w5DQ0NkZ2eje/fusreblpYGfX19nDt3Dvr6+mrrzM3NJddV2Gdw48aNYsWaq3fv3pg0aRIiIyPx7NkzJCQkoGfPnqoYAGDVqlVqPxQA8hzfq4YMGYJ+/fph4cKFWLt2LXr27Kl2wr7KxMREK8fx6vcZePmdzq8s9zue336550Fh54aenh7Ea+MyZmdnaxTP623netN7UNRzqyjncWEKO6a0tDR07twZc+fOzbNf7n98XmdoaIju3btj48aN6NWrFzZu3IiePXvCwODlcwnh4eHo06cPpk+fDl9fX1hZWWHz5s2YP39+gTG+6fMpyjl68+ZNdOrUCcOHD8esWbNga2uL48ePY/DgwcjKylJ9t/N7f16PRQ5paWmYPn06unXrlmedsbGx6t/a/vw0NWTIEPj6+mLPnj3Yt28fgoODMX/+fIwaNapY9RaXpueylBzhTTSpK89TOcDLeW+2bt0KIYTqR+nEiROwsLDA9evX0alTJ3z33XdYuHAh6tSpgxMnTmDfvn0YNmwYpk6digYNGiA0NBR2dnaqp2Hy4+HhAQ8PDwQGBqJZs2bYuHEjmjZtCgD44IMP8MEHH2DcuHHo3bs31q5dq/XEpDANGjRAdHQ0qlWrpvE+J06cwIABA1RxpqWl4ebNm3m2O3XqVJ7lWrVqFSlOQ0PDfOcnMjAwgL+/P9auXQtDQ0P06tWr0B//3J7Rr9bl6uoKQ0NDnDhxQjXVQHZ2Ns6ePVvgc/MeHh7IycnB/fv38eGHHxbpmHK96TNwd3eHUqnEkSNH1P6HWdgx5adixYrw8vLChg0b8OzZM3z88cews7MD8PJqgZOTE/777z/VVRRNdOjQAWZmZlixYgXCwsIKneDSwsICLi4uOHDgALy9vfOsr1WrFhISEpCQkKD63090dDSePHmC2rVraxyTtpQvXx6XL19WK4uKisrzoy9F3bp1sXr1aiQnJ+d71USTcyu/c6Eo53FB59SbNGjQAFu3boWLi4sqsdBEnz598PHHH+Pff//FwYMH8d1336nWnTx5EpUrV8b//vc/VdmtW7cKra98+fJqV6BzcnJw+fJl1XerKOfouXPnoFQqMX/+fNUVuN9//13jYwQAKysrODo64vTp0/joo48AAC9evMC5c+fQoEGDQvd98eIFIiIi0LhxYwBATEwMnjx5ovrdbNCgAWJiYiR9zq8r6uf3uoK+P87Ozhg2bBiGDRuGwMBArFq1Kt/EpFatWnjx4gVOnz6tevrn0aNHiImJkXS+a/K7oe1z+cKFC3j27Jnqb82pU6dgbm6udtUmvzhfzzfyHcfkq6++QkJCAkaNGoWrV69i586dCAoKQkBAALKzs2FpaYlJkyZh69atqFixImJjY5GRkYHbt28DeHmilStXDl27dsWxY8cQFxeHw4cPY/To0bh9+zbi4uIQGBiI8PBw3Lp1C/v27cP169dRq1YtPHv2DCNHjsThw4dx69YtnDhxAmfPnlV9Ae/cuYOaNWvizJkzRXrjNDVp0iScPHkSI0eORFRUFK5fv46dO3di5MiRBe5TvXp1bNu2DVFRUbhw4QI+//zzfP+HeOLECcybNw/Xrl3D8uXLsWXLFowZM6ZIcbq4uCAtLQ0HDhzAw4cP1S5bDhkyBAcPHkRYWNgbb+NUrlwZCoUCu3fvxoMHD5CWlgYzMzMMHz4cEyZMQFhYGKKjozF06FBkZGRg8ODB+dbzwQcfoE+fPujfvz+2bduGuLg4nDlzBsHBwdizZ4+kY3vTZ+Di4gJ/f38MGjQIO3bsUH3Pcn8w8zumgvTp0webN2/Gli1b8iQg06dPR3BwMJYsWYJr167h0qVLWLt2LRYsWFBgffr6+hgwYAACAwNRvXr1PLfvXjdt2jTMnz8fS5YswfXr1xEZGYmlS5cCAHx8fODu7o4+ffogMjISZ86cQf/+/eHl5aXRZXBta926NSIiIrB+/Xpcv34dQUFBeX7cpOrduzccHBzg5+eHEydO4L///sPWrVsRHh4OQLNzy8XFBUePHsWdO3dUQxcU5Tx2cXFR3Sp8+PAhMjMzNTqGESNGIDk5Gb1798bZs2cRGxuLvXv3YuDAgYUmOh999BEcHBzQp08fVKlSRe3KXPXq1REfH4/NmzcjNjYWS5Yswfbt2wuNo3Xr1tizZw/27NmDq1evYvjw4Wpj+RTlHK1WrRqys7OxdOlS/Pfff/j1118REhKi0fvyqjFjxmDOnDnYsWMHrl69iq+++kqjsZPKlCmDUaNG4fTp0zh37hwGDBiApk2bqhKVqVOnYv369Zg+fTr+/fdfXLlyBZs3b8bkyZM1jq2on9/r8vsejh07Fnv37kVcXBwiIyNx6NChAv8zWr16dXTt2hVDhw7F8ePHceHCBfTt2xcVKlRA165dNY5Dk98NbZ/LWVlZGDx4MKKjo/HXX38hKCgII0eOVLud/Lr88g3k11FKiIIfF27cuLGYNGmSOHz4sDAwMBCOjo6iTJkywt7eXlhaWqr2T0xMFP379xflypUTRkZGomrVqmLo0KEiJSVFJCUlCT8/P+Ho6CgMDQ1F5cqVxdSpU0VOTo7IzMwUvXr1Es7OzsLQ0FA4OTmJkSNHqjqq5XY0fNOjbUK8ufNrbmdFIfLvTHTmzBnx8ccfC3Nzc2FmZibq1q0rZs2aVWB7cXFxwtvbW5iYmAhnZ2exbNmyfDtGTZ8+XfTo0UOYmpoKBwcHsXjxYrV6IKHzqxBCDBs2TJQtW1b1uPCrPvzww3wfHc7PjBkzhIODg1AoFKqOcs+ePROjRo1SfY6aPC6clZUlpk6dKlxcXESZMmWEo6Oj+OSTT8TFixeFEJp3fhXizZ/Bs2fPxLhx41TfpWrVqok1a9YUekz5fecfP34sjIyMhKmpqdojkLk2bNgg6tevLwwNDYWNjY346KOPxLZt24QQ+X+fhBAiNjZWACiwo/DrQkJCRI0aNVTv2ahRo1TrNH1c+FX5dWx7/dhf/a4VdCz5feemTp0q7O3thZWVlRg3bpwYOXJkng5zb3oy5PW2b968KT799FNhaWkpTE1Nhaenp6pzoybnVnh4uKhbt64wMjJSe1xY6nn8/Plz8emnnwpra+s8jwu/Gq8QQlhZWanWC/Gy4+Inn3yiesyzZs2aYuzYsWod+/IzceJEAUBMnTo1z7oJEyaIsmXLqp54WLhwoVqH99c/+6ysLDF8+HBha2sr7OzsRHBwcJ73/k3naH4WLFggHB0dhYmJifD19RXr169/Y0f87du3q30W2dnZYsyYMcLS0lJYW1uLgIAAjR8X3rp1q6hataowMjISPj4+ak87CiFEWFiYaN68uTAxMRGWlpaicePGYuXKlar12vj8NOn8mt/3cOTIkcLV1VUYGRmJ8uXLi379+omHDx8WeMy5jwtbWVmp3u9Xn5jSpPOrEG/+3RCiaOdyfnJ/b6ZOnar6vg4dOlQ8f/78jXW9nm9InsTv8OHD+OSTT5Camgp/f39VB79vv/0WV69exbZt26RURzISQqB69er46quvEBAQUNLh6Jxjx46hTZs2SEhIUHUqJSJ6Hw0YMABPnjzRyrQNkm+ktWrVCg8fPkRqaipsbGxU5V988UWBnfvo7Xvw4AE2b96MpKQkDBw4sKTD0SmZmZl48OABpk2bhh49ejApISKSoEg9fIQQOHfuHGJjY/H555/DwsIChoaGTExKETs7O5QrVw4rV65USyBJfps2bcLgwYNRv359rF+/vqTDISJ6p0i+lXPr1i20a9cO8fHxyMzMxLVr11C1alWMGTMGmZmZReoQRURERAQUYXbhMWPGwNPTE48fP1Z7/PSTTz7BgQMHtBocERER6RbJt3KOHTuGkydP5pkR0MXFBXfu3NFaYERERKR7JF8xUSqV+T7Tffv2bbXhsImIiIikkpyYtG3bFosWLVItKxQKpKWlISgoCB06dNBmbERERKRjJHd+vX37Nnx9fSGEwPXr1+Hp6Ynr16+jXLlyOHr0qGoobyIiIiKpJCcmwMt5C0JDQ3HhwgWkpaWhQYMG6NOnj9YmIyMiIiLdVKTEhIiIiEgOkvuYBAcHq4ahf9WaNWvynS6aiIiISFOSE5OffvoJNWvWzFPu5ubGwdWIiIioWP4fhnpJKChAMh4AAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate the attention score\n",
    "attention_score = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(d_model // num_heads) # [4, 4, 16, 16]\n",
    "\n",
    "# Illustration only\n",
    "plt.imshow(attention_score[1, 1].detach().cpu().numpy(), \"Accent\", aspect=\"auto\")\n",
    "plt.title(\"Attention(Q @ K)\") #plot attention in the first head of the first batch\n",
    "plt.xlabel(encoding.decode(x_batch[0].tolist()))\n",
    "plt.ylabel(encoding.decode(x_batch[0].tolist()))\n",
    "plt.colorbar()\n",
    "pd.DataFrame(attention_score[0][0].detach().cpu().numpy())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:31:11.625621Z",
     "start_time": "2024-02-09T04:31:11.196103Z"
    }
   },
   "id": "6dd5217e3fa43d9",
   "execution_count": 26
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n0   0.827912      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n1   1.123528  0.464093      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n2   1.040623  0.605709  0.119165      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n3   0.900045  0.596818  0.359724  0.836026      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n4   1.061119  1.158659  0.633724 -0.320907  0.383561      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n5   0.071058  0.273445  0.044782  0.250272 -0.114290  0.121239      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n6   0.897961  1.322748  0.553525  0.079147  0.502846 -0.506932  0.223247      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n7   0.025299  0.779429  0.627483 -0.354743  0.218848 -0.055096  0.297431 -0.119230      -inf      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n8   0.529123 -0.196815  0.232964  0.031045  0.214278 -0.328799 -0.685732  0.060671  0.237945      -inf      -inf      -inf      -inf      -inf      -inf      -inf\n9   0.935247  0.690427  0.511019  1.063499  0.504674 -0.227777 -0.035515 -0.006277  0.174739  0.899135      -inf      -inf      -inf      -inf      -inf      -inf\n10  0.467166  0.532110  0.238954  0.085494  0.207894 -0.739952 -0.371635 -0.087244 -0.217038  0.353716  0.276079      -inf      -inf      -inf      -inf      -inf\n11  0.016316 -0.238568 -0.610145 -0.418894 -0.609519 -0.072247 -0.179781 -0.085679  0.529486  0.309438  0.002346 -0.154983      -inf      -inf      -inf      -inf\n12  0.210837  0.166079  0.430884  0.002882  0.447536  0.287130 -0.069169  0.463648 -0.251514  0.225306  0.302280  0.252891  0.182381      -inf      -inf      -inf\n13 -0.131177  0.249064  0.690767 -0.918174  0.273945 -0.226493 -0.467489  0.787192 -0.527138 -0.107953  0.469084  0.095217  0.443432  0.514454      -inf      -inf\n14 -0.268342  0.041401  0.530629 -0.301306  0.030230 -0.376372 -0.326081  0.092524 -0.112594 -0.274940  0.107115  0.171046 -0.394140 -0.418468 -0.190215      -inf\n15 -0.219957 -0.423727  0.599078 -0.366324  0.452232  0.868501 -0.051142  0.200568  0.335000  0.050063  0.305444  0.225228 -0.176068  0.332432  0.156352  0.471814",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n      <th>11</th>\n      <th>12</th>\n      <th>13</th>\n      <th>14</th>\n      <th>15</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.827912</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.123528</td>\n      <td>0.464093</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.040623</td>\n      <td>0.605709</td>\n      <td>0.119165</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.900045</td>\n      <td>0.596818</td>\n      <td>0.359724</td>\n      <td>0.836026</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.061119</td>\n      <td>1.158659</td>\n      <td>0.633724</td>\n      <td>-0.320907</td>\n      <td>0.383561</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.071058</td>\n      <td>0.273445</td>\n      <td>0.044782</td>\n      <td>0.250272</td>\n      <td>-0.114290</td>\n      <td>0.121239</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.897961</td>\n      <td>1.322748</td>\n      <td>0.553525</td>\n      <td>0.079147</td>\n      <td>0.502846</td>\n      <td>-0.506932</td>\n      <td>0.223247</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.025299</td>\n      <td>0.779429</td>\n      <td>0.627483</td>\n      <td>-0.354743</td>\n      <td>0.218848</td>\n      <td>-0.055096</td>\n      <td>0.297431</td>\n      <td>-0.119230</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.529123</td>\n      <td>-0.196815</td>\n      <td>0.232964</td>\n      <td>0.031045</td>\n      <td>0.214278</td>\n      <td>-0.328799</td>\n      <td>-0.685732</td>\n      <td>0.060671</td>\n      <td>0.237945</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>0.935247</td>\n      <td>0.690427</td>\n      <td>0.511019</td>\n      <td>1.063499</td>\n      <td>0.504674</td>\n      <td>-0.227777</td>\n      <td>-0.035515</td>\n      <td>-0.006277</td>\n      <td>0.174739</td>\n      <td>0.899135</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>0.467166</td>\n      <td>0.532110</td>\n      <td>0.238954</td>\n      <td>0.085494</td>\n      <td>0.207894</td>\n      <td>-0.739952</td>\n      <td>-0.371635</td>\n      <td>-0.087244</td>\n      <td>-0.217038</td>\n      <td>0.353716</td>\n      <td>0.276079</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>0.016316</td>\n      <td>-0.238568</td>\n      <td>-0.610145</td>\n      <td>-0.418894</td>\n      <td>-0.609519</td>\n      <td>-0.072247</td>\n      <td>-0.179781</td>\n      <td>-0.085679</td>\n      <td>0.529486</td>\n      <td>0.309438</td>\n      <td>0.002346</td>\n      <td>-0.154983</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>0.210837</td>\n      <td>0.166079</td>\n      <td>0.430884</td>\n      <td>0.002882</td>\n      <td>0.447536</td>\n      <td>0.287130</td>\n      <td>-0.069169</td>\n      <td>0.463648</td>\n      <td>-0.251514</td>\n      <td>0.225306</td>\n      <td>0.302280</td>\n      <td>0.252891</td>\n      <td>0.182381</td>\n      <td>-inf</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>-0.131177</td>\n      <td>0.249064</td>\n      <td>0.690767</td>\n      <td>-0.918174</td>\n      <td>0.273945</td>\n      <td>-0.226493</td>\n      <td>-0.467489</td>\n      <td>0.787192</td>\n      <td>-0.527138</td>\n      <td>-0.107953</td>\n      <td>0.469084</td>\n      <td>0.095217</td>\n      <td>0.443432</td>\n      <td>0.514454</td>\n      <td>-inf</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>-0.268342</td>\n      <td>0.041401</td>\n      <td>0.530629</td>\n      <td>-0.301306</td>\n      <td>0.030230</td>\n      <td>-0.376372</td>\n      <td>-0.326081</td>\n      <td>0.092524</td>\n      <td>-0.112594</td>\n      <td>-0.274940</td>\n      <td>0.107115</td>\n      <td>0.171046</td>\n      <td>-0.394140</td>\n      <td>-0.418468</td>\n      <td>-0.190215</td>\n      <td>-inf</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>-0.219957</td>\n      <td>-0.423727</td>\n      <td>0.599078</td>\n      <td>-0.366324</td>\n      <td>0.452232</td>\n      <td>0.868501</td>\n      <td>-0.051142</td>\n      <td>0.200568</td>\n      <td>0.335000</td>\n      <td>0.050063</td>\n      <td>0.305444</td>\n      <td>0.225228</td>\n      <td>-0.176068</td>\n      <td>0.332432</td>\n      <td>0.156352</td>\n      <td>0.471814</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 640x480 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAHHCAYAAACLPpP8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAByvElEQVR4nO3deXxM1/8/8Ndk3zdkIxJCLRGE1F6xhNhFS+1ibanYomj6UbEHLYIqpUWplKqlWm1Kg6hdRGwhiCBIxB5JSCJzfn/4Zb5GFnOTuRLm9Xw85vFwz733nPedmTt5u/fccxRCCAEJ9PX1kZycDHt7e7Xy+/fvw97eHrm5uVKqIyIiIlLRk7pDYXlMVlYWjIyMShwQERER6S4DTTdcunQpAEChUOCHH36AhYWFal1ubi4OHDiAmjVraj9CIiIi0hkKTW/lVKlSBQBw/fp1VKpUCfr6+qp1RkZGcHNzw8yZM9G4cWN5IiUiIqJ3nsaJSZ7WrVtj27ZtsLW1lSsmIiIi0lGSExMiIiIiuWjUxyQoKAizZs2Cubk5goKCitx20aJFWgmMiIiIdI9GicmpU6eQk5MDAIiJiYFCoShwu8LKiYiIiDSh0a2cM2fOoE6dOtDTk/x0MREREZHGNMo0vLy8cO/ePQBA1apVcf/+fVmDIiIiIt2kUWJiY2ODxMREAMC1a9egVCplDYqItEOhUGD69Omyt6NUKlGnTh3MmTNH9raKEhcXBwMDA5w7d65U4yCi4tMoMfnoo4/g4+ODKlWqQKFQwNvbG1WrVi3wRfQu+e6776BQKAocnycuLg7Tp0/HtWvXCtxv3bp18gcI4K+//nojyUdRfvnlFyQlJSEwMDDfuvPnz2PAgAGoWLEijI2N4ezsjAEDBiAuLk7j+hUKRYF1z507FwqFAkOHDoVSqUTt2rXRuXNnTJs2rUTHQ0SlR+PHhSMiInDlyhWMHTsWM2fOhKWlZYHbjRs3TqsBEpWm5s2b4/bt27h27RouX76MatWqqdb99ttv6NWrF/bt24dWrVqp7VenTh2UL18e+/fvlz3GwMBALF++vMDpIp49ewYDAwMYGGg8yHOx1K9fH40bN8b333+vVr5t2zb07dsXdnZ2GDZsGKpUqYJr167hxx9/xIMHD7B582Z07979tfUrFAqMHj0a3377raps3rx5CA4ORkBAANasWaPqA/f333+jU6dOuHLlCtzd3bV7oEQkPyHR4MGDRVpamtTdiN46V69eFQDEtm3bRIUKFcT06dPV1m/ZskUAEPv27cu3r4eHh/Dx8XkjcY4ePVoU41TWmpiYGAFA/Pvvv2rlV65cEWZmZqJmzZoiNTVVbd3du3dFzZo1hYWFhbh69epr2wAgRo8erVpesGCBACAGDRokcnNz1bbNzs4Wtra24quvvirBURFRaSn2r9nly5dFRESEyMzMFEIIoVQqtRYUUVkwa9YsYWtrK7KyssSoUaNE9erVVevWrl0rAOR77du3T7i6uuYrfzlJefjwoRg3bpyoVKmSMDIyEu7u7mLevHlqf2ATExMFAPH111+L77//XlStWlUYGRkJb29vcfz4cdV2AQEBBcaRB4AICQlRO66YmBjRoUMHYWlpKczNzUWbNm3EkSNH1LbJO76DBw+KCRMmiPLlywszMzPh7++fL8mYNm2aMDIyEtnZ2Wrln376qQAgDhw4UOD7GxUVJQCIUaNGFf1BCPXEZOHChQKAGDBgQL6kJE+PHj1E3bp1X1svEZU9khOT+/fvizZt2giFQiH09PREQkKCEEKIIUOGiKCgIK0HSFRaatasKYYNGyaEEOLAgQMCgCopSEhIEGPHjhUAxJdffik2bNggNmzYIFJSUsT27dtFpUqVRM2aNVXlu3fvFkIIkZGRIerWrSvKlSsnvvzyS7Fy5UoxaNAgoVAoxLhx41Rt5yUmXl5eolq1amL+/PliwYIFonz58qJSpUqqJODw4cOiXbt2AoCqrQ0bNqjqeTUxOXfunDA3NxdOTk5i1qxZYt68eaJKlSrC2NhYHD16VLVdXmLi5eUl2rRpI5YtWyYmTpwo9PX1xccff6z2Pvn6+ooGDRrke/+cnZ2Fm5tbke+xm5ubqFSp0ms/i7zEJCwsTAAQ/fr1E8+fPy90+9mzZws9PT3x+PHj19ZNRGWL5MRk4MCBws/PTyQlJQkLCwtVYhIRESFq166t9QCJSkN0dLQAIPbs2SOEeHFFsFKlSmrJQ3Fu5cyaNUuYm5uLS5cuqZV/8cUXQl9fX9y4cUMI8X+JSbly5cSDBw9U2/3+++8CgPjjjz9UZUXdynk1MfH39xdGRkaq81YIIW7fvi0sLS1Fy5YtVWV5iYmvr6/a1dAJEyYIfX198ejRI1VZpUqVxEcffaTW7qNHjwQA0b179wLjytOtWzcB4LW3hwGorkT17du3yKRECCHCw8MFAHHs2LEityOiskfyiGm7d+/G/PnzUalSJbXy6tWr4/r161KrIyqTNm7cCAcHB7Ru3RrAi86XvXv3xqZNm5Cbm1vserds2YIPPvgAtra2uHfvnurl6+uL3NxcHDhwQG373r17q02Y+cEHHwAArl69Krnt3Nxc7N69G/7+/mpP0Dk5OaFfv344ePAg0tLS1Pb55JNP1EZ0/uCDD5Cbm6t2rt+/fz/fpJ5PnjwBgEI7yefJW5+3fVHu3LkD4MVM5y/Pbl6QvHjyxl8ioreH5K76GRkZMDMzy1f+4MEDGBsbayUootKUm5uLTZs2oXXr1qrxewCgcePGWLhwISIjI9G+ffti1X358mWcOXMGFSpUKHB9amqq2nLlypXVlvP+4D58+FBy23fv3kVmZiZq1KiRb12tWrWgVCqRlJQEDw8Pye2LV54I0jThePLkCRQKBcqXL//a+AMCAnD79m3MnTsX5cuXx4QJEwrdNi8eTpNB9PaRnJh88MEHWL9+PWbNmgXgxYmvVCqxYMEC1f8uid5me/fuRXJyMjZt2oRNmzblW79x48ZiJyZKpRLt2rXD5MmTC1z/3nvvqS0XdmXg1URALpq0X65cuXyJirW1NZydnXHmzJki6z9z5gwqVaoEIyOj18ZiYGCAX3/9FR06dMDEiRNhY2ODIUOGFLhtXjyaJDxEVLZITkwWLFiAtm3bIjo6GtnZ2Zg8eTLOnz+PBw8e4NChQ3LESPRGbdy4Efb29li+fHm+ddu2bcP27duxcuXKIv83Xtg6d3d3pKenw9fXV2vxanpVoEKFCjAzM0N8fHy+dRcvXoSenh5cXFwkt1+zZk21K0t5unbtiu+//x4HDx5EixYt8q3/77//cO3atdfOWP4yExMT7Ny5E61bt8aIESNgY2ODHj165NsuMTERenp6+RI9Iir7JPcxqVOnDi5duoQWLVqge/fuyMjIwIcffohTp05xMCN66z19+hTbtm1Dly5d0LNnz3yvwMBAPHnyBDt37oS5uTkA4NGjR/nqMTc3L7D8448/xpEjR/DPP//kW/fo0SM8f/5ccsxFxfEyfX19tG/fHr///rvaaLV37txBeHg4WrRoASsrK8ntN23aFOfOnUNWVpZa+eeffw4zMzN8+umn+ebXevDgAUaOHAkrK6sCR3QtipWVFSIiIlCtWjX07dsXkZGR+bY5efIkPDw8YG1tLfl4iKh0FWs4SGtra/zvf//TdixEpW7nzp148uQJunXrVuD6Jk2aoEKFCti4cSNWrVoFfX19zJ8/H48fP4axsTHatGkDe3t7NGzYECtWrMDs2bNRrVo12Nvbo02bNpg0aRJ27tyJLl26YPDgwWjYsCEyMjJw9uxZ/Pbbb7h27Zrk2w8NGzYEAIwdOxZ+fn7Q19dHnz59Ctx29uzZ2LNnD1q0aIHPPvsMBgYG+P7775GVlYUFCxZIe7P+v+7du2PWrFmIiopSu8VVrVo1rF+/Hn379oWnp2e+kV8fPnyITZs2oUqVKmr1KRQK+Pj4FDlqboUKFbBnzx40b94c/v7+iIyMRKNGjQAAOTk5iIqKwmeffVas4yGiUlacR3kePnwo/vnnH7Fhwwbx008/qb2I3mZdu3YVJiYmIiMjo9BtBg8eLAwNDcW9e/fE6tWrRdWqVYW+vr7ao8MpKSmic+fOwtLSMt8Aa0+ePBHBwcGiWrVqwsjISJQvX140a9ZMfPPNN6rxSV4eYO1VeOUR4OfPn4sxY8aIChUqCIVCodEAa35+fsLCwkKYmZmJ1q1bi8OHD6ttk/e48IkTJ9TK9+3bV+Aj0nXr1lWN+fKqs2fPin79+glHR0ehp6cnAAgTExNx/vz5fNs+efJEABB9+vTJd8wvj/ya58KFC6J8+fLCzs5OnDt3TgghxN9//y0AiMuXLxcYDxGVbRrPlZPnjz/+QP/+/ZGeng4rKyu1+9sKhQIPHjzQSsJERG+PDRs2YPTo0bhx4wZsbGyK3Hb9+vUYPHgwBgwYgPXr16ut++uvv9ClSxecPn0anp6exYrF398fCoUC27dvL9b+RFS6JCcm7733Hjp16oS5c+cW+NgwEekepVKJunXrom/fvhrd5p0/fz6++OILBAcHY+7cuarySZMm4datWwgPDy9WHBcuXICnpydiY2NRp06dYtVBRKVLcmJibm6Os2fPqg3QRERERKQNkp/K8fPzQ3R0tByxEBERkY6T/FRO586dMWnSJMTFxcHT0xOGhoZq6wt7moGIiIjodSTfytHTK/wii0KhKNE8IkRERKTbJCcmRERERHIp1gBreZ49ewYTExNtxSILpVKJ27dvw9LSkhN6ERFRkYQQePLkCZydnYu8Q1BSz549Q3Z2donrMTIyKvN/h6WSnJjk5uZi7ty5WLlyJe7cuYNLly6hatWq+Oqrr+Dm5oZhw4bJEWex3b59u1jzfxARke5KSkpCpUqVZKn72bNnsLOzw9OnT0tcl6OjIxITE9+p5ERyYjJnzhz89NNPWLBgAUaMGKEqr1OnDsLCwspcYpI3/frmVZEwMzOXrZ2WXWvJVjcREb0ZaWlpcHFxUf3tkEN2djaePn2Kfv36aTSzdlH1hIeHIzs7W6PEJDQ0FNu2bcPFixdhamqKZs2aYf78+ahRo0aR+23ZsgVfffUVrl27hurVq2P+/Pno1KlTseN+HcmJyfr167Fq1Sq0bdsWI0eOVJXXq1cPFy9e1Gpw2pB3+8bMzBzmZhaytVOcyc+IiKhsehO3/o2MjEqUmEgVFRWF0aNH4/3338fz58/x5Zdfon379oiLi1NNBvqqw4cPo2/fvggNDUWXLl0QHh4Of39/xMTEyDaIoeTE5NatW6hWrVq+cqVSiZycHK0ERURERNoVERGhtrxu3TrY29vj5MmTaNmyZYH7LFmyBB06dMCkSZMAALNmzcKePXvw7bffYuXKlbLEKblnT+3atfHff//lK//tt9/g5eWllaCIiIhIM2lpaWqvrKwsjfZ7/PgxAMDOzq7QbY4cOQJfX1+1Mj8/Pxw5cqT4Ab+G5Csm06ZNQ0BAAG7dugWlUolt27YhPj4e69evx59//ilHjERERFSIVx/wCAkJwfTp04vcR6lUYvz48WjevHmRt2RSUlLg4OCgVubg4ICUlJRix/s6khOT7t27448//sDMmTNhbm6OadOmoUGDBvjjjz/Qrl07OWIkIiKiQiQlJan1czQ2Nn7tPqNHj8a5c+dw8OBBOUMrlmI9pP3BBx9gz549SE1NRWZmJg4ePIj27dtrOzaV5cuXw83NDSYmJmjcuDGOHz8uW1tERERvEysrK7XX6xKTwMBA/Pnnn9i3b99rH4l2dHTEnTt31Mru3LkDR0fHEsddmGKPHhMdHY0NGzZgw4YNOHnypDZjUrN582YEBQUhJCQEMTExqFevHvz8/JCamipbm0RERO8aIQQCAwOxfft27N27F1WqVHntPk2bNkVkZKRa2Z49e9C0aVO5wpSemNy8eRMffPABGjVqhHHjxmHcuHF4//330aJFC9y8eVPrAS5atAgjRozAkCFDULt2baxcuRJmZmZYs2aN1tsiIiJ6V40ePRo///wzwsPDYWlpiZSUFKSkpKgN9DZo0CAEBwerlseNG4eIiAgsXLgQFy9exPTp0xEdHY3AwEDZ4pScmAwfPhw5OTm4cOECHjx4gAcPHuDChQtQKpUYPny4VoPLzs7GyZMn1XoE6+npwdfXV9YewURERO+aFStW4PHjx2jVqhWcnJxUr82bN6u2uXHjBpKTk1XLzZo1Q3h4OFatWoV69erht99+w44dO2QbwwQoRufXqKgoHD58WG2kuBo1amDZsmX44IMPtBrcvXv3kJubW2CP4MIGc8vKylJ7VCotLU2rMREREb2NNJmzd//+/fnKevXqhV69eskQUcEkXzFxcXEpcCC13NxcODs7ayWokggNDYW1tbXqxXlyiIiI3h6SE5Ovv/4aY8aMQXR0tKosOjoa48aNwzfffKPV4MqXLw99fX1JPYKDg4Px+PFj1SspKUmrMREREZF8NLqVY2trqzZvQEZGBho3bgwDgxe7P3/+HAYGBhg6dCj8/f21FpyRkREaNmyIyMhIVb1KpRKRkZGFdrwxNjbW6BluIiIiKns0SkzCwsJkDqNwQUFBCAgIgLe3Nxo1aoSwsDBkZGRgyJAhpRYTERERyUOjxCQgIEDuOArVu3dv3L17F9OmTUNKSgrq16+PiIiIfB1iiYiI6O0n+amc0hAYGCjrM9NERERUNhR75FciIiIibWNiQkRERGUGExMiIiIqM5iYEBERUZmhUefXDz/8UOMKt23bVuxgiIiISLdplJhYW1ur/i2EwPbt22FtbQ1vb28AwMmTJ/Ho0SNJCcy7Z9UbaOOTN9AGERFR6dEoMVm7dq3q31OmTMHHH3+MlStXQl9fH8CLeXI+++wzWFlZyRMlERER6QTJfUzWrFmDzz//XJWUAIC+vj6CgoKwZs0arQZHREREukVyYvL8+XNcvHgxX/nFixehVCq1EhQRERHpJskjvw4ZMgTDhg1DQkICGjVqBAA4duwY5s2bx/lriIiIqEQkJybffPMNHB0dsXDhQiQnJwMAnJycMGnSJEycOFHrARIREZHukJyY6OnpYfLkyZg8eTLS0tIAgJ1eiYiISCtKNIkfExIiIiLSJsmdX+/cuYOBAwfC2dkZBgYG0NfXV3sRERERFZfkKyaDBw/GjRs38NVXX8HJyQkKhUKOuIiIiEgHSU5MDh48iP/++w/169eXIRwiIiLSZZJv5bi4uEAIIUcsREREpOMkJyZhYWH44osvcO3aNRnCISIiIl0m+VZO7969kZmZCXd3d5iZmcHQ0FBt/YMHD7QWHBEREekWyYlJWFiYDGEQERERFSMxCQgIkCMOIiIiopINsPbs2TNkZ2erlXHQNSIiIiouyZ1fMzIyEBgYCHt7e5ibm8PW1lbtRURERFRckhOTyZMnY+/evVixYgWMjY3xww8/YMaMGXB2dsb69evliJGIiIh0hORbOX/88QfWr1+PVq1aYciQIfjggw9QrVo1uLq6YuPGjejfv78ccRIREZEOkHzF5MGDB6hatSqAF/1J8h4PbtGiBQ4cOKDd6IiIiEinSE5MqlatisTERABAzZo18euvvwJ4cSXFxsZGq8ERERGRbpGcmAwZMgSnT58GAHzxxRdYvnw5TExMMGHCBEyaNEnrARIREZHukNzHZMKECap/+/r64uLFizh58iSqVauGunXrajU4bWph+xeszE1kbKGcjHW/sG/7eVnrb93DQ9b6iYjo//TM3AXzHMnXB1QycpRYp71wyowSjWMCAK6urnB1ddVGLERERKTjip+qEREREWkZExMiIiIqM5iYEBERUZnBxISIiIjKjGIlJgkJCZg6dSr69u2L1NRUAMDff/+N8+flfWqEiIiI3m2SE5OoqCh4enri2LFj2LZtG9LT0wEAp0+fRkhIiNYDJCIiIt0hOTH54osvMHv2bOzZswdGRkaq8jZt2uDo0aNaDS40NBTvv/8+LC0tYW9vD39/f8THx2u1DSIiIio7JCcmZ8+eRY8ePfKV29vb4969e1oJKk9UVBRGjx6No0ePYs+ePcjJyUH79u2RkZGh1XaIiIiobJA8wJqNjQ2Sk5NRpUoVtfJTp06hYsWKWgsMACIiItSW161bB3t7e5w8eRItW7bUaltERERU+iRfMenTpw+mTJmClJQUKBQKKJVKHDp0CJ9//jkGDRokR4wqjx8/BgDY2dkVuk1WVhbS0tLUXkRERPR2kJyYzJ07FzVr1oSLiwvS09NRu3ZttGzZEs2aNcPUqVPliBEAoFQqMX78eDRv3hx16tQpdLvQ0FBYW1urXi4uLrLFRERERNol+VaOkZERVq9eja+++grnzp1Deno6vLy8UL16dTniUxk9ejTOnTuHgwcPFrldcHAwgoKCVMtpaWlMToiIiN4SxZ7Er3LlyqhcubI2YylUYGAg/vzzTxw4cACVKlUqcltjY2MYGxu/kbiIiIhIuyQnJkOHDi1y/Zo1a4odzKuEEBgzZgy2b9+O/fv35+twS0RERO8WyYnJw4cP1ZZzcnJw7tw5PHr0CG3atNFaYMCL2zfh4eH4/fffYWlpiZSUFACAtbU1TE1NtdoWERERlT7Jicn27dvzlSmVSowaNQru7u5aCSrPihUrAACtWrVSK1+7di0GDx6s1baIiIio9BW7j8nL9PT0EBQUhFatWmHy5MnaqBLAi1s5REREpDu0NrtwQkICnj9/rq3qiIiISAdJvmLy8qO4wIurGsnJydi1axcCAgK0FhgRERHpHsmJyalTp9SW9fT0UKFCBSxcuPC1T+wQERERFUVyYrJv3z454iAiIiLSTufXt8HBh51gnmUhW/2KAdp9VLogX9qukrX+uhFLZa0fAL7//nvZ2yAioreXRomJl5cXFAqFRhXGxMSUKCAiIiLSXRolJv7+/jKHQURERKRhYhISEiJ3HERERETaG8eEiIiIqKQkJya5ubn45ptv0KhRIzg6OsLOzk7tRURERGXPgQMH0LVrVzg7O0OhUGDHjh1Fbr9//34oFIp8r7x56+QiOTGZMWMGFi1ahN69e+Px48cICgrChx9+CD09PUyfPl2GEImIiKikMjIyUK9ePSxfvlzSfvHx8UhOTla97O3tZYrwBcmPC2/cuBGrV69G586dMX36dPTt2xfu7u6oW7cujh49irFjx8oRJxEREZVAx44d0bFjR8n72dvbw8bGRvsBFULyFZOUlBR4enoCACwsLPD48WMAQJcuXbBr1y7tRkdERERFSktLU3tlZWVptf769evDyckJ7dq1w6FDh7Rad0EkJyaVKlVCcnIyAMDd3R27d+8GAJw4cQLGxsbajY6IiIiK5OLiAmtra9UrNDRUK/U6OTlh5cqV2Lp1K7Zu3QoXFxe0atVK9vHKJN/K6dGjByIjI9G4cWOMGTMGAwYMwI8//ogbN25gwoQJcsRIREREhUhKSoKVlZVqWVsXCWrUqIEaNWqolps1a4aEhAQsXrwYGzZs0EobBZGcmMybN0/17969e8PV1RWHDx9G9erV0bVrV60GR0REREWzsrJSS0zk1KhRIxw8eFDWNiQnJs+ePYOJiYlquUmTJmjSpIlWgyIiIqKyJzY2Fk5OTrK2ITkxsbe3R48ePTBgwAC0bdsWenoco42IiKisS09Px5UrV1TLiYmJiI2NhZ2dHSpXrozg4GDcunUL69evBwCEhYWhSpUq8PDwwLNnz/DDDz9g7969qr6lcpGcVfz000/IzMxE9+7dUbFiRYwfPx7R0dFyxEZERERaEh0dDS8vL3h5eQEAgoKC4OXlhWnTpgEAkpOTcePGDdX22dnZmDhxIjw9PeHj44PTp0/j33//Rdu2bWWNs1idX3v06IEnT57gt99+wy+//IImTZqgatWqGDBggOoAiYiIqOxo1aoVhBCFrl+3bp3a8uTJkzF58mSZo8qv2PdhLC0tMWTIEOzevRtnzpyBubk5ZsyYoc3YiIiISMcUOzF59uwZfv31V/j7+6NBgwZ48OABJk2apM3YiIiISMdIvpXzzz//IDw8HDt27ICBgQF69uyJ3bt3o2XLlnLER0RERDqkWH1MunTpgvXr16NTp04wNDSUIy4iIiLSQZITkzt37sDS0lKOWIiIiEjHSe5jwqSEiIiI5MLR0YiIiKjMkHwr523VsutRWFmZyla/0m68bHXn8TM9JWv90566yVo/ACijtDPrZVH0fIJlb4OIiOShM4kJERFRWVJ3yzBYKYo/E3CayAIw77XbvW2KfSvnypUr+Oeff/D06VMAKHI0OSIiIiJNSE5M7t+/D19fX7z33nvo1KkTkpOTAQDDhg3DxIkTtR4gERER6Q7JicmECRNgYGCAGzduwMzMTFXeu3dvREREaDU4IiIi0i2S+5js3r0b//zzDypVqqRWXr16dVy/fl1rgREREZHukXzFJCMjQ+1KSZ4HDx7A2Lj4nXiIiIiIJCcmH3zwAdavX69aVigUUCqVWLBgAVq3bq3V4IiIiEi3SE5MFixYgFWrVqFjx47Izs7G5MmTUadOHRw4cADz58+XI0aVefPmQaFQYPz48bK2Q0RERKVDcmJSp04dXLp0CS1atED37t2RkZGBDz/8EKdOnYK7u7scMQIATpw4ge+//x5169aVrQ0iIiIqXcUaYM3a2hr/+9//tB1LodLT09G/f3+sXr0as2fPfmPtEhER0ZtVrMTk0aNHOH78OFJTU6FUKtXWDRo0SCuBvWz06NHo3LkzfH19X5uYZGVlISsrS7Wclpam9XiIiIhIHpITkz/++AP9+/dHeno6rKysoFAoVOsUCoXWE5NNmzYhJiYGJ06c0Gj70NBQzJgxQ6sxEBER0ZshuY/JxIkTMXToUKSnp+PRo0d4+PCh6vXgwQOtBpeUlIRx48Zh48aNMDEx0Wif4OBgPH78WPVKSkrSakxEREQkH8lXTG7duoWxY8cWOJaJtp08eRKpqalo0KCBqiw3NxcHDhzAt99+i6ysLOjr66vtY2xszPFUiIiI3lKSExM/Pz9ER0ejatWqcsSjpm3btjh79qxa2ZAhQ1CzZk1MmTIlX1JCREREbzeNEpOdO3eq/t25c2dMmjQJcXFx8PT0hKGhodq23bp101pwlpaWqFOnjlqZubk5ypUrl6+ciIiI3n4aJSb+/v75ymbOnJmvTKFQIDc3t8RBERERkW7SKDF59ZHg0rR///7SDoGIiIhkIvmpnPXr16uNE5InOztbbQ4dIiIiIqkkJyZDhgzB48eP85U/efIEQ4YM0UpQREREpJskJyZCCLVB1fLcvHkT1tbWWgmKiIiIdJPGjwt7eXlBoVBAoVCgbdu2MDD4v11zc3ORmJiIDh06yBIkERER6QaNE5O8J3NiY2Ph5+cHCwsL1TojIyO4ubnho48+0nqAREREpDs0TkxCQkIAAG5ubujdu7fGQ8Triidt8ncI1rZpe+Ud0VbPp5ys9b85q95AG5+8gTaIiHSP5JFfAwIC5IiDiIiISHrnVyIiIiK5MDEhIiKiMoOJCREREZUZxU5MsrOzER8fj+fPn2szHiIiItJhkhOTzMxMDBs2DGZmZvDw8MCNGzcAAGPGjMG8efO0HiARERHpDsmJSXBwME6fPo39+/erPTLs6+uLzZs3azU4IiIi0i2SHxfesWMHNm/ejCZNmqgNTe/h4YGEhAStBkdERES6RfIVk7t378Le3j5feUZGRoFz6BARERFpSnJi4u3tjV27dqmW85KRH374AU2bNtVeZERERKRzJN/KmTt3Ljp27Ii4uDg8f/4cS5YsQVxcHA4fPoyoqCg5YiQiIiIdIfmKSYsWLRAbG4vnz5/D09MTu3fvhr29PY4cOYKGDRvKESMRERHpCMlXTADA3d0dq1ev1nYsREREpOMkXzHR19dHampqvvL79+9DX19fK0ERERGRbpKcmAghCizPysqCkZFRiQMiIiIi3aXxrZylS5cCePEUzg8//AALCwvVutzcXBw4cAA1a9bUfoRERESkMzROTBYvXgzgxRWTlStXqt22MTIygpubG1auXKn9CImIiEhnaJyYJCYmAgBat26Nbdu2wdbWVragiIiISDdJfipn3759csRBREREVLzHhW/evImdO3fixo0byM7OVlu3aNEirQRGREREukdyYhIZGYlu3bqhatWquHjxIurUqYNr165BCIEGDRrIESMRERHpCMmPCwcHB+Pzzz/H2bNnYWJigq1btyIpKQk+Pj7o1auXHDESERGRjpB8xeTChQv45ZdfXuxsYICnT5/CwsICM2fORPfu3TFq1CitB/k26PCZl+xtzH1QTdb6W+OQrPW/Kcqo+2+glVDZW9DzCZa9DSIqPR+OqAsDI7Ni7/88OxNYpcWAygjJV0zMzc1V/UqcnJyQkJCgWnfv3j3tRUZEREQ6R/IVkyZNmuDgwYOoVasWOnXqhIkTJ+Ls2bPYtm0bmjRpIkeMREREpCMkJyaLFi1Ceno6AGDGjBlIT0/H5s2bUb16dT6RQ0RERCUiOTGpWrWq6t/m5uYc7ZWIiIi0RnIfkxMnTuDYsWP5yo8dO4bo6GitBEVERES6SXJiMnr0aCQlJeUrv3XrFkaPHq2VoIiIiEg3SU5M4uLiChxIzcvLC3FxcVoJ6mW3bt3CgAEDUK5cOZiamsLT05NXZoiIiN5RkhMTY2Nj3LlzJ195cnIyDAyKNcJ9oR4+fIjmzZvD0NAQf//9N+Li4rBw4UJOIEhERCTRgQMH0LVrVzg7O0OhUGDHjh2v3Wf//v1o0KABjI2NUa1aNaxbt072OCUnJu3bt0dwcDAeP36sKnv06BG+/PJLtGvXTqvBzZ8/Hy4uLli7di0aNWqEKlWqoH379nB3d9dqO0RERO+6jIwM1KtXD8uXL9do+8TERHTu3BmtW7dGbGwsxo8fj+HDh+Off/6RNU7Jlzi++eYbtGzZEq6urvDyejHaaWxsLBwcHLBhwwatBrdz5074+fmhV69eiIqKQsWKFfHZZ59hxIgRhe6TlZWFrKws1XJaWppWYyIiInobdezYER07dtR4+5UrV6JKlSpYuHAhAKBWrVo4ePAgFi9eDD8/P7nClH7FpGLFijhz5gwWLFiA2rVro2HDhliyZAnOnj0LFxcXrQZ39epVrFixAtWrV8c///yDUaNGYezYsfjpp58K3Sc0NBTW1taql7ZjIiIiKkvS0tLUXi//57wkjhw5Al9fX7UyPz8/HDlyRCv1F6ZYnULMzc3xySefaDuWfJRKJby9vTF37lwALzrYnjt3DitXrkRAQECB+wQHByMoKEi1nJaWxuSEiIjeWa/+jQsJCcH06dNLXG9KSgocHBzUyhwcHJCWloanT5/C1NS0xG0URKPEZOfOnejYsSMMDQ2xc+fOIrft1q2bVgIDXszFU7t2bbWyWrVqYevWrYXuY2xsDGNjY63FQEREVJYlJSXByspKtfy2/w3UKDHx9/dHSkoK7O3t4e/vX+h2CoUCubm52ooNzZs3R3x8vFrZpUuX4OrqqrU2iIiI3mZWVlZqiYm2ODo65nsK986dO7CyspLtagmgYWKiVCoL/LfcJkyYgGbNmmHu3Ln4+OOPcfz4caxatQqrVr2D8zwTERGVIU2bNsVff/2lVrZnzx40bdpU1nYld359k95//31s374dv/zyC+rUqYNZs2YhLCwM/fv3L+3QiIiI3irp6emIjY1FbGwsgBePA8fGxuLGjRsAXvTRHDRokGr7kSNH4urVq5g8eTIuXryI7777Dr/++ismTJgga5zF6vwaGRmJyMhIpKam5ruCsmbNGq0ElqdLly7o0qWLVuskIiLSNdHR0WjdurVqOe9BkYCAAKxbtw7JycmqJAUAqlSpgl27dmHChAlYsmQJKlWqhB9++EHWR4WBYiQmM2bMwMyZM+Ht7Q0nJycoFAo54iIiIiItatWqFYQQha4vaFTXVq1a4dSpUzJGlZ/kxGTlypVYt24dBg4cKEc8REREpMMk9zHJzs5Gs2bN5IiFiIiIdJzkxGT48OEIDw+XIxYiIiLScZJv5Tx79gyrVq3Cv//+i7p168LQ0FBt/aJFi7QWHBEREekWyYnJmTNnUL9+fQDAuXPn1NaV5Y6wj21SIBQyjob3mcPrtymhqIpbZK3/yzFestYPAEeW3Xn9RiV0YHWY7G20+nmW7G0Aco/XI/+0EkREUklOTPbt2ydHHERERERle4A1IiIi0i2Sr5i0bt26yFs2e/fuLVFAREREpLskJyZ5/Uvy5OTkIDY2FufOnUNAQIC24iIiIiIdJDkxWbx4cYHl06dPR3p6eokDIiIiIt2ltT4mAwYM0Po8OURERKRbtJaYHDlyBCYmJtqqjoiIiHSQ5Fs5H374odqyEALJycmIjo7GV199pbXAiIiISPdITkysra3VlvX09FCjRg3MnDkT7du311pgREREpHskJyZr166VIw4iIiIi6YnJy9LT06FUKtXKrKysShQQERER6S7JnV8TExPRuXNnmJubw9raGra2trC1tYWNjQ1sbW3liJGIiIh0hOQrJgMGDIAQAmvWrIGDg0OZnriPiIiI3i6SE5PTp0/j5MmTqFGjhhzxEBERkQ6TfCvn/fffR1JSkhyxEBERkY6TfMXkhx9+wMiRI3Hr1i3UqVMHhoaGauvr1q2rteCIiIhIt0hOTO7evYuEhAQMGTJEVaZQKCCEgEKhQG5urlYDJCIiIt0hOTEZOnQovLy88Msvv7DzKxEREWmV5MTk+vXr2LlzJ6pVqyZHPERERKTDJHd+bdOmDU6fPi1HLERERKTjJF8x6dq1KyZMmICzZ8/C09MzX+fXbt26aS04IiIi0i2SE5ORI0cCAGbOnJlvHTu/EhERUUkohBCitIOQU1paGqytreH9STgMjMxkayfiu1Oy1Z0n5rdestbfoOcWWesHAOtcZ9nbeBOajnEo7RDeCkeWdS/tEIgkyfub8fjxY9nmfvu/NsJgZWVagnqewtp6vKyxlgbJfUyIiIiI5FKs2YVPnDiBffv2ITU1Nd/swosWLdJKYERERKR7JCcmc+fOxdSpU1GjRo1845hwTBMiIiIqCcmJyZIlS7BmzRoMHjxYhnCIiIhIl0nuY6Knp4fmzZvLEQsRERHpOMmJyYQJE7B8+XI5YiEiIiIdJ/lWzueff47OnTvD3d0dtWvXzjfA2rZt27QWXG5uLqZPn46ff/4ZKSkpcHZ2xuDBgzF16lT2ZyEiInoHSU5Mxo4di3379qF169YoV66crAnC/PnzsWLFCvz000/w8PBAdHQ0hgwZAmtra4wdO1a2domIiKh0SE5MfvrpJ2zduhWdO3eWIx41hw8fRvfu3VVtubm54ZdffsHx48dlb5uIiIjePMl9TOzs7ODu7i5HLPk0a9YMkZGRuHTpEgDg9OnTOHjwIDp27FjoPllZWUhLS1N7ERER0dtBcmIyffp0hISEIDMzU4541HzxxRfo06cPatasCUNDQ3h5eWH8+PHo379/ofuEhobC2tpa9XJxcZE9TiIiItIOybdyli5dioSEBDg4OMDNzS1f59eYmBitBffrr79i48aNCA8Ph4eHB2JjYzF+/Hg4OzsjICCgwH2Cg4MRFBSkWk5LS2NyQkRE9JaQnJj4+/vLEEbBJk2apLpqAgCenp64fv06QkNDC01MjI2NYWxs/MZiJCIiIu2RnJiEhITIEUeBMjMzoaenfrdJX18/3/w8RERE9G4o1iR+AHDy5ElcuHABAODh4QEvLy+tBZWna9eumDNnDipXrgwPDw+cOnUKixYtwtChQ7XeFhEREZU+yYlJamoq+vTpg/3798PGxgYA8OjRI7Ru3RqbNm1ChQoVtBbcsmXL8NVXX+Gzzz5DamoqnJ2d8emnn2LatGlaa4OIiIjKDslP5YwZMwZPnjzB+fPn8eDBAzx48ADnzp1DWlqa1gc9s7S0RFhYGK5fv46nT58iISEBs2fPhpGRkVbbISIiorJB8hWTiIgI/Pvvv6hVq5aqrHbt2li+fDnat2+v1eCIiIhIt0i+YqJUKvM9IgwAhoaG7JRKREREJSI5MWnTpg3GjRuH27dvq8pu3bqFCRMmoG3btloNjoiIiHSL5MTk22+/RVpaGtzc3ODu7g53d3dUqVIFaWlpWLZsmRwxEhERkY6Q3MfExcUFMTEx+Pfff3Hx4kUAQK1ateDr66v14IiIiEi3FGscE4VCgXbt2qFdu3bajkc2IS2rwtzMQrb6LXvGyVZ3Hh/slLX+mUdkrR4A4LO9uextRFXcInsbc9vIfxxy87GT9/sEAPu2V5O9jdY9PGRvg4jeHMm3csaOHYulS5fmK//2228xfvx4bcREREREOkpyYrJ161Y0b57/f4vNmjXDb7/9ppWgiIiISDdJTkzu378Pa2vrfOVWVla4d++eVoIiIiIi3SQ5MalWrRoiIiLylf/999+oWrWqVoIiIiIi3SS582tQUBACAwNx9+5dtGnTBgAQGRmJhQsXIiwsTNvxERERkQ6RnJgMHToUWVlZmDNnDmbNmgUAcHNzw4oVKzBo0CCtB0hERES6o1iPC48aNQqjRo3C3bt3YWpqCgsL+R7DJSIiIt1RrMQkT4UKFbQVBxEREZH0zq9EREREcmFiQkRERGUGExMiIiIqMyQnJlevXpUjDiIiInoDli9fDjc3N5iYmKBx48Y4fvx4oduuW7cOCoVC7WViYiJrfMUaYK1169b4+eef8ezZMzliIiIiIhls3rwZQUFBCAkJQUxMDOrVqwc/Pz+kpqYWuo+VlRWSk5NVr+vXr8sao+TEJCYmBnXr1kVQUBAcHR3x6aefFpltERERUdmwaNEijBgxAkOGDEHt2rWxcuVKmJmZYc2aNYXuo1Ao4OjoqHo5ODjIGqPkxKR+/fpYsmQJbt++jTVr1iA5ORktWrRAnTp1sGjRIty9e1eOOImIiKgAaWlpaq+srKwCt8vOzsbJkyfh6+urKtPT04Ovry+OHDlSaP3p6elwdXWFi4sLunfvjvPnz2v9GF5W7M6vBgYG+PDDD7FlyxbMnz8fV65cweeffw4XFxcMGjQIycnJ2oyTiIiICuDi4gJra2vVKzQ0tMDt7t27h9zc3HxXPBwcHJCSklLgPjVq1MCaNWvw+++/4+eff4ZSqUSzZs1w8+ZNrR9HnmIPsBYdHY01a9Zg06ZNMDc3x+eff45hw4bh5s2bmDFjBrp3785bPERERDJLSkqClZWVatnY2FhrdTdt2hRNmzZVLTdr1gy1atXC999/r5qWRtskJyaLFi3C2rVrER8fj06dOmH9+vXo1KkT9PReXHypUqUK1q1bBzc3N23HSkRERK+wsrJSS0wKU758eejr6+POnTtq5Xfu3IGjo6NGbRkaGsLLywtXrlwpVqyakHwrZ8WKFejXrx+uX7+OHTt2oEuXLqqkJI+9vT1+/PFHrQVJREREJWNkZISGDRsiMjJSVaZUKhEZGal2VaQoubm5OHv2LJycnOQKU/oVk8uXL792GyMjIwQEBBQrICIiIpJHUFAQAgIC4O3tjUaNGiEsLAwZGRkYMmQIAGDQoEGoWLGiqp/KzJkz0aRJE1SrVg2PHj3C119/jevXr2P48OGyxahRYnLmzBmNK6xbt26xgyEiIiL59O7dG3fv3sW0adOQkpKC+vXrIyIiQtUh9saNG2p3QR4+fIgRI0YgJSUFtra2aNiwIQ4fPozatWvLFqNGiUn9+vWhUCgghChwfd46hUKB3NxcrQZIRERE2hMYGIjAwMAC1+3fv19tefHixVi8ePEbiOr/aJSYJCYmyh2H7FrY/gUrc/mG0dXzKSdb3XmUUfdlrX/aU+315C5M873ydZjKc6in/McRJXsLgI/dTlnrj3rQTdb6AeDLN/B5H+lxSPY2gE/eQBuka5QHH0BZgr9Lyox3c/R1jRITV1dXueMgIiIiKt4Aaxs2bEDz5s3h7OysGjM/LCwMv//+u1aDIyIiIt1SrMeFg4KC0KlTJzx69EjVp8TGxgZhYWHajo+IiIh0iOTEZNmyZVi9ejX+97//QV9fX1Xu7e2Ns2fPajU4IiIi0i2SE5PExER4eXnlKzc2NkZGRoZWgiIiIiLdJDkxqVKlCmJjY/OVR0REoFatWtqIiYiIiHSU5MQkKCgIo0ePxubNmyGEwPHjxzFnzhwEBwdj8uTJkuo6cOAAunbtCmdnZygUCuzYsUNtvRAC06ZNg5OTE0xNTeHr66vRyLNERET0dpKcmAwfPhzz58/H1KlTkZmZiX79+mHFihVYsmQJ+vTpI6mujIwM1KtXD8uXLy9w/YIFC7B06VKsXLkSx44dg7m5Ofz8/PDs2bv57DYREZGukzxXDgD0798f/fv3R2ZmJtLT02Fvb1+sxjt27IiOHTsWuE4IgbCwMEydOhXdu3cHAKxfvx4ODg7YsWOH5CSIiIiIyj7JV0xmz56tGgnWzMys2EnJ6yQmJiIlJQW+vr6qMmtrazRu3BhHjhyRpU0iIiIqXZITky1btqBatWpo1qwZvvvuO9y7d0+OuJCSkgIAqomF8jg4OKjWFSQrKwtpaWlqLyIiIno7SE5MTp8+jTNnzqBVq1b45ptv4OzsjM6dOyM8PByZmZlyxChJaGgorK2tVS8XF5fSDomIiIg0VKwh6T08PDB37lxcvXoV+/btg5ubG8aPHw9HR0etBZZX1507d9TK79y5U2Q7wcHBePz4seqVlJSktZiIiIhIXsVKTF5mbm4OU1NTGBkZIScnRxsxAXgxXoqjoyMiIyNVZWlpaTh27BiaNm1a6H7GxsawsrJSexEREdHboViJSWJiIubMmQMPDw94e3vj1KlTmDFjRpF9PwqSnp6O2NhY1YBtiYmJiI2NxY0bN6BQKDB+/HjMnj0bO3fuxNmzZzFo0CA4OzvD39+/OGETERFRGSf5ceEmTZrgxIkTqFu3LoYMGYK+ffuiYsWKxWo8OjoarVu3Vi0HBQUBAAICArBu3TpMnjwZGRkZ+OSTT/Do0SO0aNECERERMDExKVZ7REREVLZJTkzatm2LNWvWoHbt2iVuvFWrVhBCFLpeoVBg5syZmDlzZonbIiIiorJPcmIyZ84cOeIgIiIi0iwxCQoKwqxZs2Bubq663VKYRYsWaSUwIiIi0j0aJSanTp1SPXFz6tQpWQMiIiIi3aVRYrJv374C/01ERESkTZIfFx46dCiePHmSrzwjIwNDhw7VSlBERESkmxSiqMdiCqCvr4/k5OR8k/fdu3cPjo6OeP78uVYDLKm0tDRYW1vD+5NwGBiZydZOxHfy3+JaLPPchT63esnbAAAfu52yt6HnU072NqYfvy17G9OeGsvehtzexGehjLovextvgp5PcGmHQPi/vxmPHz+WbYDOvDYe7poGK/PiD3+RlvEMtp1nyhpradD4qZy0tDQIISCEwJMnT9TGEsnNzcVff/0l20zDREREpBs0TkxsbGygUCigUCjw3nvv5VuvUCgwY8YMrQZHREREukXjxGTfvn0QQqBNmzbYunUr7OzsVOuMjIzg6uoKZ2dnWYIkIiIi3aBxYuLj4wPgxXw2lStXhkKhkC0oIiIi0k2Sn8rZu3cvfvvtt3zlW7ZswU8//aSVoIiIiEg3SU5MQkNDUb58+Xzl9vb2mDt3rlaCIiIiIt0kOTG5ceMGqlSpkq/c1dUVN27c0EpQREREpJskJyb29vY4c+ZMvvLTp0+jXDn5xywgIiKid5fkxKRv374YO3Ys9u3bh9zcXOTm5mLv3r0YN24c+vTpI0eMREREpCM0fionz6xZs3Dt2jW0bdsWBgYvdlcqlRg0aBD7mBAREVGJSE5MjIyMsHnzZsyaNQunT5+GqakpPD094erqKkd8REREpEMkJyZ53NzcIISAu7u76soJERERUUlI7mOSmZmJYcOGwczMDB4eHqonccaMGYN58+ZpPUAiIiLSHZITk+DgYJw+fRr79+9Xm8jP19cXmzdv1mpwREREpFsk34PZsWMHNm/ejCZNmqgNS+/h4YGEhAStBkdERES6RfIVk7t378Le3j5feUZGBufPISIiohKRnJh4e3tj165dquW8ZOSHH35A06ZNtRcZERER6RzJt3Lmzp2Ljh07Ii4uDs+fP8eSJUsQFxeHw4cPIyoqSo4YiYiISEdIvmLSokULxMbG4vnz5/D09MTu3bthb2+PI0eOoGHDhnLESERERDpCoysmQUFBmDVrFszNzXHgwAE0a9YMq1evljs2IiIi0jEaXTFZtmwZ0tPTAQCtW7fGgwcPZA2KiIiIdJNGV0zc3NywdOlStG/fHkIIHDlyBLa2tgVu27JlS60GSERERLpDIYQQr9tox44dGDlyJFJTU6FQKFDYLgqFArm5uVoPsiTS0tJgbW2Nx4/DYGVlKls7+7Y3l63uPD52O2VvQ27Nf6stexuHesbJ3saboOdTTtb6lVH3Za0fkP8Y3pQ38V5FPegmexute3jI3sbb7v/+ZjyGlZWVrG388fNRmJtZFLuejMx0dB3QRNZYS4NGV0z8/f3h7++P9PR0WFlZIT4+vsCxTIiIiIhKQqM+JkFBQcjIyICFhQX27duHKlWqwNrausAXERERUXFJ7vzapk0bdn4lIiIiWbDzKxEREZUZGiUmX3/9NUaOHInQ0FAoFAr06NGjwO3KYudXIiIienuw8ysRERGVGZLmynm586uBgeRpdoiIiIiKJHmuHB8fH1y/fh1Tp05F3759kZqaCgD4+++/cf78eUl1HThwAF27doWzszMUCgV27NihWpeTk4MpU6bA09MT5ubmcHZ2xqBBg3D79m2pIRMREdFbQnJiEhUVBU9PTxw7dgzbtm1TPa1z+vRphISESKorIyMD9erVw/Lly/Oty8zMRExMDL766ivExMRg27ZtiI+PR7du8g9CRERERKVD8v2YL774ArNnz0ZQUBAsLS1V5W3atMG3334rqa6OHTuiY8eOBa6ztrbGnj171Mq+/fZbNGrUCDdu3EDlypWlhk5ERERlnOTE5OzZswgPD89Xbm9vj3v37mklqMI8fvwYCoUCNjY2hW6TlZWFrKws1XJaWpqsMREREZH2SL6VY2Njg+Tk5Hzlp06dQsWKFbUSVEGePXuGKVOmoG/fvkXOCRAaGqo2Eq2Li4tsMREREZF2SU5M+vTpgylTpiAlJQUKhQJKpRKHDh3C559/jkGDBskRI3JycvDxxx9DCIEVK1YUuW1wcDAeP36seiUlJckSExEREWmf5Fs5c+fOxejRo+Hi4oLc3FzUrl0bubm56NevH6ZOnar1APOSkuvXr2Pv3r2vnUHR2NgYxsbGWo+DiIiI5Cf5iomRkRFWr16Nq1ev4s8//8TPP/+MixcvYsOGDdDX19dqcHlJyeXLl/Hvv/+iXLl3Ywp1IiKi0rJ8+XK4ubnBxMQEjRs3xvHjx4vcfsuWLahZsyZMTEzg6emJv/76S9b4ij1KmouLS4n7b6Snp+PKlSuq5cTERMTGxsLOzg5OTk7o2bMnYmJi8OeffyI3NxcpKSkAADs7OxgZGZWobSIiIl2zefNmBAUFYeXKlWjcuDHCwsLg5+dX6Ijuhw8fRt++fREaGoouXbogPDwc/v7+iImJQZ06dWSJUfIVE22Kjo6Gl5cXvLy8AABBQUHw8vLCtGnTcOvWLezcuRM3b95E/fr14eTkpHodPny4NMMmIiJ6Ky1atAgjRozAkCFDULt2baxcuRJmZmZYs2ZNgdsvWbIEHTp0wKRJk1CrVi3MmjULDRo0kDw8iBSlOq58q1atIIQodH1R64iIiEhz2dnZOHnyJIKDg1Vlenp68PX1xZEjRwrc58iRIwgKClIr8/PzUxupXds44Q0REdFb7NXxugp7COTevXvIzc2Fg4ODWrmDgwMuXrxYYN0pKSkFbp/XtUIOpXorh4iIiErGxcVFbfyu0NDQ0g6pRIp1xeTRo0f48ccfceHCBQCAh4cHhg4dCmtra60GR0REREVLSkpSG0qjsCEzypcvD319fdy5c0et/M6dO3B0dCxwH0dHR0nba4PkKybR0dFwd3fH4sWL8eDBAzx48ACLFi2Cu7s7YmJi5IiRiIiICmFlZaX2KiwxMTIyQsOGDREZGakqUyqViIyMRNOmTQvcp2nTpmrbA8CePXsK3V4bJF8xmTBhArp164bVq1fDwODF7s+fP8fw4cMxfvx4HDhwQOtBEhERUckFBQUhICAA3t7eaNSoEcLCwpCRkYEhQ4YAAAYNGoSKFSuqbgeNGzcOPj4+WLhwITp37oxNmzYhOjoaq1atki1GyYlJdHS0WlICAAYGBpg8eTK8vb21Gpw2KQ8+gNLcRLb6W/c4JFvdeaYfz3r9RiWpv5GzrPUDwCHEyd6Gno/8A/FNP35b9jamRd2XvQ25Kd+BYwCAqAfd5G+j4hbZ28B2eatv3cND3gaoxHr37o27d+9i2rRpSElJQf369REREaHq4Hrjxg3o6f3fzZRmzZohPDwcU6dOxZdffonq1atjx44dso1hAhQjMbGyssKNGzdQs2ZNtfKkpCRYWlpqLTAiIiLSvsDAQAQGBha4bv/+/fnKevXqhV69eskc1f+R3Mekd+/eGDZsGDZv3oykpCQkJSVh06ZNGD58OPr27StHjERERKQjJF8x+eabb6BQKDBo0CA8f/4cAGBoaIhRo0Zh3rx5Wg+QiIiIdIfkxMTIyAhLlixBaGgoEhISAADu7u4wMzPTenBERESkWyTfyhk6dCiePHkCMzMzeHp6wtPTE2ZmZsjIyMDQoUPliJGIiIh0hOTE5KeffsLTp0/zlT99+hTr16/XSlBERESkmzS+lZOWlgYhBIQQePLkCUxM/u/R29zcXPz1118FTplMREREpCmNExMbGxsoFAooFAq89957+dYrFArMmDFDq8ERERGRbtE4Mdm3bx+EEGjTpg22bt0KOzs71TojIyO4urrC2Vn+AbqIiIjo3aVxYuLj4wMASExMROXKlaFQKGQLioiIiHST5MeFXV1d5YiDiIiISPpTOURERERyYWJCREREZQYTEyIiIiozJCcmISEhuH79uhyxEBERkY6TnJj8/vvvcHd3R9u2bREeHo6srCw54iIiIiIdJDkxiY2NxYkTJ+Dh4YFx48bB0dERo0aNwokTJ+SIj4iIiHRIsfqYeHl5YenSpbh9+zZ+/PFH3Lx5E82bN0fdunWxZMkSPH78WNtxEhERkQ4oUedXIQRycnKQnZ0NIQRsbW3x7bffwsXFBZs3b9ZWjERERKQjipWYnDx5EoGBgXBycsKECRPg5eWFCxcuICoqCpcvX8acOXMwduxYbcdKRERE7zjJiYmnpyeaNGmCxMRE/Pjjj0hKSsK8efNQrVo11TZ9+/bF3bt3tRooERERvfskD0n/8ccfY+jQoahYsWKh25QvXx5KpbJEgREREZHukZyY5PUledXTp0/x9ddfY9q0aVoJTNv0WthBz8pUtvr3bW8uW915fGSuXxm1U+YWAD2fcrK38SZMe2pc2iHQG+RjJ/+5ESV7C/Lbt/287G207uEhextvyiGnHTC2KP5vSVb6uzlch+RbOTNmzEB6enq+8szMTMyYMUMrQREREZFukpyYCCGgUCjylZ8+fRp2dnZaCYqIiIh0k8a3cmxtbaFQKKBQKPDee++pJSe5ublIT0/HyJEjZQmSiIiIdIPGiUlYWBiEEBg6dChmzJgBa2tr1TojIyO4ubmhadOmsgRJREREukHjxCQgIAAAUKVKFTRr1gyGhoayBUVERES6SaPEJC0tDVZWVgBeDEf/9OlTPH36tMBt87YjIiIikkqjzq+2trZITU0FANjY2MDW1jbfK69cigMHDqBr165wdnaGQqHAjh07Ct125MiRUCgUCAsLk9QGERERvT00umKyd+9e1RM3+/bt01rjGRkZqFevHoYOHYoPP/yw0O22b9+Oo0ePwtnZWWttExERUdmjUWLi4+NT4L9LqmPHjujYsWOR29y6dQtjxozBP//8g86dO2utbSIiIip7NEpMzpw5o3GFdevWLXYwr1IqlRg4cCAmTZoEDw/NRvvLyspCVtb/jYaXlpamtXiIiIhIXholJvXr14dCoYAQosjtFAoFcnNztRIYAMyfPx8GBgaSZioODQ3lCLRERERvKY0Sk8TERLnjyOfkyZNYsmQJYmJiChxptjDBwcEICgpSLaelpcHFxUWOEImIiEjLNEpMXF1d5Y4jn//++w+pqamoXLmyqiw3NxcTJ05EWFgYrl27VuB+xsbGMDbmBGtERERvI40Sk507d6Jjx44wNDTEzp1Fz7LZrVs3rQQ2cOBA+Pr6qpX5+flh4MCBGDJkiFbaICIiorJFo8TE398fKSkpsLe3h7+/f6HbSe1jkp6ejitXrqiWExMTERsbCzs7O1SuXBnlypVT297Q0BCOjo6oUaOGxm0QERHR20OjxESpVBb475KKjo5G69atVct5fUMCAgKwbt06rbVDREREbweN58qRQ6tWrV77pM/LCutXQkRERO8GjYakf1VkZCS6dOkCd3d3uLu7o0uXLvj333+1HRsRERHpGMmJyXfffYcOHTrA0tIS48aNw7hx42BlZYVOnTph+fLlcsRIREREOkLyrZy5c+di8eLFCAwMVJWNHTsWzZs3x9y5czF69GitBkhERES6Q/IVk0ePHqFDhw75ytu3b4/Hjx9rJSgiIiLSTZITk27dumH79u35yn///Xd06dJFK0ERERGRbtLoVs7SpUtV/65duzbmzJmD/fv3o2nTpgCAo0eP4tChQ5g4caI8UWrBgT+awNzMQrb6W/c4JFvdeZRR92WtP+qBdgbHK4pPVNED9GnDTNOs129UQj4PesnehtyiKm6RvY1pT9+NUZjfxHfqTZD7M5/eyFnW+gFAKfNviDLjmaz10+tplJgsXrxYbdnW1hZxcXGIi4tTldnY2GDNmjWYOnWqdiMkIiIinVFmJ/EjIiIi3VOscUyIiIiI5FCskV9v3ryJnTt34saNG8jOzlZbt2jRIq0ERkRERLpHcmISGRmJbt26oWrVqrh48SLq1KmDa9euQQiBBg0ayBEjERER6QjJt3KCg4Px+eef4+zZszAxMcHWrVuRlJQEHx8f9Or19j+pQERERKVHcmJy4cIFDBo0CABgYGCAp0+fwsLCAjNnzsT8+fO1HiARERHpDsmJibm5uapfiZOTExISElTr7t27p73IiIiISOdI7mPSpEkTHDx4ELVq1UKnTp0wceJEnD17Ftu2bUOTJk3kiJGIiIh0hOTEZNGiRUhPTwcAzJgxA+np6di8eTOqV6/OJ3KIiIioRCQnJlWrVlX929zcHCtXrtRqQERERKS7OMAaERERlRlMTIiIiKjMYGJCREREZQYTEyIiIiozip2YZGdnIz4+Hs+fP9dmPERERKTDJCcmmZmZGDZsGMzMzODh4YEbN24AAMaMGYN58+ZpPUAiIiLSHcWaK+f06dPYv38/TExMVOW+vr7YvHmzVoMjIiKiN+/Bgwfo378/rKysYGNjg2HDhqnGMCtMq1atoFAo1F4jR46U3LbkcUx27NiBzZs3o0mTJlAoFKpyDw8PteHpiYiI6O3Uv39/JCcnY8+ePcjJycGQIUPwySefIDw8vMj9RowYgZkzZ6qWzczMJLctOTG5e/cu7O3t85VnZGSoJSpERET09rlw4QIiIiJw4sQJeHt7AwCWLVuGTp064ZtvvoGzs3Oh+5qZmcHR0bFE7Uu+lePt7Y1du3aplvOSkR9++AFNmzYtUTBERERUuo4cOQIbGxtVUgK86K6hp6eHY8eOFbnvxo0bUb58edSpUwfBwcHIzMyU3L7kKyZz585Fx44dERcXh+fPn2PJkiWIi4vD4cOHERUVJTkAuQkhAACZmRmytpOW9lTW+gFAmfFM1vozMou+f6gNacbyHgMAZOVmy97Gm3iv5JaVniV7G2nPhOxtvAlv4jv1LngXfgfTMl+cF3l/O+SUlVGyczBv/7S0NLVyY2NjGBsbF7velJSUfHdGDAwMYGdnh5SUlEL369evH1xdXeHs7IwzZ85gypQpiI+Px7Zt26QFIIrhypUrYvjw4eL9998XtWrVEv379xdnzpwpTlWyS0pKEgD44osvvvjiS+NXUlKSbH+Xnj59KhwdHbUSp4WFRb6ykJCQAtudMmXKa+u7cOGCmDNnjnjvvffy7V+hQgXx3XffaXyckZGRAoC4cuWKpPdHIcQbSAtLkVKpxO3bt2FpaalRH5i0tDS4uLggKSkJVlZWbyBCefA4yo534RiAd+M43oVjAHgcchJC4MmTJ3B2doaennxjkD579gzZ2SW/EieEyPe3rbArJnfv3sX9+/eLrK9q1ar4+eefMXHiRDx8+FBV/vz5c5iYmGDLli3o0aOHRrFlZGTAwsICERER8PPz02gfoBi3coAXf+yvXLmC1NRUKJVKtXUtW7YsTpWy0dPTQ6VKlSTvZ2VlVWZOlJLgcZQd78IxAO/GcbwLxwDwOORibW0texsmJiZqQ268CRUqVECFChVeu13Tpk3x6NEjnDx5Eg0bNgQA7N27F0qlEo0bN9a4vdjYWACAk5OTpDglJyZHjx5Fv379cP369Xz34BQKBXJzc6VWSURERGVErVq10KFDB4wYMQIrV65ETk4OAgMD0adPH9UTObdu3ULbtm2xfv16NGrUCAkJCQgPD0enTp1Qrlw5nDlzBhMmTEDLli1Rt25dSe1LTkxGjhypejLHycmJjwgTERG9YzZu3IjAwEC0bdsWenp6+Oijj7B06VLV+pycHMTHx6ueujEyMsK///6LsLAwZGRkwMXFBR999BGmTp0quW3Jicnly5fx22+/oVq1apIbexsYGxsjJCSkRD2aywIeR9nxLhwD8G4cx7twDACPg+RnZ2dX5GBqbm5uandNXFxctPZkruTOr23atMHkyZPRoUMHrQRARERElEejKyZnzpxR/XvMmDGYOHEiUlJS4OnpCUNDQ7Vtpd5LIiIiIsqj0RUTPT09KBSKQgecyVvHzq9ERERUEhpdMUlMTJQ7DiIiIiLN+5gMHToUS5YsgaWlpdwxERERkY7SeFi7n376CU+fyj8PQmlbvnw53NzcYGJigsaNG+P48eOlHZIkoaGheP/992FpaQl7e3v4+/sjPj6+tMMqkXnz5kGhUGD8+PGlHYpkt27dwoABA1CuXDmYmprC09MT0dHRpR2WxnJzc/HVV1+hSpUqMDU1hbu7O2bNmvVG5hEpiQMHDqBr165wdnaGQqHAjh071NYLITBt2jQ4OTnB1NQUvr6+uHz5cukEW4SijiMnJwdTpkyBp6cnzM3N4ezsjEGDBuH27dulF3ABXvdZvGzkyJFQKBQICwt7Y/FR2aNxYlLWf4i0YfPmzQgKCkJISAhiYmJQr149+Pn5ITU1tbRD01hUVBRGjx6No0ePYs+ePcjJyUH79u2RkSHvJIZyOXHiBL7//vu3slP1w4cP0bx5cxgaGuLvv/9GXFwcFi5cCFtb29IOTWPz58/HihUr8O233+LChQuYP38+FixYgGXLlpV2aEXKyMhAvXr1sHz58gLXL1iwAEuXLsXKlStx7NgxmJubw8/PD8+eyT/JpBRFHUdmZiZiYmLw1VdfISYmBtu2bUN8fDy6detWCpEW7nWfRZ7t27fj6NGjqgG8SIdpPKmOQiGuXLkiHj9+XOTrbdaoUSMxevRo1XJubq5wdnYWoaGhpRhVyaSmpgoAIioqqrRDkezJkyeievXqYs+ePcLHx0eMGzeutEOSZMqUKaJFixalHUaJdO7cWQwdOlSt7MMPPxT9+/cvpYikAyC2b9+uWlYqlcLR0VF8/fXXqrJHjx4JY2Nj8csvv5RChJp59TgKcvz4cQFAXL9+/c0EJVFhx3Dz5k1RsWJFce7cOeHq6ioWL178xmOjskPSDEXvvfcebG1tC3zZ2Ni8Vf8TfFV2djZOnjwJX19fVZmenh58fX1x5MiRUoysZB4/fgzgxWA5b5vRo0ejc+fOap/J22Tnzp3w9vZGr169YG9vDy8vL6xevbq0w5KkWbNmiIyMxKVLlwAAp0+fxsGDB9GxY8dSjqz4EhMTkZKSova9sra2RuPGjd/qcx14cb4rFArY2NiUdigaUyqVGDhwICZNmgQPD4/SDofKAEkjv/72229v5R84Tdy7dw+5ublwcHBQK3dwcMDFixdLKaqSUSqVGD9+PJo3b446deqUdjiSbNq0CTExMThx4kRph1JsV69exYoVKxAUFIQvv/wSJ06cwNixY2FkZISAgIDSDk8jX3zxBdLS0lCzZk3o6+sjNzcXc+bMQf/+/Us7tGJLSUkBgALP9bx1b6Nnz55hypQp6Nu3b5maEO915s+fDwMDA4wdO7a0Q6EyQlJi0rx5c9jb28sVC2nZ6NGjce7cORw8eLC0Q5EkKSkJ48aNw549e9747JvapFQq4e3tjblz5wIAvLy8cO7cOaxcufKtSUx+/fVXbNy4EeHh4fDw8EBsbCzGjx8PZ2fnt+YYdEFOTg4+/vhjCCGwYsWK0g5HYydPnsSSJUsQExPDeddIRdKtnHdZ+fLloa+vjzt37qiV37lzB46OjqUUVfEFBgbizz//xL59+1CpUqXSDkeSkydPIjU1FQ0aNICBgQEMDAwQFRWFpUuXwsDA4K0ZxM/JyQm1a9dWK6tVqxZu3LhRShFJN2nSJHzxxRfo06cPPD09MXDgQEyYMAGhoaGlHVqx5Z3P78q5npeUXL9+HXv27Hmrrpb8999/SE1NReXKlVXn+vXr1zFx4kS4ubmVdnhUSjROTFxdXaGvry9nLKXKyMgIDRs2RGRkpKpMqVQiMjISTZs2LcXIpBFCIDAwENu3b8fevXtRpUqV0g5JsrZt2+Ls2bOIjY1Vvby9vdG/f3/Exsa+Nd/D5s2b53tU+9KlS3B1dS2liKTLzMyEnp76z4S+vj6USmUpRVRyVapUgaOjo9q5npaWhmPHjr1V5zrwf0nJ5cuX8e+//6JcuXKlHZIkAwcOxJkzZ9TOdWdnZ0yaNAn//PNPaYdHpUTjWzm6MPprUFAQAgIC4O3tjUaNGqmmbx4yZEhph6ax0aNHIzw8HL///jssLS1V98ytra1hampaytFpxtLSMl+fGHNzc5QrV+6t6iszYcIENGvWDHPnzsXHH3+M48ePY9WqVVi1alVph6axrl27Ys6cOahcuTI8PDxw6tQpLFq0CEOHDi3t0IqUnp6OK1euqJYTExMRGxsLOzs7VK5cGePHj8fs2bNRvXp1VKlSBV999RWcnZ3h7+9fekEXoKjjcHJyQs+ePRETE4M///wTubm5qvPdzs4ORkZGpRW2mtd9Fq8mU4aGhnB0dESNGjXedKhUVpT2Y0FlzbJly0TlypWFkZGRaNSokTh69GhphyQJgAJfa9euLe3QSuRtfFxYCCH++OMPUadOHWFsbCxq1qwpVq1aVdohSZKWlibGjRsnKleuLExMTETVqlXF//73P5GVlVXaoRVp3759BZ4HAQEBQogXjwx/9dVXwsHBQRgbG4u2bduK+Pj40g26AEUdR2JiYqHn+759+0o7dJXXfRav4uPCpPGQ9ERERERyY+dXIiIiKjOYmBAREVGZobXEJDo6GgcOHNBWdURERKSDtNbHpFatWrh06dJbM8YEERERlT1aS0xu376NnJyct2qMBiIiIipb+FQOERERlRmS+5jExMTg7NmzquXff/8d/v7++PLLL5Gdna3V4IiIiEi3SE5MPv30U9UU6FevXkWfPn1gZmaGLVu2YPLkyVoPUA7Xrl2DQqFAbGzsG2/bzc0NYWFhRW6jUCiwY8cOAPlj3b9/PxQKBR49eiRrnHLbsWMHqlWrBn19fYwfP77QMjm0atVKq/WX5vfpbfby9/xtUFbjnT59OurXr1/aYRTb4MGDixxxd926dbCxsXlj8RRFCIFPPvkEdnZ2qnNe278nmnpTv5elQXJicunSJdVJsGXLFrRs2RLh4eFYt24dtm7dqu34JHNzc4NCoSj0NXjw4NIO8bWSk5PRsWPHAtc1a9YMycnJsLa2BqC9k/ZNJzyffvopevbsiaSkJMyaNavQspIo7Ji2bdumlfqpZIr6nheHJkm/Jt72P/Qkn4iICKxbtw5//vknkpOTUadOnXy/J9r6Hr6Otn8vy5J8c+VkZ2cXOceCEEI1gde///6LLl26AABcXFxw7949mcLU3IkTJ1RPBh0+fBgfffQR4uPjVTNumpqa4uHDh6UZ4msVNcOpkZHRWzkD6svS09ORmpoKPz8/ODs7F1omFzs7O1nrJ8287d9j0j0JCQlwcnJCs2bNVGWl8XvyJn8vX/W6HEErdfn4+IjRo0eLcePGiXLlyolWrVoJIYTYv3+/eP/994WRkZFwdHQUU6ZMETk5OaJ169Zi0KBBYt26dUJfX181r0yFChWEjY2Naqz7GzduiF69eglra2tha2srunXrJhITE1Xr9+3bJ95//31hZmYmrK2tRbNmzcS1a9eEEELExsaKVq1aCQsLC2FpaSkaNGggTpw4IXm8/bw5Gh4+fKhWnjfHxNatW0WrVq2EqampqFu3rjh8+LDadv/9959o0aKFMDExEZUqVRJjxowR6enphbZ35coV0a1bN2Fvby/Mzc2Ft7e32LNnj9o2rq6uYubMmaJPnz7CzMxMODs7i2+//VZtGwBi+/btarGeOnUq3zEVNAdFSEiImDFjhvDw8MgXX7169cTUqVPzlRc050bePBbPnj0TY8aMERUqVBDGxsaiefPm4vjx44W+B3n7TJw4UTg7OwszMzPRqFEj1dwdBcVcWJkmn8GzZ8/E5MmTRaVKlYSRkZFwd3cXP/zwQ5HH9PK8O8HBwaJRo0b5jqFu3bpixowZquXVq1eLmjVrCmNjY1GjRg2xfPnyfO/fqVOnhFKpFO7u7uLrr79Wq+/UqVMCgLh8+XKh79uPP/4oateurTrnRo8erVp3/fp10a1bN2Fubi4sLS1Fr169REpKimp9SEiIqFevnvjxxx+Fi4uLMDc3F6NGjRLPnz8X8+fPFw4ODqJChQpi9uzZam0CECtXrhSdO3cWpqamombNmuLw4cPi8uXLwsfHR5iZmYmmTZuKK1euqPYJCAgQ3bt3V6tn3LhxwsfHR7Xs4+MjxowZIyZNmiRsbW2Fg4ODCAkJydd23vdcCCGSkpJEnz59hK2trTAzMxMNGzZUzVX1unPLx8cn3+edR8p5vHbt2kLnmQIgVq9eLfz9/YWpqamoVq2a+P3339X2P3v2rOjQoYMwNzcX9vb2YsCAAeLu3bsFtvX48WNhYmIi/vrrL7Xybdu2CQsLC5GRkSGEEGLy5MmievXqwtTUVFSpUkVMnTpVZGdnq7bP++xffi9enVeqe/fuanPTFHWOFmbhwoWiTp06wszMTFSqVEmMGjVKPHnyRO29s7a2FhEREaJmzZrC3Nxc+Pn5idu3b6u2ef78uZgwYYKwtrYWdnZ2YtKkSWLQoEH5vk8vy6t3+/btolq1asLY2Fi0b99e3LhxQ227HTt2CC8vL2FsbCyqVKkipk+fLnJyclTrS/r5BQQEqH0vXF1d873fhX0Pr127Jrp06SJsbGyEmZmZqF27tti1a1ehx/zgwQMxcOBAYWNjI0xNTUWHDh3EpUuXhBCF/4YW5HW/G5qeywXlCK/Kq2v69OmifPnywtLSUnz66adq82ppmm/Ax8dHWFhYiEmTJomLFy+Kixcvips3bwozMzPx2WefiQsXLojt27eL8uXLi5CQEHH69GlRp04dYWRkJExMTMS6devElStXxIcffqj6gc/Ozha1atUSQ4cOFWfOnBFxcXGiX79+okaNGiIrK0vk5OQIa2tr8fnnn4srV66IuLg4sW7dOnH9+nUhhBAeHh5iwIAB4sKFC+LSpUvi119/FbGxsYV+iIV5XWJSs2ZN8eeff4r4+HjRs2dP4erqqvoiX7lyRZibm4vFixeLS5cuiUOHDgkvLy8xePDgQtuLjY0VK1euFGfPnhWXLl0SU6dOFSYmJqrjEuJFYmJpaSlCQ0NFfHy8WLp0qdDX1xe7d+9WbaNpYpKVlSXCwsKElZWVSE5OFsnJyeLJkyciKSlJ6OnpqSUQMTExQqFQiISEhHxxP3/+XGzdulUAEPHx8SI5OVk8evRICCHE2LFjhbOzs/jrr7/E+fPnRUBAgLC1tRX3798v9H0YPny4aNasmThw4IC4cuWK+Prrr4WxsbG4dOmSyMrKEvHx8arEMDk5udAyTT6Djz/+WLi4uIht27aJhIQE8e+//4pNmzYVeUwv/5CcO3dOAFD7w5tXlpdE/Pzzz8LJyUls3bpVXL16VWzdulXY2dmJdevWFfgZzZkzR9SuXVvtPRk7dqxo2bJloe/Zd999J0xMTERYWJiIj48Xx48fV01klpubK+rXry9atGghoqOjxdGjR0XDhg3VfjxCQkKEhYWF6Nmzpzh//rzYuXOnMDIyEn5+fmLMmDHi4sWLYs2aNQKA2sSUAETFihXF5s2bRXx8vPD39xdubm6iTZs2IiIiQsTFxYkmTZqIDh06qPbR9MfMyspKTJ8+XVy6dEn89NNPQqFQFPo9f/Lkiahatar44IMPxH///ScuX74sNm/erPrPwuvOrfv374tKlSqJmTNnqs4FIaSfx5mZmWLixInCw8NDVU9mZqYq3kqVKonw8HBx+fJlMXbsWGFhYaE6Fx4+fCgqVKgggoODxYULF0RMTIxo166daN26daGfe8+ePcWAAQPUyj766CO1slmzZolDhw6JxMREsXPnTuHg4CDmz5+v9tlLTUyKOkcLs3jxYrF3716RmJgoIiMjRY0aNcSoUaNU69euXSsMDQ2Fr6+vOHHihDh58qSoVauW6Nevn2qb+fPnC1tbW7F161YRFxcnhg0bJiwtLV+bmBgaGgpvb29x+PBhER0dLRo1aiSaNWum2ubAgQPCyspKrFu3TiQkJIjdu3cLNzc3MX36dNU2Jf38Hj16JGbOnCkqVaokkpOTRWpqar73u7DvYefOnUW7du3EmTNnREJCgvjjjz9EVFRUocfcrVs3UatWLXHgwAERGxsr/Pz8RLVq1UR2dnahv5ev0uR3Q9Nz+dUcoSABAQHCwsJC9O7dW5w7d078+eefokKFCuLLL78ssq6C8g34+PgILy8vtQa+/PJLUaNGDaFUKlVly5cvFxYWFiI3N1ekpaUJY2NjsXr1atX6p0+fqrL4DRs25Ns/KytLmJqain/++Ufcv39fABD79+8v8AAtLS1VP/ol8brE5IcfflCVnT9/XgAQFy5cEEIIMWzYMPHJJ5+o7ffff/8JPT098fTpU41j8PDwEMuWLVMtu7q6qv3ICyFE7969RceOHVXLmiYmQvzf/yZe1bFjR7UfjTFjxhSa6RZUrxBCpKenC0NDQ7Fx40ZVWXZ2tnB2dhYLFiwosJ7r168LfX19cevWLbXytm3biuDgYCHEix+AV7P8gspe9xnknZyvXpUq6piEyP/DXa9ePTFz5kzVcnBwsGjcuLFq2d3dXYSHh6vVMWvWLNG0aVMhRP7P6NatW0JfX18cO3ZMCPHiPStfvnyR32lnZ2fxv//9r8B1u3fvFvr6+mr/Q8z7vuYlnyEhIcLMzEykpaWptvHz8xNubm4iNzdXVVajRg0RGhqqWgagdhXtyJEjAoD48ccfVWW//PKLMDExUS1r+mPWokULtW3ef/99MWXKFLW2877n33//vbC0tCwy4X1VQefWq7PSFuc8fvUP/cvxvvxepaenCwDi77//FkK8+E60b99ebZ+kpCRVclyQ7du3q10dybuKkldnQb7++mvRsGHDQuN9XWKiyTmqiS1btohy5cqplvOuNr2c5C9fvlw4ODiolp2cnNR+O3JyckSlSpVem5i8mlBfuHBBAFCdY23bthVz585V22/Dhg3CyclJtayNz2/x4sWqKyV5Xn2/C/oeenp6qiVJRbl06ZIAIA4dOqQqu3fvnjA1NRW//vqrEKLg38tXafK7oem5/GqOUJCAgABhZ2en+i4LIcSKFStUeUNhdRWUb+gBQMOGDdVu71y4cAFNmzaFQqFQlTVv3hzp6ek4fvw49u/fj6ysLLRt2xbHjx/H+PHjsX79ehgaGgIATp8+jStXrsDS0hIWFhawsLCAnZ0dnj17hoSEBNjZ2WHw4MHw8/ND165dsWTJEiQnJ6vaCgoKwvDhw+Hr64t58+YhISEBcqhbt67q305OTgCA1NRU1TGsW7dOFb+FhQX8/PygVCqRmJhYYH3p6en4/PPPUatWLdjY2MDCwgIXLlzAjRs31LZr2rRpvuULFy5o89AwYsQI/PLLL3j27Bmys7MRHh6OoUOHSqojISEBOTk5aN68uarM0NAQjRo1KjTes2fPIjc3F++9957aexcVFSX5c3zdZxAbGwt9fX34+PhIqvdV/fv3R3h4OIAXfah++eUX9O/fHwCQkZGBhIQEDBs2TC2O2bNnF3o8zs7O6Ny5M9asWQMA+OOPP5CVlYVevXoVuH1qaipu376Ntm3bFrj+woULcHFxgYuLi6qsdu3asLGxUfsc3NzcYGlpqVp2cHBA7dq1oaenp1aW9x3P8/J54ODgAADw9PRUK3v27BnS0tIKjK8wL9cLvDjHXm07T2xsLLy8vAq9X6/pufWq4pzHmh6Tubk5rKys1H4z9u3bp9ZWzZo1AaDQ70qnTp1gaGiInTt3AgC2bt0KKysr+Pr6qrbZvHkzmjdvDkdHR1hYWGDq1KmvPe6iFPcc/ffff9G2bVtUrFgRlpaWGDhwIO7fv4/MzEzVNmZmZnB3d1ctv/yZP378GMnJyWjcuLFqvYGBAby9vV8bs4GBAd5//33Vcs2aNdW+/6dPn8bMmTPVjmfEiBFITk5Wi0/bn5+mxo4di9mzZ6N58+YICQnBmTNnCt32woULMDAwUHufypUrhxo1akj6O6Hp74YmXs0RClOvXj2YmZmplps2bYr09HQkJSUVWldB+YYB8OID0tTYsWNVj3bdvXsX7dq1g4eHBzZu3IiUlBRMmzYN6enpaNiwITZu3Jhv/woVKgAA1q5di7FjxyIiIgKbN2/G1KlTsWfPHjRp0gTTp09Hv379sGvXLvz9998ICQnBpk2b0KNHD43j1EReIgVA9abkdexNT0/Hp59+irFjx+bbr3LlygXW9/nnn2PPnj345ptvUK1aNZiamqJnz56lMr5L165dYWxsjO3bt8PIyAg5OTno2bOn7O2mp6dDX18fJ0+ehL6+vto6CwsLyXUV9RlcuXKlRLHm6du3L6ZMmYKYmBg8ffoUSUlJ6N27tyoGAFi9erXaDwWAfMf3suHDh2PgwIFYvHgx1q5di969e6udsC8zNTXVynG8/H0GXnynCyrL+44XtF/eeVDUuaGnpwfxyriMOTk5GsXzatt5XvceFPfcKs55XJSijik9PR1du3bF/Pnz8+2X9x+fVxkZGaFnz54IDw9Hnz59EB4ejt69e8PA4MVzCUeOHEH//v0xY8YM+Pn5wdraGps2bcLChQsLjfF1n09xztFr166hS5cuGDVqFObMmQM7OzscPHgQw4YNQ3Z2tuq7XdD782osckhPT8eMGTPw4Ycf5ltnYmKi+re2Pz9NDR8+HH5+fti1axd2796N0NBQLFy4EGPGjClRvSWl6bksJUd4HU3qyvdUDvBi3putW7dCCKH6UTp06BAsLS1x+fJldOnSBbNnz8bixYtRp04dHDp0CLt378bIkSMxbdo0NGjQAJs3b4a9vb3qaZiCeHl5wcvLC8HBwWjatCnCw8PRpEkTAMB7772H9957DxMmTEDfvn2xdu1arScmRWnQoAHi4uJQrVo1jfc5dOgQBg8erIozPT0d165dy7fd0aNH8y3XqlWrWHEaGRkVOD+RgYEBAgICsHbtWhgZGaFPnz5F/vjn9Yx+uS53d3cYGRnh0KFDqqkGcnJycOLEiUKfm/fy8kJubi5SU1PxwQcfFOuY8rzuM/D09IRSqURUVJTa/zCLOqaCVKpUCT4+Pti4cSOePn2Kdu3awd7eHsCLqwXOzs64evWq6iqKJjp16gRzc3OsWLECERERRU5waWlpCTc3N0RGRqJ169b51teqVQtJSUlISkpS/e8nLi4Ojx49Qu3atTWOSVsqVKiAc+fOqZXFxsbm+9GXom7duvjhhx/w4MGDAq+aaHJuFXQuFOc8Luycep0GDRpg69atcHNzUyUWmujfvz/atWuH8+fPY+/evZg9e7Zq3eHDh+Hq6or//e9/qrLr168XWV+FChXUrkDn5ubi3Llzqu9Wcc7RkydPQqlUYuHChaorcL/++qvGxwgA1tbWcHJywrFjx9CyZUsAwPPnz3Hy5Ek0aNCgyH2fP3+O6OhoNGrUCAAQHx+PR48eqX43GzRogPj4eEmf86uK+/m9qrDvj4uLC0aOHImRI0ciODgYq1evLjAxqVWrFp4/f45jx46pnv65f/8+4uPjJZ3vmvxuaPtcPn36NJ4+far6W3P06FFYWFioXbUpKM5X840CxzH57LPPkJSUhDFjxuDixYv4/fffERISgqCgIOTk5MDKygpTpkzB1q1bUalSJSQkJCAzMxM3b94E8OJEK1++PLp3747//vsPiYmJ2L9/P8aOHYubN28iMTERwcHBOHLkCK5fv47du3fj8uXLqFWrFp4+fYrAwEDs378f169fx6FDh3DixAnVF/DWrVuoWbMmjh8/Xqw3TlNTpkzB4cOHERgYiNjYWFy+fBm///47AgMDC92nevXq2LZtG2JjY3H69Gn069evwP8hHjp0CAsWLMClS5ewfPlybNmyBePGjStWnG5ubkhPT0dkZCTu3bundtly+PDh2Lt3LyIiIl57G8fV1RUKhQJ//vkn7t69i/T0dJibm2PUqFGYNGkSIiIiEBcXhxEjRiAzMxPDhg0rsJ733nsP/fv3x6BBg7Bt2zYkJibi+PHjCA0Nxa5duyQd2+s+Azc3NwQEBGDo0KHYsWOH6nuW94NZ0DEVpn///ti0aRO2bNmSLwGZMWMGQkNDsXTpUly6dAlnz57F2rVrsWjRokLr09fXx+DBgxEcHIzq1avnu333qunTp2PhwoVYunQpLl++jJiYGCxbtgwA4OvrC09PT/Tv3x8xMTE4fvw4Bg0aBB8fH40ug2tbmzZtEB0djfXr1+Py5csICQnJ9+MmVd++feHo6Ah/f38cOnQIV69exdatW3HkyBEAmp1bbm5uOHDgAG7duqUauqA457Gbm5vqVuG9e/eQlZWl0TGMHj0aDx48QN++fXHixAkkJCTgn3/+wZAhQ4pMdFq2bAlHR0f0798fVapUUbsyV716ddy4cQObNm1CQkICli5diu3btxcZR5s2bbBr1y7s2rULFy9exKhRo9TG8inOOVqtWjXk5ORg2bJluHr1KjZs2ICVK1dq9L68bNy4cZg3bx527NiBixcv4rPPPtNo7CRDQ0OMGTMGx44dw8mTJzF48GA0adJElahMmzYN69evx4wZM3D+/HlcuHABmzZtwtSpUzWOrbif36sK+h6OHz8e//zzDxITExETE4N9+/YV+p/R6tWro3v37hgxYgQOHjyI06dPY8CAAahYsSK6d++ucRya/G5o+1zOzs7GsGHDEBcXh7/++gshISEIDAxUu538qoLyDRTUUUqIwh8XbtSokZgyZYrYv3+/MDAwEE5OTsLQ0FA4ODgIKysr1f7Jycli0KBBonz58sLY2FhUrVpVjBgxQjx+/FikpKQIf39/4eTkJIyMjISrq6uYNm2ayM3NFVlZWaJPnz7CxcVFGBkZCWdnZxEYGKjqqJbX0fB1j7YJ8frOr3mdFYUouDPR8ePHRbt27YSFhYUwNzcXdevWFXPmzCm0vcTERNG6dWthamoqXFxcxLfffltgx6gZM2aIXr16CTMzM+Ho6CiWLFmiVg8kdH4VQoiRI0eKcuXKqR4XftkHH3xQ4KPDBZk5c6ZwdHQUCoVC1VHu6dOnYsyYMarPUZPHhbOzs8W0adOEm5ubMDQ0FE5OTqJHjx7izJkzQgjNO78K8frP4OnTp2LChAmq71K1atXEmjVrijymgr7zDx8+FMbGxsLMzEztEcg8GzduFPXr1xdGRkbC1tZWtGzZUmzbtk0IUfD3SQghEhISBIBCOwq/auXKlaJGjRqq92zMmDGqdZo+Lvyygjq2vXrsL3/XCjuWgr5z06ZNEw4ODsLa2lpMmDBBBAYG5usw97onQ15t+9q1a+Kjjz4SVlZWwszMTHh7e6s6N2pybh05ckTUrVtXGBsbqz0uLPU8fvbsmfjoo4+EjY1NvseFX45XCCGsra1V64V40XGxR48eqsc8a9asKcaPH6/Wsa8gkydPFgDEtGnT8q2bNGmSKFeunOqJh8WLF6t1eH/1s8/OzhajRo0SdnZ2wt7eXoSGhuZ77193jhZk0aJFwsnJSZiamgo/Pz+xfv3613bE3759u9pnkZOTI8aNGyesrKyEjY2NCAoK0vhx4a1bt4qqVasKY2Nj4evrq/a0oxBCREREiGbNmglTU1NhZWUlGjVqJFatWqVar43PT5POrwV9DwMDA4W7u7swNjYWFSpUEAMHDhT37t0r9JjzHhe2trZWvd8vPzGlSedXIV7/uyFE8c7lguT93kybNk31fR0xYoR49uzZa+t6Nd+QPInf/v370aNHD6SlpSEgIEDVwe/LL7/ExYsXsW3bNinVkYyEEKhevTo+++wzBAUFlXY4Oue///5D27ZtkZSUpOpUSkT0Lho8eDAePXqklWkbJN9Ia9WqFe7du4e0tDTY2tqqyj/55JNCO/fRm3f37l1s2rQJKSkpGDJkSGmHo1OysrJw9+5dTJ8+Hb169WJSQkQkQbF6+AghcPLkSSQkJKBfv36wtLSEkZERE5MyxN7eHuXLl8eqVavUEkiS3y+//IJhw4ahfv36WL9+fWmHQ0T0VpF8K+f69evo0KEDbty4gaysLFy6dAlVq1bFuHHjkJWVVawOUURERERAMWYXHjduHLy9vfHw4UO1x0979OiByMhIrQZHREREukXyrZz//vsPhw8fzjcjoJubG27duqW1wIiIiEj3SL5iolQqC3ym++bNm2rDYRMRERFJJTkxad++PcLCwlTLCoUC6enpCAkJQadOnbQZGxEREekYyZ1fb968CT8/PwghcPnyZXh7e+Py5csoX748Dhw4oBrKm4iIiEgqyYkJ8GLegs2bN+P06dNIT09HgwYN0L9/f61NRkZERES6qViJCREREZEcJPcxCQ0NVQ1D/7I1a9YUOF00ERERkaYkJybff/89atasma/cw8ODg6sRERFRifw/RnL2h11V9WYAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Apply Mask to attention scores\n",
    "attention_score = attention_score.masked_fill(torch.triu(torch.ones(attention_score.shape[-2:]), diagonal=1).bool(), float('-inf'))#[4, 4, 16, 16] [batch_size, num_heads, context_length, context_length]\n",
    "\n",
    "# Illustration only\n",
    "plt.imshow(attention_score[1, 1].detach().cpu().numpy(), \"Accent\", aspect=\"auto\")\n",
    "plt.title(\"Attention(Q,K)\")\n",
    "plt.xlabel(encoding.decode(x_batch[0].tolist()))\n",
    "plt.ylabel(encoding.decode(x_batch[0].tolist()))\n",
    "plt.colorbar()\n",
    "pd.DataFrame(attention_score[0][0].detach().cpu().numpy())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:31:43.305491Z",
     "start_time": "2024-02-09T04:31:42.713623Z"
    }
   },
   "id": "9958625c65291f4e",
   "execution_count": 27
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2         3         4         5         6         7         8         9         10        11        12        13        14        15\n0   1.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n1   0.659134  0.340867  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n2   0.488936  0.316498  0.194566  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n3   0.306845  0.226584  0.178756  0.287816  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n4   0.284609  0.313769  0.185624  0.071457  0.144541  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n5   0.159313  0.195051  0.155182  0.190583  0.132360  0.167512  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n6   0.195804  0.299437  0.138751  0.086341  0.131894  0.048049  0.099724  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n7   0.100501  0.213640  0.183524  0.068726  0.121962  0.092737  0.131933  0.086976  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n8   0.176701  0.085500  0.131407  0.107381  0.128974  0.074929  0.052436  0.110609  0.132063  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n9   0.148425  0.116194  0.097110  0.168735  0.096496  0.046389  0.056222  0.057891  0.069378  0.143160  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n10  0.127447  0.135999  0.101442  0.087011  0.098340  0.038114  0.055086  0.073207  0.064296  0.113778  0.105279  0.000000  0.000000  0.000000  0.000000  0.000000\n11  0.091145  0.070638  0.048715  0.058983  0.048746  0.083420  0.074915  0.082307  0.152265  0.122190  0.089881  0.076796  0.000000  0.000000  0.000000  0.000000\n12  0.075991  0.072665  0.094695  0.061723  0.096286  0.082016  0.057433  0.097850  0.047859  0.077099  0.083268  0.079255  0.073859  0.000000  0.000000  0.000000\n13  0.051873  0.075872  0.118007  0.023613  0.077783  0.047157  0.037058  0.129953  0.034912  0.053092  0.094544  0.065053  0.092150  0.098932  0.000000  0.000000\n14  0.055033  0.075014  0.122352  0.053248  0.074181  0.049398  0.051945  0.078949  0.064308  0.054671  0.080109  0.085398  0.048528  0.047361  0.059505  0.000000\n15  0.039804  0.032466  0.090289  0.034385  0.077958  0.118207  0.047124  0.060613  0.069334  0.052143  0.067315  0.062126  0.041590  0.069156  0.057991  0.079499",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n      <th>11</th>\n      <th>12</th>\n      <th>13</th>\n      <th>14</th>\n      <th>15</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.659134</td>\n      <td>0.340867</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.488936</td>\n      <td>0.316498</td>\n      <td>0.194566</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.306845</td>\n      <td>0.226584</td>\n      <td>0.178756</td>\n      <td>0.287816</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.284609</td>\n      <td>0.313769</td>\n      <td>0.185624</td>\n      <td>0.071457</td>\n      <td>0.144541</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.159313</td>\n      <td>0.195051</td>\n      <td>0.155182</td>\n      <td>0.190583</td>\n      <td>0.132360</td>\n      <td>0.167512</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.195804</td>\n      <td>0.299437</td>\n      <td>0.138751</td>\n      <td>0.086341</td>\n      <td>0.131894</td>\n      <td>0.048049</td>\n      <td>0.099724</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.100501</td>\n      <td>0.213640</td>\n      <td>0.183524</td>\n      <td>0.068726</td>\n      <td>0.121962</td>\n      <td>0.092737</td>\n      <td>0.131933</td>\n      <td>0.086976</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.176701</td>\n      <td>0.085500</td>\n      <td>0.131407</td>\n      <td>0.107381</td>\n      <td>0.128974</td>\n      <td>0.074929</td>\n      <td>0.052436</td>\n      <td>0.110609</td>\n      <td>0.132063</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>0.148425</td>\n      <td>0.116194</td>\n      <td>0.097110</td>\n      <td>0.168735</td>\n      <td>0.096496</td>\n      <td>0.046389</td>\n      <td>0.056222</td>\n      <td>0.057891</td>\n      <td>0.069378</td>\n      <td>0.143160</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>0.127447</td>\n      <td>0.135999</td>\n      <td>0.101442</td>\n      <td>0.087011</td>\n      <td>0.098340</td>\n      <td>0.038114</td>\n      <td>0.055086</td>\n      <td>0.073207</td>\n      <td>0.064296</td>\n      <td>0.113778</td>\n      <td>0.105279</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>0.091145</td>\n      <td>0.070638</td>\n      <td>0.048715</td>\n      <td>0.058983</td>\n      <td>0.048746</td>\n      <td>0.083420</td>\n      <td>0.074915</td>\n      <td>0.082307</td>\n      <td>0.152265</td>\n      <td>0.122190</td>\n      <td>0.089881</td>\n      <td>0.076796</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>0.075991</td>\n      <td>0.072665</td>\n      <td>0.094695</td>\n      <td>0.061723</td>\n      <td>0.096286</td>\n      <td>0.082016</td>\n      <td>0.057433</td>\n      <td>0.097850</td>\n      <td>0.047859</td>\n      <td>0.077099</td>\n      <td>0.083268</td>\n      <td>0.079255</td>\n      <td>0.073859</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>0.051873</td>\n      <td>0.075872</td>\n      <td>0.118007</td>\n      <td>0.023613</td>\n      <td>0.077783</td>\n      <td>0.047157</td>\n      <td>0.037058</td>\n      <td>0.129953</td>\n      <td>0.034912</td>\n      <td>0.053092</td>\n      <td>0.094544</td>\n      <td>0.065053</td>\n      <td>0.092150</td>\n      <td>0.098932</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>0.055033</td>\n      <td>0.075014</td>\n      <td>0.122352</td>\n      <td>0.053248</td>\n      <td>0.074181</td>\n      <td>0.049398</td>\n      <td>0.051945</td>\n      <td>0.078949</td>\n      <td>0.064308</td>\n      <td>0.054671</td>\n      <td>0.080109</td>\n      <td>0.085398</td>\n      <td>0.048528</td>\n      <td>0.047361</td>\n      <td>0.059505</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>0.039804</td>\n      <td>0.032466</td>\n      <td>0.090289</td>\n      <td>0.034385</td>\n      <td>0.077958</td>\n      <td>0.118207</td>\n      <td>0.047124</td>\n      <td>0.060613</td>\n      <td>0.069334</td>\n      <td>0.052143</td>\n      <td>0.067315</td>\n      <td>0.062126</td>\n      <td>0.041590</td>\n      <td>0.069156</td>\n      <td>0.057991</td>\n      <td>0.079499</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Softmax the attention score\n",
    "attention_score = torch.softmax(attention_score, dim=-1) #[4, 4, 16, 16] [batch_size, num_heads, context_length, context_length]\n",
    "pd.DataFrame(attention_score[0][0].detach().cpu().numpy())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:32:08.443877Z",
     "start_time": "2024-02-09T04:32:08.408750Z"
    }
   },
   "id": "7eb3f22c3466b59b",
   "execution_count": 28
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 4, 16, 16])\n",
      "torch.Size([4, 4, 16, 16])\n",
      "torch.Size([4, 4, 16, 16])\n"
     ]
    }
   ],
   "source": [
    "# Calculate the V attention output\n",
    "print(attention_score.shape) #[4, 4, 16, 16] [batch_size, num_heads, context_length, context_length]\n",
    "print(V.shape) #[4, 4, 16, 16] [batch_size, num_heads, context_length, head_size]\n",
    "A = torch.matmul(attention_score, V) # [4, 4, 16, 16] [batch_size, num_heads, context_length, head_size]\n",
    "print(A.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:32:19.745362Z",
     "start_time": "2024-02-09T04:32:19.729817Z"
    }
   },
   "id": "3a14ab6238d8f340",
   "execution_count": 29
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([4, 16, 64])"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Concatenate the attention output\n",
    "A = A.transpose(1, 2) # [4, 16, 4, 16] [batch_size, context_length, num_heads, head_size]\n",
    "A = A.reshape(batch_size, -1, d_model) # [4, 16, 64] [batch_size, context_length, d_model]\n",
    "A.shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:32:38.040254Z",
     "start_time": "2024-02-09T04:32:37.994213Z"
    }
   },
   "id": "57894d06f08e7f5e",
   "execution_count": 31
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 16, 64])\n"
     ]
    },
    {
     "data": {
      "text/plain": "          0         1         2         3         4         5         6         7         8         9   ...        54        55        56        57        58        59        60        61        62        63\n0  -0.136445 -0.210882 -0.276101 -0.118035  0.450475 -0.263781  0.436132  0.102870  0.209401  0.019048  ...  0.545384 -0.129457  0.255404 -0.189144 -0.379158 -0.097181 -0.089340 -0.002253  0.009513  0.159399\n1  -0.164441 -0.185361 -0.000391  0.165244 -0.096607 -0.270862  0.209424  0.025862 -0.192396 -0.184595  ...  0.437300  0.068226  0.381531  0.152842  0.069350  0.167859 -0.643934 -0.180995  0.159944  0.320839\n2   0.474719 -0.398138 -0.111474 -0.300319 -0.264287  0.388932  0.312517  0.285156 -0.192744 -0.303520  ...  0.173660 -0.207655 -0.009765  0.073964  0.654195  0.420884  0.251090 -0.059274  0.150397 -0.747803\n3   0.098448 -0.532183  0.211407  0.393391 -0.179980 -0.709364  0.280023  0.179076 -0.238339  0.037933  ...  0.711805 -0.268873 -0.252844  0.111075 -0.001156  0.080273 -0.627740 -0.048182  0.117691  0.446960\n4  -0.089438  0.200252 -0.287827 -0.183713  0.147652 -0.000334  0.079829 -0.007042  0.101122 -0.242136  ...  0.064852  0.154467  0.437642 -0.158713  0.048466  0.139457  0.168615 -0.168226  0.244715 -0.093203\n5   0.564638 -0.318142  0.167704  0.168587 -0.196722 -0.335033 -0.119111  0.097945 -0.235783 -0.036879  ... -0.028674 -0.224565 -0.600959 -0.251498  0.210468 -0.090708  0.226622  0.053661 -0.083247 -0.256771\n6  -0.444749  0.549815 -0.146958 -0.202845  0.441075 -0.485654 -0.253080  0.008336  0.231136 -0.198602  ... -0.058932  0.165445  0.074872 -0.446140 -0.326007 -0.236170 -0.158297 -0.221966 -0.073392  0.041948\n7   0.263858 -0.107601 -0.189519 -0.474147  0.104792  0.183744  0.241586  0.084666 -0.025186 -0.165745  ...  0.037578 -0.153609  0.134120  0.029238  0.133398  0.280861  0.142644 -0.130737  0.146001 -0.286553\n8  -0.044496  0.098924  0.047460 -0.040495  0.271419 -0.466821 -0.383319 -0.018060  0.127171  0.152121  ... -0.152705 -0.246356 -0.394520 -0.314173 -0.269650 -0.481611 -0.069016  0.335082 -0.239683 -0.122500\n9  -0.617625  0.203468 -0.390796 -0.137122 -0.049398 -0.149874 -0.387239 -0.155943 -0.094754 -0.402197  ...  0.245478  0.463289  0.313353 -0.353231  0.274220 -0.032978  0.039406 -0.038581  0.062824  0.227389\n10 -0.033957 -0.511949 -0.137513  0.213922 -0.151818 -0.557235  0.199889  0.100002 -0.060549  0.083059  ...  0.600144 -0.046886 -0.104703 -0.187639  0.349231  0.043705 -0.244877 -0.157645  0.033046  0.279628\n11  0.653269 -0.140986  0.178338 -0.210512 -0.182025  0.038061 -0.030899  0.182278 -0.222574  0.021597  ... -0.273550 -0.488285 -0.668950  0.295052  0.141214  0.126645  0.227238  0.116767  0.222692 -0.576531\n12 -0.211688  0.063018 -0.276438 -0.305115  0.344467  0.185617  0.655044  0.065778  0.180189 -0.246063  ...  0.522749  0.267338  1.103017  0.165962 -0.235553  0.214957 -0.172872 -0.353915  0.366547  0.312577\n13 -0.406272  0.177455 -0.373973 -0.269012  0.443952  0.014973  0.299664  0.060399  0.248698 -0.159796  ...  0.394637  0.312999  0.571763 -0.062308 -0.085150  0.180287  0.021733 -0.355008  0.204813  0.320210\n14 -0.072227  0.081222 -0.220820 -0.058172 -0.054144  0.140578  0.241753 -0.077895 -0.067169 -0.252682  ...  0.138118  0.071120  0.362042  0.126215  0.122908  0.191198 -0.136044 -0.281643  0.335987  0.395848\n15  0.301267 -0.102764 -0.037482 -0.191742  0.288399 -0.293867  0.417902  0.318640  0.052452  0.099422  ...  0.354607 -0.598772 -0.151706 -0.024195 -0.393862 -0.040914 -0.053562 -0.130105 -0.226092  0.267122\n\n[16 rows x 64 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>54</th>\n      <th>55</th>\n      <th>56</th>\n      <th>57</th>\n      <th>58</th>\n      <th>59</th>\n      <th>60</th>\n      <th>61</th>\n      <th>62</th>\n      <th>63</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.136445</td>\n      <td>-0.210882</td>\n      <td>-0.276101</td>\n      <td>-0.118035</td>\n      <td>0.450475</td>\n      <td>-0.263781</td>\n      <td>0.436132</td>\n      <td>0.102870</td>\n      <td>0.209401</td>\n      <td>0.019048</td>\n      <td>...</td>\n      <td>0.545384</td>\n      <td>-0.129457</td>\n      <td>0.255404</td>\n      <td>-0.189144</td>\n      <td>-0.379158</td>\n      <td>-0.097181</td>\n      <td>-0.089340</td>\n      <td>-0.002253</td>\n      <td>0.009513</td>\n      <td>0.159399</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.164441</td>\n      <td>-0.185361</td>\n      <td>-0.000391</td>\n      <td>0.165244</td>\n      <td>-0.096607</td>\n      <td>-0.270862</td>\n      <td>0.209424</td>\n      <td>0.025862</td>\n      <td>-0.192396</td>\n      <td>-0.184595</td>\n      <td>...</td>\n      <td>0.437300</td>\n      <td>0.068226</td>\n      <td>0.381531</td>\n      <td>0.152842</td>\n      <td>0.069350</td>\n      <td>0.167859</td>\n      <td>-0.643934</td>\n      <td>-0.180995</td>\n      <td>0.159944</td>\n      <td>0.320839</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.474719</td>\n      <td>-0.398138</td>\n      <td>-0.111474</td>\n      <td>-0.300319</td>\n      <td>-0.264287</td>\n      <td>0.388932</td>\n      <td>0.312517</td>\n      <td>0.285156</td>\n      <td>-0.192744</td>\n      <td>-0.303520</td>\n      <td>...</td>\n      <td>0.173660</td>\n      <td>-0.207655</td>\n      <td>-0.009765</td>\n      <td>0.073964</td>\n      <td>0.654195</td>\n      <td>0.420884</td>\n      <td>0.251090</td>\n      <td>-0.059274</td>\n      <td>0.150397</td>\n      <td>-0.747803</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.098448</td>\n      <td>-0.532183</td>\n      <td>0.211407</td>\n      <td>0.393391</td>\n      <td>-0.179980</td>\n      <td>-0.709364</td>\n      <td>0.280023</td>\n      <td>0.179076</td>\n      <td>-0.238339</td>\n      <td>0.037933</td>\n      <td>...</td>\n      <td>0.711805</td>\n      <td>-0.268873</td>\n      <td>-0.252844</td>\n      <td>0.111075</td>\n      <td>-0.001156</td>\n      <td>0.080273</td>\n      <td>-0.627740</td>\n      <td>-0.048182</td>\n      <td>0.117691</td>\n      <td>0.446960</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-0.089438</td>\n      <td>0.200252</td>\n      <td>-0.287827</td>\n      <td>-0.183713</td>\n      <td>0.147652</td>\n      <td>-0.000334</td>\n      <td>0.079829</td>\n      <td>-0.007042</td>\n      <td>0.101122</td>\n      <td>-0.242136</td>\n      <td>...</td>\n      <td>0.064852</td>\n      <td>0.154467</td>\n      <td>0.437642</td>\n      <td>-0.158713</td>\n      <td>0.048466</td>\n      <td>0.139457</td>\n      <td>0.168615</td>\n      <td>-0.168226</td>\n      <td>0.244715</td>\n      <td>-0.093203</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.564638</td>\n      <td>-0.318142</td>\n      <td>0.167704</td>\n      <td>0.168587</td>\n      <td>-0.196722</td>\n      <td>-0.335033</td>\n      <td>-0.119111</td>\n      <td>0.097945</td>\n      <td>-0.235783</td>\n      <td>-0.036879</td>\n      <td>...</td>\n      <td>-0.028674</td>\n      <td>-0.224565</td>\n      <td>-0.600959</td>\n      <td>-0.251498</td>\n      <td>0.210468</td>\n      <td>-0.090708</td>\n      <td>0.226622</td>\n      <td>0.053661</td>\n      <td>-0.083247</td>\n      <td>-0.256771</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>-0.444749</td>\n      <td>0.549815</td>\n      <td>-0.146958</td>\n      <td>-0.202845</td>\n      <td>0.441075</td>\n      <td>-0.485654</td>\n      <td>-0.253080</td>\n      <td>0.008336</td>\n      <td>0.231136</td>\n      <td>-0.198602</td>\n      <td>...</td>\n      <td>-0.058932</td>\n      <td>0.165445</td>\n      <td>0.074872</td>\n      <td>-0.446140</td>\n      <td>-0.326007</td>\n      <td>-0.236170</td>\n      <td>-0.158297</td>\n      <td>-0.221966</td>\n      <td>-0.073392</td>\n      <td>0.041948</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.263858</td>\n      <td>-0.107601</td>\n      <td>-0.189519</td>\n      <td>-0.474147</td>\n      <td>0.104792</td>\n      <td>0.183744</td>\n      <td>0.241586</td>\n      <td>0.084666</td>\n      <td>-0.025186</td>\n      <td>-0.165745</td>\n      <td>...</td>\n      <td>0.037578</td>\n      <td>-0.153609</td>\n      <td>0.134120</td>\n      <td>0.029238</td>\n      <td>0.133398</td>\n      <td>0.280861</td>\n      <td>0.142644</td>\n      <td>-0.130737</td>\n      <td>0.146001</td>\n      <td>-0.286553</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>-0.044496</td>\n      <td>0.098924</td>\n      <td>0.047460</td>\n      <td>-0.040495</td>\n      <td>0.271419</td>\n      <td>-0.466821</td>\n      <td>-0.383319</td>\n      <td>-0.018060</td>\n      <td>0.127171</td>\n      <td>0.152121</td>\n      <td>...</td>\n      <td>-0.152705</td>\n      <td>-0.246356</td>\n      <td>-0.394520</td>\n      <td>-0.314173</td>\n      <td>-0.269650</td>\n      <td>-0.481611</td>\n      <td>-0.069016</td>\n      <td>0.335082</td>\n      <td>-0.239683</td>\n      <td>-0.122500</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>-0.617625</td>\n      <td>0.203468</td>\n      <td>-0.390796</td>\n      <td>-0.137122</td>\n      <td>-0.049398</td>\n      <td>-0.149874</td>\n      <td>-0.387239</td>\n      <td>-0.155943</td>\n      <td>-0.094754</td>\n      <td>-0.402197</td>\n      <td>...</td>\n      <td>0.245478</td>\n      <td>0.463289</td>\n      <td>0.313353</td>\n      <td>-0.353231</td>\n      <td>0.274220</td>\n      <td>-0.032978</td>\n      <td>0.039406</td>\n      <td>-0.038581</td>\n      <td>0.062824</td>\n      <td>0.227389</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>-0.033957</td>\n      <td>-0.511949</td>\n      <td>-0.137513</td>\n      <td>0.213922</td>\n      <td>-0.151818</td>\n      <td>-0.557235</td>\n      <td>0.199889</td>\n      <td>0.100002</td>\n      <td>-0.060549</td>\n      <td>0.083059</td>\n      <td>...</td>\n      <td>0.600144</td>\n      <td>-0.046886</td>\n      <td>-0.104703</td>\n      <td>-0.187639</td>\n      <td>0.349231</td>\n      <td>0.043705</td>\n      <td>-0.244877</td>\n      <td>-0.157645</td>\n      <td>0.033046</td>\n      <td>0.279628</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>0.653269</td>\n      <td>-0.140986</td>\n      <td>0.178338</td>\n      <td>-0.210512</td>\n      <td>-0.182025</td>\n      <td>0.038061</td>\n      <td>-0.030899</td>\n      <td>0.182278</td>\n      <td>-0.222574</td>\n      <td>0.021597</td>\n      <td>...</td>\n      <td>-0.273550</td>\n      <td>-0.488285</td>\n      <td>-0.668950</td>\n      <td>0.295052</td>\n      <td>0.141214</td>\n      <td>0.126645</td>\n      <td>0.227238</td>\n      <td>0.116767</td>\n      <td>0.222692</td>\n      <td>-0.576531</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>-0.211688</td>\n      <td>0.063018</td>\n      <td>-0.276438</td>\n      <td>-0.305115</td>\n      <td>0.344467</td>\n      <td>0.185617</td>\n      <td>0.655044</td>\n      <td>0.065778</td>\n      <td>0.180189</td>\n      <td>-0.246063</td>\n      <td>...</td>\n      <td>0.522749</td>\n      <td>0.267338</td>\n      <td>1.103017</td>\n      <td>0.165962</td>\n      <td>-0.235553</td>\n      <td>0.214957</td>\n      <td>-0.172872</td>\n      <td>-0.353915</td>\n      <td>0.366547</td>\n      <td>0.312577</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>-0.406272</td>\n      <td>0.177455</td>\n      <td>-0.373973</td>\n      <td>-0.269012</td>\n      <td>0.443952</td>\n      <td>0.014973</td>\n      <td>0.299664</td>\n      <td>0.060399</td>\n      <td>0.248698</td>\n      <td>-0.159796</td>\n      <td>...</td>\n      <td>0.394637</td>\n      <td>0.312999</td>\n      <td>0.571763</td>\n      <td>-0.062308</td>\n      <td>-0.085150</td>\n      <td>0.180287</td>\n      <td>0.021733</td>\n      <td>-0.355008</td>\n      <td>0.204813</td>\n      <td>0.320210</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>-0.072227</td>\n      <td>0.081222</td>\n      <td>-0.220820</td>\n      <td>-0.058172</td>\n      <td>-0.054144</td>\n      <td>0.140578</td>\n      <td>0.241753</td>\n      <td>-0.077895</td>\n      <td>-0.067169</td>\n      <td>-0.252682</td>\n      <td>...</td>\n      <td>0.138118</td>\n      <td>0.071120</td>\n      <td>0.362042</td>\n      <td>0.126215</td>\n      <td>0.122908</td>\n      <td>0.191198</td>\n      <td>-0.136044</td>\n      <td>-0.281643</td>\n      <td>0.335987</td>\n      <td>0.395848</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>0.301267</td>\n      <td>-0.102764</td>\n      <td>-0.037482</td>\n      <td>-0.191742</td>\n      <td>0.288399</td>\n      <td>-0.293867</td>\n      <td>0.417902</td>\n      <td>0.318640</td>\n      <td>0.052452</td>\n      <td>0.099422</td>\n      <td>...</td>\n      <td>0.354607</td>\n      <td>-0.598772</td>\n      <td>-0.151706</td>\n      <td>-0.024195</td>\n      <td>-0.393862</td>\n      <td>-0.040914</td>\n      <td>-0.053562</td>\n      <td>-0.130105</td>\n      <td>-0.226092</td>\n      <td>0.267122</td>\n    </tr>\n  </tbody>\n</table>\n<p>16 rows × 64 columns</p>\n</div>"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the output weight matrix\n",
    "Wo = nn.Linear(d_model, d_model)\n",
    "output = Wo(A) # [4, 16, 64] [batch_size, context_length, d_model]\n",
    "print(output.shape)\n",
    "pd.DataFrame(output[0].detach().cpu().numpy())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:33:01.895820Z",
     "start_time": "2024-02-09T04:33:01.830312Z"
    }
   },
   "id": "8f4fbfce6f7f330a",
   "execution_count": 33
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Add residual connection\n",
    "output = output + X"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:34:34.038171Z",
     "start_time": "2024-02-09T04:34:34.029896Z"
    }
   },
   "id": "2835a23754fafc42",
   "execution_count": 34
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Add Layer Normalization\n",
    "layer_norm = nn.LayerNorm(d_model)\n",
    "output_layernorm = layer_norm(output)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:34:34.636527Z",
     "start_time": "2024-02-09T04:34:34.608438Z"
    }
   },
   "id": "7761664159de5538",
   "execution_count": 35
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Define Feed Forward Network\n",
    "output = nn.Linear(d_model, d_model * 4)(output_layernorm)\n",
    "output = nn.ReLU()(output)\n",
    "output = nn.Linear(d_model * 4, d_model)(output)\n",
    "output = torch.dropout(output, p=dropout, train=True)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:34:35.139721Z",
     "start_time": "2024-02-09T04:34:35.110122Z"
    }
   },
   "id": "2ce7c56ac3de91a0",
   "execution_count": 36
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 16, 64])\n"
     ]
    }
   ],
   "source": [
    "# Add residual connection & layerNorm (last time in a Transformer block)\n",
    "output = output + output_layernorm\n",
    "# Add Layer Normalization\n",
    "layer_norm = nn.LayerNorm(d_model)\n",
    "output = layer_norm(output)\n",
    "print(output.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:35:25.104704Z",
     "start_time": "2024-02-09T04:35:25.076394Z"
    }
   },
   "id": "35cce4d92eeb74ee",
   "execution_count": 38
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Until here, we finished a Transformer block,\n",
    "# We actually should pack the above Transformer block code into a call and \n",
    "# repeat the steps for num_layers times\n",
    "# but this jupyter notebook is purely for illustration purpose, so we'll skip it:\n",
    "# for _ in range(num_layers):\n",
    "#   do loop for each transformer block"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:35:25.618791Z",
     "start_time": "2024-02-09T04:35:25.608096Z"
    }
   },
   "id": "5a0de0adb842207e",
   "execution_count": 39
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "      0         1         2         3         4         5         6         7         8         9       ...    100060    100061    100062    100063    100064    100065    100066    100067    100068    100069\n0  -1.537361 -0.200670 -0.408220 -0.285989  0.503236 -0.352333  0.429026  0.618260 -0.630350  0.003683  ... -0.747295 -0.510281  0.190999  0.736991 -0.211302  0.486846  0.619305  0.239595  0.915543 -1.194285\n1  -0.106679  0.231435 -0.189576 -0.182358  0.747573 -0.745270  0.414703 -0.177571 -0.471406  0.088703  ...  0.357872  0.074848  0.632222  0.522492 -0.006226 -0.386350 -0.336347 -0.283921  0.476996 -0.246730\n2   0.019565  0.234084 -0.517373 -0.946907  0.524948 -0.830676  0.752750 -0.458348 -0.070522 -0.266437  ...  0.669110 -0.925616  1.175533  0.860116 -0.381903  0.240957 -0.381847 -0.562366  0.530819 -0.502148\n3  -0.126875 -0.180552 -0.217995 -0.290191  0.532972 -0.675587 -0.237965  0.160410 -1.149514 -1.691638  ...  0.159243 -0.107689 -0.243918 -0.826834 -0.745132 -0.772887  0.323196  0.450803 -1.003144 -0.696712\n4  -0.102389  0.727906  0.566872  0.063772 -1.096750 -0.190684  0.875384 -0.586231 -0.316142  0.186759  ...  0.360017 -0.071049  0.614276  1.576313 -0.007284  1.404002  0.039557  0.005006 -0.341690 -1.459264\n5  -0.111226  0.943119 -0.515207  0.177915 -0.563385 -0.512341  0.603554 -0.474289 -0.070351  0.149274  ... -0.806764 -0.108375  0.370318 -0.603620 -1.000170 -1.134740 -0.315730 -0.175492 -0.349457 -0.795470\n6   0.872424  0.486800 -0.425815  0.314616  0.280176  0.409603  0.146088 -0.452896  0.065016 -0.481648  ...  0.302409  0.853725 -0.519820  0.880158 -0.653103 -0.280612 -0.350723  0.611320 -0.934517 -0.676754\n7   0.741628  0.256866 -1.012910  0.433350 -0.845165  0.329714 -0.292277 -0.330230 -0.764646  0.831685  ... -0.010119  0.961887 -0.682834 -0.063657 -0.971638  0.985793  0.409012 -0.629063  0.041146 -1.439935\n8   0.226227 -0.125217 -0.642953 -0.605414  0.354313  1.338140 -0.153114  0.024755  0.146925 -1.252991  ... -0.450090  0.140722  0.167505  0.040322 -0.333780 -0.595672 -0.975344  0.171940  0.263188 -0.046738\n9  -0.862159  0.270271 -0.578958 -0.446508  0.306703  0.086267  0.488297  0.461057 -0.148345 -0.514032  ... -0.928977 -0.527919  0.292537  1.034436 -0.234091  0.578609  0.008787  0.305765  0.614632 -0.721908\n10 -0.334877  0.718262 -0.339231 -0.403899  0.039919  0.001551  0.667907  0.381174 -1.124639 -0.373244  ... -0.481928 -0.127556  0.303473  0.544919  0.430514  1.019882 -0.950689  0.138021  0.685181 -0.615615\n11 -0.694525  0.572969  0.215758 -1.143841 -0.388871 -0.107535  0.049288 -0.786526  0.536279 -0.506368  ... -0.186428 -1.487678  0.713042  0.093053  0.091143 -0.141182 -0.674878  0.657704 -0.924867  0.410927\n12  0.405904  1.368553  0.995166 -0.403671 -0.373950  0.493838  0.565539 -0.089824  0.081446  0.282199  ...  0.361010  0.858070 -0.119999  0.130150 -0.323220  0.839781 -0.990412  0.505244 -0.811275  0.134853\n13 -0.539911  0.262381  0.145610 -0.962335 -0.060307  0.812004  0.256701  0.289468 -0.314500 -0.173251  ... -0.416052 -0.145934  0.514849  0.633466 -0.178631  1.113246 -0.278801  0.262267  0.436200 -0.159290\n14  0.257532  0.128558  0.558949 -0.677057  0.045429  0.276470  0.239281 -0.226379  0.614334 -0.522354  ... -0.466851  0.808567 -0.663552  0.648689 -1.155005  0.039744 -0.427043 -0.117230 -1.050166 -0.729577\n15  0.350021 -0.158842 -0.374558 -0.575825  0.334527 -0.461471  1.042502 -0.789735  0.057360  0.060458  ...  0.112147  0.032235 -0.489684  0.233091 -0.997444 -0.356804 -0.442014 -0.286688  0.559590  0.437227\n\n[16 rows x 100070 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>100060</th>\n      <th>100061</th>\n      <th>100062</th>\n      <th>100063</th>\n      <th>100064</th>\n      <th>100065</th>\n      <th>100066</th>\n      <th>100067</th>\n      <th>100068</th>\n      <th>100069</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-1.537361</td>\n      <td>-0.200670</td>\n      <td>-0.408220</td>\n      <td>-0.285989</td>\n      <td>0.503236</td>\n      <td>-0.352333</td>\n      <td>0.429026</td>\n      <td>0.618260</td>\n      <td>-0.630350</td>\n      <td>0.003683</td>\n      <td>...</td>\n      <td>-0.747295</td>\n      <td>-0.510281</td>\n      <td>0.190999</td>\n      <td>0.736991</td>\n      <td>-0.211302</td>\n      <td>0.486846</td>\n      <td>0.619305</td>\n      <td>0.239595</td>\n      <td>0.915543</td>\n      <td>-1.194285</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.106679</td>\n      <td>0.231435</td>\n      <td>-0.189576</td>\n      <td>-0.182358</td>\n      <td>0.747573</td>\n      <td>-0.745270</td>\n      <td>0.414703</td>\n      <td>-0.177571</td>\n      <td>-0.471406</td>\n      <td>0.088703</td>\n      <td>...</td>\n      <td>0.357872</td>\n      <td>0.074848</td>\n      <td>0.632222</td>\n      <td>0.522492</td>\n      <td>-0.006226</td>\n      <td>-0.386350</td>\n      <td>-0.336347</td>\n      <td>-0.283921</td>\n      <td>0.476996</td>\n      <td>-0.246730</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.019565</td>\n      <td>0.234084</td>\n      <td>-0.517373</td>\n      <td>-0.946907</td>\n      <td>0.524948</td>\n      <td>-0.830676</td>\n      <td>0.752750</td>\n      <td>-0.458348</td>\n      <td>-0.070522</td>\n      <td>-0.266437</td>\n      <td>...</td>\n      <td>0.669110</td>\n      <td>-0.925616</td>\n      <td>1.175533</td>\n      <td>0.860116</td>\n      <td>-0.381903</td>\n      <td>0.240957</td>\n      <td>-0.381847</td>\n      <td>-0.562366</td>\n      <td>0.530819</td>\n      <td>-0.502148</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.126875</td>\n      <td>-0.180552</td>\n      <td>-0.217995</td>\n      <td>-0.290191</td>\n      <td>0.532972</td>\n      <td>-0.675587</td>\n      <td>-0.237965</td>\n      <td>0.160410</td>\n      <td>-1.149514</td>\n      <td>-1.691638</td>\n      <td>...</td>\n      <td>0.159243</td>\n      <td>-0.107689</td>\n      <td>-0.243918</td>\n      <td>-0.826834</td>\n      <td>-0.745132</td>\n      <td>-0.772887</td>\n      <td>0.323196</td>\n      <td>0.450803</td>\n      <td>-1.003144</td>\n      <td>-0.696712</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-0.102389</td>\n      <td>0.727906</td>\n      <td>0.566872</td>\n      <td>0.063772</td>\n      <td>-1.096750</td>\n      <td>-0.190684</td>\n      <td>0.875384</td>\n      <td>-0.586231</td>\n      <td>-0.316142</td>\n      <td>0.186759</td>\n      <td>...</td>\n      <td>0.360017</td>\n      <td>-0.071049</td>\n      <td>0.614276</td>\n      <td>1.576313</td>\n      <td>-0.007284</td>\n      <td>1.404002</td>\n      <td>0.039557</td>\n      <td>0.005006</td>\n      <td>-0.341690</td>\n      <td>-1.459264</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-0.111226</td>\n      <td>0.943119</td>\n      <td>-0.515207</td>\n      <td>0.177915</td>\n      <td>-0.563385</td>\n      <td>-0.512341</td>\n      <td>0.603554</td>\n      <td>-0.474289</td>\n      <td>-0.070351</td>\n      <td>0.149274</td>\n      <td>...</td>\n      <td>-0.806764</td>\n      <td>-0.108375</td>\n      <td>0.370318</td>\n      <td>-0.603620</td>\n      <td>-1.000170</td>\n      <td>-1.134740</td>\n      <td>-0.315730</td>\n      <td>-0.175492</td>\n      <td>-0.349457</td>\n      <td>-0.795470</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.872424</td>\n      <td>0.486800</td>\n      <td>-0.425815</td>\n      <td>0.314616</td>\n      <td>0.280176</td>\n      <td>0.409603</td>\n      <td>0.146088</td>\n      <td>-0.452896</td>\n      <td>0.065016</td>\n      <td>-0.481648</td>\n      <td>...</td>\n      <td>0.302409</td>\n      <td>0.853725</td>\n      <td>-0.519820</td>\n      <td>0.880158</td>\n      <td>-0.653103</td>\n      <td>-0.280612</td>\n      <td>-0.350723</td>\n      <td>0.611320</td>\n      <td>-0.934517</td>\n      <td>-0.676754</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.741628</td>\n      <td>0.256866</td>\n      <td>-1.012910</td>\n      <td>0.433350</td>\n      <td>-0.845165</td>\n      <td>0.329714</td>\n      <td>-0.292277</td>\n      <td>-0.330230</td>\n      <td>-0.764646</td>\n      <td>0.831685</td>\n      <td>...</td>\n      <td>-0.010119</td>\n      <td>0.961887</td>\n      <td>-0.682834</td>\n      <td>-0.063657</td>\n      <td>-0.971638</td>\n      <td>0.985793</td>\n      <td>0.409012</td>\n      <td>-0.629063</td>\n      <td>0.041146</td>\n      <td>-1.439935</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.226227</td>\n      <td>-0.125217</td>\n      <td>-0.642953</td>\n      <td>-0.605414</td>\n      <td>0.354313</td>\n      <td>1.338140</td>\n      <td>-0.153114</td>\n      <td>0.024755</td>\n      <td>0.146925</td>\n      <td>-1.252991</td>\n      <td>...</td>\n      <td>-0.450090</td>\n      <td>0.140722</td>\n      <td>0.167505</td>\n      <td>0.040322</td>\n      <td>-0.333780</td>\n      <td>-0.595672</td>\n      <td>-0.975344</td>\n      <td>0.171940</td>\n      <td>0.263188</td>\n      <td>-0.046738</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>-0.862159</td>\n      <td>0.270271</td>\n      <td>-0.578958</td>\n      <td>-0.446508</td>\n      <td>0.306703</td>\n      <td>0.086267</td>\n      <td>0.488297</td>\n      <td>0.461057</td>\n      <td>-0.148345</td>\n      <td>-0.514032</td>\n      <td>...</td>\n      <td>-0.928977</td>\n      <td>-0.527919</td>\n      <td>0.292537</td>\n      <td>1.034436</td>\n      <td>-0.234091</td>\n      <td>0.578609</td>\n      <td>0.008787</td>\n      <td>0.305765</td>\n      <td>0.614632</td>\n      <td>-0.721908</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>-0.334877</td>\n      <td>0.718262</td>\n      <td>-0.339231</td>\n      <td>-0.403899</td>\n      <td>0.039919</td>\n      <td>0.001551</td>\n      <td>0.667907</td>\n      <td>0.381174</td>\n      <td>-1.124639</td>\n      <td>-0.373244</td>\n      <td>...</td>\n      <td>-0.481928</td>\n      <td>-0.127556</td>\n      <td>0.303473</td>\n      <td>0.544919</td>\n      <td>0.430514</td>\n      <td>1.019882</td>\n      <td>-0.950689</td>\n      <td>0.138021</td>\n      <td>0.685181</td>\n      <td>-0.615615</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>-0.694525</td>\n      <td>0.572969</td>\n      <td>0.215758</td>\n      <td>-1.143841</td>\n      <td>-0.388871</td>\n      <td>-0.107535</td>\n      <td>0.049288</td>\n      <td>-0.786526</td>\n      <td>0.536279</td>\n      <td>-0.506368</td>\n      <td>...</td>\n      <td>-0.186428</td>\n      <td>-1.487678</td>\n      <td>0.713042</td>\n      <td>0.093053</td>\n      <td>0.091143</td>\n      <td>-0.141182</td>\n      <td>-0.674878</td>\n      <td>0.657704</td>\n      <td>-0.924867</td>\n      <td>0.410927</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>0.405904</td>\n      <td>1.368553</td>\n      <td>0.995166</td>\n      <td>-0.403671</td>\n      <td>-0.373950</td>\n      <td>0.493838</td>\n      <td>0.565539</td>\n      <td>-0.089824</td>\n      <td>0.081446</td>\n      <td>0.282199</td>\n      <td>...</td>\n      <td>0.361010</td>\n      <td>0.858070</td>\n      <td>-0.119999</td>\n      <td>0.130150</td>\n      <td>-0.323220</td>\n      <td>0.839781</td>\n      <td>-0.990412</td>\n      <td>0.505244</td>\n      <td>-0.811275</td>\n      <td>0.134853</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>-0.539911</td>\n      <td>0.262381</td>\n      <td>0.145610</td>\n      <td>-0.962335</td>\n      <td>-0.060307</td>\n      <td>0.812004</td>\n      <td>0.256701</td>\n      <td>0.289468</td>\n      <td>-0.314500</td>\n      <td>-0.173251</td>\n      <td>...</td>\n      <td>-0.416052</td>\n      <td>-0.145934</td>\n      <td>0.514849</td>\n      <td>0.633466</td>\n      <td>-0.178631</td>\n      <td>1.113246</td>\n      <td>-0.278801</td>\n      <td>0.262267</td>\n      <td>0.436200</td>\n      <td>-0.159290</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>0.257532</td>\n      <td>0.128558</td>\n      <td>0.558949</td>\n      <td>-0.677057</td>\n      <td>0.045429</td>\n      <td>0.276470</td>\n      <td>0.239281</td>\n      <td>-0.226379</td>\n      <td>0.614334</td>\n      <td>-0.522354</td>\n      <td>...</td>\n      <td>-0.466851</td>\n      <td>0.808567</td>\n      <td>-0.663552</td>\n      <td>0.648689</td>\n      <td>-1.155005</td>\n      <td>0.039744</td>\n      <td>-0.427043</td>\n      <td>-0.117230</td>\n      <td>-1.050166</td>\n      <td>-0.729577</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>0.350021</td>\n      <td>-0.158842</td>\n      <td>-0.374558</td>\n      <td>-0.575825</td>\n      <td>0.334527</td>\n      <td>-0.461471</td>\n      <td>1.042502</td>\n      <td>-0.789735</td>\n      <td>0.057360</td>\n      <td>0.060458</td>\n      <td>...</td>\n      <td>0.112147</td>\n      <td>0.032235</td>\n      <td>-0.489684</td>\n      <td>0.233091</td>\n      <td>-0.997444</td>\n      <td>-0.356804</td>\n      <td>-0.442014</td>\n      <td>-0.286688</td>\n      <td>0.559590</td>\n      <td>0.437227</td>\n    </tr>\n  </tbody>\n</table>\n<p>16 rows × 100070 columns</p>\n</div>"
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Apply the final linear layer to get the logits\n",
    "logits = nn.Linear(d_model, max_token_value+1)(output)\n",
    "pd.DataFrame(logits[0].detach().cpu().numpy())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:35:26.555049Z",
     "start_time": "2024-02-09T04:35:26.440498Z"
    }
   },
   "id": "1d4b434e547757",
   "execution_count": 40
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "      0         1         2         3         4         5         6         7         8         9       ...    100060    100061    100062    100063    100064    100065    100066    100067    100068    100069\n0   0.000002  0.000007  0.000006  0.000006  0.000014  0.000006  0.000013  0.000016  0.000004  0.000008  ...  0.000004  0.000005  0.000010  0.000018  0.000007  0.000014  0.000016  0.000011  0.000021  0.000003\n1   0.000008  0.000011  0.000007  0.000007  0.000018  0.000004  0.000013  0.000007  0.000005  0.000009  ...  0.000012  0.000009  0.000016  0.000014  0.000008  0.000006  0.000006  0.000006  0.000014  0.000007\n2   0.000009  0.000011  0.000005  0.000003  0.000014  0.000004  0.000018  0.000005  0.000008  0.000006  ...  0.000016  0.000003  0.000027  0.000020  0.000006  0.000011  0.000006  0.000005  0.000014  0.000005\n3   0.000007  0.000007  0.000007  0.000006  0.000014  0.000004  0.000007  0.000010  0.000003  0.000002  ...  0.000010  0.000008  0.000007  0.000004  0.000004  0.000004  0.000012  0.000013  0.000003  0.000004\n4   0.000008  0.000017  0.000015  0.000009  0.000003  0.000007  0.000020  0.000005  0.000006  0.000010  ...  0.000012  0.000008  0.000016  0.000041  0.000008  0.000034  0.000009  0.000008  0.000006  0.000002\n5   0.000008  0.000022  0.000005  0.000010  0.000005  0.000005  0.000015  0.000005  0.000008  0.000010  ...  0.000004  0.000008  0.000012  0.000005  0.000003  0.000003  0.000006  0.000007  0.000006  0.000004\n6   0.000020  0.000014  0.000006  0.000012  0.000011  0.000013  0.000010  0.000005  0.000009  0.000005  ...  0.000011  0.000020  0.000005  0.000020  0.000004  0.000006  0.000006  0.000016  0.000003  0.000004\n7   0.000018  0.000011  0.000003  0.000013  0.000004  0.000012  0.000006  0.000006  0.000004  0.000019  ...  0.000008  0.000022  0.000004  0.000008  0.000003  0.000023  0.000013  0.000005  0.000009  0.000002\n8   0.000011  0.000007  0.000004  0.000005  0.000012  0.000032  0.000007  0.000009  0.000010  0.000002  ...  0.000005  0.000010  0.000010  0.000009  0.000006  0.000005  0.000003  0.000010  0.000011  0.000008\n9   0.000004  0.000011  0.000005  0.000005  0.000011  0.000009  0.000014  0.000013  0.000007  0.000005  ...  0.000003  0.000005  0.000011  0.000024  0.000007  0.000015  0.000009  0.000011  0.000016  0.000004\n10  0.000006  0.000017  0.000006  0.000006  0.000009  0.000008  0.000016  0.000012  0.000003  0.000006  ...  0.000005  0.000007  0.000011  0.000015  0.000013  0.000023  0.000003  0.000010  0.000017  0.000005\n11  0.000004  0.000015  0.000010  0.000003  0.000006  0.000008  0.000009  0.000004  0.000014  0.000005  ...  0.000007  0.000002  0.000017  0.000009  0.000009  0.000007  0.000004  0.000016  0.000003  0.000013\n12  0.000013  0.000033  0.000023  0.000006  0.000006  0.000014  0.000015  0.000008  0.000009  0.000011  ...  0.000012  0.000020  0.000007  0.000010  0.000006  0.000020  0.000003  0.000014  0.000004  0.000010\n13  0.000005  0.000011  0.000010  0.000003  0.000008  0.000019  0.000011  0.000011  0.000006  0.000007  ...  0.000006  0.000007  0.000014  0.000016  0.000007  0.000026  0.000006  0.000011  0.000013  0.000007\n14  0.000011  0.000010  0.000015  0.000004  0.000009  0.000011  0.000011  0.000007  0.000016  0.000005  ...  0.000005  0.000019  0.000004  0.000016  0.000003  0.000009  0.000006  0.000008  0.000003  0.000004\n15  0.000012  0.000007  0.000006  0.000005  0.000012  0.000005  0.000024  0.000004  0.000009  0.000009  ...  0.000009  0.000009  0.000005  0.000011  0.000003  0.000006  0.000005  0.000006  0.000015  0.000013\n\n[16 rows x 100070 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>100060</th>\n      <th>100061</th>\n      <th>100062</th>\n      <th>100063</th>\n      <th>100064</th>\n      <th>100065</th>\n      <th>100066</th>\n      <th>100067</th>\n      <th>100068</th>\n      <th>100069</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.000002</td>\n      <td>0.000007</td>\n      <td>0.000006</td>\n      <td>0.000006</td>\n      <td>0.000014</td>\n      <td>0.000006</td>\n      <td>0.000013</td>\n      <td>0.000016</td>\n      <td>0.000004</td>\n      <td>0.000008</td>\n      <td>...</td>\n      <td>0.000004</td>\n      <td>0.000005</td>\n      <td>0.000010</td>\n      <td>0.000018</td>\n      <td>0.000007</td>\n      <td>0.000014</td>\n      <td>0.000016</td>\n      <td>0.000011</td>\n      <td>0.000021</td>\n      <td>0.000003</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.000008</td>\n      <td>0.000011</td>\n      <td>0.000007</td>\n      <td>0.000007</td>\n      <td>0.000018</td>\n      <td>0.000004</td>\n      <td>0.000013</td>\n      <td>0.000007</td>\n      <td>0.000005</td>\n      <td>0.000009</td>\n      <td>...</td>\n      <td>0.000012</td>\n      <td>0.000009</td>\n      <td>0.000016</td>\n      <td>0.000014</td>\n      <td>0.000008</td>\n      <td>0.000006</td>\n      <td>0.000006</td>\n      <td>0.000006</td>\n      <td>0.000014</td>\n      <td>0.000007</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.000009</td>\n      <td>0.000011</td>\n      <td>0.000005</td>\n      <td>0.000003</td>\n      <td>0.000014</td>\n      <td>0.000004</td>\n      <td>0.000018</td>\n      <td>0.000005</td>\n      <td>0.000008</td>\n      <td>0.000006</td>\n      <td>...</td>\n      <td>0.000016</td>\n      <td>0.000003</td>\n      <td>0.000027</td>\n      <td>0.000020</td>\n      <td>0.000006</td>\n      <td>0.000011</td>\n      <td>0.000006</td>\n      <td>0.000005</td>\n      <td>0.000014</td>\n      <td>0.000005</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.000007</td>\n      <td>0.000007</td>\n      <td>0.000007</td>\n      <td>0.000006</td>\n      <td>0.000014</td>\n      <td>0.000004</td>\n      <td>0.000007</td>\n      <td>0.000010</td>\n      <td>0.000003</td>\n      <td>0.000002</td>\n      <td>...</td>\n      <td>0.000010</td>\n      <td>0.000008</td>\n      <td>0.000007</td>\n      <td>0.000004</td>\n      <td>0.000004</td>\n      <td>0.000004</td>\n      <td>0.000012</td>\n      <td>0.000013</td>\n      <td>0.000003</td>\n      <td>0.000004</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.000008</td>\n      <td>0.000017</td>\n      <td>0.000015</td>\n      <td>0.000009</td>\n      <td>0.000003</td>\n      <td>0.000007</td>\n      <td>0.000020</td>\n      <td>0.000005</td>\n      <td>0.000006</td>\n      <td>0.000010</td>\n      <td>...</td>\n      <td>0.000012</td>\n      <td>0.000008</td>\n      <td>0.000016</td>\n      <td>0.000041</td>\n      <td>0.000008</td>\n      <td>0.000034</td>\n      <td>0.000009</td>\n      <td>0.000008</td>\n      <td>0.000006</td>\n      <td>0.000002</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.000008</td>\n      <td>0.000022</td>\n      <td>0.000005</td>\n      <td>0.000010</td>\n      <td>0.000005</td>\n      <td>0.000005</td>\n      <td>0.000015</td>\n      <td>0.000005</td>\n      <td>0.000008</td>\n      <td>0.000010</td>\n      <td>...</td>\n      <td>0.000004</td>\n      <td>0.000008</td>\n      <td>0.000012</td>\n      <td>0.000005</td>\n      <td>0.000003</td>\n      <td>0.000003</td>\n      <td>0.000006</td>\n      <td>0.000007</td>\n      <td>0.000006</td>\n      <td>0.000004</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.000020</td>\n      <td>0.000014</td>\n      <td>0.000006</td>\n      <td>0.000012</td>\n      <td>0.000011</td>\n      <td>0.000013</td>\n      <td>0.000010</td>\n      <td>0.000005</td>\n      <td>0.000009</td>\n      <td>0.000005</td>\n      <td>...</td>\n      <td>0.000011</td>\n      <td>0.000020</td>\n      <td>0.000005</td>\n      <td>0.000020</td>\n      <td>0.000004</td>\n      <td>0.000006</td>\n      <td>0.000006</td>\n      <td>0.000016</td>\n      <td>0.000003</td>\n      <td>0.000004</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.000018</td>\n      <td>0.000011</td>\n      <td>0.000003</td>\n      <td>0.000013</td>\n      <td>0.000004</td>\n      <td>0.000012</td>\n      <td>0.000006</td>\n      <td>0.000006</td>\n      <td>0.000004</td>\n      <td>0.000019</td>\n      <td>...</td>\n      <td>0.000008</td>\n      <td>0.000022</td>\n      <td>0.000004</td>\n      <td>0.000008</td>\n      <td>0.000003</td>\n      <td>0.000023</td>\n      <td>0.000013</td>\n      <td>0.000005</td>\n      <td>0.000009</td>\n      <td>0.000002</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.000011</td>\n      <td>0.000007</td>\n      <td>0.000004</td>\n      <td>0.000005</td>\n      <td>0.000012</td>\n      <td>0.000032</td>\n      <td>0.000007</td>\n      <td>0.000009</td>\n      <td>0.000010</td>\n      <td>0.000002</td>\n      <td>...</td>\n      <td>0.000005</td>\n      <td>0.000010</td>\n      <td>0.000010</td>\n      <td>0.000009</td>\n      <td>0.000006</td>\n      <td>0.000005</td>\n      <td>0.000003</td>\n      <td>0.000010</td>\n      <td>0.000011</td>\n      <td>0.000008</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>0.000004</td>\n      <td>0.000011</td>\n      <td>0.000005</td>\n      <td>0.000005</td>\n      <td>0.000011</td>\n      <td>0.000009</td>\n      <td>0.000014</td>\n      <td>0.000013</td>\n      <td>0.000007</td>\n      <td>0.000005</td>\n      <td>...</td>\n      <td>0.000003</td>\n      <td>0.000005</td>\n      <td>0.000011</td>\n      <td>0.000024</td>\n      <td>0.000007</td>\n      <td>0.000015</td>\n      <td>0.000009</td>\n      <td>0.000011</td>\n      <td>0.000016</td>\n      <td>0.000004</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>0.000006</td>\n      <td>0.000017</td>\n      <td>0.000006</td>\n      <td>0.000006</td>\n      <td>0.000009</td>\n      <td>0.000008</td>\n      <td>0.000016</td>\n      <td>0.000012</td>\n      <td>0.000003</td>\n      <td>0.000006</td>\n      <td>...</td>\n      <td>0.000005</td>\n      <td>0.000007</td>\n      <td>0.000011</td>\n      <td>0.000015</td>\n      <td>0.000013</td>\n      <td>0.000023</td>\n      <td>0.000003</td>\n      <td>0.000010</td>\n      <td>0.000017</td>\n      <td>0.000005</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>0.000004</td>\n      <td>0.000015</td>\n      <td>0.000010</td>\n      <td>0.000003</td>\n      <td>0.000006</td>\n      <td>0.000008</td>\n      <td>0.000009</td>\n      <td>0.000004</td>\n      <td>0.000014</td>\n      <td>0.000005</td>\n      <td>...</td>\n      <td>0.000007</td>\n      <td>0.000002</td>\n      <td>0.000017</td>\n      <td>0.000009</td>\n      <td>0.000009</td>\n      <td>0.000007</td>\n      <td>0.000004</td>\n      <td>0.000016</td>\n      <td>0.000003</td>\n      <td>0.000013</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>0.000013</td>\n      <td>0.000033</td>\n      <td>0.000023</td>\n      <td>0.000006</td>\n      <td>0.000006</td>\n      <td>0.000014</td>\n      <td>0.000015</td>\n      <td>0.000008</td>\n      <td>0.000009</td>\n      <td>0.000011</td>\n      <td>...</td>\n      <td>0.000012</td>\n      <td>0.000020</td>\n      <td>0.000007</td>\n      <td>0.000010</td>\n      <td>0.000006</td>\n      <td>0.000020</td>\n      <td>0.000003</td>\n      <td>0.000014</td>\n      <td>0.000004</td>\n      <td>0.000010</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>0.000005</td>\n      <td>0.000011</td>\n      <td>0.000010</td>\n      <td>0.000003</td>\n      <td>0.000008</td>\n      <td>0.000019</td>\n      <td>0.000011</td>\n      <td>0.000011</td>\n      <td>0.000006</td>\n      <td>0.000007</td>\n      <td>...</td>\n      <td>0.000006</td>\n      <td>0.000007</td>\n      <td>0.000014</td>\n      <td>0.000016</td>\n      <td>0.000007</td>\n      <td>0.000026</td>\n      <td>0.000006</td>\n      <td>0.000011</td>\n      <td>0.000013</td>\n      <td>0.000007</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>0.000011</td>\n      <td>0.000010</td>\n      <td>0.000015</td>\n      <td>0.000004</td>\n      <td>0.000009</td>\n      <td>0.000011</td>\n      <td>0.000011</td>\n      <td>0.000007</td>\n      <td>0.000016</td>\n      <td>0.000005</td>\n      <td>...</td>\n      <td>0.000005</td>\n      <td>0.000019</td>\n      <td>0.000004</td>\n      <td>0.000016</td>\n      <td>0.000003</td>\n      <td>0.000009</td>\n      <td>0.000006</td>\n      <td>0.000008</td>\n      <td>0.000003</td>\n      <td>0.000004</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>0.000012</td>\n      <td>0.000007</td>\n      <td>0.000006</td>\n      <td>0.000005</td>\n      <td>0.000012</td>\n      <td>0.000005</td>\n      <td>0.000024</td>\n      <td>0.000004</td>\n      <td>0.000009</td>\n      <td>0.000009</td>\n      <td>...</td>\n      <td>0.000009</td>\n      <td>0.000009</td>\n      <td>0.000005</td>\n      <td>0.000011</td>\n      <td>0.000003</td>\n      <td>0.000006</td>\n      <td>0.000005</td>\n      <td>0.000006</td>\n      <td>0.000015</td>\n      <td>0.000013</td>\n    </tr>\n  </tbody>\n</table>\n<p>16 rows × 100070 columns</p>\n</div>"
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the probabilities \n",
    "# torch.softmax usually used during inference, during training we use torch.nn.CrossEntropyLoss\n",
    "# but for illustration purpose, we'll use torch.softmax here\n",
    "probabilities = torch.softmax(logits, dim=-1)\n",
    "pd.DataFrame(probabilities[0].detach().cpu().numpy())\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:35:46.352818Z",
     "start_time": "2024-02-09T04:35:46.256747Z"
    }
   },
   "id": "3a857abd1cceecb4",
   "execution_count": 41
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "' Catholics'"
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's see the predicted token and it's original English word\n",
    "predicted_index = torch.argmax(logits[0,0]).item()\n",
    "encoding.decode([predicted_index])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:36:08.064535Z",
     "start_time": "2024-02-09T04:36:08.045119Z"
    }
   },
   "id": "5de9e0455613f790",
   "execution_count": 42
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "' the sales process. The ability to effectively communicate the value and benefits of our products'"
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's see the original input sentence\n",
    "encoding.decode(x_batch[0].tolist())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-09T04:39:04.969636Z",
     "start_time": "2024-02-09T04:39:04.925038Z"
    }
   },
   "id": "329ff8f3529b7309",
   "execution_count": 45
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Looks like the predicted token \"Catholics\" is not the correct prediction token to the original sentence, because we only did one training loop and barely trained nothing.\n",
    "# But this is the basic idea of how the Transformer works.\n",
    "# We'll continue to train the model in the next notebook and wrap the above code into a class.\n",
    "# https://waylandzhang.github.io/en/let-s-code-llm.html"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "111989a98903f4dd"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}