16_nlp_exercises.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.data import Dataset\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Input, Dense, SimpleRNN, GRU, LSTM, LSTMCell, Embedding\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.metrics import Precision, Recall\n",
    "from tensorflow.keras.losses import SparseCategoricalCrossentropy\n",
    "from tensorflow.keras.optimizers import Adam, Nadam\n",
    "from tensorflow.keras.callbacks import TensorBoard\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "import jupyterthemes as jt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from datetime import datetime\n",
    "\n",
    "jt.jtplot.style()\n",
    "\n",
    "\"\"\" Avoid error with Blas:GEMM not initializing when using GPU:\n",
    "See: https://stackoverflow.com/questions/43990046/tensorflow-blas-gemm-launch-failed\n",
    "\"\"\"\n",
    "physical_devices = tf.config.list_physical_devices('GPU') \n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "\n",
    "tf.random.set_seed(98)\n",
    "np.random.seed(99)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Embedded Reber grammars were used by Hochreiter and Schmidhuber in their paper about LSTMs. They are artificial grammars that produce strings such as “BPBTSXXVPSEPE.” Check out [Jenny Orr’s nice introduction to this topic](https://www.willamette.edu/~gorr/classes/cs449/reber.html). Choose a particular embedded Reber grammar (such as the one represented on Jenny Orr’s page), then train an RNN to identify whether a string respects that grammar or not. You will first need to write a function capable of generating a training batch containing about 50% strings that respect the grammar, and 50% that don’t."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating Reber grammars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We start creating a Reber grammar generator. I'll follow the same structure provided in the link above will allow for tokens to be passed to the class to generate the strings.\n",
    "\n",
    "Each state will be labelled as follows and has two possible transitions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Reber Grammar](images\\reber.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "reber_transitions = {\n",
    "    0: [(1, 'B')],\n",
    "    1: [(2, 'T'), (3, 'P')],\n",
    "    2: [(2, 'S'), (4, 'X')],\n",
    "    3: [(3, 'T'), (5, 'V')],\n",
    "    4: [(6, 'S'), (3, 'X')],\n",
    "    5: [(4, 'P'), (6, 'V')],\n",
    "    6: [(7, 'E')]}\n",
    "\n",
    "def move_state(cur_state, transitions=reber_transitions):\n",
    "    \"\"\"Finds possible next moves in transition table and picks one at a random\"\"\"\n",
    "    paths = transitions[cur_state]\n",
    "    return paths[np.random.choice(len(transitions[cur_state]), size=1)[0]]\n",
    "    \n",
    "def generate_string(string=''):\n",
    "    \"\"\"Iterates over possible paths until final state is reached\"\"\"\n",
    "    state = 0\n",
    "    while state != 7:\n",
    "        state, char = move_state(state)\n",
    "        string += char\n",
    "    return string\n",
    "\n",
    "def find_next_state(cur_state, char, transitions=reber_transitions):\n",
    "    \"\"\"Given a current state and a character in the next state, searches transitions for a state\n",
    "    with the corresponding next_char, if it exists.\"\"\"\n",
    "    for (next_state, next_char) in transitions[cur_state]:\n",
    "        if next_char == char:\n",
    "            return next_state, next_char\n",
    "    return -1, -1\n",
    "\n",
    "def validate_string(string, transitions=reber_transitions, verbose=False):\n",
    "    \"\"\"Iterates through a given string and checks whether the string was generated by some grammar with\n",
    "    given transitions. \n",
    "    \n",
    "    Probably can be improved\n",
    "    \"\"\"\n",
    "    next_state, next_char = move_state(0, transitions)\n",
    "    for idx, char in enumerate(string):\n",
    "        if verbose: print(f\"Next State: {next_state}; Testing {idx} : {char} vs {next_char}\")\n",
    "        if char != next_char:\n",
    "            return 0\n",
    "        try:\n",
    "            if verbose: print(f\"\\tGoing to find next state by accessing {string[idx+1]}\")\n",
    "            next_state, next_char = find_next_state(next_state, string[idx+1])\n",
    "            if verbose: print(f\"\\tReturned ({next_state}, {next_char})\")\n",
    "            if next_state == -1:\n",
    "                return 0\n",
    "        except IndexError:\n",
    "            pass # Trying to access out of bounds value, meaning we reached the end of the checks\n",
    "        if next_state == 7:\n",
    "            break\n",
    "    return 1\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's generate positive classes for our dataset and check they are valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_size = 10_000\n",
    "reber_strings = [generate_string() for _ in range(dataset_size)]\n",
    "reber_labels = [1 for _ in range(dataset_size)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all([validate_string(reber_string) for reber_string in reber_strings]), \"Some generated string is NOT REBER!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That some sample non-reber strings are invalid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_reber_tests = ['BTSSPXSE', 'BTXXVVSE', 'BPVSPSE', 'BTSSSE', 'BPTVVB']\n",
    "assert not any([validate_string(not_reber) for not_reber in not_reber_tests]), \"One of the test strings was identified as reber\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And let's generate a bunch of random strings and use the function above to mark them as not reber strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = 'BEPSTVX'\n",
    "min_length = len(min(reber_strings, key=len))\n",
    "max_length = len(max(reber_strings, key=len))\n",
    "\n",
    "# Generate N random strings with the vocab. Each time strings will have different lengths \n",
    "# that are bounded by the min/max size of the reber_lengths\n",
    "randomly_generated = [''.join(np.random.choice(list(vocab), size=np.random.randint(min_length, max_length)))\n",
    "                      for _ in range(dataset_size)]\n",
    "randomly_gen_labels = [validate_string(random_str) for random_str in randomly_generated]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000,) (20000,)\n"
     ]
    }
   ],
   "source": [
    "X = np.concatenate((reber_strings, randomly_generated))\n",
    "y = np.concatenate((reber_labels, randomly_gen_labels))\n",
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([ 9998, 10002], dtype=int64))"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data looks balanced enough! Now let's convert it to a Tensorflow Dataset and do preprocessing\n",
    "\n",
    "First we start with tokenizing at character level, converting the characters into numbers and creating a dataset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_datasets(X, y, batch_size=32):\n",
    "    tokenizer = Tokenizer(char_level=True, lower=False)\n",
    "    tokenizer.fit_on_texts(X)\n",
    "    \n",
    "    encoded = tokenizer.texts_to_sequences(X)\n",
    "    padded = pad_sequences(encoded, maxlen=max_length, padding='post', value=0) # pad with zeros\n",
    "    \n",
    "    # Recall RNN inputs have shape [batch_size, time_steps, dimensionality]\n",
    "    # Need to reshape the data to an appropriate format\n",
    "    X_full, y_full = padded[..., np.newaxis], y.reshape(-1,1)\n",
    "    X_train_full, X_test, y_train_full, y_test = train_test_split(X_full, y_full, test_size=0.05)\n",
    "    X_train, X_valid, y_train, y_valid = train_test_split(X_train_full, y_train_full, test_size=0.05)\n",
    "        \n",
    "    train_set = Dataset.from_tensor_slices((X_train, y_train)).shuffle(dataset_size).batch(batch_size).prefetch(1)\n",
    "    valid_set = Dataset.from_tensor_slices((X_valid, y_valid)).shuffle(dataset_size).batch(batch_size)\n",
    "    test_set = Dataset.from_tensor_slices((X_test, y_test)).shuffle(dataset_size).batch(batch_size)\n",
    "    return train_set, valid_set, test_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set, valid_set, test_set = create_datasets(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define and train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential([\n",
    "    Input(shape=[None, 1], name='Input'),\n",
    "    LSTM(32, return_sequences=True),\n",
    "    LSTM(32),\n",
    "    Dense(1, activation=\"sigmoid\")])\n",
    "\n",
    "model.compile(loss=\"binary_crossentropy\", optimizer=Adam(), metrics=['accuracy', Precision(), Recall()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "565/565 [==============================] - 10s 12ms/step - loss: 0.4544 - accuracy: 0.7941 - precision_2: 0.7607 - recall_2: 0.8523 - val_loss: 0.3566 - val_accuracy: 0.8505 - val_precision_2: 0.8307 - val_recall_2: 0.8828\n",
      "Epoch 2/15\n",
      "565/565 [==============================] - 6s 10ms/step - loss: 0.3806 - accuracy: 0.8277 - precision_2: 0.7829 - recall_2: 0.9020 - val_loss: 0.2004 - val_accuracy: 0.9326 - val_precision_2: 0.8966 - val_recall_2: 0.9791\n",
      "Epoch 3/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.2347 - accuracy: 0.9103 - precision_2: 0.8704 - recall_2: 0.9624 - val_loss: 0.1610 - val_accuracy: 0.9516 - val_precision_2: 0.9303 - val_recall_2: 0.9770\n",
      "Epoch 4/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.1466 - accuracy: 0.9560 - precision_2: 0.9276 - recall_2: 0.9901 - val_loss: 0.1188 - val_accuracy: 0.9674 - val_precision_2: 0.9443 - val_recall_2: 0.9937\n",
      "Epoch 5/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.1758 - accuracy: 0.9427 - precision_2: 0.9061 - recall_2: 0.9876 - val_loss: 0.1572 - val_accuracy: 0.9568 - val_precision_2: 0.9226 - val_recall_2: 0.9979\n",
      "Epoch 6/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.1077 - accuracy: 0.9709 - precision_2: 0.9489 - recall_2: 0.9955 - val_loss: 0.1174 - val_accuracy: 0.9663 - val_precision_2: 0.9407 - val_recall_2: 0.9958\n",
      "Epoch 7/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0875 - accuracy: 0.9772 - precision_2: 0.9597 - recall_2: 0.9966 - val_loss: 0.0635 - val_accuracy: 0.9863 - val_precision_2: 0.9755 - val_recall_2: 0.9979\n",
      "Epoch 8/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0698 - accuracy: 0.9836 - precision_2: 0.9709 - recall_2: 0.9971 - val_loss: 0.0548 - val_accuracy: 0.9874 - val_precision_2: 0.9775 - val_recall_2: 0.9979\n",
      "Epoch 9/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0817 - accuracy: 0.9788 - precision_2: 0.9628 - recall_2: 0.9953 - val_loss: 0.0768 - val_accuracy: 0.9832 - val_precision_2: 0.9695 - val_recall_2: 0.9979\n",
      "Epoch 10/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0487 - accuracy: 0.9880 - precision_2: 0.9790 - recall_2: 0.9975 - val_loss: 0.0518 - val_accuracy: 0.9895 - val_precision_2: 0.9815 - val_recall_2: 0.9979\n",
      "Epoch 11/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0439 - accuracy: 0.9896 - precision_2: 0.9819 - recall_2: 0.9977 - val_loss: 0.0778 - val_accuracy: 0.9800 - val_precision_2: 0.9636 - val_recall_2: 0.9979\n",
      "Epoch 12/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0415 - accuracy: 0.9894 - precision_2: 0.9823 - recall_2: 0.9965 - val_loss: 0.0229 - val_accuracy: 0.9968 - val_precision_2: 0.9958 - val_recall_2: 0.9979\n",
      "Epoch 13/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0328 - accuracy: 0.9922 - precision_2: 0.9867 - recall_2: 0.9976 - val_loss: 0.0310 - val_accuracy: 0.9937 - val_precision_2: 0.9896 - val_recall_2: 0.9979\n",
      "Epoch 14/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0288 - accuracy: 0.9922 - precision_2: 0.9888 - recall_2: 0.9955 - val_loss: 0.0313 - val_accuracy: 0.9937 - val_precision_2: 0.9896 - val_recall_2: 0.9979\n",
      "Epoch 15/15\n",
      "565/565 [==============================] - 6s 11ms/step - loss: 0.0164 - accuracy: 0.9966 - precision_2: 0.9942 - recall_2: 0.9990 - val_loss: 0.0146 - val_accuracy: 0.9989 - val_precision_2: 1.0000 - val_recall_2: 0.9979\n"
     ]
    }
   ],
   "source": [
    "with tf.device('GPU:0'):\n",
    "    model.fit(train_set, epochs=15, validation_data=valid_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32/32 [==============================] - 0s 6ms/step - loss: 0.0092 - accuracy: 0.9990 - precision_2: 0.9980 - recall_2: 1.0000\n"
     ]
    }
   ],
   "source": [
    "loss, acc, pre, rec = model.evaluate(test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train an Encoder–Decoder model that can convert a date string from one format to another (e.g., from “April 22, 2019” to “2019-04-22”).\n",
    "\n",
    "Let's first generate the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>X</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34536</th>\n",
       "      <td>1994-07-23</td>\n",
       "      <td>July 23, 1994</td>\n",
       "      <td>1994-07-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42697</th>\n",
       "      <td>2016-11-25</td>\n",
       "      <td>November 25, 2016</td>\n",
       "      <td>2016-11-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36008</th>\n",
       "      <td>1998-08-03</td>\n",
       "      <td>August 03, 1998</td>\n",
       "      <td>1998-08-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23587</th>\n",
       "      <td>1964-07-31</td>\n",
       "      <td>July 31, 1964</td>\n",
       "      <td>1964-07-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29313</th>\n",
       "      <td>1980-04-04</td>\n",
       "      <td>April 04, 1980</td>\n",
       "      <td>1980-04-04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          source                  X           y\n",
       "34536 1994-07-23      July 23, 1994  1994-07-23\n",
       "42697 2016-11-25  November 25, 2016  2016-11-25\n",
       "36008 1998-08-03    August 03, 1998  1998-08-03\n",
       "23587 1964-07-31      July 31, 1964  1964-07-31\n",
       "29313 1980-04-04     April 04, 1980  1980-04-04"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"source\":np.arange('1900-01-01', '2021-01-01', dtype='datetime64[D]')})\n",
    "df[\"X\"] = df[\"source\"].dt.strftime(\"%B %d, %Y\")\n",
    "df[\"y\"] = df[\"source\"].dt.strftime(\"%Y-%m-%d\")\n",
    "df = df.sample(frac=1.)  # Resample the dataframe to shuffle the dates\n",
    "\n",
    "X = df[\"X\"].to_numpy()\n",
    "y = df[\"y\"].to_numpy()\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the data already cleansed, we can tokenize both X and y with the same object. The `<sos>` and `<eos>` tokens will have ids 0 and 1 respectively.\n",
    "\n",
    "Since we'll be outputting probabilities and our targets are words, we have no way of calculating metrics during training (maybe?). Thus I'll use a very small test size\n",
    "\n",
    "*We also have to shift the decoder inputs by 1 so the words we give as inputs are the words that it **should** have output at the previous step* - Not so sure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_vocab = sorted(list(set(word for entry in X for word in entry)))\n",
    "output_vocab = sorted(list(set(word for entry in y for word in entry)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_sent(sent, vocab):\n",
    "    \"\"\"Converts characters to ints by using char's index \"\"\"\n",
    "    return [vocab.index(char) for char in sent]\n",
    "\n",
    "def encode_data(data, vocab):\n",
    "    \"\"\"Tokenize list of sentence and converts them to a dense tensor padded\n",
    "    with zeroes.\"\"\"\n",
    "    ids = [tokenize_sent(sent, vocab) for sent in data]\n",
    "    data = tf.ragged.constant(ids, ragged_rank=1)\n",
    "    return (data + 1).to_tensor() # Padding token will have id 0\n",
    "\n",
    "def decode_sequence(seq, vocab):\n",
    "    return ''.join(vocab[char_id - 1] for char_id in seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_encoded = encode_data(X, input_vocab)\n",
    "y_encoded = encode_data(y, output_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X :  ['July 31, 1939']\n",
      "X encoded:  tf.Tensor([[16 36 28 38  1  6  4  2  1  4 12  6 12  0  0  0  0  0]], shape=(1, 18), dtype=int32)\n",
      "X decoded:  July 31, 1939yyyyy\n"
     ]
    }
   ],
   "source": [
    "print(\"X : \", X[:1])\n",
    "print(\"X encoded: \", X_encoded[:1])\n",
    "print(\"X decoded: \", decode_sequence(X_encoded[:1].numpy()[0], input_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train, validation, test split\n",
    "train_size = len(X_encoded) * 80 // 100\n",
    "test_size = (len(X_encoded) - train_size) // 2\n",
    "\n",
    "X_train, y_train = X_encoded[:train_size], y_encoded[:train_size]\n",
    "X_valid, y_valid = X_encoded[train_size:train_size + test_size], y_encoded[train_size:train_size + test_size]\n",
    "X_test, y_test = X_encoded[train_size + test_size:], y_encoded[train_size + test_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "EMBED_SIZE = 8\n",
    "MAX_OUTPUT_LEN = y_train.shape[1]\n",
    "sos_id = len(output_vocab) + 1 # adding a start of sequence token for the shifted decoder inputs\n",
    "\n",
    "def get_sequence_lengths(y):\n",
    "    return np.full([y.shape[0]], y_train.shape[1])\n",
    "\n",
    "def shift_sequence(y, fill=sos_id):\n",
    "    \"\"\"Shifts target out by 1 to generate decoder inputs with a start of sequence token\"\"\"\n",
    "    return np.c_[np.full((len(y), 1), fill), y[:, :-1]]\n",
    "\n",
    "def create_dataset(X, y, batch_size=BATCH_SIZE):\n",
    "    input_ = Dataset.zip((\n",
    "        Dataset.from_tensor_slices(X), # Encoder inputs\n",
    "        Dataset.from_tensor_slices(shift_sequence(y)))) # Decoder inputs\n",
    "    target = Dataset.from_tensor_slices(y)\n",
    "    return Dataset.zip((input_, target)).shuffle(len(X)).batch(batch_size)\n",
    "    \n",
    "train_set = create_dataset(X_train, y_train).prefetch(1)\n",
    "valid_set = create_dataset(X_valid, y_valid)\n",
    "test_set = create_dataset(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this Encoder-Decoder Network, we'll be using [tensorflow-addons](https://www.tensorflow.org/addons/tutorials/networks_seq2seq_nmt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"date_translator\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "encoder_inputs (InputLayer)     [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "decoder_inputs (InputLayer)     [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "encoder_embedding (Embedding)   (None, None, 8)      312         encoder_inputs[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "decoder_embedding (Embedding)   (None, None, 8)      320         decoder_inputs[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   [(None, 64), (None,  18688       encoder_embedding[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "basic_decoder_2 (BasicDecoder)  (BasicDecoderOutput( 19468       decoder_embedding[0][0]          \n",
      "                                                                 lstm_2[0][1]                     \n",
      "                                                                 lstm_2[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf.nn.softmax_2 (TFOpLambda)    (None, None, 12)     0           basic_decoder_2[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 38,788\n",
      "Trainable params: 38,788\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder_inputs = Input(shape=[None], dtype=np.int32, name='encoder_inputs')\n",
    "decoder_inputs = Input(shape=[None], dtype=np.int32, name='decoder_inputs')\n",
    "sequence_lengths = Input(shape=[], dtype=np.int32, name='sequence_lengths')\n",
    "\n",
    "# Add 1 to the Embed dimension due to using 0 padding\n",
    "encoder_embedding_layer = Embedding(len(input_vocab) + 1, EMBED_SIZE, name='encoder_embedding')\n",
    "# And an extra 1 for the SoS token\n",
    "decoder_embedding_layer = Embedding(len(input_vocab) + 2, EMBED_SIZE, name='decoder_embedding')\n",
    "\n",
    "encoder_embeddings = encoder_embedding_layer(encoder_inputs)\n",
    "decoder_embeddings = decoder_embedding_layer(decoder_inputs)\n",
    "\n",
    "encoder = LSTM(64, return_state=True)\n",
    "encoder_outputs, state_h, state_c = encoder(encoder_embeddings)\n",
    "encoder_state = [state_h, state_c]\n",
    "\n",
    "sampler = tfa.seq2seq.sampler.TrainingSampler()\n",
    "\n",
    "decoder_cell = LSTMCell(64)\n",
    "output_layer = Dense(len(output_vocab) + 1) # Zero padding\n",
    "decoder = tfa.seq2seq.basic_decoder.BasicDecoder(decoder_cell, sampler, output_layer=output_layer)\n",
    "final_outputs, final_state, final_sequence_lengths = decoder(decoder_embeddings,\n",
    "                                                             initial_state=encoder_state)\n",
    "Y_proba = tf.nn.softmax(final_outputs.rnn_output)\n",
    "\n",
    "model = Model(inputs=[encoder_inputs, decoder_inputs],\n",
    "              outputs=[Y_proba],\n",
    "              name='date_translator')\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer='nadam', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup for tensorboard\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "fpath = './my_logs/date_translator'\n",
    "Path(fpath).mkdir(parents=True, exist_ok=True)\n",
    "root_logdir = os.path.join(os.curdir, fpath)\n",
    "\n",
    "def get_run_logdir():\n",
    "    import time\n",
    "    run_id = f\"char_model_batch_size_{BATCH_SIZE}_embed_dim_{EMBED_SIZE}\"\n",
    "    run_id += time.strftime(\"_%Y_%m_%d-%H\")\n",
    "    return os.path.join(root_logdir, run_id)\n",
    "\n",
    "callbacks = [TensorBoard(get_run_logdir())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from tensorboard.plugins import projector\n",
    "\n",
    "def get_last_modified_folder(loc='./my_logs/date_translator/'):\n",
    "    files = os.listdir(loc)\n",
    "    stats = [(loc+fname, os.stat(loc+fname).st_mtime) for fname in files]\n",
    "    return max(stats, key=lambda x: x[1])[0]\n",
    "\n",
    "def save_embeddings(embed_layer, name, vocab):\n",
    "    logdir = get_last_modified_folder()\n",
    "    with open(os.path.join(logdir, f'{name}_metadata.tsv'), \"w\") as f:\n",
    "        for word in vocab:\n",
    "            f.write(f\"{word}\\n\")\n",
    "    \n",
    "    weights = tf.Variable(embed_layer.get_weights()[0])\n",
    "    checkpoint = tf.train.Checkpoint(embedding=weights)\n",
    "    checkpoint.save(os.path.join(logdir, f\"{name}.ckpt\"))\n",
    "    \n",
    "    config = projector.ProjectorConfig()\n",
    "    embedding = config.embeddings.add()\n",
    "    embedding.tensor_name = f\"{name}/.ATTRIBUTES/TEST_VALUE\"\n",
    "    embedding.metadata_path = f'{name}_metadata.tsv'\n",
    "    projector.visualize_embeddings(logdir, config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1105/1105 [==============================] - 29s 23ms/step - loss: 1.4171 - accuracy: 0.4975 - val_loss: 0.8316 - val_accuracy: 0.6834\n",
      "Epoch 2/5\n",
      "1105/1105 [==============================] - 18s 16ms/step - loss: 0.7613 - accuracy: 0.7098 - val_loss: 0.5143 - val_accuracy: 0.8024\n",
      "Epoch 3/5\n",
      "1105/1105 [==============================] - 18s 17ms/step - loss: 0.4084 - accuracy: 0.8552 - val_loss: 0.1634 - val_accuracy: 0.9625\n",
      "Epoch 4/5\n",
      "1105/1105 [==============================] - 18s 17ms/step - loss: 0.1260 - accuracy: 0.9730 - val_loss: 0.0427 - val_accuracy: 0.9948\n",
      "Epoch 5/5\n",
      "1105/1105 [==============================] - 18s 17ms/step - loss: 0.0326 - accuracy: 0.9969 - val_loss: 0.0129 - val_accuracy: 0.9999\n"
     ]
    }
   ],
   "source": [
    "with tf.device('CPU:0'):\n",
    "    model.fit(train_set, epochs=5, validation_data=valid_set, callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the embeddings to examine in TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_layer = model.layers[2]\n",
    "save_embeddings(embed_layer, embed_layer.name, input_vocab)\n",
    "\n",
    "embed_layer = model.layers[3]\n",
    "save_embeddings(embed_layer, embed_layer.name, output_vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Earlier I was trying to generate predictions by simply passing the encoder input (X) shifted to the decoder. Naturally, this was not working as intended. The approach below does the prediction step by step by changing the decoder inputs at each iteration and using that to generate the next character."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_predictions(dates, sos_id=len(output_vocab)+1):\n",
    "    \"\"\"Generates predictions, one character at a time. \n",
    "    \n",
    "    We start with passing SoS character to the decoder, generating a prediction,\n",
    "    appending that to decoder input and so on, until MAX_LEN is reached. \n",
    "    \"\"\"\n",
    "    X = encode_data(dates, input_vocab)\n",
    "    y = tf.fill((len(X), 1), sos_id) \n",
    "    for index in range(MAX_OUTPUT_LEN):\n",
    "        pad_size = MAX_OUTPUT_LEN - y.shape[1]\n",
    "        X_decoder = tf.pad(y, [[0, 0], [0, pad_size]])\n",
    "        y_pred_probas = model.predict((X, X_decoder))[:, index:index+1]\n",
    "        y_pred = tf.argmax(y_pred_probas, axis=-1, output_type=tf.int32)\n",
    "        y = tf.concat([y, y_pred], axis=1)\n",
    "    return [decode_sequence(translated, output_vocab) for translated in y[:, 1:].numpy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>translation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>May 20, 2018</td>\n",
       "      <td>2018-05-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>October 28, 2009</td>\n",
       "      <td>2009-10-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>June 16, 1936</td>\n",
       "      <td>1936-06-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>September 22, 1954</td>\n",
       "      <td>1954-09-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>June 21, 1984</td>\n",
       "      <td>1984-06-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>July 06, 1949</td>\n",
       "      <td>1949-07-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>September 08, 1921</td>\n",
       "      <td>1921-09-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>November 23, 1943</td>\n",
       "      <td>1943-11-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>December 08, 1970</td>\n",
       "      <td>1970-12-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>January 21, 1911</td>\n",
       "      <td>1911-01-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>October 04, 1966</td>\n",
       "      <td>1966-10-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>May 19, 1947</td>\n",
       "      <td>1947-05-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>November 04, 1902</td>\n",
       "      <td>1902-11-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>January 01, 1960</td>\n",
       "      <td>1960-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>May 11, 2017</td>\n",
       "      <td>2017-05-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>December 20, 1951</td>\n",
       "      <td>1951-12-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>June 18, 1997</td>\n",
       "      <td>1997-06-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>August 08, 1922</td>\n",
       "      <td>1922-08-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>August 21, 1922</td>\n",
       "      <td>1922-08-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>February 20, 1947</td>\n",
       "      <td>1947-02-20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                actual translation\n",
       "0         May 20, 2018  2018-05-20\n",
       "1     October 28, 2009  2009-10-28\n",
       "2        June 16, 1936  1936-06-16\n",
       "3   September 22, 1954  1954-09-22\n",
       "4        June 21, 1984  1984-06-21\n",
       "5        July 06, 1949  1949-07-06\n",
       "6   September 08, 1921  1921-09-08\n",
       "7    November 23, 1943  1943-11-23\n",
       "8    December 08, 1970  1970-12-08\n",
       "9     January 21, 1911  1911-01-21\n",
       "10    October 04, 1966  1966-10-04\n",
       "11        May 19, 1947  1947-05-19\n",
       "12   November 04, 1902  1902-11-04\n",
       "13    January 01, 1960  1960-01-01\n",
       "14        May 11, 2017  2017-05-11\n",
       "15   December 20, 1951  1951-12-20\n",
       "16       June 18, 1997  1997-06-18\n",
       "17     August 08, 1922  1922-08-08\n",
       "18     August 21, 1922  1922-08-21\n",
       "19   February 20, 1947  1947-02-20"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actual = X[train_size + test_size:]\n",
    "translated = make_predictions(actual)\n",
    "result_df = pd.DataFrame({\"actual\": actual, \"translation\": translated})\n",
    "result_df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results look good! There's an issue however; Because the model was trained on data ranging from years ranging from 1900 to 2021, it does not perform well when given an year out of that range."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['2014-07-11', '2013-17-12']"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_predictions(['July 18, 1444', 'September 12, 1234'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using GreedyEmbeddingSampler\n",
    "The [notebook](https://github.com/ageron/handson-ml2/blob/master/16_nlp_with_rnns_and_attention.ipynb) points out that by using `GreedyEmbeddingSampler` as sampler, we can speed up prediction time. \n",
    "\n",
    "It does this by calculating the decoder outputs argmax, running the resulting token IDs through the decoder embedding layer then feeding these results to the decoder cell at the next time step. This also eliminates the need for us to pass the shifted decoder inputs as well. \n",
    "\n",
    "Below is an implementation of this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "inference_sampler = tfa.seq2seq.sampler.GreedyEmbeddingSampler(embedding_fn=decoder_embedding_layer)\n",
    "inference_decoder = tfa.seq2seq.basic_decoder.BasicDecoder(\n",
    "    decoder_cell, inference_sampler, output_layer=output_layer,\n",
    "    maximum_iterations=MAX_OUTPUT_LEN) # Avoids infinite loops\n",
    "\n",
    "batch_size = tf.shape(encoder_inputs)[:1]\n",
    "start_tokens = tf.fill(dims=batch_size, value=sos_id)\n",
    "final_outputs, final_state, final_sequence_lengths = inference_decoder(\n",
    "    start_tokens,\n",
    "    initial_state=encoder_state,\n",
    "    start_tokens=start_tokens,\n",
    "    end_token=0)\n",
    "\n",
    "inference_model = Model(inputs=[encoder_inputs], \n",
    "                        outputs=[final_outputs.sample_id]) # Use final_outputs.rnn_output if we want logits\n",
    "inference_model.compile(loss=\"sparse_categorical_crossentropy\", optimizer='nadam', metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "def speedy_predictions(dates):\n",
    "    X = encode_data(dates, input_vocab)\n",
    "    y = inference_model.predict(X)\n",
    "    return [decode_sequence(translated, output_vocab) for translated in y]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compare the two"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30.2 s ± 122 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%timeit make_predictions(actual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.4 s ± 34 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%timeit speedy_predictions(actual)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Go through [Neural Machine Translation with Attention tutorial](https://homl.info/nmttuto) - Done, I'd like to try to implement attention for the date problem above:\n",
    "\n",
    "We start by re-defining the Encoder in a class of its own"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.models.Model):\n",
    "    def __init__(self, units, vocab_size, embedding_dim, batch_size):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.units = units\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "        self.embedding = Embedding(self.vocab_size, self.embedding_dim)\n",
    "        self.lstm = LSTM(self.units, return_sequences=True, return_state=True)\n",
    "        \n",
    "    def call(self, X, hidden):\n",
    "        X = self.embedding(X)\n",
    "        encoder_outputs, state_h, state_c = self.lstm(X)\n",
    "        return encoder_outputs, [state_h, state_c]\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_size, self.units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder(64, len(input_vocab)+1, EMBED_SIZE, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder output shape (batch_size, time_steps, units) (32, 18, 64)\n",
      "Encoder state shape (batch_size, units) (32, 64)\n"
     ]
    }
   ],
   "source": [
    "hidden = encoder.initialize_hidden_state()\n",
    "sample_input = next(iter(Dataset.from_tensor_slices(X_train).batch(32)))\n",
    "encoder_out, encoder_state = encoder(sample_input, hidden)\n",
    "\n",
    "print(f\"Encoder output shape (batch_size, time_steps, units) {encoder_out.shape}\")\n",
    "print(f\"Encoder state shape (batch_size, units) {encoder_state[0].shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The output of the attention mechanism is \n",
    "$$ \\overline{\\textbf{h}}_{(t)} = \\sum_{i}\\alpha_{(t,i)}\\textbf{y}_{(i)}$$\n",
    "Where $\\textbf{y}_{(i)}$ are the outputs of the encoder at time step *i* and\n",
    "$$ \\alpha_{(t,i)} = \\frac{\\exp(e_{(t,i)})}{\\sum_{i'}\\exp(e_{(t,i')})}$$\n",
    "Are the weights of the *i-th* encoder output at decoder time step *t*.\n",
    "\n",
    "$e_{(t,i)}$ is the energy score for each encoder output given by\n",
    "$$ e_{(t,i)} = \n",
    "\\begin{cases}\n",
    "    \\textbf{h}_{(t)}^{\\intercal}\\textbf{y}_{(i)} & & \\text{dot (Luong)} \\\\\n",
    "    \\textbf{v}^{\\intercal}\\tanh(\\textbf{W}[\\textbf{h}_{(t)}; \\textbf{y}_{(i)}]) & & \\text{additive (Bahdanau)}\n",
    "\\end{cases}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LuongAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, units, **kwargs):\n",
    "        super(LuongAttention, self).__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.fc = Dense(1) # Initial TimeDistributed Dense layer\n",
    "        \n",
    "    def call(self, encoder_outputs, hidden):\n",
    "        # Hidden layer has shape (batch_size, units)\n",
    "        # Need to expand it to (batch_size, 1, units)\n",
    "        # So it has the same shape as encoder_outputs\n",
    "        hidden = tf.expand_dims(hidden, axis=1)\n",
    "        \n",
    "        # LuongAttention used dot product\n",
    "        score = self.fc(encoder_outputs * hidden)\n",
    "#         score = encoder_outputs * hidden\n",
    "        \n",
    "        # Softmax along axis 1 which is the time_steps axis\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        \n",
    "        context_vector = tf.reduce_sum(attention_weights * encoder_outputs, axis=1)\n",
    "        return context_vector, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention output shape (batch_size, units) (32, 64)\n",
      "Attention weights shape (batch_size, units) (32, 18, 1)\n"
     ]
    }
   ],
   "source": [
    "attention = LuongAttention(64)\n",
    "# decoder_inputs, attention_weights = attention(encoder_out, hidden)\n",
    "decoder_inputs, attention_weights = attention(encoder_out, tf.random.normal(shape=hidden.shape))\n",
    "\n",
    "print(f\"Attention output shape (batch_size, units) {decoder_inputs.shape}\")\n",
    "print(f\"Attention weights shape (batch_size, units) {attention_weights.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, units, vocab_size, embedding_dim, batch_size):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.units = units\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "        self.embedding = Embedding(self.vocab_size, self.embedding_dim)\n",
    "        self.lstm = LSTM(self.units, return_sequences=True, return_state=True)\n",
    "        self.attention = LuongAttention(self.units)\n",
    "        self.fc = Dense(vocab_size)\n",
    "    \n",
    "    def call(self, X, encoder_output):\n",
    "        X = self.embedding(X)\n",
    "        X, state_h, state_c = self.lstm(X)\n",
    "        X = tf.reshape(X, (-1, X.shape[2]))\n",
    "        \n",
    "        context_vector, attention_weights = self.attention(encoder_output, state_h)\n",
    "        X = tf.concat([context_vector, X], axis=-1)  # Concat attention vector\n",
    "        \n",
    "        # Remove axis 1 from LSTM output which corresponds to time_steps\n",
    "        y = self.fc(X)\n",
    "        return y, state_h, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss=0\n",
    "for t in range(1, y.shape[1]):\n",
    "    y_pred_probas, state, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)), encoder_out)\n",
    "    loss += loss_function(y[:, t], y_pred_probas)\n",
    "\n",
    "    # Next input will be what decoder should\n",
    "    # have predicted in the previous time step\n",
    "    decoder_input = tf.expand_dims(y[:, t], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder output shape (batch_size, vocab_size) (32, 11)\n"
     ]
    }
   ],
   "source": [
    "decoder = Decoder(64, len(output_vocab), EMBED_SIZE, BATCH_SIZE)\n",
    "decoder_output, state, attn_weights = decoder(tf.random.uniform((BATCH_SIZE, 1)), encoder_out)\n",
    "\n",
    "print(f\"Decoder output shape (batch_size, vocab_size) {decoder_output.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = Nadam()\n",
    "loss_obj = SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
    "\n",
    "def loss_function(y_true, y_pred):\n",
    "    mask = tf.math.logical_not(tf.math.equal(y_true, 0))\n",
    "    loss_ = loss_obj(y_true, y_pred)\n",
    "    \n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(X, y, encoder_hidden):\n",
    "    loss = 0\n",
    "    \n",
    "    with tf.GradientTape() as tape:\n",
    "        encoder_out, encoder_state = encoder(X, encoder_hidden)\n",
    "        decoder_hidden = encoder_hidden\n",
    "        decoder_input = tf.expand_dims([sos_id] * BATCH_SIZE, 1)  #initial decoder_input\n",
    "        \n",
    "        for t in range(1, y.shape[1]):\n",
    "            y_pred_probas, state, _ = decoder(decoder_input, encoder_out)\n",
    "            loss += loss_function(y[:, t], y_pred_probas)\n",
    "            \n",
    "            # Next input will be what decoder should\n",
    "            # have predicted in the previous time step\n",
    "            decoder_input = tf.expand_dims(y[:, t], 1)\n",
    "    batch_loss = (loss / int(y.shape[1]))\n",
    "    variables = encoder.trainable_variables + decoder.trainable_variables\n",
    "    gradients = tape.gradient(loss, variables)\n",
    "    optimizer.apply_gradients(zip(gradients, variables))\n",
    "    return batch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 2.3082\n",
      "Epoch 1 Batch 100 Loss 1.7554\n",
      "Epoch 1 Batch 200 Loss 1.5025\n",
      "Epoch 1 Batch 300 Loss 1.4424\n",
      "Epoch 1 Batch 400 Loss 1.4625\n",
      "Epoch 1 Batch 500 Loss 1.4119\n",
      "Epoch 1 Batch 600 Loss 1.3153\n",
      "Epoch 1 Batch 700 Loss 1.2632\n",
      "Epoch 1 Batch 800 Loss 1.2608\n",
      "Epoch 1 Batch 900 Loss 1.2119\n",
      "Epoch 1 Batch 1000 Loss 1.1806\n",
      "Epoch 1 Batch 1100 Loss 1.1076\n",
      "Epoch 1 Loss 1.4051\n",
      "Time taken for 1 epoch 50.86403679847717 sec\n",
      "\n",
      "Epoch 2 Batch 0 Loss 1.1256\n",
      "Epoch 2 Batch 100 Loss 0.9781\n",
      "Epoch 2 Batch 200 Loss 0.9926\n",
      "Epoch 2 Batch 300 Loss 0.8683\n",
      "Epoch 2 Batch 400 Loss 0.8034\n",
      "Epoch 2 Batch 500 Loss 0.6425\n",
      "Epoch 2 Batch 600 Loss 0.6130\n",
      "Epoch 2 Batch 700 Loss 0.5591\n",
      "Epoch 2 Batch 800 Loss 0.5089\n",
      "Epoch 2 Batch 900 Loss 0.5333\n",
      "Epoch 2 Batch 1000 Loss 0.5010\n",
      "Epoch 2 Batch 1100 Loss 0.4516\n",
      "Epoch 2 Loss 0.7086\n",
      "Time taken for 1 epoch 38.62775254249573 sec\n",
      "\n",
      "Epoch 3 Batch 0 Loss 0.3969\n",
      "Epoch 3 Batch 100 Loss 0.4221\n",
      "Epoch 3 Batch 200 Loss 0.3767\n",
      "Epoch 3 Batch 300 Loss 0.3874\n",
      "Epoch 3 Batch 400 Loss 0.3233\n",
      "Epoch 3 Batch 500 Loss 0.3972\n",
      "Epoch 3 Batch 600 Loss 0.3233\n",
      "Epoch 3 Batch 700 Loss 0.3631\n",
      "Epoch 3 Batch 800 Loss 0.3890\n",
      "Epoch 3 Batch 900 Loss 0.3423\n",
      "Epoch 3 Batch 1000 Loss 0.3692\n",
      "Epoch 3 Batch 1100 Loss 0.3425\n",
      "Epoch 3 Loss 0.3846\n",
      "Time taken for 1 epoch 38.43734049797058 sec\n",
      "\n",
      "Epoch 4 Batch 0 Loss 0.3317\n",
      "Epoch 4 Batch 100 Loss 0.3809\n",
      "Epoch 4 Batch 200 Loss 0.3065\n",
      "Epoch 4 Batch 300 Loss 0.3437\n",
      "Epoch 4 Batch 400 Loss 0.3638\n",
      "Epoch 4 Batch 500 Loss 0.3530\n",
      "Epoch 4 Batch 600 Loss 0.3294\n",
      "Epoch 4 Batch 700 Loss 0.3648\n",
      "Epoch 4 Batch 800 Loss 0.3696\n",
      "Epoch 4 Batch 900 Loss 0.3374\n",
      "Epoch 4 Batch 1000 Loss 0.3549\n",
      "Epoch 4 Batch 1100 Loss 0.3500\n",
      "Epoch 4 Loss 0.3448\n",
      "Time taken for 1 epoch 38.24331092834473 sec\n",
      "\n",
      "Epoch 5 Batch 0 Loss 0.3551\n",
      "Epoch 5 Batch 100 Loss 0.3421\n",
      "Epoch 5 Batch 200 Loss 0.3040\n",
      "Epoch 5 Batch 300 Loss 0.3572\n",
      "Epoch 5 Batch 400 Loss 0.2791\n",
      "Epoch 5 Batch 500 Loss 0.3619\n",
      "Epoch 5 Batch 600 Loss 0.3575\n",
      "Epoch 5 Batch 700 Loss 0.3425\n",
      "Epoch 5 Batch 800 Loss 0.3042\n",
      "Epoch 5 Batch 900 Loss 0.3075\n",
      "Epoch 5 Batch 1000 Loss 0.3252\n",
      "Epoch 5 Batch 1100 Loss 0.2826\n",
      "Epoch 5 Loss 0.3350\n",
      "Time taken for 1 epoch 38.20744514465332 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 5\n",
    "BATCH_SIZE = 32\n",
    "steps_per_epoch = len(X_train)//BATCH_SIZE\n",
    "BUFFER_SIZE = len(X_train)\n",
    "EMBED_SIZE = 8\n",
    "units = 64\n",
    "\n",
    "train_set = Dataset.from_tensor_slices((X_train, y_train)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
    "\n",
    "encoder = Encoder(units, len(input_vocab)+1, EMBED_SIZE, BATCH_SIZE)\n",
    "attention = LuongAttention(units)\n",
    "decoder = Decoder(units, len(output_vocab)+2, EMBED_SIZE, BATCH_SIZE)\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    enc_hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for (batch_no, (X, y)) in enumerate(train_set.take(steps_per_epoch)):\n",
    "        batch_loss = train_step(X, y, enc_hidden)\n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        if batch_no % 100 == 0:\n",
    "            print(f\"Epoch {epoch+1} Batch {batch_no} Loss {batch_loss.numpy():.4f}\")\n",
    "            \n",
    "    print(f\"Epoch {epoch+1} Loss {total_loss/steps_per_epoch:.4f}\")\n",
    "    print(f\"Time taken for 1 epoch {time.time() - start} sec\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_input_len = X_train.shape[1]\n",
    "max_output_len = y_train.shape[1]\n",
    "output_vocab_ = ['<sos>'] + output_vocab\n",
    "\n",
    "def evaluate(date):\n",
    "    attn_plot = np.zeros((max_output_len, max_input_len))\n",
    "    X = encode_data(date, input_vocab)\n",
    "    X = pad_sequences(X, padding='post', maxlen=max_input_len)\n",
    "    print(X.shape)\n",
    "    result = ''\n",
    "    \n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    encoder_output, state = encoder(X, hidden)\n",
    "    decoder_input = tf.expand_dims([sos_id], 0)  #initial decoder_input\n",
    "    for t in range(max_output_len):\n",
    "        y_pred_probas, state, attn_weights = decoder(decoder_input, encoder_output)\n",
    "        \n",
    "        attn_weights = tf.reshape(attn_weights, (-1,))\n",
    "        attn_plot[t] = attn_weights.numpy()\n",
    "        \n",
    "        y_pred = tf.argmax(y_pred_probas[0]).numpy()\n",
    "#         return y_pred_probas\n",
    "        result += output_vocab_[y_pred]\n",
    "        \n",
    "        # Feed the prediction back to the model\n",
    "        decoder_input = tf.expand_dims([y_pred], 0)\n",
    "    return result, X, attn_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "\n",
    "    fontdict = {'fontsize': 14}\n",
    "\n",
    "    ax.set_xticklabels([''] + [c for c in sentence], fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + [c for c in predicted_sentence], fontdict=fontdict)\n",
    "\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 18)\n",
      "92-12-12-1\n"
     ]
    }
   ],
   "source": [
    "sent = [\"December 18, 1992\"]\n",
    "result, X, attn_plot = evaluate(sent)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-358-cffcad780413>:9: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
      "  ax.set_xticklabels([''] + [c for c in sentence], fontdict=fontdict, rotation=90)\n",
      "<ipython-input-358-cffcad780413>:10: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
      "  ax.set_yticklabels([''] + [c for c in predicted_sentence], fontdict=fontdict)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAFnCAYAAACyzt3vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvbklEQVR4nO3deZxU1Z338e+tW1VNd7MrDeKGAi5xiTgqCk6EiEpMjBshQRSNmkSzuOXReZ5MJsmMWVwTJUaTqFlERB0nE+c1UTM6ETPjFqNgABENm7IjINILtdw6zx8FTbceejnnVt+y83m/XvUquqvP73z7bvWrW0Xf4Ohxk40AAADQTirpAAAAANWIJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMAinXQAoK1MJqOJEz6mvfcernT6g5vn3ff8MoFUAIC/RTRJqCpXXvEVfeTQg/Xaa68rX8gnHQcA8DesKpqkhoYh2mfvvdWnto9aWlr09tur9M47m5KOhQQccfhH9P0bbtGSJW8kHQUA8Dcu0SbpsMMO1YUXTNe+++7zgceWLVuu++6f06NPlkEQaM8999CmTZsVBIGiKOqxuatJksth9Zq1ClPV81G5vn37qlgsavv27YnMzzaJtvr166chQ/bU+vUb1NTUlHQc9LA99thD/fv1VTqdVnNLi955Z5NyudzfXIaOBEEgY+K7JG2Q1AVujzzicP3Dddfouede0FN/mKtVq1arublZdXW12n+//TRxwsd0/PHH6V++e4PefPOvFc2SSqU07XNTNfm0SQrDUFddc53OmzZVUVTSz+++V7lcz7ztM27c8crlcnr55XmSpMu+dKnmzXtVL/7ppR6ZvxqWw7777K2rr/qann3ueb3zzqYPbOx//J9nK54hDEOdc/aZOmXSx9WvX19J0pYt7+o/f/e4Hnv89xWfX6qOdYFkfeqTk3XYYR/RsmXL9ci//VYXnD9Np506SakdLyL++Mf/1d33/krFYjHhpKi0yaedojM//UkNHDiw3fdLpZKWr1ipf/vNbzVv3qu9OkMYhvrcZz+jE088QfV1dfrLgkV6YM5DWrNmbevPDBjQXz+9c6amTb8otnkTO5N07jln6XePPaEH5jzc7vtNTc16bfHrem3x69q8ZYvOOfvTuvGmH1Y0y2ennquPfvRwff8HN+v//sPXJUmPP/GkvvSFi3XB+efpnnt/VdH5JemsMz+lMz51uu795X2t39u0aZO++IXPa9CggXri909WPEM1LIeTTvp77bXXME2efKry+fc1AqZnmqSLPz9DHz3yCN0/e46WL1+hIJXSqFEj9ZkpZ2vAgP6a8+C/VjxDNawLJGfa5z6jj33sRD377PMae9yxOuig0Rqy55763vdv0rLlK7TP3sP1hUs/r/OmTdV9sx5IOi4q6PRPnKbTP3Ga7ps1RyvfektDhuypKeecrafnPqMlS97UMcccrauu+Ip+9vN79dzzL/baDNM+9xkde8zfadb9cyRJk0+dpB987591+8w79cq8+RWZU0qwSdp//33183t+0eHP/O//Pqd//MY/VDzLuHHH6447fqolb7zZeuZiyZI3dNfP7tG1X7+qR56QTpl0sm6b+RMtWLCo9Xv/+si/6803l+qSiy/skSapGpbDpJMn6o6f/EzPPvd8xefanXEnjNWNN/1Qr7d5q/ett97Wxg0bdeUVX+mRJqka1sWXvnCxfvsf/6n16zdUfK5qc9hhh3b5ZxctWhz7/CeeOE4zf3ynFi9eot899oTuvOM23XDjrXpt8euSpL8uXaZ77v2Vrr7qazRJFZb0tvCJyafqJ3f9TIsXL5EkrVmzVm+9tUo33fhdfemyr+nR//hPrV+/XlM/c27FGpRqyHDC8WP14zvuaj0uP/fcC7pwxnRdc/XXdNvtP9GfX36lIvMm1iTV1NSosbHj99Tf29ao/v37VTxL/379tPW99z7w/Vwup2w2U/H5Jamurk6bN235wPc3bNyo/v3790iGalgO27Y16q233uqRuXanubnF+tmflpaWHntroxrWxXHHHaPf/PY/emSuanPRjPO1zz57d+ln4zy1v1NNtkbvbd0mSWpsbJIxRs3Nze1+ZnsuZ/0zGYhX0ttCbW2tWlrafyayublZ9XV1qq+v17Zt27Rs+QoNHjwo9rmrKUOfPjXaurX9MfHX982WJF15xZd16w9navmKFbHPm+geZkyps59QEAQVz7FgwSKdecYn9bO7f7Ejl1Ftba2mfe4zWliBVwY2i19foqlTz9Gdd93d+iG4mpoaTTn3rB778Ho1LIdf/nqWLr3k8/rNvz+qDRs2Kiq1b1Y2bNhYkXkbGoa0/vuJ3z+pL1/+Rf161mwtW7Zcxhjtt+++uujC8/Wvj/ymIvO/XzWsi9899oS+cMnn9djjv9fGd95RoVBo93il1kU1+H//+G1d+bUva0jDEP3Tt/7lA797pc1/9VVdfPEMPfb473Xi+HFqbGzSlCln666f3q13392qAQP664Lzp2nRotd6NNffoqS3hQULFuqLl35et//4Tq1fv0E1NVldeslFWrduvbZt26Z+/fpqyrln669Ll/XqDK8veUNTp56rO+/6ebt18Ov7ZquutlZfv+aKihyfE/vg9pzZv9J9sx5QSwf/a6iutlYXnD+tIt15W4MGDdL/ueYKNTQMUX19vdauXac99hisDRs36qabf9Qjf45gyJ576hvfuE6DBg7Quh1vbwxtaNCmzZt08y239chbHtWwHObM/lWHj1dqW+hs3p7I0Nbf8rqoFul0Wt+9/ttauHCR7p/9YI/OXVdXp0svuVBHffRIbWts1C9+cZ+GDRuqGRecp6amZtXX1+ntt1fpBzfeqq1bt/ZotrZqa/vowhnnq1Ao6IUX/1SRt5uqQZLbQt++ffV/vn6lDj5otBqbmlRXW6tNmzbr1h/N1MqVb+k73/5H1WSzuu32n2j9hso8T1RDhoaGIfqHa69RQ8MQ3XDTrR/Y1mZccJ4+MflUSfEemxJrkn4881apizN/7cqvVzbMDocddqj2Hj5cYRhqzdq1+stfFsb6Xwk7E4ahjjzycO09fLiKxaLWrVuvV/+yoEczSMkuhz333KPDxyvVHHQ2b09ksPlbXBfVZO/hw3XoRw7WU089nXQUSeUnigMPGKEtW97VG2/+tcePDe9XX1+nr199he766T0644zT9ctfzUo8U6UkvS0cMGJ/NTQ0aOvWrXrzr0tbPxJQV1f3gbdie2uGMAw1auSBWrN2rbZta/zA46NHjdRxxx2r2Q/E18gm1iQBAABUs+r5q30AAABVhCYJAADAoiL/u62uvq8KXJwUAAB8CGQyWTU3ffBzTrE3SXX1fTX1gsvjLgsAAFAxD8+66wONUuxN0s4zSP911X0qbu/+35Mwaak4IaP03IKChC5JRIbqyJD0/GQgQzXNH2eGoCbrniFlVBgvZZ6VgpL737ELhw31yFDS9pEb1Gdpg4KS+6dGSuvc/7u6CY3y44yyzwUKIrfl8MirftflbGoJ9blrj9KDN89Xfa3bxa/POeRI5/l7yz6R7pPRqbfNsL4DVrE/JlncXlCxpftvuZm0VCxIasknu9DJkHiGpOcnAxmqaf44M/g0NyY0KhakYLucmwNJMjn3X8CkSioWCirmin5NksML+dYMoVGxYJTa7t4k1WR2/3cCu6JQCFUsFJRNb1dNxq1Jcnme3qk37RO7wwe3AQAALGiSAAAALGiSAAAALGiSAAAALGiSAAAALGiSAAAALGiSAAAALGiSAAAALLr0xyT79euriy+aoSOPPFwtLdv12OO/12OP/77S2QAAABLTpSbp69dcqZpsVt/7wc2q7dNHX778izLG6PEn/qvS+QAAABLR6dttBxwwQocecrBm3nGXli1brkWvLdbsBx7Upz/9yZ7IBwAAkIhOm6ShDUPU2NiotWvXtX5v5cq3NXjQIA3Zc8+KhgMAAEhKp2+3bd36nmpra9WnTx9t316+GN+eQ/aQJPXr308b33nHOs6ky7fuMmH7+ySQoToyJD0/GchQTfPHmiE0MWXwqJMqeY/1qSGVL1LrO9anRmOz34psakm1u3fh8jzdOraX7BMdLYNOF8+bf12qTZs269JLLtQ99/5atbW1+sy5Z5cHh7tPVZyQKV+Z11E0Mes+OCZkqI4MSc9PBjJU0/zVkqE43m98Qeu9M+RGbvQrMNo7ggrjJNdm8dNf/Tv/AJKmXTvGffAk//mrYXv0ypDJ7PahTpukYrGoH/5opq644iv65b0/VXNzsx6Y87BGjx6llpaW3ReeW5Ba8t3OasLyLxs+nVcQdXt4LMhQHRmSnp8MZKim+ePMEGTdn1BMWG6Q0s/KK0O411D3DKmSciM3qmbpEAUl97MopXUb3DOERoVxUuY5KYgCpxqPvPpn5/ml8hmkadeO0Zyb56m+1u2s2jmHHOE8f2/ZJ9K1kqbs5rGuFFi+YqWuvuY6DRjQX42NTRo2bKhKpZLe2bRpt2OCYvnmKoj8xseBDNWRIen5yUCGapo/jgxB6PakXmZ2ZXBsDiR5NTdta/jU8clfZhREgXOdvnXxdBb1tSXnWnFsyx/2faKjcZ1uXfX1dfrnb39T/fv309at7ymKIh3zd0dr+fIVamnZ7pYIAACgynXaJDU1NaumJqsLzj9PQxsadPzYY3XuOWfq3/790Z7IBwAAkIgunae8beZPNHjwIN104/f02alT9PO7f6GXX55X6WwAAACJ6dJnktatW6/rv3tDpbMAAABUDS5wCwAAYEGTBAAAYEGTBAAAYEGTBAAAYEGTBAAAYEGTBAAAYEGTBAAAYEGTBAAAYEGTBAAAYNGlv7gNAOg9TC7nPjYtSVmZfF7yuPJ7cflKvwyjsyqufDuxq8/vXA6lXM45w2nDj/LPMEk655AjnDOsuW6c8/xBVNSei+Zr3ZXHyYTu7cSYsxc6jy1tN1pzY04NT/ZTqk/gVCOMMtJS+2OcSQIAALCgSQIAALCgSQIAALCgSQIAALCgSQIAALCgSQIAALCgSQIAALCgSQIAALCgSQIAALCgSQIAALCgSQIAALDo0sVWhjY06MIZ03XwwQcpl8vp+Rde1IMPPaJCoVDpfAAAAInotEkKw1DXXXu1Vq1erW99+3r1H9BPl33pUknSrPvnVDwgAABAEjp9u23UqJEaNmyo7rzrbq1es0aLFy/Rww//RieOP6En8gEAACSi0yZpzZq1uuHGW5XL5Vq/Z2RUV1dX0WAAAABJ6vTttm3btmnBwkWtXwdBoMmnTtKCha91OM6ky7fuMmH7+ySQoToyJD0/GchQTfOTgQyVyBBERY+xUbt7V6Xtxn1szrS7dxGUdj82OHrc5G5VvnDGdJ388Qn6xje/o1WrVn/g8Uw2q+kXX6lHH7lPRT7YDQAAqlg6k9GZU2Zo9i9uVyGfb/9YdwpdOGO6Tj3lZP3otjusDVK7wnMLUku+w5+xMaEUTcwqfDqvwK85dUaG6siQ9PxkIEM1zU8GMlQiw7orj3OeP4gi7fH6Am065AiZ0P101pGfWuw8tpQzWndbXsOuyipVEzjVCEsZaZX9sS41SUEQ6LIvXqITTxyn22feqT+//ErnY4rlm6sg8hsfBzJUR4ak5ycDGappfjKQIc4MJnT4XMwHaoRedVJ93JqbdjVqAuc6qWj347r0W11w/jSNH3+Cbv3hTL0yb75TCAAAgA+TTpuk0aNG6pOnT9YDcx7W0mXLNWDAgNbHtm7dWtFwAAAASem0SRo79lhJ0nnTpuq8aVPbPTZt+kUqlUqVSQYAAJCgTpuk+2c/qPtnP9gTWQAAAKoGF7gFAACwoEkCAACwoEkCAACwoEkCAACwoEkCAACwoEkCAACwoEkCAACwoEkCAACwoEkCAACw8L/8L3YvFXqMbVPDsZVNZTPu80syoZFklKqpUZB2vLryXkPd50+VVNR6pfffV0HJvZ+P9ujnPFZBJGmpgjGHKjDu6zPc3Og8trwc1ik9Yj+v5WC2vuc+NjQqqknh4EEKOrhidkdKjU3O80uSQiNJCrJZBaHj9lhX5zx9eRk0Kxw40HkZSFLksR7iOC5IUqq2j/PY8nGhqFRdnddyMLmc81jt2BWDMC2f68ebknEfHMe6KEXu88dk+E3POY81aak4Katht/9JQdE9w/qb3MeatKRJWW04ZZtzhnRtVvqp/THOJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFjQJAEAAFikK1XYpMu3bo8L298nIbYMHi1oHBlMaNwHtxnvU8ekSt5jfWpIkgki77E+NaQqWQ5e69G0u+/p+cvj29671fJaBjHsD5LbcXFXhvb37nWqYDl4Hdv8a0iSSfgYLb9dOvHny6TnjytDR/tkt3bXs848Q2efdUbr1z+44Ra9vuQN688WJ2RULHSnenvRxKz74JhURYaTfLY+v4PYToVxPrXWe8+fG7nRs4J/huLgFV7jC3t4R1DuwA3+RTzlj2pOOoKK431G++fPj2nxrOD/jOJ3XJCkoneGwli/Fw5xvJFR/Fjyb4b4rYt4uoukn6uSnt87Qyaz24e61SQ9+dQf9PwLL7Z+vXnzlt0XnluQWvLdKS+p3A1GE7MKn87L88W7s9gypNx3ABOWd77wmcg5Qyrjd6LQhEaFcVLmOSmIArcMwxrc50+VlBu5UTVLhygouR8Mo8F93TMEkYqDVyi9eYQCj5cq4ZYm9wypknIHblDNsgav5WDe2+aRwSh/VLOy8+sUlNy2hVKj+zKQyvtEcbyUflbu+0Rdncf8RvkxLcrOq3XeHyQp8lkPMRwXJCnVp8Yjg1FhbKTMi6HXcjC57j8/7MpQbpDSfyx5LQdjfM6oxbAuSp5nqBN+vkx6/rgypGslTdnNY90p1NTUpKamrh3ogmL55iqI/MbHwTtDDC9yfDIEKfcD2C5GQRQ4Hwx9ntTb1vCp49PctK3hU6caloM8ntB2ZfDYFrznNzvquNfyzyCv/UGK57jme2yqhuWgGJ5Uy9uCRwHPt7taM7iuixjm987QC+b3zdDRuOTPVQIAAFQhmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAAALmiQAAACLdNIBerWSx+WpS5IUlms4Xim6tN3vMtsmLUlZlXI556srl5av9Jt/dFbFlW/7XWF6uftQk5Y0KSszb7HkkcEnvklLGpVVccVbXsshHDTIfXBgyvcl47w9bpl6tPv8klSKVL9pvt49+ygpFTqVOOGql5ynj7ZLi65PadSjjQr7OJfR4mM8Lv1uJCmUTGnHv92UmprcI+w8LjQ3J3bl9/KvnpWJin47l48YjtGofpxJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsOhWk5ROp3XLTd/XEYcfVqk8AAAAVaHLF7jNZDK64quXa99996lkHgAAgKrQpSZp772H64qvXq4gCCqdBwAAoCp0qUn6yKGHaNFri/XgQ49o1q/v6VJhky7fusuE7e+TQIbqyJD0/L0tgwmN91ifGipF7mPbjveoE213n37nWJ8akttxsXVsXNujx2rsTfsEGT7c88eVoaN9sku765NP/aHbkxYnZFQsdHtYq2hi1n1wTMhQHRmSnr+3ZCiq2TtDfkyL89j6TfO955ek+i0LnMcuut7//6q8frNnjZP9M0QTMt41vDP0gn2CDL1jfu8Mmd3vTx6vaTqWnluQWvLdHmfC8i8bPp1X4PnC0xUZqiND0vP3tgzhwIEeGYzyY1qUnVerIHJ7233zaQc5zy9JKkWq37JATYOOkFJuLxuP+/IrztNH28sN0iHXlhT2cS6jJRPcV6IJyw1SOLfgtz0an7OKvWefIMOHe/64MqRrJU3ZzWPOyToRFMs35/GR3/g4kKE6MiQ9f2/J4NrcvL+Gcx3HxsZax7GWT3PTtoZPnTi2I+/t0ePtttgyxIAM1ZEh6fl9M3Q0jr+TBAAAYEGTBAAAYEGTBAAAYEGTBAAAYNHtD25/dtqMSuQAAACoKpxJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsOj2X9zusiAo37o9ru149+lTffs6jzWhkZRXqm9fBZF7iGD4UPcMQUlFrVI46kAFxrGXNcZ5/l0ZVisceYB7hnUb3ecPjYrKKezXz2s9RI1NzmNbX0akQr+XFKXIY3A8oi1bnMeatCRlFb37roKiW42Bs553nn9nhuKkrAY89CfnDItn+c2vSVktOanoPL8k5U4/1iNEpJRZqPykMVIQOpdZO9790B8UIw1/fr7e+qfjZNLuGUbds8Z5rEmVVNR6pfffV0HJfccsrlzlPDaOY0MQui8/SdKO4UE66/R0K0mmkPfL0MtxJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCiSxfwGdrQoAtnTNfBBx+kXC6n5194UQ8+9IgKhUKl8wEAACSi0yYpDENdd+3VWrV6tb717evVf0A/XfalSyVJs+6fU/GAAAAASej07bZRo0Zq2LChuvOuu7V6zRotXrxEDz/8G504/oSeyAcAAJCITs8krVmzVjfceKtyuVzr94yM6urqOhxn0uVbd5mw/b0rExrvsT41JElByT3DjrHGo4bklz+WDFWwHly2w10Z2t8781iEsWXwQIYY5zeR/1ifGpKCYuAxNmp378qkPI6PO8b61JCq4Njg+angODIYj8Nr0vtkXBk62g463US2bdumBQsXtX4dBIEmnzpJCxa+1uG44klpFQvuSz+akHEeW5b3HC8VjvX9zNUq7wz5/dd41/DOMGKt++ADYpj/GN916b8HRyf51oghw8Ssdw0yJD9/yiz0zpDSYq/XQMOf946gvV5a4DV++2j/DLmRG/0KjK6GY4O/4t/7jPbfn5LeJ70zZHbfb3S7j55xwXkaMWJ/feOb3+nw59LPFKWW7jcZJiw3SOHcggKPFyqp+nrnsSY0KhxbUOaljILI4xXXsAb3DEFJ+f3XKLtyuALj+HLD5yXCzgwj1iq7Yi/3DBvecZ8/NMofk1f2z1mv9RA1NXtkKB8Ew2cir+1RJffBJiwfAMKn834ZPJAhvvnzpxztESJSSotV0qFS4P7kvO5491MoQTHSXi8t0Npjj5BJu2c48D73F18mVVJu5EbVLB2ioOR+Oqb4tvuL0DiODUHKr8EyYblBSv+PnDOYovuL0KT3ybgypGslTdnNY90pdOGM6Tr1lJP1o9vu0KpVqzv82aBYvrkKIt/x7k+qbWt4NUmujcX7aiTVJMWSoRrWg8d2tCuDZx2/dwXiyRADMsQwv0dzI6l8BikIver4NDdta/jU8Wlu2tbwqZP0scF3U2iXwbVJqYbjYwy81kMH47rUJAVBoMu+eIlOPHGcbp95p/788ituSQAAAD4kutQkXXD+NI0ff4Ju/eFMvTJvfoUjAQAAJK/TJmn0qJH65OmT9cCch7V02XINGDCg9bGtW7dWNBwAAEBSOm2Sxo49VpJ03rSpOm/a1HaPTZt+kUqlGD5sAQAAUGU6bZLun/2g7p/9YE9kAQAAqBpc4BYAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCCJgkAAMCiS9duc2KM21Xodw4xZte/HZS2bXMea9KSlFWpsdHvysZvNvtlGBEqWrbSOUMqm3GeX5JMaKQDJLNqjRQFbhn2GuoeIFWStF7B4IF+V/o+aD/3DEEkaamCjx6kwLhfsjvc3Og81qRKKmqd0iP281oOZut77mNDo6KaFA4epMBxWyg1NjnPL0kKyweEIJtVEDpuj3V1ztOXl0GzwoEDnZeBJNU84X6BcJOWih8Plf3vV72OTQc+08c9Q2iUHyeNuHWh13KIcjmPDJJGpxStWlveRT+kTCHvN95IUlammJd8nquwW5xJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsKBJAgAAsEhXqrBJl2/dHhe2v09CbBk8WtA4MpjQuA9uM96njkmVvMf61JAkE0TeY31qSFWyHLzWo2l339Pzl8e3vXer5bUMYtgfJLfj4q4M7e/d61TBcvA6tvnXkCST8DFafrt04s+XSc8fV4aO9slu7a5nnXmGzj7rjNavf3DDLXp9yRvWny1OyKhY6E719qKJWffBMamKDCf5bH1+B7GdCuN8aq33nj83cqNnBf8MxcErvMYX9vCOoNyBG/yLeMof1Zx0BBXH+4z2z58f0+JZwf8Zxe+4IElF7wyFsX4vHOJ4I6P4seTfDPFbF/F0F0k/VyU9v3eGTGa3D3WrSXryqT/o+RdebP168+Ytuy88tyC15LtTXlK5G4wmZhU+nZfni3dnsWVIue8AJizvfOEzkXOGVMbvRKEJjQrjpMxzUhAFbhmGNbjPnyopN3KjapYOUVByPxhGg/u6ZwgiFQevUHrzCAUeL1XCLU3uGVIl5Q7coJplDV7Lwby3zSODUf6oZmXn1ykouW0LpUb3ZSCV94nieCn9rNz3ibo6j/mN8mNalJ1X67w/SFLksx5iOC5IUqpPjUcGo8LYSJkXQ6/lYHLdf37YlaHcIKX/WPJaDsb4nFGLYV2UPM9QJ/x8mfT8cWVI10qaspvHulOoqalJTU1dO9AFxfLNVRD5jY+Dd4YYXuT4ZAhS7gewXYyCKHA+GPo8qbet4VPHp7lpW8OnTjUsB3k8oe3K4LEteM9vdtRxr+WfQV77gxTPcc332FQNy0ExPKmWtwWPAp5vd7VmcF0XMczvnaEXzO+boaNxyZ+rBAAAqEI0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABY0SQAAABbppAP0aiWPy1OXJCks13C8UnRpu99ltk1akrIq5XLOV1cuLV/pN//orIor3/a7wvRy96EmLWlSVmbeYskjg098k5Y0Kqviire8lkM4aJD74MCU70vGeXvcMvVo9/klqRSpftN8vXv2UVIqdCpxwlUvOU8fbZcWXZ/SqEcbFfZxLqPFx3hc+t1IUiiZ0o5/uyk1NblH2HlcaG5O7Mrv5V89KxMV/XYuHzEco1H9OJMEAABgQZMEAABgQZMEAABgQZMEAABgQZMEAABgQZMEAABgQZMEAABgQZMEAABgQZMEAABgQZMEAABgQZMEAABg0a0mKZ1O65abvq8jDj+sUnkAAACqQpcvcJvJZHTFVy/XvvvuU8k8AAAAVaFLTdLeew/XFV+9XEEQVDoPAABAVejS220fOfQQLXptsb75rX+pdB4AAICq0KUzSU8+9YduFzbp8q3b48L290kgQ3VkSHr+3pbBhMZ7rE8NlSL3sW3He9SJtrtPv3OsTw3J7bjYOjau7dFjNfamfYIMH+7548rQ0T7psbt2rDgho2LBfXw0MRtfGDJ8qDMkPX9vyVBUs3eG/JgW57H1m+Z7zy9J9VsWOI9ddL3/f+h9/WbPGif7Z4gmZLxreGfoBfsEGXrH/N4ZMrvfnyrWJKXnFqSWfLfHmbD8y4ZP5xV4vvB0RYbqyJD0/L0tQzhwoEcGo/yYFmXn1SqI3D6buPm0g5znlySVItVvWaCmQUdIKbeXjcd9+RXn6aPt5QbpkGtLCvs4l9GSCe4r0YTlBimcW/DbHo3PWcXes0+Q4cM9f1wZ0rWSpuzmMedknQiK5Zvz+MhvfBzIUB0Zkp6/t2RwbW7eX8O5jmNjY63jWMunuWlbw6dOHNuR9/bo8XZbbBliQIbqyJD0/L4ZOhrHH5MEAACwoEkCAACwoEkCAACw6PZnkj47bUYlcgAAAFQVziQBAABY0CQBAABY0CQBAABY0CQBAABY0CQBAABY0CQBAABY0CQBAABY0CQBAABY0CQBAABYdPsvbndZEJRv3R7Xdrz79Km+fZ3HmtBIyivVt6/XldOD4UPdMwQlFbVK4agDFRjHXtb4Xeq7nGG1wpEHuGdYt9F9/tCoqJzCfv281kPU2OQ8tvVlRCr0e0lRijwGxyPassV5rElLUlbRu+86X2l74KznneffmaE4KasBD/3JOcPiWX7za1JWS04qel3xPHf6sR4hIqXMQuUnjZGC0LnM2vHuh/6gGGn48/P11j8dJ5N2zzDqnjXOY02qpKLWK73/vgpK7jtmceUq57FxHBuC0H35SZJ2DA/SWaenW0kyhbxfhl6OM0kAAAAWNEkAAAAWNEkAAAAWNEkAAAAWNEkAAAAWNEkAAAAWNEkAAAAWNEkAAAAWNEkAAAAWNEkAAAAWNEkAAAAWXbqAz9CGBl04Y7oOPvgg5XI5Pf/Ci3rwoUdUKBQqnQ8AACARnTZJYRjqumuv1qrVq/Wtb1+v/gP66bIvXSpJmnX/nIoHBAAASEKnb7eNGjVSw4YN1Z133a3Va9Zo8eIlevjh3+jE8Sf0RD4AAIBEdNokrVmzVjfceKtyuVzr94yM6urqKhoMAAAgSZ2+3bZt2zYtWLio9esgCDT51ElasPC1DseZdPnWXSZsf+/KhMZ7rE8NSVJQcs+wY6zxqCH55Y8lQxWsB5ftcFeG9vfOPBZhbBk8kCHG+U3kP9anhqSgGHiMjdrduzIpj+PjjrE+NaQqODZ4/tepODIYj8Nr0vtkXBk62g6Co8dN7tYiunDGdJ388Qn6xje/o1WrVn/g8Uw2q+kXX6lHH7lPRT7YDQAAqlg6k9GZU2Zo9i9uVyGfb/9YdwpdOGO6Tj3lZP3otjusDVK7ws8UpZbuN0kmlKIJGYVzCwo8Xqik6uudx5rQqHBsQZmXMgoij1dcwxrcMwQl5fdfo+zK4QqM48sNn5cIOzOMWKvsir3cM2x4x33+0Ch/TF7ZP2e91kPU1OyRQYpOChU+E3ltjyq5DzahFE3MKnw675fBAxnimz9/ytEeISKltFglHSoF7i+d1x3vfgolKEba66UFWnvsETJp9wwH3rfWeaxJlZQbuVE1S4coKLmfjim+vcY9QwzHhiDldwrGhFLx76X0/8g5gynmO/+hDubvDceFdK2kKbt5rCsFgiDQZV+8RCeeOE63z7xTf375lc7HFMs3V0HkO979SbVtDa8mybWxeF+NpJqkWDJUw3rw2I52ZfCs4/euQDwZYkCGGOb3aG4kld9FD0KvOj7NTdsaPnV8mpu2NXzqJH1s8N0U2mVwbVKq4fgYA6/10MG4LjVJF5w/TePHn6BbfzhTr8yb75YCAADgQ6TTJmn0qJH65OmT9cCch7V02XINGDCg9bGtW7dWNBwAAEBSOm2Sxo49VpJ03rSpOm/a1HaPTZt+kUqlGN5HAAAAqDKdNkn3z35Q989+sCeyAAAAVA0ucAsAAGBBkwQAAGBBkwQAAGBBkwQAAGBBkwQAAGBBkwQAAGBBkwQAAGBBkwQAAGBBkwQAAGDRpQvcOjHG7Sr0O4cYs+vfDkrbtjmPNWlJyqrU2Oh3ZeM3m/0yjAgVLVvpnCGVzTjPL0kmNNIBklm1RooCtwx7DXUPkCpJWq9g8EC/K30ftJ97hiCStFTBRw9SYNwv2R1ubnQea1IlFbVO6RH7eS0Hs/U997GhUVFNCgcPUuC4LZQam5znlySF5QNCkM0qCB23x7o65+nLy6BZ4cCBzstAkmqeeMU9Q1oqfjxU9r9f9To2HfhMH/cMoVF+nDTi1oVeyyHK5TwySBqdUrRqbXkX/ZAyhbzfeCNJWZliXvJ5rsJucSYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAgiYJAADAIl2pwiZdvnV7XNj+PgmxZfBoQePIYELjPrjNeJ86JlXyHutTQ5JMEHmP9akhVcly8FqPpt19T89fHt/23q2W1zKIYX+Q3I6LuzK0v3evUwXLwevY5l9DkkzCx2j57dKJP18mPX9cGTraJ7u1u5515hk6+6wzWr/+wQ236PUlb1h/tjgho2KhO9XbiyZm3QfHpCoynOSz9fkdxHYqjPOptd57/tzIjZ4V/DMUB6/wGl/YwzuCcgdu8C/iKX9Uc9IRVBzvM9o/f35Mi2cF/2cUv+OCJBW9MxTG+r1wiOONjOLHkn8zxG9dxNNdJP1clfT83hkymd0+1K0m6cmn/qDnX3ix9evNm7fsvvDcgtSS7055SeVuMJqYVfh0Xp4v3p3FliHlvgOYsLzzhc9EzhlSGb8ThSY0KoyTMs9JQRS4ZRjW4D5/qqTcyI2qWTpEQcn9YBgN7uueIYhUHLxC6c0jFHi8VAm3NLlnSJWUO3CDapY1eC0H8942jwxG+aOalZ1fp6Dkti2UGt2XgVTeJ4rjpfSzct8n6uo85jfKj2lRdl6t8/4gSZHPeojhuCBJqT41HhmMCmMjZV4MvZaDyXX/+WFXhnKDlP5jyWs5GONzRi2GdVHyPEOd8PNl0vPHlSFdK2nKbh7rTqGmpiY1NXXtQBcUyzdXQeQ3Pg7eGWJ4keOTIUi5H8B2MQqiwPlg6POk3raGTx2f5qZtDZ861bAc5PGEtiuDx7bgPb/ZUce9ln8Gee0PUjzHNd9jUzUsB8XwpFreFjwKeL7d1ZrBdV3EML93hl4wv2+GjsYlf64SAACgCtEkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWNAkAQAAWKSTDtCrlTwuT12SpLBcw/FK0aXtfpfZNmlJyqqUyzlfXbm0fKXf/KOzKq582+8K08vdh5q0pElZmXmLJY8MPvFNWtKorIor3vJaDuGgQe6DA1O+Lxnn7XHL1KPd55ekUqT6TfP17tlHSanQqcQJV73kPH20XVp0fUqjHm1U2Me5jBYf43HpdyNJoWRKO/7tptTU5B5h53GhuTmxK7+Xf/WsTFT027l8xHCMRvXjTBIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIBFt5qkdDqtW276vo44/LBK5QEAAKgKXb7AbSaT0RVfvVz77rtPJfMAAABUhS41SXvvPVxXfPVyBUFQ6TwAAABVoUtN0kcOPUSLXlusBx96RLN+fU+XCpt0+dZdJmx/nwQyVEeGpOfvbRlMaLzH+tRQKXIf23a8R51ou/v0O8f61JDcjoutY+PaHj1WY2/aJ8jw4Z4/rgwd7ZNd2l2ffOoP3Z60OCGjYqHbw1pFE7Pug2NChurIkPT8vSVDUc3eGfJjWpzH1m+a7z2/JNVvWeA8dtH1/v9X5fWbPWuc7J8hmpDxruGdoRfsE2ToHfN7Z8jsfn/yeE3TsfTcgtSS7/Y4E5Z/2fDpvALPF56uyFAdGZKev7dlCAcO9MhglB/Touy8WgWR29vum087yHl+SVIpUv2WBWoadISUcnvZeNyXX3GePtpebpAOubaksI9zGS2Z4L4STVhukMK5Bb/t0ficVew9+wQZPtzzx5UhXStpym4ec07WiaBYvjmPj/zGx4EM1ZEh6fl7SwbX5ub9NZzrODY21jqOtXyam7Y1fOrEsR15b48eb7fFliEGZKiODEnP75uho3H8nSQAAAALmiQAAAALmiQAAAALmiQAAACLbn9w+7PTZlQiBwAAQFXhTBIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIAFTRIAAIBFt/+YZJcL98k4jTNpSZmM0rXJXVWYDNWRIen5e1uG0HGflCQTGkWZjNJ9MgqiwKlGJvR8TZYySmcyyqRTUuBWKxV5LINISmdSSkUlpSLnMkrXhu4Z0pIyaaVrA7/t0RjPDL1jnyDDh3v+uDJ01K8ER4+b7L63WNTV99XUCy6PsyQAAEBFPTzrLjU3Nbb7XuxNklRulAqFfNxlAQAAYpfJZD/QIEkVervNNhEAAEA1KuTtJ3b44DYAAIAFTRIAAIAFTRIAAIAFTRIAAIDF/wdBRjyqf9Cw2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_attention(attn_plot[:len(result), :len(sent[0])], sent[0], result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Obviously the result isn't coming out correctly, though i'm not sure where I'm going wrong here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 11\n",
    "Use one fo the recent Language Models to generate more convincing Shakesperean text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this I'll use [HuggingFace Transformers](https://huggingface.co/transformers/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "All model checkpoint layers were used when initializing TFGPT2LMHeadModel.\n",
      "\n",
      "All the layers of TFGPT2LMHeadModel were initialized from the model checkpoint at gpt2.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFGPT2LMHeadModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import GPT2Tokenizer, TFGPT2LMHeadModel\n",
    "from transformers import tf_top_k_top_p_filtering, pipeline\n",
    "from pprint import pprint\n",
    "\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "model = TFGPT2LMHeadModel.from_pretrained(\"gpt2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First, you know Caius Marcius is chief enemy to the people. ...  He has been imprisoned for his treason and he will be executed if we do not destroy him before then.\"\n",
      "The king's words were a little harsh but they also made me think about what might happen in our own country after that war ended (which was probably as soon as it began). I'm glad my father died long ago so this time around there are some things which may change: one day all of us who have served with Nerva or other princes should die fighting because their family became enemies! The next year when these guys start getting shot at again by bandits just imagine how much more dangerous life would become than being killed on your way home from work each morning while trying desperately hard to get back into civilization without any help whatsoever… This could possibly lead to something worse like mass murder? Or maybe even genocide!? It seems such an unlikely scenario either... Even though many nobles simply refuse to go along no matter where\n"
     ]
    }
   ],
   "source": [
    "text = \"First, you know Caius Marcius is chief enemy to the people.\"\n",
    "max_len = 200\n",
    "temp = 0.6\n",
    "\n",
    "tokenized = tokenizer(text, add_special_tokens=False, return_tensors=\"tf\")\n",
    "input_ids = tokenized.input_ids\n",
    "attn_mask = tokenized.attention_mask\n",
    "\n",
    "output = model.generate(\n",
    "    input_ids=input_ids,\n",
    "    attention_mask=attn_mask,\n",
    "    do_sample=True,\n",
    "    top_k=0,\n",
    "    top_p=0.9,\n",
    "    repetition_penalty=1.1,\n",
    "    max_length=max_len,\n",
    "    temperature=temp)\n",
    "\n",
    "print(f\"{text} ... {tokenizer.decode(output[0])[len(text):]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "homl",
   "language": "python",
   "name": "homl"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}