text_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/mohsenMahmoodzadeh/Image-Caption-classification-with-tensorflow-Keras/blob/master/Phase_01_Text_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "olAUeBAWg-MQ",
    "outputId": "4f17a73f-1dd1-4aeb-c0dd-31f93ab7843e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wDg-HnzJZDW_"
   },
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KLxWduGEM_xv"
   },
   "outputs": [],
   "source": [
    "!pip install tensorflow-text\n",
    "!pip install tensorflow-hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "oKcYXY-YF7ZS"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import string\n",
    "import re\n",
    "import pandas as pd\n",
    "import seaborn as sn\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text\n",
    "\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "\n",
    "tf.get_logger().setLevel('ERROR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FXPTesswMbn5"
   },
   "outputs": [],
   "source": [
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5oh3d7MHZWji"
   },
   "source": [
    "## Import Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "pVH4xDDdg1S1"
   },
   "outputs": [],
   "source": [
    "TRAIN_DATA_DIR = \"/content/drive/MyDrive/dataset/train/sentences/\"\n",
    "TEST_DATA_DIR = \"/content/drive/MyDrive/dataset/test/sentences/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "tTI4GIlqDqjb"
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "AUTOTUNE = tf.data.AUTOTUNE\n",
    "seed = 1337"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ds7U72yJsVJD",
    "outputId": "3584d046-ec1e-4af4-910e-6c198024787b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 532 files belonging to 19 classes.\n",
      "Using 426 files for training.\n"
     ]
    }
   ],
   "source": [
    "raw_train_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
    "    TRAIN_DATA_DIR,\n",
    "    batch_size = batch_size,\n",
    "    label_mode=\"categorical\",\n",
    "    validation_split=0.2,\n",
    "    subset='training',\n",
    "    seed= seed \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ArHI4lydsYG3",
    "outputId": "b46b3259-758c-4193-cc8b-7bae4706c284"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 532 files belonging to 19 classes.\n",
      "Using 106 files for validation.\n"
     ]
    }
   ],
   "source": [
    "raw_val_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
    "    TRAIN_DATA_DIR,\n",
    "    batch_size = batch_size,\n",
    "    label_mode=\"categorical\",\n",
    "    validation_split=0.2,\n",
    "    subset='validation',\n",
    "    seed= seed \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "U2b9uhWIsa7K",
    "outputId": "fd88bf86-0bf5-4a2c-da05-da22911d7918"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 380 files belonging to 19 classes.\n"
     ]
    }
   ],
   "source": [
    "raw_test_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
    "    TEST_DATA_DIR,\n",
    "    batch_size=batch_size,\n",
    "    label_mode=\"categorical\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "65pqci2363_B",
    "outputId": "6b155370-b739-49bf-ff20-bd4e9a4c4a45"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of batches in raw_train_ds: 14\n",
      "Number of batches in raw_val_ds: 4\n",
      "Number of batches in raw_test_ds: 12\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of batches in raw_train_ds: {raw_train_ds.cardinality()}\")\n",
    "print(f\"Number of batches in raw_val_ds: {raw_val_ds.cardinality()}\")\n",
    "print(f\"Number of batches in raw_test_ds: {raw_test_ds.cardinality()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "C5WTiEfgP8ST"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mT0M1-hNGo-z"
   },
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NeBnJwkz7CKZ"
   },
   "outputs": [],
   "source": [
    "def custom_standardization(input_data):\n",
    "    lowercase = tf.strings.lower(input_data)\n",
    "    stripped_html = tf.strings.regex_replace(lowercase, \"<br />\", \" \")\n",
    "    return tf.strings.regex_replace(\n",
    "        stripped_html, f\"[{re.escape(string.punctuation)}]\", \"\"\n",
    "    )\n",
    "\n",
    "max_features = None\n",
    "sequence_length = 256\n",
    "\n",
    "vectorize_layer = tf.keras.layers.TextVectorization(\n",
    "    standardize=custom_standardization,\n",
    "    output_mode=\"int\",\n",
    "    output_sequence_length=sequence_length,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YQZ60mVR7mE4"
   },
   "outputs": [],
   "source": [
    "text_ds = raw_train_ds.map(lambda x, y: x)\n",
    "vectorize_layer.adapt(text_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tfHwoSjDAw3r"
   },
   "outputs": [],
   "source": [
    "max_features = vectorize_layer.vocabulary_size()\n",
    "vocab = vectorize_layer.get_vocabulary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YAFcIQUZ8M6b"
   },
   "outputs": [],
   "source": [
    "def vectorize_text(text, label):\n",
    "    text = tf.expand_dims(text, -1)\n",
    "    return vectorize_layer(text), label\n",
    "\n",
    "train_ds = raw_train_ds.map(vectorize_text)\n",
    "val_ds = raw_val_ds.map(vectorize_text)\n",
    "test_ds = raw_test_ds.map(vectorize_text)\n",
    "\n",
    "train_ds = train_ds.cache().prefetch(buffer_size=10)\n",
    "val_ds = val_ds.cache().prefetch(buffer_size=10)\n",
    "test_ds = test_ds.cache().prefetch(buffer_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xhuAg2a0exhS"
   },
   "outputs": [],
   "source": [
    "word_index = dict(zip(vocab, range(len(vocab))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4BsrfZjFGwrC"
   },
   "source": [
    "## Word Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rG2zjdln_SZ2",
    "outputId": "70a9098f-4ab8-4b93-acd5-b7e33e298103"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2022-01-24 21:28:25--  http://nlp.stanford.edu/data/glove.6B.zip\n",
      "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
      "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
      "--2022-01-24 21:28:25--  https://nlp.stanford.edu/data/glove.6B.zip\n",
      "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
      "HTTP request sent, awaiting response... 301 Moved Permanently\n",
      "Location: http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip [following]\n",
      "--2022-01-24 21:28:25--  http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip\n",
      "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
      "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 862182613 (822M) [application/zip]\n",
      "Saving to: ‘glove.6B.zip’\n",
      "\n",
      "glove.6B.zip        100%[===================>] 822.24M  5.19MB/s    in 2m 40s  \n",
      "\n",
      "2022-01-24 21:31:06 (5.13 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget http://nlp.stanford.edu/data/glove.6B.zip\n",
    "!unzip -q glove.6B.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "W_4gpSGG_XJE",
    "outputId": "d7859927-5eb1-4bf1-9bd8-8ad63a3638ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "path_to_glove_file = '/content/glove.6B.100d.txt'\n",
    "\n",
    "embeddings_index = {}\n",
    "with open(path_to_glove_file) as f:\n",
    "    for line in f:\n",
    "        word, coefs = line.split(maxsplit=1)\n",
    "        coefs = np.fromstring(coefs, \"f\", sep=\" \")\n",
    "        embeddings_index[word] = coefs\n",
    "\n",
    "print(\"Found %s word vectors.\" % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2LNpJgch_ahP",
    "outputId": "b9e41d6e-2923-48e9-f106-a1c3afea4e29"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 2016 words (66 misses)\n"
     ]
    }
   ],
   "source": [
    "num_tokens = len(vocab) + 2\n",
    "embedding_dim = 100\n",
    "hits = 0\n",
    "misses = 0\n",
    "\n",
    "embedding_matrix = np.zeros((num_tokens, embedding_dim))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "        hits += 1\n",
    "    else:\n",
    "        misses += 1\n",
    "print(\"Converted %d words (%d misses)\" % (hits, misses))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sjFao1X_G1SA"
   },
   "source": [
    "## Building model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "S83_eEdSA9y7"
   },
   "outputs": [],
   "source": [
    "def create_model(max_features, embedding_dim):\n",
    "    \n",
    "  num_classes = 19\n",
    "      \n",
    "  inputs = tf.keras.Input(shape=(None,), dtype=\"int64\")\n",
    "  x = tf.keras.layers.Embedding(num_tokens, \n",
    "                                embedding_dim, \n",
    "                                embeddings_initializer= tf.keras.initializers.Constant(embedding_matrix),\n",
    "                                trainable=False\n",
    "                                )(inputs)\n",
    "  \n",
    "  x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128, \n",
    "                                                         return_sequences=True, \n",
    "                                                         dropout=0.1,\n",
    "                                                         recurrent_dropout=0.1))(x)\n",
    "  \n",
    "  x = tf.keras.layers.Conv1D(64, kernel_size=3, padding=\"valid\", kernel_initializer=\"glorot_uniform\")(x)\n",
    "  x = tf.keras.layers.BatchNormalization()(x)\n",
    "  x = tf.keras.layers.Conv1D(32, kernel_size=3, padding=\"valid\", kernel_initializer=\"glorot_uniform\")(x)\n",
    "  x = tf.keras.layers.BatchNormalization()(x)\n",
    "  \n",
    "  x = tf.keras.layers.GlobalMaxPooling1D()(x)\n",
    "\n",
    "  preds = tf.keras.layers.Dense(num_classes, activation=\"softmax\")(x)\n",
    "\n",
    "  model = tf.keras.Model(inputs, preds)\n",
    "\n",
    "  return model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2rOrLgm_G69A"
   },
   "source": [
    "## Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FS_LW1ThBfiK"
   },
   "outputs": [],
   "source": [
    "model = create_model(max_features, embedding_dim)\n",
    "model.compile(\n",
    "    optimizer='adam',\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kKcwhaZj8b1n",
    "outputId": "87947fbd-7152-42c8-f90a-3cd05989e61e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "14/14 [==============================] - 28s 2s/step - loss: 3.6891 - accuracy: 0.0915 - val_loss: 3.3183 - val_accuracy: 0.0943\n",
      "Epoch 2/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 2.4938 - accuracy: 0.2136 - val_loss: 2.9485 - val_accuracy: 0.0943\n",
      "Epoch 3/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 1.9737 - accuracy: 0.4061 - val_loss: 2.6677 - val_accuracy: 0.1132\n",
      "Epoch 4/30\n",
      "14/14 [==============================] - 20s 1s/step - loss: 1.5329 - accuracy: 0.5446 - val_loss: 2.4942 - val_accuracy: 0.2642\n",
      "Epoch 5/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 1.1833 - accuracy: 0.6667 - val_loss: 2.3439 - val_accuracy: 0.3585\n",
      "Epoch 6/30\n",
      "14/14 [==============================] - 20s 1s/step - loss: 0.8604 - accuracy: 0.7840 - val_loss: 2.2872 - val_accuracy: 0.4434\n",
      "Epoch 7/30\n",
      "14/14 [==============================] - 20s 1s/step - loss: 0.6103 - accuracy: 0.8662 - val_loss: 2.2136 - val_accuracy: 0.6321\n",
      "Epoch 8/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.5211 - accuracy: 0.8944 - val_loss: 2.1697 - val_accuracy: 0.5755\n",
      "Epoch 9/30\n",
      "14/14 [==============================] - 20s 1s/step - loss: 0.3466 - accuracy: 0.9366 - val_loss: 2.0595 - val_accuracy: 0.6038\n",
      "Epoch 10/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.2602 - accuracy: 0.9671 - val_loss: 1.9793 - val_accuracy: 0.7358\n",
      "Epoch 11/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.1901 - accuracy: 0.9883 - val_loss: 1.9395 - val_accuracy: 0.6887\n",
      "Epoch 12/30\n",
      "14/14 [==============================] - 21s 2s/step - loss: 0.1625 - accuracy: 0.9836 - val_loss: 1.8441 - val_accuracy: 0.6981\n",
      "Epoch 13/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.1145 - accuracy: 0.9977 - val_loss: 1.7819 - val_accuracy: 0.7264\n",
      "Epoch 14/30\n",
      "14/14 [==============================] - 22s 2s/step - loss: 0.0932 - accuracy: 0.9977 - val_loss: 1.7228 - val_accuracy: 0.7075\n",
      "Epoch 15/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0686 - accuracy: 1.0000 - val_loss: 1.5935 - val_accuracy: 0.7358\n",
      "Epoch 16/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0535 - accuracy: 1.0000 - val_loss: 1.5582 - val_accuracy: 0.7264\n",
      "Epoch 17/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0476 - accuracy: 1.0000 - val_loss: 1.5032 - val_accuracy: 0.7358\n",
      "Epoch 18/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0428 - accuracy: 1.0000 - val_loss: 1.3899 - val_accuracy: 0.7642\n",
      "Epoch 19/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0364 - accuracy: 1.0000 - val_loss: 1.3344 - val_accuracy: 0.7358\n",
      "Epoch 20/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0306 - accuracy: 1.0000 - val_loss: 1.3081 - val_accuracy: 0.7547\n",
      "Epoch 21/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0286 - accuracy: 1.0000 - val_loss: 1.2680 - val_accuracy: 0.7453\n",
      "Epoch 22/30\n",
      "14/14 [==============================] - 20s 1s/step - loss: 0.0277 - accuracy: 1.0000 - val_loss: 1.2204 - val_accuracy: 0.7453\n",
      "Epoch 23/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0247 - accuracy: 1.0000 - val_loss: 1.1888 - val_accuracy: 0.6981\n",
      "Epoch 24/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0216 - accuracy: 1.0000 - val_loss: 1.1494 - val_accuracy: 0.7264\n",
      "Epoch 25/30\n",
      "14/14 [==============================] - 21s 1s/step - loss: 0.0199 - accuracy: 1.0000 - val_loss: 1.1256 - val_accuracy: 0.7453\n",
      "Epoch 26/30\n",
      "14/14 [==============================] - 21s 2s/step - loss: 0.0184 - accuracy: 1.0000 - val_loss: 1.0750 - val_accuracy: 0.7264\n",
      "Epoch 27/30\n",
      "14/14 [==============================] - 21s 2s/step - loss: 0.0154 - accuracy: 1.0000 - val_loss: 1.0637 - val_accuracy: 0.7453\n",
      "Epoch 28/30\n",
      "14/14 [==============================] - 21s 2s/step - loss: 0.0144 - accuracy: 1.0000 - val_loss: 1.0585 - val_accuracy: 0.7453\n",
      "Epoch 29/30\n",
      "14/14 [==============================] - 21s 2s/step - loss: 0.0136 - accuracy: 1.0000 - val_loss: 1.0606 - val_accuracy: 0.7170\n",
      "Epoch 30/30\n",
      "14/14 [==============================] - 21s 2s/step - loss: 0.0131 - accuracy: 1.0000 - val_loss: 1.0192 - val_accuracy: 0.7358\n"
     ]
    }
   ],
   "source": [
    "epochs = 30\n",
    "es = tf.keras.callbacks.EarlyStopping(patience=2)\n",
    "history = model.fit(\n",
    "    train_ds, \n",
    "    validation_data= val_ds, \n",
    "    callbacks= [es],\n",
    "    epochs= epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 563
    },
    "id": "jXXPUVYRHJNx",
    "outputId": "63e12bc5-5d4c-4c6a-b4ff-399411a465b8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Training and validation loss')"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEICAYAAACgQWTXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVf74/9c7nYQQQi8BQk/oJTRRBLEAVhQVFBV3XT6L665+tnysa1vdn+u6rr/dtayuXQERGygiuiJF6SWANKlJCJAQkpCQnpzvH2cCQ0iZJDOZzOT9fDzyyMzcO+e+70zynjPve+65YoxBKaWUfwjwdgBKKaXcR5O6Ukr5EU3qSinlRzSpK6WUH9GkrpRSfkSTulJK+RFN6n5MRL4UkTvcva43icghEbnUA+0aEenluP2KiPzRlXXrsJ1bRWRZXeNUqiai49QbFxHJdbobDhQCpY77/2OMeb/ho2o8ROQQcJcx5hs3t2uA3saYfe5aV0RigYNAsDGmxB1xKlWTIG8HoM5ljGlefru6BCYiQZooVGOhf4+Nh5ZffISIjBeRFBG5X0SOAW+KSLSIfC4i6SKS6bgd4/Sc70TkLsftWSKyWkSec6x7UEQm13Hd7iKyUkRyROQbEXlRRN6rIm5XYvyTiHzvaG+ZiLRxWn6biBwWkQwRebia12eUiBwTkUCnx6aKyDbH7ZEiskZEskTkqIj8S0RCqmjrLRF5yun+HxzPSRWRn1VY90oR2SIip0QkWUQed1q80vE7S0RyRWRM+Wvr9PwLRGSDiGQ7fl/g6mtTy9e5lYi86diHTBH51GnZtSKy1bEP+0VkkuPxc0pdIvJ4+fssIrGOMtTPRSQJ+Nbx+IeO9yHb8TfS3+n5zUTkb473M9vxN9ZMRL4QkV9X2J9tIjK1sn1V1dOk7ls6AK2AbsBs7Pv3puN+VyAf+Fc1zx8F7AHaAM8Cr4uI1GHducB6oDXwOHBbNdt0JcZbgDuBdkAI8HsAEekHvOxov5NjezFUwhizDjgNXFKh3bmO26XA/zr2ZwwwEbi7mrhxxDDJEc9lQG+gYj3/NHA70BK4EpgjItc5lo1z/G5pjGlujFlToe1WwBfAPxz79jzwhYi0rrAP5702lajpdX4XW87r72jr744YRgLvAH9w7MM44FBVr0clLgbigSsc97/Evk7tgM2Ac7nwOWA4cAH27/j/gDLgbWBm+UoiMhjojH1tVG0ZY/Snkf5g/7kuddweDxQBYdWsPwTIdLr/HbZ8AzAL2Oe0LBwwQIfarItNGCVAuNPy94D3XNynymJ8xOn+3cBSx+1HgflOyyIcr8GlVbT9FPCG43YkNuF2q2Ld+4BPnO4boJfj9lvAU47bbwDPOK3Xx3ndStp9Afi743asY90gp+WzgNWO27cB6ys8fw0wq6bXpjavM9ARmzyjK1nv3+XxVvf357j/ePn77LRvPaqJoaVjnSjsh04+MLiS9cKATOxxCrDJ/6WG/n/zlx/tqfuWdGNMQfkdEQkXkX87vs6ewn7db+lcgqjgWPkNY0ye42bzWq7bCTjp9BhAclUBuxjjMafbeU4xdXJu2xhzGsioalvYXvn1IhIKXA9sNsYcdsTRx1GSOOaI48/YXntNzokBOFxh/0aJyHJH2SMb+KWL7Za3fbjCY4exvdRyVb0256jhde6Cfc8yK3lqF2C/i/FW5sxrIyKBIvKMo4RzirM9/jaOn7DKtuX4m/4AmCkiAcAM7DcLVQea1H1LxaFKvwP6AqOMMS04+3W/qpKKOxwFWolIuNNjXapZvz4xHnVu27HN1lWtbIzZiU2Kkzm39AK2jLMb2xtsATxUlxiw31SczQUWAV2MMVHAK07t1jS0LBVbLnHWFTjiQlwVVfc6J2Pfs5aVPC8Z6FlFm6ex39LKdahkHed9vAW4FluiisL25stjOAEUVLOtt4FbsWWxPFOhVKVcp0ndt0Viv9JmOeqzj3l6g46e70bgcREJEZExwNUeinEhcJWIXOg4qPkkNf/NzgXuxSa1DyvEcQrIFZE4YI6LMSwAZolIP8eHSsX4I7G94AJHffoWp2Xp2LJHjyraXgL0EZFbRCRIRG4G+gGfuxhbxTgqfZ2NMUexte6XHAdUg0WkPOm/DtwpIhNFJEBEOjteH4CtwHTH+gnANBdiKMR+mwrHfhsqj6EMW8p6XkQ6OXr1YxzfqnAk8TLgb2gvvV40qfu2F4Bm2F7QWmBpA233VuzBxgxsHfsD7D9zZeocozHmR+BX2ER9FFt3TanhafOwB+++NcaccHr899iEmwO85ojZlRi+dOzDt8A+x29ndwNPikgO9hjAAqfn5gFPA9+LHXUzukLbGcBV2F52BvbA4VUV4nZVTa/zbUAx9ttKGvaYAsaY9dgDsX8HsoEVnP328EdszzoTeIJzv/lU5h3sN6UjwE5HHM5+D2wHNgAngb9wbg56BxiIPUaj6khPPlL1JiIfALuNMR7/pqD8l4jcDsw2xlzo7Vh8mfbUVa2JyAgR6en4uj4JW0f9tKbnKVUVR2nrbuBVb8fi6zSpq7rogB1ul4sdYz3HGLPFqxEpnyUiV2CPPxyn5hKPqoGWX5RSyo9oT10ppfyI1yb0atOmjYmNjfXW5pVSyidt2rTphDGmbVXLvZbUY2Nj2bhxo7c2r5RSPklEKp6FfA4tvyillB/RpK6UUn5Ek7pSSvmRRnXlo+LiYlJSUigoKKh5ZVWjsLAwYmJiCA4O9nYoSqkG0qiSekpKCpGRkcTGxlL1tRuUK4wxZGRkkJKSQvfu3b0djlKqgdRYfhGRN0QkTUR2VLFcROQfIrLPcQmqYXUNpqCggNatW2tCdwMRoXXr1vqtR6kmxpWa+lvApGqWT8Zevqo39hJrL9cnIE3o7qOvpVJNT43lF2PMShGJrWaVa4F3jJ1vYK2ItBSRjo45nJVqcMYYCorLOF1UwunCEk4XlpJXVEJuYQl5RaX2d2EJp4tKKSwu9Xa4qgmaGN+ewV0qu2ZJ/bmjpt6Zcy/3leJ47LykLiKzsb15unateAEZ78vKymLu3LncfXeN1yM+x5QpU5g7dy4tW1b9Jj366KOMGzeOSy+teN1iVZXSMsOWpEy+3Z3G8j3p7D52Ck9MVaRfaFRDa9cirFEndZcZY17FMbVmQkJCo5tJLCsri5deeum8pF5SUkJQUNUv1ZIlS2ps+8knn6x3fE1B5ukiVuxNZ/meNFbsTScrr5jAACGhWzRzLu5JUKBro3CbBQcSERpIREiQ/R0aRHhIEM1DgwgPCbS/QwMJCQzQMpXyK+5I6kc49xqOMdTtGote98ADD7B//36GDBlCcHAwYWFhREdHs3v3bvbu3ct1111HcnIyBQUF3HvvvcyePRs4O+VBbm4ukydP5sILL+SHH36gc+fOfPbZZzRr1oxZs2Zx1VVXMW3aNGJjY7njjjtYvHgxxcXFfPjhh8TFxZGens4tt9xCamoqY8aM4euvv2bTpk20aePqdYy9yxjDD/szeGP1QTYnZdIqIoR2kWG0axFKu8jQM7fbOt2ODA1i59FTLHf0xrckZVJmoE3zECbGteeSuHZc2LsNUc10WKZSrnBHUl8E3CMi84FRQLY76ulPLP6Rnamn6h2cs36dWvDY1f2rXP7MM8+wY8cOtm7dynfffceVV17Jjh07zgwJfOONN2jVqhX5+fmMGDGCG264gdatz70O8k8//cS8efN47bXXuOmmm/joo4+YOXPmedtq06YNmzdv5qWXXuK5557jP//5D0888QSXXHIJDz74IEuXLuX111936/57SkFxKYu2pvLG9wfZfSyH1hEhXNavPTkFJaTlFLI5KZO0U4UUlpSd99zgQKG41H5pGxQTxa8v6c2EuHYM6hxFQID2oJWqrRqTuojMA8YDbUQkBXtB22AAY8wr2IvnTsFevzEPe71DvzBy5Mhzxnj/4x//4JNPPgEgOTmZn3766byk3r17d4YMGQLA8OHDOXToUKVtX3/99WfW+fjjjwFYvXr1mfYnTZpEdHS0W/fH3dJOFfDe2sO8vy6JjNNFxHWI5Nlpg7hmcCfCggPPWdcYw6mCEtJzCkg7VUhaTiFpOQVk5BbRs11zxvdtS7vIMC/tiVL+w5XRLzNqWG6wFwd2q+p61A0lIiLizO3vvvuOb775hjVr1hAeHs748eMrHQMeGhp65nZgYCD5+fmVtl2+XmBgICUlJW6O3LN2HMnmjdUHWbwtlZIyw8S49vzswljG9Kj6HAMRIapZMFHNgunVLrKBI1aq6WhUZ5R6W2RkJDk5OZUuy87OJjo6mvDwcHbv3s3atRUvlF5/Y8eOZcGCBdx///0sW7aMzMxMt2+jrowx/HdXGq+uOsD6gycJDwnk1lHdmHVBLLFtImpuQCnVIDSpO2ndujVjx45lwIABNGvWjPbt259ZNmnSJF555RXi4+Pp27cvo0ePdvv2H3vsMWbMmMG7777LmDFj6NChA5GR3u/VpmTm8ehnP/Lt7jQ6t2zGI1fGc2NCFz14qVQj5LVrlCYkJJiKF8nYtWsX8fHxXomnMSgsLCQwMJCgoCDWrFnDnDlz2Lp1a73arM9rWlpmeOuHQ/xt2R6Mgd9d3odZF8S6PKxQKeV+IrLJGJNQ1XLtqTciSUlJ3HTTTZSVlRESEsJrr73mtVh2HMnmoU+2sy0lmwl92/Kn6wYQEx3utXiUUq7RpN6I9O7dmy1btng1hvyiUl74Zi//WX2Q6PAQ/nXLUK4c2FFP0FHKR2hSV2es2JvOI59uJ/lkPjNGduGBSfFEhWvdXClfokldcSK3kKc+38mnW1Pp0TaCD2aPZlSP1jU/USnV6GhSb+KW/XiM//toG6cLS7h3Ym/untCT0KDAmp+olGqUNKk3YW9+f5AnP9/JwM5RPH/TYD0pSCk/oGPT6qF58+YApKamMm3atErXGT9+PBWHblb0wgsvkJeXd+b+lClTyMrKcl+gFZSVGf70+U6eWLyTy+Lb88HsMZrQlfITmtTdoFOnTixcuLDOz6+Y1JcsWVLt3Oz1UVBcyq/mbub11QeZdUEsL88cTrMQLbco5S80qTt54IEHePHFF8/cf/zxx3nqqaeYOHEiw4YNY+DAgXz22WfnPe/QoUMMGDAAgPz8fKZPn058fDxTp049Z+6XOXPmkJCQQP/+/XnssccAO0lYamoqEyZMYMKECYCdyvfEiRMAPP/88wwYMIABAwbwwgsvnNlefHw8v/jFL+jfvz+XX355lXPMODt5uohb/7OOL3cc45Er43ns6n4E6kyISvmVxltT//IBOLbdvW12GAiTn6ly8c0338x9993Hr35l5ydbsGABX331Fb/5zW9o0aIFJ06cYPTo0VxzzTVVjtt++eWXCQ8PZ9euXWzbto1hw85eh/vpp5+mVatWlJaWMnHiRLZt28ZvfvMbnn/+eZYvX37evOmbNm3izTffZN26dRhjGDVqFBdffDHR0dEuT/Fb7nDGaWa9uYEjWfm8dOswpgzsWJtXTinlI7Sn7mTo0KGkpaWRmppKYmIi0dHRdOjQgYceeohBgwZx6aWXcuTIEY4fP15lGytXrjyTXAcNGsSgQYPOLFuwYAHDhg1j6NCh/Pjjj+zcubPaeFavXs3UqVOJiIigefPmXH/99axatQpwfYpfgC1JmVz/0g9k5hUx965RmtCV8mONt6deTY/ak2688UYWLlzIsWPHuPnmm3n//fdJT09n06ZNBAcHExsbW+mUuzU5ePAgzz33HBs2bCA6OppZs2bVqZ1yrk7xm19cyi2vraVdZBhv3TmCHm2b13mbSqnGT3vqFdx8883Mnz+fhQsXcuONN5KdnU27du0IDg5m+fLlHD58uNrnjxs3jrlz5wKwY8cOtm3bBsCpU6eIiIggKiqK48eP8+WXX555TlVT/l500UV8+umn5OXlcfr0aT755BMuuugil/flRG4hJ3OL6Ns+ko/vvkATulJNQOPtqXtJ//79ycnJoXPnznTs2JFbb72Vq6++moEDB5KQkEBcXFy1z58zZw533nkn8fHxxMfHM3z4cAAGDx7M0KFDiYuLo0uXLowdO/bMc2bPns2kSZPo1KkTy5cvP/P4sGHDmDVrFiNHjgTgrrvuYujQodWWWsAOWTx2qoATuYWEBQcwb/ZowkP0rVaqKdCpd/2IMYZT+cUczS6gqLSMNs1DyUo9RL9++poq5S906t0moqC4lNSsfHILSwgLDqRHdHOahwWRXe9LgCulfIkmdR9XUlpGWk4hGblFBARA55bNaBURolPlKtVENbqkbozRhOQCYwwnTxdx/FQBpWWGVhGhtG8Res5VibxVWlNKeU+jSuphYWFkZGTQunXVV6VXcLqwhNSsfPKLS4kIDaJTVLPzTvU3xpCRkUFYWJiXolRKeUOjSuoxMTGkpKSQnp7u7VAapdIyQ3Z+MXlFpQQFCFHNgikKCeTQicrXDwsLIyYmpmGDVEp5VaNK6sHBwXTv3t3bYTRad729kZV70/nl+J7MubinTsSllDpPo0rqqmrf7Unjm13HeWByHL+8uKe3w1FKNVJ6RqkPKCop48nPd9K9TQR3jo31djhKqUZMk7oPeOuHgxxIP82jV/XTS80ppaqlSb2RSztVwP//zU9MjGvHhLh23g5HKdXIaVJv5P6ydA/FpYY/XtXP26EopXyAJvVGbNPhTD7anMLPL+pObJsIb4ejlPIBmtQbqbIyw+OLfqR9i1DumdDL2+EopXyEJvVG6sNNyWw/ks1DU+KJCNWRp0op12hSb4Sy84t5dukeErpFc83gTt4ORynlQ7QL2Ai98M1eTuYV8fY1I3UOHKVUrWhPvZHZezyHd9Yc5paRXRnQOcrb4SilfIxLSV1EJonIHhHZJyIPVLK8q4gsF5EtIrJNRKa4P1T/Z4w9ONo8NIjfXd7X2+EopXxQjUldRAKBF4HJQD9ghohUHDT9CLDAGDMUmA685O5Am4KlO47xw/4Mfnd5H1pFhHg7HKWUD3Klpz4S2GeMOWCMKQLmA9dWWMcALRy3o4BU94XYNOQXlfLUF7uI6xDJLSO7ejscpZSPcuVAaWcg2el+CjCqwjqPA8tE5NdABHBpZQ2JyGxgNkDXrpq4nL2yYj9HsvKZP3v0OVcvUkqp2nBX9pgBvGWMiQGmAO+KyHltG2NeNcYkGGMS2rZt66ZN+77kk3m8smI/Vw3qyOgerb0djlLKh7mS1I8AXZzuxzgec/ZzYAGAMWYNEAa0cUeATcGfl+wiQISHpsR7OxSllI9zJalvAHqLSHcRCcEeCF1UYZ0kYCKAiMRjk7pek84F+9Jy+HLHMWaP60Gnls28HY5SysfVmNSNMSXAPcBXwC7sKJcfReRJEbnGsdrvgF+ISCIwD5hl9FL2LnlvbRLBgcJtY7p5OxSllB9w6YxSY8wSYEmFxx51ur0TGOve0PxfXlEJH21OYfKAjrRpHurtcJRSfkCHWXjR4sRUcgpKmDlae+lNUn4mlJZ4OwrlZzSpe4kxhnfXHqZv+0hGxEZ7OxzVkE6fgEW/gb90h1fGwoHvvB2R8iOa1L0kMSWbHUdOMXN0V520q6koLYF1/4Z/DoMt78Gw26CkAN65Fj6YCZmH67+NvJNQUlT/dpTP0lkaveS9tYcJDwnkuqGdvR2KaggHVsDSByBtJ3S/GCY/C+3ioLgA1vwTVj0PP30NY++DsfdCSLjrbZcUwZ4vYPM7sH85BARB+37QcbDjZwi07w/BOrqqKdCk7gVZeUUsTkzlhuExRIYFezsc5UlZSbDsEdj5GbTsCje/B3FXQfm3s+AwGPcHGDwDlv0RVjwDW9+HK56G+GvOrleZ9D02kSfOg7wMaBEDF/0WykrgaCLsWmyXA0ggtI1zSvSDoXVPOP8cwfMFBEIzL5UIy8rAlEGgG1NVWRnkn3RtXQmA8Fbu23YD0KTuBQs3pVBYUsbMUXqA1G8V58P3/4DVf7f3JzwMF/y66t5yVAzc+CaM+Dl8eT8suB26j3P06J1OSis6DT9+apN18lrbK+87BYbdAT0n2ARczhjITrYJ/mgipG6FfV9D4tza78+Iu2DyXyGggSq22Udg61zY8g7kpkP/qTD8DugyqvoPuhrbfB+2vGs/bF3V61KY9Ay06V237TYw8dZw8oSEBLNx40avbNubysoME59fQauIED6ac4G3w6kdY+r+D9VYuXufjLE95K8ehuwkm4wu+xO07FLzc8uVlsCmN+Hbp6AwB0b+AvpdC9sWwPaFUJQDrXvDsNttD795LabcMAZyjtkkn51c8/oAx7bZD5Ght8HV//BcYi8thp+Wwaa37YePKbOlqpZd7AdZUS606XN2vyNcOGm9tBj2fmXjd26zzyQIdOFb8ul0WPsyFOfB6Dkw7v8grEXNz6tOfhYEhda5HCYim4wxCVUu16TesFb/dIKZr6/j7zcPZurQGG+H47riAvj3OOgx3vZaGqrH5inpe2Dpg3D4+wpliSG2Hu3KP5wxtsdX3hMu/zmdBu36weS/2N52XeWdhG//BBvfBAwENbMfEsNuh66jG+4D1hhY/mdY+axNpte+eO43gvrK2G97z1vnQu5xaN4Bht5qP0RadbfrFObCzvJvKOsgIBjiHN9Qekw4/+8xY79dd+tc+35EdoQht8LQmWfbdFVuGvz3CXtwu3l7uPQJGHRz7f4HjIHDP9iYdn4KVz5v97EONKk3Mr98dxPrDmaw5sGJhAW78R/D03Z8DAvvtLeHznT02Hwo/nIF2bDiWVj3CgRHwMAbbAI4mggFWXadyurP7fvbXlvqlnMTeGXPiR0Lg6a7rw58bIc9wNrnCgjz4tWwVjwLy5+GAdNg6r/rt3/F+Wdr/odW2devzxX2A6vXZdW3nbb77LGE/JMQ1dX+TQ6cBkc22Z7+4dVObd5hSyj1fT9SNsGXf7DbiBlpP7Q7D6v+Oblp9oNl8ztwcj+EtoCBN8LI2fZAeR1oUm9EjmUXMPYv33LXhd150Ncm75p7s01iQ2fCyr/apHXdSw2T2LOS4OtH7T9Tv2vtP2lt65tlZbaW/M3jdpz4sNth4qNnv8JX2uveahN5RYEhtidel969r1v1vO219rsWbnjdtRKGs7JS2Py2LS3lZUB0rKOccgu06Fi7tkoKYbdj1M+B5Wcfr0+bNSkrsx8m3zxu/zaG3QaXPHpuCaysFPb91+7n3qX2wHXXC2xM/a6t3cimStSU1PVAaQOavyGJ0jLDLaN8bC753HQ73O6Ce+CSRyAwFJY/BWXFMPVV945McFbxYGPXUbDmJfjhn7X7J6nYw7r1Q+g09Nx1RCC6m/3p55jSyBjIOWoT/PEd9qt3x8HQNh6CmuiVqS76rf1QW/awTV7T3nT9tTi8xr4Px7ZD7EUw7vcQO67upbygUBhwvf3JPGQTfIeB0O1Cz5UHAwJs2ST+qrPf+H78DCY8CL0vh20f2DLNqSMQ3gZG323/ThvwIKv21BtIcWkZF/7lW+I6tODtn430dji1s/YVWHo/zFlje6QAq1+Abx6zw+6mvVH7Hlt1qjvYmHPc9rg3vwMnD5z9Ojvsdug05Nx2ctPgmydgq6MWetmTMPAm3z8e0Bise9Um6N5XwE3v2KGZVTmVCl8/BtsX2GGXVzwF/a7zj4Pu6Xvt/8b+bx0PiC31DLvdHoz1wIe/ll8aiaU7jvLL9zbz2u0JXNavvbfDqZ1Xx9te2S9Xnfv4mhfhq4eg75V2OF6QGyYlS9tlh/QdXAHt+jsONl50/nrG2IOcm9+xY8BLCmwvetjt0G+q/Yq84i+2tz96Dlz8fxAaWf/41Fkb34DP/xd6ToTp759ffioptH8jK5+zJYix98KF90FIhHfi9RRj7AibE3ttB6Q2I53qQJN6IzHzP+s4kJ7LqvsvITDAh3oo6XvgxZFwxZ9hzK/OX77+NVjye9d6bNXJz7JJeN2/bfK95BEYfqdrpZ38TNj2oa1hHt9x9nEfG1/skza/C4t+bT94Z8w/m7D3LLVn0GYetCdbXfG0rXWretOaeiNw8MRpVu87we8u6+NbCR0gcb4dRTBgWuXLR/7CngDz+X0wfwZMn1u7A4ZlZbY88s0T9sDZ8FlwyR8hohaX9WsWDaNm21hSt8CuRdBltB354A9f8RuzYbfZ0tunc+D9G+2H//Kn7XjzNn1g5sfQa6K3o2xSNKk3gPfXHiYoQLh5pGe/lrldWZk94aXnJRBZTcko4U77j/3ZPTD3pnN7bBUVF9jheeUjTA7/ACf22CR828e2hFJXInaIWU3DzJR7DZ5uP9g/ng2vXgwhkXD50zDqf9x7rEW5RJO6hxUUl/LhphSuGNCBdpF1LE14y+HVcCoFLnui5nWHzrQnhHz6S3hvGty6wM6bcfxHe3p6eRJP32XrqwChUdBxkB0FMfBG7VX7soHT7Af5wVW2dl5dJ0B5lCZ1D/t821Gy84t9c56XxPm21xV3pWvrD77Z1sA/+gW8MNCe6GPK7LLw1nY8d+/Lzo7vjo7VRO5P+k62P8qrNKl72LtrD9OrXXNG9/Ctmd4oyrOjSvpfV7sa+YAb7JmaifOgbd+zCbxFZ03gSjUATeoetONINonJWTx2dT/fuxDG7i/sBEqDptf+uX0n2R+lVIPTszA86L21h2kWHMj1w3xo4q5y2+ZDVBfoptcTV8qXaFL3kFMFxXy2NZVrh3QiqpmPjQDIOWbPkBukZ18q5Wv0P9ZDPtl8hPziUmaO9sEDpNsX2gOcdSm9KKW8SpO6BxhjmLc+iUExUQzo7MWpUusqcT50GgZt+3g7EqVULWlS94AtyVnsPpbD9BE+Nhsj2Lm7j2+3F0NQSvkcTeoeMH99EuEhgVwzpJO3Q6m9bfPt2YEDbvB2JEqpOtCk7mY5BcUsTjzKNYM70TzUx0aMlpXaibF6X167uVeUUo2GJnU3W5SYSn5xKdNH+mDp5cB3kHvMXn9RKeWTNKm72fz1ycR1iGRwjA8eIN32gb0GZh89cUgpX6VJ3Y12HMlm+5FsZozs6pkzSHPT7JmenlCYa6821H9q3edEV0p5nSZ1N5q3PonQoACuG9LZMxtY/XeYf4u9lJi77VoMxXk66kUpH6dJ3U3yikr4bGsqVw7sSKXvxukAABV4SURBVFS4h84gPbjS/l56P+xd5t62E+fZWRO7jHJvu0qpBqVJ3U0+33aU3MISZozy0AHS0yfspdou/C207w8L77Rjyt0h+4j9wBg0XWdSVMrHaVJ3k/nrk+jVrjkJ3aI9s4FDq+3vvlNgxgf2Op5zb4ac4/Vve/sCwNi5XpRSPs2lpC4ik0Rkj4jsE5EHqljnJhHZKSI/ishc94bZuO05lsPmpCymj+jiuSl2D66EkObQaQhEdYZbPoD8kzBvup37vK6MsdMCdBkFrXu6L16llFfUmNRFJBB4EZgM9ANmiEi/Cuv0Bh4Exhpj+gP3eSDWRmve+iRCAgM8O8XuoVXQdczZaz52HAw3vG4vtPzJ/9jridbF0URI361j05XyE6701EcC+4wxB4wxRcB84NoK6/wCeNEYkwlgjElzb5iNV0FxKZ9sOcLl/dvTKiLEMxs5dRRO7IXu4859PG4KXPE07FoE/3XhOqLOyspg87vw/jQIDrdDGZVSPs+VpN4ZSHa6n+J4zFkfoI+IfC8ia0Wk0rNXRGS2iGwUkY3p6el1i7iRWbrjGNn5xczw5Bmk5fX07hedv2z03ZDwM/j+Bdj8jmvtJW+A/0yERfdAqx7ws6UQ7mOX21NKVcpdk5MEAb2B8UAMsFJEBhpjspxXMsa8CrwKkJCQYNy0ba+atz6Jrq3CGdPDg3OlHFppz/TsMOj8ZSIw+VnIPASf/y+07AY9Lq68nZzj8M3jkDgXmneA61+DgTfqiBel/IgrPfUjQBen+zGOx5ylAIuMMcXGmIPAXmyS92sH0nNZd/Ak00d2ISDAg4nx4ErodiEEBFa+PDAYbnwLWveCBbdB+t5zl5cUwQ//hH8Oh+0fwtj74Ncb7WgXTehK+RVXkvoGoLeIdBeREGA6sKjCOp9ie+mISBtsOeaAG+NslD7YkExQgDBtuAcPkGYl2V54xXp6RWFRcMsCCAyBuTfC6Qz7+L5v4OULYNkj0G0M/GodXPaEHRKplPI7NZZfjDElInIP8BUQCLxhjPlRRJ4ENhpjFjmWXS4iO4FS4A/GmAxPBu5tRSVlLNyUwsT4drSL9OBcKQdX2d+V1dMriu4G0+fBW1fC/BkQ3hr2LLF181sWQJ8rPBenUqpRcKmmboxZAiyp8NijTrcN8FvHT5Pw9c7jZJwu8vwUu4dWQXgbaBvv2vpdRsDUl2HhzyA4Ai593B5MDQr1ZJRKqUbCx67i0HjM35BE55bNGNe7rec2Yoytp8deCAG1OPl3wA3QorM9aNqio+fiU0o1OjpNQB0kn8xj1U8nuDEhhkBPHiA9eQBOHXGt9FJR19Ga0JVqgjSp18EHG5IJELgpoUvNK9dH+ayM3asYoqiUUhVoUq+lktIyFmxMZnzfdnRq2cyzGzu0yo4nb93Ls9tRSvkNTeq19O3uNNJyCpk+wsO9dGPsyJfu43QsuVLKZZrUa2n+hmTaRYZySVw7z24ofQ+cTqtbPV0p1WRpUq+F9JxCvtuTxrThMQQFevilO1NPr+GkI6WUcqJJvRa+2JZKmYHrhnroGqTODq2EqK72EnNKKeUiTeq1sCgxlbgOkfRp7+FT7MvK7MyM2ktXStWSJnUXJZ/MY3NSFtcM6eT5jR3fAfmZWk9XStWaJnUXLd6WCsDVgxogqR9yzPcSq0ldKVU7mtRdtGhrKsO6tqRLq3DPb+zgSmjV016LVCmlakGTugv2Hs9h97EcrhncAL300hI4/IOWXpRSdaJJ3QWLtqYSIHBlQ5RejiZC4Sk9SKqUqhNN6jUwxrAoMZULerahbWQDTF97yDE+XevpSqk60KReg8SUbJJO5jXMqBew9fS28dDcw2esKqX8kib1GizamkpIYABX9O/g+Y2VFEHSWq2nK6XqTJN6NUrLDIu3pTK+b1uimgV7foNHNkFxntbTlVJ1pkm9GusOZJCeU9hwpZdDqwCBbmMbZntKKb+jSb0aixJTiQgJZGJc+4bZ4MGV0GEAhLdqmO0ppfyOJvUqFJWU8eWOY1zWrz3NQgI9v8HiAkher1c5UkrViyb1Kqzcm052fnHDlV5S1kNpoQ5lVErViyb1KixKTKVleDAX9mrbMBs8uBIkELpd0DDbU0r5JU3qlcgrKuHrnceZMrAjIUEN9BIdXAWdhkBYi4bZnlLKL2lSr8TXO4+TX1zaMHO9ABTmwpGNOpRRKVVvmtQrsTgxlQ4twhgZ20CjUJLXQlmJ1tOVUvWmSb2CrLwiVuxN56pBHQkIkIbZ6MGVEBAMXUc3zPaUUn5Lk3oFS3cco7jUNNyoF7D19JgECIlouG0qpfySJvUKFiWmEts6nIGdoxpmg0e3QeoW6HlJw2xPKeXXNKk7STtVwJoDGVwzuBMiDVB6MQaWPgjNomHkbM9vTynl9zSpO/l821GMoeFKL7s/h8OrYcJD0Kxlw2xTKeXXNKk7+SwxlX4dW9CrXaTnN1ZSCMv+CG3jYPidnt+eUqpJ0KTucDjjNInJWQ3XS1//KmQehCuehsCghtmmUsrvaVJ3WJyYCsDVDXHC0ekTsOJZ6HUZ9LrU89tTSjUZmtQdFiWmktAtms4tm3l+Y8v/DEWnbS9dKaXcyKWkLiKTRGSPiOwTkQeqWe8GETEikuC+ED1v97FT7D2e2zCll+M7YdObMOLn0Lav57enlGpSakzqIhIIvAhMBvoBM0SkXyXrRQL3AuvcHaSnLd1xjACBKQM7enZDxsCyhyE0EsY/6NltKaWaJFd66iOBfcaYA8aYImA+cG0l6/0J+AtQ4Mb4GsSa/Rn07xRFm+ahnt3QT1/D/m/h4gf06kZKKY9wJal3BpKd7qc4HjtDRIYBXYwxX1TXkIjMFpGNIrIxPT291sF6QkFxKVuSsxjdw8NJtrQYvnoIWveCEXd5dltKqSar3gdKRSQAeB74XU3rGmNeNcYkGGMS2rZtoItP1GBLUhZFJWWM7tHasxva+AZk/ASXPwVBIZ7dllKqyXIlqR8Bujjdj3E8Vi4SGAB8JyKHgNHAIl85WLrmQAYBAgmenGY376Qd8dJjPPSZ5LntKKWaPFeS+gagt4h0F5EQYDqwqHyhMSbbGNPGGBNrjIkF1gLXGGM2eiRiN1t7wNbTo5oFe24jK56FwlNwxZ+hIeaUUUo1WTUmdWNMCXAP8BWwC1hgjPlRRJ4UkWs8HaAnFRSXsjXJw/X0Ez/Bhtdg2O3Qvr/ntqOUUoBL56cbY5YASyo89mgV646vf1gNY3NSJkWlHq6nL3sEgprBhEc8tw2llHJo0meUrj1wkgCBEd091FPf/y3sXQrjfg/NG8eBYaWUf2viST2DAZ2jaBHmgXp6aQl89TBEx8LoOe5vXymlKtFkk/rZerqHSi9b34O0nXDZkxDk4ZOalFLKockm9c2Hy+vpHii9FBfYES8xIyHep48lK6V8TJOdyHutJ8enb34bTh2B617SIYxKqQbVZHvqaw+c9Ew9vSgPVv0Nul0I3S92b9tKKVWDJpnU84tK2ZqcxRhP1NM3vg65x+11R7WXrpRqYE0yqW/x1Pj0wlxY/YKdDiB2rHvbVkopFzTJpH62nh7t3obXvwp5J/REI6WU1zTRpH6SgZ2jiHRnPb3gFPzwD+h9OXQZ4b52lVKqFppcUi+vp7u99LL2ZcjPtLV0pZTykiaX1D0y30t+Jqx5EeKugk5D3deuUkrVUpNL6msPZBAYIO6tp//wLyjM1uuOKqW8rkkm9QHurKefzoB1r0D/qdBhgHvaVEqpOmpSSf1sPd2NZ5F+/wIUnbYXk1ZKKS9rUkl9c1ImxaXGffX0nOOw/jUYeCO0i3NPm0opVQ9NKqmfqad3c1M9ffXfobQIxmsvXSnVODS5pO62evqpVNj4BgyeAa171r89pZRygyaT1N1eT1/1NzClcPEf3NOeUkq5QZNJ6psO23q6WybxykqCTW/D0NvslY2UUqqRaDJJ/ez4dDf01Ff+1c7AOO739W9LKaXcqEkl9YGdo2geWs/rgpw8AFveh+GzICrGLbEppZS7NImknldUQmKKm+Z7WfEsBAbDRb+rf1tKKeVmTSKpbz6c5RifXs/SS95J2LYAht8JkR3cE5xSSrlRk0jqbqun711qR7wMusk9gSmllJs1maQ+KMYN9fRdi6FFjM7EqJRqtPw+qbutnl6YC/v+C/FX67VHlVKNlt8n9fLx6fVO6vu+htJCm9SVUqqR8vuk7rb5XnYthvA20HW0ewJTSikPaAJJ/SSDYqKIqE89vaQQ9i6DuCkQEOi+4JRSys38OqnnFZWQ6I7rkR5YAUU5EH+NewJTSikP8eukvulwJiVlbqin71oEoS2g+zj3BKaUUh7i10l97YEMgupbTy8tgT1LoM8VEBTqvuCUUsoD/Dypu6GenrQG8jJ01ItSyie4lNRFZJKI7BGRfSJy3mV+ROS3IrJTRLaJyH9FpJv7Q62dtFMFbE3OYmyvNvVraPfnEBQGvS51T2BKKeVBNSZ1EQkEXgQmA/2AGSLSr8JqW4AEY8wgYCHwrLsDra0PN6VQWma4flg9ZlI0xg5l7DkRQiLcF5xSSnmIKz31kcA+Y8wBY0wRMB+41nkFY8xyY0ye4+5awKtz0paVGT7YkMzoHq3o3qYeyTh1M5w6oqUXpZTPcCWpdwaSne6nOB6rys+BLytbICKzRWSjiGxMT093PcpaWnMgg6STecwY2bV+De1aDAFB9iCpUkr5ALceKBWRmUAC8NfKlhtjXjXGJBhjEtq2bevOTZ9j3vokWoYHc0X/ekyPawzsXASxF0G4m65rqpRSHuZKUj8CdHG6H+N47BwicinwMHCNMabQPeHVXkZuIV/9eIypQzsTFlyPsz/Td8PJ/RB/lfuCU0opD3MlqW8AeotIdxEJAaYDi5xXEJGhwL+xCT3N/WG67uPNRyguNW4ovXwOCMRpUldK+Y4ak7oxpgS4B/gK2AUsMMb8KCJPikj5efN/BZoDH4rIVhFZVEVzHmWMYd6GJIZ3i6ZP+8j6NbZrEXQZqVc4Ukr5FJfOyjHGLAGWVHjsUafbjWIQ94ZDmRxIP81fp/WsX0OZh+DYNrj8KbfEpZRSDcWvziidvz6JyNAgrhzUsX4N7frc/tbSi1LKx/hNUs/OK+aL7Ue5dmgnwkPccNm69gOhVXf3BKeUUg3Eb5L6J1tSKCwpY/qIeh4gzTkOyev0hCOllE/yi6RujGH+hmQGdo5iQOeo+jW25wvAaFJXSvkkv0jqW5Oz2H0sp/7DGMGWXlr1hHbx9W9LKaUamF8k9fnrkwkPCeSaIZ3q11B+JhxcaU84EnFPcEop1YB8PqnnFpaweFsqVw/qRPP6zJsOsPcrKCvRy9YppXyWzyf1RVtTySsqZfrILjWvXJNdiyGyE3QaVv+2lFLKC3w+qc9bn0Rch0iGdGlZv4aKTsO+/9rSS4DPvyxKqSbKp7PXjiPZbD+SzfQRXZD61sD3/RdK8nXUi1LKp/l0Up+/IYnQoACmDnXDNTl2LYZmraDrBfVvSymlvMRnk3peUQmfbUnlyoEdiQoPrl9jJUX2IGnfKRBYz4OtSinlRT6b1L/YdpScwhKm13dsemkJbH4bCrO19KKU8nk+2y2dvyGZnm0jGBEbXbcGMg/Blvdgy/uQkwqte0OP8W6MUCmlGp5PJvW9x3PYdDiTh6fE1+4AaUkh7P4cNr8DB74DCYBel8GUv9rrkAbWs4yjlFJe5pNJfd76JIIDheuHVXf9ayfHd8KWdyFxnj1rNKorTHgYhtwCUW44yKqUUo2EzyX1wsMbidj8Dv9fpwhabz9Y/colhbBnCaRsgIBgOwZ92O3QfbyORVdK+SWfS+r71i/h9+ZtSMdeYK8mbfrC5U/D4OkQ0cbT4SmllFf5XFI/Fncn/z49jhduGkJAQE31dIHQSJ2cSynVZPhcUp84sAsTB7phnhellPJDWlhWSik/okldKaX8iCZ1pZTyI5rUlVLKj2hSV0opP6JJXSml/IgmdaWU8iOa1JVSyo+IMcY7GxZJBw7X8eltgBNuDKcx8Ld98rf9Af/bJ3/bH/C/fapsf7oZY9pW9QSvJfX6EJGNxpgEb8fhTv62T/62P+B/++Rv+wP+t0912R8tvyillB/RpK6UUn7EV5P6q94OwAP8bZ/8bX/A//bJ3/YH/G+far0/PllTV0opVTlf7akrpZSqhCZ1pZTyIz6X1EVkkojsEZF9IvKAt+OpLxE5JCLbRWSriGz0djx1ISJviEiaiOxweqyViHwtIj85fkd7M8baqGJ/HheRI473aauITPFmjLUlIl1EZLmI7BSRH0XkXsfjPvk+VbM/Pvs+iUiYiKwXkUTHPj3heLy7iKxz5LwPRCSk2nZ8qaYuIoHAXuAyIAXYAMwwxuz0amD1ICKHgARjjM+eMCEi44Bc4B1jzADHY88CJ40xzzg+fKONMfd7M05XVbE/jwO5xpjnvBlbXYlIR6CjMWaziEQCm4DrgFn44PtUzf7chI++TyIiQIQxJldEgoHVwL3Ab4GPjTHzReQVINEY83JV7fhaT30ksM8Yc8AYUwTMB671ckxNnjFmJXCywsPXAm87br+N/YfzCVXsj08zxhw1xmx23M4BdgGd8dH3qZr98VnGynXcDXb8GOASYKHj8RrfI19L6p2BZKf7Kfj4G4l905aJyCYRme3tYNyovTHmqOP2MaC9N4Nxk3tEZJujPOMTZYrKiEgsMBRYhx+8TxX2B3z4fRKRQBHZCqQBXwP7gSxjTIljlRpznq8ldX90oTFmGDAZ+JXjq79fMbbG5zt1vsq9DPQEhgBHgb95N5y6EZHmwEfAfcaYU87LfPF9qmR/fPp9MsaUGmOGADHYykRcbdvwtaR+BOjidD/G8ZjPMsYccfxOAz7BvpH+4Lij7lle/0zzcjz1Yow57viHKwNewwffJ0ed9iPgfWPMx46HffZ9qmx//OF9AjDGZAHLgTFASxEJciyqMef5WlLfAPR2HA0OAaYDi7wcU52JSITjIA8iEgFcDuyo/lk+YxFwh+P2HcBnXoyl3soTn8NUfOx9chyEex3YZYx53mmRT75PVe2PL79PItJWRFo6bjfDDgjZhU3u0xyr1fge+dToFwDHEKUXgEDgDWPM014Oqc5EpAe2dw4QBMz1xf0RkXnAeOw0oceBx4BPgQVAV+wUyzcZY3zi4GMV+zMe+5XeAIeA/3GqRTd6InIhsArYDpQ5Hn4IW4f2ufepmv2ZgY++TyIyCHsgNBDb4V5gjHnSkSfmA62ALcBMY0xhle34WlJXSilVNV8rvyillKqGJnWllPIjmtSVUsqPaFJXSik/okldKaX8iCZ1pZTyI5rUlVLKj/w/OSsAN9Qid7wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEICAYAAACgQWTXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1fn48c+TnSxANtYEEgHZkSUgCiICKiCCu6i1YlW+Uq22tQu1v7pVv19bl1qroihabVVE3FsXUEFAAWWXfUfCGtawJCHL8/vj3sAQswxhkpuZPO/Xa15z594z9z43A8+cOffcc0RVMcYYExrCvA7AGGNM4FhSN8aYEGJJ3RhjQogldWOMCSGW1I0xJoRYUjfGmBBiSd2US0Q+EZGbAl3WSyKyWUSG1MB+VUTausvPi8if/ClbjePcICLTqhtnJfsdKCLZgd6v8UaE1wGYwBGRwz4vY4ECoNh9/T+q+rq/+1LVYTVRNtSp6u2B2I+IZACbgEhVLXL3/Trg92do6idL6iFEVeNLl0VkM3Crqn5etpyIRJQmCmNMaLHml3qg9Oe1iPxeRHYCr4hIooj8R0RyRGS/u5zm856ZInKruzxGROaIyONu2U0iMqyaZTNFZJaIHBKRz0XkWRH5dwVx+xPjn0Xka3d/00QkxWf7jSKyRUT2isgfK/n7nC0iO0Uk3Gfd5SKyzF3uIyJzReSAiOwQkWdEJKqCff1TRB72ef1b9z3bReRnZcpeIiKLRSRXRLaKyAM+m2e5zwdE5LCInFP6t/V5/7ki8p2IHHSfz/X3b1MZEenovv+AiKwQkZE+24aLyEp3n9tE5Dfu+hT38zkgIvtEZLaIWH7xgP3R649mQBLQGhiL89m/4r5uBeQBz1Ty/rOBNUAK8FdgkohINcq+AXwLJAMPADdWckx/YrweuBloAkQBpUmmEzDB3X8L93hplENV5wNHgEFl9vuGu1wM/Mo9n3OAwcDPK4kbN4ahbjwXAu2Asu35R4CfAo2BS4BxInKZu22A+9xYVeNVdW6ZfScB/wWeds/tSeC/IpJc5hx+9LepIuZI4CNgmvu+XwCvi0h7t8gknKa8BKAL8KW7/h4gG0gFmgL3AjYGiQcsqdcfJcD9qlqgqnmquldV31HVo6p6CHgEOL+S929R1RdVtRh4FWiO85/X77Ii0groDdynqsdUdQ7wYUUH9DPGV1R1rarmAVOA7u76q4D/qOosVS0A/uT+DSryJnAdgIgkAMPddajqQlWdp6pFqroZeKGcOMpzjRvfclU9gvMl5nt+M1X1e1UtUdVl7vH82S84XwLrVPVfblxvAquBS33KVPS3qUxfIB541P2MvgT+g/u3AQqBTiLSUFX3q+oin/XNgdaqWqiqs9UGlvKEJfX6I0dV80tfiEisiLzgNk/k4vzcb+zbBFHGztIFVT3qLsafYtkWwD6fdQBbKwrYzxh3+iwf9Ymphe++3aS6t6Jj4dTKrxCRaOAKYJGqbnHjONNtWtjpxvG/OLX2qpwUA7ClzPmdLSIz3Oalg8Dtfu63dN9byqzbArT0eV3R36bKmFXV9wvQd79X4nzhbRGRr0TkHHf9Y8B6YJqIbBSR8f6dhgk0S+r1R9la0z1Ae+BsVW3IiZ/7FTWpBMIOIElEYn3WpVdS/nRi3OG7b/eYyRUVVtWVOMlrGCc3vYDTjLMaaOfGcW91YsBpQvL1Bs4vlXRVbQQ877Pfqmq523GapXy1Arb5EVdV+00v0x5+fL+q+p2qjsJpmnkf5xcAqnpIVe9R1TOAkcCvRWTwacZiqsGSev2VgNNGfcBtn72/pg/o1nwXAA+ISJRby7u0krecToxTgREi0t+9qPkQVf97fwO4G+fL4+0yceQCh0WkAzDOzximAGNEpJP7pVI2/gScXy75ItIH58ukVA5Oc9EZFez7Y+BMEbleRCJE5FqgE05TyemYj1Or/52IRIrIQJzPaLL7md0gIo1UtRDnb1ICICIjRKSte+3kIM51iMqau0wNsaRefz0FNAD2APOAT2vpuDfgXGzcCzwMvIXTn7481Y5RVVcAd+Ak6h3AfpwLeZUpbdP+UlX3+Kz/DU7CPQS86MbsTwyfuOfwJU7TxJdlivwceEhEDgH34dZ63fcexbmG8LXbo6RvmX3vBUbg/JrZC/wOGFEm7lOmqsdwkvgwnL/7c8BPVXW1W+RGYLPbDHU7zucJzoXgz4HDwFzgOVWdcTqxmOoRu5ZhvCQibwGrVbXGfykYUx9YTd3UKhHpLSJtRCTM7fI3Cqdt1hgTAHZHqaltzYB3cS5aZgPjVHWxtyEZEzqs+cUYY0KINb8YY0wI8az5JSUlRTMyMrw6vDHGBKWFCxfuUdXUirZ7ltQzMjJYsGCBV4c3xpigJCJl7yQ+iTW/GGNMCLGkbowxIcSSujHGhBDrp26MCYjCwkKys7PJz8+vurCpUkxMDGlpaURGRp7S+yypG2MCIjs7m4SEBDIyMqh4/hTjD1Vl7969ZGdnk5mZeUrvteYXY0xA5Ofnk5ycbAk9AESE5OTkav3qsaRujAkYS+iBU92/ZdAl9TU7D/HoJ6s5lF/odSjGGFPnBF1S37rvKM9/tYF1uw97HYoxpg45cOAAzz333Cm/b/jw4Rw4cKDSMvfddx+ff/55dUOrVUGX1Ns1daZZXL/Lkrox5oSKknpRUVGl7/v4449p3LhxpWUeeughhgwZclrx1ZagS+ppibFER4Sxbvchr0MxxtQh48ePZ8OGDXTv3p3evXtz3nnnMXLkSDp16gTAZZddRq9evejcuTMTJ048/r6MjAz27NnD5s2b6dixI7fddhudO3fmoosuIi8vD4AxY8YwderU4+Xvv/9+evbsSdeuXVm92pkUKicnhwsvvJDOnTtz66230rp1a/bsOa2JqKol6Lo0hocJZ6TGW/OLMXXYgx+tYOX23IDus1OLhtx/aecKtz/66KMsX76cJUuWMHPmTC655BKWL19+vEvgyy+/TFJSEnl5efTu3Zsrr7yS5OST5yJft24db775Ji+++CLXXHMN77zzDj/5yU9+dKyUlBQWLVrEc889x+OPP85LL73Egw8+yKBBg/jDH/7Ap59+yqRJkwJ6/v4Kupo6QLsm8ay3pG6MqUSfPn1O6uP99NNPc9ZZZ9G3b1+2bt3KunXrfvSezMxMunfvDkCvXr3YvHlzufu+4oorflRmzpw5jB49GoChQ4eSmJgYwLPxX9DV1MFJ6h8u3c7RY0XERgXlKRgT0iqrUdeWuLi448szZ87k888/Z+7cucTGxjJw4MBy+4BHR0cfXw4PDz/e/FJRufDw8Crb7GtbcNbU3YulG3Yf8TgSY0xdkZCQwKFD5V9rO3jwIImJicTGxrJ69WrmzZsX8OP369ePKVOmADBt2jT2798f8GP4o8qkLiIxIvKtiCwVkRUi8mA5ZcaISI6ILHEft9ZMuI62TZykbhdLjTGlkpOT6devH126dOG3v/3tSduGDh1KUVERHTt2ZPz48fTt2zfgx7///vuZNm0aXbp04e2336ZZs2YkJCQE/DhVqXKOUnFua4pT1cMiEgnMAe5W1Xk+ZcYAWap6p78HzsrK0upOklFYXELHP33KbQPO4PdDO1RrH8aYwFq1ahUdO3b0OgzPFBQUEB4eTkREBHPnzmXcuHEsWbLktPZZ3t9URBaqalZF76myQVqdrF96VTLSfXg6W3VkeBiZKXF2sdQYU2f88MMPXHPNNZSUlBAVFcWLL77oSRx+XWUUkXBgIdAWeFZV55dT7EoRGQCsBX6lqlvL2c9YYCxAq1atqh00OO3qq3ZY84sxpm5o164dixcv9joM/y6UqmqxqnYH0oA+ItKlTJGPgAxV7QZMB16tYD8TVTVLVbNSUyucN9UvbZsksGXvEfILi09rP8YYE0pOqfeLqh4AZgBDy6zfq6oF7suXgF6BCa9ibZvEU6KwaY/1gDHGmFL+9H5JFZHG7nID4EJgdZkyzX1ejgRWBTLI8rQ73gPG2tWNMaaUP23qzYFX3Xb1MGCKqv5HRB4CFqjqh8BdIjISKAL2AWNqKuBSmSlxhAl2sdQYY3xUWVNX1WWq2kNVu6lqF1V9yF1/n5vQUdU/qGpnVT1LVS9Q1dWV7/X0xUSG0zo5jvXWV90YUw3x8c6v/e3bt3PVVVeVW2bgwIFU1fX6qaee4ujRo8df+zOUb00KyjtKS7VtEs86G4LXGHMaWrRocXwExuoom9T9Gcq3JgV9Ut+05wiFxSVeh2KM8dj48eN59tlnj79+4IEHePjhhxk8ePDxYXI/+OCDH71v8+bNdOnidOjLy8tj9OjRdOzYkcsvv/yksV/GjRtHVlYWnTt35v777wecQcK2b9/OBRdcwAUXXACcGMoX4Mknn6RLly506dKFp5566vjxKhriNxCCejSsdk3iKSpRtuw9enzoAGNMHfDJeNj5fWD32awrDHu0ws3XXnstv/zlL7njjjsAmDJlCp999hl33XUXDRs2ZM+ePfTt25eRI0dWOP/nhAkTiI2NZdWqVSxbtoyePXse3/bII4+QlJREcXExgwcPZtmyZdx11108+eSTzJgxg5SUlJP2tXDhQl555RXmz5+PqnL22Wdz/vnnk5iY6PcQv9URfDX13avg/Z9DUQHtmjjjKli7ujGmR48e7N69m+3bt7N06VISExNp1qwZ9957L926dWPIkCFs27aNXbt2VbiPWbNmHU+u3bp1o1u3bse3TZkyhZ49e9KjRw9WrFjBypUrK41nzpw5XH755cTFxREfH88VV1zB7NmzAf+H+K2O4KupH9oJS16HZt1o09MZN2zdrsMMLXs7lDHGO5XUqGvS1VdfzdSpU9m5cyfXXnstr7/+Ojk5OSxcuJDIyEgyMjLKHXK3Kps2beLxxx/nu+++IzExkTFjxlRrP6X8HeK3OoKvpt7mAsgcALMeI1bzaNm4gfVVN8YAThPM5MmTmTp1KldffTUHDx6kSZMmREZGMmPGDLZs2VLp+wcMGMAbb7wBwPLly1m2bBkAubm5xMXF0ahRI3bt2sUnn3xy/D0VDfl73nnn8f7773P06FGOHDnCe++9x3nnnRfAsy1f8NXUAQbfDy8NhnkTaNd0oCV1YwwAnTt35tChQ7Rs2ZLmzZtzww03cOmll9K1a1eysrLo0KHyUV3HjRvHzTffTMeOHenYsSO9ejk3x5911ln06NGDDh06kJ6eTr9+/Y6/Z+zYsQwdOpQWLVowY8aM4+t79uzJmDFj6NOnDwC33norPXr0CGhTS3mqHHq3ppzO0LsATL4BNs3iyU5TeOG7A6x8aCjhYeVf/DDG1Lz6PvRuTajO0LvB1/xSatD/g4JDDD84mYKiErL3H636PcYYE+KCN6k36QhnjebMLW/SlH12E5IxxhDMSR1g4B8QLeHuiHetXd2YOsCr5txQVN2/ZXAn9cTWSNbPuCZiJvu31vjAkMaYSsTExLB3715L7AGgquzdu5eYmJhTfm9w9n7xNeA3FH33Kv23vgCM8DoaY+qttLQ0srOzycnJ8TqUkBATE0NaWtopvy/4k3p8E75tei0Ddr6K7liKND/L64iMqZciIyPJzMz0Oox6L7ibX1w7u9zGAY2j4LMHvQ7FGGM8FRJJvXWLFkwoGknM5i9gy1yvwzHGGM+ERFJv1zSBV4sv4khUCnz+ANiFGmNMPRUSST0pLoq4uASmpdwEW+fBumleh2SMMZ7wZ+LpGBH5VkSWisgKEflRw7WIRIvIWyKyXkTmi0hGTQRbmbZN4nmzaCAkZsIXf4YSmzjDGFP/+FNTLwAGqepZQHdgqIj0LVPmFmC/qrYF/gb8JbBhVq1d03hW785DL7gXdn0PK96t7RCMMcZz/kw8rapaertmpPso22g9CnjVXZ4KDJaKphapIW1T48nNLyKn9Qho2gW+fBiKC2szBGOM8ZxfbeoiEi4iS4DdwHRVnV+mSEtgK4CqFgEHgeRy9jNWRBaIyIJA36DQrqk7C1LOURj0J9i/CRb/K6DHMMaYus6vpK6qxaraHUgD+ohIteYZUtWJqpqlqlmpqanV2UWF2rlzlK7bfRjOvBjSz4av/gqFgZtRxBhj6rpT6v2iqgeAGcDQMpu2AekAIhIBNAL2BiJAf6UmRNMwJoJ1uw+BCAx5AA7tgHkTajMMY4zxlD+9X1JFpLG73AC4EFhdptiHwE3u8lXAl1rLo/qICG2bxJ8Ygrf1udD+EvjqL5CzpjZDMcYYz/hTU28OzBCRZcB3OG3q/xGRh0RkpFtmEpAsIuuBXwPjaybcyrVrksB63yF4R/wNImPh3dug6JgXIRljTK3yp/fLMlXtoardVLWLqj7krr9PVT90l/NV9WpVbauqfVR1Y00HXp52TePZe+QY+464CTyhKVz6d9ixFGb91YuQjDGmVoXEHaWl2roXS0+qrXcaCd1vgNlPwNZvPYrMGGNqR0gl9dJujet2Hzp5w9BHoWEavDsWCmyGJGNM6AqppN6iUQyxUeE/nq80piFc/jzs3wzT/uhJbMYYUxtCKqmX9oBZX958pRn9oN9dsPCfsObTWo/NGGNqQ0gldaDipA5wwR+dIQQ+vBOO7KndwIwxphaEXFJv1ySBnbn55OaXM+5LRDRcMRHyD8JHd9u468aYkBOCSb2cHjC+mnaGwffB6v/AktdrMTJjjKl5IZfUj3drLHux1FffOyDjPPjk987FU2OMCREhl9TTk2KJigj7cbdGX2FhcNlzIGHw3u1QUlx7ARpjTA0KuaQeHia0Sa3kYmmpxq1g+GPww1z45unaCc4YY2pYyCV1cNrV11WV1AG6XQudRsGXj8COZTUfmDHG1LCQTOptm8STvT+Po8eKKi8oAiOegthk527T/NzaCdAYY2pISCb10h4wG3YfqbpwbJLTvr53HbwyHHJ31HB0xhhTc0IzqTd1e8DkVHKx1FfbwXD9W7BvI0y60MZfN8YErZBM6q2T44gIkx+PAVOZtkPg5v9CUQFMugh+mFdzARpjTA0JyaQeGR5GZkqcfxdLfbXoAbdOh7gUeG0UrPqoZgI0xpgaEpJJHaoYA6YyiRnws2nQrCu8dSPMnxjw2IwxpqaEbFJv1ySeLXuPkF9YjRuL4pLhpx9C+2HwyW9h+v1QUhL4II0xJsD8mXg6XURmiMhKEVkhIneXU2agiBwUkSXu476aCdd/bZsmUKKwea8fPWDKExUL1/wLsn4GXz8F799u85waY+q8CD/KFAH3qOoiEUkAForIdFVdWabcbFUdEfgQq6e0W+O6XYfp0Kxh9XYSHgGXPAkNW8KXf4bDu5xEH1PN/RljTA3zZ+LpHaq6yF0+BKwCWtZ0YKcrMyWOMOHUL5aWJQIDfgOjnoPNc6wvuzGmTjulNnURyQB6APPL2XyOiCwVkU9EpHMF7x8rIgtEZEFOTs4pB3sqYiLDyUiJY8nWA4HZYY8bTvRlnzgQtnwTmP0aY0wA+Z3URSQeeAf4paqWvZ9+EdBaVc8C/gG8X94+VHWiqmapalZqamp1Y/bb0M7N+Hr9Hnbn5gdmh22HOF0eo+LgnyPgm3/YRBvGmDrFr6QuIpE4Cf11VX237HZVzVXVw+7yx0CkiKQENNJquLJXGsUlyvtLtgVup007w9gZ0GE4TPt/MOVGZyYlY4ypA/zp/SLAJGCVqj5ZQZlmbjlEpI+7372BDLQ62qTG07NVY6YuzEYDWaOOaeRcML3oEVj9sdMcs3N54PZvjDHV5E9NvR9wIzDIp8vicBG5XURud8tcBSwXkaXA08BoDWgWrb6reqWzdtdhvt8W4Nq0CJx7J4z5Lxw7Ci8NgSVvBPYYxhhzisSr3JuVlaULFiyo8ePk5hfS++HPubZ3Og+N6lIzBzm8G6b+DDbPhp43wbC/QmRMzRzLGFOvichCVc2qaHvI3lFaqmFMJEO7NOODJdspKKqhaevim8CN70P/X8OiV+Hli2zuU2OMJ0I+qQNc1SuNg3mFfLFqd80dJDwChtwP1012EvoLA2D1f613jDGmVtWLpH5umxSaN4rh7QVba/5g7YfB2K+gcWuYfD38rQt89EtY8wkcq+aQBcYY4yd/hgkIeuFhwhU9WzJh5gZ25+bTpGENt3cnZcIt0+H7KbD2M/j+bVj4CoRHQ0Z/aHcRnHkRJJ1Rs3EYY+qdkL9QWmpjzmEGPfEVfxjWgf85v02tHRdwJt74YS6snQbrpjlT5wEkt4V2F0O7C6F1P4iIqt24jDFBp6oLpfUmqQNcOeEbcvMKmfarAbjd6r2xbyOsm+4k+E2zobgAGiRBt2ug+w3QvJt3sRlj6rR63/vF11W90li3+zDLsj2+AzTpDDj7f+An78DvN8HoN+GM82HBy/DCefB8f5g3AY54fv+WMSbI1Kukfkm35kRHhDF1YbbXoZwQFecMOXD1P+GeNTD8cQiLgE/HwxPtYfINzkXW4kKvIzXGBIF6ldRL+6x/uHR79WZEqmmxSdDnNhg7E8Z949Tmt86HN0fDk53gsz/Czu+tm6QxpkL1qk0dYPa6HG6c9C3PXt+TS7o1r/Xjn7LiQlj/OSz+N6z9FEqKoEEipPWG9D6Qfja06AnR8V5HaoypBVW1qdeLLo2+SvusT124NTiSenik0/e9/TA4ssdpitk6H7Z+61xoBZAwaNrlRJJP6+1MoO3lxWBjjCfqXVKv9T7rgRSXAj1vdB4Aefshe4GT4LO/haWT4buX3LJNoOtVMOC3TrOOMaZeqHdJHeDKnmk8O2MD7y3eVvt91gOpQaLTx73dhc7rkmLYvcqpyW+eA/Ofd0aOPP/30PtW6wdvTD1Qry6UljojNZ6s1omBH2fda2Hh0KwL9L4Frn4Fbp8DLXrAZ3+A5/raWDTG1AP1MqlDHeqzXpOadoYb34Pr33YS/uTr4dVLYccyryMzxtSQepvUh3drTkxkHeuzXhNEnHFmxn3j9IHftcIZQfKDO+DQTq+jM8YEWL1N6g1jIhnauQ73WQ+08EinD/xdi+GcO2DpW/B0T/jqMSjM8zo6Y0yA1NukDs5UdzU+znpd06AxXPwI3DEf2g6CGQ87yX3WY84MTsaYoObPxNPpIjJDRFaKyAoRubucMiIiT4vIehFZJiI9aybcwDqnTTIt3D7r9U5yG7j2384cqylt4cuHnbtW377Z6TljF1SNCUr+dGksAu5R1UUikgAsFJHpqrrSp8wwoJ37OBuY4D7XaU6f9TSem7k++PqsB0pGf+exZ50zoNiS12HFu5DaAbJ+BmeNhphGXkdpjPFTlTV1Vd2hqovc5UPAKqBlmWKjgNfUMQ9oLCJBcLsmXNkrjRKFdxdv8zoUb6W0g6H/B79eDaOehchY+OR38EQH+PAu2LHU6wiNMX44pTZ1EckAegDzy2xqCfi2YWTz48SPiIwVkQUisiAnJ+fUIq0hmSlxodlnvbqiYqHHT2DsDGdgsS5XwrIpTo+ZFwfD1393xoDPz/U6UmNMOfy+o1RE4oF3gF+qarX+R6vqRGAiOAN6VWcfNeGqXmmMf/d7FmzZT+8Mu6X+uBY9YNQzcNGfnSEIFv4Tpt/nbhSndt+ix4lHs27Ol4IxxjN+JXURicRJ6K+r6rvlFNkGpPu8TnPXBYWR3Vvw6KereXnOJkvq5WmQCH3HOY8je2D7Eti+GLYvgk2zYNlbTjkJg9SOToJP7+OMPRMV523sxtQzVSZ1ceZ9mwSsUtUnKyj2IXCniEzGuUB6UFV3BC7MmhUbFcH1fVrx/Fcb2LrvKOlJVtusUFwKtBviPErl7oAdbqLftsgZInjJv+GLB+GcO53+8dEJ3sVsTD1S5XjqItIfmA18D5S4q+8FWgGo6vNu4n8GGAocBW5W1UoHS/dqPPWK7DyYT/+/fMlPz8ngvks7eR1OcFN1BhWb9ZgzFnxMY+j7c2fSjwaNvY7OmKBmE0+fgl+9tYRpK3Yy997BNIyJ9Dqc0LBtIcx6HNZ8DNENoc9Y545WGw7YmGqxiadPwS39MzlyrJgp39XDm5FqSstecN2b8D+z4YyBMPtxeKqrc8H1cN3oAWVMKLGk7qNLy0b0yUzila83U1RcUvUbjP+ad4Nr/wU/nwdnDoVv/uEk90//APs2eR2dMSHDknoZt/bPZNuBPD5bscvrUEJTk45w1SS44zvofDnMfwGe7g7P94eZf4FdK22IAmNOg7Wpl1Fcogx6YibJcVG8+/N+XocT+g5shZUfwKqPnIurKCSdAR0vhQ6XOs03YVb3MKaUXSithle/2cz9H67g3Z+fS89WiV6HU38c2gVr/usk+E2zoKQIEppDh0ucJN+6nzOEsDH1mCX1ajhSUMQ5//cF552ZyrPXB8WAk6Enbz+snQarP4J1n0NRntM1st1F0H4YtB1sA42ZeqmqpF4vJ56uSlx0BNf1acVLczax7UAeLRs38Dqk+qdBIpx1rfM4dhQ2fOHMsbr2M/h+CoRFOKNLnjkM2g+FxAyvIzamTrCaegW2H8jjvL/O4Jb+mdw7vKPX4ZhSJcWw9VtY+wms+QT2rHXWN+nk1ODPHGbt8CakWfPLafjFm4uZuXo3c+8dTHy0/aipk/ZucIYlWPMJbPkGtBjimjhDE/T7JUREeR2hMQFlNx+dhlv6Z3KooIi3F9jNSHVWchvnDtUx/4HfbYArXnJq6jMegRcHwY5lXkdoTK2ypF6J7umNyWqdyMtfb6K4xPpO13kNEqHb1XD9ZBj9JhzZDS9eAF8+AkXHvI7OmFphSb0Kt/TPZOu+PKavtJuRgkqH4c7dq12vhll/hYkDnVEkjQlxltSrcFHnZqQlNuDlOXYre9CJTYLLn4frp0DePmfmpi8egqICryMzpsZYUq9CeJhwc79Mvt28j2XZB7wOx1THmRc7tfazroPZT8AL5zujRxoTgiyp++GarDTioyOYZLX14NWgMVz2LNwwFfIPwktDYPr9UJjvdWTGBJQldT8kxEQyunc6/122gx0H87wOx5yOdhfCHfOcybW/fgqe7Q3vjYO5z8LGmc50fcYEMet87aebzs3g5a838eo3Wxg/rIPX4ZjTEdMIRv4DOl3mJPMNX8DSN05sj28GTTs7j2Zdnefkdtbn3QQFS+p+Sk+KZWiXZpzHxtwAABfBSURBVLwxfwu/GNSWOLsZKfi1Hew8wJmwY9dy2LXCfSyH+c9DsdsVMizS6f/edojznubd7a5VUyf5M/H0y8AIYLeqdiln+0DgA6C0wfldVX0okEHWFbf0P4OPv9/JO4uy+ek5GV6HYwIpPhXiL4A2F5xYV1wIe9c7SX7nMmfkyBkPO4/YFGgzyGnOaTPImZDbmDrAn+rmP3EmlX6tkjKzVXVEQCKqw3q1TiSrdSLPzljPVb3SiI2y2npIC490JvVo0hG6XuWsO5wDG2c4E2qv/8IZXAyBFt3dWvwQaJkF4fZvw3ijyt+PqjoL2FcLsQSFPwzvwK7cAl6cZT1h6qX4VOh2DVwxEX6zDm6bARf8EcKjne6SL18MT7SH715yBh8zppYFqlHwHBFZKiKfiEjnigqJyFgRWSAiC3JygnPS4V6tkxjetRkvzNrA7lzrDlevhYVBy55w/m/hls/gdxvh6lchtQP89x54YQBsnuN1lKaeCURSXwS0VtWzgH8A71dUUFUnqmqWqmalpqYG4NDe+P3QDhQWl/DEtLVeh2LqkgaJ0PkyZ3Cxq1+F/Fz45yUw5SY48IPX0Zl64rSTuqrmquphd/ljIFJEQvqqUevkOG46J4MpC7eyakeu1+GYukbESe53fus0zaz9DJ7pDTP+15nww5gadNpJXUSaiYi4y33cfe493f3Wdb8Y1I5GDSL5349X4dWY9KaOi2wA5/8OfrEAOoyAr/4Cz2TB91PB/s2YGlJlUheRN4G5QHsRyRaRW0TkdhG53S1yFbBcRJYCTwOjtR5kuUaxkdw1qB2z1+1h5trgvD5gakmjNLhqEtz8KcQmwzu3wCvDYMdSryMzIchmPjoNx4pKuOhvXxEZHsYnd59HRLjdjGKqUFIMi//ljBZ5dJ9zI1O7i5yukMltvI7OBAGb+agGRUWEMX5YB9btPsxbNjuS8UdYOPQaA79YBP1/Bfs2wSe/g3/0hL93h49/C2unWdu7qTarqZ8mVeXaF+axcc9hZv72ApvL1Jy6fRth3eewfjpsmg1FeU6/94z+Tg2+3YWQ3Na5AGvqPZt4uhYs3XqAUc9+zZ0XtOU3F7f3OhwTzArzYcvXzh2r66bD3nXO+oYtncSelAmJGe7DXW7Q2MOATW2zpF5L7p68mE+X72TGbwbSonEDr8MxoWL/ZifB/zDPWd63CY6WGR64QeLJib5JJ+emqMRMG3QsBFlSryXZ+48y6ImvGNG1OU9e293rcEwoKzjkJPjSJL9/M+x3nw/8ACVFTrnoRs6YNC16OEm+RU+nJ4414wS1qpK6NQAHSFpiLLf0z2TCzA3c3C+TrmmNvA7JhKroBGec92Zdf7ytuBByVjuTbG9bBNsXwdxnTiT62BQ3wfdwhhJOP9uab0KM1dQD6FB+IQMfm0nbJvFMHtsXsRqRqQsK853hg7cvOpHs96wBLQEEmnWB1v2h9bnQuh/EJXsdsamE1dRrUUJMJL+88Ez+9P5ypq/cxUWdm3kdkjEQGQNpvZxHqYLDTpLfMhe2zIGF/4T5E5xtqR2c5F6a5Bs29yRsUz1WUw+wouISLn5qFqrw2a8GEGk3JJlgUHTMqcVv+dp5/DAfjh1ytiWd4cz0lNoBUs+ElPbOjVIR0d7GXE/ZhVIPfLFqF7e8uoAHLu3EmH6ZXodjzKkrLoJd38Pmr2HLN7B7BezfArj5QsKd7pWpHSDlTJ+EfyZExXkaeqiz5hcPDOrQhHPbJPP3L9ZxeY80GsVGeh2SMacmPMK5mNqiB5x7p7Pu2FFner+cNU6bfM5qyFkLaz89cSEWgdT2zvuauz1vmnWFqFjPTqW+sZp6DVm5PZcR/5jNDWe35s+X/WhqV2NCR3Ghc1dszmrYtRJ2LHGacg7vcrZLmFOTL/2SaNEDmnZ2RrE0p8xq6h7p1KIhPz0ng1fnbuaarHTr4mhCV3ikUztPbQ+dRp1Yn7vDSe7bFzuJft00WPK6s03CoXEraJwOjVr5LKc7yw1bOPs1p8xq6jUoN7+QQY9/RcvEBrw37lzCwqyLo6nHVCF3G2xf4iT5fRudm6UObIXDO08uK2GQ0MJJ9EltIKMfZJ4PjVp6E3sdYjV1DzWMieSPl3TgV28t5a0FW7muTyuvQzLGOyLOHa2N0qDjiJO3FeY7Cf/AD3Bwq5PoS5fXfAxL/u2US2oDmQNOPOJCepK1arGkXsMu696SN7/dyl8+Xc3FnZuRFBfldUjG1D2RMU43yfLGlC8pgd0rYdMs2PSVM3PUwlecbU27uAn+fKdffUzD2o27DrLml1qwZuchhj89m6t7pfHold28DseY4FZc5DTfbPoKNn4FW+dDUb7TZBPXxEns0QkQ3dBdbnjyckxDiIp3+tmHR7nP0RAR5TyHR568DpzJTVRBi93lYueO3BKf58gY55pADd9Jbs0vdUD7Zgn8rF8GL87exDW90+nZKtHrkIwJXuERkJblPM67x2m6yf4ONs+BQ9shPxcKcp2Bz3K3O8v5uVB4pOZja5DkjKmTluU8t+hZ68MuVFlTF5GXgRHAblX9Ud88d9LpvwPDgaPAGFVdVNWB61NNHeBwQRGDn5hJakI0H9zRn3C7aGpM7SoucpN9rjNMQvEx51FU4PNc4HTR9F0HzoxVEu7UwkuXj68Lc5bzDzpDL2xbBLtXcfxGrcRMJ8GXJvtmXU+rO2cgaur/BJ4BXqtg+zCgnfs4G5jgPhsf8dER/GlEJ+58YzGvz9/CT8/J8DokY+qX8AiITXIeNeYW56ngkDOxePYC2LbQGQ9/+VRnW1gEnPcbuOAPNRJBlUldVWeJSEYlRUYBr6lT5Z8nIo1FpLmq7ghQjCHjkq7Nmdx2K499tobhXZuTEm9jZxgTkqITnOkIM/qfWHdop5Pgty10hj+uIYEYbaol4Dvrcra77kdEZKyILBCRBTk5OQE4dHARER4c1Zn8wmL+7+PVXodjjKlNCc2gwyUw+D448+IaO0ytDiGoqhNVNUtVs1JTU2vz0HVGm9R4bjvvDN5ZlM13m/d5HY4xJsQEIqlvA9J9Xqe560wF7hzUlpaNG/Cn95dTVFzidTjGmBASiKT+IfBTcfQFDlp7euVioyK479JOrN55iFfnbvE6HGNMCKnyQqmIvAkMBFJEJBu4H4gEUNXngY9xujOux+nSeHNNBRtKLurUlIHtU/nb9LWM6Nacpg1jvA7JGBMC7I5SD23Ze4QL/zaLoZ2b8fR1PbwOxxgTBKrqp25zrXmodXIc485vw4dLt/PN+j1eh2OMCQGW1D02bmAbWifH8pu3l7LvyDGvwzHGBDlL6h6LiQznmet6sufIMe6evJjiEm+aw4wxocGSeh3QNa0RD4/qwux1e/jb9LVeh2OMCWKW1OuIa3qnM7p3Os/MWM/0lbu8DscYE6QsqdchD4zsTNeWjfj1W0vYtKcWhgk1xoQcS+p1SExkOBN+0pPwcGHcvxdy9FiR1yEZY4KMJfU6Ji0xlqdH92DNrkPc++73eHUfgTEmOFlSr4MGnJnKPReeyftLtvOveTaMgDHGf5bU66ifD2zLkI5NeOijlSzcYqM5GmP8Y0m9jgoLE564pjstExvw89cXkXOowOuQjDFBwJJ6HdaoQSQTbujFwbxC7nxjkQ3Ta4ypkiX1Oq5Ti4b83xVdmb9pH3/9bI3X4Rhj6jhL6kHg8h5p3Ni3NRNnbeTj722oemNMxSypB4k/jehEj1aNuWfKUh77bDW7D+V7HZIxpg6ypB4koiLCeOEnvRhwZgrPzdxA/7/MYPw7y9iQc9jr0IwxdYhNkhGENuYc5qU5m5i6MJtjRSUM6diU288/g6yMJK9DM8bUsKomybCkHsT2HC7gtW8289q8LRw4WkjPVo0ZO6ANF3ZqSniYeB2eMaYGBGTmIxEZKiJrRGS9iIwvZ/sYEckRkSXu49bTCdr4JyU+ml9f1J5vxg/iwZGdyTlcwO3/XsiQJ7/i9flbyC8s9jpEY0wtq7KmLiLhwFrgQiAb+A64TlVX+pQZA2Sp6p3+Hthq6oFXVFzCpyt2MnHWRpZlH6Rzi4a8cnNvmiTYpNbGhIpA1NT7AOtVdaOqHgMmA6MCFaAJnIjwMEZ0a8EHd/TjhRt7sWnPEa547hs22sVUY+oNf5J6S2Crz+tsd11ZV4rIMhGZKiLp5e1IRMaKyAIRWZCTk1ONcI0/RISLOzfjzdv6knesmCsnfMPiH/Z7HZYxphYEqkvjR0CGqnYDpgOvlldIVSeqapaqZqWmpgbo0KYiZ6U35p1x55IQE8l1L87ji1U2o5Ixoc6fpL4N8K15p7nrjlPVvapaOuLUS0CvwIRnTldGShzvjDuXM5smMPZfC5n87Q9eh2SMqUH+JPXvgHYikikiUcBo4EPfAiLS3OflSGBV4EI0pys1IZo3b+tLv7YpjH/3e/7++TqbfMOYEFVlUlfVIuBO4DOcZD1FVVeIyEMiMtItdpeIrBCRpcBdwJiaCthUT1x0BJNuyuKKni352+drufe95TbqozEhyG4+qmdUlcc+W8NzMzcwpGNT/nFdDxpEhXsdljHGTwG5+ciEDhHhd0M78ODIznyxehc3vDSP/UeOeR2WMSZALKnXUzedm8Fz1/dk+fZcrpjwDR8u3U6hNccYE/QivA7AeGdY1+Ykx0fz+3eWcdebi2nRKIabzs1gdJ9WNGoQ6XV4xphqsDZ1Q0mJ8uXq3Uyas4m5G/cSGxXONVnp3Nwvg9bJcV6HZ4zxYaM0mlOyYvtBJs3ZxEdLt1NUogzp2JRb+2fSJzMJERv50RivWVI31bI7N5/X5m7h9flb2H+0kC4tG3JL/0wu6dqCqAi7FGOMVyypm9OSd6yY9xZvY9KcjWzIOULj2EhGdGvO5T1a0rNVotXejallltRNQJSUKLPX7+GdhdlMW7mT/MISWiXFcln3Fozq0ZI2qfFeh2hMvWBJ3QTc4YIiPlu+k/eXbOPr9XsoUTgrrRGX9WjJiG4tSE2I9jpEY0KWJXVTo3bl5vPR0u28t3gbK7bnEh4mnNcuhUu6NqdDs4a0TomlYYx1jzQmUCypm1qzdtch3l+8jQ+WbGfbgbzj65PiomidHEtmchytk+PISIl1npNjaRwb5WHExgQfS+qm1pWUKOt2H2bz3iNs3nOEzXuPsmXvEbbsPcr2g3n4/pNr1CCSLi0b0icjmbPPSKJ7emNiIm0sGmMqUlVStztKTcCFhQntmyXQvlnCj7blFxazdd/R44l+454jLP7hAE99sRb9HKIiwuie3pi+mUmcfUYyPVsl2oBjxpwCq6mbOuHg0UK+3byP+Rv3Mn/TPlZsP0iJQmS40C2tMWdnJtGrdSLpSbE0axRDQnSEdac09ZLV1E1QaBQbyYWdmnJhp6YA5OYXsnDLfuZv3Mf8TXuZOGsjRSUnKiBxUeE0bRRD80YxNGvYwHkufd0ohpaNG9CoQaQlflPvWFI3dVLDmEguaN+EC9o3AeBIQRGrduSy42A+Ow/mszPXed5xMI+5G/aw61ABxSUn/+pMiI6gZWID0hJjSU9yntMSG5CeGEtaUgPrlWNCkiV1ExTioiPIykiqcHtxibLncAE7Duaz40Ae2w7kkb0/j+z9R8nef5S5G/Zw5FjxSe9pGBNBy8RYkuOiaBwbSVJcFImxUc5zXBSJsZHHXyfFRREdEWY1f1Pn+ZXURWQo8HcgHHhJVR8tsz0aeA1nwum9wLWqujmwoRpTsfAwoWnDGJo2jKF7euMfbVdVDhwtJHt/HlvdRL91Xx47Duax78gxth1wng/mFVZ4jDCB6IhwoiPDiAoPIzoyjOiIcJ9l93VEmPMIDyMyXIgM933tLEe626Ijw4mOCCOmiufoiDCiI8OP79O+XExFqkzqIhIOPAtcCGQD34nIh6q60qfYLcB+VW0rIqOBvwDX1kTAxlSHiDi177gouqY1qrBcUXEJB/MK2X/0GPuOFLLvyDF3+Rh5x4opKCrmWFEJBe7DWS52XheWcCCvkILCYgqLSygsVo4VlVBYXMKxYqfsseISTrdvgghER5R+sZz4Uil9HREmhIsg4nzZhYc5XwLh7mtnWQgLA8EpJyKIu2/B97W7HxHCw8XZd1jpcxgRYUJYWNn17nN4WDnrw056XRpjmM9xwtzXYQJhcqKcs6502YnNt5zv+ZY9ToTP3yHU+VNT7wOsV9WNACIyGRgF+Cb1UcAD7vJU4BkREbUp602QiQgPIzk+muT4mhvqoLhEKSw+8aWQX+h8KZQ+FxQVU1DoPOf7PJd+KRQUFlNQ7HyJHDvp2Xl/cYlSXKKUqHOc/EKlWJ1fK77bStx1CmjZZdRd56wvUSgqUYpLStxnPf5c9lpGXVaa9MPdL4OTvszkxJfHSevcZfD5EoTjXxDHt8mJ7U7ZE+8vXVG6fF2fVtx63hk1co7+JPWWwFaf19nA2RWVUdUiETkIJAN7AhGkMaHESSzhIXOT1YmkX0JRsVKsSnGx+iT/kpO+BIqKnXUl6nzBlb6/dLnYfV2iSon7HvU5Tok6Xzi+zyXq3PRWolrmS8f9Eir+8Xrnvb5fYCe+1Er0xDFLq6Ynfdn5vHY2cry8+/J4udK/0fGvPoWUGqw01OqFUhEZC4wFaNWqVW0e2hhTQ0407YQTbV0vPOfPbAfbgHSf12nuunLLiEgE0AjngulJVHWiqmapalZqamr1IjbGGFMhf5L6d0A7EckUkShgNPBhmTIfAje5y1cBX1p7ujHG1L4qfyy5beR3Ap/hdGl8WVVXiMhDwAJV/RCYBPxLRNYD+3ASvzHGmFrmVwuYqn4MfFxm3X0+y/nA1YENzRhjzKmyGYSNMSaEWFI3xpgQYkndGGNCiCV1Y4wJIZ5NkiEiOcCWar49hdC7WzXUzinUzgdC75xC7Xwg9M6pvPNpraoV3ujjWVI/HSKyoLKZP4JRqJ1TqJ0PhN45hdr5QOidU3XOx5pfjDEmhFhSN8aYEBKsSX2i1wHUgFA7p1A7Hwi9cwq184HQO6dTPp+gbFM3xhhTvmCtqRtjjCmHJXVjjAkhQZfURWSoiKwRkfUiMt7reAJBRDaLyPciskREFngdz6kSkZdFZLeILPdZlyQi00Vknfuc6GWMp6qCc3pARLa5n9MSERnuZYynQkTSRWSGiKwUkRUicre7Pig/p0rOJ5g/oxgR+VZElrrn9KC7PlNE5rs57y13CPSK9xNMberuJNhr8ZkEG7iuzCTYQUdENgNZqhqUN02IyADgMPCaqnZx1/0V2Keqj7pfvomq+nsv4zwVFZzTA8BhVX3cy9iqQ0SaA81VdZGIJAALgcuAMQTh51TJ+VxD8H5GAsSp6mERiQTmAHcDvwbeVdXJIvI8sFRVJ1S0n2CrqR+fBFtVjwGlk2AbD6nqLJxx9H2NAl51l1/F+Q8XNCo4p6ClqjtUdZG7fAhYhTO3cFB+TpWcT9BSx2H3ZaT7UGAQMNVdX+VnFGxJvbxJsIP6g3QpME1EFrrzuIaCpqq6w13eCTT1MpgAulNElrnNM0HRVFGWiGQAPYD5hMDnVOZ8IIg/IxEJF5ElwG5gOrABOKCqRW6RKnNesCX1UNVfVXsCw4A73J/+IcOd2jB42vkqNgFoA3QHdgBPeBvOqROReOAd4Jeqmuu7LRg/p3LOJ6g/I1UtVtXuOHNB9wE6nOo+gi2p+zMJdtBR1W3u827gPZwPM9jtcts9S9s/d3scz2lT1V3uf7oS4EWC7HNy22nfAV5X1Xfd1UH7OZV3PsH+GZVS1QPADOAcoLGIlM5SV2XOC7ak7s8k2EFFROLcCz2ISBxwEbC88ncFBd/JyG8CPvAwloAoTX6uywmiz8m9CDcJWKWqT/psCsrPqaLzCfLPKFVEGrvLDXA6hKzCSe5XucWq/IyCqvcLgNtF6SlOTIL9iMchnRYROQOndg7OnLFvBNs5icibwECcYUJ3AfcD7wNTgFY4Qyxfo6pBc+GxgnMaiPOzXoHNwP/4tEfXaSLSH5gNfA+UuKvvxWmHDrrPqZLzuY7g/Yy64VwIDcepcE9R1YfcHDEZSAIWAz9R1YIK9xNsSd0YY0zFgq35xRhjTCUsqRtjTAixpG6MMSHEkroxxoQQS+rGGBNCLKkbY0wIsaRujDEh5P8Dvxh0dvFwGDQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = history.history['accuracy']\n",
    "val_acc = history.history['val_accuracy']\n",
    "\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(len(acc))\n",
    "\n",
    "plt.plot(epochs, acc, label = \"training\")\n",
    "plt.plot(epochs, val_acc, label = \"validation\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.title('Training and validation accuracy')\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, label = \"training\")\n",
    "plt.plot(epochs, val_loss, label = \"validation\")\n",
    "plt.legend(loc=\"upper right\")\n",
    "plt.title('Training and validation loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n37xsovmHBYx"
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "iuChAYtI8jER",
    "outputId": "61c46b19-b571-4aca-937c-35447c8e599a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/12 [==============================] - 3s 194ms/step - loss: 1.0467 - accuracy: 0.7368\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.0466973781585693, 0.7368420958518982]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(test_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5Pr8TJ6eHOsc"
   },
   "source": [
    "## Saving model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ne0K2uJ6HSrw"
   },
   "outputs": [],
   "source": [
    "model.save('glove_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LUsZrF__Htoi"
   },
   "source": [
    "## Text Classification With Bert\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7yfV3dw1S9eo"
   },
   "outputs": [],
   "source": [
    "label_map = {\n",
    "    0 : 'aeroplane', 1 : 'bicycle', 2 : 'bird', 3 : 'boat', \n",
    "    4 : 'bus', 5 : 'car', 6 : 'cat', 7 : 'chair', 8 : 'cow',\n",
    "    9 : 'diningtable', 10 : 'dog', 11 : 'horse', 12 : 'motorbike',\n",
    "    13 : 'person', 14 : 'pottedplant', 15 : 'sheep', 16 : 'sofa',\n",
    "    17 : 'train', 18 : 'tvmonitor'\n",
    "  }\n",
    "\n",
    "CATEGORIES = list(label_map.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "UwkkneeDWlt7"
   },
   "outputs": [],
   "source": [
    "def read_file(filename):\n",
    "    f = open(filename, 'r')\n",
    "    lines = f.readlines()\n",
    "    return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "D7Nhv7E_X-wy"
   },
   "outputs": [],
   "source": [
    "def prepare_train_data(TRAIN_DATA_DIR, CATEGORIES):\n",
    "    train_data = []\n",
    "    path = \"\"\n",
    "    for category in CATEGORIES:\n",
    "        temp = []\n",
    "        path = os.path.join(TRAIN_DATA_DIR, category)\n",
    "        class_num = CATEGORIES.index(category)\n",
    "        for filename in os.listdir(path):\n",
    "            file_path = os.path.join(path, filename)\n",
    "            try:\n",
    "                lines = read_file(file_path)\n",
    "                for line in lines:\n",
    "                    temp.append(line)\n",
    "\n",
    "            except Exception as e:\n",
    "                print(str(e))\n",
    "        \n",
    "        for line in temp:\n",
    "            train_data.append((line, class_num))\n",
    "    \n",
    "    return train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "U5H4r1ulYAYY"
   },
   "outputs": [],
   "source": [
    "def prepare_test_data(TEST_DATA_DIR, CATEGORIES):\n",
    "    test_data = []\n",
    "    path = \"\"\n",
    "    for category in CATEGORIES:\n",
    "        temp = []\n",
    "        path = os.path.join(TRAIN_DATA_DIR, category)\n",
    "        class_num = CATEGORIES.index(category)\n",
    "        for filename in os.listdir(path):\n",
    "            file_path = os.path.join(path, filename)\n",
    "            file = file_path\n",
    "            try:\n",
    "                lines = read_file(file)\n",
    "                for line in lines:\n",
    "                    temp.append(line)\n",
    "\n",
    "            except Exception as e:\n",
    "                print(str(e))\n",
    "        \n",
    "        for line in temp:\n",
    "            test_data.append((line, class_num))\n",
    "    return test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "fR6tN4RjYDfz"
   },
   "outputs": [],
   "source": [
    "training_data = prepare_train_data(TRAIN_DATA_DIR, CATEGORIES)\n",
    "test_data = prepare_test_data(TEST_DATA_DIR, CATEGORIES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "b64IMh2UZqy3"
   },
   "outputs": [],
   "source": [
    "def extract_X_and_y(data):\n",
    "  data_dict = {}\n",
    "  X = []\n",
    "  y = []\n",
    "  for example in data:\n",
    "    X.append(example[0])\n",
    "    y.append(example[1])\n",
    "  data_dict['X'] = X\n",
    "  data_dict['y'] = y\n",
    "  return data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "NowGoVsiZsBz"
   },
   "outputs": [],
   "source": [
    "training_dict = extract_X_and_y(training_data)\n",
    "X_train = training_dict['X']\n",
    "y_train = training_dict['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "XbEY9b4AZyyb"
   },
   "outputs": [],
   "source": [
    "test_dict = extract_X_and_y(test_data)\n",
    "X_test = test_dict['X']\n",
    "y_test = test_dict['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "_bU8DH3TZ8wp"
   },
   "outputs": [],
   "source": [
    "def clean_data(text):  \n",
    "  text = text.replace(\"\\n\", \" \")\n",
    "  text = text.replace(\"\\r\", \" \")\n",
    "  text = text.replace(\"\\t\", \" \")\n",
    "  word_tokens= text.lower().strip().split()\n",
    "  le=WordNetLemmatizer()\n",
    "  stop_words= set(stopwords.words(\"english\"))     \n",
    "  word_tokens= [le.lemmatize(w) for w in word_tokens if not w in stop_words]\n",
    "  \n",
    "  cleaned_data=\" \".join(word_tokens)\n",
    "  return cleaned_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "Zqxn1pQtaAhL"
   },
   "outputs": [],
   "source": [
    "cleaned_X_train = []\n",
    "for tr_example in X_train:\n",
    "  cleaned_data = clean_data(tr_example)\n",
    "  cleaned_X_train.append(cleaned_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "uIS7X7LdaENB"
   },
   "outputs": [],
   "source": [
    "cleaned_X_test = []\n",
    "for test_example in X_test:\n",
    "  cleaned_data = clean_data(test_example)\n",
    "  cleaned_X_test.append(cleaned_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "8BjXzdojaHQ7"
   },
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(cleaned_X_train, y_train, test_size=0.3, shuffle=True, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "tE68GufPLvPy"
   },
   "outputs": [],
   "source": [
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "0eGONYiWLgU4"
   },
   "outputs": [],
   "source": [
    "bert_preprocess = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\")\n",
    "bert_encoder = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "id": "VIqfAkajLroR"
   },
   "outputs": [],
   "source": [
    "def get_sentence_embeding(sentences):\n",
    "    preprocessed_text = bert_preprocess(sentences)\n",
    "    return bert_encoder(preprocessed_text)['pooled_output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "CoGxIkeUMeOT"
   },
   "outputs": [],
   "source": [
    "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "preprocessed_text = bert_preprocess(text_input)\n",
    "outputs = bert_encoder(preprocessed_text)\n",
    "\n",
    "l = tf.keras.layers.Dropout(0.1, name=\"dropout\")(outputs['pooled_output'])\n",
    "l = tf.keras.layers.Dense(19, activation='softmax', name=\"output\")(l)\n",
    "\n",
    "model = tf.keras.Model(inputs=[text_input], outputs = [l])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "id": "hE7lquEpMltU"
   },
   "outputs": [],
   "source": [
    "METRICS = [\n",
    "      tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n",
    "]\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=METRICS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "A6o44OxCNUyF"
   },
   "outputs": [],
   "source": [
    "es = tf.keras.callbacks.EarlyStopping(patience=2)\n",
    "epochs = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "di5z6kbwMqly",
    "outputId": "0c9b2c4c-6772-41b9-f47c-eb12aed6c912"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "WARNING: AutoGraph could not transform <function Model.make_train_function.<locals>.train_function at 0x7f73665a68c0> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: closure mismatch, requested ('self', 'step_function'), but source function had ()\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "59/59 [==============================] - ETA: 0s - loss: 2.6167 - accuracy: 0.9475 WARNING: AutoGraph could not transform <function Model.make_test_function.<locals>.test_function at 0x7f7366244d40> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: closure mismatch, requested ('self', 'step_function'), but source function had ()\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "59/59 [==============================] - 1026s 17s/step - loss: 2.6167 - accuracy: 0.9475 - val_loss: 2.6200 - val_accuracy: 0.9474\n",
      "Epoch 2/5\n",
      "59/59 [==============================] - 1020s 17s/step - loss: 2.5698 - accuracy: 0.9475 - val_loss: 2.5761 - val_accuracy: 0.9476\n",
      "Epoch 3/5\n",
      "59/59 [==============================] - 1020s 17s/step - loss: 2.5432 - accuracy: 0.9475 - val_loss: 2.5975 - val_accuracy: 0.9476\n",
      "Epoch 4/5\n",
      "59/59 [==============================] - 1019s 17s/step - loss: 2.5230 - accuracy: 0.9476 - val_loss: 2.4954 - val_accuracy: 0.9478\n",
      "Epoch 5/5\n",
      "59/59 [==============================] - 1020s 17s/step - loss: 2.4958 - accuracy: 0.9475 - val_loss: 2.4804 - val_accuracy: 0.9478\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(np.array(X_train), tf.keras.utils.to_categorical(y_train), \n",
    "          validation_data=(np.array(X_val), tf.keras.utils.to_categorical(y_val)),\n",
    "          callbacks= [es],\n",
    "          epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NOerlUpFqMVv",
    "outputId": "d65aaf22-6acf-44bb-83eb-8854d7cf22f6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84/84 [==============================] - 1094s 13s/step - loss: 2.6400 - accuracy: 0.9474\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[2.6400163173675537, 0.9474471211433411]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(np.array(X_test), tf.keras.utils.to_categorical(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "id": "yxmgMQLgSLh-"
   },
   "outputs": [],
   "source": [
    "model.save('bert_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "tPis2I95xtEy"
   },
   "outputs": [],
   "source": [
    "# y_predicted = model.predict(X_test)\n",
    "# y_predicted = y_predicted.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "id": "J-0NFHa3PSuE"
   },
   "outputs": [],
   "source": [
    "# y_predicted = y_predicted.reshape((2658, 19))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "id": "hZfObTDpPxUI"
   },
   "outputs": [],
   "source": [
    "# y_pred = np.argmax(y_predicted, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "id": "kCBVvABwJJE4"
   },
   "outputs": [],
   "source": [
    "# import numpy as np\n",
    "# from sklearn.metrics import confusion_matrix, classification_report\n",
    "# from matplotlib import pyplot as plt\n",
    "# import seaborn as sn\n",
    "\n",
    "# # y_predicted = np.where(y_predicted > 0.5, 1, 0)\n",
    "# cm = confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "id": "h-FGXbiVP7Yg"
   },
   "outputs": [],
   "source": [
    "# sn.heatmap(cm, annot=True, fmt='d')\n",
    "# plt.xlabel('Predicted')\n",
    "# plt.ylabel('Truth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "id": "ja9OuXT1JeR6"
   },
   "outputs": [],
   "source": [
    "# print(classification_report(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyOr3mgz3gCrGyQDAUGfkZNC",
   "collapsed_sections": [],
   "include_colab_link": true,
   "mount_file_id": "1STuQdII_pJMX3l-x6ARJdHEYb7N0JML5",
   "name": "Phase_01_Text_Classification.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}