diff --git a/LICENSE b/LICENSE deleted file mode 100644 index ab2fb0a..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 ml-mipt - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/Natural Language Processing exam program MSAI 21f .pdf b/Natural Language Processing exam program MSAI 21f .pdf deleted file mode 100644 index ff5d6ac..0000000 Binary files a/Natural Language Processing exam program MSAI 21f .pdf and /dev/null differ diff --git a/homeworks/assignment01_three_headed_network/README.md b/homeworks/assignment01_three_headed_network/README.md deleted file mode 100644 index 937d25a..0000000 --- a/homeworks/assignment01_three_headed_network/README.md +++ /dev/null @@ -1,2 +0,0 @@ -Assignment on more complex network: -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/homeworks/assignment01_three_headed_network/assignment01_three_headed_network.ipynb) diff --git a/homeworks/assignment01_three_headed_network/assignment01_three_headed_network.ipynb b/homeworks/assignment01_three_headed_network/assignment01_three_headed_network.ipynb deleted file mode 100644 index 8d45b31..0000000 --- a/homeworks/assignment01_three_headed_network/assignment01_three_headed_network.ipynb +++ /dev/null @@ -1,906 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "13pL--6rycN3" - }, - "source": [ - "## Homework01: Three headed network in PyTorch\n", - "\n", - "This notebook accompanies the [week02](https://github.com/girafe-ai/natural-language-processing/tree/master/week02_cnn_for_texts) practice session. Refer to that notebook for more comments.\n", - "\n", - "All the preprocessing is the same as in the classwork. *Including the data leakage in the train test split (it's still for bonus points).*" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "P8zS7m-gycN5" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "\n", - "import nltk\n", - "import tqdm\n", - "from collections import Counter" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you have already downloaded the data on the Seminar, simply run through the next cells. Otherwise uncomment the next cell (and comment the another one ;)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# uncomment and run this cell, if you don't have data locally yet.\n", - "\n", - "# !curl -L \"https://www.dropbox.com/s/5msc5ix7ndyba10/Train_rev1.csv.tar.gz?dl=1\" -o Train_rev1.csv.tar.gz\n", - "# !tar -xvzf ./Train_rev1.csv.tar.gz\n", - "\n", - "# data = pd.read_csv(\"./Train_rev1.csv\", index_col=None)\n", - "\n", - "# wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/advanced_f20/homeworks_advanced/assignment1_02_Three_headed_network/network.py" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 143 - }, - "colab_type": "code", - "id": "vwN72gd4ycOA", - "outputId": "7b9e8549-3128-4041-c4be-33fb6f326c78" - }, - "outputs": [], - "source": [ - "# run this cell if you have downloaded the dataset on the seminar\n", - "data = pd.read_csv(\"../../week02_CNN_n_Vanishing_gradient/Train_rev1.csv\", index_col=None)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 265 - }, - "colab_type": "code", - "id": "UuuKIKfrycOH", - "outputId": "e5de0f94-a4f6-4b51-db80-9d11ddc1db31" - }, - "outputs": [], - "source": [ - "data['Log1pSalary'] = np.log1p(data['SalaryNormalized']).astype('float32')\n", - "text_columns = [\"Title\", \"FullDescription\"]\n", - "categorical_columns = [\"Category\", \"Company\", \"LocationNormalized\", \"ContractType\", \"ContractTime\"]\n", - "target_column = \"Log1pSalary\"\n", - "\n", - "data[categorical_columns] = data[categorical_columns].fillna('NaN') # cast missing values to string \"NaN\"\n", - "\n", - "data.sample(3)\n", - "\n", - "\n", - "data_for_autotest = data[-5000:]\n", - "data = data[:-5000]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RUWkpd7PycOQ" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "595it [00:00, 5946.62it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tokenized:\n", - "2 mathematical modeller / simulation analyst / o...\n", - "100002 a successful and high achieving specialist sch...\n", - "200002 web designer html , css , javascript , photosh...\n", - "Name: FullDescription, dtype: object\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "239768it [00:49, 4858.44it/s]\n" - ] - } - ], - "source": [ - "tokenizer = nltk.tokenize.WordPunctTokenizer()\n", - "# see task above\n", - "def normalize(text):\n", - " text = str(text).lower()\n", - " return ' '.join(tokenizer.tokenize(text))\n", - " \n", - "data[text_columns] = data[text_columns].applymap(normalize)\n", - "\n", - "print(\"Tokenized:\")\n", - "print(data[\"FullDescription\"][2::100000])\n", - "assert data[\"FullDescription\"][2][:50] == 'mathematical modeller / simulation analyst / opera'\n", - "assert data[\"Title\"][54321] == 'international digital account manager ( german )'\n", - "\n", - "# Count how many times does each token occur in both \"Title\" and \"FullDescription\" in total\n", - "# build a dictionary { token -> it's count }\n", - "from collections import Counter\n", - "from tqdm import tqdm as tqdm\n", - "\n", - "token_counts = Counter()# \n", - "for _, row in tqdm(data[text_columns].iterrows()):\n", - " for string in row:\n", - " token_counts.update(string.split())\n", - "\n", - "# hint: you may or may not want to use collections.Counter" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2598827" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "token_counts.most_common(1)[0][1]" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 215 - }, - "colab_type": "code", - "id": "GiOWbc15ycOb", - "outputId": "1e807140-5513-4af0-d9a9-9f029059a553" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total unique tokens : 201127\n", - "('and', 2598827)\n", - "('.', 2471477)\n", - "(',', 2266256)\n", - "('the', 2036428)\n", - "('to', 1977039)\n", - "...\n", - "('dbms_stats', 1)\n", - "('dbms_output', 1)\n", - "('dbms_job', 1)\n", - "Correct!\n", - "Vocabulary size: 33795\n", - "Correct!\n", - "Correct!\n" - ] - } - ], - "source": [ - "print(\"Total unique tokens :\", len(token_counts))\n", - "print('\\n'.join(map(str, token_counts.most_common(n=5))))\n", - "print('...')\n", - "print('\\n'.join(map(str, token_counts.most_common()[-3:])))\n", - "\n", - "assert token_counts.most_common(1)[0][1] in range(2500000, 2700000)\n", - "assert len(token_counts) in range(200000, 210000)\n", - "print('Correct!')\n", - "\n", - "min_count = 10\n", - "\n", - "# tokens from token_counts keys that had at least min_count occurrences throughout the dataset\n", - "tokens = [token for token, count in token_counts.items() if count >= min_count]# \n", - "# Add a special tokens for unknown and empty words\n", - "UNK, PAD = \"UNK\", \"PAD\"\n", - "tokens = [UNK, PAD] + sorted(tokens)\n", - "print(\"Vocabulary size:\", len(tokens))\n", - "\n", - "assert type(tokens) == list\n", - "assert len(tokens) in range(32000, 35000)\n", - "assert 'me' in tokens\n", - "assert UNK in tokens\n", - "print(\"Correct!\")\n", - "\n", - "token_to_id = {token: idx for idx, token in enumerate(tokens)}\n", - "assert isinstance(token_to_id, dict)\n", - "assert len(token_to_id) == len(tokens)\n", - "for tok in tokens:\n", - " assert tokens[token_to_id[tok]] == tok\n", - "\n", - "print(\"Correct!\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JEsLeBjVycOw" - }, - "outputs": [], - "source": [ - "UNK_IX, PAD_IX = map(token_to_id.get, [UNK, PAD])\n", - "\n", - "def as_matrix(sequences, max_len=None):\n", - " \"\"\" Convert a list of tokens into a matrix with padding \"\"\"\n", - " if isinstance(sequences[0], str):\n", - " sequences = list(map(str.split, sequences))\n", - " \n", - " max_len = min(max(map(len, sequences)), max_len or float('inf'))\n", - " \n", - " matrix = np.full((len(sequences), max_len), np.int32(PAD_IX))\n", - " for i,seq in enumerate(sequences):\n", - " row_ix = [token_to_id.get(word, UNK_IX) for word in seq[:max_len]]\n", - " matrix[i, :len(row_ix)] = row_ix\n", - " \n", - " return matrix" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 179 - }, - "colab_type": "code", - "id": "JiBlPkdKycOy", - "outputId": "3866b444-1e2d-4d79-d429-fecc6d8e02a8" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Lines:\n", - "engineering systems analyst\n", - "hr assistant\n", - "senior ec & i engineer\n", - "\n", - "Matrix:\n", - "[[10705 29830 2143 1 1]\n", - " [14875 2817 1 1 1]\n", - " [27345 10107 15 15069 10702]]\n" - ] - } - ], - "source": [ - "print(\"Lines:\")\n", - "print('\\n'.join(data[\"Title\"][::100000].values), end='\\n\\n')\n", - "print(\"Matrix:\")\n", - "print(as_matrix(data[\"Title\"][::100000]))" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 53 - }, - "colab_type": "code", - "id": "DpOlBp7ZycO6", - "outputId": "30a911f2-7d35-4cb5-8991-60457b1e8bac" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "DictVectorizer(dtype=, separator='=', sort=True,\n", - " sparse=False)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sklearn.feature_extraction import DictVectorizer\n", - "\n", - "# we only consider top-1k most frequent companies to minimize memory usage\n", - "top_companies, top_counts = zip(*Counter(data['Company']).most_common(1000))\n", - "recognized_companies = set(top_companies)\n", - "data[\"Company\"] = data[\"Company\"].apply(lambda comp: comp if comp in recognized_companies else \"Other\")\n", - "\n", - "categorical_vectorizer = DictVectorizer(dtype=np.float32, sparse=False)\n", - "categorical_vectorizer.fit(data[categorical_columns].apply(dict, axis=1))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yk4jmtAYycO8" - }, - "source": [ - "### The deep learning part\n", - "\n", - "Once we've learned to tokenize the data, let's design a machine learning experiment.\n", - "\n", - "As before, we won't focus too much on validation, opting for a simple train-test split.\n", - "\n", - "__To be completely rigorous,__ we've comitted a small crime here: we used the whole data for tokenization and vocabulary building. A more strict way would be to do that part on training set only. You may want to do that and measure the magnitude of changes.\n", - "\n", - "\n", - "#### Here comes the simple one-headed network from the seminar. " - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 53 - }, - "colab_type": "code", - "id": "TngLcWA0ycO_", - "outputId": "6731b28c-07b1-41dc-9574-f76b01785bba" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Train size = 191814\n", - "Validation size = 47954\n" - ] - } - ], - "source": [ - "from sklearn.model_selection import train_test_split\n", - "\n", - "data_train, data_val = train_test_split(data, test_size=0.2, random_state=42)\n", - "data_train.index = range(len(data_train))\n", - "data_val.index = range(len(data_val))\n", - "\n", - "print(\"Train size = \", len(data_train))\n", - "print(\"Validation size = \", len(data_val))" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2PXuKgOSycPB" - }, - "outputs": [], - "source": [ - "def make_batch(data, max_len=None, word_dropout=0):\n", - " \"\"\"\n", - " Creates a keras-friendly dict from the batch data.\n", - " :param word_dropout: replaces token index with UNK_IX with this probability\n", - " :returns: a dict with {'title' : int64[batch, title_max_len]\n", - " \"\"\"\n", - " batch = {}\n", - " batch[\"Title\"] = as_matrix(data[\"Title\"].values, max_len)\n", - " batch[\"FullDescription\"] = as_matrix(data[\"FullDescription\"].values, max_len)\n", - " batch['Categorical'] = categorical_vectorizer.transform(data[categorical_columns].apply(dict, axis=1))\n", - " \n", - " if word_dropout != 0:\n", - " batch[\"FullDescription\"] = apply_word_dropout(batch[\"FullDescription\"], 1. - word_dropout)\n", - " \n", - " if target_column in data.columns:\n", - " batch[target_column] = data[target_column].values\n", - " \n", - " return batch\n", - "\n", - "def apply_word_dropout(matrix, keep_prop, replace_with=UNK_IX, pad_ix=PAD_IX,):\n", - " dropout_mask = np.random.choice(2, np.shape(matrix), p=[keep_prop, 1 - keep_prop])\n", - " dropout_mask &= matrix != pad_ix\n", - " return np.choose(dropout_mask, [matrix, np.full_like(matrix, replace_with)])" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 251 - }, - "colab_type": "code", - "id": "I6LpEQf0ycPD", - "outputId": "e3520cae-fba1-46cc-a216-56287b6e4929" - }, - "outputs": [], - "source": [ - "a = make_batch(data_train[:3], max_len=10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "But to start with let's build the simple model using only the part of the data. Let's create the baseline solution using only the description part (so it should definetely fit into the Sequential model)." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "from torch import nn\n", - "import torch.nn.functional as F" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "# You will need these to make it simple\n", - "\n", - "class Flatten(nn.Module):\n", - " def forward(self, input):\n", - " return input.view(input.size(0), -1)\n", - "\n", - "class Reorder(nn.Module):\n", - " def forward(self, input):\n", - " return input.permute((0, 2, 1))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To generate minibatches we will use simple pyton generator." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "def iterate_minibatches(data, batch_size=256, shuffle=True, cycle=False, **kwargs):\n", - " \"\"\" iterates minibatches of data in random order \"\"\"\n", - " while True:\n", - " indices = np.arange(len(data))\n", - " if shuffle:\n", - " indices = np.random.permutation(indices)\n", - "\n", - " for start in range(0, len(indices), batch_size):\n", - " batch = make_batch(data.iloc[indices[start : start + batch_size]], **kwargs)\n", - " target = batch.pop(target_column)\n", - " yield batch, target\n", - " \n", - " if not cycle: break" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "iterator = iterate_minibatches(data_train, 3)\n", - "batch, target = next(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "# Here is some startup code:\n", - "n_tokens=len(tokens)\n", - "n_cat_features=len(categorical_vectorizer.vocabulary_)\n", - "hid_size=64\n", - "simple_model = nn.Sequential()\n", - "\n", - "simple_model.add_module('emb', nn.Embedding(num_embeddings=n_tokens, embedding_dim=hid_size))\n", - "simple_model.add_module('reorder', Reorder())\n", - "simple_model.add_module('conv1', nn.Conv1d(\n", - " in_channels=hid_size,\n", - " out_channels=hid_size,\n", - " kernel_size=2)\n", - " )\n", - "simple_model.add_module('relu1', nn.ReLU())\n", - "simple_model.add_module('adapt_avg_pool', nn.AdaptiveAvgPool1d(output_size=1))\n", - "simple_model.add_module('flatten1', Flatten())\n", - "simple_model.add_module('linear1', nn.Linear(in_features=hid_size, out_features=1))\n", - "# " - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Title': array([[11439, 1467, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1],\n", - " [18664, 7252, 195, 24093, 18670, 12351, 13242, 195, 12724,\n", - " 195, 10720],\n", - " [26688, 10702, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1]], dtype=int32),\n", - " 'FullDescription': array([[30411, 26324, 33079, ..., 1, 1, 1],\n", - " [18664, 7252, 195, ..., 195, 0, 80],\n", - " [26688, 10702, 10364, ..., 1, 1, 1]], dtype=int32),\n", - " 'Categorical': array([[1., 0., 0., ..., 0., 0., 0.],\n", - " [0., 0., 0., ..., 0., 0., 0.],\n", - " [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "batch" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Remember!__ We are working with regression problem and predicting only one number." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0.0493],\n", - " [0.1251],\n", - " [0.0742]], grad_fn=)" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Try this to check your model. `torch.long` tensors are required for nn.Embedding layers.\n", - "simple_model(torch.tensor(batch['FullDescription'], dtype=torch.long))" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(3, 653)" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "batch['FullDescription'].shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And now simple training pipeline (it's commented because we've already done that in class. No need to do it again)." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHdBJREFUeJzt3X1wHPWd5/H3dx6ksR5sWbZiDAJkhzwRWB4iCByJk4XkyNMGsqES2NxiCISqvSSbLHfZheNuk9RRlRCuwmYvKYg3gThXITHL+gILHCwBcpCHcywbGxvMg0NskPCDbEu2bD3OzPf+6B5ZksexrBlpRt2fV5VK3T0909/R2J/5zbd7us3dERGR6EpUugAREZleCnoRkYhT0IuIRJyCXkQk4hT0IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScalKFwCwcOFCb2trq3QZIiKzyrp16/a4e8ux1quKoG9ra6Ojo6PSZYiIzCpmtn0y66l1IyIScQp6EZGIU9CLiERcVfToRUTKYWRkhM7OTgYHBytdSlllMhlaW1tJp9NTur+CXkQio7Ozk8bGRtra2jCzSpdTFu7O3r176ezsZMmSJVN6DLVuRCQyBgcHWbBgQWRCHsDMWLBgQUmfUhT0IhIpUQr5glKf06wO+rXb9nH7Yy+Sz+tyiCIiRzOrg37Da71876nfc3A4W+lSRERoaGiodAlFzeqgb8wE+5L7BhX0IiJHM8uDPjjU6KCCXkSqiLvzla98hTPOOIMzzzyTVatWAbBjxw6WLVvG2WefzRlnnMEzzzxDLpfjmmuuGV33jjvuKHs9s/rwyobREf1IhSsRkWrz9X99nhfeOFDWxzz9xLl89c/eecz1Vq9ezYYNG9i4cSN79uzhvPPOY9myZdx7771ceuml3HLLLeRyOfr7+9mwYQNdXV1s3rwZgN7e3rLWDLN+RB8G/ZBG9CJSPX71q19x1VVXkUwmWbRoEe973/tYu3Yt5513Hvfccw9f+9rX2LRpE42NjSxdupRXX32VL37xizz66KPMnTu37PXM6hF9Y6169CJS3GRG3jNt2bJlPP300zz88MNcc8013HjjjVx99dVs3LiRxx57jLvuuov77ruPu+++u6zbneUj+qBHr9aNiFST9773vaxatYpcLkd3dzdPP/00559/Ptu3b2fRokV87nOf4/rrr2f9+vXs2bOHfD7PJz/5SW699VbWr19f9npm94g+bN1oZ6yIVJNPfOIT/Pa3v+Wss87CzPjWt77FCSecwMqVK7n99ttJp9M0NDTw4x//mK6uLq699lry+TwA3/jGN8pez6wO+rqaJAlT60ZEqsPBgweB4Just99+O7fffvu425cvX87y5cuPuN90jOLHmtWtGzOjoTbFQe2MFRE5qlkd9BD06Q+oRy8iclQRCPqUWjciMso9eue+KvU5RSLotTNWRCC4QMfevXsjFfaF89FnMpkpP8as3hkLQetmd1+0riYjIlPT2tpKZ2cn3d3dlS6lrApXmJqqYwa9md0NfAzY7e5nhMuagVVAG7AN+JS791hw0uTvAB8B+oFr3H1adyc3ZlJs3a0RvYhAOp2e8lWYomwyrZsfAR+asOwm4Al3fwvwRDgP8GHgLeHPDcCd5Snz6Jrra+g5NDzdmxERmbWOGfTu/jSwb8Liy4CV4fRK4PIxy3/sgf8HNJnZ4nIVW8zChlr6hrIMjuSmczMiIrPWVHfGLnL3HeH0TmBROH0S8PqY9TrDZUcwsxvMrMPMOkrppzXX1wCwT6N6EZGiSj7qxoPd28e9i9vdV7h7u7u3t7S0THn7C8Kg33tQQS8iUsxUg35XoSUT/t4dLu8CTh6zXmu4bNqMjuj7FfQiIsVMNegfBAonbFgOPDBm+dUWuADYP6bFMy3qw1MVH9JpEEREiprM4ZU/Bd4PLDSzTuCrwDeB+8zsOmA78Klw9UcIDq3cSnB45bXTUPM4DWHQ63w3IiLFHTPo3f2qo9x0SZF1Hfh8qUUdD43oRUT+uFl/CoT62iSgoBcROZpZH/S1qSTppHFwSMfRi4gUM+uDHoL2jUb0IiLFRSPoaxT0IiJHE4mgb6pL6zh6EZGjiETQt86fQ2fPQKXLEBGpSpEI+pPn19HZ0x+piw2IiJRLJIK+bWE9gyN5tu4+WOlSRESqTiSC/tJ3ngDA41t2VbgSEZHqE4mgb2msJZNO6AIkIiJFRCLoIbh2bJ8uEi4icoTIBP3cTIoDgyOVLkNEpOpEJ+jnaEQvIlJMZIK+MZPmwIBG9CIiE0Um6OdmUhrRi4gUEZmgb8yk1aMXESkiMkE/d06KAxrRi4gcITpBn0kznM0zOKLz0ouIjBWhoA8uKag+vYjIeJEJ+sZMGkB9ehGRCSIT9HPnaEQvIlJMdIK+MKLXsfQiIuNEJugLrRuN6EVExotM0BdaN+rRi4iMF5mgb1TrRkSkqMgEfX1NkoSpdSMiMlFkgt7MdBoEEZEiSgp6M/sbM3vezDab2U/NLGNmS8xsjZltNbNVZlZTrmKPZe4cndhMRGSiKQe9mZ0E/DXQ7u5nAEngSuA24A53Pw3oAa4rR6GT0TSnht19gzO1ORGRWaHU1k0KmGNmKaAO2AFcDNwf3r4SuLzEbUxae9t8Orb1MDCs892IiBRMOejdvQv4H8BrBAG/H1gH9Lp7oX/SCZxU7P5mdoOZdZhZR3d391TLGOfit7+JoWye3/x+T1keT0QkCkpp3cwHLgOWACcC9cCHJnt/d1/h7u3u3t7S0jLVMsY5f0kzyYTx7Gu9ZXk8EZEoKKV18wHgD+7e7e4jwGrgIqApbOUAtAJdJdY4abWpJPU1Sfp05I2IyKhSgv414AIzqzMzAy4BXgCeAq4I11kOPFBaicenMZPm4JB69CIiBaX06NcQ7HRdD2wKH2sF8HfAjWa2FVgA/LAMdU5afW2SQ0M6xFJEpCB17FWOzt2/Cnx1wuJXgfNLedxS1NemODSsoBcRKYjMN2MLGmpTHNSIXkRkVOSCvr4mpdaNiMgY0Qv62hSHtDNWRGRU5IK+MZPSic1ERMaIXNA31aXpG8ySzeUrXYqISFWIXNA31wcny+zVBUhERIAIBn1TXRD0PYeGK1yJiEh1iFzQz68LLinY068RvYgIRDLowxF9v0b0IiIQxaCvV+tGRGSs6AW9WjciIuNELujnpJPUphJq3YiIhCIX9GbG/LoatW5EREKRC3oIvjSlEb2ISCCSQd+2oJ4tO/pw90qXIiJScZEM+otOW0BX7wCbuvZXuhQRkYqLZNBfds5J1CQTPPzcjkqXIiJScZEM+rmZNHW1SQZGdLpiEZFIBj1AbSrBcFZnsBQRiWzQ1yjoRUSACAd9bSrJkIJeRCS6QV+TTCjoRUSIcNDXphMMZbUzVkQkskGfMOOZV/awY/9ApUsREamoyAb9uu09APz9A89XuBIRkcqKbNCbBb/zeZ0GQUTiLbJBn0oESW+FxBcRiamSgt7MmszsfjN70cy2mNmFZtZsZo+b2Svh7/nlKvZ4jOSCkXwysm9lIiKTU2oMfgd41N3fDpwFbAFuAp5w97cAT4TzFZPQiF5EYm7KQW9m84BlwA8B3H3Y3XuBy4CV4WorgctLLbIUynkRibtSRvRLgG7gHjN71sx+YGb1wCJ3L5w2ciewqNQip+KzFy0BoH9Yx9KLSLyVEvQp4FzgTnc/BzjEhDaNB1f+KHrYi5ndYGYdZtbR3d1dQhnF/f2fnc5Fpy3g4GC27I8tIjKblBL0nUCnu68J5+8nCP5dZrYYIPy9u9id3X2Fu7e7e3tLS0sJZRxdQ22Kg0MKehGJtykHvbvvBF43s7eFiy4BXgAeBJaHy5YDD5RUYQkaatP0aUQvIjGXKvH+XwR+YmY1wKvAtQRvHveZ2XXAduBTJW5jyhozKfoGRyq1eRGRqlBS0Lv7BqC9yE2XlPK45VJo3bi7vjglIrEV6a8TNWRS5B1dUlBEYi3SQd+YCT6w6MgbEYmzSAd9Q20Q9AcU9CISY5EO+rlz0gAc0A5ZEYmxSAd9Uxj0+/sV9CISX9EO+roaAHoHhitciYhI5UQ76MMRfa9G9CISY5EO+rkKehGRaAd9MmHMzaTYP6CgF5H4inTQQ9CnV9CLSJxFPujnzUnT26+dsSISX5EP+qa6NL0a0YtIjEU+6OfNSes4ehGJtcgHfVNdmh61bkQkxiIf9Cc11dHTP8LabfsqXYqISEVEPug/fd7JAHRs66lwJSIilRH5oG+ur6GxNsWuA4OVLkVEpCIiH/QAi+Zl2LlfQS8i8RSLoD9hboadGtGLSEzFIugXzc2odSMisRWLoD9hXi27+4bI5b3SpYiIzLh4BP3cDLm8s+fgUKVLERGZcbEI+sXz5gDwRu9AhSsREZl5sQj6JS31APxhz6EKVyIiMvNiEfSnNNeRShhbdx+sdCkiIjMuFkGfTiY47U0NbOraX+lSRERmXCyCHuC8tmbWbe/RkTciEjslB72ZJc3sWTN7KJxfYmZrzGyrma0ys5rSyyzdW09opH84pyNvRCR2yjGi/xKwZcz8bcAd7n4a0ANcV4ZtlOykpgwAXTryRkRipqSgN7NW4KPAD8J5Ay4G7g9XWQlcXso2yuXEJh1iKSLxVOqI/h+AvwXy4fwCoNfds+F8J3BSidsoizc1BiP6PX1q3YhIvEw56M3sY8Bud183xfvfYGYdZtbR3d091TImLZMOnupgNn+MNUVEoqWUEf1FwMfNbBvwM4KWzXeAJjNLheu0Al3F7uzuK9y93d3bW1paSihjcmpTSQCGRhT0IhIvUw56d7/Z3VvdvQ24EnjS3T8DPAVcEa62HHig5CrLIJkw0kljKJurdCkiIjNqOo6j/zvgRjPbStCz/+E0bGNKalNJhtS6EZGYSR17lWNz918CvwynXwXOL8fjllttKsHgiEb0IhIvsflmLARBv+H13kqXISIyo2IV9G/sH+T5Nw7w5Iu7Kl2KiMiMiVXQF7y+T1+aEpH4iGXQz6lJVroEEZEZE8ugz6QV9CISH7EM+mxOh1iKSHzEMuh1LL2IxEmsgv7Oz5wLwG2Pvoi7LkAiIvEQq6Bf9tbgnDq9/SP09o9UuBoRkZkRq6CvTcXq6YqIADEL+lTy8NPN6tqxIhITsQr6sbJ57ZAVkXiIb9DnNKIXkXiIb9CrdSMiMRHfoNeXpkQkJmIX9Hf9h+BYeo3oRSQuYhf0yUTwlNWjF5G4iF3Qp5IGwIiOuhGRmIhf0CeCoM+pdSMiMRHDoA+e8oh2xopITMQu6NNJjehFJF5iF/TJsHWjnbEiEhexC/p0Uq0bEYmX2AV9Sq0bEYmZ+AV9onB4pYJeROIhhkFf+MKUWjciEg+xC/rRnbEa0YtITEw56M3sZDN7ysxeMLPnzexL4fJmM3vczF4Jf88vX7mlK+yMfWTTjgpXIiIyM0oZ0WeB/+TupwMXAJ83s9OBm4An3P0twBPhfNUo7Iz95Uvd7D4wWOFqRESm35SD3t13uPv6cLoP2AKcBFwGrAxXWwlcXmqR5VTYGQvQsb2ngpWIiMyMsvTozawNOAdYAyxy90JfZCewqBzbKJdMOjk6/dq+/gpWIiIyM0oOejNrAP4F+LK7Hxh7m7s7UHSvp5ndYGYdZtbR3d1dahmTlkkn+eV/fj+NmRTb9yroRST6Sgp6M0sThPxP3H11uHiXmS0Ob18M7C52X3df4e7t7t7e0tJSShnHrW1hPW9uaeDlXX0E70UiItFVylE3BvwQ2OLu3x5z04PA8nB6OfDA1MubPgsbalm3vYfvPrm10qWIiEyrUkb0FwF/CVxsZhvCn48A3wQ+aGavAB8I56vO/oFhAH6xZVeFKxERmV6pqd7R3X8F2FFuvmSqjztTbvno6Vz+vV/zrlObK12KiMi0it03YwvOPrmJ+pokdrS3KhGRiIht0APUpBI6XbGIRF6sgz6dTDCcVdCLSLQp6DWiF5GIi3nQG6vXdzE4kqt0KSIi0ybWQb8t/Gbs1//1hQpXIiIyfWId9AWPv6Bj6UUkuhT0QC6vPr2IRJeCnsNXnRIRiSIFPZDQt6ZEJMIU9Iy/GImISNQo6AHTiF5EIkxBDxwcypLP67z0IhJNCnpg/8AI//WBzZUuQ0RkWsQ66L//l+8anb53zWsVrEREZPrEOugvfecJXH72iUBwOgQRkSiKddADDITnuRnJOe23Ps7+gZEKVyQiUl4K+pHD34rdc3CYV3b1jb99OMe3H39ZJz4TkVlrypcSjIqB4ey4+S07+2hva2Ykl+eh597gjd5B/vGJV5ibSXH9e5dWqEoRkamL/Yj+v3zkHePm/9vPg6Nvvv9/f8/frNrI6vWdAPT2T76lk887T764C3cdsikilRf7oD/nlPlHLMvm8nT1DgDQ3TcEwHef2son7/zNMR9v5/5Blt/zOz77ow7+eV3n6HJ354o7f8NDz71RpspFRCYn9kFfTP9IjsJg/MDg4dbOuu09AHT29HPz6k109vSPu99QNsenV/yWZ17ZAzCu37/v0DAd23v4wr3PTnP1IiLjxb5HX0z/UI6RXPG2y4HBEf7in9bw2r5+duwf4EfXnj962+d/sp7tew+H/8Ghw28Sf9hzCID5delpqlpEpDiN6IHPvPuUcfMv7jzAa/sOFV33rl/+ntf2BWG+5tV95PPOxtd7+fmzXfxiy+5x63Zs6+FrDz7Ps6/1jH4aWNhQO26dfN7J5x1359db9+DuPPXibjq27SvX0xORmLNq2GHY3t7uHR0dFa1h/8AIZ33930bnUwnj5OY6Tm6u48R5GX629vWi9zODyfwJG2tT9A1laT91Pvf/1b/D3XnPbU/R1TvAn76thUvfeQI3rd7Ed//inNH2zrZvfpTe/mF29w3x1kWNZXmeIhIdZrbO3duPtZ5aN6E56eS4+YZMipXXns8pC+oYyub43LKl3Lx6E7/7w/iR9tiQL4R5MYXlHdt7aLvpYebXpekJj+R56qVuTmyaAzD6aQHg0c07+eufPstwLs8X/vQ0LnzzAi46bWHJz1VE4kUj+pC7s+TmR0bnf3vzxSyeN+eIddZu6+HQcJamOWlW/mYbP99w+Ciab/75mdy0etPo/AVLm1m3vYc/P6eVVR3FPxEcrw+evojPXrSE89rmk807ubxTX6v3a5E4quiI3sw+BHwHSAI/cPdvTsd2ymnsOek/1d56RMgX1jl/SfPo/DmnzOeOT5/NE1t2c/2PO8aNti9Y2sz/uu7d5PJOOpngc8uW0NkzwNMv76F1/hz+6ZlX2bF/sGgtS1vqSScSvDThW7oQXMh87MXM00njpf/+YZ7r2s93n9zKbZ88kwUT9gOISLyVfURvZkngZeCDQCewFrjK3V842n2qYUQPwY7RRzbv4APvWERmQitnsh7Y0MWpC+o5q3XepC5osu/QMNevXMuftDZRV5Pk0c07eeALF7Hi6Vf5n09undQ2WxqDYC8c83/r5WfQXF/Dr7fuoad/mC9/4K20LainJqV97yJRMtkR/XQE/YXA19z90nD+ZgB3/8bR7lMtQV9N9g+M8OWfPcu7Tp1Px/Ye3OHqC0/l5tWbOK+tmYc37Ri3fjJhXLC0mV9v3XvUx2yur+GdJ86lNpXkzW+qJ5NKks3nSSYSnNJcR00qwQtvHOAdixtpzKQYGM5zwrxa5mbS5B1qUwnm19UAMJLPM5TNk8s5tekEyYSRTiSwBCTNyLuTMCOZMIZzeUayeZIJI5EwkuFy9+C7BzWpBDXJBKlkgpFcnp5Dw5gZmXSC+poUiYSNfsu48M/V4fCy8PlZ+HfI5X30gu9j32xzeSdh0D+c48DgCM31NdQkE5gFjz+UzVObSoy7TzaXxzn8nLITLlBTWNWwI5YVajq83IosO/J2iQ/3oP2aSk5tEFbJ1s1JwNiGdCfw7mnYTqTNm5PmnjHH6Bf87pZFAHx3TPA9/8YBkgnj9BPn8vKuPoZG8ry0q49nXunGgNb5dazdto+Xd/WxfW8/AyM5frFl1xGPXWkJCy7UPjFMJ8ssCOSx9zcLjqBKmDGUzZNO2rjvSCQs2BE/MJKjcLeEQToZvPkcGs5S6YuPHX4zGbtszBtL0XWLv9sc9U0GK7LsyO2ZBW+mifDNMXjDDdaZ+MaLj/s1evucmiTZvFMbfsIcGD7+EwYe75vi8b6HHu9b7vHUUxgEAYzk8tx6+RlcdvZJx7nF41OxvXhmdgNwA8App5xyjLVlorH/8c5snTe6vHAY5pmt87jiXa1HvX9v/zB7Dg7RtqCeA4NZ3ugdIJmw8I0gS3N9LQcGRugPg66hNsVQNj96Gmd3H61hKJujoTbFSC74TsBwLk9NMjE6Aq5NJUgnE+TyTj4cweTccQ9CdjiXZzgb/OTdWdhQSzppDI7kOTScHQ2SiSFmdvg/pBkMZ/Nk804mnSQfPr6HNWTzTjppZHPO/Poa6muS7Ng/SMKMgZEcdTXJMNhzpJPGcDb4xNKYSVETftJwIJNOjtZR7MPw2E/IY2/3osv++Lpjb/AjFx31MSazbrHJY9UOjL6GeXcMI1F4TSYEXbHXKpgPPq1m0kmGs3kcp65memPoeLsWx/u+frxNkcK/1UTCyOedk5qO3B9YbtPxF+4CTh4z3xouG8fdVwArIGjdTEMd8kc01dXQFLZhmutraK4Ppt+xeG4lyxKRaTAde+fWAm8xsyVmVgNcCTw4DdsREZFJKPuI3t2zZvYF4DGCwyvvdvfny70dERGZnGlpjrn7I8Ajx1xRRESmnQ6sFhGJOAW9iEjEKehFRCJOQS8iEnEKehGRiKuK0xSbWTewfYp3XwjsKWM500E1lq7a64Pqr7Ha6wPVeLxOdfeWY61UFUFfCjPrmMxJfSpJNZau2uuD6q+x2usD1Thd1LoREYk4Bb2ISMRFIehXVLqASVCNpav2+qD6a6z2+kA1TotZ36MXEZE/LgojehER+SNmddCb2YfM7CUz22pmN1WwjrvNbLeZbR6zrNnMHjezV8Lf88PlZmb/GNb8nJmdOwP1nWxmT5nZC2b2vJl9qQprzJjZ78xsY1jj18PlS8xsTVjLqvDU15hZbTi/Nby9bbprDLebNLNnzeyhKq1vm5ltMrMNZtYRLqum17nJzO43sxfNbIuZXVhl9b0t/NsVfg6Y2ZerqcYpcfdZ+UNwCuTfA0uBGmAjcHqFalkGnAtsHrPsW8BN4fRNwG3h9EeA/0NwsZ0LgDUzUN9i4NxwupHg4u2nV1mNBjSE02lgTbjt+4Arw+V3AX8VTv9H4K5w+kpg1Qy91jcC9wIPhfPVVt82YOGEZdX0Oq8Erg+na4CmaqpvQq1JYCdwarXWOOnnUukCSngRLgQeGzN/M3BzBetpmxD0LwGLw+nFwEvh9PeBq4qtN4O1PgB8sFprBOqA9QTXGt4DpCa+5gTXO7gwnE6F69k019UKPAFcDDwU/ueumvrCbRUL+qp4nYF5wB8m/h2qpb4i9f574NfVXONkf2Zz66bYRcin9wq7x2eRu+8Ip3cCi8LpitYdthDOIRgxV1WNYVtkA7AbeJzgE1uvu2eL1DFaY3j7fmDBNJf4D8DfAvlwfkGV1QfBJU//zczWWXBdZqie13kJ0A3cE7a/fmBm9VVU30RXAj8Np6u1xkmZzUE/a3jwVl/xw5vMrAH4F+DL7n5g7G3VUKO759z9bIKR8/nA2ytZz1hm9jFgt7uvq3Qtx/Aedz8X+DDweTNbNvbGCr/OKYIW553ufg5wiKANMqoa/h0ChPtaPg7888TbqqXG4zGbg35SFyGvoF1mthgg/L07XF6Rus0sTRDyP3H31dVYY4G79wJPEbRCmsyscCW0sXWM1hjePg/YO41lXQR83My2AT8jaN98p4rqA8Ddu8Lfu4H/TfCGWS2vcyfQ6e5rwvn7CYK/Wuob68PAenffFc5XY42TNpuDvtovQv4gsDycXk7QFy8svzrcW38BsH/MR8JpYWYG/BDY4u7frtIaW8ysKZyeQ7APYQtB4F9xlBoLtV8BPBmOtKaFu9/s7q3u3kbwb+1Jd/9MtdQHYGb1ZtZYmCboMW+mSl5nd98JvG5mbwsXXQK8UC31TXAVh9s2hVqqrcbJq/ROglJ+CPZ4v0zQy72lgnX8FNgBjBCMWq4j6Mc+AbwC/AJoDtc14HthzZuA9hmo7z0EHzWfAzaEPx+pshr/BHg2rHEz8Pfh8qXA74CtBB+ja8PlmXB+a3j70hl8vd/P4aNuqqa+sJaN4c/zhf8TVfY6nw10hK/zz4H51VRfuN16gk9f88Ysq6oaj/dH34wVEYm42dy6ERGRSVDQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJx/x/1al1il/BnyQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# from IPython.display import clear_output\n", - "# from random import sample\n", - "\n", - "# epochs = 1\n", - "\n", - "# model = simple_model\n", - "# opt = torch.optim.Adam(model.parameters())\n", - "# loss_func = nn.MSELoss()\n", - "\n", - "# history = []\n", - "# for epoch_num in range(epochs):\n", - "# for idx, (batch, target) in enumerate(iterate_minibatches(data_train)):\n", - "# # Preprocessing the batch data and target\n", - "# batch = torch.tensor(batch['FullDescription'], dtype=torch.long)\n", - "\n", - "# target = torch.tensor(target)\n", - "\n", - "\n", - "# predictions = model(batch)\n", - "# predictions = predictions.view(predictions.size(0))\n", - "\n", - "# loss = loss_func(predictions, target)# \n", - "\n", - "# # train with backprop\n", - "# loss.backward()\n", - "# opt.step()\n", - "# opt.zero_grad()\n", - "# # \n", - "\n", - "# history.append(loss.data.numpy())\n", - "# if (idx+1)%10==0:\n", - "# clear_output(True)\n", - "# plt.plot(history,label='loss')\n", - "# plt.legend()\n", - "# plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Actual homework starts here\n", - "__Your ultimate task is to code the three headed network described on the picture below.__ \n", - "To make it closer to the real world, please store the network code in file `network.py` in this directory. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0eI5h9UMycPF" - }, - "source": [ - "#### Architecture\n", - "\n", - "Our main model consists of three branches:\n", - "* Title encoder\n", - "* Description encoder\n", - "* Categorical features encoder\n", - "\n", - "We will then feed all 3 branches into one common network that predicts salary.\n", - "\n", - "\n", - "\n", - "This clearly doesn't fit into PyTorch __Sequential__ interface. To build such a network, one will have to use [__PyTorch nn.Module API__](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import network" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Re-run this cell if you updated the file with network source code\n", - "import imp\n", - "imp.reload(network)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = network.ThreeInputsNet(\n", - " n_tokens=len(tokens),\n", - " n_cat_features=len(categorical_vectorizer.vocabulary_),\n", - "\n", - " # this parameter defines the number of the inputs in the layer,\n", - " # which stands after the concatenation. In should be found out by you.\n", - " concat_number_of_features= \n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "testing_batch, _ = next(iterate_minibatches(data_train, 3))\n", - "testing_batch = [\n", - " torch.tensor(testing_batch['Title'], dtype=torch.long),\n", - " torch.tensor(testing_batch['FullDescription'], dtype=torch.long),\n", - " torch.tensor(testing_batch['Categorical'])\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert model(testing_batch).shape == torch.Size([3, 1])\n", - "assert model(testing_batch).dtype == torch.float32\n", - "print('Seems fine!')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now train the network for a while (100 batches would be fine)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Training pipeline comes here (almost the same as for the simple_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, to evaluate the model it can be switched to `eval` state." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.eval()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_submission(model, data, batch_size=256, name=\"\", three_inputs_mode=True, **kw):\n", - " squared_error = abs_error = num_samples = 0.0\n", - " output_list = []\n", - " for batch_x, batch_y in tqdm(iterate_minibatches(data, batch_size=batch_size, shuffle=False, **kw)):\n", - " if three_inputs_mode:\n", - " batch = [\n", - " torch.tensor(batch_x['Title'], dtype=torch.long),\n", - " torch.tensor(batch_x['FullDescription'], dtype=torch.long),\n", - " torch.tensor(batch_x['Categorical'])\n", - " ]\n", - " else:\n", - " batch = torch.tensor(batch_x['FullDescription'], dtype=torch.long)\n", - "\n", - " batch_pred = model(batch)[:, 0].detach().numpy()\n", - " \n", - " output_list.append((list(batch_pred), list(batch_y)))\n", - " \n", - " squared_error += np.sum(np.square(batch_pred - batch_y))\n", - " abs_error += np.sum(np.abs(batch_pred - batch_y))\n", - " num_samples += len(batch_y)\n", - " print(\"%s results:\" % (name or \"\"))\n", - " print(\"Mean square error: %.5f\" % (squared_error / num_samples))\n", - " print(\"Mean absolute error: %.5f\" % (abs_error / num_samples))\n", - " \n", - "\n", - " batch_pred = [c for x in output_list for c in x[0]]\n", - " batch_y = [c for x in output_list for c in x[1]]\n", - " output_df = pd.DataFrame(list(zip(batch_pred, batch_y)), columns=['batch_pred', 'batch_y'])\n", - " output_df.to_csv('submission.csv', index=False)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "generate_submission(model, data_for_autotest, name='Submission')\n", - "print('Submission file generated')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Both the notebook and the `.py` file are required to submit this homework.__" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "CNN_for_texts.ipynb", - "provenance": [], - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Py3 research env", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/assignment01_three_headed_network/network.py b/homeworks/assignment01_three_headed_network/network.py deleted file mode 100644 index 07decef..0000000 --- a/homeworks/assignment01_three_headed_network/network.py +++ /dev/null @@ -1,50 +0,0 @@ - -import numpy as np -import pandas as pd - -import torch -from torch import nn -import torch.nn.functional as F - -import tqdm - - -class ThreeInputsNet(nn.Module): - def __init__(self, n_tokens, n_cat_features, concat_number_of_features, hid_size=64): - super(ThreeInputsNet, self).__init__() - self.title_emb = nn.Embedding(n_tokens, embedding_dim=hid_size) - # - - self.full_emb = nn.Embedding(num_embeddings=n_tokens, embedding_dim=hid_size) - # - - self.category_out = # - - - # Example for the final layers (after the concatenation) - self.inter_dense = nn.Linear(in_features=concat_number_of_features, out_features=hid_size*2) - self.final_dense = nn.Linear(in_features=hid_size*2, out_features=1) - - - - def forward(self, whole_input): - input1, input2, input3 = whole_input - title_beg = self.title_emb(input1).permute((0, 2, 1)) - title = # - - full_beg = self.full_emb(input2).permute((0, 2, 1)) - full = # - - category = # - - concatenated = torch.cat( - [ - title.view(title.size(0), -1), - full.view(full.size(0), -1), - category.view(category.size(0), -1) - ], - dim=1) - - out = # - - return out \ No newline at end of file diff --git a/homeworks/assignment02_attention_scores/README.md b/homeworks/assignment02_attention_scores/README.md deleted file mode 100644 index 3d9f15b..0000000 --- a/homeworks/assignment02_attention_scores/README.md +++ /dev/null @@ -1 +0,0 @@ -Please, refer to week04 attention notebook and finish the concat and general attention scores. diff --git a/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_par1_Embedding_based_MT-checkpoint.ipynb b/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_par1_Embedding_based_MT-checkpoint.ipynb deleted file mode 100644 index 0f42882..0000000 --- a/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_par1_Embedding_based_MT-checkpoint.ipynb +++ /dev/null @@ -1,753 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eulvfJWl7ueY" - }, - "source": [ - "# Lab 1\n", - "\n", - "\n", - "## Part 1: Bilingual dictionary induction and unsupervised embedding-based MT (30%)\n", - "*Note: this homework is based on materials from yandexdataschool [NLP course](https://github.com/yandexdataschool/nlp_course/). Feel free to check this awesome course if you wish to dig deeper.*\n", - "\n", - "*Refined by [Nikolay Karpachev](https://www.linkedin.com/in/nikolay-karpachev-b0146a104/)*" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fV4rIjxa7uei" - }, - "source": [ - "**In this homework** **YOU** will make machine translation system without using parallel corpora, alignment, attention, 100500 depth super-cool recurrent neural network and all that kind superstuff.\n", - "\n", - "But even without parallel corpora this system can be good enough (hopefully), in particular for similar languages, e.g. Ukrainian and Russian. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idSYq2GU7uew" - }, - "source": [ - "### Frament of the Swadesh list for some slavic languages\n", - "\n", - "The Swadesh list is a lexicostatistical stuff. It's named after American linguist Morris Swadesh and contains basic lexis. This list are used to define subgroupings of languages, its relatedness.\n", - "\n", - "So we can see some kind of word invariance for different Slavic languages.\n", - "\n", - "\n", - "| Russian | Belorussian | Ukrainian | Polish | Czech | Bulgarian |\n", - "|-----------------|--------------------------|-------------------------|--------------------|-------------------------------|-----------------------|\n", - "| женщина | жанчына, кабета, баба | жінка | kobieta | žena | жена |\n", - "| мужчина | мужчына | чоловік, мужчина | mężczyzna | muž | мъж |\n", - "| человек | чалавек | людина, чоловік | człowiek | člověk | човек |\n", - "| ребёнок, дитя | дзіця, дзіцёнак, немаўля | дитина, дитя | dziecko | dítě | дете |\n", - "| жена | жонка | дружина, жінка | żona | žena, manželka, choť | съпруга, жена |\n", - "| муж | муж, гаспадар | чоловiк, муж | mąż | muž, manžel, choť | съпруг, мъж |\n", - "| мать, мама | маці, матка | мати, матір, неня, мама | matka | matka, máma, 'стар.' mateř | майка |\n", - "| отец, тятя | бацька, тата | батько, тато, татусь | ojciec | otec | баща, татко |\n", - "| много | шмат, багата | багато | wiele | mnoho, hodně | много |\n", - "| несколько | некалькі, колькі | декілька, кілька | kilka | několik, pár, trocha | няколко |\n", - "| другой, иной | іншы | інший | inny | druhý, jiný | друг |\n", - "| зверь, животное | жывёла, звер, істота | тварина, звір | zwierzę | zvíře | животно |\n", - "| рыба | рыба | риба | ryba | ryba | риба |\n", - "| птица | птушка | птах, птиця | ptak | pták | птица |\n", - "| собака, пёс | сабака | собака, пес | pies | pes | куче, пес |\n", - "| вошь | вош | воша | wesz | veš | въшка |\n", - "| змея, гад | змяя | змія, гад | wąż | had | змия |\n", - "| червь, червяк | чарвяк | хробак, черв'як | robak | červ | червей |\n", - "| дерево | дрэва | дерево | drzewo | strom, dřevo | дърво |\n", - "| лес | лес | ліс | las | les | гора, лес |\n", - "| палка | кій, палка | палиця | patyk, pręt, pałka | hůl, klacek, prut, kůl, pálka | палка, пръчка, бастун |" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cNM3_fjr7ue2" - }, - "source": [ - "But the context distribution of these languages demonstrates even more invariance. And we can use this fact for our for our purposes." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YLppwa527ue6" - }, - "source": [ - "## Data" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lYBGKAUn7ue_" - }, - "outputs": [], - "source": [ - "import gensim\n", - "import numpy as np\n", - "from gensim.models import KeyedVectors" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MwGoVhRA7ufP" - }, - "source": [ - "In this notebook we're going to use pretrained word vectors - FastText (original paper - https://arxiv.org/abs/1607.04606).\n", - "\n", - "You can download them from the official [website](https://fasttext.cc/docs/en/crawl-vectors.html). We're going to need embeddings for Russian and Ukrainian languages. Please use word2vec-compatible format (.text)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "u1JjQv_97ufT" - }, - "outputs": [], - "source": [ - "uk_emb = KeyedVectors.load_word2vec_format(\"cc.uk.300.vec\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ffzuept_7ufd" - }, - "outputs": [], - "source": [ - "ru_emb = KeyedVectors.load_word2vec_format(\"cc.ru.300.vec\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nTkXfT0W7ufk" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([ru_emb[\"август\"]], topn=10)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vdBA8lcg7ufs" - }, - "outputs": [], - "source": [ - "uk_emb.most_similar([uk_emb[\"серпень\"]])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_yJvcKXO7uf0" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([uk_emb[\"серпень\"]])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pNdYAR1q7uf6" - }, - "source": [ - "Load small dictionaries for correspoinding words pairs as trainset and testset." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "35d_DAK67uf8" - }, - "outputs": [], - "source": [ - "def load_word_pairs(filename):\n", - " uk_ru_pairs = []\n", - " uk_vectors = []\n", - " ru_vectors = []\n", - " with open(filename, \"r\") as inpf:\n", - " for line in inpf:\n", - " uk, ru = line.rstrip().split(\"\\t\")\n", - " if uk not in uk_emb or ru not in ru_emb:\n", - " continue\n", - " uk_ru_pairs.append((uk, ru))\n", - " uk_vectors.append(uk_emb[uk])\n", - " ru_vectors.append(ru_emb[ru])\n", - " return uk_ru_pairs, np.array(uk_vectors), np.array(ru_vectors)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wkNL602WHJyO" - }, - "outputs": [], - "source": [ - "!wget -O ukr_rus.train.txt http://tiny.cc/jfgecz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uoclU6JcHCcn" - }, - "outputs": [], - "source": [ - "!wget -O ukr_rus.test.txt http://tiny.cc/6zoeez" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "05BqsdSK7ugD" - }, - "outputs": [], - "source": [ - "uk_ru_train, X_train, Y_train = load_word_pairs(\"ukr_rus.train.txt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zQOZw51r7ugL" - }, - "outputs": [], - "source": [ - "uk_ru_test, X_test, Y_test = load_word_pairs(\"ukr_rus.test.txt\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ZBBNvpz7ugQ" - }, - "source": [ - "## Embedding space mapping (0.3 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_Dhk5gL7ugS" - }, - "source": [ - "Let $x_i \\in \\mathrm{R}^d$ be the distributed representation of word $i$ in the source language, and $y_i \\in \\mathrm{R}^d$ is the vector representation of its translation. Our purpose is to learn such linear transform $W$ that minimizes euclidian distance between $Wx_i$ and $y_i$ for some subset of word embeddings. Thus we can formulate so-called Procrustes problem:\n", - "\n", - "$$W^*= \\arg\\min_W \\sum_{i=1}^n||Wx_i - y_i||_2$$\n", - "or\n", - "$$W^*= \\arg\\min_W ||WX - Y||_F$$\n", - "\n", - "where $||*||_F$ - Frobenius norm." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "acOjDdtL7ugY" - }, - "source": [ - "$W^*= \\arg\\min_W \\sum_{i=1}^n||Wx_i - y_i||_2$ looks like simple multiple linear regression (without intercept fit). So let's code." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Lb-KN1be7uga" - }, - "outputs": [], - "source": [ - "from sklearn.linear_model import LinearRegression\n", - "\n", - "# YOUR CODE HERE\n", - "# mapping = ...\n", - "# -------" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "X7tqJwoY7ugf" - }, - "source": [ - "Let's take a look at neigbours of the vector of word _\"серпень\"_ (_\"август\"_ in Russian) after linear transform." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "31SrFSbn7ugi" - }, - "outputs": [], - "source": [ - "august = mapping.predict(uk_emb[\"серпень\"].reshape(1, -1))\n", - "ru_emb.most_similar(august)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "okSkjk597ugo" - }, - "source": [ - "We can see that neighbourhood of this embedding cosists of different months, but right variant is on the ninth place." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o2uY6Y9B7ugt" - }, - "source": [ - "As quality measure we will use precision top-1, top-5 and top-10 (for each transformed Ukrainian embedding we count how many right target pairs are found in top N nearest neighbours in Russian embedding space)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zptuho8LAfIE" - }, - "outputs": [], - "source": [ - "def precision(pairs, mapped_vectors, topn=1):\n", - " \"\"\"\n", - " :args:\n", - " pairs = list of right word pairs [(uk_word_0, ru_word_0), ...]\n", - " mapped_vectors = list of embeddings after mapping from source embedding space to destination embedding space\n", - " topn = the number of nearest neighbours in destination embedding space to choose from\n", - " :returns:\n", - " precision_val, float number, total number of words for those we can find right translation at top K.\n", - " \"\"\"\n", - " assert len(pairs) == len(mapped_vectors)\n", - " num_matches = 0\n", - " for i, (_, ru) in enumerate(pairs):\n", - " # YOUR CODE HERE\n", - " precision_val = num_matches / len(pairs)\n", - " return precision_val" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "duhj9hpv7ugy" - }, - "outputs": [], - "source": [ - "assert precision([(\"серпень\", \"август\")], august, topn=5) == 0.0\n", - "assert precision([(\"серпень\", \"август\")], august, topn=9) == 1.0\n", - "assert precision([(\"серпень\", \"август\")], august, topn=10) == 1.0" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0-iyd5gP7ug5" - }, - "outputs": [], - "source": [ - "assert precision(uk_ru_test, X_test) == 0.0\n", - "assert precision(uk_ru_test, Y_test) == 1.0" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U-ssEJ3x7uhA" - }, - "outputs": [], - "source": [ - "precision_top1 = precision(uk_ru_test, mapping.predict(X_test), 1)\n", - "precision_top5 = precision(uk_ru_test, mapping.predict(X_test), 5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7K-hy7a6Ksn2" - }, - "outputs": [], - "source": [ - "print(precision_top1)\n", - "print(precision_top5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hf6Ou8bx7uhH" - }, - "source": [ - "## Making it better (orthogonal Procrustean problem) (0.3 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4oLs-drN7uhK" - }, - "source": [ - "It can be shown (see original paper) that a self-consistent linear mapping between semantic spaces should be orthogonal. \n", - "We can restrict transform $W$ to be orthogonal. Then we will solve next problem:\n", - "\n", - "$$W^*= \\arg\\min_W ||WX - Y||_F \\text{, where: } W^TW = I$$\n", - "\n", - "$$I \\text{- identity matrix}$$\n", - "\n", - "Instead of making yet another regression problem we can find optimal orthogonal transformation using singular value decomposition. It turns out that optimal transformation $W^*$ can be expressed via SVD components:\n", - "$$X^TY=U\\Sigma V^T\\text{, singular value decompostion}$$\n", - "$$W^*=UV^T$$" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_KSaRJFGMFiJ" - }, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DdFQ7qti7uhL" - }, - "outputs": [], - "source": [ - "def learn_transform(X_train, Y_train):\n", - " \"\"\" \n", - " :returns: W* : float matrix[emb_dim x emb_dim] as defined in formulae above\n", - " \"\"\"\n", - " # YOUR CODE GOES HERE\n", - " # compute orthogonal embedding space mapping\n", - " # mapping = ...\n", - "\n", - " return mapping" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7X7QfYDd7uhQ" - }, - "outputs": [], - "source": [ - "W = learn_transform(X_train, Y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OVOFYYa37uhX" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([np.matmul(uk_emb[\"серпень\"], W)])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r297sYP37uhb" - }, - "outputs": [], - "source": [ - "print(precision(uk_ru_test, np.matmul(X_test, W)))\n", - "print(precision(uk_ru_test, np.matmul(X_test, W), 5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hvUZ72U5AfJg" - }, - "source": [ - "## Unsupervised embedding-based MT (0.4 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLyuVfHBLrJn" - }, - "source": [ - "Now, let's build our word embeddings-based translator!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tPAURW1CMuP7" - }, - "source": [ - "Firstly, download OPUS Tatoeba corpus." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "F80kUKzQMsDu" - }, - "outputs": [], - "source": [ - "!wget https://object.pouta.csc.fi/OPUS-Tatoeba/v20190709/mono/uk.txt.gz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0CGFZoxCUVf1" - }, - "outputs": [], - "source": [ - "!gzip -d ./uk.txt.gz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2MV3VvoVUX5U" - }, - "outputs": [], - "source": [ - "with open('./uk.txt', 'r') as f:\n", - " uk_corpus = f.readlines()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tU7nPVf0UhbI" - }, - "outputs": [], - "source": [ - "# To save your time and CPU, feel free to use first 1000 sentences of the corpus\n", - "uk_corpus = uk_corpus[:1000]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FLN8dBOXAfJ1" - }, - "outputs": [], - "source": [ - "# Any necessary preprocessing if needed\n", - "# YOUR CODE HERE" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FGksC7l_NMi9" - }, - "outputs": [], - "source": [ - "def translate(sentence):\n", - " \"\"\"\n", - " :args:\n", - " sentence - sentence in Ukrainian (str)\n", - " :returns:\n", - " translation - sentence in Russian (str)\n", - "\n", - " * find ukrainian embedding for each word in sentence\n", - " * transform ukrainian embedding vector\n", - " * find nearest russian word and replace\n", - " \"\"\"\n", - " # YOUR CODE GOES HERE\n", - "\n", - " return \" \".join(translated)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4hbbMy-tNxlf" - }, - "outputs": [], - "source": [ - "assert translate(\".\") == \".\"\n", - "assert translate(\"1 , 3\") == \"1 , 3\"\n", - "assert translate(\"кіт зловив мишу\") == \"кот поймал мышку\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ia6I2ce7O_HI" - }, - "source": [ - "Now you can play with your model and try to get as accurate translations as possible. **Note**: one big issue is out-of-vocabulary words. Try to think of various ways of handling it (you can start with translating each of them to a special **UNK** token and then move to more sophisticated approaches). Good luck!" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ap1W7ZCeOAVU" - }, - "outputs": [], - "source": [ - "for sent in uk_corpus[::10]:\n", - " print(translate(sent))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Great! \n", - "See second notebook for the Neural Machine Translation assignment." - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "homework.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Py3 research env", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_part2_NMT-checkpoint.ipynb b/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_part2_NMT-checkpoint.ipynb deleted file mode 100644 index cae6998..0000000 --- a/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_part2_NMT-checkpoint.ipynb +++ /dev/null @@ -1,941 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Lab 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Part 2: Neural Machine Translation in the wild\n", - "In the third homework you are supposed to get the best translation you can for the EN-RU translation task.\n", - "\n", - "Basic approach using RNNs as encoder and decoder is implemented for you. \n", - "\n", - "Your ultimate task is to use the techniques we've covered, e.g.\n", - "\n", - "* Optimization enhancements (e.g. learning rate decay)\n", - "\n", - "* CNN encoder (with or without positional encoding)\n", - "\n", - "* attention/self-attention mechanism\n", - "\n", - "* pretraining the language model\n", - "\n", - "* [Byte Pair Encoding](https://github.com/rsennrich/subword-nmt)\n", - "\n", - "* or just fine-tunning BERT ;)\n", - "\n", - "to improve the translation quality. \n", - "\n", - "__Please use at least three different approaches/models and compare them (translation quality/complexity/training and evaluation time).__\n", - "\n", - "Write down some summary on your experiments and illustrate it with convergence plots/metrics and your thoughts. Just like you would approach a real problem." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# You might need to install the libraries below. Do it in the desired environment\n", - "# if you are working locally.\n", - "\n", - "# ! pip install subword-nmt\n", - "# ! pip install nltk\n", - "# ! pip install torchtext" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Thanks to YSDA NLP course team for the data\n", - "# (who thanks tilda and deephack teams for the data in their turn)\n", - "\n", - "import os\n", - "path_do_data = '../../datasets/Machine_translation_EN_RU/data.txt'\n", - "if not os.path.exists(path_do_data):\n", - " print(\"Dataset not found locally. Downloading from github. Loading special files as well\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/advanced_f20/datasets/Machine_translation_EN_RU/data.txt -nc\n", - " path_do_data = './data.txt'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists('./utils.py'):\n", - " print(\"utils file not found locally. Downloading from github.\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/advanced_f20/homeworks_advanced/Lab1_NLP/utils.py -nc\n", - "\n", - "if not os.path.exists('./my_network.py'):\n", - " print(\"network file not found locally. Downloading from github.\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/advanced_f20/homeworks_advanced/Lab1_NLP/my_network.py -nc" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "\n", - "import torchtext\n", - "from torchtext.datasets import TranslationDataset, Multi30k\n", - "from torchtext.data import Field, BucketIterator\n", - "\n", - "import spacy\n", - "\n", - "import random\n", - "import math\n", - "import time\n", - "\n", - "import matplotlib\n", - "matplotlib.rcParams.update({'figure.figsize': (16, 12), 'font.size': 14})\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "from IPython.display import clear_output\n", - "\n", - "from nltk.tokenize import WordPunctTokenizer\n", - "from subword_nmt.learn_bpe import learn_bpe\n", - "from subword_nmt.apply_bpe import BPE\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Main part\n", - "__Here comes the preprocessing. Do not hesitate to use BPE or more complex preprocessing ;)__" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer_W = WordPunctTokenizer()\n", - "def tokenize(x, tokenizer=tokenizer_W):\n", - " return tokenizer.tokenize(x.lower())" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "SRC = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "TRG = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "dataset = torchtext.data.TabularDataset(\n", - " path=path_do_data,\n", - " format='tsv',\n", - " fields=[('trg', TRG), ('src', SRC)]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "train_data, valid_data, test_data = dataset.split(split_ratio=[0.8, 0.15, 0.05])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training examples: 40000\n", - "Number of validation examples: 2500\n", - "Number of testing examples: 7500\n" - ] - } - ], - "source": [ - "print(f\"Number of training examples: {len(train_data.examples)}\")\n", - "print(f\"Number of validation examples: {len(valid_data.examples)}\")\n", - "print(f\"Number of testing examples: {len(test_data.examples)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "SRC.build_vocab(train_data, min_freq = 3)\n", - "TRG.build_vocab(train_data, min_freq = 3)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unique tokens in source (ru) vocabulary: 9267\n", - "Unique tokens in target (en) vocabulary: 6699\n" - ] - } - ], - "source": [ - "print(f\"Unique tokens in source (ru) vocabulary: {len(SRC.vocab)}\")\n", - "print(f\"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here are tokens from original (RU) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['',\n", - " '29',\n", - " 'соль',\n", - " 'комо',\n", - " '―',\n", - " 'электрическая',\n", - " 'ming',\n", - " 'утренний',\n", - " 'детском',\n", - " 'таунус']" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "SRC.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And from target (EN) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['', 'king', 'buffets', 'catch', 'media', 'schedule', 'maraunenhof']" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "TRG.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And here is example from train dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'trg': ['laundry', 'service', 'is', 'provided', '.'], 'src': ['помимо', 'этого', ',', 'гостям', 'предоставляются', 'услуги', 'прачечной', '.']}\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[9]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check the length distributions:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Train data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfUAAAEICAYAAABGRG3WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAer0lEQVR4nO3df7ReVX3n8fdHIr+0kCApxQRNKhlbZC0rzUhcOB3HOPyybVhr1MFxDdGmTVdLW9vpTAvTrmFGZQbXOEVYKpURSrAWpFRLRqw0RRlXp8OPoA7yQ0rKryQFuZIAVqs19jt/nH3hId6be3Pvzb3PPff9WutZ95y99zlnn3Of/XzP2c9+zklVIUmS5r8XzHUFJEnSzDCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST1hUNcBkWRFkkqyaA62/c4kfznb25XmQpKrkrxvGsv/XZIfnck6tfU+nORNM73eSWx3zj57hoFBXfPaQm/AGg5zFcD2V5Jbkvz8YFpVvbiqHpyrOk3XfDn2s8WgrudJctBc10HqG086NVsM6vNIkt9OsjPJN5Pcn2RtSz8kyQeT/G17fTDJIS3vB7qi25Xt8W36qiSXJflskm8B/yLJcUk+lWQkyZNJPjSw7M8luS/J7iQ3JXn5JOt+ZJIrkjzW9uF9oycQo3VM8oG23oeSnDGw7MokX2z7/RdJPpzkD1v2F9vfp1o34usGlhtzfdJMSvJx4GXA/2rvwd8a6EHakORR4POt7B8neTzJ0+09/aqB9VzV3ts3tvf6bUle0fKS5OIkTyR5JslXk5w4Rl2WJPlMa7u72/Tylnch8M+AD7V6fqilD34eHJnk6rb8I0l+N8kLWt4+2+kEx+gFSc5L8jftM+W6JEe1vNFjtT7Jo0m+keR3BpY9LMmmts372vHdMd6xH9jsO8ZaX+9Vla958AJeCWwHXtrmVwCvaNPvAW4FfhhYCvwV8N6W907gL/daVwHHt+mrgKeBU+hO8l4E/D/g4jZ9KPD6VnYdsA34cWAR8LvAX41T3xVtO4va/KeBj7Z1/jBwO/CLA3X8HvALwEHALwF/C6Tl/1/gA8DBwOuBZ4A/HGs7k1mfL18z/QIeBt40MD/6vry6vecPa+k/B/wQcAjwQeArA8tcBTwJvLa1r08A17a804A7gcVAWhs8dmC597XplwD/Cji8beePgT8d2MYtwM/vVffBz4OrgRvasiuAvwY2tLz9aleDxwR4N91n1PK27x8FrtnrWP1P4DDg1cB3gR9v+RcB/xtY0pa/C9gxiWM/5vr6/przCvia5D8KjgeeAN4EvHCvvL8BzhyYPw14uE2/k4mD+tUDea8DRhgIkgN5fzbawNv8C4BvAy8fo+xow1oEHNMa1WED+W8HvjBQx20DeYe3ZX+E7ix8D3D4QP4fMnFQH3N9c/1/9NXP1z4Cy4/uY5nFrcyRbf4q4GMD+WcCX2vTb6QLsGuAF+y1nqtoQX2MbfwEsHtg/hbGCep0gfofgBMG8n4RuKVN71e74vlB/T5g7UDesXQnCIsGjtXygfzbgbPb9IPAaQN5P8/kgvqY6+v7y+73eaKqtgG/Dvxn4Ikk1yZ5act+KfDIQPFHWtpkbR+YPg54pKr2jFHu5cAlSZ5K8hSwi+6qYdkE63858ELgsYFlP0p3xT7q8dGJqvp2m3xx249dA2l713c8461Pmk3PvleTHJTkotYF/QxdMAI4eqD84wPT36a9Z6vq88CHgA/Ttf/Lkxyx98aSHJ7ko63r/Bm6r6cWZ3JjZY6ma6d7f5YMtu+ptquXA58eaP/3Ad+nO+H/gXUzsO90nwGDbX4y7X9f6+s1g/o8UlV/VFWvp2sgBby/Zf1tSxv1spYG8C26M2oAkvzIWKsemN4OvCxjD+zZTtdlvnjgdVhV/dUEVd9Od6V+9MByR1TVqyZYDuAx4Kgkhw+kHTdO3aW5Mt77cDD939B9hfUm4Ei6K0roTown3kDVpVX1k8AJwD8B/sMYxX6T7qu6k6vqCOCn9trGvtrLN+iunvf+LNk5mfpNYDtwxl6fHYdW1WTW/Rhdt/uo4/bK9zNggEF9nkjyyiRvTDcA7jvA3wP/2LKvAX43ydIkRwP/ia6LGrrvx1+V5CeSHEp3pb8vt9M1oouSvCjJoUlOaXm/D5w/OrinDap560R1r6rHgD8H/keSI9qgmVck+eeTWPYRYCvwn5Mc3AbC/cxAkRG64zDjv7OV9sPXmfg9+EN0J7dP0p1o/9fJrjzJP01ycpIX0p2of4fn2v/e2/h7uoGjRwEXTLaeVfV94DrgwiQ/lG4Q7L/juc+S6fj9tt6Xt/1ZmmTdJJe9ju5zZ0mSZcCv7JU/mWO/YBjU549D6AaMfIOuW+mHgfNb3vvoAt9dwFeBL7U0quqv6QbS/QXwALDPm7K0hv0zdN+xPQrsAP51y/s0Xe/Ata1r725gsqPKz6Eb6HYvsBu4nu57tcl4B913/U+2/fok3YfjaBfghcD/aV17aya5Tmkm/Te6E+unkvz7ccpcTdedvZOuHdy6H+s/gm7g1+62jieB/z5GuQ/SDQ77Rlv/5/bKvwR4SxtJfukYy/8q3UnDg3SfFX8EXLkf9RzPJcBm4M+TfLPV7eRJLvseus+hh+g+x66ntf9mMsd+wRgdXSzNG0k+STeAaO+rEEk9l+SX6Aa9TdjTtxB5pa6h17oeX9G67U+n+17yT+e6XpIOvCTHJjmltf9X0o0b+PRc12tYeZcjzQc/AnyK7je4O4Bfqqovz22VJM2Sg+l+LbMSeAq4FvjInNZoiNn9LklST9j9LklST8zb7vejjz66VqxYMdfVkIbanXfe+Y2qWjrX9dgX27I0OZNpzxMG9SRXAj8NPFFVJ7a0o+h+VrSC7q5Ib6uq3UlC99OFM+nu4PPOqvpSW2Y93b3Cobul4aaW/pN0tzk8DPgs8O6axHcCK1asYOvWrRMVkxa0JI9MXGpu2ZalyZlMe55M9/tVwOl7pZ0H3FxVq4Cb2zx0v1le1V4bgctaRUZvgnAy3cMKLkiypC1zGd0DAkaX23tbkiRpEiYM6lX1Rbp7fA9aB2xq05uAswbSr67OrXT3HD6W7gEjW6pqV1XtBrYAp7e8I6rq1nZ1fvXAuiRJ0n6Y6kC5Y9qtP6G7u9noTfmX8fyb7e9oaftK3zFG+piSbEyyNcnWkZGRKVZdkqR+mvbo93aFPSu/i6uqy6tqdVWtXrp0qMf+SJI066Ya1L/eus5pf59o6Tt5/hN0lre0faUvHyNdkiTtp6kG9c3A+ja9HrhhIP2cdNYAT7du+puAU9tTdpYApwI3tbxnkqxpI+fPGViXJEnaD5P5Sds1wBuAo5PsoBvFfhFwXZINdE8Melsr/lm6n7Nto/tJ27sAqmpXkvcCd7Ry76mq0cF3v8xzP2n7s/aSJEn7acKgXlVvHydr7RhlCzh3nPVcyRiP8KuqrcCJE9VDkiTtm7eJlSSpJ+btbWJn04rzbpywzMMXvXkWaiJpumzP6jOv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1KUFJMmVSZ5IcvdA2lFJtiR5oP1d0tKT5NIk25LcleSkgWXWt/IPJFk/kP6TSb7alrk0SWZ3D6WFzaAuLSxXAafvlXYecHNVrQJubvMAZwCr2msjcBl0JwHABcDJwGuBC0ZPBFqZXxhYbu9tSTqADOrSAlJVXwR27ZW8DtjUpjcBZw2kX12dW4HFSY4FTgO2VNWuqtoNbAFOb3lHVNWtVVXA1QPrkjQLDOqSjqmqx9r048AxbXoZsH2g3I6Wtq/0HWOk/4AkG5NsTbJ1ZGRk+nsgCTCoSxrQrrBrFrZzeVWtrqrVS5cuPdCbkxYMg7qkr7euc9rfJ1r6TuC4gXLLW9q+0pePkS5plhjUJW0GRkewrwduGEg/p42CXwM83brpbwJOTbKkDZA7Fbip5T2TZE0b9X7OwLokzYJFc10BSbMnyTXAG4Cjk+ygG8V+EXBdkg3AI8DbWvHPAmcC24BvA+8CqKpdSd4L3NHKvaeqRgff/TLdCPvDgD9rL0mzxKAuLSBV9fZxstaOUbaAc8dZz5XAlWOkbwVOnE4dJU2d3e+SJPWEQV2SpJ4wqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9Ma2gnuQ3ktyT5O4k1yQ5NMnKJLe15yl/MsnBrewhbX5by18xsJ7zW/r9SU6b3i5JkrQwTTmoJ1kG/BqwuqpOBA4CzgbeD1xcVccDu4ENbZENwO6WfnErR5IT2nKvonv28keSHDTVekmStFBNt/t9EXBYkkXA4cBjwBuB61v+3s9mHn1m8/XA2nZ/6HXAtVX13ap6iO6WlK+dZr0kSVpwphzUq2on8AHgUbpg/jRwJ/BUVe1pxQafp/zsM5hb/tPASxj/2cw/wGcwS5I0vul0vy+hu8peCbwUeBFd9/kB4zOYJUka33S6398EPFRVI1X1PeBTwCnA4tYdD89/nvKzz2Bu+UcCTzL+s5klSdJ+mE5QfxRYk+Tw9t34WuBe4AvAW1qZvZ/NPPrM5rcAn29PgdoMnN1Gx68EVgG3T6NekiQtSFN+9GpV3ZbkeuBLwB7gy8DlwI3AtUne19KuaItcAXw8yTZgF92Id6rqniTX0Z0Q7AHOrarvT7VekiQtVNN6nnpVXQBcsFfyg4wxer2qvgO8dZz1XAhcOJ26SJK00E0rqEtSH60478YJyzx80ZtnoSbS/jGoz5DJfAiAHwSSpAPHe79LktQTBnVJknrC7ndJvTHZr8GkvvJKXZKknjCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST1hUJckqScM6pIASPIbSe5JcneSa5IcmmRlktuSbEvyySQHt7KHtPltLX/FwHrOb+n3JzltrvZHWogM6pJIsgz4NWB1VZ0IHET3eOT3AxdX1fHAbmBDW2QDsLulX9zKkeSEttyrgNOBjyQ5aDb3RVrIDOqSRi0CDkuyCDgceAx4I3B9y98EnNWm17V5Wv7aJGnp11bVd6vqIWAbYzyKWdKBYVCXRFXtBD4APEoXzJ8G7gSeqqo9rdgOYFmbXgZsb8vuaeVfMpg+xjLPSrIxydYkW0dGRmZ+h6QFyqAuiSRL6K6yVwIvBV5E131+QFTV5VW1uqpWL1269EBtRlpwDOqSAN4EPFRVI1X1PeBTwCnA4tYdD7Ac2NmmdwLHAbT8I4EnB9PHWEbSAWZQlwRdt/uaJIe378bXAvcCXwDe0sqsB25o05vbPC3/81VVLf3sNjp+JbAKuH2W9kFa8Hz0qiSq6rYk1wNfAvYAXwYuB24Erk3yvpZ2RVvkCuDjSbYBu+hGvFNV9yS5ju6EYA9wblV9f1Z3RlrADOqSAKiqC4AL9kp+kDFGr1fVd4C3jrOeC4ELZ7yCkiZk97skST1hUJckqScM6pIk9YRBXZKknjCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST3hHeVm2YrzbpywzMMXvXkWaiJJ6huv1CVJ6gmDuiRJPTGtoJ5kcZLrk3wtyX1JXpfkqCRbkjzQ/i5pZZPk0iTbktyV5KSB9axv5R9Isn78LUqSpPFM90r9EuBzVfVjwKuB+4DzgJurahVwc5sHOIPu2cqrgI3AZQBJjqJ7MtTJdE+DumD0RECSJE3elIN6kiOBn6I9X7mq/qGqngLWAZtasU3AWW16HXB1dW4FFic5FjgN2FJVu6pqN7AFOH2q9ZIkaaGazuj3lcAI8AdJXg3cCbwbOKaqHmtlHgeOadPLgO0Dy+9oaeOlz4rJjEaXJGk+mE73+yLgJOCyqnoN8C2e62oHoKoKqGls43mSbEyyNcnWkZGRmVqtJEm9MJ2gvgPYUVW3tfnr6YL811u3Ou3vEy1/J3DcwPLLW9p46T+gqi6vqtVVtXrp0qXTqLokSf0z5aBeVY8D25O8siWtBe4FNgOjI9jXAze06c3AOW0U/Brg6dZNfxNwapIlbYDcqS1NkiTth+neUe5XgU8kORh4EHgX3YnCdUk2AI8Ab2tlPwucCWwDvt3KUlW7krwXuKOVe09V7ZpmvSRJWnCmFdSr6ivA6jGy1o5RtoBzx1nPlcCV06mLJEkLnXeUkySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiQAkixOcn2SryW5L8nrkhyVZEuSB9rfJa1sklyaZFuSu5KcNLCe9a38A0nWj79FSTPNoC5p1CXA56rqx4BXA/fRPaTp5qpaBdzMcw9tOgNY1V4bgcsAkhwFXACcDLwWuGD0REDSgWdQl0SSI4GfAq4AqKp/qKqngHXAplZsE3BWm14HXF2dW4HF7QFOpwFbqmpXVe0GtgCnz+KuSAuaQV0SwEpgBPiDJF9O8rEkLwKOaQ9eAngcOKZNLwO2Dyy/o6WNly5pFhjUJUH3HIiTgMuq6jXAt3iuqx149vkNNRMbS7IxydYkW0dGRmZilZIwqEvq7AB2VNVtbf56uiD/9datTvv7RMvfCRw3sPzyljZe+vNU1eVVtbqqVi9dunRGd0RayAzqkqiqx4HtSV7ZktYC9wKbgdER7OuBG9r0ZuCcNgp+DfB066a/CTg1yZI2QO7UliZpFkz3eeqS+uNXgU8kORh4EHgX3Yn/dUk2AI8Ab2tlPwucCWwDvt3KUlW7krwXuKOVe09V7Zq9XZAWNoO6JACq6ivA6jGy1o5RtoBzx1nPlcCVM1s7SZNh97skST1hUJckqSfsfpekKVhx3o0Tlnn4ojfPQk2k53ilLklSTxjUJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ4wqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ7w0atDyEc6SpKmYtpX6kkOSvLlJJ9p8yuT3JZkW5JPJjm4pR/S5re1/BUD6zi/pd+f5LTp1kmSpIVoJrrf3w3cNzD/fuDiqjoe2A1saOkbgN0t/eJWjiQnAGcDrwJOBz6S5KAZqJckSQvKtIJ6kuXAm4GPtfkAbwSub0U2AWe16XVtnpa/tpVfB1xbVd+tqoeAbcBrp1MvSZIWouleqX8Q+C3gH9v8S4CnqmpPm98BLGvTy4DtAC3/6Vb+2fQxlnmeJBuTbE2ydWRkZJpVlySpX6Yc1JP8NPBEVd05g/XZp6q6vKpWV9XqpUuXztZmJUmaF6Yz+v0U4GeTnAkcChwBXAIsTrKoXY0vB3a28juB44AdSRYBRwJPDqSPGlxGkiRN0pSv1Kvq/KpaXlUr6Aa6fb6q3gF8AXhLK7YeuKFNb27ztPzPV1W19LPb6PiVwCrg9qnWS5KkhepA/E79t4Frk7wP+DJwRUu/Avh4km3ALroTAarqniTXAfcCe4Bzq+r7B6BekiT12owE9aq6BbilTT/IGKPXq+o7wFvHWf5C4MKZqIskSQuVt4mVJKknDOqSJPWEQV3Ss7ztszS/GdQlDfK2z9I8ZlCXBHjbZ6kPDOqSRs3abZ+95bN0YBjUJc36bZ+95bN0YByIm89Imn+87bPUA16pS/K2z1JPeKUuaV+87bM0jxjUJT2Pt32W5i+73yVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTPnp1nlpx3o0Tlnn4ojfPQk0kScPCoC5JB8hkTr7BE3DNHLvfJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ4wqEuS1BMGdUmSemLKQT3JcUm+kOTeJPckeXdLPyrJliQPtL9LWnqSXJpkW5K7kpw0sK71rfwDSdZPf7ckSVp4pnOlvgf4zao6AVgDnJvkBOA84OaqWgXc3OYBzgBWtddG4DLoTgKAC4CTgdcCF4yeCEiSpMmbclCvqseq6ktt+pvAfcAyYB2wqRXbBJzVptcBV1fnVmBxkmOB04AtVbWrqnYDW4DTp1ovSZIWqhn5Tj3JCuA1wG3AMVX1WMt6HDimTS8Dtg8stqOljZc+1nY2JtmaZOvIyMhMVF2SpN6YdlBP8mLgT4Bfr6pnBvOqqoCa7jYG1nd5Va2uqtVLly6dqdVKktQL0wrqSV5IF9A/UVWfaslfb93qtL9PtPSdwHEDiy9vaeOlS5olDnyV+mE6o98DXAHcV1W/N5C1GRhtyOuBGwbSz2kfBmuAp1s3/U3AqUmWtA+MU1uapNnjwFepB6bz6NVTgH8LfDXJV1rafwQuAq5LsgF4BHhby/sscCawDfg28C6AqtqV5L3AHa3ce6pq1zTqJWk/tRPsx9r0N5MMDnx9Qyu2CbgF+G0GBr4CtyYZHfj6BtrAV4AkowNfr5m1nZEWsCkH9ar6SyDjZK8do3wB546zriuBK6daF0kzZzYGvibZSHeFz8te9rKZq7y0wHlHOUnPmq2Brw56lQ4Mg7okwIGvUh8Y1CU58FXqiekMlJPUHw58lXrAoN5jK867cVLlHr7ozQe4Jhp2DnyV+sHud0mSesKgLklSTxjUJUnqCYO6JEk9YVCXJKknHP0uSXNsMr9U8Vcqmgyv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk948xl54wtJ6gmv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hAPlJGkecECrJsMrdUmSesKgLklST9j9rkmx60+Shp9BXdLQm8xJpSS73yVJ6g2DuiRJPWFQlySpJ3r9nbrfw80uB9NJc8s2KK/UJUnqiaG5Uk9yOnAJcBDwsaq6aI6rpAPAK4n+sy1Lc2cognqSg4APA/8S2AHckWRzVd07tzWTtD9sy8Nvsl9LenI9Pw1FUAdeC2yrqgcBklwLrAP8IFiAZnIshB9Ms8623BMz1Q5tg7NrWIL6MmD7wPwO4OS9CyXZCGxss3+X5P4x1nU08I0Zr+HssO4zLO+fVLGhrPskTKbeL5+NigyYybYMw/u/sV6TlPcPX52a+VivCdvzsAT1Samqy4HL91UmydaqWj1LVZpR1n1uzNe6z9d6w+TaMgzvPlqvyRvGOkF/6zUso993AscNzC9vaZLmF9uyNIeGJajfAaxKsjLJwcDZwOY5rpOk/WdblubQUHS/V9WeJL8C3ET3M5grq+qeKa5uwi69IWbd58Z8rfvQ1XuG2zIM4T421mvyhrFO0NN6papmqiKSJGkODUv3uyRJmiaDuiRJPdGroJ7k9CT3J9mW5Ly5rs++JDkuyReS3JvkniTvbulHJdmS5IH2d8lc13UsSQ5K8uUkn2nzK5Pc1o79J9sgqaGTZHGS65N8Lcl9SV43j475b7T3yt1Jrkly6Hw57lMxDO152NvpMLbDYWxjw9J2klyZ5Ikkdw+kjXls0rm01e+uJCdNZhu9Cep57vaUZwAnAG9PcsLc1mqf9gC/WVUnAGuAc1t9zwNurqpVwM1tfhi9G7hvYP79wMVVdTywG9gwJ7Wa2CXA56rqx4BX0+3D0B/zJMuAXwNWV9WJdIPQzmb+HPf9MkTtedjb6TC2w6FqY0PWdq4CTt8rbbxjcwawqr02ApdNagtV1YsX8DrgpoH584Hz57pe+1H/G+jul30/cGxLOxa4f67rNkZdl7c33xuBzwChuwPSorH+F8PyAo4EHqINEB1Inw/HfPRObUfR/WrlM8Bp8+G4T3F/h7I9D1M7HcZ2OIxtbNjaDrACuHuiYwN8FHj7WOX29erNlTpj355y2RzVZb8kWQG8BrgNOKaqHmtZjwPHzFG19uWDwG8B/9jmXwI8VVV72vywHvuVwAjwB63L8mNJXsQ8OOZVtRP4APAo8BjwNHAn8+O4T8XQtechbKfD2A6Hro3Ng7Yz3rGZUhvoU1Cfl5K8GPgT4Ner6pnBvOpOz4bqN4dJfhp4oqrunOu6TMEi4CTgsqp6DfAt9uoGHMZjDtC+Z1tH96H5UuBF/GA3ng6QYWunQ9wOh66Nzae2MxPHpk9Bfd7dnjLJC+k+KD5RVZ9qyV9PcmzLPxZ4Yq7qN45TgJ9N8jBwLV3X3yXA4iSjNzMa1mO/A9hRVbe1+evpPoCG/ZgDvAl4qKpGqup7wKfo/hfz4bhPxdC05yFtp8PaDoexjQ172xnv2EypDfQpqM+r21MmCXAFcF9V/d5A1mZgfZteT/cd3tCoqvOranlVraA7xp+vqncAXwDe0ooNXb0BqupxYHuSV7aktXSPBB3qY948CqxJcnh774zWfeiP+xQNRXse1nY6rO1wSNvYsLed8Y7NZuCcNgp+DfD0QDf9+GZrsMIsDUA4E/hr4G+A35nr+kxQ19fTdbPcBXylvc6k+17sZuAB4C+Ao+a6rvvYhzcAn2nTPwrcDmwD/hg4ZK7rN06dfwLY2o77nwJL5ssxB/4L8DXgbuDjwCHz5bhPcX/nvD3Ph3Y6bO1wGNvYsLQd4Bq67/W/R9ersWG8Y0M38PHD7f3/VbrR+xNuw9vESpLUE33qfpckaUEzqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9YVCXJKkn/j90WDhb6Ns32gAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in train_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in train_data.examples])\n", - "\n", - "print('Length distribution in Train data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Test data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe0AAAEICAYAAAByPazKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAfXElEQVR4nO3df7RdZX3n8fdHIij+4GdETMCkktqiq1YmBVzYjiVWAa241qDFsWPUdNJatLbaarBdpctKJ06dIi4tNRUKTK1AqdaMUjFFrdNpQYNaFFBJEUjSYCK/bKVq0e/8sZ8Lh8tN7k3uz33O+7XWWWfv53n23s++5z7nu/ezn7N3qgpJkrTwPWq+KyBJkqbGoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFb05JkWZJKsmgetv3qJH8/19uV5kOSi5O8YxrL/1uSH5nJOrX13pbk+TO93ilsd96+e+aTQVu9MKoNVAvLfAWovZXkM0l+aTCtqh5fVbfOV52mqy9/+9lm0B5RSfab7zpIw8aDSs02g/YClOStSbYn+dckX0uyqqUfkOTdSf6lvd6d5ICW94iu4nZmekybvjjJBUmuSvId4GeTHJXkw0l2JbkryXsHln1tkpuT3JPk6iRPnWLdD0pyYZIdbR/eMXaAMFbHJO9q6/1GklMHll2e5LNtv/82yfuS/HnL/mx7v7d18z1nYLkJ1yfNpCT/Gzga+D/tf/AtAz1Aa5LcAXyqlf3LJHcmua/9Tz9jYD0Xt//tj7f/9euSPK3lJcl5SXYm+XaSLyd55gR1OSTJx1rbvadNL2155wI/Dby31fO9LX3w++CgJJe25W9P8jtJHtXy9thOJ/kbPSrJuiT/3L5TrkhyaMsb+1utTnJHkm8l+e2BZR+b5JK2zZvb33fb7v72A5t95UTrG1pV5WsBvYCnA1uBp7T5ZcDT2vTbgWuBJwGLgX8Afr/lvRr4+3HrKuCYNn0xcB9wEt3B2uOAfwLOa9OPAZ7byp4ObAF+HFgE/A7wD7up77K2nUVt/iPA+9s6nwR8DvjlgTr+B/Dfgf2A1wH/AqTl/yPwLmB/4LnAt4E/n2g7U1mfL18z/QJuA54/MD/2f3lp+59/bEt/LfAE4ADg3cCXBpa5GLgLOL61rw8Cl7W8FwLXAwcDaW3wyIHl3tGmDwP+C3Bg285fAn89sI3PAL80ru6D3weXAh9tyy4Dvg6saXl71a4G/ybAG+m+o5a2fX8/8KFxf6s/BR4LPAv4HvDjLX898HfAIW35G4BtU/jbT7i+YX3NewV8jftA4BhgJ/B84NHj8v4ZOG1g/oXAbW361UwetC8dyHsOsIuBIDiQ9zdjDbjNPwq4H3jqBGXHGs4i4IjWaB47kP8K4NMDddwykHdgW/bJdEfRDwAHDuT/OZMH7QnXN9+fo6/hfO0hcPzIHpY5uJU5qM1fDHxgIP804Ktt+mS6AHoi8Khx67mYFrQn2MZPAvcMzH+G3QRtukD8feDYgbxfBj7TpveqXfHwoH0zsGog70i6A4BFA3+rpQP5nwPObNO3Ai8cyPslpha0J1zfsL7sHl9gqmoL8OvA7wE7k1yW5Ckt+ynA7QPFb29pU7V1YPoo4PaqemCCck8Fzk9yb5J7gbvpjvqXTLL+pwKPBnYMLPt+ujPuMXeOTVTV/W3y8W0/7h5IG1/f3dnd+qS59OD/apL9kqxvXcTfpgs2AIcPlL9zYPp+2v9sVX0KeC/wPrr2vyHJE8dvLMmBSd7fura/TXf56OBMbazK4XTtdPx3yWD73td29VTgIwPt/2bgB3QH9I9YNwP7TvcdMNjmp9L+97S+oWTQXoCq6i+q6rl0DaCAd7asf2lpY45uaQDfoTsiBiDJkyda9cD0VuDoTDxwZitdl/bBA6/HVtU/TFL1rXRn2ocPLPfEqnrGJMsB7AAOTXLgQNpRu6m7NF929384mP5f6S4xPR84iO6MELoD38k3UPWeqvpPwLHAjwK/NUGxN9NdSjuhqp4I/My4beypvXyL7ux3/HfJ9qnUbxJbgVPHfXc8pqqmsu4ddN3iY44al+93AAbtBSfJ05OcnG6A2XeBfwd+2LI/BPxOksVJDgd+l64LGbrr089I8pNJHkN3pr4nn6NrJOuTPC7JY5Kc1PL+BDh7bPBMG7TyssnqXlU7gE8C/yvJE9uglKcl+c9TWPZ2YDPwe0n2bwPNfn6gyC66v8OM/85U2gvfZPL/wSfQHbzeRXcg/QdTXXmSn0pyQpJH0x2If5eH2v/4bfw73cDMQ4FzplrPqvoBcAVwbpInpBtk+iYe+i6Zjj9p631q25/FSU6f4rJX0H3vHJJkCfD6cflT+dsPPYP2wnMA3YCMb9F1+zwJOLvlvYMusN0AfBn4Qkujqr5ON1Dtb4FbgD3edKQ13J+nu8Z1B7AN+IWW9xG6s/vLWtfbV4Cpjsp+Fd1AspuAe4Ar6a5rTcUr6a6139X263K6L7+xLrpzgf/Xut5OnOI6pZn0P+gOnO9N8pu7KXMpXXfzdrp2cO1erP+JdAOr7mnruAv4wwnKvZtu8NW32vo/MS7/fOCMNhL7PRMs/wa6g4Jb6b4r/gK4aC/quTvnAxuBTyb511a3E6a47Nvpvoe+Qfc9diWt/TdT+dsPvbFRu9KCk+RyugE6488iJA25JK+jG1Q2aU/dKPFMWwtG6xp8WutWP4XuuuBfz3e9JM2+JEcmOam1/6fTXbf/yHzXa6Hx7j1aSJ4MfJjuN6jbgNdV1Rfnt0qS5sj+dL82WQ7cC1wG/PG81mgBsntckqSesHtckqSeWNDd44cffngtW7ZsvqshLXjXX3/9t6pq8XzXY09sz9LU7Kk9L+igvWzZMjZv3jzf1ZAWvCS3T15qftmepanZU3u2e1ySpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ5Y0HdEm0nL1n18RtZz2/oXzch6JM2uqbR527P6xjNtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoSyMkyUVJdib5yrj0NyT5apIbk/zPgfSzk2xJ8rUkLxxIP6WlbUmybi73QRplI3NzFUkAXAy8F7h0LCHJzwKnA8+qqu8leVJLPxY4E3gG8BTgb5P8aFvsfcDPAduAzyfZWFU3zdleSCPKoC2NkKr6bJJl45JfB6yvqu+1Mjtb+unAZS39G0m2AMe3vC1VdStAkstaWYO2NMvsHpf0o8BPJ7kuyd8l+amWvgTYOlBuW0vbXfojJFmbZHOSzbt27ZqFqkujxaAtaRFwKHAi8FvAFUkyEyuuqg1VtbKqVi5evHgmVimNtEmD9kQDV5L8YRu0ckOSjyQ5eCDPgStSv2wDPlydzwE/BA4HtgNHDZRb2tJ2ly5plk3lTPti4JRxaZuAZ1bVTwBfB86GRwxcOQX44yT7JdmPbuDKqcCxwCtaWUnz76+BnwVoA832B74FbATOTHJAkuXACuBzwOeBFUmWJ9mfrs1vnJeaSyNm0oFoEw1cqapPDsxeC5zRph24Ii1gST4EPA84PMk24BzgIuCi1pv2fWB1VRVwY5Ir6NrpA8BZVfWDtp7XA1cD+wEXVdWNc74z0giaidHjrwUub9NL6IL4mMEBKuMHrpww0cqSrAXWAhx99NEzUD1JY6rqFbvJ+sXdlD8XOHeC9KuAq2awapKmYFoD0ZL8Nt0R+AdnpjoOXJEkaXf2+Uw7yauBFwOrWlca7HmAigNXJEmahn06005yCvAW4CVVdf9AlgNXJEmaJZOeae9m4MrZwAHApvZzzmur6leqyoErkiTNkqmMHp9o4MqFeyg/1ANXlq37+JTK3bb+RbNcE0nSqPGOaJIk9YRBW5KknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoSyMkyUVJdib5ygR5b05SSQ5v80nyniRbktyQ5LiBsquT3NJeq+dyH6RRZtCWRsvFwCnjE5McBbwAuGMg+VS6J/WtANYCF7Syh9I9OOgE4HjgnCSHzGqtJQEGbWmkVNVngbsnyDqP7nG7NZB2OnBpda4FDk5yJPBCYFNV3V1V9wCbmOBAQNLMM2hLIy7J6cD2qvqncVlLgK0D89ta2u7SJ1r32iSbk2zetWvXDNZaGk0GbWmEJTkQeBvwu7Ox/qraUFUrq2rl4sWLZ2MT0kgxaEuj7WnAcuCfktwGLAW+kOTJwHbgqIGyS1va7tIlzTKDtjTCqurLVfWkqlpWVcvourqPq6o7gY3Aq9oo8hOB+6pqB3A18IIkh7QBaC9oaZJmmUFbGiFJPgT8I/D0JNuSrNlD8auAW4EtwJ8CvwpQVXcDvw98vr3e3tIkzbJF810BSXOnql4xSf6ygekCztpNuYuAi2a0cpIm5Zm2JEk9YdCWJKknDNqSJPWEQVuSpJ6YNGhP9ICBJIcm2dQeFrBp7L7DPmBAkqTZM5Uz7Yt55H2F1wHXVNUK4Jo2Dz5gQJKkWTNp0N7NAwZOBy5p05cALx1I9wEDkiTNgn29pn1EuzMSwJ3AEW3aBwxIkjRLpj0Qrd2AoSYtOPX1+YABSZImsK9B+5ut25v2vrOl+4ABSZJmyb7exnQjsBpY394/OpD++iSX0Q06u6+qdiS5GviDgcFnLwDO3vdqD4dl6z4+aZnb1r9oDmoiSeqDSYN2e8DA84DDk2yjGwW+HriiPWzgduDlrfhVwGl0Dxi4H3gNdA8YSDL2gAHwAQOS9tFUDnalYTVp0N7DAwZWTVDWBwxIkjRLvCOaJEk9YdCWJKknDNqSJPXEvo4e1yQcLKOFKMlFwIuBnVX1zJb2h8DPA98H/hl4TVXd2/LOBtYAPwB+raqubumnAOcD+wEfqKr1c70v0ijyTFsaLRfzyFsIbwKeWVU/AXyd9nPMJMcCZwLPaMv8cZL9kuwHvI/uWQPHAq9oZSXNMoO2NEImepZAVX2yqh5os9fS3fwIumcJXFZV36uqb9D9lPP49tpSVbdW1feBy1pZSbPMoC1p0GuBv2nTPktAWmC8pi0JgCS/DTwAfHCm1llVG4ANACtXrpyxZxTMJe9cqIXEoC2JJK+mG6C2qt0kCfb8zACfJSDNA7vHpRHXRoK/BXhJVd0/kLURODPJAUmWAyuAz9HdjnhFkuVJ9qcbrLZxrustjSLPtKURsptnCZwNHABsSgJwbVX9SlXdmOQK4Ca6bvOzquoHbT2vB66m+8nXRVV145zvjDSCDNrSCNnNswQu3EP5c4FzJ0i/iu4BQZLmkN3jkiT1hEFbkqSeMGhLktQTBm1JknrCoC1JUk84elzSyPJpfOobz7QlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknphW0E7yG0luTPKVJB9K8pj25J/rkmxJcnl7ChDtSUGXt/TrkiybiR2QJGlU7HPQTrIE+DVgZVU9k+5pP2cC7wTOq6pjgHuANW2RNcA9Lf28Vk6SJE3RdLvHFwGPTbIIOBDYAZwMXNnyLwFe2qZPb/O0/FVpzwGUJEmT2+egXVXbgXcBd9AF6/uA64F7q+qBVmwbsKRNLwG2tmUfaOUPG7/eJGuTbE6yedeuXftaPUmShs50uscPoTt7Xg48BXgccMp0K1RVG6pqZVWtXLx48XRXJ2lAkouS7EzylYG0Q5NsSnJLez+kpSfJe9o4lBuSHDewzOpW/pYkq+djX6RRNJ3u8ecD36iqXVX1H8CHgZOAg1t3OcBSYHub3g4cBdDyDwLumsb2Je29i3nkwfU64JqqWgFc0+YBTgVWtNda4ALogjxwDnACcDxwzliglzS7phO07wBOTHJguza9CrgJ+DRwRiuzGvhom97Y5mn5n6qqmsb2Je2lqvoscPe45MHxJuPHoVxanWvpDsiPBF4IbKqqu6vqHmATM9DLJmly07mmfR3dgLIvAF9u69oAvBV4U5ItdNesL2yLXAgc1tLfxENH85Lm1xFVtaNN3wkc0aYfHIfSjI1R2V36IzhGRZpZ03rKV1WdQ9dNNuhWui6z8WW/C7xsOtuTNLuqqpLMWA9YVW2gO5hn5cqV9qxJ0+Qd0SR9s3V70953tvQHx6E0Y2NUdpcuaZYZtCUNjjcZPw7lVW0U+YnAfa0b/WrgBUkOaQPQXtDSJM2yaXWPS+qXJB8CngccnmQb3eWt9cAVSdYAtwMvb8WvAk4DtgD3A68BqKq7k/w+8PlW7u1VNX5wm6RZYNCWRkhVvWI3WasmKFvAWbtZz0XARTNYNUlTYPe4JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JACS/EaSG5N8JcmHkjwmyfIk1yXZkuTyJPu3sge0+S0tf9n81l4aDQZtSSRZAvwasLKqngnsB5wJvBM4r6qOAe4B1rRF1gD3tPTzWjlJs2xaQTvJwUmuTPLVJDcneU6SQ5NsSnJLez+klU2S97Qj8xuSHDczuyBphiwCHptkEXAgsAM4Gbiy5V8CvLRNn97mafmrkmQO6yqNpOmeaZ8PfKKqfgx4FnAzsA64pqpWANe0eYBTgRXttRa4YJrbljRDqmo78C7gDrpgfR9wPXBvVT3Qim0DlrTpJcDWtuwDrfxh49ebZG2SzUk279q1a3Z3QhoB+xy0kxwE/AxwIUBVfb+q7uXhR+Djj8wvrc61wMFJjtznmkuaMa1H7HRgOfAU4HHAKdNdb1VtqKqVVbVy8eLF012dNPKmc6a9HNgF/FmSLyb5QJLHAUdU1Y5W5k7giDb94JF5M3jULml+PR/4RlXtqqr/AD4MnER3cL2olVkKbG/T24GjAFr+QcBdc1tlafRMJ2gvAo4DLqiqZwPf4aGucACqqoDam5XanSbNizuAE5Mc2K5NrwJuAj4NnNHKrAY+2qY3tnla/qdae5c0i6YTtLcB26rqujZ/JV0Q/+ZYt3d739nyHzwybwaP2h9kd5o091o7vhL4AvBluu+GDcBbgTcl2UJ3zfrCtsiFwGEt/U2MO2CXNDsWTV5kYlV1Z5KtSZ5eVV/joSPzm+iOwNfzyCPz1ye5DDgBuG+gG13SPKuqc4BzxiXfChw/QdnvAi+bi3pJesg+B+3mDcAH2w0XbgVeQ3eEfkWSNcDtwMtb2auA04AtwP2trCRJmqJpBe2q+hKwcoKsVROULeCs6WxPkqRR5h3RJEnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEsCIMnBSa5M8tUkNyd5TpJDk2xKckt7P6SVTZL3JNmS5IYkx813/aVRMN3naWuWLVv38UnL3Lb+RXNQE42A84FPVNUZSfYHDgTeBlxTVeuTrAPWAW8FTgVWtNcJwAXtXdIs8kxbEkkOAn4GuBCgqr5fVfcCpwOXtGKXAC9t06cDl1bnWuDgJEfOcbWlkeOZtiSA5cAu4M+SPAu4HngjcERV7Whl7gSOaNNLgK0Dy29raTsG0kiyFlgLcPTRR89a5efbVHrEwF4xTZ9n2pKgO4A/Drigqp4NfIeuK/xBVVVA7c1Kq2pDVa2sqpWLFy+escpKo8qgLQm6M+VtVXVdm7+SLoh/c6zbu73vbPnbgaMGll/a0iTNIoO2JKrqTmBrkqe3pFXATcBGYHVLWw18tE1vBF7VRpGfCNw30I0uaZZ4TVvSmDcAH2wjx28FXkN3YH9FkjXA7cDLW9mrgNOALcD9raykWWbQlgRAVX0JWDlB1qoJyhZw1qxXStLD2D0uSVJPTDtoJ9kvyReTfKzNL09yXbtT0uWtq40kB7T5LS1/2XS3LUnSKJmJ7vE3AjcDT2zz7wTOq6rLkvwJsIbubklrgHuq6pgkZ7ZyvzAD25ekXvAOh5quaZ1pJ1kKvAj4QJsPcDLdz0XgkXdQGruz0pXAqlZekiRNwXS7x98NvAX4YZs/DLi3qh5o82N3SYKBOyi1/Pta+YdJsjbJ5iSbd+3aNc3qSZI0PPY5aCd5MbCzqq6fwfp4ByVJknZjOte0TwJekuQ04DF017TPp3twwKJ2Nj14l6SxOyhtS7IIOAi4axrblyRppOzzmXZVnV1VS6tqGXAm8KmqeiXwaeCMVmz8HZTG7qx0Riu/V/cxliRplM3G77TfCrwpyRa6a9YXtvQLgcNa+psY9zACSZK0ZzNyR7Sq+gzwmTZ9K3D8BGW+C7xsJrY33lQfizes/BmJhsWot2VpMt4RTZKknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0Jb0IB+1Ky1sBm1Jg8YetTtm7FG7xwD30D1iFwYetQuc18pJmmUGbUmAj9qV+sCgLWmMj9qVFjiDtiQftSv1xIzce1xS7/moXakHPNOW5KN2pZ4waEvaEx+1Ky0gdo9Lepj5ftSupN3zTFuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPXEPgftJEcl+XSSm5LcmOSNLf3QJJuS3NLeD2npSfKeJFuS3JDkuJnaCUmSRsF0zrQfAN5cVccCJwJnJTmW7naG11TVCuAaHrq94anAivZaC1wwjW1LkjRy9jloV9WOqvpCm/5X4Ga6Z+yeDlzSil0CvLRNnw5cWp1r6Z4edOQ+11ySpBEzI9e0kywDng1cBxxRVTta1p3AEW16CbB1YLFtLW38utYm2Zxk865du2aiepIkDYVpPzAkyeOBvwJ+vaq+neTBvKqqJHv1uL6q2gBsAFi5cqWP+pshy9Z9fErlblv/olmuiSRpX03rTDvJo+kC9ger6sMt+Ztj3d7tfWdL3w4cNbD40pYmSZKmYDqjx0P3TN2bq+qPBrI2Aqvb9GrgowPpr2qjyE8E7hvoRpckSZOYzpn2ScB/A05O8qX2Og1YD/xckluA57d5gKuAW4EtwJ8CvzqNbUuaQf6EU+qHfb6mXVV/D2Q32asmKF/AWfu6PUmzauwnnF9I8gTg+iSbgFfT/YRzfZJ1dD/hfCsP/wnnCXQ/4TxhXmo+ZKYy/sSxJ6PLO6JJ8iecUk8YtCU9jD/hlBYug7akB43/CedgXrvEtdc/4ayqlVW1cvHixTNYU2k0Tft32pKGw55+wllVO/wJ58Lhde/R5Zm2JH/CKfWEZ9raax7lD6Wxn3B+OcmXWtrb6H6yeUWSNcDtwMtb3lXAaXQ/4bwfeM3cVlcaTQZtPcxUb3eq4eJPOKV+sHtckqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFbkqSe8I5okjSEvN3wcDJoS9KImuptiw3uC4dBW7PCo3xJmnle05YkqScM2pIk9YTd45o3Xk+TpL1j0NaC5/VxaX7ZBheOOe8eT3JKkq8l2ZJk3VxvX9LMsC1Lc29Oz7ST7Ae8D/g5YBvw+SQbq+qmuayHho9nAnPLtqzxpnq5azK20z2b6+7x44EtVXUrQJLLgNMBG7pm3UIM7AuxTlNkW9ascKzLns110F4CbB2Y3wacMFggyVpgbZv9tyR3Ad+am+rNq8MZ/v1c8PuYd87IamZ0P6dYp6fO1PamaNK2DBO256/NQd3myoL/f94HvdmnvWyrvdmvZrftecENRKuqDcCGsfkkm6tq5TxWaU6Mwn6Owj7C6OznVIxvz8NkGD/nYdwnGK79muuBaNuBowbml7Y0Sf1iW5bmwVwH7c8DK5IsT7I/cCawcY7rIGn6bMvSPJjT7vGqeiDJ64Grgf2Ai6rqxkkWG8qutQmMwn6Owj7CCOznPrblYTOMn/Mw7hMM0X6lqua7DpIkaQq897gkST1h0JYkqScWdNAextskJjkqyaeT3JTkxiRvbOmHJtmU5Jb2fsh813UmJNkvyReTfKzNL09yXftML2+DmHorycFJrkzy1SQ3J3nOsH6Wo2rY2+ywtdFhb5MLNmgP3CbxVOBY4BVJjp3fWs2IB4A3V9WxwInAWW2/1gHXVNUK4Jo2PwzeCNw8MP9O4LyqOga4B1gzL7WaOecDn6iqHwOeRbevw/pZjqphb7PD1kaHu01W1YJ8Ac8Brh6YPxs4e77rNQv7+VG6+zd/DTiypR0JfG2+6zYD+7aUroGcDHwMCN1diRZN9Bn37QUcBHyDNqBzIH3oPktfD/t8h6bNDlsbHYU2uWDPtJn4NolL5qkusyLJMuDZwHXAEVW1o2XdCRwxT9WaSe8G3gL8sM0fBtxbVQ+0+b5/psuBXcCfte7FDyR5HMP5WYqhbLPD1kaHvk0u5KA91JI8Hvgr4Ner6tuDedUdDvb6t3hJXgzsrKrr57sus2gRcBxwQVU9G/gO47rdhuGzVGfY2uyQttGhb5MLOWgP7W0SkzyarvF/sKo+3JK/meTIln8ksHO+6jdDTgJekuQ24DK67rfzgYOTjN3Up++f6TZgW1Vd1+avpPvCGLbPcuQNaZsdxjY69G1yIQftobxNYpIAFwI3V9UfDWRtBFa36dV01816q6rOrqqlVbWM7rP7VFW9Evg0cEYr1uv9rKo7ga1Jnt6SVtE9mnKoPstRN6xtdhjb6Ci0yQV9R7Qkp9Fdcxm7TeK581ylaUvyXOD/Al/moetIb6O7RnYFcDRwO/Dyqrp7Xio5w5I8D/jNqnpxkh+hO6o/FPgi8ItV9b35rN90JPlJ4APA/sCtwGvoDoaH8rMcRaPQZoepjQ57m1zQQVuSJD1kIXePS5KkAQZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9cT/B/odrf8G1COhAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in test_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in test_data.examples])\n", - "\n", - "print('Length distribution in Test data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model side\n", - "__Here comes simple pipeline of NMT model learning. It almost copies the week03 practice__" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "device(type='cuda', index=1)" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "device" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def _len_sort_key(x):\n", - " return len(x.src)\n", - "\n", - "BATCH_SIZE = 128\n", - "\n", - "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE, \n", - " device = device,\n", - " sort_key=_len_sort_key\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "[torchtext.data.batch.Batch of size 128]\n", - "\t[.trg]:[torch.cuda.LongTensor of size 55x128 (GPU 1)]\n", - "\t[.src]:[torch.cuda.LongTensor of size 59x128 (GPU 1)]\n", - "torch.Size([59, 128]) torch.Size([55, 128])\n" - ] - } - ], - "source": [ - "for x in train_iterator:\n", - " break\n", - "print(x)\n", - "print(x.src.shape, x.trg.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "import my_network\n", - "Encoder = my_network.Encoder\n", - "Decoder = my_network.Decoder\n", - "Seq2Seq = my_network.Seq2Seq" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(SRC.vocab)\n", - "OUTPUT_DIM = len(TRG.vocab)\n", - "ENC_EMB_DIM = 256\n", - "DEC_EMB_DIM = 256\n", - "HID_DIM = 512\n", - "N_LAYERS = 2\n", - "ENC_DROPOUT = 0.5\n", - "DEC_DROPOUT = 0.5\n", - "\n", - "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n", - "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n", - "\n", - "# dont forget to put the model to the right device\n", - "model = Seq2Seq(enc, dec, device).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Seq2Seq(\n", - " (encoder): Encoder(\n", - " (embedding): Embedding(9267, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - " (decoder): Decoder(\n", - " (embedding): Embedding(6699, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (out): Linear(in_features=512, out_features=6699, bias=True)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - ")" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def init_weights(m):\n", - " # \n", - " for name, param in m.named_parameters():\n", - " nn.init.uniform_(param, -0.08, 0.08)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The model has 14,880,299 trainable parameters\n" - ] - } - ], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [], - "source": [ - "PAD_IDX = TRG.vocab.stoi['']\n", - "optimizer = optim.Adam(model.parameters())\n", - "criterion = nn.CrossEntropyLoss(ignore_index = PAD_IDX)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, clip, train_history=None, valid_history=None):\n", - " model.train()\n", - " \n", - " epoch_loss = 0\n", - " history = []\n", - " for i, batch in enumerate(iterator):\n", - " \n", - " src = batch.src\n", - " trg = batch.trg\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " output = model(src, trg)\n", - " \n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - " \n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - " \n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - " \n", - " loss = criterion(output, trg)\n", - " \n", - " loss.backward()\n", - " \n", - " # Let's clip the gradient\n", - " torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " history.append(loss.cpu().data.numpy())\n", - " if (i+1)%10==0:\n", - " fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8))\n", - "\n", - " clear_output(True)\n", - " ax[0].plot(history, label='train loss')\n", - " ax[0].set_xlabel('Batch')\n", - " ax[0].set_title('Train loss')\n", - " if train_history is not None:\n", - " ax[1].plot(train_history, label='general train history')\n", - " ax[1].set_xlabel('Epoch')\n", - " if valid_history is not None:\n", - " ax[1].plot(valid_history, label='general valid history')\n", - " plt.legend()\n", - " \n", - " plt.show()\n", - "\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "def evaluate(model, iterator, criterion):\n", - " \n", - " model.eval()\n", - " \n", - " epoch_loss = 0\n", - " \n", - " history = []\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for i, batch in enumerate(iterator):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - "\n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - "\n", - " loss = criterion(output, trg)\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "def epoch_time(start_time, end_time):\n", - " elapsed_time = end_time - start_time\n", - " elapsed_mins = int(elapsed_time / 60)\n", - " elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n", - " return elapsed_mins, elapsed_secs" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "train_history = []\n", - "valid_history = []\n", - "\n", - "N_EPOCHS = 10\n", - "CLIP = 1\n", - "\n", - "best_valid_loss = float('inf')" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHwCAYAAABUqPIVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdeXxU1f3/8dfJAmFPAggikOCGInuCG7hiFTfcW6y20tri1tp+269V237VWv3Vqm1ta9WqdWndt7buxVYUFyqbgKIoLkEWhbAJAQJJ5vz+uDOTmclMZu5kMnfuzPvpAzOZucuZJclnPvM5n2OstYiIiIiISKsirwcgIiIiIpJrFCSLiIiIiMRQkCwiIiIiEkNBsoiIiIhIDAXJIiIiIiIxFCSLiIiIiMRQkCx5xRjzgjHmvDT3rTPGHJPpMYmIiIj/lHg9ABFjTEPEt92BnUBL8PsLrLUPpnosa+3xmRybiIiIFCYFyeI5a23P0GVjTB3wHWvtv2O3M8aUWGubszk2ERERKUwqt5CcZYw50hizyhhzuTHmC+BeY0yFMeZZY0y9MWZT8PLgiH1eMcZ8J3h5ujHmdWPMzcFtPzXGpJRpNsZ0NcbcYoxZE/x3izGma/C2fsHzbjbGbDTGvGaMKQredrkxZrUxZqsx5gNjzOROeGhERESkkylIllw3EKgEqoAZOK/Ze4PfDwV2ALe2s/9BwAdAP+BG4C/GGJPCeX8GHAyMBcYABwI/D972Y2AV0B8YAPwUsMaY4cD3gAnW2l7AcUBdivdTREREcoiCZMl1AeBqa+1Oa+0Oa+0Ga+2T1trt1tqtwPXAEe3sv8Jae5e1tgW4H9gdJ7BN5hzgWmvtOmttPfAL4BvB25qCx6my1jZZa1+z1lqcOuquwAhjTKm1ts5a+3Fa91pEREQ8pSBZcl29tbYx9I0xprsx5s/GmBXGmC3AbKDcGFOcYP8vQhestduDF3sm2DbSIGBFxPcrgtcB3AR8BMw0xnxijLkiePyPgB8C1wDrjDGPGGMGISIiIr6jIFlynY35/sfAcOAga21v4PDg9amUULixBqekI2Ro8DqstVuttT+21u4JTAV+FKo9ttY+ZK2dFNzXAr/O8LhEREQkCxQki9/0wqlD3myMqQSu7qTzPAz83BjT3xjTD7gKeADAGHOSMWbvYG3zlzhlFgFjzHBjzNHBCX6NwXEGOml8IiIi0okUJIvf3AJ0A9YD/wVe7KTzXAfMB5YA7wALg9cB7AP8G2gA5gC3WWtn4dQj3xAc2xfAbsCVnTQ+ERER6UTGmW8kIiIiIiIhyiSLiIiIiMRQkCwiIiIiEkNBsoiIiIhIDAXJIiIiIiIxFCSLiIiIiMQo8XoA8fTr189WV1d7PQwREdcWLFiw3lrb3+txZJN+Z4uIX7X3Ozsng+Tq6mrmz5/v9TBERFwzxqxIvlV+0e9sEfGr9n5nq9xCRERERCSGgmQRERERkRg5WW4hIiKdwxhTB2wFWoBma21tzO1HAv8EPg1e9ZS19tpsjlFEJBcoSBYRKTxHWWvXt3P7a9bak7I2GpEOaGpqYtWqVTQ2Nno9FMlhZWVlDB48mNLS0pT3UZAsIiIivrVq1Sp69epFdXU1xhivhyM5yFrLhg0bWLVqFcOGDUt5P9Uki4gUFgvMNMYsMMbMSLDNIcaYxcaYF4wxB2RzcCJuNTY20rdvXwXIkpAxhr59+7r+tEGZZBGRwjLJWrvaGLMb8JIxZpm1dnbE7QuBKmttgzHmBOAfwD6xBwkG2DMAhg4dmo1xiySkAFmSSec1okyyiEgBsdauDn5dB/wdODDm9i3W2obg5eeBUmNMvzjHudNaW2utre3fv6DWThHJaUceeWTcvuW33HIL27dvd328q666in//+98pb3/ffffxve99L+5tJ5xwAps3b064b7pj7CwKkkVECoQxpocxplfoMnAs8G7MNgNNMOVijDkQ5+/EhmyPVUTis9YSCARc79deANrS0pJwv2uvvZZjjjnG9fnief755ykvL094ezpBcntj7ygFySIihWMA8LoxZjEwF3jOWvuiMeZCY8yFwW3OBN4NbvMHYJq11no0XhFf+OUvf8nw4cOZNGkSZ599NjfffDMAH3/8MVOmTKGmpobDDjuMZcuWATB9+nQuvfRSDj30UPbcc0+eeOKJ8LFuuukmJkyYwOjRo7n66qsBqKurY/jw4Xzzm99k5MiRrFy5kosuuoja2loOOOCA8HaJ/OEPf2DNmjUcddRRHHXUUQD07NmTH//4x4wZM4Y5c+Zw7bXXMmHCBEaOHMmMGTMI/dhPnz49PL7q6mquvvpqxo8fz6hRo8L3J9aaNWuYMmUK++yzDz/5yU/C11dXV7N+/Xq2bdvGiSeeyJgxYxg5ciSPPvpo3DE+/PDDjBo1ipEjR3L55ZeHjxM59uuvv55TTz01fNtLL73EaaedlsKzlpxqkkVECoS19hNgTJzr74i4fCtwazbHJZIpv3hmKe+t2ZLRY44Y1JurT048f3XevHk8+eSTLF68mKamJsaPH09NTQ0AM2bM4I477mCfffbhrbfe4uKLL+bll18G4PPPP+f1119n2bJlTJ06lTPPPJOZM2eyfPly5s6di7WWqVOnMnv2bIYOHcry5cu5//77OfjggwG4/vrrqayspKWlhcmTJ7NkyRJGjx4dd4yXXnopv/3tb5k1axb9+jnVU9u2beOggw7iN7/5jXM/R4zgqquuAuAb3/gGzz77LCeffHKbY/Xr14+FCxdy2223cfPNN3P33Xe32WbRokW8/fbbdO3aleHDh/P973+fIUOGhG9/8cUXGTRoEM899xwAX375JX369Ika45o1a7j88stZsGABFRUVHHvssfzjH//g1FNPjRq7tZb999+f+vp6+vfvz7333su3v/3t9p/UFCmTLCIiIpKmN954g1NOOYWysjJ69eoVDiwbGhp48803Oeussxg7diwXXHABn3/+eXi/U089laKiIkaMGMHatWsBmDlzJjNnzmTcuHGMHz+eZcuWsXz5cgCqqqrCATLAY489xvjx4xk3bhxLly7lvffeczXu4uJizjjjjPD3s2bN4qCDDmLUqFG8/PLLLF26NO5+p59+OgA1NTXU1dXF3Wby5Mn06dOHsrIyRowYwYoVK6JuHzVqFC+99BKXX345r732Gn369GlzjHnz5nHkkUfSv39/SkpKOOecc5g9e3absRtj+MY3vsEDDzzA5s2bmTNnDscff7yrxyIRZZJFREQkL7SX8c22QCBAeXk5ixYtint7165dw5dDpQ3WWq688kouuOCCqG3r6uro0aNH+PtPP/2Um2++mXnz5lFRUcH06dNdtzcrKyujuLgYcNroXXzxxcyfP58hQ4ZwzTXXJDxeaNzFxcU0NzcnvW/xttt3331ZuHAhzz//PD//+c+ZPHlyOIvtduwA3/rWtzj55JMpKyvjrLPOoqQkM+GtMskiIiIiaZo4cSLPPPMMjY2NNDQ08OyzzwLQu3dvhg0bxuOPPw44AfDixYvbPdZxxx3HPffcQ0NDAwCrV69m3bp1bbbbsmULPXr0oE+fPqxdu5YXXngh6Th79erF1q1b494WCoj79etHQ0NDVI10Z1izZg3du3fn3HPP5bLLLmPhwoVtxnjggQfy6quvsn79elpaWnj44Yc54ogj4h5v0KBBDBo0iOuuu45vfetbGRunMskiIiIiaZowYQJTp05l9OjRDBgwgFGjRoXLBx588EEuuugirrvuOpqampg2bRpjxrSZFhB27LHH8v7773PIIYcAzgS1Bx54ICprCjBmzBjGjRvHfvvtx5AhQ5g4cWLScc6YMYMpU6YwaNAgZs2aFXVbeXk53/3udxk5ciQDBw5kwoQJbh8GV9555x0uu+wyioqKKC0t5fbbb487xhtuuIGjjjoKay0nnngip5xySsJjnnPOOdTX17P//vtnbJwmFyct19bW2ng9/kREcp0xZoG1ttbrcWSTfmeLl95///2MBkbpaGhooGfPnmzfvp3DDz+cO++8k/Hjx3s6pkLzve99j3HjxnH++ecn3Cbea6W939nKJIuISPZt+Bj67uX1KEQyYsaMGbz33ns0NjZy3nnnKUDOspqaGnr06BHu1JEpCpJFJKf86LFF9Opawi9OGen1UKSz1H8Id0yCkafD8TdCWW+vRyTSIQ899JDXQyhoCxYs6JTjauKeiOSUj+u38cn6bV4PQzpT5TCY9ENY8qgTLH/2ltcjEhFpQ0GyiIhkV3EpHPVT+NYLgIV7p8CsX0FL/HZSIiJeUJAsIrnFWnJwPrF0hqEHw4VvwOivwas3OMHyxk+8HpWICKAgWURyjAUsipILRllvOO0OOPOeYK3yYfD2g+idkoh4TUGyiOQUaxUfFaSRZ8BFb8DuY+GfF8Pj02H7Rq9HJeI7Rx55JJloyRh5nBNOOIHNmze32eaaa67h5ptvbnP99OnT4y5IsmbNGs4888yE59y8eTO33XZbB0adWQqSRSSnWFRuUbDKh8B5T8Pkq2HZs3D7RPh0ttej6nyBACx+FP50MLx5q9ejkRxnrSUQCGT1nM8//zzl5eUdPs6gQYPaXc0vnSA50dLYmaAgWURyigLkAldUDIf9CL7zb+jSHe6fCjP/D5p3eT2yzLMWPvwX/Pkw+PsM2FQHr/xKGXQf+uUvf8nw4cOZNGkSZ599dji7+vHHHzNlyhRqamo47LDDWLZsGeBkWi+99FIOPfRQ9txzz6jA8aabbmLChAmMHj2aq6++GoC6ujqGDx/ON7/5TUaOHMnKlSu56KKLqK2t5YADDghvl8iLL77IWWedFf7+lVde4aSTTgJI6TjV1dWsX78egOuvv559992XSZMm8cEHHyQ85+zZs9vcv7q6OkaOdNp7Ll26lAMPPJCxY8cyevRoli9fzhVXXMHHH3/M2LFjueyyy7DWctlllzFy5EhGjRrFo48+Gh7/YYcdxtSpUxkxYgRXXXUVt9xyS/jcP/vZz/j973/f7mOSCvVJFpGco5pkYdA4uGA2/Otn8OYf4JNX4Iy7of9wr0eWGZ/9F/59DXw2Byr3dGqy++8Ptx/q3N9jrvF4gD71whXwxTuZPebAUXD8DQlvnjdvHk8++SSLFy+mqamJ8ePHU1NTAziLjNxxxx3ss88+vPXWW1x88cW8/PLLAHz++ee8/vrrLFu2jKlTp3LmmWcyc+ZMli9fzty5c7HWMnXqVGbPns3QoUNZvnw5999/PwcffDDgBKuVlZW0tLQwefJklixZwujRo+OO8ZhjjmHGjBls27aNHj168OijjzJt2jTXx1mwYAGPPPIIixYtorm5Oeq+xop3/yLdcccd/OAHP+Ccc85h165dtLS0cMMNN/Duu++yaNEiAJ588kkWLVrE4sWLWb9+PRMmTODwww8HYOHChbz77rsMGzaMuro6Tj/9dH74wx8SCAR45JFHmDt3bsLnLFUKkkUkp6gmWcK69ICTb4F9vgJPfx/+fAQcdx3Ung/GeD269KxdCv/5JXz4AvQcACf+FsZ/02mLB05t9lt3wsGXQM/+3o5VUvLGG29wyimnUFZWRllZGSeffDLgLFX95ptvRmVwd+7cGb586qmnUlRUxIgRI1i7di0AM2fOZObMmYwbNy58jOXLlzN06FCqqqrCATLAY489xp133klzczOff/457733XsLgtqSkhClTpvDMM89w5pln8txzz3HjjTe6Ps5rr73GaaedRvfu3QGYOnVqwscl3v2LdMghh3D99dezatUqTj/9dPbZZ58227z++uucffbZFBcXM2DAAI444gjmzZtH7969OfDAAxk2bBjgZLr79u3L22+/zdq1axk3bhx9+/ZNOLZUKUgWkZxig/9EwvY7EfaogX9cDM/9GJa/BFNv9VcQuWmFU0qx+BHo2hsmXwUHXei8EYh0xOWw9Cl48/dw7HXejNXP2sn4ZlsgEKC8vDycFY3VtWvX8GUbzAxYa7nyyiu54IILoratq6ujR4/W18qnn37KzTffzLx586ioqGD69Ok0Nja2O55p06Zx6623UllZSW1tLb169UrrOKmKd/8iff3rX+eggw7iueee44QTTuDPf/4ze+65Z8rHj3w8AL7zne9w33338cUXX/Dtb387/YFHUE2yiOQUa62iZGmr10A45wmY8mv4eBbcfogTLOe6hnp44XL4Yw0s/TtMvBR+sAgO+3HbABmg/74w6qsw927Y2jb7Jrln4sSJPPPMMzQ2NtLQ0MCzzz4LQO/evRk2bBiPP/444PxuW7x4cbvHOu6447jnnntoaGgAYPXq1axbt67Ndlu2bKFHjx706dOHtWvX8sILLyQd5xFHHMHChQu56667wqUWbo9z+OGH849//IMdO3awdetWnnnmmaTnTeSTTz5hzz335NJLL+WUU05hyZIl9OrVi61bt4a3Oeyww3j00UdpaWmhvr6e2bNnc+CBB8Y93mmnncaLL77IvHnzOO6449IeVyRlkkVExB+KiuDgC2HYYfDkd+HBM+HAGfCVa6G0m9eji9a4Beb8CebcCk07YNy5Tpa4zx7J9z3iJ/DO4/DGLTDlV50/VumQCRMmMHXqVEaPHs2AAQMYNWoUffr0AeDBBx/koosu4rrrrqOpqYlp06YxZsyYhMc69thjef/99znkkEMA6NmzJw888ADFxcVR240ZM4Zx48ax3377MWTIECZOnJh0nMXFxZx00kncd9993H///WkdZ/z48Xzta19jzJgx7LbbbkyYMCHpeRN57LHH+Nvf/kZpaSkDBw7kpz/9KZWVlUycOJGRI0dy/PHHc+ONNzJnzhzGjBmDMYYbb7yRgQMHhidARurSpQtHHXUU5eXlbR6vdJl4KXCv1dbW2kz0+BMR/znud7Pp3a2Exy881OuhpMUYs8BaW+v1OLLJk9/ZTY3wn1/Af2+D/vs5k/oGjsruGOJp3gnz74HZN8H2DTDiFDj6/6Bf23rLdv3jEidQ/sFi6L1754w1T7z//vvsv//+no6hoaGBnj17sn37dg4//HDuvPNOxo8f7+mYCk0gEGD8+PE8/vjjceubIf5rpb3f2Sq3EJGck4Pv3SXXlJY5WdZzn4Idm+Cuo50ew1nuHxsWaIFFDzllFS9eAQNGwndfhq/+1X2ADHD4/4Jtgdd/m/mxSsbNmDGDsWPHMn78eM444wwFyFn23nvvsffeezN58uSEAXI6VG4hIjnFqgGcuLH3ZLhojtP9YubP4KOX4NTbofeg7JzfWvjgBfjPtVD/vrNi4NQ/wl5Hdey4lcNg7Dmw4D6Y+APoMzgjw5XO8dBDD3k9hII2YsQIPvnkk4wfV5lkEckpTgs4hcniQo++MO1BOPn3sHKu02v4vX92/nlXvAn3HAePnA0tu+Cs++C7szoeIIccfpnzA/HabzJzPBFxRUGyiOQUhceSFmOgZjpc8BqUV8Fj34R/XgI7GzJ/ri/egQfPgnuPh82fOcH5JW/BAac5kwszpXwI1JwHC//mtJCThPTGWpJJ5zWiIFlEcoq1KreQDui3N5z/Ekz6Ebz9oLPk86oFmTn2xk+drhp3HAYr34JjfgHfX+gE56HFQDJt0o/AFDkTASWusrIyNmzYoEBZErLWsmHDBsrKylztp5pkEckpFk3ckw4q6QLHXA17HwN/vwD+8hU48ko47EdQlEZrqIZ18OqNTn1wUQlM+qFTJ9ytIuNDb6PPHlD7LZh7lzP+ytQXWygUgwcPZtWqVdTX13s9FMlhZWVlDB7srrZfQbKI5BzFyPmtsamFOZ9sYPiAXgwq78T+xtUT4cLXnVX6Zl0HH/0bTr8TKqpSHOiX8OYfYc5t0NzolD4c/pPst2Sb9D9OgP7qTXDa7dk9tw+UlpaGlycWySSVW4hIblEqOe9t2LaLb907j5lLv+j8k3UrhzP/AqffBevegzsmwZLH2t+nqdEJjn8/xilz2Pc4+N48OOl33vQs7jUQJnwHljwC6z/K/vlFCpSCZBHJKQqP898e5d0Y1KeM+Ss2Ze+ko7/qZJV3GwFPfReeOB92bI7epqXZmST3x/Ew8+cwaBzMeAXOuhf67pW9scYz8QdQUgav/trbcYgUEAXJIpJTNHGvMNRUVzK/blN2J1tVVMH05+Con8PSvztZ5bo3nE8u3n/GaR339PeczO03n4Zv/N0JlHNBz93gwO86q/DVf+D1aEQKgoJkEckpqrYoDBOqK/hiSyOrN+/I7omLS+CIy+D8mc4kvPtOhNsOhkfPBRuAr/4NvvMf2POI7I4rFYf+ALr0gFdu8HokIgVBQbKI5BRr0Zp7BaCmyukMMb8uiyUXkQbXOuUX478BzTudVfIu/i+MmOr0XM5FPfrCQRc4WfC1S70ejUjeU5AsIjlHmeT8t9/A3vTsWsL8FRu9G0TXnk5w/INFMP6bTpY51x3yPejaS9lkkSxQkCwiOcViFSQXgOIiw7ih5d5lkv2qeyUcfDG8/zR8vsTr0YjkNQXJIpJTFCAXjtqqSj5Yu5UvdzR5PRR/OfgiKOujbLJIJ0s5SDbGFBtj3jbGPBvntq7GmEeNMR8ZY94yxlRH3HZl8PoPjDHHZWbYIpKvnJpkKQQTqiuwFt7+TNlkV7qVwyHfhw+egzVvez0akbzlJpP8A+D9BLedD2yy1u4N/A74NYAxZgQwDTgAmALcZoxJY01QESkkWW0LJp4ZO7Sc4iKjkot0HHSBsyz2rP/n9UhE8lZKQbIxZjBwInB3gk1OAe4PXn4CmGyMMcHrH7HW7rTWfgp8BBzYsSGLSD5TgFw4uncp4YBBvb2dvOdXZb3h0Eth+UxYOc/r0YjkpVQzybcAPwECCW7fA1gJYK1tBr4E+kZeH7QqeJ0ErdvSyPqGnV4PQySnKE4uHDVVFSxauZmmlkR/XiShA2dA977wirLJIp0haZBsjDkJWGetXdCZAzHGzDDGzDfGzK+vr+/MU+WUHz++mKv++a7XwxDJGYqPC0ttVSWNTQGWrtni9VD8p2tPmPhD+PhlWDHH69GI5J1UMskTganGmDrgEeBoY8wDMdusBoYAGGNKgD7AhsjrgwYHr2vDWnuntbbWWlvbv39/V3fCz7Y0NrO1sdnrYYjkDC0mUlhqq0OLiqjkIi0TvgM9dlM2WaQTJA2SrbVXWmsHW2urcSbhvWytPTdms6eB84KXzwxuY4PXTwt2vxgG7APMzdjo84FVT1iRSOqTXFgG9C5jSGU3Td5LV5fuMOl/4NPZ8OlrXo9GJK+k3SfZGHOtMWZq8Nu/AH2NMR8BPwKuALDWLgUeA94DXgQusda2dGzI+cUCAUUEImFqAVd4JlRVMn/FJk3aTFftt6DnQHjlVyroF8kgV0GytfYVa+1JwctXWWufDl5utNaeZa3d21p7oLX2k4h9rrfW7mWtHW6tfSGzw/c/a/U7TSSSRR0uCk1NdQXrG3ayYsN2r4fiT6Xd4LAfw4o34NNXvR6NSN7Qinses8H/REQK1YTqSgDmr1DJRdrGfxN67+H0TdabTJGMUJDsMWWSRaKp3KLw7N2/J73LSjR5ryNKy+Dw/4WVb8HH//F6NCJ5QUFyDlBAIBJJUXKhKSoy1FRVKJPcUWPPhT5DlU0WyRAFyR6zFgUEIhGUSS5MtdWVfLSugU3bdnk9FP8q6QJHXAarFzgr8YlIhyhI9pi6W4hE08S9wlRb5fRLXqBscseMORsqqmHW9comi3SQgmSPWatpeyKRFCAXpjFDyiktNiq56KjiUjjicvh8MSx7zuvRiPiaguQcoKBAJJp+IgpPWWkxI/foo8l7mTDqq1C5l9M3ORDwejQivqUg2WOqvxSJ5pRbeD0K8cKE6kqWrPqSxiatOdUhxSVw5BWw9l14/2mvRyPiWwqSPaYleEWiOW8c9UNRiGqqKtjVEuDd1V96PRT/G3kG9Ns3mE3Wmw6RdChI9pjTJ1kBgUiItXrjWKhqgpP3VJecAUXFTja5fhks/bvXoxHxJQXJHlMHOJFo+nkoXP16dmXPfj2YX6cgOSNGnAb994dXblA2WSQNCpI9pqyZSAytQlnQaqoqWLBioz5hy4SiIjjqStiwHN55wuvRiPiOgmSPOZlk/TEQEQFn8t6m7U18XL/N66Hkh/1OhgGj4NUboKXZ69G4Z606dIhnFCTnACVMRFppMZHCVlMdrEtWK7jMKCqCo34KGz+BJY96PRp3Vs6DPx0E950Au/SmSbJPQbLX9NGySBQtsFPY9uzXg8oeXTR5L5OGHw+7j4VXfw0tTV6PJrmmRnjpKrjnWNi5BVa+BY+d54+xS15RkOwxLUstEk0/DYXNGENNVYUyyZlkDBz1M9i8AhY95PVo2rdqAfz5cHjj9zDuG3DJXDjpd/DRS/D0pcoqSVYpSPaYPlYWiWb16UrBq62qoG7Dduq37vR6KPljn6/AHrUw+yZo3uX1aNpq3gn/vgb+coxTWnHuUzD1D1DWG2qmw5E/hcUPOduIZImCZI9pdTGRaDb4nxSu2upKABao5CJzjHFqk79cCW//1evRRFsdzB6//jsYew5c/CbsPTl6myN+ArXfhjdugf/e7s04peAoSPaYVhcTaUtvHAvbyD1606WkSCUXmbbX0TDkYJj9G6fu12vNO+E/18LdX4HGLXDOE3DKrVDWp+22xsAJN8P+J8OLV6ilnWSFgmSPaVlqyWefbdjuuqTIeeMohaxrSTFjB5dr8l6mhbLJW9fAwvu9Hcuat+HOI+G138CYs+HiOU5JSHuKiuH0u6FqEvz9Qvj45awMVQqXgmSPWauJe5KfPqlv4PCbZrHws82u9tNPg4DTCu7d1V+yY5dWisuoYYc7QeZrv4GmHdk/f/MuePk6uGsy7NgEX38cTv0TdCtPbf/SMpj2IPTbFx79hhNsi3QSBckeU9ZM8tXmHU67pi93uJwkpIl7gjN5rzlgWbzK3ZssScIYZxW+hrUw/57snnvNIid7PPsmGP1VJ3u877Huj9OtHM59ErpVwoNnOT2gRTqBguRcoIBA8lC6ga4NrkMpha2myllURJP3OkH1JBh2hDNRLhuLdDTvgln/D+6eDNs3wNmPwGl3QLeK9I/Ze3f4xlMQaIG/nQYN6zI3XtFd3JIAACAASURBVJEgBcke08IJkr+cV7bbYFkt4ASgvHsX9tmtJ/M0ea9zHPVT2FYP8+7u3PN88Q7cdbSzkMnIM5zs8fDjM3PsfvvAOY87AfIDZziT/0QySEGyx7QEr+S7dF7e+okQcFrBLVixiUBAr4iMG3ow7DUZXr8Fdm7N/PFbmuCVG5zyioa1MO0hOP1O6F6Z2fMMroWv/hXWvQePnut0zBDJEAXJOUC//yUfhYJjty9vvXGUkNqqCrY2NvPhuk4I4sRZhW/HRph7Z2aP+8W7Tvb4lV/BAafBJW/Bfidm9hyR9vkKTL0VPn3V6XoRCHTeuaSgKEj2mPokS74Kvardt4DTz4M4JgQXFZlfp7rkTjG4BvadAm/8ITOlCi1N8OpNTvZ46+fwtQfgjLsznz2OZ+zZcMwvYOlT8K+fqmZLMkJBssfUJ1nyVfoT91RuIY4hld3o36urFhXpTEdeCY2b4a07Onacte85E/NmXQcjpsLFbzkLf2TTxB/AwZfAW7c7K/OJdFCJ1wModJqkJPkqlBF2XW6hnwkJMsZQW1WhRUU606CxsN9J8OatcOCM1PsVh7Q0OwHpKzc4K+V99a8w4pTOGWsyxsCx18G2dfDva6DHbjDuHG/GInlBmWSPKRaQfNVabpHGvoqSJai2upJVm3bwxZc5sIxyvjryCtj5Jcz5k7v91i2DvxwDL//SqTm+5C3vAuSQoiI45TbY8yh4+vvw4b+8HY/4moJkjzlZMwUEhWLjtl1c+dQ7NDbl/ypirS9r969v/URISG2wX/L8FSq56DQDRznB7X9vh+0pPM4tzU6P5T8fBps/gzPvha/eDz36df5YU1HSBb72N+d+PXYerJzn9YjEpxQke86qu0UB+fULy3h47mc8vXiN10PJSXrD2PmMMXXGmHeMMYuMMfPj3G6MMX8wxnxkjFlijBnvxThDRgzqTbfSYk3e62xHXAG7GuDNP7a/Xf0HcM+xTjnDvlOc2uORp2dliK507QXnPAG9BsJDZ0H9h16PSHxIQbLH1N2isARCdboFEAzaNBYTCW+b/w+P146y1o611tbGue14YJ/gvxnA7VkdWYzS4iLGDilXJrmzDRjhBLtv/Rm2rW97e6AF3vg93HGYswz0GX9x6o979s/+WFPVs7+zKl9RKTxwOmxRckLcUZDsMacnrNejkGwxxvlaEM95Gn2SFSPnhFOAv1rHf4FyY8zuXg5oQnUF763ZQsPOZi+Hkf+OuAKadzjBcKT1y+Ge4+Clq5yexJfMhVFntv5Cy2WVe8K5T8COzc6qfDv0iYSkTkGyx7QsdWExOH9UCuE5T2fini2gTLuHLDDTGLPAGDMjzu17ACsjvl8VvM4zNdWVBCws+myzl8PIf/33hVFnwdy7nKWeAy1O+cUdk5xA+fS7nd7HPXfzeqTu7D4Gpj3o3IeHvw5NO7wekfiEgmSPKZNcWAopk9yR5agL4OHx0iRr7XicsopLjDGHp3MQY8wMY8x8Y8z8+vr6zI4wxrih5RijyXtZccTl0LILXrwS7j0eZv4c9jra6Vwx+ix/ZI/j2fMIOP3P8NkcePI7zhsAkSQUJHtM3S0KSzhILoAwMFyTXAD31U+stauDX9cBfwcOjNlkNTAk4vvBwetij3OntbbWWlvbv3/n1qX2Litlv4G9WaB+yZ2v714wZhq8+wTUL4PT7oRpDzkT4Pxu5Bkw5QZY9iw89+PCyFZIh2gxkRygH9NCEiy3KIAnPXQf05m4VwiPjxeMMT2AImvt1uDlY4FrYzZ7GvieMeYR4CDgS2vt51keahu1VRU8tXAVzS0BSoqV3+lUx1wDvfeA2m9Db0/L0TPv4AuhYS28/lsn8D/yCq9HJDlMv2k8Zq1VJrmAtGaSC4e7iXvKPneyAcDrxpjFwFzgOWvti8aYC40xFwa3eR74BPgIuAu42JuhRqutrmDbrhaWfbHV66Hkv567wdE/y78AOWTyVTD2XHjlVzDvL16PRnKYMskesxRWwFTowtV8BfDGqHXiXur3VZnkzmWt/QQYE+f6OyIuW+CSbI4rFbXVlQDMr9vIyD36eDwa8TVj4OTfw7Z6eP5/nTcF+5/s9agkBymT7DWrgKCQFFImuSOfkBTC4yPu7FHejd37lDFfdcmSCcUlcNZ9sEcNPHE+1L3h9YgkBylI9pjT3UIhQaEwhVSTnM4+BfC4SPpqqyuZX7dJvzMlM7p0h68/BhVV8PDZsHap1yOSHKMg2WNOTbLXo5BsaW0BVwBPekdKJwrg4RH3aqsq+GJLI6s3q8+tZEj3Sjj3KejSw1lsZPNnXo9IcoiCZI+pJrmwhGqSC+E5T2cSnibuSXtqqysA1ApOMqt8CJz7JDRth7+dDts2eD0iyREKkj2mPsmFxZgCKrdIZzERTdyTduw3sDc9u5Ywr06LikiGDRgBZz/iZJIf+irs2ub1iCQHKEj2mFXOrCAV0nPuqk9yzFeRSMVFhnFDy5lfp0yydIKqQ+HMe2DNQnh8OrQ0eT0i8ZiCZI9ZdbcoKIVUk5zeYiL5/7hIx9RWVfLB2q18uUMBjHSC/U+CE38Ly2fC05fqD3SBU5DsMQsE9ENYMExrp+S8l05WOJ3eylJYaqsrsBbe/kzZZOkktd+CI38Kix+Cf1/j9WjEQwqSvWb10XIhac0kezuObFCfZOkMY4eUU1xkNHlPOtcRP3GW5X7jFvjv7V6PRjyiFfdygSKCgtHa3SL/n3StuCedoUfXEkbs3luT96RzGQMn3OysyvfiFdCjP4w60+tRSZYpk+wxq6l7BaWwMsnBr6526oyRSL6pra5g0crNNLUEvB6K5LOiYjj9bqiaCH+/ED6e5fWIJMuSBsnGmDJjzFxjzGJjzFJjzC/ibPM7Y8yi4L8PjTGbI25ribjt6UzfAb/TxL3CEm4B5/E4ssN9lKw3jJKK2qpKGpsCLF2zxeuhSL4rLYNpD0G/feHRc2HJYxDQm7NCkUomeSdwtLV2DDAWmGKMOThyA2vt/1hrx1prxwJ/BJ6KuHlH6DZr7dSMjTxPaOJeYQmXWxTQU+5qMREbebmAHiRxJbSoyHyVXEg2dCt3Fhvpuzc89V246yj4dLbXo5IsSBokW0dD8NvS4L/2/nqdDTycgbEVBGuVOysooXKLAnjW01pMpIP7S2EY0LuMIZXd1C9Zsqf37vDdWXDanbBtPdx/Mjz0Naj/wOuRSSdKqSbZGFNsjFkErANesta+lWC7KmAY8HLE1WXGmPnGmP8aY07t8IjzjEXBQCEJtYArhOe8deJex/YXiae2qpL5KzbpEwfJnqIiGPM1+P58OOYaWPEm3HYIPPND2LrW69FJJ0gpSLbWtgRLKQYDBxpjRibYdBrwhLW2JeK6KmttLfB14BZjzF7xdjTGzAgG0/Pr6+td3AV/0+/3wmIKp01yWhP3IgMeBT/SntrqCtY37OSzjdu9HooUmtJuMOl/4NJFMOE78Pbf4A/j4NUbtZx1nnHV3cJauxmYBUxJsMk0YkotrLWrg18/AV4BxiU49p3W2lprbW3//v3dDCsvKCDwr+ornuOap5emtG1rTXL+P9+hkpJ0lqUWSaa2qhKAeSq5EK/06Asn3AiXzIW9j4ZZ18Mfa2DhXyHQknx/yXmpdLfob4wpD17uBnwFWBZnu/2ACmBOxHUVxpiuwcv9gInAe5kZuv9FZ808HIh02H1v1qW0XSG2gEt3nwJ4iKQD9tmtJ73LSliwQpP3xGN994KvPQDf/hf0GQxPfx/uOAw++rfXI5MOSiWTvDswyxizBJiHU5P8rDHmWmNMZLeKacAjNjpFtj8w3xizGCcDfYO1VkFyUOQjpQ4XhSFck+zxOLKhdVlqF90t0BtHSU1RkaGmqkKZZMkdQw+G81+Cs+6Dpm3wwBnw11Phi3e8HpmkKemKe9baJcQpkbDWXhXz/TVxtnkTGNWB8eU1m+Cy5K9CyiSHuLqvUZnkAnqQJC211ZXM+uADNm3bRUWPLl4PR8T5JX/AaTD8RJj/F3j1105WeezX4aifQZ89vB6huKAV9zykcovCU1DLUgdf1OneU/1MSDK1VU6/5AUrlE2WHFPSBQ6+CC59Gw79PrzzuFOv/J9fQqMWwfELBck5ohCCJiGcSlYAGJ8eFnFjzJBySosN8xUkS67qVgHH/hK+Nx/2Pwleu9nphDH3Lmhp8np0koSCZA9p4YTC05pJzn/h17SLF7d+DsSNstJiRu7RR5P3JPdVVMEZdzsLkvTfD57/X6fH8rLn9IsvhylI9lD0ErzejUM8UABPeLgFXBr7QEE8RJIBtVUVLF71JTub1XJLfGCP8TD9WTj7EeeTxUe+DvedCKsXeD0yiUNBsoeiAoKCyC3mH7f9jkPbBwrg6Q4vJuKmT7Im7olLtdWV7GoO8O7qL70eikhqjIHhx8NFc+DE38L6D+Guo+GJb8OmOq9HJxEUJHtImWT/cxvshjZvKYAnvDVIdtMCru3+Iu2pCU7eUys48Z3iEphwvjO57/DLYNnzcOsE+NfPYIdez7lAQXKOUDzgT277WwfCmWQ948noEZJU9OvZlWH9ejBfQbL4VddecPTP4dKFMOqrMOdP8PuxztfmnV6PrqApSPZQdCZZIYEfuQ12Q5sHCqDewsZ8TWkf/RxIGmqrKliwYqNeP+JvvQfBqX+CC1+HPWrgXz91MsvvPqWP1jyiINlD0TXJ4kduf2+FYuMCiJFb+ySnW5OsPwqSotrqCjZtb+Lj+m1eD0Wk4waOhG88Bec+5WSZn/gW3H0MrJjj9cgKjoJkD0UFBAHvxiHpcxvHhd4YtRRAlJxOJjne/iLJ1FZXAqgVnOSXvSfDBbPhlNtgy2q4dwo8cg6s/8jrkRUMBckeil6WWiGBH6VbblEQWdI07qIms0o69uzXg8oeXTR5T/JPUTGMOwe+v9CpW/7kFbjtIPj7RbDkcdiyxusR5rUSrwdQyLQstf+5D5KDmeQCeMLDfZJddbeI6gEnkhJjDDVVFVqeWvJXl+5OB4zx58ErNzjLXC9+yLmtYhhUTYTqiVB1KJRXhVd3lY5RkOwhm+Cy+Ifb562wapK9HoEUktqqCl56by31W3fSv1dXr4cj0jl67gYn/RZOuAm+eAdWvAEr3oQPnoNFDzjb9B7sBMtVh0L1JOi7t4LmNClIzhEF8fF7HnJbS15I3S1CtJiIZENttdMvecGKTUwZOdDj0Yh0sqJiGDTW+XfIJRAIQP2yYND8hlOW8c5jzrY9dgsGzcFsc//9oUjVtqlQkOyhyICggGKmvKI+yYmlcw+1mIika+QefehSUsSCFRsVJEvhKSqCASOcfwd+1/kFuuFjWPG6k2muewPe+4ezbbcKGBrKNE+EAaOchU2kDT0qXlLWzPfSDXZbCqCbSXiSoovXdlSdfqYHJHmta0kxYwb30eQ9EXDKK/rt7fyrme5ct2mFEzCHAucPnnOu79ILhh7cWp6x+1go6eLZ0HOJgmQPaZKS/7mvSXY/mc2vWifuudkn4nIBPEaSWbXVldz92ifs2NVCty7FXg9HJLdUVDn/xp7tfL9lTTBoDtY1/+cXzvWl3WHwhNbyjD1qobTMu3F7SEGyh6xiZN9Le8W9AggAWzPJ7vcRSUdtVQW3v2JZvGozB+/Z1+vhiOS23oNg1JnOP4Bt6yOC5jfglV8BFoq7OIFyqDxj8IHQtaenQ8+WvAmSv9zRxOKVmzl83/5eDyVlqr/0P/cr7oVawHXCYHJMeDGRNO9rATxEkmE1Va2T9xQki7jUox+MmOr8A9ixCT57qzVofv138NrNUFTilGSEyjMGjYee/om93MibIPn7D7/N7A/rmf/zY+jX0x/tfyI/Ti6EzGI+cp1JTnM/X0rrPqp3uKSvvHsX9tmtJ/PqtPKeSId1q4DhU5x/ADsbYOVbrdnmt+6AN//g3NZzAAwcBQNGtn7tu7fvJwT6e/QRPl3fAMD2nS3gk08B1CfZ/1wvSx3qblFA7UzcTdxLbz+RkNrqCp5b8jmBgKWoSL1hRTKma09nqey9JzvfN+2A1Qvg88Xwxbuw9h345FUINDm3l5TBbvtHB84DR0JZH+/ug0t5EySXBnv+7fJR24DoJXgVEPiRapITS1ZuYa3l8fmrOGnM7nTvUhK1T9tvRFJTW1XJw3NXsnxdA8MH9vJ6OCL5q7SbU25RPan1uuZdsP5DWPuus9jJ2nfhg+fh7b+1blM+1Gk7N3BkawBdXpWTvZvzJ0gudh7c5oCPgmR9tOx7adck++dlmrZkj03dhu385MkldO9azEmjB6W0j0gyoUVF5tVtVJAskm0lXZzgd+BIGDPNuc5a2PpFa+AcCp4/fKF1Ra4uvWDAARGB82gnC92lu3f3hXwKkkucj9Wamn30V9ZHQ5X40s0kF8InB8nuY3PwnUJzxCzGqDeOnTMsyXNDK7vTv1dXFqzYxLkHV3k9HBExBnrv7vzb5yut1+/aDvXvBwPnd53AefGjsOvu4H5FULlXdOA8cCT02j1ry2znT5Bc7MNyi8jLigh8yX0m2fnaUgBPeGu5Rfz7GkhSelIAD5F0AmMMtVUVmrwnkuu6dIc9apx/IYEAbF4RzDoHA+fVC2Dp31u36VYZzFaPbq1z7je8UxZAybsguclHQXKkQqhRzUfuu1uElqXujNF0nuaWAC3W0rUk9QUaWrPm8W9vLT2JX3akiXuSrpqqCl549wu++LKRgX0KcxEEEV8qKoLKYc6//U9uvb7xS1i71Amcv1jiBM/z7obmxuB+pdB/OHz9MeizR8aGk0dBcrDcwkdBci4tJrJy43YAhlR6W//jN26D3fDEPZ9FyTf+6wMWfbaZxy48JOV9bMzXWK2rD0bsk+CyiBsTqisBmL9iY7jeXUR8rKyP05e56tDW61qaYePHrXXO696DHpnt15xHQXJw4p6PVmmInrjn7bgPu3EWAHU3nOjpOPzG7fMWbgHnswjwiy8b+WJLo6t9kj028Tp9KHssmTBiUG+6lRYzv26TgmSRfFVc4mSP+w9vXTUww3Kv30aafFmTnEOZZEmP2+ctXJPss0yyJf0ANlm5RSBRJjmts4k4fw/GDilnwYpNXg9FRHwsb4LkLj6sSdbEPf9Ld8U9vz3fAWs7sLx0uhP3fPYgSU6pra7gvc+3sG1ns9dDERGfypsg2Z81yblTbiHpCbXlTrUbTXiymt+eb5vO6oLRX2O11iSru4VkXm11JS0By6KVm70eioj4VB4FycFyi2Y/BckRl70bhnRAKNBLuWOjT1fcczLJ6XXySLSXTVJuIdIR44aWYwxqBSciacubILnEh0FyJAUHhSFch+u3mmTr/o1cstd0vHILTdyTTOldVsrwAb1UlywiacubILlLsNxip4+CZPWE9b9wJjnFeovWjg6dNaLOYXFfkxzePNFiIoEkE/d89hhJ7plQXcnCFZvCqzuKiLiRN0FyiR+7W0S1gPNwIJK2UICXarlFvAU0/CBg3b+RC9ckt3NMiM6qR01m1RtH6aDa6gq27Wph2RdbvR6KiPhQHgXJTpjip3ILt1mzloDlhheWsW6ru3610nlaM8mpbR96mv1Wk2zTmLgXuW/869v2jI6ezJre+URCaoOLiqjkQkTSkTdBcpHxYZAccTmVoGnlxu3c8erHvPbh+s4blLjivuODPxcTAet+dcEkmeDWmuRE+4t0zB7l3di9T5km74lIWvImSA7FHH4Kkt0Kt8zyeBzSKhT0mhQLLvxak+yMN91yi0R9kuNkkqP299mDJDmptrqS+XWb9HoSEdfyJ0gO/nn1VU2yy4+Wky2+INkXDnZd9kn2X3eLDiwm4qJPsl7akmm1VRV8saWR1Zt3eD0UEfGZ/AmSfZhJdjtJycYJKsRbbvsk+7YmmXRawCXrk+x8jX6/ED+rLJKumqoKQHXJIuJeHgXJfswkx7+cSLIaTsk+t7Fu6Lnz24p7Aev+zVnyPsnxJu6lvr9IKvYb2IueXUuYX6cgWUTcyaMg2fnqp0xyZK4slcxieAUzBQ85I1yTnGp3i3C5RWeNqHNY674hW2j7xOUW0V8j94n3nUg6SoqLGDe0XJP3RMS1/AmSg1/9FCS7XZY6FFj57aP6fNbaJ9ntxD1/PYfWpl9HnXTiXoLj+uwhkhxWW1XJB2u3sqWxyeuhiIiP5E+QHMok+6ncIvJySuUWqknONe77JPuzBZxNY2mP8F101SfZ/dhEkqmtrsBaWKi6ZBFxIX+C5FB3C59mklPJJfu1fVg+c/tUhD4N8NF7OSD42kuzT3Ki3eKWWyRoByfSEWOHlFNcZDR5T0RcyZ8g2Yc1yW6Xpc50FnLHrhYWrdyckWMVKvfdLfz5aUAgnZrkFCfuJQqMffYQSQ7r0bWEEbv3Vl2yiLiSR0Gyv7tbpJIdznR3ix89tohT//QGG7ftyswBC1DrxL3UwmS/drew6XS3CO+bqCY59DVBdwvlkiWDaqoqWLRyM00++hshIt7KnyA5+NVXmeQEHzMnkuma5Lc/c7LIO5tbMnK8QhQqn0g1kxx6ofpuMRHSeHMWfr0mujn0yUjkeVSfLJ1jQnUljU0B3luzxeuhiIhP5E+Q7MNyi0ipxAM2SdDhViibWZTqrDNpI/xUuF1xz2cBoNMCLs1McsJjOl/9NolR/Km22llURCUXIpKq/AmSg3+Kd/ooSHabNcv0stShbKaC5PQVzIp7NvOZ3UC8N31Rn65k9nxS2Ab0LmNIZTdN3hORlOVPkOzHFnBJ6i/nfLyBv82pa7N9prKQbtuXSVvua5Kd7Vt8lkpOo7lF+PWabDGRyMciOl7212Mkua+2qpJ5dZt8N3FWRLyRNEg2xpQZY+YaYxYbY5YaY34RZ5vpxph6Y8yi4L/vRNx2njFmefDfeZm+AyGhv7N+LbeIFw+cfdd/+b9/Lg1/H28Z344IJAliJLnwYiIpr7gX/dUvAtZ2oAVcool77fdJ9ttjJLmvpqqC9Q07+Wzjdq+HIiI+UJLCNjuBo621DcaYUuB1Y8wL1tr/xmz3qLX2e5FXGGMqgauBWpw/sQuMMU9bazvh8y7nL6qfZi67726R2Yl7oXILZezS5/YNS+i582V3C7c1yUneECSbuCeSaROqKwGYV7eJqr49PB6NiOS6pJlk62gIflsa/JfqX7LjgJestRuDgfFLwJS0RppE6A9xs48+xo6qSXaxmEim4qtwgOefhyznhB7C/K9Jtq7LfJJt3vpJhjpaSHbss1tPepeVsGCFJu+JSHIp1SQbY4qNMYuAdThB71txNjvDGLPEGPOEMWZI8Lo9gJUR26wKXhfvHDOMMfONMfPr6+td3AVH6I+rn2o93X60nOnOCC0ZPl4haq3rdleTnE7fYS9Z0uiTnGTzZOVDoat3Nrfw8NzPfPV4SW4qKjLUVFUwv06T90QkuZSCZGtti7V2LDAYONAYMzJmk2eAamvtaJxs8f1uB2KtvdNaW2utre3fv7/b3cOZ2JaA9c0f0wST+hNvn/HuFqFz++PxykWuM8kuS2xyRRolyUlXF4y7LHWc/d/4aD1XPvUO732u/raZEEx6vG2MeTbObQnnl+SL2upKlq9rYPN2LaIkIu1z1d3CWrsZmEVMyYS1doO1dmfw27uBmuDl1cCQiE0HB6/LuMg/tH4puYj+mDn5mDNek6xMcoe57RAS+Vj76VOPgLVpl0Ik2s3GySTHK71oavFnR5Ac9gPg/XZuf9RaOzb47+5sDSpbaqucfslqBSciyaTS3aK/MaY8eLkb8BVgWcw2u0d8O5XWX8D/Ao41xlQYYyqAY4PXZZz1YfARlTVLYciZbgEXLrfwyePVWV75YB1bGpvS2tdt4BgZBPqpLrm1Ht7FmJPU0IcnjibMJEfvX+Av04wwxgwGTsRJZhSkMUPKKS02zFPJhYgkkUomeXdgljFmCTAPpyb5WWPMtcaYqcFtLg22h1sMXApMB7DWbgR+GdxvHnBt8LqMiywZ8E8mOeJyCh9mhzPJGSqP8GMdd6at29LI9Hvn8f2H3k5r/9ZAN3Eq+f3Pt7BuSyMQW27hn8c9NmB1s08icRfHibOTzfAnKAXuFuAnQHttgOLNL2mjo/NIvFJWWswBg/po8p6IJJW0BZy1dgkwLs71V0VcvhK4MsH+9wD3dGCMqYnMJLf45Y+pu1n9nZVR88ubis7Q2OTECp+sb0iyZXzhmuR2yi2O//1rdC0p4oPrjo96g+Onhz0cqKa1T6Ka5LZlFNGrUEaXA/no4cpJxpiTgHXW2gXGmCMTbPYM8LC1dqcx5gKc+SVHx9vQWnsncCdAbW2tr56eCdUV3D9nBTubW+haUuz1cEQkR+XPinsRl5sD/umVHOKuu0Vm/x4VciY5FNym+5Cmuix1aLl0v9Ykp1NukbxPcpLbw1/blmVIWiYCU40xdcAjwNHGmAciN2hnfkleqamqZFdzgHdXf+n1UEQkh+VPkBzxF9QvmdEknzK30Vkr5PkpWOssmZ6Ulvg87iZr5opAOpnkmK+Jjplsxb3Wl6d/Hq9cZK290lo72FpbDUwDXrbWnhu5TTvzS/JKbbUzeU+t4ESkPXkTJPuyu0Xk5RQCpvDHzxm+f36qjc20VLtSJOK2u4UfJ5hCmjXJSbaNV5McvU90LXIBv0w7VSrzS/JNv55dGdavhybviUi7UlmW2hci/376pSbZ/bLUqW/rhl/eVHSmdLO6oYfOpNgpOfIsTT55nULy+uL2941/fbwWhPE21cKQmWetfQV4JXg5pfkl+aa2qoJ/v78Wa23KiwGJSGHJm0xydLmFP2qSowMz285toS0y290ixE8ZzUwL/XFM9xGwLjPJkVnTxqaWNM+afeksiW6TFFzE61oRr09y6DiF3qpQMqu2uoJN25v4uH6b10MRkRyVP0FyxGW/BH3R5RbRt+1qaRvou6lJ3rGrhVc/TK0tk18er84Qim1TDf5e/bCew2+cxc7mFlf7WprMDwAAIABJREFUhVgLXUucH7vGZh8FyaGvaZRbJM4kR3+NPE/k5daVIUUyp6aqEkCt4EQkobwJkiP/gvrlY+z2Ju7tam4bJMdboSyR59/5nPPumUv91p1Jt/VL5r0zpZqdv/aZpXy2cTsrN24HUu9uEXmebl2cllOh9nN+0JFyi0SSdWuJXYZdNcmSSXv170FF91JN3hORhPImSI784+2XzGh0T9jo2+IFyW5awIWylPEy0m2O659YLePcvlJKi50fmV3N0fW0qdY0BgLQvdQJknfs8k8mOZ3OKskm3IWOGdUnOaq7RXRHjUyXGUlhM8ZQU1XJfC1PLSIJ5E2QHBno+SYzGjVxLzoAiFtuEQhtm/zQrUFN8o1bPE7RvbPqS6bfO5emFAL6TIu3NHJ7ugRLJUJjTfb4xrs9nEn2VblFdMCa2j7R+7a5PW4QHacWXzP3pJNMqK7g0/XbWN+Q/BM3ESk8eRMk+zOTHP8yJCi3CH1N4e65aZvV4vGbiv99fDGvfFDPx/XprXqXCam+ZEKZ5NYguf3tY1+LAWvp3sVpKrPThxP33LQLTF6T3PaTkXglSIqRpbOoX7KItCd/guSIv6B+aWkW76PlkJ0RQXJsLXJqPZWJ2qc9HiRwo3j5MbrbJHppsVNWsSu8gp5zgKIEP0mxWXproVupH2uSo7+62jfB9XH7JMc5Z7ztRDJh5B596FJSpMl7IhJX/gTJEZebszxx78O1W9m2s9n1fu0Fh7uiguTQ19Rrkt1MdorMJHu5ClyqvYYzqfWxTO1+h2uSW6KXmU409tgkfcC2Ttzb4atMsvt0brI3P3H7JEdlkmPfHKZ+7vbMq9vIA/9dkZmDia91LSlmzOA+WlREROLKnyA5KpOcvQxdS8By7O9mc/GDCzt0nNgAICqTHPzqZjERN9m35gQTp7LFywShmxIWgC7BIDmUBU624l6bTDLQPdzdwj9BcrjG3UWUnCz7HPvmr+0GUV8y9nnDkwtW8fv/LM/Q0cTvDhxWyburv2TdlkavhyIiOSZvgmQ8qknesqMJgIWfuc9ERK+4FzNxLyJIjq3dTCXwje0M0J6o7gIpbJ9poXN6seiVm8cJWifu7Uxx0l3s6o/W+rQFXHiJaDf7RO8bKzRpMrpPctvXYrxFRzoiYK3ayUnYWTVDaLGWv+nTBRGJkTdBcsBCUTDIymZN8pfBILl3WanrfePVX4ZEBmGxGblU/sC7qV+ODJK9rPsMxciPzP2Ml5etzco53XQBgdZyi1D7tlCglyi+j1eTXFbqx3KL4Nc09km0U9ya5DjbZnrinnNeRcniqO7Xg2P2H8AD/13hq093RKTz5U2QbK2lJBjAZDOTHAqSe5WVuN43agnemNviZZLdZNTclGYk6lObLbH35y+vf8pj81Zl6+wR/0+uNFxuEQySQzXJCdLg8bpbFBtD15IiX3W3SG/yXIo1yQk+yWgz+TRDr82AtSl3M5HCcP6kYWza3sRTC1d7PRQRySH5EyQDpcFUcjb77XYoSI68nEqfZFc1yW4m7uVIJtm0jiFbHS/cdm3oUuIMckdsTXKC7WMfT4vziUdZabHPslZplFskiW1bJ6K2vc7ZL/r2TL0mrPV2gqrknoOGVXLAoN785fVPot60iUhhy58g2UJpiZeZZPflFpF/81PJJLurSY7etz1et8yLPbu1qfct7ii35RYlRdGZ5GSTytpkkgMWYwxlpUUdrkm21mbtD7qNCVS372pm+r1z+WzD9hT2TVCTnOQ12qarS4be+yqTLLGMMXznsGF8XL+NV5fXez0cEckR+RMk0xrAeFGTnF4mOXGUvDNuC7jor+0eO412cakeO+PC53Tysc7EqiwFfy7LLUKPVThITlICExskW5yMeVlpcYdrkh+dt5LDbpzVoWOkKrbkYeXGHbzyQT2LV21OuE+yTHK8FnBR+6d4HLcCyiRLHCeOGsRuvbpyz+ufej0UEckR+RMkW0uX4EIP/qlJbr2c6e4WrRnS5OPIvXKL7AXrbs8TGySHy1qSbB95PoOhWwbKLVZv3sHqzTuyEvDFZsxTeXORvE9ycLtkE/fCpR6ZuZ/qbiHxdCkp4rxDq3lt+XqWfbHF6+GISA7IoyCZ8MS9bGaStzQ6QXKX4mLX+0YvnBBtV5w+yel1t0i+ba60gAtxPg7PzkjcTgoLLVSzI2biXqLhtskkW0uRga6lxTTGWXrcjUwvstGe2PKdUOlDe4FrstdrvE87olrAxWSaM3U3rfVyjUfJZeccNJSy0iJlk0UEyKcgGUtJMJPcnMWJe6E+yekEdZF7xO4eOXHPBi+mlUlOIRzImUxy8Gs2a5LdfpQfaukWqieOrdWNFft4Bmyw3KKkqMOZ5FSCx4adzSnVDSc/V3RAnkrf5Njsc6JjJl5xL/65OyoQ0BLXEl959y6cWTOYf7y9hvqtO70ejoh4LH+CZNu6GpoX5RbpiDeTPyR6xb3oYCKVuxevc0Ai3q+4FxtIZi+TnGgMiYQmyu1oU5Mcf/vY92sWS5ExdOvS8XKLVCZnfu3Pczj8plkdOo9zsqgvKZ07duJdrKR9kttkolVuIZ3vWxOHsasloKXLRSS/guRwJtmDIDnTmeRAVHY3tH3qtZnJss6Rx4/uk5z96CFeuUW2JKspjhV6bbWpSU6x3CJgAQNlJZkIkpNnWJeuyUxtZdte3UR9jTu+JHXL8R676Bg5+pyZ+rEOWGWSJbG9+vdk8n67aXEREcmjIBkbXujBi0xyR//oxsumxt6WSvaudf/QvvFvj1wJzuvFRMLnDn7NZhDjps4b4k3cS2371hNCUYZawLkpv+moRBnkjpw7tGuyn1e3z1Hy86omWdp3/qRhbNi2i38u0uIiIoUsb4LkgIXSUAu4Fi+CZPf7tjdxL/L78MfSgdQzask+6k5Uh+xFhi02CHL6/2bp3OGvqd3vlnAmOaYmOcHjFvupRsBaDJlZTCT2dbBk1WbWbW3s0DETiRPrR32Nv1P7G8ULtON1usj0YiIBa72ZoSq+cchefdl/99785fVP1S5QpIDlTZCMhaIiZ1JUc7YiLCKDpXR+kSbO4Mab8e9mln+yj6gjg7eomuQUjt15Wsec7e4WqZ4uVGO8I8UWcPH6JDuZ5I73SY4NMqfe+gbH/m52h46Z6rliyy/iSfYGJN6nHVHlFjHBccYm7qncQpIwxnD+pGF8uLaB15av93o4IuKRvAmSLRaDoaTIZLUmOZzdTSMujwoOEqXqoM1H3G5qkhNmkiOy7YEEWeVsiQ2CAln8ONx1d4vgE912MZH427ftbmHDi4ns7PCKe6Fjtl63eXv6E0nbPVfMOVOqSY55bN74aH3UZKi4JRtxLma6BVw2X1/iXyeP2Z1+PbvyF7WDEylY+RMkWyeLXFJUlNWa5JZ4f+hTZBNcjj1ebCDnpiY5cSY5EHG5nbqPLArfv0D2Vtxze4dD7y1CnyAkKwVo2yfZyVKVFJmouvB0JFvtL6NiXoOpTKaLvemcu9/i5/94t/X2eOUWcfokZ/p+WmWSJQVdS4o575AqXv2wnuVrt3o9HBHxQP4EyYSCZJPVmuRQEJRWsUWcbHG872OzwqlkrVvLAJIHb9H1ycmPnWnxspPZGkf4PCmeLxCuSQ5mkpOUAkRm6UPPn8F5rXY0UHPTErCj2nS3CF7fXk42WbY5EO7/3XafeMfJFLWAk1Sdc3AVXUuKuOcNZZNFClH+BMnW6T9bXGzCH4ln57zO1/QyyYnLHOIld910FEgWoDQnCJKTfRBdt34bm7fvSnp+N2JrT7PZJ9nt8xd6rHY2x6y4l2j7OJ8IFBmDMabDgZrb1QI7Iva9RCqTSJMF0slKgmJf94/MXcnhN85KccSJuSlbksJW2aMLp48fzJMLV7OhQYuLiBSavAmSQ3+sS4oMTdkstwhlktM4Zbv1nJEBdExAksq5kgXUUZnkOIFcItPvncsf/vNR8gGkIbLGNlvxi9tAKfS4NbU4JSHJFxNp+0bIGCgy6Z0/UmwQ2ZnarrgXGkTqb9jaXB8+dtvr4p3rs43b+WxjJlYPbH9cIpHOn1TNruYAD771mddDEZEsy5sg2Sm3MBQXmahJaZ0tYzXJ7ZRbxF6XyrSjZF0bojLJLW0DuUQadjazpbGTJodFZHWzleVzOykstr90sg8touttHUXGySZHnj8dsaUPqWyb/rmijxNvSWm35wzdnrhPd+y5MpQB7sCnP1J49t6tF0cO789f56wIf4IkIoUhb4JkrNN/tqSoyJvuFmllkhOXOUTe1qYmOYVzJSsjiCxJcZNJDlhojl1rOUMi63uz9RS6WcUQogO65oCNeN4SPc6tl1szyQYTc106Wmt608/mpsLGCfRbl4xOfOD2H5nImur4PwexJUNu2/Ul4naVRZHzJw1jfcNOnl60xuuhiEgW5U2Q7GSSnaWps1mTnKlVz2J3jzeZyc25kq64F4i87CZItp1WzhJ5P7OW5XOZSY4cV1NLIIXHue1jawwUBestOnI3I4PGQJLnpCOPpo37Woz+2t5JE07cixP0/n/23jzejqO6Fl7Vfc69V7ItyYM84HnAgDFgg/GADWFyAiExCYRA+GwMmCFfSML3SMiD5D14EJKQkADJBwmTAQdihzkhgAFjTIzxIAvbeMajLHmQLVuzdIdzuuv90b2rdlXt6j7n3itd6aiWf/6doburqvv0Va9atfbe8goKKc5Os7PGzqxUmDAaOOu4A/CUg/ZJxUUSEvYwjA5J1lXWgHwn5knWWjOSNAu7BTvEH7JUBW+YbAZ2KV7eOZYCrs3KUZZ63u0sUtntnfUcGtafyjOn9AvdqkpKqfwUFGq3xdyUZHbvtbUzp/LRznvtvTYd16wly0py2K+/31xJSvIkJwwLKi5y59otuPreJxZ6OAkJCTsJo0OSUWW32Jkp4NpSVw2DwG4hvJ9NMZGYqO4ElA2RAk7rHVfRkFcW3GnZLYbUJR0luSwDr64PKXMI9yTPyQbBiGpbzuV5s1t4E7VGu0ULGZXyJDcdP5d0i1K/iSQnDINzTnoSDth7LBUXSUjYgzAyJLksq2XsfCd6kqXMBcNA8l/azzp430Y6ONoC0qIp4AZQJHvzrSSbvnc+gRm2H8eTXLDsFpH9eUYLOlQpzIsneRjlfS415vifU5AvuaHZQaw7fvv8IJ4SUHp9x8U34Fs3Ptg6/rBft/2EhEEw0c1x7ulH4sd3PoZ7Htu60MNJSEjYCRgZklxTEXR3oifZtUQMf/ygBMNXKwfzJDfv6weg2ePa2t2RSrK8BN+GstS4ZpZLoMOSVJ8kty3d06pGppT5/TKlrJI85Hg5OMlsqzI5H4o1b8f3yTcdF51AkMc4Zrfw73sv3eJ3b34E/+Mrv2g/Ab/fpCQnzBLnnn4kxjoZvpCKiyQk7BEYHZKsda0k7zxPsus3nYWS7BBh93iJgA/nSW4eVz+a9q1dSZ5vOwsf62wyD1x73xP4vc9eizvXbh6+7yH3L+r7DKjsFm0WGLJBKLi/2872JM/NbiH1PfjvFLs2kpIsBu6Z/QfobADMZiKWkAAAB+w9jt8+6VB844YHsWHb/BZVSkhI2PUwMiQZQJ0CTrWqavOFuZZz5of4xzsE2lt2HoSQD1PkYjglWc/7JIQrjrPJPLB1ug8A2D4zixyms7BbjHeqP5vKbhFvZs367fjl2i0Aag9yvZOjJM9BlOe/8TD5mofvJ3w/kJLccG349pjdx5Lj6t1ccpK7/Q4/EUtIILz5rKMx1Stx8YpUXCQhYdQxMiRZa1gleWcF7gk5cIeBFBBl2wv7GUYBa1OduWWC21PafaQ7ME+yHs53bcfUPHnYPtPHn3/rFrEIyrC/W1lqjHdyAFUKOGNFEJp5/t9dYYN8lOtPng8lmV+rViV51r14dotgwjb7PqWVGNdu4fYxX351M/adlykyYYTwlIP3wfOffAAuunoVZvrpJkpIGGWMDkk22S2yHeaZ9TFnT7LzXnvbQmICQ1AGH9tAZamHCEDcIYF7pjk9KyWZlO3Ydbno6gdw8XWr8dkr72voe/C+JrqZeV8OmLs3s0LyDvAk6wGyW8zd1lG1U78O0G7bhGfQvx/axrNbzMf5pMC9hNnigrOOxmNbpvGdm1NxkYSEUcbIkORS82IiO/bhd8PqDfjSNau8SnVzY8n+4dIStxToFG26ZanbsViwOUVzMGFlL5jL9b30lkdwxS8fi7Q/uxy2NJ7YuKZ6lQ1DkXzLMLSSrF0leVALjIJylORsXjzJljS2FROZj/LXTp8DqbrNtgY5awbv1+2fe7Dncg8OUlI7IaEJv3L8cjz5wL3xuZ+m4iIJCaOMkSHJWmso1HmSd/DT71X/fDX+93/e5hCT2XFkrha7kIpQDBOV30be+Ni58t6krlFTvTko9Z+68j583sszynXy2SjJRdl8TK+2h3SzkCQP+7NxT7JTca/lOKXs9VNKGcI+H4F7VYXClp3nQpKF9/aebDiOkVrpPhTJd0M7dl+If+Nr1m/H534arhY0jSshYTZQSuHNZx2N2x/ZjGvvW7/Qw0lISNhBGB2SDAC1J7lNZdo63cemydCfOiwK4SE/DJxDvOP5x7DiXntfbYSaj527J5qapjbn4vkuyjL4ffiyvC7d7wYBEabYMbS92wlv92F/tqLUGO/awD2jlra0w1PAKVhP8mx42oZtM3jvN28xCrnWAxQTmQNL1sJKwyCWBb5FsuiINg4ncC8y+YmsZlx66yP40HfvwBbBe+72m5TkhLnjt08+FPvtlYqLJCSMMkaGJENTdovMKIcxnPKhy/CsD/xwzl1Ky8XDgB/RVJbaKq2DP9zbCDVvY9DAPTpmLoF7RRm3RfBUZsPmSW46hoJrurlEkof73YpSY6K2W/TLciCyCFT3Ju3heJJnQdR+/sAGXLJiNe54ZEvdht6xdgtBSx7mXgTk3NpS6XWn34hKryFnWCm8ANcYrGqdWHLC7DHRzXHuaUfg8jsfxf2Pb1vo4SQkJOwAjAxJ1qiWwLJMtT4kp3rzE9g3TDlnCY7/sqEsta8KD0IgfR9nbDvQlDPZhVGS5+IHLSU/qV1qn02hh7bAPSJoY/k82C20xkSXPMl64PEqL7vFXDzJJiVafV6DlPGei7VA9MdrYWOkT62BXr9ZSZa86LH7PVY8xZZib5kwzGK1IiFBwrlnHIluloqLJCSMKkaHJGuNTAG5mh3xmA2GKefchjBwL2x7GALZ5ruU7BxAM2mk3eZEknU8E4PWw1lKeJtNxxBBk5Xkgbup+ijh5Eke3JOsTF+Zwpw8yaX3O2i0e5LnoiRLWSjaJmHVuOhVY0ZYfZA8yeLxwt+GpExTe4Nm+kgkOWGuOHCfCZxz0pPwtZUPYuP2VFwkIWHUMDIkudTVknamdmIxEYE8DIOmwL0m9W4QQt5GqHnfg6aAo21tdpYmFDpUku2yuh6IfPkgJTymINJ4OwJJHrostWae5LK0v0VLMxlXkqFAmvZsiJohg6SgN1hYzDFz8SQLffPfLD5O+yrdMw75ZoQ/7Mu7XyCfr7VbtCjJs5iIJSTE8OYzj8Zkr8AlK9Ys9FASEhLmGa0kWSk1oZRaoZT6hVLqNqXUB4R93qWUul0pdbNS6nKl1JFsW6GUuqn+/9vzfQIEDW3sFvzZd//j23DUe76LH93+6Pz32aKEtR8vv/fb8x/q8+JJZpzFVcSb2qw2ziVwT7JbaPZmmImAP67YdZFUzKDvAUCpxyY61m4xCFm0x1evld1i9p5kOoQrya3Xaw580LUF0evgSjLg3jPSRKgp9Z/k15fuQWu3iI+J75cocsJ84IQnLcGZx+2Pi65eNScBISEhYdfDIEryNIAXa62fBeAkAC9TSp3u7XMjgFO01s8E8HUAf8e2TWqtT6r/P2deRi1AGyXZJYY3rt4AADsk6XsxYH7hGCSFTmrPVyuHym4xQN/9MiQwYpv1+c5FqZeUZD6m2RAY60luVpIlpXk4Ml692uwWPE9y+7GWJCtkGX0//LW0k5XS9N1mMZivPMlmVWMAXy9XnflERZrASb+79l75Buke8m1JMcxmIpaQ0IQLzjoaazdP4Xu3PLLQQ0lISJhHtJJkXWFr/bFb/6+9fa7QWm+vP14L4LB5HeUA0BqACu0W9DYTCknMFXP2JDd4gSWVbRglORb0ZLeHS93SONwxVVvnkidZsgZwMjWXPMmx34BUR4lIDvOzUT+mmEipHXK3YdsM3nHxDdHy16S+ZkxJnosnmauvbT/JvNktvAIhAxW2gXYUNskTLE42IvewhuyLp9+31XqSPMkJ84wXHn8gjlm+Fy68KhUXSUgYJQzkSVZK5UqpmwA8BuAyrfV1DbtfAOBS9nlCKbVSKXWtUuq35jDW9nEizG5hctPuAJI8r2Wpg39YOXF1ieMwnuR4dgv73q2+V71f9fg2nPKhH+HhjZNBmzqi5A2CslFJ1kypHJ4kt9ktRPVxCPJI528D90onQPJTV96L7978CL587QPhwdqOj9+Ks8uK4h5UFRNpI4bD98Pbtw253w3SrO9JlpRkybZi3nudxO6/Qb3Gs8mgkpDQhCxTePOZR+PmBzfh+lUbFno4CQkJ84SBSLLWutBan4RKIT5VKXWitJ9S6lwApwD4CPv6SK31KQBeD+DjSqljI8e+rSbTK9etWzfUSQDVgy+rU2tJy8NCsbU5oy3PaxuaPcnhtkEzKTjjiSnJkMdO7y65fjUe3zqN/7jpIXFMs/XeFWWY3cKqkrMjMAPbLeaoJFM/43UKuD7zJAMw4Xiyr9b6hnme5Nk4Y/32NdonLXMJUnMVX9Ypmicz2rlf4pO+2PhiqyGllrNbDLrSkgL3EnYEXv3sw7BscRcXXtVe9TEhIWH3wFDZLbTWGwFcAeBl/jal1EsB/AWAc7TW0+yYh+rX+wD8BMDJkbY/o7U+RWt9yvLly4cZVn28DYiSKuHtSLtFN7fq9fptM/jaysGinLVATgmiyjaEFcFfko9tB+TAPbIUTPfCZXL/mGEQC7oCyJNs3w/cZtm8zE4ErUl9HATWblGXpS6ZJxmuQuyDnxvA7RaD90+QSOOOVJK18969B5ua5dYM125RvXKe2xy4F06qZE9y9Tqw3aJxr4SE4bBoLMf/c9oR+OHtj+KBJ1JxkYSEUcAg2S2WK6WW1e8XATgbwJ3ePicD+DQqgvwY+35fpdR4/f4AAGcCuH3+hm+hoSu7hVJikY8dwJHNw7uTZebB+45/uwHv/vrNA/0j6ZAPYUnZ72coK4JHrIPN7GspcI+IoBRwBcw+w0VRhoSOTwJm40luK0vda7JbDNFP6ZHkvldMpOkWq/bjSjL/fjj4p6EHIMlzQSlMouibpn7NJi2T5Jgv3uznt2M+24p7fIWozZvu95+U5IT5xhvOOAqdTOELP1u10ENJSEiYBwyiJB8C4Aql1M0ArkflSf6OUuqDSinKVvERAHsD+JqX6u1pAFYqpX6BSoH+sNZ6x5BkpiRLBLPNk7xuyzTe+81bMN0vBu6TFMpOpkw/j26ecra1jdkfp/TZJySDPNtbPcmRPMn0joggV5L5mGYbvFeUYUlhruwNo5b744rbLeLbh6FJNO6xDs9uEe4nkbTKSlK9V8pO2mZzGaVMKO0Wg3CHlavW4yX/8BNMzgx+zw9zL3KfsVTVsdTV347bXkjIw0mBvWf5CpGpRJg8yQkLhIOWTOA3n/kkfHXlGmyaDAN4ExISdi902nbQWt8MwSKhtX4fe//SyLFXA3jGXAY4KGi5O1PuQ5LetXmS3/eft+LSW9fihU9Zjl97+sED9UkP6pzZLai/QZTrtme0Uq5PdxgvZRtxjNkt/OA0Pmngbc1WSS51gxKu+TkO3qYpJtKqJAtdzoKM55lCN1folW7YH/3mUpNac3+8MpO22WSdkPzrs7Fb/OV3bse967bhzrWbcfIR+w50bFBMpElJZufGVyQ40e7kCv2SVS7U0vHhpIB+c06SJRuHhJQCLmFH4s1nHY1v3vgQvnL9arztBWIITkJCwm6Ckam4VynJCjlTdavvw4ephK3TfQDARB2UNQhMJTdmtxjGA23HFj6wS62RG9Zlv2MfG9FKMiP+Yvqa1NKZfonP/fQ+vOZTV3sWjdkpyaUWlGTzymwDQ/CXdiV5fgL3zKRIKXSyrM5uMZg6zX3DVBly2P55W05/A9gtpO2ksEvlumPH+uS4qVe7r5/dgivJmfOdFo4PlGTYDClKsFsMnN2ica+EhNnhxEOX4vRj9sMXf7bK5DJPSEjYPTFCJFlDoSLKTo5hwbsoYXu95Lx4bHCS3DckQzmEAGj2p/rIlAoe2FpXaYWAZitGDG22BUdJ5t5QoyTXgXv9Eveu24Z7122bFyW5KBtSwGmrAg7nSY4XCwGAXr8pBdzgMCQ5U+jkCr1CJqeiksy+V8qWpZ5VVhTvc6nb1VOpF34+gx7r50ke6F5E3JNMfTcWsQnuf/ubi3aL1sC9wceekDAbXHDWMXh40xQuvXXtQg8lISFhDhgdkgxrt5AD95qJAJFkKjk8CEgR7eSKqVOhwtUG30cNuEqy/1AfxIrQZluQvJ+AJT+dvOp7pl8apdIJ3JuDkhyQGDbBmEsxkXie5Dh5mk0/ld0iQ78sRXIqWSgcJVmBVdwbuPvomB0FPgJpM/2GnTaSLPmEW4Ilq3HZ43tiWWrNPMnhOO3x4XgKYfKb8iQn7Cp4yVMPxFH7L8bnUnGRhITdGqNDknWV3cK3W3Bi0oTJmcpuMYxHlB7UnSxrTGEFVHYOX+m0yqK0hG4VPkMcTSng9jG2EYE28kzHTfeLiuB5CvAggYkSGpVk8AnBcG1Wx8gHka96znYL5knuZKrKkyzsR21y8sn7cTzJs3iAioFsrSQ53E6TvLYJpJSzWwvbwk7ti1xMJLzHnSsamTBpNnbXkzzYBGs2E7GEhGGQZQpvPuto/GLNRtywOhUXSUjYXTF4X9o5AAAgAElEQVQ6JBmUNaCyW/ikoM0jTEryUAFjTInzSSkngttn+jjx/T/A337fyZzHyhRH7BauJXkoJbltSTnWhH/cTFGiKBFcU35+26b7g6W8q9XoGKHz8/0OSiBtCrgISe4NZrdo669kxKybZ+gVWjyGvvFtDJIneVbppoUJlZhCjV9LoRmyzLRf57Ad2138WE54yfLC+3OUZDMBlHr12tVo8SRHh+RsTxw5YUfid55zGJYu6uLCq+5f6KEkJCTMEqNDkjWRD/sZ4IF0zcdPzsTVxhhMCrg8CwKa+IN682SlUvPqdXyMmQof2KXg1xzGS9m+/B4jqu7YYnYLrgye//kV+JWP/KR1THwCIRE4brfg+7ehbCBHRamby1I7qw7N/RC57xhPcilf5/q7MS8gju6XLLOe9flQkv3fRtpP6mZYUlm14xLrJteNZvdSrwivs9ZVZhg6ByAWuOdPClieZPaHTe02eZJjNqOEhPnG4rEOfu/UI/D9W9dizfrtCz2chISEWWCESLKuslsoz6JgiGgzS95GdouhPKoVQ+g6nmQ7HrMfLdN7Y6A9qrHpYFteG1ftErdLlpvge0dj28PvrcoHVIF7hSHJ9iCeoWLlA4MtJ7oBgmLvbtDlkEqytD9PYddUpS223enHkNzablFG8iTXr+TrJsz06X7J5r3injR2KbUfhwl4bJ1Q8b7c75rsSZq98rzavGqfn91CujF9Iq5h//ZEu0XDRXUIPzT+8ju34we3peCqhB2D8593JDKl8MWrVy30UBISEmaBESLJFBBVPTQLj+y1pZuYzRIskbM8U4HqJWbY8ORsM7SokuyOTVL0YhimmIh7nPs606+IoJ+LtzeL1EactPHAP66UD6Ps2v100D6BF0ORbB5uf4Nd0zxit/AP972+PGWgXfGYu5KsI+20qfJ0/7Zmg/DyW/Dvmg7VjPj2+uFYpMA96Xj/XtWa50mu/P7v/ebN2DzVax2Tf02+ccOD+O+71sUPSEiYAw5ZugiveOYh+Mr1a7BlKhUXSUjY3TA6JBkwZakBd6kXGCxvMdBOzDgZ6QsV97ilwOxX2v3cMVuSLQbueQUnyiEIpBgI5WyPHRkqyaWuyGCb6joocQdcdZArjr7SNwistzbcNsWLoYjZLVh/Ld0F2S2KUjwmZhOwSrIN3JuNkhxW3NPiBKBNSS4arhuH81t5k6hBOb6UAk4O3GN9we7njEfzPMkKn/vpfbhkxRr86I7HADQHMfpBvUWhzXVISNgRuOCso7F1uo+vXL9moYeSkJAwJEaHJGttUsABPLuDVZwGQZua6GZ4sMqgIQ1CO2ZpOKIkSyngtNYsTzKc10HG6ZOZ2PbY93TdpnsFylIHSrKUJ7nV0xtRknnfs/EkN2W3mOm3KMns/cBKcu1J7peR7BaQFVpzv+QZq843PEEL75WIH7ul7d6gdgshcM+/T+Tj7KuUi1trbSwpvu/eeS+cr81uAUz23LLaTefjtI+qHb+4TULCfOKZhy3DqUfthy+k4iIJCbsdRockw80a4AclDa4kNz8w+QPVpIDLVfCQ583M1EvNvieZoKTsFoAQuNesDErnEQ/Qa1aYaTMpyU2Be4S2ZXvOiyV10s/3O6gn2Vprwm08YKytLHXrb18wJTnLooF7MXvtjJlUsRUPbyxf/Nn9rcuyUnENMbuFcI056PdqTx8Xvh+kah2/Ds7kh407N55k2ibt5/bCK+5lSjmWGqDNk8wnYVU7s835nZAwKN581tF4aOMkfnj7ows9lISEhCEwOiRZu57k0nuYt+WC5e00oe+oodX7bp6xh29IPIhQ+inBuMotBWP5xURElS2C2aa58r2mM/0qBZzWsoXEH3PzmNqV5GHOkdCkJPN+5DRpbHwtXKkwv1etJBdaHGOM3E3XqvZYJwtWPADgmnufwP/5r9vx/v+8rXEcwYRKy8VEJPWWoy11nm3fvvcnX42qrXnVzqBL9nuRBakwYwnbkaxIPE/ydN/94Zoma769pl+WSUlO2OE4+4SDcMR+i1M6uISE3QyjQ5KhkSkVBETR86+JIruFDtrUxJK9t8qiT0qlIDdfzaZdeOAf30aEX/IktxFI6Ri3/WYl2XiSi1IkRLMhyZy08fe81PGslOQGssdtIbLdYvD+iERXdosMvbK5LLVP1sj60clkTzLl6t44ObyS3Jbd4qY1G3HzgxvFcbb68BvtFu6+L/v4lTj3c9cFG6U0blq3eJIjanU1YbN/hzMeSR40cK+obUTJk5ywo5FnCm868yj8/IENuDEVF0lI2G0wMiS51ABUaLegB22TkLxtuu+20wBXSQ5TwEk2B1pmzyJXO1Nh4F6prT2D+MAwnmRTnCG2PbLBt3bM1CngAHcyIXnr2u0WLkGxfdq+3XNsbM6OxWRpiG/z+/f7rvprmSCVdkWgm6l44F4kVR9PAacEJZneta15hH3KijZv+6+/dwf+7vu/FNtrsidU7fC+3cmnf83uXLsFV93zeD0qO17/OtNnyiU9qPpbtcvzJLtp/qQxOceye4TaSEpyws7Aa045HPtMdJKanJCwG2FkSDJ0nd0iYrcwqpnwQNzqkOQ2JZkrxKQkZ4YR0FZO2Mx+kTzJXIk227RmSrL9btBxti2H09c0JBtIRu3bfek9P3cxcG9AuwIQJ0WzqbjHl++DPtmgJCV5GFIeBO7FFMgIgbSBe9aT7FZhbp/QAeE9XGr53Pg1nilKM1kL9hvwXnK+Y+p//Dj76tsc6NpQ4J5kt7AOJl+FtuqvZLcotcbF163G11aG2QT4b2KLzCRPcsKOx97jVXGRS29di4c2Ti70cBISEgbAyJBkDTe7BT3YiReYzwIJ2zZd8IYawT2uppiIkALOsVv027JbyL5Lmyc5JIHthM7tI9zuEvdOg7WjNEotOyeBWAyTGcRRkulVu4RsQI7c6K11A/fiZA8YZPzVa67IbhEJ3KP9tcbZJxyE804/EgC3W8ieZPuumSX7Pbop0dzvzTFavj60bdD+/EnUIBMZjfA60ye/mIhr7XBVa348/eYKED3JX/v5GnzzBrfCJe8HsBO9pCQn7Cyc/7yjAAAXpeIiCQm7BUaHJGsvu4UhyZTmCvXnkGhStb3qu8GVZF5MxFeuJU9yqCTXalimAubDA/eMSi2ojjG0ZbegbzNDkmuyIlg7aDlbyuzBMUyWhBhhdTJgDKoka5lM+f00+YcH6c/Jk5w1BO5pm1f6hEOW4NjlewGw90GXKcm+wgoMoCQLEyqjQvP9vHnMIOq9BLeEuHszNh1qbSfuddKsz65RktnGlvFpMOUZVZpCd7zVxK5t5YB+j+iKQELCPOPQZYvw8hMPxiXXrXZWMBMSEnZNjA5JRkUuco98+CWLJVLEH5JDeZKpmAjPbkFkmREU60mOBO4JnmQNnqkjJN5t/NGSdnk75ZUmVtVpsHaQ0s6XpXtinuTZKcmcd/EW+qXGRy+7C09snW5sl34HqX/uo24rgNL223OSnCkFXf/nnYbTVqasBWi6sJ5ku5+j0wJo9yRLyiovrmHG612P2Pm1VtzjfzPeJKopMDQ2qePbOp4nWVKt/S60dv+uJSW50HKpbj6OnrFbJJKcsPNwwVlHY8t0X7QDJSQk7FoYHZKsq+wWJiCKHqLmYVp9L6XFalMbObjdggfueRzZU5JrctXoSfYJDUsBZ87R3d6EQTzJjCMjz+OEnIo1uBMEwW4xjCdZUCe5VxUAfnr34/iny+/G+1pSovk5sTn6rXmS7fu2ADb6vTu5AlR1vtIh/DwyZYkr5fPlnmSXSFavrdkKGxRSfqhP/tr86dHuhPex4ETCZK/w7lf3vVGSvYmg1K+UGYMmbFoLdgutUZSx1QqLnrFbJE9yws7DyUfsi+ccuS8+/7P70wQtIWEXx8iQ5FK7dgujEnt+1VhWBftdC1ESgteqintuP6Ldwi/7R0vkKgzcg25WkgcNMoudjk2Z53qSbdCZ3XdyhpRkdu6zsFuUZXjt3DG51397bYOJBZz5/TblApYmItUx/H3z+IlUdbMMCiHJJWjWVpYpQ1yN3SLLTKYT18pQQbVoyZKSbPOBx88ndn7tmVLCvw9ppYKfy5apvus154q7dgvxAG42Gj+I1IcG/7sGZvq+3UJXdgvhHnU9yXumkqyUypVSNyqlviNsG1dKfUUpdY9S6jql1FE7f4SjjwvOOhpr1k/islRcJCFhl8bIkGQAgFIs72r1lV9ogj/wjZI8BPmMVdzzfbH8Ad+L2S0AE2zok3OxmAg/tk1J9vZ7fOs0Nm23+XdLXfVNhMTPVysV/mjPbtE8plhxCz5ZcbzQtfI6ljffpuY3FvqnsY93sgEC9xq7MeplpQTTOYTnVGrrreYTkZl+iawueDMbT/KL/v4nuGTFatGTLE8A3O9ic41Wu4Xw3jbNJ4OcJPdsOkH4qf0sgR3reIF72k50Y7m+OcmW7RaI2i0cT3L9QbIOjTjeCeCOyLYLAGzQWh8H4GMA/nanjWoPwq+ecBAO23cRPp/SwSUk7NIYCZLMg5b8/LPWu1h9XwjkbCi7BWMa9JDt8GIiVHGPp9/qU+CeP26rfvvPc42wLPVQSrLn8TzlQz/Csz90mdu3spolBe6FJMiSCFdJnlt2C0mJ5goswMo4+xcu0m5T4N5YJxPHN4yFpVfY31spIqfseOE+yjN7T/aK0nhwlbcfYO8diST3ihL3P74N9z++TVCSLQHmKnRgt4gG7olf23EJLNlXlAE3X/HmKTcoybeVmN8lzwHYSZeGzVAj9l/33eRJLpuUZPYdZZ3Zk5RkpdRhAF4B4HORXV4J4KL6/dcBvEQNWq40YWB08gxvfN5RWLFqfVDkJyEhYdfBiJDk6lWxYiJ+2rImJXkY8in5cjuMzFJToifZqyZSpa1TYgq4Uod5kofJ/CCNw7WaaMeeYgm53W7HX3Xcaw3caxxS1P/LRUmXdNmUaU1oKktN44wqyQNYbW55cBO2Tvft751nNnCP2QNsukHuSbZVIGeK0nhwlXJ/WzoOCO0WazdN4YEntgOoMjlouON0ylJzu4U3j5HS+jWdt7Tdz4/Mt3GyuoWRZK39AEmbeYKUZPpMkzd674+7+mzzJJc6zG5B7cdKnxPMCsme5Un+OIA/AxA76UMBrAEArXUfwCYA+++coe1ZeO1zD8fe46m4SELCrozRIMn1q0LcbmFUY4EQ82XooTzJxu+amT7sg90eYz3J3rhrJRmSkqyt8mxUYcG/GkNTWjQ6XspuIR3Xl5TkWVTca1eStUjOuwMqydJvRxaJsajdgu8btl2WGr/5iatw3oXXmTF38kpJLjUpn2HwGfGuSq23dotuTQr9fN7OWLzTPf1vLsdLP/rfACoiGiqr9h5xAvd8u4Wg0kv7+eBbjRdZUO9nHJLcc2xCfhd0fcZyd1KroQNHdnC+0E6sge9ZL2oVWbodpcnrnqIkK6V+A8BjWuufz1N7b1NKrVRKrVy3bt18NLlHYZ+JLl773MPx3ZsfwSObUnGRhIRdEaNBko1qZ8kHPfh8ldFXU8Pvmvtyi4lop8+qSEKoavZaAs8kT7LWTN0Ftc+8wy0Pdskb7bZfZwOpP4d9hYRKmiC4fbaQ5BYl0/fWmuIbLSTZlKVuVJLzWdktqO0bV28077tZBkAZckrecU7WbbEWa5+Y7pdGFZeItZTr2MdMvxQq7llltilwj3/kxHLQIFDehlGS2X5cSd461XfyJLurNXa8lA6P/3bKm0AEw9Nu/vPQblH9Lm2WoD2wmMiZAM5RSq0C8O8AXqyU+rK3z0MADgcApVQHwFIAT0iNaa0/o7U+RWt9yvLly3fcqEcYb3zeUSi1xkVXP7DQQ0lISBAwEiSZnnGKpduynuTS+eySItTfuQ/wJrhlqcsqZy5Tr6UlYlv+1m2L1NxMqVBpq0msM06EwXwxtBUTKWsVm8ZOXlnJa0qQ0t9JfUbHFFGSOeHizZLHtdsauOcWjHHGXNjgvzZi3xb81mceaZrY0G/I99XsPQ/Sm+mXRhW3qQqrV+2QxGrj11auwRu/sMIZz3S/DEijBr+X2di985V88n7fEvhmfxLo2i2s7WHLVN87zm2PxkbKul0tYYF7wt9S9dnNkxyq1BUJl1wU0grPnlJMRGv9Xq31YVrrowC8DsCPtdbnert9G8D59fvfqffZMy7QAuDw/RbjZScejIuvewDbUnGRhIRdDiNBkm3AkwpIpB/UJQXpzTa7Rb/wcjNr7ai+hF4/VJdpjAqyJ1lrq6By4upnoYghRjD4dq4kdxo8yeZ8vXP34S9b94sSX71+TaDqA7LqG1OS20lyfEJAYx7vZqL67pO3WNuAELhXN+CrwlqDKbvKC9yrPtgMDhVe/o8/xed+en91TP3du79+M37yS3cZe7oflsKmQDX/fJryJLcVWeGQrpH/Cgh2C3a8Pxmh32XMKyYChNdGslvQ/jN9ebJWlPJEzrHzGE/yns0BlVIfVEqdU3+8EMD+Sql7ALwLwHsWbmR7Bi446xhsnurjGzc8uNBDSUhI8NBZ6AHMB/hDlGK8iKQEeZIF5XC2xUSKUpvqazQOKUAwVtlLo5JzlVBxjyvJnLj63uEY2oo9lNr2DTSngDPn66jogurqcZKvrFyDv/jWrdgy3ccFZx3tTBx4W3YJX0Nr6xewJHnA7BaCcki//0Qnx5bpXrCdn2ZrXt2yWjlQdVo3yk9M185WjbPqZs5TwBVlbdUI1efV67eb37Ypl8B0vwhJo+YEPT4R4fckJ5dD2S1Mn+F9wm0Pm6f6diwaDtPWsNeqawL3bLv++UsTSPpdt8+4QXtVW7W9owwvpDR5LfaswD0AgNb6JwB+Ur9/H/t+CsBrFmZUeyaec+S+OOnwZfj8Vffj3NOODFKFJiQkLBxGQkkmONktPAJsyukK/mPJFxqDnwaNq7F86VdS7QLiqSkFXEhUNLhP2CrJfsaLGEyAVcOOPPOCryTLdgtGwAawW9C1uuexrcH2gZRkkz2kRUluIvakWHaywO5y/+Pb8DgreS3aLdgxmyZ7lsjW49VAkN2iKJndgnmSe30dKslsMkfXt+kRWQXu+aRRs2sgj93f5pLklruJE1xmKak+y21OzhRRJZl7lMPAPXv+klpN+zSp35T+re0e7SclOWEXwQVnHY1VT2zH5Xc+ttBDSUhIYBgpJZkXbvCD8mRrBe0D9l1zXz0veC1T4fKw385MVEm2xN5/oGtt/cd//b07ce196x3Vso3MW3+svF9pFDvyJA+gJJP6lyuTI9rZ7h2z11h1exERdewWEesD/5qKibRlaW3Kk0wTlLGOa7d4bPMUXvT3P3H2lY7n5/Tghklj/VBKgVL++RMznqO3suNYJXnxWO6ck2MDqt83paWd6ZfihEqyyQSTFmEC4h8jwa+Wx9t2lWSr6hYeKeY9aM3vpTAFXOZNDqVCO03ElgL3tHAZpcnrnpLdImHXxctPPBiHLluEC6+6D2efcNBCDychIaHGSCjJ9ODjeX+JAwTFRCQleQi7BSezvULXy+/hsa7dwiXstn9de5LlFHB82e3Hdz7mZLxoeq7zfM2x06ndFmbsvlorHcd9pFIKOJ/MbK/z1xqS7Kh4Qgdae6SrVuBbSExTnmTKQNLNldP/Ry+7q3X8vG2gIskdFnhHyrdvVSnY9edlqad7hQmQ5FYarStSTcS1TUkOrDmMlJcamOoV+JtL7wgCgfj5cdW3JfmKo0jzTCSAOzGkSU3VpnY2lt57v+Iev86+khz8/LrZIlHUgXtyXmz7nv4u95TAvYRdF508w/nPOxLX3rcetz60aaGHk5CQUGMkSDI94pSynmTfbmE8lEJBDinFVQxcSS5qj6ohPJFiH1TZSwzcq5fjJU9y7imKpZY9yd+68UF8+xcPi+cQDdyDduwWgSdZUopZzmE5cM/9vL0maUSSncmI6P91idxkTbKb8vjyUtaxYiKdrCr+wfu8d91WYfzSmDhJ3m4mEwrKWG39tIOl5koymCdZs2Iitn2/2yYlebpfiEoyH+e/Xbcan/7v+/CJK+6Jnt8wdgstvKemHOLNboB+qR27hV+0hfrseoF7VGCH9xUUT4FuJLY06WjLZmKyW+yBnuSEXQ+vfe4R2Hu8g7/41i3YmjJdJCTsEhgNkmyUZBUsffNUUUCkLLX3AG+CW1DDVswDgA9//w6zTUo1JdotQIF7CLb5ARyOksye6//jK7/AH19yI+u7nfSXRNDJbjGAJ5kmCGMdOZ2af37b6qCqdVumDXEhiMVEPMI4WR/fpHS6QZfh9n6dUSLPXCVZJsTh8fxaTvVKc51MRhIdph0sS0sCnYp7/UL0JPskrcleUpFbf0IlT0A2TfaC/Qg9J3CyhSSL95P2PlsleVE3r+wOTHV2J26IZregFY7YuKvza7ZIUOCenNLPvu+zqn1tqxUJCTsaSxd18bHXnoRbH96Mt160ElO9MCg1ISFh52I0SHL96palRv1qH4SAnMmibCFaHD1PLcuVzZN8yYo1Zpvk//TFL10TLKmYSFnqoEKftLQvwV3ajijJ2k1P5ts4JC8zXyJvS68FAJMzlRoy1Svx+NYZL6hMIMneeI2S3KD09YXf09+eZ1VqwDZC3Wa3ACDbLbxiIoW27WcZTwGnraeZjcPvo9Vu4efb9iYXlA3Ef8jya75281T0HH3wzUZJpr8vwZO8eCxHvywdJdjxNcOmrCO7Es8xTasrMCTbHd8Nqzdg5QMbGsfrK8mrn9iOX/vYlVi3xQZqOmnw2paQEhJ2As4+4SD8/WueiWvuewJ/dMmNoq0tISFh52E0SDJ7vuWe3cJXkt3Apup1mBRwfmlmSgkWjslVnIGQGFIJXkon5m5DYLfQ2v3u5gc34gs/u998Puo938Xfff/OQEmWyF/lh7aTCpOTuSF1nKk4l2diCjifaGxj6bk2T/W8zALSmFxCNqySLOdJLtHNM2SZavWei0qy13eXeYopL7a1W9i23ewWNnDPKtFcSfZIclMKuF4h2nb4tSff86SXHo0f97N7HmfWnXh/dQ+sLyKz2ttiPeSLxnIUpaceO75m+7v5E5jqGM9u4Y3vn39yb+NoKyW/vp/qdu96dAt++egWrHp8m9lvmFzRCQk7C7998mH4wDlPx2W3P4o/+/rNaZUjIWEBMRIkmZ6mPJOA9SRXD0JjrRCIEud7bf8eOQU1So0sk5W/gfIkawCKylIj2ObbLXwl+ZxP/Awf+K/bnX18AsELL/jtczKWZ5n5ntr3YTzJkcA9v5vtzFc33Svbi4lA9iQ3TVzcUtfC9rLycedKiZlNYm3FviNiCWXHa7IxCHaLnKUILErtkOxqHG7O6KrphuwWRVhxjxcTAayFYdJTkm0GCY2r7nkcZz35AHN8ExwlWbvf8W1Ekvca66AoS2fCxZXkUtuUdVm9EmOvs65tQM33YhOcuAEzWa7G1ivdlSD+fqpX4Gsr17TaTxISdjTOf95ReNfZx+ObNz6ED37n9nRPJiQsEEaCJJvsFkKe5GZPcqjwtv1jxAOeqhRw1gftjsm+j+ZJriF5kqXAPceT3DDM0iOOsQAmXi2w4xM9SZWtycd4J2P5p3lf7jG80MNM4WZlKIoS///ld2PtJrvszwO6AG63aCDJAiHiqAL3aiLGSLSkTlPf37n5Ybz3m7eIfXdyG7iHWvk2GVXMxMyWRFbe/WHsFmzFYygluV8KKxLu79XtyPcIfV71xHas2zKNs44bjCTzzcFKg2O3qE56Yix3Jp7+CkGl8FbvSUnm50Q+fUmtHgSSQkzXmAf8OX/LRYnL73gM7/76zSavd0LCQuKPXnwcLjjraHzx6lX42I/uXujhJCTskRgJkkyPPQVrR6CHsJ8ezM1uUe/TojByOJ7koqwrqoX7uUUx4sSzsjxUKu0H/us23PXoluqcIkqyn8/YR565hFtDDmDSpm9ljqN+q/bDtiW7RVMwnEOS+66SfOvDm/EPl93lBBz6/RKJeWD9drz9Syvx3ZsfCcbUFnRZBe5lyDM/oFFS16vv/vDiG3HJitXifl0TeGfbC7NbuNt4dj3jaTZ9Cp7kBpKsNTDtMfwqm4b9LE3agDCg75Cli8x4myDlSeal0gkz/RJjnQydrMr77ZBr9p4ryXmm6v3d/fgZSMvNpx61H/70V48Xx+tXxQQsOe4XspL8g9vWmiwsvgKfkLAQUErhf73iaXjNcw7DP11+Ny686v6FHlJCwh6HESkmQkpymLPYLzQhZbdwlNcWwuAH7mVZTEnWwTFinuRaaXxsyzS+8LNV+MLPVuGev3o5SuMZ5kvbVl2OKd7dXAXnE/P/8uvVzV31sSmIbayTYfNULzgnn4xvm+ljopthqlcGqcsoqIwTEq1l8n/dfU9gul/iB7c9ihc/9WVYVBfkCPsPDkWf2y243UNS1z11mfLtctBkQjkk2ctuwRTxPFOOfYJSyNkCNGEfRBGlSoxA5Uvm0Fr2wAfnV+9D154KmwxltwheuZJcYDzPqkwipXYItZMhA0BZX+zcqPzWFkWBe/b4cExZFk+VN9MPVxfob3CGXRv+t/w/v3ELnnX4suD7hISFhFIKf/OqZ2DLVB9/+Z3bsc9EB797yuELPayEhD0Go6UkC3aLME9ySOqkUtUx+HaLnBFNDtluEY7bVNxjD++f3v14vc0l4BqsGllknN08c/I1a60Dzyt9T95PQCiBLfCEvpAnuSm7xORMgX0XjwGoK8UJPm2ulmto8bx4/l1+/YH2oMt+UanvgwTu+WR161Q/INPdjAfuVd/ZstT2njPZLbz7o+ulgBM9yQrOPj6meu414GWp+Th8lDWZJpJMk422wCCpip+0MjPdLzHezYIJCeBaJjRTvnPlpuerglm9wD1hTDGbE+ApyV4hH64k+8Gn6+qMH9P9RJITdh108gz/+Hsn4flPPgDv+cbN+P6t4YpaQkLCjsFokGS2ROt7dpsq7lyqBxQAACAASURBVMllqZsJA1ei+kXpEHMO3g8VE+HffeQHd+Jfr3mgogPKDSha9cQ2Q2L982zLSDCWZ0F2C1FJhks0cuW221SWeiy3eZIdT7DHLbbN9LGMkWQnLV59TXJ2jpVXVVa9Cd+/7RF8+doHgjHxsXNUSnImBO6F+/rq+eapXjBZ8O0SgEt4q1cbLKmUq3jyFHK0b5AnmdqVfDwI7QC+57epZLPWlgRaJTm6u3g8P4YfOtMvMd7J0clJSdZmH/+eNMVWMirLbrdR/m5ppYfA80/78Fd7AJgy6vza+Irx+u0z5jwSEnYljHdyfOrc5+BZhy/DH19yE666+/GFHlJCwh6B0SDJIEKiAn+o70meazER/mAtNS0XC2MSPMmc0H3yinvZmF3l7eGNk5UnWYUEoS1P8ljHJcnxogo1CSfVkvL5Cl5TglNMRKhW5vezfbrAvou79TVwPcnUVu4oye2E7X9+4xb8r/+41XzmpCeWAs4UE2lTkutTmehWP+jmqV6Y3YJyDDLia/3c9l6j91VFRnu8CdxTNOa48usHblLWiiD/sTe5aCzZzJTkwe0W8koEfwUq8j3Wycz9bJRgHQadcjtKntnxa8BMHJvSEcYmp4BrtzABvML96mdoIYU+keSEXRF7jXfwxTeeimOW74W3fWklblgdzxWekJAwPxgNksyWvf0UcH3P6tBmt2gjaf4DNI96ku37tuwWmXKXfh/aOFl7ksOsF7bintxWJ3ePafQkg9kt6lRl3JPczd3z4p7knjcJkc5v20zf2C2me252C7JQcJU15kluQpvPmFLAZZnv1Rb2rb/be7yy6m+Z6gf7UVlpTnzpHKJ2C6Y7+yngpDzJ9JFPIP7k7OPx/539ZAASSXYnQlIOa74vVcab6A6mJPNLwKvo+dtm+gXG8jpwTzNPch08yv30Jk+yCtPzKeUmwYspybEAx76Q5s0P4APi12kmeZITdlEsXdzFv15wKpbvM443fn4F7ly7eaGHlJAw0hgtkgwVzdIgRePzYL5Oi0JL8JdoY2oWJ2xTDanMekUZtPHQhkmnSAWHyWccGV83z7zqZqHnlb7ndgsKQDSeZK1NkBmByMe4oyTLxLMoNaZ6JZbVSvK0pyTTZIOfY+VJnj1Jln67XlFauwXfV8ypTDaEiiRvnuwFkxFrt7ADN6sXZiIGx27Bz9EvJvLwpimsXr/d6YOuAT/uecftj+MP3AeA4ElG+2SBUJbAVF0Zb6KbV6sVLSzZCdxjxLfaZjdy/7cUPJiZtHduMRHuF+cTXpi+QmQNSrKz2kN2i8J99feLHZ+QsKvhwH0m8OULTsPisQ7Ou3CFUyAnISFhfjEaJBkhIbH5UV0VV0r35hbpkPuY6hX45BX3YNtM4SissYc1EZ1+URoPqETipvulo4gt6uZ4qLZbSFJZG5kfy7NA+ZNKSJe1lMyDxBTLpqBZXwTRk+yQM7sv+WadwD02Lpo4ZIGSLJ5WFK2eZB64pwfLA71XrSRvFgL3bHYK+52v7jt2C6U8T7Jblvri61bjT7/6C6cPbkUgjOU5xjqy3cLPL91rsFuUzG4x0c3rlIHNF93PTFGdq/sZqP62KKVbVY3QEl8377d2SLIfuOf3JU2cfE8y/1PhRNjep6HdIkaGk90iYVfH4fstxpffcir6RYlzL7zOyTefkJAwfxgNkmyU5Hh2C3o2SoVDytIW1ogpmdfdvx4f+cEvseL+JzDesSnI2uwW2xmhkRS+mb6rJB++3yI8vrUKIJKUZPKlxtTCQT3J0K4VIK9TanEFvuPZLXieZDG7BXtP1faMktwvnO1E1PxLNxe7hexJ1qZgBd+fH0fXmfreq/bqbmGeZFN0xQu8q95795xmFeUyL7tFFh6/hVUmrNqpXjlJ7nYUxjtyJT1/ciGtHJhtWhsleqKTiYVsfPDNpUdm/XSDVEGvX3K7BZy836W2k9VMeX5xzQP3tNnfh6r7IZCNBZCLifRMnmRX+ZaQSHLC7oDjDtwHF735VGzc3sN5F16HDdtmFnpICQkjh9EgyfWrUmGKtKDinkOqUH9nyW6MpE3WhTGmeqVR9IB4lD2RgO3T1XG8Sp0Pfvzh+y522vaxZFHHGbuPbp4Fy+OxinsKnpIMV2nt5O7tQePvdjKjyBWRwD0qJLKvkN2ikylD9FwlWbcSNh9tGSv6ZYlunhlyWwjEi86TrtNiUpIn+2Zytaj275KSzNXhoJiI40l2z9EoyQ2eWltBkivJGcbrMUieZH5vNWa3KKvj80yhk2dicGhwjA7/ZkwX7NBqsom6OIgbuMdXa7S2EyqawJhUjZDyJEtKsnt9xltIsilLzdMJRpTklAIuYXfBMw9bhs++4RQ8sH473viFFdjqTbgTEhLmhtEgyYxUWH+oRsnVLG2/J5hgPq1rMhMnn9N9S0zGGUnOM3c53bZdNbRtpvpHa5+JbpQAchK1315j5r3EofaZ6Drt++jmYT5gSTHT2u3XKOlmuzaqJ6FXVNepmyn0Ch0EnTkp3gobHNbJFK68ax3+6fK76zFmZtLhj2l4T3I8uwZQ2y2Y2m9WFNi+dJ4mLVl92punemY/CnLrCkpy7inJPHtDLE8ybZMgXYJunpn7zr+PtHc+3FKwmBVeAarfaLpfYqJuyy8J3TYe7X3n+9A56XVtP+514hX3uJKsdRWwqgDcsHoDNmybEf9u/Mlpl/1N9gt3TAArSz3AZCIF7iXsTjjj2P3xz69/Nm59eDPeetHKYBKdkJAwe4wISa5eFeQHMX0G5OwW9HBvWnrm//AQYQLiSjJ1TUrykokOilKjV5T4+QPrnX1V5GEv5cldUpPkmCrdDTzJkepy2s3DTEq6taBAUJIrawh9X3ptSxk9OrnCWCfDDas3GnV5rJOZJX8nuwVmY7eg85bJXr/UdQq4en/Bk9zxLCz0yu0WRkkWA/eE7BaGcMsp4Ko2ZEirHmOdzFnB8Pd3SbJ9f/CSiWDfqV5h7uGs4Z4nOJvJbmFsFxaF1p59wsrN3G6h4SrJ/goO3RI3rN6I8z5/nTimLHN/gzF2XWckJdkE7oXbfPT6w92DCQkLjZeecBD+4TXPwrX3P4E/vPjGFHyakDBPGC2SzIKkQgJXEw+BOFPAUdbgSeYZBcY9Iit6kktfSe6g0BqX3f4oXv0v1zj7+svqTSC7RYzYSMVEwrLHoSKYKTgp4Cq7RehJJhIEVITDzRgRKnjdXDnXq/ousyng+Jj04IF7l9/xKG5/eLNRTX2bCR9HJ8tEImvH426j61fZLUhJrs5BCtyzhUFgjrd2C+WcZWcAJZmO5Xl8x5iSHEC72Tq4J/kgnySXlSeZSLIawG7h3E/eq3uvaUcZ5qs4GjaDDF+ByJWnJNdtTdWWh1sfklNcqUBJth8c3zEF7BWh3SKGmSIpcQm7H37r5EPxwXOejh/d8Sj+7Os3t64QJSQktKOz0AOYD5jsFmBBWKX20pPZ781x7DtKhxYjDFxJduwWCqK31HiSa5K890QHWmtsmuwF+/KHve939kF2ixiZV0oFgVaSYkZ2i7625aEzL3Cvm4WeZKUsqex711guFhIqoPz68VOkfLqD4IKLVgIAvnTBqQBgMir4qFLAqSADBd+ViK+fsWPLdM+8pxLORPR8FR5wFWBjt8iAjPEyR0lu8SQ7RL6TYayMK8l8IsR/k+X7jHv7VingxmvSP1h2i3BsJoCPbStKjfEOI8l0PGxQnxkvKe11CripXokHN2w3+bt59gvp/vXLUo9FPMkmX7qnKDchBe4l7K4474yjsGmyh7//4V1YMtHB/znn6aIdMCEhYTC0KslKqQml1Aql1C+UUrcppT4g7DOulPqKUuoepdR1Sqmj2Lb31t//Uin1a/M7/ApWSXYJC1fU5DzJ9oFvSbLch6sk8+wWmUhmaUzbarvFPuNdFKUWH8D8eO5Zlf5tWzLRrCT7S+8aMjGgstcmk0KVUoCRH0FJLijFV00qfSWZE7WaqHQzFZBkUmWBUEke0m1hyE+V1cPd9up/uRoPbpg0FfcAO3nh4+74SnL9E22atJ5ka7dwi4Hw91yJdu0WLMCsZRIEsPSB3G6RZ0HeakKp3UkT9yRTdhFCoTWmewUmOtxu0UKSnbHJr1XbFemlYiL8fEhlpmOcinsKuOa+J3DW316B7TOFmzJP8jKBAvfsZze7Bb8PXXLci/3hMCSSnLA74x0vOg5vff7RuOiaB/Cxy+5a6OEkJOzWGERJngbwYq31VqVUF8BVSqlLtdbXsn0uALBBa32cUup1AP4WwGuVUicAeB2ApwN4EoAfKaWO11rP63omPfYUi3gvtW+tqF4lDy1lt2haep7igXuM5FWp08L9qZ3tzG5RavkB7JJkTqLCdpcsqj3JkXHyHL30WSwrXPfrlk9WznF0Tejwfu1JdpTkwu2LwJVAPqkAbLEOYO6e5Ec2VvlBKy+2e+zPH9hQj8FOZHguY0LX9yTX2yqSXO3jB+5xGLuFadtN4xYjc1Elub5FfEuIX6aa4PvO+fuli1ySTHYLuoeVUmhzIIh5kj1vMrWdq4ooF4WGYj97qd2y1NQn2S0I22f6zsSpm2ditommyYfjO9ZEjl3bRRNS4F7C7gylFP7815+GzZN9/NOP78GSRV285fnHLPSwEhJ2S7QqybrC1vpjt/7fZzKvBHBR/f7rAF6iKvbzSgD/rrWe1lrfD+AeAKfOy8jdMQJwg6TKUovEQSpNXGpdBwLFlcyY3aITUZKpP6MkT9h8wT6iJEoI7aLAvV5E7eIlkYGKkHBFkqd449ksKCUZz9CglEvMyG6RGyXZv8a2X1LsOnkW+KwXscBHhxRHCH0TblpTEeFuHk+x1xWUZN4PbfPtFhu3W7vFREMKOEmlttktmiZBMU+ymzmkm1de+1wg6LS/GzTZQJK1xnTfKsl51p5RxPuJvPbctnOmJHO7hYa9z3nxkyxzr8NUr3CWF/KIklyle7SfOy0p4KSy1DHMpMC9hN0cSin89auegV9/xsH40HfvwFevX7PQQ0pI2C0xUOCeUipXSt0E4DEAl2mt/ZDzQwGsAQCtdR/AJgD78+9rPFh/N6/gD2put5AIsVR8oiiramBZpqKEIW63iBcTue6+J/DD29cCqJRkICwEAQxntyBv7JRAtqt+dUA83TRp5mtnUpAreJ5k7RRnAWzgHtkTekXpLO2LdotcsFuwtGTO7wH3twTigYxU8OOWOrCrk1e2gS9f+wD+8OIbnH2pLDXvj/eT1TYdX2XeMtU3yr+f3YJzN2u3sNdBynUMuJ7zuCfZHR9dg5iSzMfMzxEAzjzugKDtKnDP2kaGC9xzrxGvkFdNooRiIroO6lPWbkFj7GSZY+vZPlM4U8O43UJF/27EFHDGbuFOLqX2k5KcMArIM4WPvfYkPP/JB+A937wZ37vlkYUeUkLCboeBSLLWutBanwTgMACnKqVOnO+BKKXeppRaqZRauW7duiGPtoTEBgf5kffVaxH5jgLXYpbF6VjgXhZLAafx2s9ci2vvW18Xgqirpc1IS8f2fZvSSESDk3aOsnQzHZRaJg2VkuzaLRTcgKzMU+sKypPM7Bax7BYme4HgSV7E7CrcfkJkioNbWziozS1TVSAkleO++cGNuO5+N8VevyxttTfByqBQKc1+4RkA2LB9ph6zlyeZtU+/H6/gaDy3DWSuyZPMJx+UFjBiSQ6CM+k87v3rX8eTli5y9i3KMAVcu92Cf3C/C5RkVZelLq3tp/Ik2wmshlXJfaXdn0T6vniCX0xkjE1cxRRwxm7h3V9CxpCZyAQ0IWF3w3gnx6fPew5OPmJfvPPfb8SVdw37bE1I2LMxVAo4rfVGAFcAeJm36SEAhwOAUqoDYCmAJ/j3NQ6rv5Pa/ozW+hSt9SnLly8fZlhOnmSbjssjcIzA2O9gvstrq8bwnmS5mAjve9FYbshtu5LcrDTSd7GE8UUQuCcr6tSWa7dQbIm8TvcmKckZeXhLMYMIYMlIN88CBZR7kh0rCDQ8oc+o9j6ZocPIr0pKsh/EBgCPbp5y8iT725WiKnFURdBue6Iu9UoKPllNeA5royTTyoS251GVaebn064kF96926Yka095pvPIFKAyf1+NqX7hpIBrtVuw923ZLaiYSL/UbsBffT9Vx1JGmdrS43iS/cA9+Z+oIP90RHH2lWTfkzzezYNjUuBewihh8VgHnz//uTh2+d54+5d+HuTpT0hIiGOQ7BbLlVLL6veLAJwN4E5vt28DOL9+/zsAfqyrJ++3Abyuzn5xNIAnA1gxX4MnWKLHiomU2lNUyVoRfmeD1GaR3ULJ5YVdj2hpiIBIbtnxbSngqJ1Y6VxeZbD6DI/IcqXYLrXbID1tjsvqpXMCLafTEnXP9yRzu0V98SuF1h0rL8bC/aNah5MUsgWEJLnajxR+ypMspbx7dPO0k4HC307ea6Mkl9r0u35rRZJpYiQryaGVg65FlTRkWE+y+5vRMTF/Ls+mAVS/CwWx+kcUWjt2i8FSwIWrL/bV3cY96ybdHqr7iWe3oNzkgEv+J327RURJ5qtGgFx4B7C/Sc97JYhKcrJbJIwYli7u4ksXnIaDlozjTV+4Hnc8IucfT0hIcDGIknwIgCuUUjcDuB6VJ/k7SqkPKqXOqfe5EMD+Sql7ALwLwHsAQGt9G4CvArgdwPcBvGO+M1tU/VSv3Kc4mN1C49r7nsCGbT1TnCDuSY4oybnsSebtbJ8pzD4SSY7le5U4FO073aAkk4ZHuYOljB7kObaeZOV8psA+Tsx6ZYlM2SCpfuHlouZL/qQkZxlmvCVuHrjn2C2AYJJCJGaRV16Z+iJCQ0VU/IBNoFKSrfqtgz4Uqt+R2iy0xv57VfmFjZLsBe7BIWjumLjVwLfjtGUvAap7h6cvpImTX0DD7I/qtxljWTqIePqrHGVZ3YPjTgo4eRx2PG5f/Dt+KBFfOkVzb+g6mwqPFyitspw5SnLf+XtoSgHHN8VUdhuMKWe3mBCU5FRxL2EUsXyfcXz5Ladh8VgH5124Avc/vm2hh5SQsMtjkOwWN2utT9ZaP1NrfaLW+oP19+/TWn+7fj+ltX6N1vo4rfWpWuv72PF/pbU+Vmv9FK31pTviJEyQFCx3KXR7WeqZQuN1n7kWK1atR541BzE5JJkRnU7Ek1xojcVjboAfELNb2Peu3SJsmPaN2S140BeRD+7DfOclN+K6+54wSrKxqij3/Gk7Jx/0nQncK0svKLA69p7HtmB9TS47uQoycSwas+foq9yUv5lAJMYnM77doptntRfYzZsNAO/+taewDBZl8BtnWfU7ciV5/73HAADrt01XYx4kcI/ZLQpmt+Dn4wbuxZVhGssBe4/jmYctNdskNVnrKhUf94rTb+/vXmqNaScFXDydoGm/psLVJNL9zg8YrFK6uSn1NChwD+YzWTMAl+CWGlg8Hv7d+MiUa3NqUtkBm/GjN4AneTopyQkjisP2XYwvv+VUlFrj3M9dh0c2TS70kBISdmmMWFlqOIowEeJurhwFlcDJGwVYDWS3YITNf1gTSg0cuswGTRFpmZwZIruFMA5qp8luQedolGR2zpff+Riuu399tQ9Tzv2cyJQWz1/GzpStxFeUbp5k8vu+9KNX4q++d4cZg18KmCvJjrJXq/9cPSQSM9HxSbIl84D9jfn5ZkrhHS86Fq969mFO6Wk61lonVB3AxpXkiiT7nmR+jL0mdvXC7yMIMHNS/MkoS0sw/+RXj8c/vu7koC9pfyLg/aI05Ng3XPRLjZmitCngVDyji2mfVhsyxchx9Z1r7anIeaAk1/tZu4V27RbePUbpEqtzkcfmB/zF7Ba+F9m/FzlJPuGQJTjhkCXJk5ww0jjuwH1w0ZtOxabJHs67cIURNBISEkKMBkm2rmQAMISnMGQxs55kwTcLuEvBHFO9Ake957u4nXm43DzJoZJMS/+Lx22Ams1KIeVJZiS5xZMcs2284Ywj8bRDllTlgBnxLTUC+wGVDFawy+WUyo4+UxVCfxm7IkHkSXZVWa0RlN3u5FlIklngHlf2yG7BSRMpyH6WC5/Xdeprrtn5am2X9KnN+9Ztw6W3VGn5bM5jCtyjpXmNZYvHkCnrSfbzJLtKcvXKS16b3N2ZS1MHSQHXL0vcsXazM25znpKSDI1eUZpViL5jt3D3pUkaz27hB0sG7ZtJqF154Pm2CaWmdHr+b6XN/QQAazdNYf22GXNuPsGldIlAfDLoW08i1uWgwmLf+1sg28nLnn4wvvfO5+PoA/ZK2S0SRh7POGwpLjz/FKxZvx3nf36FyRKUkJDgYjRIsre8ToowPfw7Gcv/K6TKAmolOUNQJmW7oPxyJZnnSR7LM6z68Ctw5P6L68DBqrG/fOXTjbrWarfI4iTq5ScezOwWLnl4xqFLcdi+i5zCEnmd99knBqWuTKKu3cKdJJQ6zDxQjZVV3PM8yUWp8cimKWf/Tq6CJe5F0cA9XSvJ9hrElGQ/GHCMBe5ZNdyeF7X5xatX4U++9gszNsB6km1Z6krlXLqoiy3TVcVES5JD8pl5144HB4b5fO25xQTc61dtwJu+cL3Tn+lLIMkU6McrB2Yxktyrzme8M7zdgqvO5mdnhxpPsvKPr/6ne+mff3Iv/vOmhw2R989xyQAk2b+urYF7hUuWCePdDFe++0X4+OtOAlBNYlLgXsKegNOO2R//cu6zcccjm/GWi1ZGLXwJCXsyRoIkE+gxmWVuVS9KDwa42S34Uq7NkywQSg/ck8xJMnE7aqfUGi992oE474yjmALcVpaaL+Pbfa5894vwydc/25A+v3Ifpd7imQ5oclB6xKBf2wGUsufnTxIqFTbMzesE7pVlUNVwrU+SMxWQDu5J9rNbaO2eN5HTsU7mED7fFtPNlfnNeZo3aosIMZUJB6z1Qdfnb/MkV8ctWzxmx1GTSjp3rg/zUugApeGD6T/2+7YFzAGhkix7kis7wTi3WxCZ9+wW0/X9RysWNJFqgmbn4nNjR0muyXmeh6o/ZZDhsL7puN0ipupWNhb+uZkk+5MqQidTOGL/xfY+y7MUuJewx+DFTz0I//C7z8KKVevxjn+7IVj1S0jY0zESJNk+xD27BZHFPGMKKfMks38QiMz4xMUnmIC79N/JbGBWh+XQpQA6pVwiIGe3sO9jgV0T3cyxOvhkO6+38ewNZJ8IlOTabsHtFXlWESquJIt2C2VTwPULbRQ6VfubAyU5E+wWjifZt1top8QwkZc8U47K7qNb/8ZFieAaVOMIrxsRZyJwPPAuzxT2ZnaZJXV550UsvzDBLwroll2OB+4NUoLbzxMsZXGg1QLHbhEJ3KMJS4cR1PbsFlYV9+0W/FCrJLud6jq9RUD4jRXG7W8fdt1jqi5Nav22fMRsFuY47/omJTlhT8MrTzoUf/nKE3H5nY/hT7/2C/GZl5Cwp2IkSLItAVx9NnYLCtDKLBGQKpMBNnDNJy7SUjTPk8xJECcmulYzM2+bZLdwPMmRwC6f9PhlqSmnMffDUg7cwlPRKLCMB+qRtYI4a2mUZJd8KGXJZb/UJiBqvJOh0BprvWjpTqbw9hcc63zneJJLV0n2FccJlp84lsEAqCZC3I9t7Q52O+BOUuhal2VFRintGimiXPU96fBl+OTrn40zjt3fXC+Cr2K6gXsuSeZE31/6lzCQkoxqstHtKNOuCdzzxkZBafx+ahsHz5ZC1gse4Gn2K7UziXLb0AGRjQfuMZIc9SQPFrjnFxPx0fW8Id08S4F7CXsczj39SLz7156C/7zpYbz/27e1ri4lJOwpGAmSTH/OliTD2B2Aym9qyBMP3HOUZOUoZQSJQPhKsh8cRko2D1aih7gcuGffx4qJ2ICxuJKcKcqTbL/j6cTMeZe6LvzAylLXJNnk+tUAEFOSq7H0CltxbyzPUJRwlGQi2f/vC4/Fqg+/AicfsQx7jeUNSnKlALvZLaySHCssUfWvnGIk9LtZT3J4/bvMbpFl1m5BimjHs9W84pmHiAptkItYW1tP7mU/4WRuMCW5nSSXWjt5kntFaT3J3r6k/OfsvmwvJmL79smxk92CVh+8MdYW+IDI8r8XDm63iPF3v+Ie/VQ+6eXBmFLfnVxQkhNJTtgD8QcvPBZvf8Ex+NK1D+AffnjXQg8nIWGXQKd9l10fRPTIf0kPfiIq3SzDtrqGCS/SwAPKiGQGnmThecmzW/Bl38whHm4AFZES3ufv/8qxeNsLjsFXV64x38XKUpsgs/o735NM1giuYhr7hRC4p7UOsltUvtzqhEkFbw3cq89nrJNDa421my1J9gnIt/7gTADALQ9uMt/5acJ4RgrAksROljnXxgdZaqg5Uqh9lX+KESBqW+vKqmDLUldj4Ona/MlCk90CcFc3YgL4IGKNdP19FPWkhwfu0XH+7mR9sfmew4lhME6WJ9lP/caPrfoNx0w2mtkoyTFUnuTQblHZe+zfBt1fvuUnVwoFdFDOmuwWVb7u+KQsIWHUoJTCe17+VGye6uETV9yDpYu6eOsLjlnoYSUkLChGSkkm2cwEsAmBe9yvyYN5bJ7k4ewWPAWc9XnaLAuWpIXj3ndxF/vtNeaQKK4cOp5LIhQUuCcpybXSR3kvSfnzlWSTAk6xCYaypBqwhUN8UlbZLWzgXr8soVSl4BWlxqOcJEfYIQ/c81FqtxQxJ1Kx9siPzS02NBmi8ROx5yohnYeu/bLGalKSkmz78/mSlCeZg0hZplQQPEeYL7sFnZPJk+xkt3D3p3HNxm5RtUUrDeRJtscWWju+eQPt5kkm0EdJSf7r334GTjx0SXRMvpJMKrW/2sD/7jnyyP40AfYzsiQk7AlQSuFDv/UMvOKZh+CvvncHPvXf9y70kBISFhQjoiRXr/S4U+RJLklRzcz7fr0sPVkWQXYLIFzeFe0WHb4MnxkiwpePiaTHlpT9/QkxYtbx9vVtG0TWH9o4iXf++00AKgV9IaRjsAAAIABJREFUUhdynmTtBm0RKTRp0HRVTERSMomoF2VFPshyUmo3ZVeM1EqlgAEb8Mb7NOSnhSSTn5zuBVKSrSc8JOZEnDV0nSe5VpLrMZAy66uWQHtmBU5GY0rybOwW0jWgvkj55v5kGit1Rb9Px7tXG2FWakL1m99atAoQy8ARs1tISvLrTzsCD6zfhlsf2gwJqvbgm7bMZMj9nf1iIn7fgd2i/jzZKxzrU0LCnoI8U/jY756ETCl8+NI7sWHbDN7z8qemlZWEPRIj8hRwVcNMVQSZVOBubpeU+4V2FDcCKcmcBKzbMh0UxwBc33DOltOJh+WqKtBAHk1AVgBtzl1ZlZTK7lq7hfvQ9xW85x61L048dGlUSS7JbuF5kvuMJPtEhM6RzrPUVeBeJ8uQZfTZ9hWzRywek+dm1bK8SwTNMnquAkLD96Hfzg/U8rNbcJjAvZrA9U3gnmsrkX67psA9t/94+elZ2S1Ekkz3eagk++Oz5N163NvtFjblWuhJZkpyqc195B6vnbLU/rjCPMmVJ3mswV6Tsb+76nzktujfAP9vgHbz7RbHH7wPAODnD6yP9p2QMOoY62T4+GtPwnmnH4lPX3kf3vONW4KJZkLCnoCRIMl2Obh6tVkdartFxu0WrDIZD9yryR9/6D/3r36EV//L1QCA5xy5L/7i158GwCV/eZ6ZBWdewY2UZBNMKJFkplSacXCSzPYlokWVxvzgIiKKhD940XEY67h5kwmF5oF7tn1edY6UZp/Y8H4oKLCTWz+0Y2GJ2S2iSnLoSabrlmdZNHCPVPSCWWyCwD3hWCJUVAqbpwnMFJiSHB7r/mbhmGxqPBVYNQgDKcneuKVUZ5SyjCZvRVk6+/EjZnwlOWsfh5kwqbAstfUoazPZCFLAaTsRkc7Nn0SQJ9lPf8fhl4Ontv2JGa0O8MnbWJ5F9z/9mP2w93gHP7zt0WjfCQl7AvJM4YOvfDr++CVPxldWrsE7Lr4hFRxJ2OMwEiTZ2i34ErL9vipLXb3vFxpj9cO5x8gj+W/Xb5/BbQ/bwDLC215wDJ5//AF43rH745ClE+b7SjW2gXK2f2tpoP18+BaK6j3Ye5ldZUo5qdOAmuQHAW/xPMkmBRydRxYqyZkKia5y7BaVctyp/dB8YgLEleTxyDK2RtVeVwiY62TxPMm8EIxZMSCSzM7Ph+mn9sv62S1ou6gkIyRoHP2ybLTaUD9t8G0iUl9EfB0lOeJtDz3Jqr3inq6VZPCAPc22syqPSiH3iL1G9dsGgXtsssWxd02SuWXER6UkyysOHDQP5pO3sU5mVWzvHh3v5PiVpyzHj+98LNp3QsKeAqUU3nX28Xj/b56AH9z2KN70hetTCeuEPQojQpJt8Bm9lk4xEasS9kpmtyhc1VMphRtXb8TvfebaoI9cKTz14CW4+K2nO57aDiNXtjgCz25h2/chKcl8v5gCKS2R+8vcnSxDpqpr43uSOYG0OZXrXMEsBZwUuJcpS9RIOe7kmSGpfu5pcfyZwiVvPR2vOvlQ53uyS3SdwD3bVqw98kRrzZbXC9eTLBHsjvEk26BFzSY3XZYBIjiHlslMr2C5isVRD2a3CFPAhfsYT3JNKvusb38AYZ7k6p7/5BX3RMdQ6mpSQF5/f+yVUmztJdKEsBQC98AmW4RF3dyQ/abiMbSiQqD3gd2iLOtJof2um9spjp8yDgCO3G+xCX5NSEgA3nTm0fj4a0/C9avW4/WfvQ5PbJ1e6CElJOwUjAZJrl+5akjliYGKjNJDvShLRpJ9T3L1fuu0LV1stjcEjRWeksztFk1qonmgR3zIsYAviTzz8thARQCrcYQleWlsPHCPAq5cT7IcuEdfaV0HidVL7GXpXtOmvMZnHLs/li8Zd77TqEguV/d44J5EaGifzCNcdB7GriEcaz3J2lhN+G/ZFSYxBCUQNI5+UQbVFmeDMLtF+CdLJNkqyaXTp2O3KNx7lZr/yA9+6bSptcYPb1uLux/dgk/9973G0mGKifB9wYq3ZGGAJU08fFsFTWTo2Fc9+1D87984wWyP/d7VuN37XbJPVBlXwpWUrmCR4hjv5OiX4eQyIWFPxm+dfCg++4ZTcPdjW/CaT12DhzZOth+UkLCbYzRIsseSrd2iDmhiftMeW8537BYZ99qGWS1iFb0kkpzVFoySEQOJZEuFKVwlOd6njyxzlTWrrtp80QTKblHZLWyAG8/wYDzJAkmm74paOc5rQl59tp01KYE0Rg4KMpTyE+cNgXs8DZ8fuOcXE+Ew2S20/R1pYlWlnIvbLdCSAq5fMnvB7DlyqCR7bWVKDtxzMoQIdgvJ6rONTQ7/46aH8LYv/RwXXLQSQFVxUCkYduxU2tNubm7/b0XD5t3mMJkn6nvm5MOX4fWnHWHPvSVwj192utY8x/JYnuG7tzyMB57Y5hzbzTNnlckHFQtKRUUSEly86KkH4ksXnIZ1W6fx6n++Gnc/umWhh5SQsEMxGiTZy25BhM1UPWMkuV+Ust3CW76Vig9IyDOFo5fvBQB4x4uOZf25xEB63kvL+XwMMW4lppMT7RZk+wjLUmu4tg0iN3Ta5EmWUp9x1bZXlOhmmanWxycXTWWkacwuqhLart2iVgjrPiRQSW7A/m5+CjhJMeRKMqnodKkypYwnVpyUcCVZ2F5VvQv3HRZteZLzTAV5krkXHnDJZOBJZu09zpZQH9pQqURr6wqKX37LaY6H3bdbmImikpRkmPuNo1e6SrJ/Hflkyb9VYkryElatb9tMgUc3T+PtX/65227HkmRJrSbPvF+wJyEhAXjuUfvhq28/A4XWeM2nr8FNazYu9JASEnYYRoMkm8C9CjkpqNoqbMTdCqZU9pw8yW4wlk+SY6Joniksmehi1YdfgZedeEi1r3ItDdV3kpLcnN0itkwftVtwkpyTKh6WpTZV+ZS9dr6SXKWIC7NbcCWZ/M6dXNXFOLRzTZuWy6sxuxeV7BuO3YIp8bH2yFoC2KV1Q7walGQizpWSnGH9thn86zWr6v6sEi79Dm22mMoXPLjdIpYD2if3oUdc2ewWgk0FiGW3CO89TpJ9W4ZNs2ftOARe6VApedLAM70QfNU/yAmdx/8W+GSNj3PJoi7a0M1tZcmY3QII0ywmJCRUeNohS/D13z8DSya6eP1nr8VVdz++0ENKSNghGC2SzEgJBWEBLECrJnE2TzJLAafcVF3+A7JJSfZBAU68LHWTJzmeJ1ns0uwz5pFJPpRuXittGkIKOAC12mhU+MzN8FCl7JLy9HIluZpoUEEVnhsYaF4ur7aHiqMfuEf7dJj9gUDnnys34wbA7BZm3GEqti67LzqZwvptM/ibS++sz0GhqQx2W+BevywNWRzEkhzL+OF7qYOUcJligXv8fpDHJwXuEdZtmQ7267BzUKjui3vXbcWWKde3zzO8cLJLzfvqNmB/I5rM+tu7uXw+tK9jU1KkJIc5uI8/cJ+gXfrTb1SSe4kkJyTEcOT+e+Hrv38GjthvMd70xRX43i2PLPSQEhLmHaNBkiFkt+BLwBmpqm6eZN8awB/S26fdpdZo4J7AgDJVkQYnBZykZLbYLWIKpLEgeLYEPpa8zm4hK8klKyZiz8PPk0y5cd1zY/7f2oPczSvFmYgVFQtptVsE3lXtTCz4NcjzUEnmxT5oP5r49I3dwh7je6T97Bb+2Lga78Pxw7JjaUzbpovWFHAc45Hc0b666reVZwo9kwKO/f58P/aWVGdz77H21221GR0MmWbXWKkqpeDvfuoap8gOD5L1/44U28cH/UZEsP0JQLdBSfatQMZuISjJfuBqN8+skixMhMaS3SIhYSAcuGQCX3nbGXjWYcvwjotvwMXXrV7oISUkzCtGgyT7dotMgac+I3JEhNHaLVwlmXOo7T1XKYsG7glKFHmgyddL7Uv7VdvC75wT8mB9zpwUu2Ps1MopzzxAIE+yUix1V6aQZ5lR98hPLeVJpvRbdI2JGE3X15NS5LXZLSTvalGruv716GRhECGRGSpLDQCFCQZzJ068LYLjSRYmA2Nmezh2bs3hhy5dNAagKg++j6BqxhBVkgfwJJPFZRi7RZOSfNejWwwJpt8iV8qkDFy/3U2PptlqRaYU3NLq9UTDm/wAPHCPCLZ7DbiSvGjMnUT4KwPUJfckX/rO5wMIV4XGmP1KWi2wnuSkJCcktGHp4i6+dMFpeOHxy/Hn37oFn7ziHiePekLC7ozRIMn1Kz00yW5RenYLKpsspYCj4wjbfCU5ogZKXtLKbmErkAEuuaG3UkW3tnLHfB+eO9gPZOrktsBGWEzELn/TFutJtp5THhBnr4PdnwfuZSyAbHFNaKR0ZU5bgZJc/Sb8OFtMJAtUP17swyrJrt3CvybS8VqHk52MKclSKjD+0/A+li6qiPHDGycNYRtESR6LkOQwu0X4mdThbsxuwT6YwD1jA7L7rdsyjV5R4lc/diW+ccOD1X7cMlKvFvjPP+5JpvvIH0epQ9tJz0sB558b/72X7+2mC/RXOawn2U5MnnbIEhy8ZCKoEsaLlIh2i27yJCckDINFYzk+84ZT8FsnPQkf+cEv8aHv3hHY/BISdkcMLnXtwrDFROyDv9QI1KIqMMzaLXgKOCq9S5icmYvdQtX922Alzu/2Gutgy3TfJSBCHzFqxYuW5EqhgA6KbXSqSEQxnR2Vb66UQdtmnnueZBWWGDYe6zpQr/IQV3moZ+rlaSLJ3WHtFvVYHZJFSnKugvaIWPIUcJYk13YLdt19wtk1kyeZjCpSkkWSLFsbli0eA7AN/VIbwtZyGQDEleRgkhJ4xDnhy8T9+BE9ISCP8PjW6YAYUlEaVSvJk0JZWg0wu4U06XNXVQjmt/JsUfZ87OcDl4zj7se22vNWXiaYus+lnt0izxSmeqHdgp+fj5TdIiFheHTzDB/93ZOwbPEYLrzqfmzc3sPfvvoZrbEpCQm7MkaEJFev9Mz0SyRbT3JttzCVyezDk1fHA4BtM57dYojAvby2MXBP8iFLF+HZRyzDYfsuxsbJHq68a10QuCcFJ0ngFo48U0AhKMmZDdybnCkw1smM0tsvw+IOWVYdQ+p7rJgI5ZClcty9QmOiWx1HiiYtjTcVE5GunYbNu2z3sfv6/9iSvSDLwhRwppiI59PmsAQptALkGZDXd5RUttklaPb9MkbSiLDF8l075xIhyb6P2r9mnNyPO0qyvDrR8zzJyiPJfm5g7WVokYLZdOnZLbJQ0RaLiXiZSMJJjG1nv71cJdm/3+m67DPhkuROrgKyy20pYp7kZLdISJgVskzh/b95AvbbawwfvewubJrs4ROvP9mpUpuQsDthNEiyCdyzD3Ne4KDDFMN+qY06xu0WfsDYwEqy8D31XzD1bK/xDr75B2cCAP74khurcXlpuKTgpMv/5FcCWwgvkGEUwcwrJpIrE7i3dbqPJRNdk+KrrDN/+JaBTCkzcSCC749pok6PRWo9BUL2ijKwW0gqHUdgVdFVUKG7XG/PNUaizIQAlnBJFgl/aZ3a04KSzFPdiYF7jifZvl+62JI0a7cIDg9AaccIR+2/GP/65tNELy616Zd6drNBhJYHQPIk222TM0WQ+rBXaidLh6wkuxNS/rMrNtHw5wp+xT3f8uIqvuHfhutJrj4smYgryWN5hpmidGwpcnaL2m6RslskJAwNpRT++CVPxr6Lu3jft2/DGz6/Ap87/5TgbzMhYXfASKyD+EpyJ1Po9bVRt3jgXq8ojYe1x6Le+6V2lqV9JTm2YhRNAVfKAWEAS2vmZbcIlGMFHLt8bzzlYDeFFZEQXmVPslsoKEOSlzKvpi0m4p5H6EkOz5uU5Kzet19oQ9aJgC3qDqoke3mSUf0OkkKYZ1nQHqmvnMzThILIntQWgchSqXVA0HhZ6lLgSlLQGFCdOymVvpJ89gkHhQ3Vx/h2i4lujiP2Xxzsyz3a/jnF7BbccGGyW2Rhhhc+0SH0i9L0qaBEkkzWIiCuJFPebf84APjfv3ECnv/kA3D60fs7251Ucl6f3JOslJ08THT91QKrJNM1HmuzW3ST3SIhYa4474yj8I+vOxk3PLABr/v0tU6KyYSE3QWjRZLrJ+nSRV1smuyBBFhT/KKsSBTlEOaqWVk2e5Jj1gfpIZtnvPRzeNxY7hIcTnQH6dPxJDtKMiPJtZKsAWyZ6jupsSiokZOWLPAka0dNJZDKRhk8+nUxkUwxkjxgCrgwu0U1sXGyW9Tn1M2VUbEJXTbZsJ5kr4qbZ0GR+i+1EBCXKeZZbgnc85Rv8iLza37te1+CT7z+5KAdAHjX2cfjTWce7Y4tVjiF/d6AO+54nmT73leSuaWgX+rAYtAr7GqIUgiC4ID6d2NZUvifBF3/amVCPCUcd+De+NIFoWruWFC8Y/nESMFO5vy/tzzLjJJM5FfKw+30WzeWylInJMwN5zzrSfjc+afg/se34TWfuhpr1m9f6CElJAyF0SDJ9Ss9NPddPIYN22eskszITlUhrgpqK7jdwgssCrJbRJ7wkqMgUypaIAGwD2ZSuBVTxDhiFNPkDmYk1i+r3anXo7UGtkz1nICmQhOB523CU5JdtY6uoeNJ1hr9ojQlsEmlXEwp4IbMbkF2mDxXOOnwZXj/b57gZAc5/3lH4fwzjjT7O0oyI7yADcp0LSiep9mUcdai3cLk026xW/jWBvLFcpJ88NKJwFJB+O1nH4qXPPVAvPzEg3HyEcvq820O5KPxuoF78pj472w8yQJJ7vVDJblXlE4fnCTz/OOmBLynJPObeABrtgO3kElot+A2pacdsgQnHroEBy+dCNqgMRP55Yq7mALOKMmJJCckzBUvfMqB+PJbTsOG7T38zqeuxi/XblnoISUkDIyRIMm+0rdscRcbJ3uG3BA5KrVGryzRrW0KPLuF70kO8iQPoSQr5u2VuLVZKo/YLairWG5mvj1nZMnJaKCUsT1M90vHD1aaPMmuFSHPsjqoz3qW/Xy6pOYqpVCU5PGur2c96Rg0cC9QkmGzW/zHO87Em848mtkLFA7fbzFe+JQDzf7ck+wrwXT9+TmGSjJLASeUf6b7Rkr56Sq17nWnqm9S9TcJnawi+f9y7nNw6lH7AXAtHBx0nkcesBhLJjrO5IdbNtzsFsxuESjJFYGc6GbolTb4ktCv82DTefIVFhqL60l21W1+GoOkwuOg33fZoq4Y1Monl089eAm+80fPx97j7jXPM2XI7rjJ3x33OgOpLHVCwnzjOUfu+3/bO/MwKarr/b+nl9mHYRvGYXPYF9kFBGQREEVRcddEoyQajcbdmK9mVZNfNO5J3GLcTeJuohITRYMYkaiogLKoGIiKC6gRFQRmub8/qm7Vrapb1dVr9fScz/PM0z3VtZyuru5669R7z8EDp06GEMDRv1+GV/77v6hDYphQlIRIdtstOleVYVdLG7bvNISuPBHue80SSxCpg9QA016g7A233cJP8Om8yvGYUqdXcxJOWplkvd1CvY2s36aSSVasF+5tdasus557M8nCYxlQ7QdtQmbrnJYDK5McM8R2S6tht1BFmSWSU9gt3PEK077hqJOseJIBZzbStq14/cqt2kyySyQrHfe87Z6Dm6E461kr74nIyiDrur/pcF/cAP6DHuU+mzG4Hqsu3h/ViigMM3DPXQJOiubaiqTWk6xePLoH7skQheJJJtO2444X8D+e/ZCCvUt1mfcui5JJDqoeoqv+UZZIkUnmEnAMk3OG7FaLh0+bgi5VSRx/64tY8taWqENimJSUhEiWhguZMetiVhj4ZJvRGUyeKP+33ewiFpeVHOwUYUur05O83V3dws8frBEzMSJtCTJJwuNJdorlVCd/tVZxTBHM7m11UUSy2mShpVV4PKKq/7ilrc0q/WVPM96PzCRbdgvTvqKKoSpr4F561S10pcA89gKNB9ddm1eN15ktdzcjMUWy8JaAU+0WOvw9yXaFBXfNXj90A9387D3u/aDO5hDJGuEN2MJTHs/y/9qKBFpahdaHq4pktWiIlUlWRLJ64aYu644jDE3dqnH8pL74wwnjNXYLSnkxCTj3oxS/6sVPoEjm6hYMk1P6dK3Cg9+bgqbu1Tj5rpfx2MoPog6JYQIpCZHszSQb4uRTs+SZt1Ob0dJWrW7R5hJK2911klOIFse8RNYAMt1itk1AjjaylzPehy1KdOhKwMViXiHftdpbsxeQHdKcA/cML6mZNW6zPcnWoEdzH8tMMpm+6xazZJu67zKtk+zOcsq41GnqdtT96F6Xzu7iV0JO20wkFiyS/e0W9gVJ2JJHzm6M5Jmmm5c0AlHNkAbt+hjZx7OaSd7V2oZdrd7sqTUozi1UrTsPzhJwuu6SRszO9V54wFD/IM11/fLQkRhQX+NZVhXJ7gucHx84DMft1ReA8xhMajzJumOUiFCWiLHdgmHyQH1tOe4/dRLG9u2Cs+97Dfcs2xh1SAzjS4nUSTZQ7RYA8Jkrkyyx7RZuT7I9T9hMsk48E+mbWUikzcKvBFzc5+RvbVOKasWLGyev3UJtwKAKNl0JuFjMFmDNisD3q5Mcj5Ftt4jFHKKsyqxukcpu4bYUtLY5/bIAUFVubK+63K7PLLHaese8WXTd/vf1JEN4RKlqP9GjF4Ixogwyyd51+Q4UdYljR2bdz27h0vrqfpciuVNFAi0au4Vjm66QrAsowFHdQmcfccf0t7OmYo+eddr3qMMrkgGK6V/77vT+SozefeIYuOdjaylXmu8wDJNbOlUkcfd3JuKMP7+Knz66Gp9ta8ZZswemfbeJYfJNSWWS1eoWAPDptl2OGqqSpFkyrEUduCecWbntruoW6WSS40Se7LZKv/pq9KyrsG7rej3J5rLaLaqeZDsudzMRAOha5eNJbhOekneqtaK5pU2Z5lyn7UkmtFrNRJzd8CrLXJnyFO9D0qKxW4zt0xl3LJiAcX27AHCKLrUttTeTLO0W9jRvnWWZCfXGEjeziX6os7tFYe+uVah1DaoLQmeN8BPJ8v3ohKuv3cJ1JKnr3tli2y3aBDwtnB3bck2XF2htbcKyYRgXVkq80D9PdWx4cW5dPQ6CTqsJTTCpOu4BhkhmTzLD5I+KZBw3Hb8nDh/XC9c+/RYueXyNVZGKYYqFksgkW4OGzP9tu8UurVdXlixzrCNFdYtUmT3HNM3tc5WDRvXEQaN6euZxCyC/TLIqpBIxwzqiWiMktRUJQ8y2CWedZCHg/ilS7RbS9hAjbwbd2XHPyCTHY+Ro4lCZDJlJVgRKr86VVnkyt+icOdSuaKGuslwpAecW83Y2XM0kuxtNyJFnGpGcIpPsEGmuQXLHTuiDA0bsFiiy/dB5jZ2vOx/VbTtLwKmxOtehvi/LblFuHB9f7XQe94CdiXZnedTj3LJbkFF1IhknNLd6B4dKUtXQ9sSgySS7L4516Kwf6ufiL5LjbLdgmDyTjMdw1ZGj0aWqDLc9vwGfb9+FK48aHWh1Y5hCUhIi2Wu3ME74W79uRlki5jnBSmGp0urKrLozyX52C52QcgiDEFpAzm+d0Mk53Y0tlIwLAD97RixG6FKVxCdf7fLYLSDct+TJujXdrJRPc2fQrY57ZNotzIF7ag1gqy11ih86dfs15Qkr+xskTnV1gd2eaMCugR1ot7CqW6Q/cC/m8xnL5brXlHsXCoFO/KpYrddddx3c+0Dnc7ZeU4ShOnAP8HrxAfvYd4ckRWqbsLtbyrhqK5L4bNsuXwGb6gLKjc6TLNfRv77adzln1z7jeVi7BYtkhsk/sRjhJ/OGoWt1Ga588k1s/boZNx63p6e5EMNEQUlcrg3qUYPT9xlgeZHLE3FLqOl8tYk4eUSIp5mIZ+CeftvudsaAU1CHyZi5M8d+HlD3/HKQlC2YvAt0NStcdHK1pW5zlYADbEGxS2npXF2WcGTeKpK2J1lmIZMxctToratMoktVEr26VAa+b7n8waN7mgMp5eCvcOJU9ST7WTfU9+hpS60M3Gtx9Z6Okb7ygcQ96NGanp728yA/S7/VuNcv43B3XHQMJnStQ413p1nSTTZA+Wqn12LgF5P0kAthN1yR+1jWK/aLKd1MsnfQIFBdnsAdCybgjm9P9F3OcbFgfpypOu4BRrZ5p6a7IMMwuYeI8P2ZA/H/DhuBZ9/agm/d9iK2ft0cdVgMUxoieVhjJ/xw7lBH9q6zaS8wbv8650/EndnleSMbccURo1zVLUKWgNNM9xu45IfXbuGc7rdNKZDt/73zSn92bYqBe2rczUpliGMn9sGDp0625lEtDlY5sTh5Mskv/mhfHDyqMfB9T+zXFT+ZNwyXHz4SgF2RIqzNIbCZSJvMhsMzv/v/NuEtfaa2pdbHoY8p24EnbsuN53WNWASM90LK2/PruAc4j095F6ZGZpJ1dgvr+HTt41a7hbnqSQbgaerhjiNV5ZOgZdVYZg7tYV0I6nAOijSep6qTDBiNR9yNVRiGyS/H7bU7rv/GOKx8/3Mc8/tl2PzFjqhDYjo4JSGSdciMp67yQdI17dpjxqCpe7XjROwRyT7CTTc96Fa3DmvgXshMsmrPiJO+PJqka3UZasoTjpisEnCaDDugNEIhQm1FEqP7dLbmsferPegrGYs5PMnxmDHoLZVgjMcIJ0/rj+ryhNnBz78Bi0RfJ9lrC2nVVLdwf1ZSQAnhHMQpYwiyi/hZatJMkHqwa2D7ve6KwxTNibhTPgddqKnC8YFTJ+OMmQNRY1YPcd9BARS7hWu63GcCsO0W5rxSdPt0qM4gk+wkbPc+3V2JVG2pAdNuwXWSGabgzBvViNsXTMC7n23HkTcvw7ufbo86JKYDU7Ii2boVrxm45741LU/YanvrVkU0yYFxOnQne11JryDIJY6D7BPq9Lg5WM+deVYXG9bYCQN71DjibG0zmokQgEdOn4Lz5wx2vJddrd4srERmkuNkt/uNu+wWmQy6INgZ7EBPsvJSuSKS/WouB3msZulPAAAgAElEQVSS1ThbXFnDOJGjCoI3Xr34Trf1shvblh782be57CRuT7Ij0+1ah2oRGtGrDj/Yf4i1L7YF2S1cK2qx7BbOOskAUJvCbpFudQvPoMGQu9k5mNG+AyPxE+tc3YJhomPaoHr86eS98MWOZhxx8wtY/cHWqENiOiilK5IT9gnRfXJPxmPaTKDf3VU/q4Xfa6mqW7ixbrG7BmP5LWk3ECGHQNRllM+aPQh//f7eDlHRJozsKRFhXN8uOHP2IMd7kSXgdBcGaobe8iTHCeVJ226RbpbQ2JadmQxaXt9MRFMnWddMxKctNWCLakkqu0XMx9qQzluv0gxMCXsXQV7DqW2sHRd+GnFqD/Lzfu1l1lwO3NtnSL2yvHNbErnPVE+y3JYcCOgn3DM5RlTCZ5K9n03Q91nCA/cYJlrG9u2CB0+djDgRDr1hKa74xzrtwGKGySclK5KlEIgReTJCsi218bp98lczySqBt/9TZZLTGrjn/N+/woHxKDPJqmgG9OLa22yjzderajfi8K7H4Uk296tR3UIpq5WpSA5R3ULnMQ4euKdmDV2eZOX/5lb3wD3vOh1xKHvZz5+ciud+OBPPnD/DuV7XXQXvdg1kET9L+MbJV4Ra2WZXO3SVMvOCYNvOVtSWJ3DntyeiUhmkqW7LjXHR5RbJSfM19Y6Mf1Y/Fe79EXY3J5TYZSRhkthcAo5homdQQy0WnjUV88f0wo3PvoM51zyHp1Z/ZP3eMEy+KVmRXGYJAu8tZNWioAoGty/Vmj/NW+jOLl+p53fHkk51C2d7XufrumUkbcIbm8ysqtUt3FjVLdSBezGyphvrycRuQdo6yW6cmWSZRfXWNG7VVLdwZ4bl3QbA+9kbdyDCiXWdIA1D95pyDKivcUyzs7Z+25UXdOb/5vREzFmyThe73Ec6gSovKrftanF4vY2YjMdenau0MQkI6y6MVd3CzCRvUwYCqltN25Psmj1dT3IiFtP61P0Y2KMGQ3arTStGhmFyT/eaclx11Gg8+L3JqClP4JR7XsFJdy1nrzJTEEpWJEsBFCPy3KIx7BbejJ1fJjndE7qqxcIN3HNmji37ha8n2Xx02y0CxLXuPXg7sZl1klu8VgVJmZW9hTWwKRl3epKztVsEVT7QDdzT+c51nmTvwD075kNG93S8lkrnx3w+4+w9ycGZZDndnblNxGOIx8iycMQ1dhApjnWfTzIhPcktyn6F43FYo140trWpJeCMabK6xZc77O+ew/pSoIF78lhKxMnjmw7i3DmDccM3x6UVY3uAiCqI6CUiWklEq4noEs08C4hoCxGtMP9OjiJWhlGZ0NQVC8+aip/MG4YX//Mp5ly7BL995m0eO8DkldIVycrAPXftV7WZiKPqg08mOV3NF3dk9FLPTy4xIv/3W1RtSy0tF0DwgD/de/Bkks0Jza1eq4K1jCK0dlkD7ZzNRDKyW0AtPRckku3n5QFtqVvbNJ5k1zzqwLwRveqw8fJ5djwpPzh99jhLq22ITLLxKDyeZDODa4pTXe3mZIDdImlO27az1RbJLlE9rLGTNiYB4alu0cnMJKuWBQq4YEmFN5Mcbjm5nUQsPZFcwuwEMEsIMRrAGABziWiSZr77hRBjzL9bCxsiw+hJxmM4eVp/PHP+Pth3eAOuWfQW5l73Lzz31paoQ2NKlJIXyaq1QqIOclLFRKuf3SLtTHJ6mUV3LFa20Gd+uy11zOHHDfKOakWfa5rtSQ4jVu2Be4kYWZ34gPRr4MpY5P4Pqnygq5Ns7APnfC2aW+tuG0hQnOpnuEdPrzjMVybZupvgc4kkp0pPsu03dopk0ohkaYEIzCTvarGtSq47HL4iWdh3Ydwl4FTUzaZbT9o9f9jlLYtJPGbFmK59qpQQBl+Z/ybNPzZ4Mu2K3eoqcMM3x+Gek4xGQifc/hK+/6dX8eHWryOOjCk1Slgk24Lx1OkDrDJngBy4ZzxXB9blym6ha2AQhPsWuzyJ+w4ktLLgzkYaQZ5kHe65rI57AXYLNWaZJUzEY6hwZJIzKwEnB+4Fe5Lt50HVLZo1Zez82lLrkDG88pN98dD3pnjj9RHG2TYTcfvLPa/H5LFh/C8/q927Gq2Zq8qdg+3UdcqKE3pPsswkt3gzyebyfk07hIAnS1tTnvTOmMW+cS/JmeTMIaI4Ea0AsBnAIiHEi5rZjiCiVUT0EBH1KXCIDBOKaYPq8Y9zpuH8OYPx9NqPMfvqJbjluXc8A7EZJlNKWCTbJ/rKsrhV5sx4zRZVziYb+nWlmx101mFNPb8t2I1H+5Z6sGiPxwjDGjthj151jjjDhqurHw3o/by6GOzMrzOTnKknublNiu7gDLZEvVvgtVuk9iQng9pfm/N2qylHpaZUm7omR4bUd43hSOVHl5PlBdRJU/vhogOG4oojRwEAqpKy9Jo3plpTuGozyUqLbtuqBM/8T54zHXcsmOBYVjanAez9puu4l4029dZJTjOTHCM0dKoAAFSVeWPrSAghWoUQYwD0BjCRiEa4ZnkcQJMQYhSARQDu0q2HiE4houVEtHzLFr7dzURDecI4vz993gxM7t8Nv3piHeb99l94acNnUYfGlAAle7ZIum4ZqyRi+oF7ObNbOKodpF7WLYyshhE+ol2N/SxF/Lu9yWG3K5EZYF0W1o3D6+sauJcJBNtnG7a6haPjnivYFm11C2eMQfsp1S15h8XCkbUNXCwlttdY/7plwzDnG92ns6MjohT0jvfmySQHd6ErS7jtFvZ8Q3ar9Xw+AnaNcbnM+KYuOGrP3qitSOL2pRucsWdA1tUt4jFcfsQozBnegOEa+0xHRAjxOREtBjAXwBvK9E+V2W4FcIXP8rcAuAUAxo8fz5YNJlL6dK3CbQsmYNGaj3HxY6tx9O+X4fBxvfCjA4ehe0151OEx7ZTSzyRrTqZxxcOq6iZ/e0PmIjlcMxFnVtsWycHxuMVKqsoI3vW4/jf3hW4A3e7dnOW/3N3T1IF7mRC2hq761uoqE6gqi6NHbYW3TnKI6hZBpHKMOAfr6QVzJtheY/165Or9FEl1uV2iz71Mp8qgTLI9rdxn4J5EbUEOAKfesxw7mlsd20rGY7jyqNHoV1/tiSMT3IuG/UranmRCXWUSh4/rnXkQJQAR1RNRZ/N5JYA5ANa55mlU/j0EwNrCRcgw2TFneAOePm8Gvj9zAB5f+QFmXfUs7lm20TcJxjBBlGwmWTZH0ImWZNwWdXkZuJdmZtGvBFybj63Kz3tsrSdknG4hJjOMVp1kRQs9fd4Mh2hX32MiTh7hlC5O+0JAJlnZbk15Ei9cOAudKpLY8tVOx3y6hii7dapAbXkCX+50lgQcqqmHmyqTrL7srCSRnUiWh6CvJ1leQPkcq5VJTac78zHIk+zIJLsuMN37olfnSvzowKGoTMbx00dX4+MvdmLT58aAGff3Tf0vm12TeSbZtluovPrTOdYA1Q5GI4C7iCgOI0nygBBiIRFdCmC5EOIxAGcR0SEAWgB8BmBBZNEyTAZUlsVxwf5DcdjY3vj5Y2/gp4+uxgPL38cvDx3huPPGMKlIKZLNQRt3A2iAkcC6RQjxG9c8FwA4TlnnMAD1QojPiGgjgC8BtAJoEUKMz134/tjdxXSv2aJOPam7RfLQ3Wqx7qMvsxTJqZe1vJ/m4wEjGrHuoy/RvVY/UEqKFndc8v+wQs09m+VJbjGzmYrE8VgVXJnf7DPJ9vNgT7LzeeeqMk88gN2WWt0Xh4zuiVnDemDUxU9Z0/5xzjQ01lV6tpPqM9dVj3DHlwnSh+5rt5AXUD5JEVknWZfprtVUvpBo7RY+xxMR4ZTpA7Dyvc+taTKT7BbU6r/ZXEC4rRphx4ZadZJdC/gNQix1hBCrAIzVTP+Z8vwiABcVMi6GyQcDe9TgjyfthcdXfYhfLlyDQ29cim9O7IsL9h9inTsYJogwp5oWAOcLIYYDmATg+0Q0XJ1BCHGlrKkJ48d1iRBCdc3PNF8viEAGUnmSyarG4By451Qeo3rLAXHpbTvzttTG45mzBmLlz/ZDj9oK7fzu2rj2epyPqXALD7k+mWEL9CQrG0nGY5awyhQ1liC7hV8lCb+21G4rRKcKZ9WFobt1Ql2ltxJDKkHnl/nOxncL2J7kVBdXwsdwIatb7GpV6xMbx0S1KZJ3aUZ+qxcm7mYifoNPR/Wuw2WHjwQA7GiWdx8CrDK+r6Qm20yyu9siwzAdAyLCIaN74pnzZ+DbU/rhvpffw6yrl+DB5e/53pFjGElKZSOE+FAI8ar5/EsY/rReAYt8A8C9uQkvc6TdQtueNx5ztFeWuDPJI3sbt2XSzSSnO5BLhmANyIsR6qo0JbRc63QLkqBmIkHblaRV3UK1G2iqS6SLw74QWCfZuV07HrdITl3GLohU78fP65xtJlleqPmtxu64p39dVrfYrjTQIRgDG2Xt4h27vB2qAu0WPm+KiNDQyRgQI7teuWdVLxqyqSHtrW4RbrmgLoMMw3QcaiuS+NnBw/H4GVPRr3s1LnhoFY7+/TKs/fCLqENjipi00n9E1ATjVp2uriaIqArGSOmHlckCwFNE9AoRnRKw7pyWE5InfZ2WUEuWOeskO+drMger6U7u35jYF3sP7KbddqZ1ksOUi3PM7y7hZmWk9cut/38H4Mbj7Fa77vlkNtGukxyQ0XVlkrPFYbcImUl2WC9cIbS2ei0j6ZCOJ9lpAclOjMljMOXAPR+VLAfubVeEsJFJJqss2w5NG1fHwL1ksN3CuZwxr6yZ7RajTruF72pS4l40rHVDrW7BMAwzvGcnPHjqZFxxxCj855NtOOh3z+MXC9fgK9dYFYYB0hi4R0Q1MMTvOUIIv0uvgwEsdVktpgohNhFRDwCLiGidEOI594K5LickO4ipt1MaOpXj4y92GnYLM5OccIhkY95L5++B+ppyy8eoy0LJ28w6dC2Bg0i7KoVPqTf3AEA3iXjM8X7dAlK+z+27jB+LoLJujlJsuRDJSixhS8A5K2I4Y2jWlIBLh3SqW/j5kzNBiOAsvtyW311CWQJOfoYyJiKg2qwP/HWKTHJX06tnVV0JIZLt6hYukaw8zy6T7Pw/3TrJbLdgGEYSixGOntAH++3RgCuefBO3L92Ahas+wE8PGo55IxuzHoDNlA6h1A0RJWEI5D8JIR4JmPVYuKwWQohN5uNmAH8BMDGzUNPD9tfaauKh703BFUeOcnSI09VJHtunCw4Y2eg7Mj4VmQ7cC1s+zB7o5yek/JdVvafu+aTQlFfU8kJCh7rtZCL7H5SwmWRdaTNAk0mWnuQ0P7tu1fqBgG78RH22meRUdZLtEnA+nuQyTSYZZIjkgEyyus/loDapm4N2oZVJbs53Jjkzu4XdeIczyQzDOOlcVYZfHTYSj5w2Bd1rynHGn1/Dt257Ce9s+Sr1wkyHIOWZg4xLqtsArBVCXBMwXx2AGQAeVaZVE1GtfA5gPyhF6/NJmSaT3KdrFY4eb3RYrUh6s8S2sILjtXSFlipEQ3khpUgOm0n2sWe0pchCul9zXy1LASpFcmVZ6o50QO7tFoGDv5RNBXmS7VrP6cXx8GlTcMkhe6R8T36iPutMMryl63Qb9q9uYXqSFZEcixmfuywBJwfZOVeriGSz8L5l3wnYiWWW3UKfSfbbRrpknUlmTzLDMD6M7dsFj50xFZfO3wMr3/8cc697Dlc9+ab2rhvTsQhjt9gbwLcAvE5EK8xpPwLQFwCEEDeb0w4D8JQQYpuybAOAv5gnxwSAPwsh/pGLwFMhRU6rj3dTZknVjJwUmWrbZyC1P9WNn2821fxh7wj7iXdbJKdeFvD6PONmANt2SruFfyZZ3UZQe+ewZFLdIqhRiK6ZSBiaulejqXt1yvnU9daUJ1AWj2FXa1vOPMl+67E9yfrlBzcYNZ+nDLD98nLgnpVJTvHDL7PpoewW5l0Eq7qFa1b1c81mz7iXDbub3d9lhmEYHfEY4YTJTThgRCMue2Itrl+8Hn9dsQkXH7wH9h3eEHV4TESkFMlCiOcR4vwmhLgTwJ2uaf8BMDrD2LJCimS/Ei8ykyzFFGBnkqUoSGSaSVZEY5jsWbqeZDmbW0ymElhAcCUGub5tZmWEQLuFmknOsd0i2JOsX4bIsBRI8diapSc5FepqiYC9B3bD4je3WIMeM8UqQ+gTtxSdfgP3+nWvxis/2ddRB9jyJJuD+nR2CxXbbpH6uJTH+o6WVsRIc7yrdwiIcOLk3fF1c/rZmYwzyeaFXy7udjAMU/rU15bjmmPG4OgJffDTv76Bk+9ejn2H9cDPD94DfbpWpV4BU1KU7JlDDtRJlUlWa8ZaItMlDtLNJOffk6wXL7Ic14K9m/xjC7JbSJG8S3qSww3cy7UAcQ/C89uu+/2r761Z1nrOsm5xmDgIhFlDewAAPvpiR1brTVUnOVUmGQC61ZS7BhOSo7pFc2vAwrAzyenYLXY0t2ovbtwXE5fMH4Erjkz/utlbAi7c52pXt+BMMsMw4ZnUvxueOHsafnTgULzwzqeYc+0SXP/Pty1rGdMxKNm21Jbdwi+TbFoJ1Exy95oybPhkm1XVQZ5Y892W2qqPHHIzdpMH5wK1FUlsvHxexrFZnuQdpic5ZCZZZqDv+s7EjIuzBzUGcc5nP/e05Y6RdaVji82MwkmNmsWOGSUBv25uxTET+ma1WpHCMrPv8AY0PvM2TprWL/Q6yVyf/Dy7BNTgBoAult3C+D/oGkjeRdjZ0pbyrkk2n4U3kxxuOa6TzDBMpiTjMZwyfQAOGtUTv1i4Blc99RYeXfEBLj9iJPbcvWvU4TEFoORFsp9ok7Vgm5VM8o3H7YnFb25G7y7O+sjp2i0yziSnWQIukxO/471oRGaMwlW3kKtJxskSRzMG16cdjxWK8jyTOsmAPuOfrUfYPw77OcEor3fK9AFZr9eqk+yTAe9eU45lF81Oa50xMvYDEeH6b47FqF6dA+e3ulWGsFuoJeB0+98pnDP/LNz7I+wgQHvgXsneNGMYJs/07FyJm47fE4vXbcaP//I6jrx5Gb41aXdcsP8Q1FYEJx2Y9k3JnjlS2S3koDRVJNfXllvVLwAlC5XmuT19kZyeGE/Xw6ySCMgkG6/HrMYQgSLZqj+bm0MobHULuxa0v11EN79KqkxqGNx2hlyRjww4kX0hc9ConujbLZyvLj2R3JbSbhFJJtn88rLdgmGYbJk5tAeeOm8GTpzchHv+/V/MueY5LFrzcdRhMXmkZEVymWW30L8u/bZBA60yHRnvEKIh9rAtesOt389uEW5Zp5fWjT3QKbjVtHtwY7bItaRaX1DWXbcoafb/kh/OxEs/Ti8b61lviu1mitWWOofCW9ot0l6OUh//aiMZ7f533LjIJpPsJG1PMtstGIbJATXlCVx8yB545LQpqKtM4rt3L8f3//QqNn+Z3XgUpjgpWZFsddxLMXAvaBBTmExa0HJhl7W8n2FP/BkOKHTHpltcDu6qCCj/BtjvqyygK186hBFk6ny62cJmkjtVJNGjtiKDKPXrzeXgwANHNjoec4EcuJeKymQcA3vUWP/L5GtwMxH1glBnt7Cf5zaTnJ7dgttSMwyTS8b27YLHz5yKH+w3GIvWfox9r16C+19+17fyENM+KVlPstVxzyeVbIvk3GeS0x+4Jx/DbccSihmojlR2i9qKBDZ/uRMVZSlEshQfOfJ6hs0kA7bH1k2q2/25xJkhzd16h+xWm3LwZboQhTsO37hkf8f/YZrpxGN26b1Ux0I2+8nT7jrkYWd1zWS7BcMwOaYsEcMZswbhgJGNuOiR1/F/D7+Ov7y2CZcdPgr9QtTbZ4qfkk2vWAP3fC7qrDrJAdUYMu2458zWhvckhxXjcv5MbiGr70WXAZWDEILKvwGK3SJH4sOupBBuf+ntFuGm5YJ8ieR8QAh3HMZjpD12U3XRk9+1Ks2FlaOZSA53VNqZZLZbMAyTJwbU1+C+707CZYePxOoPvsD+1z2HGxavD0zCMe2DkhXJ0gaQqgRcEJnaGtT5wyybqSc5k0yys06y9/VOlaZITmm3MNeXM/ER/ra4IZK903VCKH/NRPJjt8gHMaJQ3ng3YY9/2fJZVzLQcTGRfgjKepxLhz3s4jm+48EwDKMjFiN8Y2JfPHPeDMwe2gNXPvkmDv7d81jx3udRh8ZkQcmeOcK2pQ4inmmdZNWnmY9mIrHMxDuQOstdW2F6klPsH3fDlWyRqwnr4dZmjUN6knNBu8ok++yvVIS9kyL9/yktOhEM3EvwwD2GYQpIj04VuOn4PXHLt/bE/7bvwuE3LsWlj6/BNrO0KtO+KGGRbJwUU7WlDsLqOJbmyT3hEKKp5w9zW1vFtmekFRYA5y1xnW7oZIrkoEYixrbTy36nIj1PMmn3q37gXpaBBcSge16MZCqS7QuX4PnkBWlliu9UNrvJvWzYdcWtEnAl+1PHMEwRst8eu2HReTPwzb364valG7Dftc9h8Zubow6LSZOSPXPkJJMcy0yMOgRUSJUWj1HozHC6zUdUqsvtsZpau4XpSS4P6UnOld0iPU+yfr5CNhMhn+fFSCIWy+hzioe8Y1FmiWSd3SI3FxOZZ5KN+ZI8cI9hmALTqSKJXx46Eg9+bzIqkjF8+46XcfZ9r+HTr3ZGHRoTkpKtbiFFsl81lvIQpctyUic55KK/OmwEJjSFa3OZ7kA/FfV96wfuJUKtOx17RBhkLGFKyvkO3CukJ7kd2S1OndEfn361K+3lwja5kQK0UjtwT/887VhcMfDAPYZh2gsTmrriibOn4cbF7+DGZ9fjube24CfzhuPwcb1yOqCZyT0lnElOJfJSH5iZ2i3SrZMMAMdM6Iv+9TWpZ4TiB87gxO/sFOd9XVa3aAmoHw1kfgHhH5fxGCbjR6T//FK3Rc4d+eq4lw/26FmH6Rm0DA/b2dG2W3ivuZ0XE7nMJIdbrmt1GX4ybxjmjshd3WmGYZh0KU/Ece6cwfjbWdPQr3s1zn9wJU64/SW8++n2qENjAihhkZz9W4uZNWDTFYK5usXshwwn2+xY0MC9XSlK18RznKGzRXKITHJMX91CXjSE9dJmiyHW87uNKJEfRaqPxBLJZd4ZnSXgMo+l3FVtJZ2a4idP64/d6rJrHsMwDJMLBjfU4qHvTcEv5u+B1979HPtdtwS3PPeOb08HJlo6tEg+bGwvXH3U6MB5ErFw3cr8yIdQs+wWGcYVJCKlJzlVfcdsGppo15d2CThNJtlcNGlWNMj3gDq/OEqF0JnkhL8nWbe+TJg3qhF3LJiQ8fIMwzDFQixG+NbkJiw6bzqmDuyOXz2xDofeuBRvbNoadWiMi5IVyWGyv9ceMwZH7Nk7cJ4YUVaWgrxkkrOwWwBAdZmRLQ7yJKcSyZlaUXwxV1MWwm6RauCebHCSbwFLKP5Be9kQtsxf6DrJWeys6vIEZg7tkfkKGIZhiozGukr84YTxuOGb4/DR1p2Yf8NSXPb3tfh6V2vUoTEmJSuSAWBEr0644ohRWa3juL12xz5D0vdzSvJpt8hUvMsycLrQaqTdoiWV3cJ8zNnAPYMwdwDIpwRczOWTzneS168UXakQtoJJmzk6VlcnWV2Sx84xDMM4ISLMG9WIZ86bgSPH9cbvl/wH+1/3HJau/yTq0BiUuEheeOY0HD2hT1br+NnBwzFtUOYimfKwh8PeBvdDimSdaJHez+YUA/dsu0VGIfiuL5zdQv/epaiTQjvvVgifAYSlQtgmN7KrZapMcmnn3RmGYTKnriqJXx85Cvd+dxLiMcJxt76I8x9Yif9tS78yEZM7SlokFwO5yrSqZJ9JllUIvMv36FQOAJg/pmfgOnJeJ9l8DGe3CB64lyhQJrnj2C2C52sJEMmO9ZXyzmIYhskBkwd0w9/PnobT9xmAR1dswr7XLMGjKzZB+NWzZfIKi+Q8kx+7RXYD94IyyZ0qklhz6f44a9agwHXEQ/pVwyJXkwiRmo4RabObhc4kdxi7RYo3aWWSNXYL9TMo5aw7wzBMrqhIxvHDuUPx2BlT0btLJc6+bwW+c+fL2PT511GH1uFgkZxn8qELsmkmAgBVZtc9P9FSVZZIeYvdnbXNFvmekiGaifi1WZYD9mTDlLxnkn3iKBXCDtyTmWRdF8vKFG3QGYZhGD3De3bCI6fvjZ8eNBwvbvgMc65Zgtuf32AlJpj8wyI5z+SnuoXxmKlIrjaFS6rBeYExZGn5cGMP3MvCbmHu6/Jk3PF/voiRrj5I6RC2416QJ1mdVtp7i2EYJvfEY4STpvbDU+dOx8R+XXHpwjU4/KYXsO6jL6IOrUPAIjnP5LVOcpae5G27WjKOQd6Cz9ktdHM1yVB2C5+BezFnJjnvzURQ2haCsM1EWtqMiy2d3ULNLpfwrmIYhskrvbtU4Y4FE/CbY8fgvc+2Y95vn8cvFq7BFzuaow6tpGGRnGfykc3cZ3AP/GC/wejfvTqj5avLDeGyfWfmIlne7MlZxz1Iu0W4TLJOnMp9XZEsjCe51Dvuha2iIluYazPJZSySGYZhcgERYf6YXnjmvBk4enwf3L50A2ZdtQSPvPo+D+zLEyyS80yuOtKp1FUlccasQRmvWwqXbVkULJc+1Jx13JOZ5FB1kvXZTTlNlrHLvye5tA0EYUWyLBeo8yRXlbHdgmEYJpd0qS7DZYePxF9P3xu9ulTivAdW4ujfL8OaD9iCkWtYJHdAZMe9bLr6yKvWnFW3MB/DiGT/ttTOTHK+rRBU8nWSw9l6WgPsFmp2OVc1tRmGYRhgdJ/O+MtpU3D54SOxfvNXOOh3/8LFj63G1q/ZgpEr+LTVATlsbC/UlCdw2LheGa9DDtYKMc4uFHYmORd2C/8Sd7nEbwBhqSCvV7Kpk1zBA/cYhmHyRixGOHZiXyz+wT44bq/dcfeyjZh11bN4YPl7aOMqGFnDIrkD0qdrFd64ZH8MqK/JeB25tltIwtotdJv1DrX8cRYAABZJSURBVNzLcyYZJZ5JDlkCTloqyjTl+8qVabk4VIY01Ga/EoZhmBKjc1UZfnHoCDx2xlQ0da/GDx9ahSNufgFvbNoadWjtmkTqWRjGi7Rb5KqjoBTdWdktZAk405Oc73EMRJT/jURIWLvFn787CU+v+Rg15d6fE/UiIheHyqNn7I2dzZmXLmQYhillRvSqw4OnTsYjr23C5X9fi4Ovfx7fnNgXF+w/BJ2ryqIOr93BIpnJiFZTp+SqTnKrJZJD2C1ienEec3mSW/J8q8kIoXQzyfGQA/cG1NdgwIzUdyVykXWvSMa1AwQZhmEYg1iMcOSevTFneAOue/ot3L3sv3ji9Q/xw7lDccz4PnkpKFCqsN2CyYhWmUnO0ZdNlhEL25Zap7fiLk+yrN+bLwy7RV43ESmW3SLHDWMYhmGY/FNXmcTPD94DC8+cikE9anHRI6/jsBuXYsV7n0cdWruBRTKTEXJAQK5EcrOZmg7TltrPbmFnkk2R3JrfTHLpd9xzPma/vlLeWwzDMMXJsMZOuP/USbjumDH4YOsOHHbjUlz48Cp8tm1X1KEVPSySmYzoZzYyGdGzLifrk3aLslDVLfTlxOw6ydJukedMsk/nv1JBXgDlyndewruKYRimqCEiHDq2F/55/gycPLUfHnrlfcy86lnc8+//WudfxguLZCYjpg+ux1PnTsdR43vnZH3NbeHtFidOacJxe+3umS6XlSI53198P9tHqWA1E8lRKrmULygYhmHaA7UVSfx43nD8/expGN7YCT/96xs45Prn8cp//xd1aEUJi2QmYwY31OasBJpsSBHGbjF/TC8cOLLRM12KsHLTbtGcZ7sFUNo+27Ad9xiGYZj2xaCGWvz5u3vh+m+Oxadf7cIRN72AHzy4Ep98tTPq0IoKFslMUSAFbRi7hR/SblGo6gel3nFP7s8QVflCwSOqGYZhigciwkGjeuKZ82fgezMG4NEVmzDzqmdx59INaGnlUpsAi2SmSJBfyDB2Cz+sgXshstG5oMPYLXLcepxhGIYpHqrLE7jwgKH4xznTMaZPZ1z8+Boc9Lvn8dKGz6IOLXJYJDNFgVUnOQuBG3fZLfJNqQ/cy5VItqtklO6+YhiGae8MqK/B3d+ZiJuPH4cvd7Tg6N8vw7n3r8DmL3ZEHVpksEhmigJpt0hmcUve3ZY635R6JtmqbpGlTULaX0p5XzEMw5QCRIS5Ixrx9HkzcOasgfjbqg8x6+oluPVf/7FKtXYkWCTniZuOG4ezZw+KOox2Q0saA/f8GN27M6YPri+YSCaUtoXAaiaSpbq1RHLWETEMwzCFoLIsjvP3G4Knzp2OCU1d8Mu/rcW83/4Ly975NOrQCgqL5DxxwMhGnDtncNRhtBtarLbUmR+S+w5vwN3fmZjVOtKixO0W0r6SbSZZXrQ0cy1OhmGYdkVT92rcvmACbj1hPL5ubsU3/vBvnHnva/hoa8ewYLBIZooCuy119qKzrIB2i1JOj04b3B3nzRmMgT1qslpP365VAOwujQzDMEz7gYiw7/AGLDp3Bs7ZdxCeWv0RZl39LG5e8g52tZS2BYNFMlMUWB33ciBwywqUSS51u0WniiTOmj0o60zyTcfviauOGo0+plhmGIZh2h8VyTjO2Xcwnj5vBqYM6I7L/74Oc3/zHBau+qBk/coskpmiQH7BcmGVyMbXnA4xopK2W+SKrtVlOHLP3HRmZBiGYaKlT9cq3HrieNyxYAIggDP+/BqmX7EYNyxej8+27Yo6vJySiDoAhgFsT3JO7BaFyiSXttuCYRiGYXyZObQHpg+ux7NvbsadL2zElU++id888zYOHdMTJ05pwh4966IOMWtYJDNFgWwm0q7sFkQgttkyDMMwHZR4jDB7WANmD2vA2x9/ibuWbcTDr2zCA8vfx8R+XfHtKU2YM7wBiUINqM8xLJKZoiCXmeRkojD5XQLX/mUYhmEYABjUUItfHjoSF+w/FA8ufw93LduI0/70KnrWVeBbk5tw7IQ+6FJdFnWYadE+pT1TcrTkoOOepKB2C1bJDMMwDGNRV5nEydP649kfzMQfThiPfvXV+PU/1mHSZc/gwodXYe2HX0QdYmg4k8wUBVZ1ixwI3GyrMYQlRgQh2G/BMAzDMG7iMcKc4Q2YM7wBb370Je58YSP+8tr7uO/l9zCpf1csmNIPc4Y3FOycnQkpFQkR9SGixUS0hohWE9HZmnn2IaKtRLTC/PuZ8tpcInqTiNYT0YW5fgNMadCaQ7tFobK7RECM78UwDMMwTCBDdqvFZYePxL8vmo2LDhiK9z77Gt/74yuYfsVi3PLcO9i6vTnqELWEySS3ADhfCPEqEdUCeIWIFgkh1rjm+5cQ4iB1AhHFAdwAYA6A9wG8TESPaZZlOjiVyTi+bm4t6itKNzxwj2EYhmHC07mqDKfOGICTpvbD02s3484XNuBXT6zDtYvexmHjemHBlCYMbqiNOkyLlCJZCPEhgA/N518S0VoAvQCEEboTAawXQvwHAIjoPgDzQy7LdCAeO2NvLF3/Sbvy+PLAPYZhGIZJn0Q8hrkjdsPcEbth7Ydf4K4XNuLhV97Hn198F3sP7IYFU/ph1tAekSfO0vIkE1ETgLEAXtS8PJmIVgL4AMAPhBCrYYjp95R53gewV0aRMiXNoIZaDCqiq8cwGHWSWSUzDMMwTKYMa+yEy48Yhf+bOxT3vvwu7ln2X3z37uXo07USJ05uwlHj+6CuMhlJbKEdlURUA+BhAOcIIdxDE18FsLsQYjSA3wH4a7qBENEpRLSciJZv2bIl3cUZpuDEiCUywzAMw+SCLtVlOH2fgfjXD2fixuPGobFTJX75t7WY9Ktn8JO/vo71m78seEyhMslElIQhkP8khHjE/boqmoUQTxDRjUTUHcAmAH2UWXub0zwIIW4BcAsAjB8/np2eTNHDdguGYRiGyS2JeAwHjmzEgSMb8camrbjrhY14YPn7+OO/38W0Qd3x7b2bsM/gHogVwIoRproFAbgNwFohxDU+8+xmzgcimmiu91MALwMYRET9iKgMwLEAHstV8AwTJTEixFglMwzDMExeGNGrDlceNRrLLpyFC/Yfgrc//grfuXM5Zl79LG5/fgO+2JHfqhhhMsl7A/gWgNeJaIU57UcA+gKAEOJmAEcCOI2IWgB8DeBYYRSQbSGiMwA8CSAO4HbTq8wweeWFC2ehPAeNSQIhcHULhmEYhskz3WrK8f2ZA3HK9P54cvVHuGPpRly6cA2ufupNHLlnb5wwpQkD6mtyvt0w1S2eB4Ktl0KI6wFc7/PaEwCeyCg6hsmQnp0r874NtlswDMMwTOFIxmM4aFRPHDSqJ1a9/znufGEj7n3pPdy17L+YMbgelx0+Mqfnf+64xzAZcvi4XmhtizoKhmEYhul4jOrdGdccPQYXHTAM9770Lp54/UN0rS7L6TZYJDNMhhwzoW/UITAMwzBMh6a+thxnzR6EM2cNzHmvBW6qyzAMwzAMw7Rr8tGMjEUywzAMwzAMw7hgkcwwDMMwDMMwLlgkMwzDMAzDMIwLFskMwzAMwzAM44JFMsMwDMMwDMO4YJHMMAzTQSCiCiJ6iYhWEtFqIrpEM085Ed1PROuJ6EUiaip8pAzDMNHDIplhGKbjsBPALCHEaABjAMwlokmueU4C8D8hxEAA1wL4dYFjZBiGKQpYJDMMw3QQhMFX5r9J80+4ZpsP4C7z+UMAZlM+CpAyDMMUOSySGYZhOhBEFCeiFQA2A1gkhHjRNUsvAO8BgBCiBcBWAN006zmFiJYT0fItW7bkO2yGYZiCwyKZYRimAyGEaBVCjAHQG8BEIhqR4XpuEUKMF0KMr6+vz22QDMMwRQCLZIZhmA6IEOJzAIsBzHW9tAlAHwAgogSAOgCfFjY6hmGY6GGRzDAM00Egonoi6mw+rwQwB8A612yPATjRfH4kgH8KIdy+ZYZhmJInEXUADMMwTMFoBHAXEcVhJEkeEEIsJKJLASwXQjwG4DYA9xDRegCfATg2unAZhmGig0UywzBMB0EIsQrAWM30nynPdwA4qpBxMQzDFCNst2AYhmEYhmEYFyySGYZhGIZhGMYFi2SGYRiGYRiGcUHFOGiZiLYA+G+ai3UH8EkewsknHHNhaG8xt7d4AY5ZZXchRIcqHJzhbzZQPMdNscQBFE8sxRIHwLHoKJY4gOKJJdM4fH+zi1IkZwIRLRdCjI86jnTgmAtDe4u5vcULcMxMZhTLZ1AscQDFE0uxxAFwLMUcB1A8seQjDrZbMAzDMAzDMIwLFskMwzAMwzAM46KURPItUQeQARxzYWhvMbe3eAGOmcmMYvkMiiUOoHhiKZY4AI5FR7HEARRPLDmPo2Q8yQzDMAzDMAyTK0opk8wwDMMwDMMwOaEkRDIRzSWiN4loPRFdGHU8fhDRRiJ6nYhWENFyc1pXIlpERG+bj10ijvF2ItpMRG8o07QxksFvzf2+iojGFUm8FxPRJnM/ryCiA5XXLjLjfZOI9i90vGYMfYhoMRGtIaLVRHS2Ob2Y97NfzEW5r4mogoheIqKVZryXmNP7EdGLZlz3E1GZOb3c/H+9+XpTIePtaBTLb7bu9yOiOLTfr4hi0X53IownTkSvEdHCiOPwnL8jjKUzET1EROuIaC0RTY4ghiHK7/4KIvqCiM4pdBxKPOeax+sbRHQvEVXkZMVCiHb9ByAO4B0A/QGUAVgJYHjUcfnEuhFAd9e0KwBcaD6/EMCvI45xOoBxAN5IFSOAAwH8HQABmATgxSKJ92IAP9DMO9w8PsoB9DOPm3gEMTcCGGc+rwXwlhlbMe9nv5iLcl+b+6rGfJ4E8KK57x4AcKw5/WYAp5nPTwdws/n8WAD3F3ofd5S/YvrN1v1+RBSH9vsVUSza706E++Y8AH8GsDDiz8hz/o4wlrsAnGw+LwPQOeJ44gA+glFvOIrt9wKwAUCl+f8DABbkYt2lkEmeCGC9EOI/QohdAO4DMD/imNJhPowDHubjoRHGAiHEcwA+c032i3E+gLuFwb8BdCaixsJEauATrx/zAdwnhNgphNgAYD2M46egCCE+FEK8aj7/EsBaGF/yYt7PfjH7Eem+NvfVV+a/SfNPAJgF4CFzunsfy33/EIDZREQFCrejUTS/2Wn+fuQzjnS/X/mMxe+7U3CIqDeAeQBujWL7xQgR1cG4uLsNAIQQu4QQn0cbFWYDeEcIkUlDoVyRAFBJRAkAVQA+yMVKS0Ek9wLwnvL/+4joxyUEAsBTRPQKEZ1iTmsQQnxoPv8IQEM0oQXiF2Mx7/szTGvC7YqFpejiNW/rj4WRrWkX+9kVM1Ck+9q8TbsCwGYAi2BkLz8XQrRoYrLiNV/fCqBbIePtQER+bBQzmu9XFDE4vjtCiKhiuQ7ADwG0RbR9Fd35Owr6AdgC4A7ThnIrEVVHGA9g3H27N6qNCyE2AbgKwLsAPgSwVQjxVC7WXQoiuT0xVQgxDsABAL5PRNPVF4Vxn6Coy420hxgB3ARgAIAxML4wV0cbjh4iqgHwMIBzhBBfqK8V637WxFy0+1oI0SqEGAOgN4zs5dCIQ2KYQIJ+EwqJ+7tDRCMKHQMRHQRgsxDilUJv24fA83cBScCwCN0khBgLYBsMe14kmOM6DgHwYIQxdIFxN6ofgJ4Aqono+FysuxRE8iYAfZT/e5vTig7zagdCiM0A/gLjxP2xvHVuPm6OLkJf/GIsyn0vhPjY/JFvA/AH2Lf5iyZeIkrCOBn+SQjxiDm5qPezLub2sK/NW5GLAUyGYVVJaGKy4jVfrwPwaYFD7SgUzbFRTPj8JkSK8t2ZG8Hm9wZwCBFthGHJmUVEf4wgDgC+5+8oeB/A+0p2/yEYojkqDgDwqhDi4whj2BfABiHEFiFEM4BHAEzJxYpLQSS/DGCQOWq9DEba/7GIY/JARNVEVCufA9gPwBswYj3RnO1EAI9GE2EgfjE+BuAEMpgE4xbHh7oVFBKXX/cwGPsZMOI91qxk0A/AIAAvRRAfwfCTrRVCXKO8VLT72S/mYt3XRFRPRJ3N55UA5sDweS4GcKQ5m3sfy31/JIB/mtl8Jve0i9/sQhLwmxBFLLrvzrpCxyGEuEgI0VsI0QTjGPmnECIn2cF0CTh/FxwhxEcA3iOiIeak2QDWRBGLyTcQodXC5F0Ak4ioyvwuzYbxe589uRj9F/UfjNH/b8HwHP446nh8YuwPYxT3SgCrZZwwfI/PAHgbwNMAukYc570wbps3w7hiPckvRhijoG8w9/vrAMYXSbz3mPGsgnHybVTm/7EZ75sADohoH0+FYaVYBWCF+Xdgke9nv5iLcl8DGAXgNTOuNwD8zJzeH4ZYXw/j9mC5Ob3C/H+9+Xr/KI6NjvJXLL/Zut+PiOLQfr8iikX73Yn4eNkHEVa38Dt/RxjPGADLzc/orwC6RBRHNYw7bnVFcIxcAuNi7g3zvFSei/Vyxz2GYRiGYRiGcVEKdguGYRiGYRiGySkskhmGYRiGYRjGBYtkhmEYhmEYhnHBIplhGIZhGIZhXLBIZhiGYRiGYRgXLJKZdgsRtRLRCiJaSUSvElFg8XAi6kxEp4dY77NEND53kTIMwzDKb7b8y1mnOCJqIqJIahczpUsi9SwMU7R8LYzWqSCi/QFcBmBGwPydAZwO4MYCxMYwDMM4sX6zGaY9wJlkplToBOB/AEBENUT0jJldfp2I5pvzXA5ggJnBuNKc9//MeVYS0eXK+o4iopeI6C0imlbYt8IwDNNxIKKNRHSF+Vv8EhENNKc3EdE/iWiV+Zve15zeQER/MX+3Vyp3EeNE9AciWk1ET5kdAxkmYziTzLRnKoloBYxuaY0AZpnTdwA4TAjxBRF1B/BvInoMwIUARijZ5wMAzAewlxBiOxF1VdadEEJMJKIDAfwcRm94hmEYJnPkb7bkMiHE/ebzrUKIkUR0AoDrABwE4HcA7hJC3EVE3wHwWwCHmo9LhBCHEVEcQA2ALgAGAfiGEOK7RPQAgCMA/LEwb40pRVgkM+0Z1W4xGcDdRDQCRhvnXxHRdABtAHoBaNAsvy+AO4QQ2wFACPGZ8toj5uMrAJryEz7DMEyHIshuca/yeK35fDKAw83n9wC4wnw+C8AJACCEaAWwlYi6ANgghJAinH+7maxhkcyUBEKIZWbWuB7AgebjnkKIZiLaCCPbnA47zcdW8PeEYRgm3wif5+mwU3neCoDtFkxWsCeZKQmIaCiAOIBPAdQB2GwK5JkAdjdn+xJArbLYIgDfJqIqcx2q3YJhGIYpHMcoj8vM5y8AONZ8fhyAf5nPnwFwGgAQUZyI6goVJNOx4AwZ055R/W0E4EQhRCsR/QnA40T0OoDlANYBgBDiUyJaapYJ+rsQ4gIiGgNgORHtAvAEgB9F8D4YhmE6Am5P8j+EELIMXBciWgUjG/wNc9qZAO4gogsAbAHwbXP62QBuIaKTYGSMTwPwYd6jZzocJESmdzUYhmEYhmGyw7TEjRdCfBJ1LAyjwnYLhmEYhmEYhnHBmWSGYRiGYRiGccGZZIZhGIZhGIZxwSKZYRiGYRiGYVywSGYYhmEYhmEYFyySGYZhGIZhGMYFi2SGYRiGYRiGccEimWEYhmEYhmFc/H8jUJGRag2ougAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 10 | Time: 1m 10s\n", - "\tTrain Loss: 2.998 | Train PPL: 20.040\n", - "\t Val. Loss: 4.710 | Val. PPL: 111.007\n" - ] - } - ], - "source": [ - "for epoch in range(N_EPOCHS):\n", - " \n", - " start_time = time.time()\n", - " \n", - " train_loss = train(model, train_iterator, optimizer, criterion, CLIP, train_history, valid_history)\n", - " valid_loss = evaluate(model, valid_iterator, criterion)\n", - " \n", - " end_time = time.time()\n", - " \n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut1-model.pt')\n", - " \n", - " train_history.append(train_loss)\n", - " valid_history.append(valid_loss)\n", - " print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Let's take a look at our network quality__:" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [], - "source": [ - "del utils" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "import utils\n", - "import imp\n", - "imp.reload(utils)\n", - "generate_translation = utils.generate_translation\n", - "remove_tech_tokens = utils.remove_tech_tokens\n", - "get_text = utils.get_text\n", - "flatten = utils.flatten" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "batch = next(iter(test_iterator))" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Original: there is a 24 - hour front desk at the property .\n", - "Generated: the property offers a 24 - hour front desk . .\n", - "\n", - "Original: this property also features free wifi .\n", - "Generated: free wifi access . . . .\n", - "\n" - ] - } - ], - "source": [ - "for idx in [1,2]:\n", - " src = batch.src[:, idx:idx+1]\n", - " trg = batch.trg[:, idx:idx+1]\n", - " generate_translation(src, trg, model, TRG.vocab)" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [], - "source": [ - "from nltk.translate.bleu_score import corpus_bleu\n", - "\n", - "# \"\"\" Estimates corpora-level BLEU score of model's translations given inp and reference out \"\"\"\n", - "# translations, _ = model.translate_lines(inp_lines, **flags)\n", - "# # Note: if you experience out-of-memory error, split input lines into batches and translate separately\n", - "# return corpus_bleu([[ref] for ref in out_lines], translations) * 100" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "59it [00:03, 18.87it/s]\n" - ] - } - ], - "source": [ - "original_text = []\n", - "generated_text = []\n", - "model.eval()\n", - "with torch.no_grad():\n", - "\n", - " for i, batch in tqdm.tqdm(enumerate(test_iterator)):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output.argmax(dim=-1)\n", - " \n", - " original_text.extend([get_text(x, TRG.vocab) for x in trg.cpu().numpy().T])\n", - " generated_text.extend([get_text(x, TRG.vocab) for x in output[1:].detach().cpu().numpy().T])\n", - "\n", - "# original_text = flatten(original_text)\n", - "# generated_text = flatten(generated_text)" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "14.139920232081806" - ] - }, - "execution_count": 111, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "corpus_bleu([[text] for text in original_text], generated_text) * 100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Baseline solution BLEU score is quite low. Try to achieve at least __24__ BLEU on the test set. \n", - "The checkpoints are:\n", - "\n", - "* __22__ - minimal score to submit the homework, 30% of points\n", - "\n", - "* __27__ - good score, 70% of points\n", - "\n", - "* __29__ - excellent score, 100% of points" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "homework.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Py3 research env", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_part2_NMT_old-checkpoint.ipynb b/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_part2_NMT_old-checkpoint.ipynb deleted file mode 100644 index 4e586b1..0000000 --- a/homeworks/lab01_nlp/.ipynb_checkpoints/Lab1_NLP_part2_NMT_old-checkpoint.ipynb +++ /dev/null @@ -1,900 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eulvfJWl7ueY" - }, - "source": [ - "# Lab 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Part 2: Neural Machine Translation in the wild\n", - "In the second part of the homework you are supposed to get the best translation you can for the EN-RU translation task.\n", - "\n", - "Basic approach using RNNs as encoder and decoder is implemented for you. \n", - "\n", - "Your ultimate task is to use the techniques we've covered, e.g.\n", - "* [Byte Pair Encoding](https://github.com/rsennrich/subword-nmt)\n", - "\n", - "* CNN encoder (with or without positional encoding)\n", - "\n", - "* attention/self-attention mechanism\n", - "\n", - "* pretraining the language model\n", - "\n", - "* or just fine-tunning BERT)\n", - "\n", - "to improve the translation quality. \n", - "\n", - "__Please use at least three different approaches/models and compare them (translation quality/complexity/training and evaluation time).__\n", - "Write down some summary on your experiments and illustrate it with convergence plots/metrics and your thoughts. Just like you would approach a real problem." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# ! pip install subword-nmt\n", - "# ! pip install nltk\n", - "# ! pip install torchtext\n", - "# ! wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/advanced/homeworks/Lab1_NLP/data.txt\n", - "\n", - "# Thanks to YSDA NLP course team for the data\n", - "# (who thanks tilda and deephack teams for the data in their turn)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "\n", - "import torchtext\n", - "from torchtext.datasets import TranslationDataset, Multi30k\n", - "from torchtext.data import Field, BucketIterator\n", - "\n", - "import spacy\n", - "\n", - "import random\n", - "import math\n", - "import time\n", - "\n", - "import matplotlib\n", - "matplotlib.rcParams.update({'figure.figsize': (16, 12), 'font.size': 14})\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "from IPython.display import clear_output\n", - "\n", - "from nltk.tokenize import WordPunctTokenizer\n", - "from subword_nmt.learn_bpe import learn_bpe\n", - "from subword_nmt.apply_bpe import BPE\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Main part\n", - "__Here comes the preprocessing. Do not hesitate to use BPE or more complex preprocessing ;)__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer_W = WordPunctTokenizer()\n", - "def tokenize(x, tokenizer=tokenizer_W):\n", - " return tokenizer.tokenize(x.lower())" - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [], - "source": [ - "SRC = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "TRG = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "dataset = torchtext.data.TabularDataset(\n", - " path='data.txt',\n", - " format='tsv',\n", - " fields=[('trg', TRG), ('src', SRC)]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [], - "source": [ - "train_data, valid_data, test_data = dataset.split(split_ratio=[0.8, 0.15, 0.05])" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training examples: 40000\n", - "Number of validation examples: 2500\n", - "Number of testing examples: 7500\n" - ] - } - ], - "source": [ - "print(f\"Number of training examples: {len(train_data.examples)}\")\n", - "print(f\"Number of validation examples: {len(valid_data.examples)}\")\n", - "print(f\"Number of testing examples: {len(test_data.examples)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [], - "source": [ - "SRC.build_vocab(train_data, min_freq = 3)\n", - "TRG.build_vocab(train_data, min_freq = 3)" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unique tokens in source (ru) vocabulary: 9285\n", - "Unique tokens in target (en) vocabulary: 6770\n" - ] - } - ], - "source": [ - "print(f\"Unique tokens in source (ru) vocabulary: {len(SRC.vocab)}\")\n", - "print(f\"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here are tokens from original (RU) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['',\n", - " 'общими',\n", - " 'ferienwohnung',\n", - " 'закат',\n", - " 'campo',\n", - " 'шампанское',\n", - " 'louis',\n", - " 'уэверли',\n", - " 'диннер',\n", - " 'стеклянными']" - ] - }, - "execution_count": 97, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "SRC.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And from target (EN) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['', '46', 'cheeses', 'columbia', 'macerata', 'rouge', 'mactan']" - ] - }, - "execution_count": 98, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "TRG.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And here is example from train dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'trg': ['you', 'can', 'find', 'a', 'restaurant', '1', 'km', 'from', 'the', 'grain', 'bauernhof', ',', 'while', 'a', 'supermarket', 'is', '5', 'km', 'away', '.'], 'src': ['расстояние', 'от', 'фермерского', 'дома', 'grain', 'bauernhof', 'до', 'ресторана', 'составляет', '1', 'км', ',', 'до', 'супермаркета', '—', '5', 'км', '.']}\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[9]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check the length distributions:" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Train data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAEICAYAAAB2yHz3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcjUlEQVR4nO3dfbBlVXnn8e9PWl5VaKRDoBvtTmBM0Coj6QEcMhmHJrxmglWjFpnU2Did9FRCEpNJok2SGmZUZpoaJwgVZUKEAMaAhJjQA44MAZmUlQJpxCAvElreujsgzatG4wv6zB97XTi09/Y917597tl9vp+qU3fvtdbee+19zzrP2eusvXeqCkmS1B8vW+gKSJKkuTF4S5LUMwZvSZJ6xuAtSVLPGLwlSeoZg7ckST1j8NZOS/JwkhMWYLvLk1SSRaPetjRqSS5L8oGdWP4fk/zIfNaprdf2vwAM3uqNhfqQkKb05T2Y5JYkvziYVlWvqKoHF6pOO6svx35UDN4TKskeC10HaXczqWeBGj2D9xhK8t4kW5N8Lcn9SVa19L2SfCjJP7TXh5Ls1fLOTPLZ7dZTSQ5v05cluSjJp5J8HfjXSfZJ8j+TPJLkuSSfTbJPK39skr9N8mySv0vyliHr/rIk65J8OclTSa5OcmDLm+rmWp3k0SRPJvm9gWX3SXJ5kmeS3JfkPUm2tLyPAa8B/nfr/nvPwGZ/Ybr1SfNpuvfgwHt6TZJHgZtb2T9P8nhrV3+T5PUD67ksyYeTXN/a+G1JfrTlJcn5SZ5I8tUkX0zyhmnqsjjJdUm2tfZyXZJlLe9c4F8Cf9jq+YctffDzYP8kV7TlH0ny+0le1vLObJ8FH2zrfijJKUMeI9v/qFSVrzF6Aa8DNgOHtvnlwI+26fcBtwI/BCwB/hZ4f8s7E/jsdusq4PA2fRnwHHAc3Ze2vYEPA7cAS4E9gH8B7NXmnwJObWV/ps0vmaHODwMntOl3tzoua+v6I+DKgX0p4I+BfYA3At8Cfrzlrwf+H7C4LX8XsGW67QyzPl++5vu1g/fgFcB+wD4t/T8Ar2xt4EPAFwaWuay1p6OBRcDHgata3knAHcABQIAfBw4ZWO4DbfrVwL8F9m3b+XPgrwa2cQvwi9vVffDz4Arg2rbscuDvgTUt70zgO8Avtc+FXwb+Achsx8T2P8L34kJXwNd2/xA4HHgCOAF4+XZ5XwZOHZg/CXi4TZ/J7MH7ioG8lwH/BLxxmjq8F/jYdmk3AKtnqPNg470PWDWQd0j7IFg00NiWDeR/DjijTT8InDSQ94tDNt5p1+fL13y/dvAe/JEdLHNAK7N/m78M+OhA/qnAl9r08XSB9FjgZdut5zJa8J5mGz8BPDMwfwszBG+6gPxt4MiBvP8I3NKmzwQ2DeTt25b94dmOie1/dC+7zcdMVW0CfgP4L8ATSa5KcmjLPhR4ZKD4Iy1tWJsHpg+iO/v+8jTlXgu8vXWZP5vkWeCn6BribF4L/OXAcvcB3wUOHijz+MD0N4BXtOlDt6vj4PSOzLQ+aVReeK8m2SPJ+tZ1/FW6oANdm5sy7Xu2qm4G/pCuV+yJJBcnedX2G0uyb5I/al3eXwX+Bjggw41lOQh4Od//WbJ0uvpV1Tfa5DDtyvY/IgbvMVRVf1ZVP0XXEAo4r2X9Q0ub8pqWBvB1um/IACT54elWPTD9JPBN4EenKbeZ7sz7gIHXflW1fojqbwZO2W7Zvatq6xDLPkbXXTblsB3UX1oIM70HB9P/HXA6Xe/Z/nRniNB1g8++gaoLq+ongSOBfwb8zjTFfovuJ7ZjqupVwE9vt40dtZUn6c6Gt/8sGaaNzsb2PyIG7zGT5HVJjk83EO2bdF3b32vZVwK/n2RJkoOA/wz8acv7O+D1SX4iyd50Z+4zqqrvAZcCf5Dk0Ha28Oa23T8F/k2Sk1r63kneMjUgZhb/Czg3yWvb/ixJcvqQu381cHYbjLMU+NXt8r8CzPt1qtIcDPMefCXdb69P0X2h/m/DrjzJP09yTJKX030h/yYvtv/tt/FPwLNtQNg5w9azqr5L19bOTfLK1lb/Ey9+luwM2/+IGLzHz150AzeepOsO+iHg7Jb3AWAj3UCOLwKfb2lU1d/TDWj7a+AB4CUjz2fw2209twNP053hv6yqNtOdOfwusI3u2/TvMNz75QJgA/B/k3yNbvDKMUMsR6v/FuChth/X0H0ITvnvdF9enk3y20OuU5pPw7wHr6Drht4K3EvXBob1KroBWM+0dTwF/I9pyn2IbpDWk239n94u/wLgbW3k9oXTLP9rdF8OHqT7rPgzui/zO8v2PyJpP/JLYyfJL9MNPvlXC10XSaNl+98xz7w1NpIckuS4dq3o6+h+1/vLha6XpF3P9j833g1I42RPuutCVwDPAlcBH1nQGkkaFdv/HNhtLklSz9htLklSz4x1t/lBBx1Uy5cvX+hqSGPvjjvueLKqlix0PXbE9iwNZ5j2PNbBe/ny5WzcuHGhqyGNvSSPzF5qYdmepeEM057tNpckqWcM3pIk9YzBW5KknjF4S5LUMwZvSZJ6xuAtSVLPGLwlSeoZg7ckST1j8JYkqWfG+g5r82n5uutnLfPw+tNGUBNJu9ow7R1s8+ovz7wlSeoZg7ckST1j8JYkqWcM3pIk9YzBW5KknjF4S5LUMwZvSZJ6xuAtSVLPGLwlSeoZg7ckST1j8JYkqWcM3pIk9YzBW5KknpmYp4oNwyePSZL6wDNvaTeU5NIkTyS5eyDtwCQ3Jnmg/V3c0pPkwiSbktyV5KiBZVa38g8kWT2Q/pNJvtiWuTBJRruH0mQzeEu7p8uAk7dLWwfcVFVHADe1eYBTgCPaay1wEXTBHjgHOAY4GjhnKuC3Mr80sNz225K0Cxm8pd1QVf0N8PR2yacDl7fpy4G3DqRfUZ1bgQOSHAKcBNxYVU9X1TPAjcDJLe9VVXVrVRVwxcC6JI2AwVuaHAdX1WNt+nHg4Da9FNg8UG5LS9tR+pZp0r9PkrVJNibZuG3btp3fA0mAwVuaSO2MuUawnYuramVVrVyyZMmu3pw0MQze0uT4Suvypv19oqVvBQ4bKLespe0ofdk06ZJGxOAtTY4NwNSI8dXAtQPp72yjzo8Fnmvd6zcAJyZZ3AaqnQjc0PK+muTYNsr8nQPrkjQCXuct7YaSXAm8BTgoyRa6UePrgauTrAEeAd7Rin8KOBXYBHwDeBdAVT2d5P3A7a3c+6pqahDcr9CNaN8H+D/tNRLD3I9B2t0ZvKXdUFX9/AxZq6YpW8BZM6znUuDSadI3Am/YmTpK+sHZbS5JUs8YvCVJ6pmhgneS30xyT5K7k1yZZO8kK5Lc1m6P+Ikke7aye7X5TS1/+cB6zm7p9yc5adfskiRJu7dZg3eSpcCvAyur6g3AHsAZwHnA+VV1OPAMsKYtsgZ4pqWf38qR5Mi23OvpbqX4kSR7zO/uSJK0+xu223wRsE+SRcC+wGPA8cA1LX/7Wy1O3YLxGmBVu5zkdOCqqvpWVT1EN7L16J3fBUmSJsuswbuqtgIfBB6lC9rPAXcAz1bV863Y4O0RX7ilYst/Dng1M99q8SW8naIkSTs2TLf5Yrqz5hXAocB+7MInCHk7RUmSdmyYbvMTgIeqaltVfQf4JHAc3ZOHpq4TH7w94gu3VGz5+wNPMfOtFiVJ0hwME7wfBY5Nsm/77XoVcC/wGeBtrcz2t1qcugXj24Cb200gNgBntNHoK+ieAfy5+dkNSZImx6x3WKuq25JcA3weeB64E7gYuB64KskHWtolbZFLgI8l2UT3POEz2nruSXI1XeB/Hjirqr47z/sjSdJub6jbo1bVOXT3Rh70INOMFq+qbwJvn2E95wLnzrGOkiRpgHdYkySpZwzekiT1jMFbkqSeMXhLktQzBm9JknrG4C1JUs8YvCVJ6hmDtyRJPWPwliSpZwzekiT1jMFbkqSeMXhLktQzBm9JknrG4C1JUs8M9UhQvWj5uutnLfPw+tNGUBNJ0qTyzFuSpJ7xzFvSxLInTX3lmbckST1j8JYmSJLfTHJPkruTXJlk7yQrktyWZFOSTyTZs5Xdq81vavnLB9Zzdku/P8lJC7U/0qQyeEsTIslS4NeBlVX1BmAP4AzgPOD8qjoceAZY0xZZAzzT0s9v5UhyZFvu9cDJwEeS7DHKfZEmncFbmiyLgH2SLAL2BR4DjgeuafmXA29t06e3eVr+qiRp6VdV1beq6iFgE3D0iOovCYO3NDGqaivwQeBRuqD9HHAH8GxVPd+KbQGWtumlwOa27POt/KsH06dZ5iWSrE2yMcnGbdu2ze8OSRPM4C1NiCSL6c6aVwCHAvvRdXvvMlV1cVWtrKqVS5Ys2ZWbkiaKwVuaHCcAD1XVtqr6DvBJ4DjggNaNDrAM2NqmtwKHAbT8/YGnBtOnWUbSCBi8pcnxKHBskn3bb9ergHuBzwBva2VWA9e26Q1tnpZ/c1VVSz+jjUZfARwBfG5E+yAJb9IiTYyqui3JNcDngeeBO4GLgeuBq5J8oKVd0ha5BPhYkk3A03QjzKmqe5JcTRf4nwfOqqrvjnRnpAln8JYmSFWdA5yzXfKDTDNavKq+Cbx9hvWcC5w77xWUNBS7zSVJ6hmDtyRJPWPwliSpZwzekiT1jMFbkqSeMXhLktQzBm9JknrG4C1JUs8MFbyTHJDkmiRfSnJfkjcnOTDJjUkeaH8Xt7JJcmGSTUnuSnLUwHpWt/IPJFk98xYlSdJMhj3zvgD4dFX9GPBG4D5gHXBTVR0B3NTmAU6hu9fxEcBa4CKAJAfS3dnpGLq7OZ0zFfAlSdLwZg3eSfYHfpp2v+Oq+nZVPUv3aMHLW7HLgbe26dOBK6pzK90Tiw4BTgJurKqnq+oZ4EZ28eMIJUnaHQ1z5r0C2Ab8SZI7k3w0yX7AwVX1WCvzOHBwm14KbB5YfktLmyn9JZKsTbIxycZt27bNbW8kSZoAwwTvRcBRwEVV9Sbg67zYRQ5Ae0xgzUeFquriqlpZVSuXLFkyH6uUJGm3Mkzw3gJsqarb2vw1dMH8K607nPb3iZa/FThsYPllLW2mdEmSNAezBu+qehzYnOR1LWkV3XN8NwBTI8ZXA9e26Q3AO9uo82OB51r3+g3AiUkWt4FqJ7Y0SZI0B8M+z/vXgI8n2ZPu2b/vogv8VydZAzwCvKOV/RRwKrAJ+EYrS1U9neT9wO2t3Puq6ul52QtJkibIUMG7qr4ArJwma9U0ZQs4a4b1XApcOpcKSpKkl/IOa5Ik9YzBW5KknjF4S5LUMwZvSZJ6xuAtSVLPGLwlSeoZg7ckST1j8JYkqWcM3pIk9YzBW5KknjF4S5LUMwZvaYIkOSDJNUm+lOS+JG9OcmCSG5M80P4ubmWT5MIkm5LcleSogfWsbuUfSLJ65i1K2hUM3tJkuQD4dFX9GPBG4D5gHXBTVR0B3NTmAU4BjmivtcBFAEkOBM4BjgGOBs6ZCviSRmPYR4JqDpavu36ocg+vP20X10R6UZL9gZ8GzgSoqm8D305yOvCWVuxy4BbgvcDpwBXtSYG3trP2Q1rZG6ce6ZvkRuBk4MpR7Ys06TzzlibHCmAb8CdJ7kzy0ST7AQdX1WOtzOPAwW16KbB5YPktLW2mdEkjYvCWJsci4Cjgoqp6E/B1XuwiB6CdZdd8bTDJ2iQbk2zctm3bfK1WmngGb2lybAG2VNVtbf4aumD+ldYdTvv7RMvfChw2sPyyljZT+vepqouramVVrVyyZMm87Yg06Qze0oSoqseBzUle15JWAfcCG4CpEeOrgWvb9AbgnW3U+bHAc617/QbgxCSL20C1E1uapBFxwJo0WX4N+HiSPYEHgXfRfYm/Oska4BHgHa3sp4BTgU3AN1pZqurpJO8Hbm/l3jc1eE3SaBi8pQlSVV8AVk6TtWqasgWcNcN6LgUund/aSRqW3eaSJPWMwVuSpJ4xeEuS1DMGb0mSesbgLUlSzxi8JUnqGYO3JEk9Y/CWJKlnDN6SJPWMwVuSpJ4xeEuS1DMGb0mSesbgLUlSzxi8JUnqGYO3JEk94/O8JWkHlq+7ftYyD68/bQQ1kV409Jl3kj2S3Jnkuja/IsltSTYl+USSPVv6Xm1+U8tfPrCOs1v6/UlOmu+dkSRpEsyl2/zdwH0D8+cB51fV4cAzwJqWvgZ4pqWf38qR5EjgDOD1wMnAR5LssXPVlyRp8gwVvJMsA04DPtrmAxwPXNOKXA68tU2f3uZp+ata+dOBq6rqW1X1ELAJOHo+dkKSpEky7Jn3h4D3AN9r868Gnq2q59v8FmBpm14KbAZo+c+18i+kT7PMC5KsTbIxycZt27bNYVckSZoMswbvJD8LPFFVd4ygPlTVxVW1sqpWLlmyZBSblCSpV4YZbX4c8HNJTgX2Bl4FXAAckGRRO7teBmxt5bcChwFbkiwC9geeGkifMriMJEka0qxn3lV1dlUtq6rldAPObq6qXwA+A7ytFVsNXNumN7R5Wv7NVVUt/Yw2Gn0FcATwuXnbE0mSJsTOXOf9XuCqJB8A7gQuaemXAB9Lsgl4mi7gU1X3JLkauBd4Hjirqr67E9uXJGkizSl4V9UtwC1t+kGmGS1eVd8E3j7D8ucC5861kpIk6UXeHlWSpJ4xeEuS1DMGb0mSesbgLUlSzxi8pQnjQ4ak/jN4S5PHhwxJPWfwliaIDxmSdg87c5MW7aTl666ftczD608bQU00QaYeMvTKNj/0Q4aSDD5k6NaBdU77kCHoHjQErAV4zWteM397IU04z7ylCTHqhwyBDxqSdhXPvKXJ4UOGpN2EZ97ShPAhQ9LuwzNvST5kSOoZg7c0gXzIkNRvdptLktQzBm9JknrG4C1JUs8YvCVJ6hmDtyRJPWPwliSpZwzekiT1jMFbkqSeMXhLktQzBm9JknrG4C1JUs8YvCVJ6hmDtyRJPWPwliSpZwzekiT1jMFbkqSeMXhLktQzBm9JknrG4C1JUs8YvCVJ6hmDtyRJPWPwliSpZ2YN3kkOS/KZJPcmuSfJu1v6gUluTPJA+7u4pSfJhUk2JbkryVED61rdyj+QZPWu2y1JknZfw5x5Pw/8VlUdCRwLnJXkSGAdcFNVHQHc1OYBTgGOaK+1wEXQBXvgHOAY4GjgnKmAL0mShrdotgJV9RjwWJv+WpL7gKXA6cBbWrHLgVuA97b0K6qqgFuTHJDkkFb2xqp6GiDJjcDJwJXzuD+SNHLL110/a5mH1582gppoUszpN+8ky4E3AbcBB7fADvA4cHCbXgpsHlhsS0ubKX37baxNsjHJxm3bts2lepIkTYRZz7ynJHkF8BfAb1TVV5O8kFdVlaTmo0JVdTFwMcDKlSuHWucw33olSdpdDHXmneTldIH741X1yZb8ldYdTvv7REvfChw2sPiyljZTuiRJmoNhRpsHuAS4r6r+YCBrAzA1Ynw1cO1A+jvbqPNjgeda9/oNwIlJFreBaie2NEmSNAfDnHkfB/x74PgkX2ivU4H1wM8keQA4oc0DfAp4ENgE/DHwKwBtoNr7gdvb631Tg9ck7Xpe9intPoYZbf5ZIDNkr5qmfAFnzbCuS4FL51JBSfNm6rLPzyd5JXBHu+rjTLrLPtcnWUd32ed7eelln8fQXfZ5zMBlnyuBauvZUFXPjHyPpAnlHdakCVFVj1XV59v014DByz4vb8UuB97apl+47LOqbgWmLvs8iXbZZwvYU5d9ShoRg7c0gUZx2Wfbjpd+SruAwVuaMNtf9jmY1372mpfLPtv6Lq6qlVW1csmSJfO1WmniGbylCeJln9LuweAtTQgv+5R2H0PfYU0Lw3smax5NXfb5xSRfaGm/S3eZ59VJ1gCPAO9oeZ8CTqW77PMbwLugu+wzydRln+Bln9LIGbylCeFln9Luw25zSZJ6xuAtSVLP2G2+G/B3cUmaLJ55S5LUM555Sxobw/QiSfLMW5Kk3jF4S5LUMwZvSZJ6xuAtSVLPGLwlSeoZg7ckST1j8JYkqWe8zluSRmDYa9i9G6KG4Zm3JEk9Y/CWJKlnDN6SJPWMv3lPCH9vk6Tdh2fekiT1jMFbkqSesdtcksbIMD9x+fOWPPOWJKlnDN6SJPWMwVuSpJ7xN2+9hL+3SdL488xbkqSeMXhLktQzBm9JknrG37w1Z/4uLi0s26A885YkqWdGfuad5GTgAmAP4KNVtX7UdZC082zL482z893bSIN3kj2ADwM/A2wBbk+yoaruHWU9tOv5wbF7sy1LC2vUZ95HA5uq6kGAJFcBpwM2+Ak07GNKh+EXgZGzLe8GbIP9NergvRTYPDC/BThmsECStcDaNvuPSe4HDgKeHEkNdw3rv4vlvB1mj339ZzFM/V87iooMmLUtw4ztedC4/m+s19wclPPGs16M6fFix/WatT2P3WjzqroYuHgwLcnGqlq5QFXaadZ/YVn/hTNdex40rvtmvebGes3NfNRr1KPNtwKHDcwva2mS+sW2LC2gUQfv24EjkqxIsidwBrBhxHWQtPNsy9ICGmm3eVU9n+RXgRvoLi+5tKruGWLRGbvdesL6LyzrP892oi1vb+z2rbFec2O95man65Wqmo+KSJKkEfEOa5Ik9YzBW5Kknhn74J3k5CT3J9mUZN1C12c2SQ5L8pkk9ya5J8m7W/qBSW5M8kD7u3ih6zqTJHskuTPJdW1+RZLb2v/gE22A0thKckCSa5J8Kcl9Sd7cl+Of5Dfb++buJFcm2btvx39Y49K2x7nNjmtbHNc2Ni7tJ8mlSZ5IcvdA2rTHJ50LW/3uSnLUMNsY6+CdF2/BeApwJPDzSY5c2FrN6nngt6rqSOBY4KxW53XATVV1BHBTmx9X7wbuG5g/Dzi/qg4HngHWLEithncB8Omq+jHgjXT7MvbHP8lS4NeBlVX1BrqBYGfQv+M/qzFr2+PcZse1LY5dGxuz9nMZcPJ2aTMdn1OAI9prLXDRUFuoqrF9AW8GbhiYPxs4e6HrNcd9uJbu/s/3A4e0tEOA+xe6bjPUd1l7Yx0PXAeE7k5Ai6b7n4zbC9gfeIg2GHMgfeyPPy/etexAuitBrgNO6tPxn8O+jm3bHpc2O65tcVzb2Li1H2A5cPdsxwf4I+Dnpyu3o9dYn3kz/S0Yly5QXeYsyXLgTcBtwMFV9VjLehw4eIGqNZsPAe8BvtfmXw08W1XPt/lx/x+sALYBf9K6Gz+aZD96cPyraivwQeBR4DHgOeAO+nX8hzWWbXvM2uy4tsWxbGM9aD8zHZ8fqC2Me/DurSSvAP4C+I2q+upgXnVfr8buGr0kPws8UVV3LHRddsIi4Cjgoqp6E/B1tuu+G+Pjv5ju4R4rgEOB/fj+rjftIuPUZse8LY5lG+tT+5mP4zPuwbuXt2BM8nK6D4GPV9UnW/JXkhzS8g8Bnlio+u3AccDPJXkYuIquu+4C4IAkUzf0Gff/wRZgS1Xd1uavofug6cPxPwF4qKq2VdV3gE/S/U/6dPyHNVZtewzb7Di3xXFtY+PefmY6Pj9QWxj34N27WzAmCXAJcF9V/cFA1gZgdZteTfe72lipqrOrallVLac71jdX1S8AnwHe1oqNZd2nVNXjwOYkr2tJq+geUzn2x5+uu+/YJPu299FU3Xtz/OdgbNr2OLbZcW6LY9zGxr39zHR8NgDvbKPOjwWeG+hen9koBxT8gD/6nwr8PfBl4PcWuj5D1Pen6LpD7gK+0F6n0v1edRPwAPDXwIELXddZ9uMtwHVt+keAzwGbgD8H9lro+s1S958ANrb/wV8Bi/ty/IH/CnwJuBv4GLBX347/HPZ1LNr2uLfZcWyL49rGxqX9AFfS/e7+HbqeijUzHR+6gYgfbu3gi3Sj5WfdhrdHlSSpZ8a921ySJG3H4C1JUs8YvCVJ6hmDtyRJPWPwliSpZwzekiT1jMFbkqSe+f+0vTrOPh+1DAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in train_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in train_data.examples])\n", - "\n", - "print('Length distribution in Train data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Test data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe0AAAEICAYAAAByPazKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAc9ElEQVR4nO3dfbBlVXnn8e9PEHwNDXIlbTdtEyUkaI2R6QCOSQZF5UWTtmqMhZNMGkOmZxJMzGiijUmFqUQyWMkEsTQkrXSAxICEmNAjjISghLEihMZ3QELLi90dsC/yYqJRgz7zx14th8u9fW/fc1/Ovvf7qTrVe6+19t5r97nrPGuvtc8+qSokSdLoe9JiV0CSJM2MQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2hpbkniSvWITjrk1SSfZf6GNLCy3JRUneOcT2/5LkB+ayTm2/tv8FZNBWbyzWh4O0R1/+BpNcn+QXBtOq6hlVdddi1WlYffm/n28G7WUqyX6LXQdpqVluV31aeAbtEZTk7Ul2JfnnJHckObGlH5jk3Un+qb3eneTAlnd6kk9M2E8leX5bvijJBUmuTvJ14GVJnprkfye5N8kjST6R5Kmt/PFJ/j7Jw0k+m+SEGdb9SUk2JflSkq8muTzJIS1vz3DWhiRfTvJAkt8Y2PapSS5O8lCS25O8LcnOlvenwBrg/7RhvrcNHPZnJtufNJcm+xsc+Js+I8mXgY+1sn+R5P7Wrm5I8oKB/VyU5H1Jrmpt/KYkz2t5SXJekt1Jvpbk80leOEldDk7ykSTjrb18JMnqlncO8OPAe1s939vSBz8PDkpySdv+3iS/meRJLe/09lnw+23fdyc5ZYb/R7b/+VZVvkboBRwF7ACe09bXAs9ry78N3Ag8GxgD/h74nZZ3OvCJCfsq4Plt+SLgEeCldJ21pwDvA64HVgH7Af8BOLCtfxU4tZV9ZVsfm6LO9wCvaMtvbnVc3fb1x8ClA+dSwPuBpwIvAr4F/HDLPxf4O+Dgtv3ngJ2THWcm+/Pla65fe/kbvAR4OvDUlv7zwDNbG3g38JmBbS5q7elYYH/gg8BlLe8k4BZgBRDgh4GVA9u9sy0/C/hPwNPacf4C+OuBY1wP/MKEug9+HlwCXNm2XQv8I3BGyzsd+Dfgv7bPhV8E/gnIdP8ntv8F+Btc7Ar4mvCGwPOB3cArgCdPyPsScOrA+knAPW35dKYP2pcM5D0J+FfgRZPU4e3An05IuwbYMEWdBxvt7cCJA3kr2wfA/gONbPVA/j8Ap7Xlu4CTBvJ+YYaNdtL9+fI116+9/A3+wF62WdHKHNTWLwI+MJB/KvDFtvxyugB6PPCkCfu5iBa0JznGjwAPDaxfzxRBmy4Qfxs4eiDvvwHXt+XTge0DeU9r237/dP8ntv/5fzk8PmKqajvwq8D/BHYnuSzJc1r2c4B7B4rf29JmasfA8qF0V9tfmqTcc4GfbkPjDyd5GPgxugY4necCfzWw3e3Ad4DDBsrcP7D8DeAZbfk5E+o4uLw3U+1PWijf+1tNsl+Sc9sQ8dfogg10bW6PSf9mq+pjwHvpRsF2J9mc5PsmHizJ05L8cRva/hpwA7AiM7tX5VDgyTzxs2TVZPWrqm+0xZm0K9v/PDNoj6Cq+vOq+jG6BlDAu1rWP7W0Pda0NICv0/WIAUjy/ZPtemD5AeCbwPMmKbeD7kp7xcDr6VV17gyqvwM4ZcK2T6mqXTPY9j66YbE9Dt9L/aXFMNXf4GD6fwbW042WHUR3RQjdcPf0B6h6T1X9e+Bo4AeBX5+k2FvpptKOq6rvA35iwjH21lYeoLv6nfhZMpM2Oh3b/zwzaI+YJEcleXm6G8y+STeE/d2WfSnwm0nGkhwK/BbwZy3vs8ALkvxIkqfQXalPqaq+C2wB/iDJc9rVwUvacf8M+MkkJ7X0pyQ5Yc+NLtP4I+CcJM9t5zOWZP0MT/9y4Kx2k80q4E0T8r8CzPn3TKV9MJO/wWfSza1+la4j/bsz3XmSH01yXJIn03XEv8lj7X/iMf4VeLjd6HX2TOtZVd+ha2vnJHlma6tv4bHPkmHY/ueZQXv0HEh3Q8YDdMM+zwbOannvBLbR3aDxeeBTLY2q+ke6G9X+FrgTeNyd5FP4tbafm4EH6a7on1RVO+iuFN4BjNP1nn+dmf29nA9sBf4myT/T3ZRy3Ay2o9V/J3B3O48r6D789vhfdJ2Wh5P82gz3Kc2lmfwNXkI33LwLuI2uDczU99HdWPVQ28dXgd+bpNy76W6+eqDt/6MT8s8HXtfuxH7PJNv/Ml2n4C66z4o/p+vED8v2P8/SJu+lkZPkF+luKvmPi10XSQvL9j85r7Q1MpKsTPLS9l3Po+jm7f5qseslaf7Z/mfGp/dolBxA973OI4CHgcuAP1zUGklaKLb/GXB4XJKknnB4XJKknhjp4fFDDz201q5du9jVkEbeLbfc8kBVjU1XLskW4DXA7qp64UD6LwNn0j0I46qqeltLPws4o6X/SlVd09JPprtTeD+6p3tN+x1+27M0M3trzyMdtNeuXcu2bdsWuxrSyEty7/SlgO5RmO+l+1rSnm1fRvcVvxdV1beSPLulHw2cBryA7mlVf5vkB9tm76N7Jv1O4OYkW6vqtr0d2PYszcze2vNIB21Jc6uqbkiydkLyLwLnVtW3WpndLX093Q9ZfAu4O8l2uh+5gO7Z1HcBJLmsld1r0JY0POe0Jf0g8OPpfiLy75L8aEtfxeOf/7yzpU2V/gRJNibZlmTb+Pj4PFRdWl4M2pL2Bw6h+2WpXwcuTzKj52RPp6o2V9W6qlo3NjbtlLukaTg8Lmkn8OHqvv/5D0m+S/dLULt4/I82rOaxH5WYKl3SPPJKW9JfAy8DaDeaHUD3TOutwGlJDkxyBHAk3e8V3wwcmeSIJAfQ3ay2dVFqLi0zXmlLy0iSS4ETgEOT7KT7dagtwJYkXwC+DWxoV923Jrmc7gazR4Ez2y9EkeRNwDV0X/naUlW3LvjJSMuQQVtaRqrqDVNk/ewU5c8Bzpkk/Wrg6jmsmqQZcHhckqSeMGhLktQTDo/vo7WbrppRuXvOffU810TSQphJm7e9a6F4pS1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFbkqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJ6YN2km2JNmd5AuT5L01SSU5tK0nyXuSbE/yuSTHDJTdkOTO9towt6chSdLSN5Mr7YuAkycmJjkceBXw5YHkU4Aj22sjcEErewhwNnAccCxwdpKDh6m4JEnLzbRBu6puAB6cJOs84G1ADaStBy6pzo3AiiQrgZOAa6vqwap6CLiWSToCkiRparOa006yHthVVZ+dkLUK2DGwvrOlTZU+2b43JtmWZNv4+PhsqidpCk53Sf22z0E7ydOAdwC/NffVgaraXFXrqmrd2NjYfBxCWs4uwukuqbdmc6X9POAI4LNJ7gFWA59K8v3ALuDwgbKrW9pU6ZIWkNNdUr/tc9Cuqs9X1bOram1VraUb6j6mqu4HtgI/14bVjgceqar7gGuAVyU5uPXIX9XSJC0yp7uk/pjJV74uBT4JHJVkZ5Iz9lL8auAuYDvwfuCXAKrqQeB3gJvb67dbmqRF5HSX1C/7T1egqt4wTf7ageUCzpyi3BZgyz7Wb0Gt3XTVYldBWmiD013w2HTXsex9uuuECenXL0BdpWXPJ6JJy5jTXVK/GLSlZcTpLqnfph0el7R0LKfpLmkp8kpbkqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFbkqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPeFPc0pattZuumqxqyDtk2mvtJNsSbI7yRcG0n4vyReTfC7JXyVZMZB3VpLtSe5IctJA+sktbXuSTXN/KpIkLW0zGR6/CDh5Qtq1wAur6t8B/wicBZDkaOA04AVtmz9Msl+S/YD3AacARwNvaGUlSdIMTRu0q+oG4MEJaX9TVY+21RuB1W15PXBZVX2rqu4GtgPHttf2qrqrqr4NXNbKSpKkGZqLOe2fBz7UllfRBfE9drY0gB0T0o+bbGdJNgIbAdasWTMH1ZO0R5ItwGuA3VX1wpb2e8BPAt8GvgS8saoebnlnAWcA3wF+paquaeknA+cD+wEfqKpzF/pcpuN8tZaioe4eT/IbwKPAB+emOlBVm6tqXVWtGxsbm6vdSupchNNdUm/N+ko7yel0PfYTq6pa8i7g8IFiq1sae0mXtECq6oYkayek/c3A6o3A69ry96a7gLuT7JnugjbdBZBkz3TXbfNYdUnM8kq7DY29DfipqvrGQNZW4LQkByY5AjgS+AfgZuDIJEckOYCu9751uKpLmgc/D/zftryKJ05rrdpL+hMk2ZhkW5Jt4+Pj81BdaXmZyVe+LgU+CRyVZGeSM4D3As8Erk3ymSR/BFBVtwKX0/W4PwqcWVXfaTetvQm4BrgduLyVlTQinO6SRt+0w+NV9YZJki/cS/lzgHMmSb8auHqfaidpQTjdJfWDjzGVljmnu6T+WDaPMfXrH9L3prtOAA5NshM4m+5u8QPpprsAbqyq/15VtybZM931KG26q+1nz3TXfsAWp7ukhbFsgrYkp7ukvnN4XJKknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqSemDdpJtiTZneQLA2mHJLk2yZ3t34NbepK8J8n2JJ9LcszANhta+TuTbJif05EkaemayZX2RcDJE9I2AddV1ZHAdW0d4BTgyPbaCFwAXZAHzgaOA44Fzt4T6CVJ0sxMG7Sr6gbgwQnJ64GL2/LFwGsH0i+pzo3AiiQrgZOAa6vqwap6CLiWJ3YEJM0zR86kfpvtnPZhVXVfW74fOKwtrwJ2DJTb2dKmSn+CJBuTbEuybXx8fJbVkzSFi3DkTOqtoW9Eq6oCag7qsmd/m6tqXVWtGxsbm6vdSsKRM6nv9p/ldl9JsrKq7muNeHdL3wUcPlBudUvbBZwwIf36WR57yVi76appy9xz7qsXoCZa5uZt5EzS3Jpt0N4KbADObf9eOZD+piSX0Q2dPdIC+zXA7w4Mob0KOGv21R59MwnI0qipqkoyZyNnSTbSDa2zZs2audqttGzN5CtflwKfBI5KsjPJGXTB+pVJ7gRe0dYBrgbuArYD7wd+CaCqHgR+B7i5vX67pUlafF9pI2bsw8jZZOlP4HSXNLemvdKuqjdMkXXiJGULOHOK/WwBtuxT7SQtBEfOpJ6Y7fC4pB5qI2cnAIcm2Ul3F/i5wOVtFO1e4PWt+NXAqXQjZ98A3gjdyFmSPSNn4MiZtGAM2tIy4siZ1G8+e1ySpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKkn/J62JA1ppr814A8AaVheaUuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4YKmgn+R9Jbk3yhSSXJnlKkiOS3JRke5IPJTmglT2wrW9v+Wvn4gQkSVouZh20k6wCfgVYV1UvBPYDTgPeBZxXVc8HHgLOaJucATzU0s9r5SRJ0gwNOzy+P/DUJPsDTwPuA14OXNHyLwZe25bXt3Va/olJMuTxJc0RR86k0TfroF1Vu4DfB75MF6wfAW4BHq6qR1uxncCqtrwK2NG2fbSVf9Zsjy9p7jhyJvXDMMPjB9NdPR8BPAd4OnDysBVKsjHJtiTbxsfHh92dpJlz5EwaccMMj78CuLuqxqvq34APAy8FVrRGD7Aa2NWWdwGHA7T8g4CvTtxpVW2uqnVVtW5sbGyI6kmaqfkaObMTLs2tYYL2l4Hjkzyt9bBPBG4DPg68rpXZAFzZlre2dVr+x6qqhji+pDkyXyNndsKluTXMnPZNdMNinwI+3/a1GXg78JYk2+l63he2TS4EntXS3wJsGqLekubWvIycSZpb+09fZGpVdTZw9oTku4BjJyn7TeCnhzmepHnzvZEz4F/pRs628djI2WVMPnL2SRw5kxaMT0ST5MiZ1BNDXWlLWjocOZNGn1fakiT1hEFbkqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hE9Ek6QFsnbTVdOWuefcVy9ATdRXXmlLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1xFBBO8mKJFck+WKS25O8JMkhSa5Ncmf79+BWNknek2R7ks8lOWZuTkGSpOVh2Cvt84GPVtUPAS8Cbgc2AddV1ZHAdW0d4BTgyPbaCFww5LElzSE74dLom3XQTnIQ8BPAhQBV9e2qehhYD1zcil0MvLYtrwcuqc6NwIokK2ddc0lzzU64NOKGeYzpEcA48CdJXgTcArwZOKyq7mtl7gcOa8urgB0D2+9safcNpJFkI92HAGvWrBmiekuDjz3UQhjohJ8OXScc+HaS9cAJrdjFwPXA2xnohAM3tqv0lQNtX9I8GGZ4fH/gGOCCqnox8HUe64UD0Bp07ctOq2pzVa2rqnVjY2NDVE/SPhjshH86yQeSPJ1974Q/TpKNSbYl2TY+Pj6P1ZeWh2GC9k5gZ1Xd1NavoAviX9kz7N3+3d3ydwGHD2y/uqVJWnx2wqUemHXQrqr7gR1JjmpJJwK3AVuBDS1tA3BlW94K/Fy7geV44BGH0qSRYSdc6oFhf5rzl4EPJjkAuAt4I11H4PIkZwD3Aq9vZa8GTgW2A99oZSWNgKq6P8mOJEdV1R081gm/ja7zfS5P7IS/KcllwHHYCZcWxFBBu6o+A6ybJOvEScoWcOYwx5M0r+yESyNu2CttSUuEnXBp9PkYU0mSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesInoknSCFm76appy9xz7qsXoCYaRQZtSb0yk6AmLVUOj0uS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknhg7aSfZL8ukkH2nrRyS5Kcn2JB9KckBLP7Ctb2/5a4c9tiRJy8lcXGm/Gbh9YP1dwHlV9XzgIeCMln4G8FBLP6+VkzRC7IRLo22ooJ1kNfBq4ANtPcDLgStakYuB17bl9W2dln9iKy9pdNgJl0bYsFfa7wbeBny3rT8LeLiqHm3rO4FVbXkVsAOg5T/Syj9Oko1JtiXZNj4+PmT1JM2UnXBp9M36MaZJXgPsrqpbkpwwVxWqqs3AZoB169bVXO1X0rT2dMKf2dZn3AlPsqcT/sDgDpNsBDYCrFmzZl4rv5z4fPLla5gr7ZcCP5XkHuAyuh75+cCKJHs6A6uBXW15F3A4QMs/CPjqEMeXNEcGO+Fzud+q2lxV66pq3djY2FzuWlqWZh20q+qsqlpdVWuB04CPVdXPAB8HXteKbQCubMtb2zot/2NV5ZW0NBrshEs9MB/f03478JYk2+mGyy5s6RcCz2rpbwE2zcOxJc2CnXCpH+bkpzmr6nrg+rZ8F3DsJGW+Cfz0XBxP0oJ5O3BZkncCn+bxnfA/bZ3wB+kCvaR55u9pS3ocO+HS6PIxppIk9YRBW5KknnB4fAnwO5uStDx4pS1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFbkqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJ/zBkGViJj8qAv6wiCSNMq+0JUnqiVkH7SSHJ/l4ktuS3JrkzS39kCTXJrmz/XtwS0+S9yTZnuRzSY6Zq5OQJGk5GOZK+1HgrVV1NHA8cGaSo4FNwHVVdSRwXVsHOAU4sr02AhcMcWxJc8hOuNQPsw7aVXVfVX2qLf8zcDuwClgPXNyKXQy8ti2vBy6pzo3AiiQrZ11zSXPJTrjUA3Myp51kLfBi4CbgsKq6r2XdDxzWllcBOwY229nSJu5rY5JtSbaNj4/PRfUkTcNOuNQPQwftJM8A/hL41ar62mBeVRVQ+7K/qtpcVeuqat3Y2Niw1ZO0j+yES6NrqK98JXkyXcD+YFV9uCV/JcnKqrqv9bx3t/RdwOEDm69uaZJGxMROeJLv5VVVJdnnTjiwGWDdunX7tK2GM5OvefoVz/4Z5u7xABcCt1fVHwxkbQU2tOUNwJUD6T/XbmA5HnhkoAcvaZHtrRPe8u2ES4tsmOHxlwL/BXh5ks+016nAucArk9wJvKKtA1wN3AVsB94P/NIQx5Y0h+yES/0w6+HxqvoEkCmyT5ykfAFnzvZ4kubVnk7455N8pqW9g67TfXmSM4B7gde3vKuBU+k64d8A3riw1ZWWJx9jKslOuNQTPsZUkqSeMGhLktQTBm1JknrCOW1JWqb8yd7+8UpbkqSeMGhLktQTBm1JknrCOW09zkznuKbjHJgkzT2vtCVJ6gmDtiRJPeHwuCRpr/yZz9HhlbYkST1h0JYkqScM2pIk9YRz2poXzoFJy4ttfmEsiaA9V98tliRplDk8LklSTxi0JUnqiQUP2klOTnJHku1JNi308SXNDduytPAWdE47yX7A+4BXAjuBm5NsrarbFrIeGg1zeS+CN7gsrPlqy96fsrTZ5oe30DeiHQtsr6q7AJJcBqwHDNoaineuLjjbshbVcv1xo4UO2quAHQPrO4HjBgsk2QhsbKv/kuSOKfZ1KPDAnNdwcSylc4ERPZ+8a1abjeS5TOK5C3y8adsyzLg99+X/eBie44jax8+FhTrHKdvzyH3lq6o2A5unK5dkW1WtW4AqzbuldC6wtM5nKZ3LYphJe14O/8ee49IwCue40Dei7QIOH1hf3dIk9YttWVoECx20bwaOTHJEkgOA04CtC1wHScOzLUuLYEGHx6vq0SRvAq4B9gO2VNWts9zdtEPoPbKUzgWW1vkspXOZM7blfeY5Lg2Lfo6pqsWugyRJmgGfiCZJUk8YtCVJ6oleBu0+Pz4xyeFJPp7ktiS3JnlzSz8kybVJ7mz/HrzYdZ2pJPsl+XSSj7T1I5Lc1N6fD7UblUZekhVJrkjyxSS3J3lJn9+XPuhzW57KUmzjk1kq7X4qo/p50LugPfD4xFOAo4E3JDl6cWu1Tx4F3lpVRwPHA2e2+m8CrquqI4Hr2npfvBm4fWD9XcB5VfV84CHgjEWp1b47H/hoVf0Q8CK6c+rz+zLSlkBbnspSbOOTWSrtfiqj+XlQVb16AS8BrhlYPws4a7HrNcT5XEn3/OY7gJUtbSVwx2LXbYb1X033x/ty4CNA6J4YtP9k79eovoCDgLtpN2cOpPfyfenDa6m15b2cZ6/b+BTntCTa/V7Ob2Q/D3p3pc3kj09ctUh1GUqStcCLgZuAw6rqvpZ1P3DYIlVrX70beBvw3bb+LODhqnq0rffl/TkCGAf+pA35fSDJ0+nv+9IHS6YtT2WJtPHJLJV2P5WR/TzoY9BeEpI8A/hL4Fer6muDedV140b+u3hJXgPsrqpbFrsuc2B/4Bjggqp6MfB1Jgx99eV90WhYCm18Mkus3U9lZD8P+hi0e//4xCRPpmvMH6yqD7fkryRZ2fJXArsXq3774KXATyW5B7iMbqjsfGBFkj0P7unL+7MT2FlVN7X1K+gabR/fl77ofVueyhJq45NZSu1+KiP7edDHoN3rxycmCXAhcHtV/cFA1lZgQ1veQDcPNtKq6qyqWl1Va+neh49V1c8AHwde14r15VzuB3YkOaolnUj3M5O9e196pNdteSpLqY1PZim1+6mM8udBL5+IluRUujmVPY9PPGeRqzRjSX4M+H/A53lsPugddHNelwNrgHuB11fVg4tSyVlIcgLwa1X1miQ/QNcDPwT4NPCzVfWtxazfTCT5EeADwAHAXcAb6Tq2vX1fRl2f2/JUlmobn8xSaPdTGdXPg14GbUmSlqM+Do9LkrQsGbQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoS5LUE/8fozUqvHgTO/4AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in test_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in test_data.examples])\n", - "\n", - "print('Length distribution in Test data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model side\n", - "__Here comes simple pipeline of NMT model learning. It almost copies the week03 practice__" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "device(type='cuda')" - ] - }, - "execution_count": 103, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "device" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [], - "source": [ - "def _len_sort_key(x):\n", - " return len(x.src)\n", - "\n", - "BATCH_SIZE = 128\n", - "\n", - "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE, \n", - " device = device,\n", - " sort_key=_len_sort_key\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "[torchtext.data.batch.Batch of size 128]\n", - "\t[.trg]:[torch.cuda.LongTensor of size 47x128 (GPU 0)]\n", - "\t[.src]:[torch.cuda.LongTensor of size 47x128 (GPU 0)]\n", - "torch.Size([47, 128]) torch.Size([47, 128])\n" - ] - } - ], - "source": [ - "for x in train_iterator:\n", - " break\n", - "print(x)\n", - "print(x.src.shape, x.trg.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "import my_network\n", - "Encoder = my_network.Encoder\n", - "Decoder = my_network.Decoder\n", - "Seq2Seq = my_network.Seq2Seq" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(SRC.vocab)\n", - "OUTPUT_DIM = len(TRG.vocab)\n", - "ENC_EMB_DIM = 256\n", - "DEC_EMB_DIM = 256\n", - "HID_DIM = 512\n", - "N_LAYERS = 2\n", - "ENC_DROPOUT = 0.5\n", - "DEC_DROPOUT = 0.5\n", - "\n", - "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n", - "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n", - "\n", - "# dont forget to put the model to the right device\n", - "model = Seq2Seq(enc, dec, device).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Seq2Seq(\n", - " (encoder): Encoder(\n", - " (embedding): Embedding(9285, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - " (decoder): Decoder(\n", - " (embedding): Embedding(6770, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (out): Linear(in_features=512, out_features=6770, bias=True)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - ")" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def init_weights(m):\n", - " # \n", - " for name, param in m.named_parameters():\n", - " nn.init.uniform_(param, -0.08, 0.08)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The model has 14,939,506 trainable parameters\n" - ] - } - ], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [], - "source": [ - "PAD_IDX = TRG.vocab.stoi['']\n", - "optimizer = optim.Adam(model.parameters())\n", - "criterion = nn.CrossEntropyLoss(ignore_index = PAD_IDX)" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, clip, train_history=None, valid_history=None):\n", - " model.train()\n", - " \n", - " epoch_loss = 0\n", - " history = []\n", - " for i, batch in enumerate(iterator):\n", - " \n", - " src = batch.src\n", - " trg = batch.trg\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " output = model(src, trg)\n", - " \n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - " \n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - " \n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - " \n", - " loss = criterion(output, trg)\n", - " \n", - " loss.backward()\n", - " \n", - " # Let's clip the gradient\n", - " torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " history.append(loss.cpu().data.numpy())\n", - " if (i+1)%10==0:\n", - " fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8))\n", - "\n", - " clear_output(True)\n", - " ax[0].plot(history, label='train loss')\n", - " ax[0].set_xlabel('Batch')\n", - " ax[0].set_title('Train loss')\n", - " if train_history is not None:\n", - " ax[1].plot(train_history, label='general train history')\n", - " ax[1].set_xlabel('Epoch')\n", - " if valid_history is not None:\n", - " ax[1].plot(valid_history, label='general valid history')\n", - " plt.legend()\n", - " \n", - " plt.show()\n", - "\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 112, - "metadata": {}, - "outputs": [], - "source": [ - "def evaluate(model, iterator, criterion):\n", - " \n", - " model.eval()\n", - " \n", - " epoch_loss = 0\n", - " \n", - " history = []\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for i, batch in enumerate(iterator):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - "\n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - "\n", - " loss = criterion(output, trg)\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 113, - "metadata": {}, - "outputs": [], - "source": [ - "def epoch_time(start_time, end_time):\n", - " elapsed_time = end_time - start_time\n", - " elapsed_mins = int(elapsed_time / 60)\n", - " elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n", - " return elapsed_mins, elapsed_secs" - ] - }, - { - "cell_type": "code", - "execution_count": 114, - "metadata": {}, - "outputs": [], - "source": [ - "train_history = []\n", - "valid_history = []\n", - "\n", - "N_EPOCHS = 10\n", - "CLIP = 1\n", - "\n", - "best_valid_loss = float('inf')" - ] - }, - { - "cell_type": "code", - "execution_count": 115, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsIAAAHwCAYAAACsSAniAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd5wU9f3/n5+7A066FEHKCSoqCNxxwKFixyg2sEbArhGjv6hJjDGmqLF9LZioSWxJjEYFQUmUpqIRRRTpRZqiCNKkSYeTu9vP74/Zfju7MzuzO7O776cP3DKf+cx7ZnZvX/Oed1FaawRBEARBEASh0Cjy2gBBEARBEARB8AIRwoIgCIIgCEJBIkJYEARBEARBKEhECAuCIAiCIAgFiQhhQRAEQRAEoSARISwIgiAIgiAUJCKEhZxDKVWslNqjlCpLY90jlVJSM1AQBEEQBBHCQuYJitbQv4BSan/U68vtzqe1rtNaN9Vaf5sJewVBEARBKAxKvDZAyH+01k1Dz5VSq4GfaK3fNxuvlCrRWtdmwzZBEARBEAoX8QgLnqOUekApNVYpNUYptRu4Qil1vFLqM6XUDqXURqXUU0qpBsHxJUoprZTqEnz9SnD520qp3UqpmUqprha33UkpNUkp9b1SaqVS6rqoZccppeYrpXYppTYppR4Lvt9YKTVaKbUtaN9spVQb1w+MIAiCIAgZRYSw4BcuBEYDLYCxQC1wG9AGGAgMBm5Msv4I4A9AK+Bb4H6L2x0LfAN0AC4DHlVKnRJc9hfgMa11c+BI4I3g+9cCjYFOQGvgZqDa4vYEQRAEQfAJIoQFvzBDaz1Rax3QWu/XWs/RWs/SWtdqrVcBzwOnJFn/Da31XK11DfAqUJFqg0GvcRXwG611tdZ6PvAv4MrgkBqgm1KqtdZ6t9Z6VtT7bYAjg/HKc7XWe9LbbUEQBEEQvEKEsOAX1ka/UEodo5SarJT6Tim1C7gPQ3ya8V3U831AU7OBUXQAtmqt90a9twboGHx+LdAD+CIY/nBO8P0XgfeBcUqp9Uqph5VSEm8vCIIgCDmGCGHBL8SXNHsOWILhdW0O3A0ol7e5AWijlGoS9V4ZsB5Aa/2F1noYcAjwODBeKVWqtT6gtb5Xa90dOBEjrMN29QtBEARBELxFhLDgV5oBO4G9SqnuJI8PTgut9TfAXOAhpVQjpVQFhhf4FQCl1JVKqTZa60DQFg0ElFKnK6V6KqWKgF0YoRIBt+0TBEEQBCGziBAW/MrtwNXAbgzv8NgMbecyoBtGaMUbwG+11h8Gl50DLA9WshgFXKa1PoARUvEfDBG8FCNMYnSG7BMEQRAEIUMoraXJliAIgiAIglB4iEdYEARBEARBKEhECAuCIAiCIAgFiQhhQRAEQRAEoSARISwIgiAIgiAUJCKEBUEQBEEQhILEs25Ybdq00V26dPFq84IgCI6YN2/eVq11W6/tyBbyN1sQhFzG7G+2Z0K4S5cuzJ0716vNC4IgOEIptcZrG7KJ/M0WBCGXMfubbTk0QilVrJRaoJSalGBZI6XUWKXUV0qpWUqpLumbKgiCIAiCIAiZx06M8G3AcpNl1wPbtdZHAn8GHnFqmCAIgiAIgiBkEktCWCnVCTgX+IfJkKHAS8HnbwCDlFLKuXmCIAiCIAiCkBmsxgg/AfwaaGayvCOwFkBrXauU2gm0BrY6tlAQBEEQhIKnpqaGdevWUV1d7bUpgo8pLS2lU6dONGjQwNL4lEJYKXUesFlrPU8pdaoT45RSI4GRAGVlZU6mEgRBEAShgFi3bh3NmjWjS5cuyE1nIRFaa7Zt28a6devo2rWrpXWshEYMBIYopVYDrwGnK6VeiRuzHugMoJQqAVoA2xIY+LzWup/Wul/btgVTdUgQBEEQBIdUV1fTunVrEcGCKUopWrdubeuuQUohrLW+S2vdSWvdBRgGfKC1viJu2ATg6uDzS4JjtGUrBEEQBEEQUiAiWEiF3c9I2p3llFL3KaWGBF/+E2itlPoK+CXwm3TnFQRBEARBEJJz6qmnJqzt/cQTT7Bv3z7b89199928//77lse/+OKL/OxnP0u47JxzzmHHjh2m66ZrYyawJYS11h9qrc8LPr9baz0h+Lxaa32p1vpIrXWV1npVJowVBEEQBEEoBLTWBAIB2+slE5l1dXWm6913332cccYZtreXiClTptCyZUvT5ekI4WS2OyFtj7AgCIIgCEIhcf/993P00Udz4oknMnz4cEaNGgXA119/zeDBg+nbty8nnXQSK1asAOCaa67h1ltv5YQTTuDwww/njTfeCM/12GOP0b9/f3r37s0999wDwOrVqzn66KO56qqr6NmzJ2vXruWmm26iX79+HHvsseFxZjz11FNs2LCB0047jdNOOw2Apk2bcvvtt1NeXs7MmTO577776N+/Pz179mTkyJGEIlmvueaasH1dunThnnvuobKykl69eoX3J54NGzYwePBgunXrxq9//evw+126dGHr1q3s3buXc889l/Lycnr27MnYsWMT2jhmzBh69epFz549ufPOO8PzRNv+4IMPcsEFF4SXvffee1x44YUWzlpyPGuxLAiCIAiCkA5/nLiUZRt2uTpnjw7Nuef8Y02Xz5kzh/Hjx7No0SJqamqorKykb9++AIwcOZJnn32Wbt26MWvWLG6++WY++OADADZu3MiMGTNYsWIFQ4YM4ZJLLmHq1KmsXLmS2bNno7VmyJAhTJ8+nbKyMlauXMlLL73EcccdB8CDDz5Iq1atqKurY9CgQSxevJjevXsntPHWW2/lT3/6E9OmTaNNmzYA7N27lwEDBvD4448b+9mjB3fffTcAV155JZMmTeL888+vN1ebNm2YP38+Tz/9NKNGjeIf/6jfSmLhwoUsWLCARo0acfTRR3PLLbfQuXPn8PJ33nmHDh06MHnyZAB27txJixYtYmzcsGEDd955J/PmzePggw/mzDPP5M033+SCCy6IsV1rTffu3dmyZQtt27blX//6F9ddd13yk2oB8QgLgiAIgiCk4JNPPmHo0KGUlpbSrFmzsHjcs2cPn376KZdeeikVFRXceOONbNy4MbzeBRdcQFFRET169GDTpk0ATJ06lalTp9KnTx8qKytZsWIFK1euBOCwww4Li2CAcePGUVlZSZ8+fVi6dCnLli2zZXdxcTEXX3xx+PW0adMYMGAAvXr14oMPPmDp0qUJ17vooosA6Nu3L6tXr044ZtCgQbRo0YLS0lJ69OjBmjVrYpb36tWL9957jzvvvJOPP/6YFi1a1Jtjzpw5nHrqqbRt25aSkhIuv/xypk+fXs92pRRXXnklr7zyCjt27GDmzJmcffbZto5FIsQjLAiCIAhCTpHMc5ttAoEALVu2ZOHChQmXN2rUKPw8FIagteauu+7ixhtvjBm7evVqmjRpEn79zTffMGrUKObMmcPBBx/MNddcY7uhSGlpKcXFxYBRgu7mm29m7ty5dO7cmXvvvdd0vpDdxcXF1NbWpty3ROOOOuoo5s+fz5QpU/j973/PoEGDwt5ou7YDXHvttZx//vmUlpZy6aWXUlLiXMaKR1gQBEEQBCEFAwcOZOLEiVRXV7Nnzx4mTZoEQPPmzenatSuvv/46YIjcRYsWJZ3rrLPO4oUXXmDPnj0ArF+/ns2bN9cbt2vXLpo0aUKLFi3YtGkTb7/9dko7mzVrxu7duxMuC4neNm3asGfPnpiY5UywYcMGGjduzBVXXMEdd9zB/Pnz69lYVVXFRx99xNatW6mrq2PMmDGccsopCefr0KEDHTp04IEHHuDaa691xUbxCAuCIAiCIKSgf//+DBkyhN69e9OuXTt69eoVvtX/6quvctNNN/HAAw9QU1PDsGHDKC8vN53rzDPPZPny5Rx//PGAkRT2yiuvxHg/AcrLy+nTpw/HHHMMnTt3ZuDAgSntHDlyJIMHD6ZDhw5MmzYtZlnLli254YYb6NmzJ+3bt6d///52D4MtPv/8c+644w6Kiopo0KABzzzzTEIbH374YU477TS01px77rkMHTrUdM7LL7+cLVu20L17d1dsVF71vejXr59OVP9OEAQhF1BKzdNa9/Pajmwhf7MFr1m+fLlr4idd9uzZQ9OmTdm3bx8nn3wyzz//PJWVlZ7aVGj87Gc/o0+fPlx//fWmYxJ9Vsz+ZotHWBAEQcgIWmvWbd9P51aNvTZFEFxh5MiRLFu2jOrqaq6++moRwVmmb9++NGnSJFwBww1ECAuCkHXW79jPwIc/4JnLKzm716FemyNkiD+/9yX/mPENs347iGalDbw2RxAcM3r0aK9NKGjmzZvn+pySLCcIQtZZsn4nAOPnr/fYEiGTnHbMIew7UMeERRu8NkUQBCEhIoQFQfAMpby2QMgkFZ1b0v3Q5oyZ/a3XpgiCICREhLAgCEKeoZRarZT6XCm1UClVL8NNKXWqUmpncPlCpZT1wp727GBEVWeWrN/F4nU7MrEJQRAER4gQFgQh63hUrKbQOE1rXZGkssXHweUVWuv7MmXE0D4dOahBsXiFBUHwJSKEBUHwDImMyH+alzbg/PJDeWvhBnZX13htjiDkDaeeeipulDSMnuecc85hx476d2/uvfdeRo0aVe/9a665JmFTjg0bNnDJJZeYbnPHjh08/fTTDqx2DxHCgiB4gLiEM4wGpiql5imlRpqMOV4ptUgp9bZSKqP9akcMOEyS5gTBJlprAoFAVrc5ZcoUWrZs6XieDh06JO1al44QNmvz7BQRwoIgZJ1QaIQky2WME7XWlcDZwP9TSp0ct3w+cJjWuhz4C/BmokmUUiOVUnOVUnO3bNmStjHlnVrQ/dDmjJ71LV41cRIEN7j//vs5+uijOfHEExk+fHjYS/r1118zePBg+vbty0knncSKFSsAw2N66623csIJJ3D44YfHiMPHHnuM/v3707t3b+655x4AVq9ezdFHH81VV11Fz549Wbt2LTfddBP9+vXj2GOPDY8z45133uHSSy8Nv/7www8577zzACzN06VLF7Zu3QrAgw8+yFFHHcWJJ57IF198YbrN6dOn19u/1atX07NnTwCWLl1KVVUVFRUV9O7dm5UrV/Kb3/yGr7/+moqKCu644w601txxxx307NmTXr16MXbs2LD9J510EkOGDKFHjx7cfffdPPHEE+Ft/+53v+PJJ59MekxSIXWEBUHwDCXBERlBa70++LhZKfVfoAqYHrV8V9TzKUqpp5VSbbTWW+PmeR54HozOcunaE0qa+8NbS/l8/U56d3LucRIKnLd/A9997u6c7XvB2Q+bLp4zZw7jx49n0aJF1NTUUFlZSd++fQGj0cazzz5Lt27dmDVrFjfffDMffPABABs3bmTGjBmsWLGCIUOGcMkllzB16lRWrlzJ7Nmz0VozZMgQpk+fTllZGStXruSll17iuOOOAwxB2qpVK+rq6hg0aBCLFy+md+/eCW0844wzGDlyJHv37qVJkyaMHTuWYcOG2Z5n3rx5vPbaayxcuJDa2tqYfY0n0f5F8+yzz3Lbbbdx+eWXc+DAAerq6nj44YdZsmQJCxcuBGD8+PEsXLiQRYsWsXXrVvr378/JJxvX7/Pnz2fJkiV07dqV1atXc9FFF/Hzn/+cQCDAa6+9xuzZs03PmRXEIywIgpBHKKWaKKWahZ4DZwJL4sa0V8rwxyulqjB+C7Zl0q5Q0tzoWZI0J+Qmn3zyCUOHDqW0tJRmzZpx/vnnA0bb5U8//ZRLL72UiooKbrzxRjZu3Bhe74ILLqCoqIgePXqwadMmAKZOncrUqVPp06cPlZWVrFixgpUrVwJw2GGHhUUwwLhx46isrKRPnz4sXbqUZcuWmdpYUlLC4MGDmThxIrW1tUyePJmhQ4fanufjjz/mwgsvpHHjxjRv3pwhQ4aYjk20f9Ecf/zxPPTQQzzyyCOsWbOGgw46qN6YGTNmMHz4cIqLi2nXrh2nnHIKc+bMAaCqqoquXbsChse6devWLFiwIHz8WrdubWqbFcQjLAhC1pGb4xmlHfDfoM4tAUZrrd9RSv0UQGv9LHAJcJNSqhbYDwzTGY5ZCCXNTVi0gd+d2106zQnOSOK5zTaBQICWLVuGvZvxNGrUKPw89DXTWnPXXXdx4403xoxdvXo1TZo0Cb/+5ptvGDVqFHPmzOHggw/mmmuuobq6Oqk9w4YN469//SutWrWiX79+NGvWLK15rJJo/6IZMWIEAwYMYPLkyZxzzjk899xzHH744Zbnjz4eAD/5yU948cUX+e6777juuuvSNzyIeIQFQfAMiRF2H631Kq11efDfsVrrB4PvPxsUwWit/xpcVq61Pk5r/Wk2bAslzb21UJLmhNxj4MCBTJw4kerqavbs2cOkSZMAaN68OV27duX1118HDDG4aNGipHOdddZZvPDCC+zZsweA9evXs3nz5nrjdu3aRZMmTWjRogWbNm3i7bffTmnnKaecwvz58/n73/8eDouwO8/JJ5/Mm2++yf79+9m9ezcTJ05MuV0zVq1axeGHH86tt97K0KFDWbx4Mc2aNWP37t3hMSeddBJjx46lrq6OLVu2MH36dKqqqhLOd+GFF/LOO+8wZ84czjrrrLTtCiEeYUEQBCErRCfNXT6gDCVXQkIO0b9/f4YMGULv3r1p164dvXr1okWLFgC8+uqr3HTTTTzwwAPU1NQwbNgwysvLTec688wzWb58OccffzwATZs25ZVXXqG4uDhmXHl5OX369OGYY46hc+fODBw4MKWdxcXFnHfeebz44ou89NJLac1TWVnJZZddRnl5OYcccgj9+/dPuV0zxo0bx8svv0yDBg1o3749v/3tb2nVqhUDBw6kZ8+enH322Tz66KPMnDmT8vJylFI8+uijtG/fPpx0GE3Dhg057bTTaNmyZb3jlQ7Kqwzefv36aTfq3wmCkHtM+XwjN786n7N7tueZKxInYPgdpdS8JM0q8g63/ma//Nka/vDmEib8bKAkzQm2WL58Od27d/fUhj179tC0aVP27dvHySefzPPPP09lZaWnNhUagUCAyspKXn/9dbp165ZwTKLPitnfbAmNEAQh62Tr+vtAbYA/vLmE7/ceyM4GhZQMreggSXNCzjJy5EgqKiqorKzk4osvFhGcZZYtW8aRRx7JoEGDTEWwXSQ0QhCErKOD6XKZvjO+cvNuXv5sDQOPbMPgnu0zuzHBEs1LGzCkvIMkzQk5yejRo702oaDp0aMHq1atcnVO8QgLguAZma4jHPE8S50KPzF8QJkkzQmC4AtECAuCkHWynZogzcz8hXSaE9JFPi9CKux+RkQIC4LgHRkOjZDfTH+ilGLEgDKWbdzF4nU7vTZHyBFKS0vZtm2biGHBFK0127Zto7S01PI6EiMsCELeIz+b/mNoRQcemrycMbO/pbyzVI8QUtOpUyfWrVvHli1bvDZF8DGlpaV06tTJ8ngRwoIgZJ1sCdNQUp44kPxH1pPmPn8DGjaFo86STi45SoMGDcKtdgXBLSQ0QhAEz8i0HAkJYC0+YV+StaS5j/8E46+HMZfBMwMNURyoy+w2BUHICUQIC4KQt4j89TflnVrQI9NJc588Bf/7I/S6FC54FgI1hij+a3+Y/zLUSo1pQShkRAgLgpB1QqIn0y12Q9uR0Ah/opRieCaT5mb+Dd77Axx7kSGCK4bDzbPgx/+Ghk1gws/gqT4w6zmo2e/+9gVB8D0ihAVB8IxsRWqKDvYvFwQ7zY2Z7XKnuVnPwbu/hR5D4aK/Q3EwJaaoyHjvxulw+RvQohO8/Wt4ohfM+DNU73LXDkEQfI0IYUEQ8paQAJZyS/6lWVTS3O7qGncmnf13Q9wecx5c/M+ICI5GKej2I7juHbhmMrTvBe/fC0/0hGkPwb7v3bFFEARfI0JYEISsky1dKvo3N3A1aW7uv2DKr+Doc+CSf0FximoUSkGXE+HK/8INH0CXk+CjR+DPPWHq72H3d85tEgTBt4gQFgTBMzJfxUqUcC7gWtLc/Jdh0s+h21lw6YtQ0tDe+h37wrBX4aaZcMw5RozxE71h0i9h+5r07RIEwbeIEBYEIW8Jl08TPexrXEmaWzgaJtwCR55hJMOVNErfoHY94OJ/wM/mQvkwmP9v+Esl/Pcm2PJl+vMKguA7RAgLggvMW7OduoCoLatku66v1BH2P6GkudGz0kiaWzQW3rwZDj8VLnsFGlhvr5qU1kfAkKfgtkXQ/wZY+l/4WxWMuwo2LnZnG4IgeIoIYUFwyPxvt3PxM5/y1P9Wem1KzpFuZITWmv97ezkrvkue4S/yN3dIO2nu8zfgzZ9C15Ng2GhocJD7xrXoCGc/DD//HE78BXw9DZ47CV69FL6d5f72BEHIGiKEBcEhm3ZWA6QUZYJ77Nxfw3MfrWL4858lHSehEbnFiAFl7K+p402rSXNL/wv/GQllJ8Dw16Bh48wa2LQtnHGPIYhP/z2smwsvnAkvnmeIY/mgCULOIUI4D5i4aAMLvt3utRkFSyjhS34DrRM6Vk4baqSKRpGGGrlFbztJc8smwBvXQ+cqGDHWaJCRLQ5qCSffAb9YAmc9BNu+gpcvgH8MghWTIRDIni2CIDhChHAe8Mg7K3jlM5eL0Qs2MMScaC3rhIVwprcT9yj4G6UUIwaUsXzjLhYlS5pbMRneuNao8nD569CoafaMjKZhEzj+/xkxxOc9AXu3wmsj4NmBRshGoM4buwRBsIwI4TwgENDSMMBDMl8CTDBDPvf5x9BQpzmzpLkv3oFxV8OhFXDFeGjULLsGJqKkEfS7Fm6ZDxc+bwjg8dfDX/sZFSdqD3htoSAIJogQzgM04vHykpAOFk1mHaeHSln0JUdihOXk5ApJk+ZWvgfjrjS6wF35Hyht7o2RZhSXQPllcPNn8OOXDZE+4RZ4qgI+exYO7PPaQkEQ4hAhnAdoLT/0XhKJc5VzYJsMe9NDZdPkzOQWCZPmvvofvHY5HNI9KIJbeGdgKoqKoMcQGPkRXD4eWpbBO3fCk73h4z9BtSTWCoJfECGcB2ipkuoL5Fok+6Q85BIknJPUS5pb9aERe9v2KLjyTTjoYK9NtIZS0O0MuO4duPZtaN8b/vdHeLIctkq5RUHwAyKE8wDDI+y1FYWL+IPt4/gOhkVPspyT3CQ6ae7rOe/A6GHQ6gi48i1o3Mpr89LjsBMMT/YNHxgxxJNvlz/cguADUgphpVSpUmq2UmqRUmqpUuqPCcaUKaWmKaUWKKUWK6XOyYy5QiLkT6m3RMqnyZmwi9VYX6fIPZPcY2hFB05q+CVlb18DB3eBqydAk9Zem+Wcjn1h0B/gm49gyXivrRGEgseKR/gH4HStdTlQAQxWSh0XN+b3wDitdR9gGPC0u2YKydBafua9RKpG2Mfx59XiBNJQI3dptnke/yh5hLWBVuweNh6atPHaJPfod51R9eLd30m8sCB4TEohrA32BF82CP6L/1nRQCh9twVgsS2Q4AaSLOctSuoI2yfcUMOdecwXS7JcTrJ2DrxyCbrZoQz74Xe8ubLWa4vcpagYzv0T7NkEH/6f19YIQkFjKUZYKVWslFoIbAbe01rHN1e/F7hCKbUOmALc4qqVQlKkfJrHSGe5tElXB1u9ByIe4Rxk3Tx45SJo2pZG103ikA6HWes0l2t06gt9r4FZz8F3n3ttjSAULJaEsNa6TmtdAXQCqpRSPeOGDAde1Fp3As4BXlZK1ZtbKTVSKTVXKTV3y5YtTm0XgmitRQn7ADkF2SMscFONy7glgqtsWAAvX2gkxF09CdWiI8OrLHSay1UG3W20a558u7RlFgSPsFU1Qmu9A5gGDI5bdD0wLjhmJlAK1Avo0lo/r7Xup7Xu17Zt2/Qs9hlTl37Hpl3VntpgeITlJ98rJEQ4+9j9tMv3IwfYuAj+fQEc1AKungQtOgJG0lzjhsWMnrXGYwMzQONW8KP7YO0sWDTaa2sEoSCxUjWirVKqZfD5QcCPgBVxw74FBgXHdMcQwnnv8tVac9Or8xk3Z63HdsitXy8JNdTIu1u3GcQtYZrqmIeWy6nxOd99Dv8eanRiu3oStOwcXhTqNDdx0UZ2xXeaywfKR0Dn4+C9u2Hf915bIwgFhxWP8KHANKXUYmAORozwJKXUfUqpIcExtwM3KKUWAWOAa3QBqAKtoS6gqQl4u6uB/D/UvkY8wumTbrKc1T8v0k8jB9i0zBDBDRrD1RPh4MPqDRleZXSae2vBeg8MzDBFRXDu47B/B/zvPq+tEYSCoyTVAK31YqBPgvfvjnq+DBjormn+J/zj6rUQFY+wL5BzkD0sH2o5J/5m8wp46XwobmiI4FZdEw7r3akFx3ZozquzvuWK4w6LamueJ7TvCQN+Cp89DX2uNBLpBEHICtJZzgW8/q2VGGFvCTfUkHNgGbcuGlIny0nZCN+y5UtDBBcVG+EQrY8wHaqUYnhVGSu+252fSXMAp/4GmraDyb8wOs8JgpAVRAg7wC/RH1pr+Z33kGx1R8snQh/XdI+d3c+7fD18xravDREMhghuc2TKVfI6aQ6gtDkMfshIGpz7gtfWCELBIELYAeH4Q+8jI+SH3gd4/TnIRdKOEZY6wrnL96vgxfMgUGu0TW57lKXV8j5pDuDYi+DwU+F/98OezV5bIwgFgQhhB0RqmXqfLCc/9N6hpKGGbRwfK5stlgWfsH01vHg+1FYbIviQ7rZWHzEgj5PmwPhjcs4oqNkHU//gtTWCUBCIEHYBr39sje3LL75XhJyaXl8Q5SJOc55Sffcid23k3HjO9jWGCK7Za4jgdsfanqJXx0jSXN6e0zbdYOCtsPg1WD3Da2sEIe8RIeyAkPDx+s+x19sveCREOOtY/cyH6whnzhTBCjvWGjHBP+yEq96C9r3SmiY6aW7h2h0uG+kjTvoVtCiDyb+CujwNAxEEnyBC2AG+iT+U8mmeEkr4knNgHafec7thSXJuPGTnekME798BV74Jh5Y7mi6UNDdm9rcuGehDGjaGsx+BLcvhs2e8tkYQ8hoRwnmAlpvynhIpnybYx5k73WpohOARuzYaInjfNrjyv9Cx0vGUBZE0B3DMOXDU2fDhw8bFhCAIGUGEsAt4LUONFsvyk+85cgqyhu2qEdzK6RwAACAASURBVBm0RUjCnL/Dnk1wxXhXm0TkfdJciLMfBl0H797ltSWCkLeIEHZAWHt6/Csb0F5L8cLGjWS5Jet3su9ArTsG5QCh7076LZYtjwyOl2+IJ5z2O7hhGnSucnXagkiaAzi4C5z8K1j2Fnz1vtfWCEJeIkLYAX5Klsvn3wK/E2r3mu45qK6p46KnP2X8vHUuWuVvIg01nK2fcpx8L7ylqNhynWA7KKUYMaAAkuYATrgVWh8JU+6AmmqvrRGEvEOEsAMiyXI+CI3w1ILCxun5rw1oDtQF2HuggNqquvSdkc994TKkvACS5gBKGsE5jxmNSD550mtrBCHvECHsAL90ljNs8IERBYrTCJlwia8COoWO+2lYPFh++o4K7tKstAFDKwogaQ7giNPh2Avh48cNQSwIgmuIEM5xRAD7h3TPRURIF865zFaMsF+6PwqZYXhVgSTNAZz1EBQ3gLfvlCs7QXAREcIO8EOxfvl76D1OKxMU4jl07QIuZfm0wvO2FxIFkzQH0LwDnHoXrJwKKyZ7bY0g5A0ihB3gh9uugQK8re43HHsb/dKYJYtka1cL6ZgWIgWVNAcw4EY45FjDK3xgr9fWCEJeIELYAX647VqIt9X9SrqiqxDPXTg0Is26EXaPdeEd4cIhlDQ3elaeJ82BERpx7uOwax1Mf8xrawQhLxAh7AJeep180+a5kHEpNCLvb+1G4ThZzmpDjdBj4RzagiOcNLd4Q/4nzQEcdjxUXA6f/gW2fOG1NYKQ84gQdoIPflwlBtJ7wofeabJcAZ3DkOhPN1kuPE+KL2Ekjr+ADm4BMryqjOqaAG8WQtIcwI/ug4ZNYfLthfWHQxAygAhhB/jhx9UP4RmCgePyae6Z4nsioRHO1hcNIAD07tSSnh2bM7oQkuYAmrSBQXfD6o/h8ze8tkYQchoRwg7wwy3tQvib73ecirKC9Ag7lP12O8sV0rEtVIZXGUlzCwohaQ6g7zXQoRKm/g6qd3ptjSDkLCKEXcDT8mkSGuE5jkVdAZ67LFVPEwqIcKe5QkiaA6N99bmPw57NMO0hr60RhJxFhLAD/ODJc1rDNpvcOmYBd7+1xGszMka6gjh8MZMTZ9EdQnuq0gwStt5ZLnShWDjHtlApuKQ5gI6V0P96mP08bFzstTWCkJOIEHaAHxJxdL0n/mX1tr2s3rbPazNcx/Ht9wK8fe90X+2GRgiFQcElzQGc/ns4qBVM/iUEAl5bIwg5hwhhB/jDI+y9GLeK1vnpmXNeCsydeXKJbIWTSIxwYVFwSXMABx0MZz4A6+bAwle8tkYQcg4RwjmOH8S4VXJBrDsh7WS5XDqJLuFajHCKiQrxIqPQKbikOYDyYVB2Arx3D+z73mtrBCGnECHsAD/E5+qA9zZYJRCItITOJ5yWP4vECAvWsXe08vBjJ5gwtKJjYSXNgVGQ+9zHjeoR79/rtTWCkFOIEHaAHyo25JKXVZOfgiTi0E0zWS4Pj0kqAoFshUYU4MEtcJo2Kgknze3cXyBJcwDtesBxN8H8l2DtHK+tEYScQYSwE3yQqeaHWsZW0VoXpOhLRQFGRkRVjXBnnlTLc+mCUXDOiKrDqK4J8NbCAkqaAzj1N9Csg5E4V1frtTWCkBOIEHYBbz3CsY9+Jx9DI5xWfcilhEe3iHSWS7N8ms2B+fixE8zp1alF4SXNATRqBoMfgu8Ww9x/em2NIOQEIoQd4AdPXlhE5cDf+oDOT6nnVgWEXDiHbpGtYybx14VLQSbNAfS4AI44HT54AHZv8toaQfA9IoQd4AfhkkseYa3JDUNtEkmazE7b4HzAeR3hQjpaQjqEkuZGF1LSHBjxRueMgtpqmPp7r60RBN8jQtgBfugIFsghd6Khg/1vZ7o4LZ+WA6fQNZLt6p1vLOa12e6Il0IsTScYhJLmJhVa0hxA6yNg4M/h83HwzcdeWyMIvkaEsAN8IWB8UMLNKlprHBYL8CVOy+j54YIq6ySJi/7oyy3MXbPdyuqpNxP3KBQWBZs0B3DSL6HlYTD5dqg94LU1guBbRAi7gA90cE4gneVM1s+/Q5KSZLusSV1dRDrLCVYo2KQ5gAYHwdmPwtYv4LOnvbZGEHyLCGEH+OGuay790OdpiHCYtOsI13uS/yT73Grtnne8oLzsQkJGVB1WmElzAEcPhqPPhY8egR1rvbZGEHyJCGEH+KHsVS7dVs/f0AiHneUcrp+LJCujp8P/M8fu5z0Xvh9CZhhS0YEmhZg0F+Lsh42ry3fv8toSQfAlIoQd4AcvbCDHPMI5YahNnHp0nXamy0VS7WmqetMSGiFYpWmjEoZUdCzMpDmAlmVwyh2wfCJ8OdVrawTBd4gQdgOpI2yJfK0jHCJ9j3DsYyGQrCOiERrh0nbiHoXCZERVGdU1Ad5cUIBJcwDH3wJtjoK374Ca/V5bIwi+QoSwC3iaLJdTVSPyU+w5b3NdeKERyRtduNiKOx8/cBZQSq1WSn2ulFqolJqbYLlSSj2llPpKKbVYKVXphZ3ZIpQ0N2Z2ASbNAZQ0NGoLb18NM57w2hpB8BUihB3gXAC5hx9sSIXWedpi2aUuaQVFkn224hG2XT6tEI8xnKa1rtBa90uw7GygW/DfSOCZrFrmAaFOcwsLMWkO4PBToOclMOPPsO1rr60RBN8gQtgBfmjfmms/8Llmrx2c+YPz+9jEk2yfNRZihCVZzilDgX9rg8+AlkqpQ702KpMMKe9A44bFjHGpWUtOctaDUNwQptxRWH9wBCEJIoRdwNPyaTn0A5+vMcJOY3zdatGcSySruKJdDBKOdJZzZ74cQgNTlVLzlFIjEyzvCETX01oXfC9vaVbagCHlHZi4aCO7qgswaQ6gWXs4/Xfw9f9g+QSvrREEXyBC2AF+uKDOqaoR0lDDZP3cSXh0i2T7atSbdqtqRAEd1FhO1FpXYoRA/D+l1MnpTKKUGqmUmquUmrtlyxZ3LfSA4VVl7K+p462FG7w2xTv63wDtesE7d8EPe7y2RhA8R4SwA/zgbPJDLWOrWOkYlsukew7y+ZiYkewCLlFS5cK1Oxj17heRMRa344fvqBdordcHHzcD/wWq4oasBzpHve4UfC9+nue11v201v3atm2bKXOzRu9OLeh+aHPGFGKnuRDFJXDu47BrvdFoQxAKHBHCDoiULvOyoUbIFs9MsIybHcP8hGuhEblwEl0iWXy90XgldskFf/uEv077KmaMpe0U4LFVSjVRSjULPQfOBJbEDZsAXBWsHnEcsFNrvTHLpmYdpRQjqjqzbOMuPl+/02tzvKNsAPS50mi9vHm519YIgqeIEHYBPyTL5cLPfCBfy6c5DG3wQ9JltkkZGuHywcjHz10S2gEzlFKLgNnAZK31O0qpnyqlfhocMwVYBXwF/B242RtTs8/QPh0pbVBU2ElzAGf8ERo1g9GXwaKxUFfrtUWC4AkphbBSqlQpNVsptUgptVQp9UeTcT9WSi0Ljhntvqnp8fWWPVQ9+D6bdlW7Preu98QLvPdKW6e+py8fcLpLeXhILJNw35PkytltR12Ih1ZrvUprXR78d6zW+sHg+89qrZ8NPtda6/+ntT5Ca91La12v1nC+0ry0Aef37sBbCzew54cCFn9NWsNlr0DDpvDfkfC3Klg4WgSxUHBY8Qj/AJyutS4HKoDBwVtpYZRS3YC7gIFa62OBn7tuaZqs2baXzbt/YP0O97vp+CHbP5c8wm52DMtHCkkQRy7cElSNwPxY2A1DsSuchcJgWFUZ+w7UMXFRASfNAXQ5EX46IyiIG8ObN8Ff+8L8l6GuQCtrCAVHSiEc9ByEUksbBP/F/67cAPxNa709uM5mV610QGZjBL3/eQ14b4JldPh/+UUkTttZslw+xk+bkWxPjeOYeES6dxQK6SJDSE1lWUuObtdMwiMAioqg+/lw48cwbAyUtoQJP4O/VMK8F6H2gNcWCkJGsRQjrJQqVkotBDYD72mtZ8UNOQo4Sin1iVLqM6XUYLcNTZfQD2AmBaMv6gjnwA+93+sIf75uJw9NWW5b0Dr1OhZy+TTzhhom6yV4Zmk7vv7kCdlGKcXwqs4sXreTJYWcNBeNUnDMOTDyQxgxDhq3gYm3GYJ4zj+h9gevLRSEjGBJCGut67TWFRgldqqUUj3jhpRgtOo8FRgO/F0p1TJ+Hi9qUoY8SJkQGU6rBbhqg3cmWMbvLZb/t2ITz09fRV2aV03OG2oUDsnEf7J605n8PguFxYV9OtGoRJLm6qEUHHUW3PABXD7eaMIx+ZfwVB+Y/XeocT/fRhC8xFbVCK31DmAaEO/xXQdM0FrXaK2/Ab7EEMbx62e9JmXo9zITAiwS5eiDGOEcUAZa+7uOcCDgjmfX/nrBRx8fG7dJXjXC/EjavWgoRG+7YI0WjRtwbq9DeWvhBvYdkASxeigF3c6A69+DK/8LLTrDlF/BUxUw6zmocT/vRhC8wErViLYh765S6iDgR8CKuGFvYniDUUq1wQiVWOWqpWkSCY3IgBD2g0c4h0pvGZGf/rU03S59zqtGmCeO5SvhY52wxXLqZDmriAAWkjF8QBl7fqhl0qK8L6GcPkrBEafDde/AVROg1RHw9q/hyXKY+Tc4sM9rCwXBEVY8wocC05RSi4E5GDHCk5RS9ymlhgTHvAtsU0otw/AY36G13pYZk+2SwdAIH4hQP4hxq2gNgYDXVphTFzyI2U7IyoFTlwGShEZgfg7shkY4TWQU8pt+hx3MkYc0ZbSER6RGKTj8FLh2MlwzGdoeDe/+Fp7sDZ88BQf2em2hIKRFSaoBWuvFQJ8E798d9VwDvwz+8xUhz5OfY1OdkEvJQH4XI4F0Y4MdXhDl0sWMWyQNcUgaNhFa316ynCAkwkiaK+P+SctYvnEX3Q9t7rVJuUGXE41/a2YabZrf+wN88gSccAv0vwEaNfXaQkGwTN53lstk1Qg/CJh4AXygNsC4uWvTFnWZRONvMRxI0yPs/HNQeHGsKWOETZan7a1Pay2hELioT0caFhfxmniF7XPY8XDVm0YccYc+8P698EQvmD4Kqnd5bZ0gWCL/hXAGO69FpvRDspzx+OxHX/PrNxYzwYeF4v3eUKMuGLbhVNDaXiuHvPpukbJqRNyxUCq4LHSOXNiOIAAc3KQhZ/dqz38WrGf/gTqvzclNOlfBFePhJ/+DTv3hg/sNQfzRo1At5ekEf5P3QjjdBCgr+OFHNr7iwPd7D8Q8+omA9neLZa88woVcNSJhshzmseR2v3OFeJEh2Gd4VRm7q2uZ/LkkzTmiUz+4fBzcMA0OOwGmPQh/7gXT/g/2b/faOkFISN4LYZ2muLG1jYzNbGHbcftVUmS4ztKthZtJjNAIr60wJ5yIZXM9p7tUmHWEkyzT9QuoBR3CSatNCEK6DOjaisPbNJGawm7RsRKGj4Ebp0PXk+Cjh+GJ3vDBA7Dve6+tE4QY8l4Ih8hsjLB3P8qBOBuKg0K41odCGN+HRjjz8KefLOfno5IZklWMS3TBpIKxEWl3/Su8QyzYQCnFsKrOzFuznS837fbanPzh0HIY9ir8dAYccRpMf8wImXj/j7DXJ4WlhIIn74VwJusI+4NYL2ZRUAhnan+NphjpV1fw82mIv6iwSkRsOUvk8vOxcZtklTYSxZLHe4TtXnUU0KEV0uTiyk40KFbiFc4E7XvBj/8NN82EbmfCjD8bgvi9u2FPdrrMCoIZeS+Es9Ji2f2p7dsQfAyFRtTWuW9VIKDpetcUHn47vp+KxfW1v72fgTQ9wmlqs8j6BXi7P9Uxjv+chJPlbIav+KGyi5AbtG7aiLOObc9/5q+nukaS5jJCux5w6b/g5s/gmHPg078YdYjf/R3s3uS1dUKBkrKOcK6TjfAFPyTLhSgKKoa6DBgVuqh4bvoq7jqnu+31jdhP/5J2Qw2HOxU+Kn4+OC6Tyotu2lnO7nZsjhcKmxFVZUxavJG3l2zkwj6dvDYnfznkGLj4H3DKnUaptc+ehtnPQ2lLry0zaNIGev8YykdAs3ZeWyNkmPwXwsHHjMQI+6mzXNCKkEc4E3WEnU5pxH76V5qkmywXIu1dKzwdbOpFN/P4KhSg7XeWC48rpKMrpMtxh7fmsNaNGTN7rQjhbNCmG1z0HJzya5j3Ivzgk/jsLSuMmsj/ux+OOgv6XGmEdBTnvWQqSPL+rGayaoQfkuXik4GKizOXLOfUn6t1Zi5I3CJ08WC7fJrDWtWRGGEfHxyXMfX4pvhO2Q0j8UOJQyF3KCpSDOtfxiPvrOCrzXs48hDpkJYVWh8BZ97vtRWxbF0JC16GhWPgiynQtB2UDzdEcZsjvbZOcJG8jxHOZLKcH35bI+WkDIpDoRFmhVgd4OQQOk0oywZpJ2I59ZQXskdYm7wfv0IwRjjdmH8ff+wEn3FJ306UFCnpNFfotOkGP7oPfrkMho2Gjn2NmOa/9oUXzoYFr8KBvV5bKbhA/gvhPPcIxe9fcbiOcAa25UgIBx/dMSUjhGKEHUY4pLGen49KZjALgYjcwYl9X4WX292OfduEwqZts0aceWw7xs9fxw+1kjRX8BQ3gGPONeoi/3IZnHEv7NkEb90Mo46GCbfCurnyxyaHyX8hnMGMfF/UKI2LgSzOYPk0J3PmQqhm+qER8U/sUYiVDcx2NXIszapGJF/fbL5CvNgQ0md4VRnb99Xw7lKpZCBE0aw9nPgLuGUeXPs2dD8fPn8d/jEInj4eZv5N6iPnIHkvhEOepQxECvjiRzb+FnOkoUYGQiOcrJuFDn9OcXrb3akn2b9HJgOYxAKbHUtF7AWe5RAbP1ysCjnHwCPa0LnVQYyZJeERQgKUMlpIX/gM3P4FnP8kNGwC7/4WHj8axl0FK9+HgNxRyAXyP1mObCTLuT61bRtCZDI0wskxjI9l9iOhY5buftqNf37lszXU1AXo2qZJWuvnMmYVOlKFMtk9QgV5kSE4JpQ099i7X7Bqyx4ObytJc4IJpc2h7zXGv03LYMErsPg1WPYWNO8IFSOgzxVwcBePDRXMyHuPcDbEqpf6JV5QlBT5NFkuB2K1rYa6TFy0gf97e3lkvTTL6P3+zSX8ceKyghRrqapGxF+MqPhkuUwZJghBLu3bieIixdg5a702RcgV2vWAwQ/BL1fApS/BId2NOslPlsNL58Pi16Gm2msrhTjyXwgHHzNzS977n+NIaITxTKkMlk9zEiOcweodblFnUQjfMmYBz320Kvza8S4VoBJO1UQklVC2eqz8cNdGyE0OaV7KGd0P4Y156zhQm4FbbEL+UtIQjr0ArhgPv1gCp/0Otq+G//wEHj8KJv8KNi7y2kohSP4L4Qx6kPzQGtds/zLRUMMNMeFnPVIXarFs0cqd+2tiXqd7fCIeZT8fHXcx++6YxwiHlts7VoV4bAX3GFZVxra9B3hvmSTNCWnSopPRMOTWRXDVBKMxx/x/w3Mnw7Mnwey/w/7tXltZ0BSAEDYeM1lH2A8tluM9ZXUZsMlZjLC9IOFAQPPk+yvZvvdA2tu0i1XvYeg2/brt+4zxofXTFFuF6K00rxqROBkudKfD7vWdXQ+yIERzcre2dGx5EGOkprDglKIiOPwUo7X0r76Ac0YZ70/5lVGG7Y3rYdWHmcnsF5JSAEI4lCyXibmDj+5PbcOI4EOcpywjMcJO1rXpPZ/x1Vb+/P6X/P7NJQ62ao86i+XTOrQ4CIB12/cbbzisTFCIt+/N9tns/bBH2GaseQEdUiEDFBcpLuvfmRlfbWXNNmmeILjEQQdD1Q3w04/hxunQ92r46j3491B4qgI+ehR2rvPayoIh/4Vw6DGj2XKZmzr1pmNDI0KCv86B8v9s1Ta6/GYyX26K7fvuLFnOwKpZofJvew/Upr9Rm1htqHFoi1IgSggHSffw+OHOQvZJLGhNveuhZLng9Z3tEnf2hgtCmB/360yRQpLmhMxwaDmc8xjc/iVc/E+jusS0B+HPPeGVi2HuC7B5uXiKM0jel0+L1BHOXPKYl/GH4e9GnCfNiRAO/cFfvG4nR7VrFn4/+mKipi5Ag2Lr11F2WyyH6sZmUxxatbH5QQ2A+qER6TfU8P5zlG3Mk+GSe3ztHqOIh7lwjq3gLu1blHL6MYcwbu46fvGjo2z93RMEyzQohV6XGP+2rzZaOC8aA1+9bywvbQllxwX/HQ8d+kBJI09NzhfyXghnNDTC/SltE29DSCg4qRqxaZdR3uWQZrFfsugZq2vqbP0g2K4jHOokZnkLzgkny6XYaCh0Yu33+y2NT0UheoTNPL9mIb3xLZatHqp0S9sJQjTDq8p4f/lc/rd8E4N7Huq1OUK+c3AXOP13cNpvYfs38O1n8O1M4/HLd4wxxY2gY2VEGHeuMkIuBNvkvRAOkYkfQj/EdsZXjXAjNCIkhEsbFMe8Hx07W10ToFmpjUltHqv4KgHZIHz3IKUQNh537jcS+Zx6dH0Ra55lzLr4mXlwI8ly9u4sFNRBFTLGKUe15dAWpYyevVaEsJA9lIJWhxv/KkYY7+3dCmtnRYTxp3+BGX82lh3SIyKMy46DFp0j2d2CKXkvhLNxa9TL39p6MdAutDLevPuHhHNEv6yusdc6Mlokaq3DwsaMVMszQcCioA2YeI7TP+SFp9ZMj1WKCyb7HuEU2xMEC5QUF/Hjfp156oOVrP1+H51bNfbaJKFQadIGjjnX+AdwYB9smB8Rxp+/YcQVg9HZLloYH9IDiorN5y5Q8lYIz1i5lW7tmobFTWbKp9n0TsUxb812GhYX0atTi/RtiBMGocdaB/XTdlfXxswN8NCU5eyLSlyzLYR17HM/XqRaDY2I1Bsm4aNd/HBnIduYHTOzUIb4znLpbk8Q0uXH/Q0hPG7uWm4/82ivzREEg4aNocuJxj+AQB1sXhYJp1gzE5aMN5Y1am6EUITjjCuN9QucvBTCdQHNFf+cxZGHNOWSvp2AzMQIp2iOlZKHpiynWWkJL15b5diIcL1kiyXArM0cmeOTr7bGdFeqrrGXwRptjxXLvNDJkdCIFB7huNvzbsUIF5JcMzt2qe7g2PXw2k3SFAQzOrY8iFOPasvYOWu5bVA3SiRpTvAjRcXQvpfxr+oG44/lzrWxccYfPBAc2wA6VETFGR8HTVp7a78H5KUQ3rbHuLX/3c5qXzfUqK0LOPLcQnQSWqwnLd1kuejY4ngvbl3UG9W1dkMjIgS0ptii1M2mfjELeag3zqTMWrpiqxA9wmaYeYqLVKiKiD3/uxxTwU2GV5Ux8uV5fLBiM2ce295rcwQhNUpByzLjX+8fG+/t+x7WzYkI41nPGbHGAG2Oig2nOLirP2/hukheCuFQjGvbZo1Mk3L8QEA7L5kVL6LiPcN22Rq8iIieCwzxFz2n09CIVHjxvbP6WQnEH3NiH+1SiJUNIsc4Pg498R2N0MfBdme5hFsRhPQ4/ZhDOKRZI16bs1aEsJC7NG4FR51l/AOoqYaNCyPCeNkEow00QNN20HkAdDnJKO3WuJV3dmeIPBXCRtWDtk0j5b8ycWvUabZ/QGsXbqvHThASEHVpTrx9X6SlcfzctTFC2F5oREyynI0jls3aupGGGsm3GYkldue2e67Uun3mw6/p2qaxK1nzZh3iUt1lses9t+lAFoSkhJLmnv7wK9bv2E/Hlgd5bZIgOKdBaaRGMRgNCrZ+ERHG386E5RPgvbsNr/KAG6Hdsd7a7CJ5GeS0aVfEI5zZOsLO7mkHtAvxpSZiPN2Qi+jmNdHHLOseYQ8aakTiq1OMMw2NsL6taNGbK1rtkXdW8NNX5rsyl9nn1kzoxifL2T1WhdSsRMgsl/XvjAbGSac5IV8pKoJDukO/6+Ci5+Hnn8NPPzFE8OKx8MwJ8OJ5sHyikZyX4+SlEN4cFMKtmzbMbIywQ4+w1tqxXfF3mEPTpVtHOL7MWYhAfIxwlkIjsiqELXpm3Qi3CcQcj8zv5Pod+9m5rybj27FK2tUf7HqERQALLtO5VWNO6taWcXPXOqrXLgg5RfueMOQp+OVyOOOPRve7sVfAkxXwyZNG3HGOkp9COBgaUVJUZLlJghPS1TFaO/cCxjdzcBoaESNY47ZTF+UttpuMZzc0ItxQI5uhEXFl0czHERyXviCuqasfWpJJPTzw4Q84ZdS0zG3AJmbhIPGf4wjxyXJ2t2PbREEwZURVZzburOajLzd7bYogZJfGreDEn8OtC+HHL8PBhxkhE3/qARNuhU1LvbbQNnkphEOhETr4H2QoRjj8mG5ohHMlXC9ZLvh+2h7hGPUb+360OLHr0bPrEfaifprVrmVulE1LdH4yrdV2+MgjbBoOkjI0IjQs/c+fIDhlUPd2tGnaiNGzJDxCKFCKS6DHELhmUjBs4tK4sIlJORM2kZdCeEvQI6x1Zj1CTsV1wJXQiFgvppuhEfG1f81Kq1khXRGd3dAIawK3Li6WOJ0LoegY7lxIlnPdNpPpzC4u4+8QWE+Wi71jIghu0KC4iEv7deKDFZv4bme11+YIgre07wlD/hIXNnF5zoRN5KUQ/j6m8oGBH+sIuxMaEXqMvaWcbvk0M89tQOs4IezAI2xhfDhZztZWnBGKVkh16MLVJRx4hmujshJzQaSlW5faDNOqERY9woLgNcP6dyagYdxc8QoLAlA/bKJlWSRsYuJtsGmZ1xYmJC+FcEhjaK0zWjUiRLpCOBBln9vbTle4mHWA0zrWI+zkeNpJlssmVsuhuXFNlci77mOHcExXQTcwrRpB8u9rxMNrcTtR2/Ozx13IPQ5r3YQTj2zD2DmSNCcIMYTCJq6dHAmbWPQaPHO8L8Mm8lMIR/1YZvS2s8mPuVUC2rlAj4+1DO1n+qEREeLDGeriXtua1yT22JZBGabO4kVTpI6w8TodE2sSCWEfe4Z/cFsIm72vE4+IL6dn9fscGjZ95RZ6/3Eqe36otWeoICRhWFVn1u/Yz8crt3htiiD4k5iwiXvh+2+MBui6zwAAIABJREFUsImnKuCTp2D/dq8tzHMhrJNloTvHaSJeQDuXPvFxrSF95UrViLjnAQceYbsxwl40dIxUjUhuX+RCK/3QiLroGGH7q2edH2y21E6Fmfc9VbiR7Quw4IzVNQF2V9eyfe+BFGsIgnXO7NGe1k0aMmb2t16bIgj+pnErOPEXcNsiI2yiRRm89wd4vLvnYRN5KYQjolDXE4iZ2I6j9R1PkmBOYoWWPZsSu251nEfYdoywyfNU47PpJdWRjSbFauONZMTECLtQhSLTuB4aYfa+SehDunWl48e7LeiFwqZhSRGX9O3E+8s3s3mXJM0JQkqShU28dL4nYRN5KYQj2fzRTRLc345zIexc5sU3wAi9TtsjHP086kV8Fzz7oRH2RLQXcbPx1SBMx8Uny6VTNSI6NCL06GMh7HZohHkMcOgxPjQitJ6ziwa390MQLuvfmbqA5vV567w2RRByi/iwiW2rPAmbyEshHO1hy2RoRGR76a1nxAg7s6te+ELwddrJcibhD/Fiz+709j3CiT2DmSQ+5MF8HMFxwceYcxC77uxvvmft9/vqzRHTAjsHYoTd9giTQtCaCuXwo8UY4bjXIoQFtzm8bVOOP7w1r835Nu1qPYJQ0JiFTfypB0z8OWxentHN56UQjohLHREZmSyf5qChhpuRERrCyqI2Qecy+/OZxwQ78QhbWteD35Nw6TmLoRGJbIzftR8/N5OTHp1Wb1xM1QgHscbZIhRSUFzkTvS22XfH1COsYjvLWT1W8eNcF/SCgJE0t/b7/Xzy9VavTRGE3CUmbGIG9LwYFo2Bp48zwiZWTM5I2ESeCmHjUWvr4iYdnMZ2xocbWGXrnh+YuGhDwm2HXgZ0erWEkyXLmY2zO68VkRswEUSZJFINIpVH2NxbbfUCoSYmRhjT+fzCDzWGvQ2KXRLCph7f5HcC7H8cYlcQj7CQCc46tj0HN24gSXOC4Bbte8HQv8aGTbw2wgibmPWcq5vKUyEcFRqhY99zEx33aHt9nV5nuetfnMMtYxbw/d4DsXV/4+aLFltm1NQFqK6JXGGZeW7rZfdnPFnOi9CI4LZTbDQSS1xfOFu1ty5BjHCmcONi4oe6kBB2509GqoYa8QelXovlND3CP9RIspzgPqUNirm4shNTl25iy+4fvDZHEPKHmLCJf0OLzrBhoaubyEshHF2X1alYtba99EMj0mFDsKVnTV2gnsCMntJKLeFH3l7BVS/MjpkjEfHv244RjhpvZb+znSwXSBCqYDo2zrZ0kggTtVjO1IfUjbshIY9woxKXhLDJ+Y3c0YgPjYh9P91dOpBmyJAgpGJYVRm1Ac34+ZI0JwiuU1wCPYbCtVPg/CddnTovhXAij3BGYoQdTpluaERxUBUEdOwERnJghBoLJdQ27qqOKftjFg4RL0zsiviATiD8kpDtMIE6G/YlS6qzelgStVjOVLKcG12vQjHCrnmEzUIjUghdu3tS3yMsQljIDEce0pSqLq14bbYkzQlCRilp6Op0KX/VlFKlSqnZSqlFSqmlSqk/Jhl7sVJKK6X6uWqlTaK9RuEY4Yz8/jnzTqUbGlEUdZs41iMcO5+VhDnDhsjr2BbLUWESgXghbM/mGIFtaXx2f0hiG34kH5uss5x1IVxfeCdb9zfjF/PJV+kl4rgRFhRKMrMqhGvrAjz94VcxYTfRRJzgOvH78aERxCfLWdun+PnFIyxkkuEDOrN62z4+W7XNa1MEQbCIlV+1H4DTtdblQAUwWCl1XPwgpVQz4DZglrsm2sfJLet0tpOuEo734FollEEfCOj6Htyo11ZKqNUFdJz4jRB98RA/k/0Y4WjhZyM0wtZW0idmX1PYF//5ihX51ixO1Fku2ZqvzVnLp2lmpLvx2Q8lmTW0GBqxZMMuHn3nC2aaCAKzRNPosKbE64WWW0NihIVscnbPQ2lxUAPGzFnrtSmCIFgk5a+aNtgTfNkg+C/R79D9wCOA5+11osMhQj+4mbxTle7UgSj7rLBtzw8Mf/4zvg+2iY2uipHIFisCKD48wyzxK34q++XTzOdKOJ4EKjOD2PIIJ/FKWv2c1cYqb9P5wHknOzdCI+x6hOuC+2d2i9j8tCbe1/gYYatfuvhh4hEWMklpg2IuquzIu0u+Y9seSZoThFzA0q+aUqpYKbUQ2Ay8p7WeFbe8EuistZ6cARttEx0aEbnVmgGPsMO540MbUvHlpj3MXLWN/UGvVnwdYq3NG2KYER+eYSqK68UI2zC83rzWx2fLIxzbiS/5VqM/X+f/ZQavzFoTWdNqslyiznIpbEs37tCVZLmQR9hi+bTQNs0bY0SOYcz7Jgcj0lnO0uZNkRhhIdMMryrjQF2A/8xf77UpgiBYwJIQ1lrXaa0rgE5AlVKqZ2iZUqoI+BNwe6p5lFIjlVJzlVJzt2zZkq7NKYkua+U0yzwZTsWa3YYa9bu71b+BHBvaYC00wuzWfmxoSex6tj3C0fNaOGJeVo1Idth01DnTGlZu3h1TLsmquXUJYoRNbQvf1UhTCHuQLBcIRL6DiTATvJG3YxfUa6hh8UjHb148wkKmOapdM/oedjBj5nyb9VwHQRDsYysFXGu9A5gGDI56uxnQE/hQKbUaOA6YkChhTmv9vNa6n9a6X9u2bdO3OgXRjb9Cf4cyU0fY2ZxG0QdbSjiG+NAKjf1OdfFtnk1FsUkHMKskE9UJx9ub3jEx7aSTbDy+I1y9GFeLOqsmpnxaSDSa2BYIbdva3PHUufDZD4VGKIv9NMJebFOPsMn7JhdAKsVyM+I/t9JQQ8gGw6vKWLVlL7O/+d5rUwRBSIGVqhFtlVItg88PAn4ErAgt11rv1Fq30Vp30Vp3AT4Dhmit52bI5qToOFEXepXRGOE0507k0U0+PvZ1vDCKjxm2FiMcXzUidj6zbTtqqGHRLmO97EjiRC2PExF/fOoJYavJcjHl02If6431hUc4GPNrVYCGx5l5hBOf39Brs321/XmI9wiLEBaywLm9DqVZaYl0mhOEHMCKR/hQYJpSajEwByNGeJJS6j6l1JDMmmefGKFC5Ac3k1Uj0hVrGnt2xW+nNhBIWpbMimip51U2TZaLD8tIPXf8dhLNa0amQyPmrfme1Vv3Rm3PWmhE/H7UD1extv1E5dPMdtZq62cz3LgIDAlIqzakam1utsum4UbxneVi1jG3KX5JKMRDEDLJQQ2LubBPR6Ys+Y7tweRmQRD8SUmqAVrrxUCfBO/fbTL+VOdmpU98W+DoChJuk0K/pMR2jHB8aEQg0f4mFrWmNgTivcix9plu265H2GLoQdQoG2Ptc/EzMwFY/fC5QHxDjWQe4cSfLyvrRlNro3xayKObboiDG6ERIQFpdaZIspy92IhUoRHRzXKi17EasiGhEUK2GNa/jH/PXMN/Fqzn+hO7em2OIAgm5F1nuXoeuwx6FlPFdqZaN9Gt9WTEi4raQP0Wy2ZhDmbU1QvPSCx+65Vps73P9gS6W+drV3UNt45ZwI59yb0yVpPX4sfV77hnza5Yj3Dyz1GqeNtUeBEaEUmWS7w8lfg3I9FdmGRzxH/WRAgL2aJHh+ZUdG7Ja7MlaU4Q/EzeCeGYvzc6dcyhV6Tjqa4f+lC/fJr9xhU6RiiZhVok2rYdkoVwJMKtmO5lG3YxYdEGPl+/M+k4syTBenbFNRmJH5lOjHAqwnWE0zwobtYRtnqFkqrSRXRll2jMPr/xVSNi1rERGiExwkI2GVFVxsrNe5i3ZrvXpgiCYEJeCeFvtu5l/reRPzjRiWB+a/2eVlm3uMG1dTppNQdLHuG48mmxyWDuhUZYrcoQHmNSZ9YuqTyTIaLFYjKNmjo0wppdsVUjgo8me+s4Wc6V0IhgjHDUe99s3cutYxYkFJfhWN4UIRDJ3o/+zISrRqQYl2o74hEWssl55YfStFEJoyVpThB8S14J4dNGfciIv0d6fUSHHmQkRtjB3KmEQuJ1YgfXxQsxHZ8smHryeuXTEoRGOOmgFpkr8TbMx5tv2w5WRaTVZL7QfEUq8TGwam608I50qks+Nu3QCBc++5Fkuch7c775ngmLNrBpV/1mkuEEP7OqERYudGI9wsZj+MImwVyJtxOLtFgWsknjhiUMrejA5MUb2bmvxmtzBEFIQF4J4Xg0EY9pJjzCTryW6ZQHi9cz8be8ddzNeit337XWpt7aZO2pnZRPs2SXrdnNqbPoEY739KYaV1ykEoYqWBWd0THCiRLAYuYMhB7TDY2Iff3ttn22z9+BcIywNbvDnx2Tc20Wu28WQqOCPmEnFx8gDTWE7DO8qowfagO8uVA6zQmCH8lvIRylCzNaPi2NqXXYLuvr1PMIB+rH9yZLcEuEkSxXX9xE25bQI2xTT6SKwf30q628PHN11Hh3zpfVrmzR+ijZ0NB+FxephHPGeCqTTFQbtcFIaISJbQ5DI6IvmBau3cHJj03jlc/WJFmjPjWB+h7hZJUhEpU5i8Zsn806G0aWJ1mYcDux46TFspBtenZsQe9OLRgjSXOC4EvyWwiT2tvmznbsTx72JidZ9Zute1m5aXfUOrHUBWK3HL2/qeYOYZRPM7MxOCbBcvvJcsntGvGPWfzhraX1xjg9byG9meqCwyw8pN58wXElRUUJy5KZJR7Wt0vXe272I+lmaMSabUbt5Fk2O16Fyr3FePaTCPR020KbXciFQiMijTgSr1NvvrjX4hEWvGBY/zJWfLebBWt3eG2KIAhx5LcQ1tFizl8e4YioMV/5tFEf8qM/T6+3vRDxiW7xyVvWYoTjaw/HzhcaE4/dXbYqXCLjdcxjukREZCqPcH1PeCJCQtfwCCffdrJtJgqNSDVPuvWAo+cvLlLh9774bjd/m/aVpTlq6uo31EgkSuO3aSbutcmXJ9UeJlot6WckbpE01BC8YEhFBxo3LGbMLEmaEwS/kddCGLRrnsXEs6dPqs5bCbeXIDQivp5qrMfOmh1mTTRSrWcHuwLdPY9wckEWwqonPTSuxEKMcLItRodGpIr9TSQql2/cxdMfWhOx0SK/OOhara3TnP+XGTz27heWyqvVho9jtF3Us6u+zYnn03GP4fdTnIeEF2VJdXDsQimfJnhB00ZG0tykxRvZVS1Jc4LgJ/JaCGfeI2zuEUu5brgsq/W140fWqxpB4mSmZNSrGpFAFNsVH4lIFftpNr/Ts2a1GYXVi4HQuKIildBDazVGuzaBB9pseEioRgvWCYs28Og7X8QIanObE3uEQ2ECloRwyCOcIJ48YehMipAUs2MTc+cg6lWojnAi8W3nMyLl0wSvGF5Vxv6aOt5auMFrUwRBiCK/hTDpxyra2k4aU6dTRzhxaEScwIz2vFoRwvXqCNcXrFY9c0m3E+MRTo1bZyuQRmhEspGhcSVFKuVxsRojnKqCSCJRWR0sA7bPQjmw6PWKQh5hG6EZ0eMTdS5MtH6qBD+zz1YijzNE1xGu/71J2lAjbpF4hAWv6NWxBT0Obc7oWZI0Jwh+Ir+FcJRLOLOhEfYnDwt0G7ERiapGxHvQbCfLxd12j50v8XatxMfGE/2H35qn2p0TZjXRLOaHKalH2FgW8qzWm8eiXdENNVLWEY46R3UBzUdfbqE6WP1g/4HUQjhadNcG6nuBrXmEYz8n0c8T2Z3qbokOP8aPqD8/RCfLmc+VcDtxCwvJI6yUKlZKLVBKTUqw7Bql1Bal1MLgv594YWMhoZRi+IAylm/cxeJ1yTtdCoKQPfJaCAd0lAcpExfgDkR2qvJSSTYXpjbOm4uOHWOps1z87e0E3uH4acxKhyUj1oNnfQW3GmqkmsdyslxUjHAirAr+RLGwZqOjQyNemPENV78wm0mLjdure3+oNTc2ZEfUDh2oi8wVnt+SR7h+Z7nkVSOMR/NkucTbiY0lT7S8/vfZToxwXUBbCifJE24DlidZPlZrXRH8949sGVXIDK3owEENinltjiTNCYJfyGshrIm+rey+EnbSUEOnUj/J1gkSSJQsl0DIJiP++CQMjYjTDcVK2Reo2vRF7LC4MAGnZ81yi+UEns6E46KqRiTCqkCLieVOkdAXHYu7Yed+AHZXGwJ4nxWPcNS8NUGPaHRoRF1d6qNcU1f/OCaLbU5dtjDxcrMLpnBnuURfm2RCOMGyQiihppTqBJwLiMD1Ec1LG3B++aG8tXADeyxcxAqCkHnyWwjr6M5ymXAJp08CB2zqdeIGx3uEtbafLBdfIi3RGvHzGBUTrNmcaI6Aht3VNfzkpTn12vOm03o6GVabUZjFppqNMxPCdpPuom1M5REOaE2jkuKYZVaEcPS2Qp7dgF2PcILyacnEbqrYbCsxwokOSGQ9e5/z/8/em8dbcpXXoWtXnXPv7W611Bpas9SNwIyyBhBCkjG2MdhgE+xAYoYAcuKY9/LLw/HDeQYPIfHwfp5t7PjFCbEdiyEMBhxjsAEzRiAhEJpAaEBIrXmmpZ7vPadqvz92fXt/+9vfrlP3dku37+29+Ilz7qmqXbvqnNNn1ar1rY/jCPEJvxPALwLoO9hXG2NuMMZ82BhzxpM0ryMer7vwTOxbavA319yz2lMpKCjAOifCQPjhfEJaLB/E7fuVFPHJ27ytTT3CQ5UygvSnxsRaJ2jVSqwRgrDf9tAefOamh/AN4ZXzRG0lVwoKhnqEl1ssV1f6V0cq9Dksx8vNSeXcKN7vvqWVWSOiYrkBX45Jm34Wwncr3T4U0unj5faYI7ihxXK6Zd/stWXr3SdsjHkFgIestV/vWe3vAGy31p4D4B8BXJYZ683GmKuNMVc//PDDT8Bsjzycd8YWnHP6MbjsyjtL0VxBwWGAdU2EWa3cE/IPzsFwtZUon4ki3NiUGAxUNnPz0Fosa8Vyuw9M8cef+fagQisxLVhrs2P73N9DZI0Y2lAjJqb5denCoc58cwZ7V7kiPMMw7lXtFphPiPDyiuWCNSKQweUowvw89Z3boQ015NKcIBy8/goR7vVip68dAW2Wvw/AK40xOwB8AMCLjTHv5StYax+11i52f/45gOdpA1lr32WtvcBae8HWrVufyDkfMTDG4NKLt+O2h/bgyu88utrTKSg44rG+iTBs9gf3kIx/ELfxW0H4Bm1j5d82OrBDbY3IHV9lDK68/VH80Wduxa2sBfSQ/dA+chm0snHDQRfLHeKGGtYTYf2rM5RQa+9Tbm1fLGctFsbLt0ZEHuGO0O7aH5TkWRczbWvVC7e+eTfifZTIXaBqdyT07dLX+vcUsNSs7+5y1tpfstaebq3dDuC1AD5nrX0DX8cYcwr785XoL6orOMT48XNOwXGb5vBXV+xY7akUFBzxWN9E2Pbfvj10+1n+2CuxbMj9TJP4tOUQBIdE1YsIq04ieWLC8E50/LnNElQq3DoYtT3er064JXj9VN/FCa03PpTFcjNIP1dXU0V4tjXCKkR4576lMP4MgXTCVtC81LpKG89dImfHyFlL+sYbqrwTDqx/RViFMebXjTGv7P78OWPMjcaY6wH8HICfXr2ZHXlYGNd47fPPwGduehD37Ny32tMpKDiisf6JcPd8ucVdg8Y/iNv3gXguZ38x0oYasUt4aGc57ZE/l6PwQrHh1wAxQw++13itqSY9HgSagakhGjHtW68aQISHFsuFuwM6wjFA8QgPsUaE5+QRptQJYLY1IqcY256LjFkqd3aPmfMnr9Niwry8z8qRkBpBsNZ+wVr7iu75O6y1H+ue/5K19jnW2nOttT9krb15dWd65OFfXLQNAPC+q0qUWkHBamJ9E2FmjXhiWiyvfNtZ3cT0/Qn1VE2NyK+vzkMos1pjCc0j7LcfeBKkUp1Lc+grblucNoMUUA5NEdbOC1coe5Vc1llOw9BiuaHpC3x501rfGY6wj0UwLU4b9dg0a0S0fIZczpt/aJYOTd3t6zoHwJ8cudhm/vAXtNp2fYqw8toR4BEuWAM4bcsGvPTZJ+EDX73Ld4osKCh48rGuiXBrMz+chwg2eTIcs6rq1f2JdRvm3dTWGaKCS6LYKuRD7jcmwrP3wcei8XKFVr5xg0IOX/7Oy/Hsd3xq2A5pPLUjWrpeoxA8iT2LU3zptkcADItP67dG6M/VddnFipzb3k4R3n1ggmf86ifxZ1/8TnZ7IBTLxXPpnwBvQKFaI7Q5zyD3WaVY+fy518UF24Cx+PocpAg3rcVP/OmX8NdX390zQkHBE4dLL96Onfsm+PgN96/2VAoKjlisayIMy72mTwATDrtZ8VbL8RfLNRvZUMPGaw0ZWfo8JWF16xy8IiyVxBxRCqkR3RzYjG5/ZO+gfUXjKfvpi/vqwy986Dr8ly84oplvqGHV5+n+FEU4847RMTTWJmOSNeLaux4DAHz2pod69zVVDnSWIsy30aw36vmcUaQoG6do42ubaqP1eoSV1xYnDd724Rvw2/9wE66/53E8undJWaug4InHxU89Ht9z4lG47IodJUqtoGCVsG6IsF5hzq0RT8hOs/uehZW0WE5sBNYmJKC1QGX09TXIFstDbtnXhnuEhx2BVPpyHf/S1IhBw2ehNXZQFcwBubrffnCPfz7KpkaE531T1xpv5I7Vq+dt7PcFQrEcEeFzT9+S3R7Q/bGzrRFum1FlVMVbm/fQHOHEGpG5YPEXZUoaRd9Frja3pabFB6++G//98jsAAHO5LLyCgicYxhi86ZLt+Ma9j+Paux9b7ekUFByRWDe/ANoPrrV5VfNQwIrH5WB2C1plfxn1NMzHEX9SK//oH2/F9/3252aMKW45K0RO7reaYY1471fuxP1dK2A5Fu1LEnCCVIQPFtp++nJv+/Y9qsNx57lTeiGhrsWWzYoa48qrHNMrwnfvBABsXhhltwdWao1wy8d1Jaw3+XlrSjzHEMuEqv76x37luA/SIzw/Xjf/DBasQbzq/NOweX6Ed5cotYKCVcG6+QXQioAswg/mE+IRPgjVkvt3h6uqQhFOUiPcMVNB1c0P7Ma9j+3v7Wkvi9M0gpFaI/hxxMse2n0Av/q/vol/+T++lp27tbpSC6Se3kOlCM/0CA8oluPZwTlFOIph65n7kNzij15zD/70c9+Oki9Sa4R7b2+8b1cyrjanlRTLkW97XBvERD89FsKsItVc4krOWtLXKr1v9nwZ3ciQqnhRhAtWE5vmR3j1807HJ75xPx7evTh7g4KCgkOKdfMLoP2YW2YdeCJzhFeC5bTY9euJvxslR7i1qX/1lgfyTS8kmYkUYcU3DMSEMIk/64jsY/sm8dzFreygGKbkXq4vsRwrilaUp9/Kn63k8qSIXHxabpxP3/gA9rILEs0CII/rrR+6Hr//6VujqDl5vkkRpqpz7XsQKcJN/3INtM3cqFaV9ZV0lpuVneyWpc+1i8++FtF8/IWRa0ayKCr050WTkoKCJxtvungbJo3FB75aotQKCp5srBsirBUBWRxaa8Q3730cv/nxbykV7MsfW7MgLGcbgFIjYgXNWht5eAHgpvt3ZcbjipuD5nFNFGE2fI7IiCkkSnMuJs2nRvQUEy4qt/dzGG6N4HPVwS8wcvFpOWX5ze/5Oj7xjVAZrrUqzu03tFi2CdHdt9hE+9Iygfm+uBp63Ka5bv+ZHXegi5u52qiKbZ9HOPvZziyILuyU9+RgCooWOguEVIRlk5KCgicbZ209Ci96+la876q71Ls2BQUFTxzWzS/AVPzjUZnuFvwhusUOAJ+7+SH8+Zfu8ApZ+HFe/ljLbYWsrTeVOcLdI6mVxNVufkAnwpxUyZbP7vzp527UowjnjkSqezk1cYgivCwirCjCun0gJXgSowFEWLsgI/B5Lyc+jR8Dn/vGuRp7O2tEX6YvPzb+I3vKMQvJcg3eGjGqovdXs9P4ZRnrCyHbjMam6wDcahH/7Z7n586XzXtFWFgjChEuOAxw6cXb8MCuA/j0jQ+u9lQKCo4orJtfAPljXlem8wg7HAoibMUPf/h7BWMp4w7dP6GV1oiOYJJySafk1gf2QENfAkDNEgIkMeT2WEl0aF1JE6UVIKRGxOtNW0F4lHnLW9t90Ii1NuYQYsUV4Vx8WtNyshsP1GbI9tCGGq2N19m8MPJqbWi6oc0pbDRl1ojzz9yizlOC3pO5uorTNTz5zs85N7R/PbmQ0t8HqyzXtunDeGRQmaIIFxye+MFnnIgzjtuAy67csdpTKSg4orBufgGkEleTJNy9fCisEfLH3Y94kIrw0B9yuZZUhAH3d9J9bKIXy6lZtpYUYcNsJfF2sSKsz92IOchUhpke4Z5zcmAZncE0pdTOIItZj3A9mwhzoplcuETnO309myPcWv/IrQ8bxnUoIutpJc1fWmpaXPiU4/Den3kBfvK806Lxc/DxabWuCPdZTWbFp/XNVXy0o/E0/7C+n7CwMgZzoyq5o1CIcMHhgLoyeONF2/DVO76btbMVFBQceqybX4BEETZO0eyrNl8ucpXwK/MID7u1m9smzEMqsult+5zlTCfC7u/KGHb7WirCLEc4mWNm7mIdf+tcvG9DcoQXp8tXhHMeYdo/ryHLvR28SDCvCOcJdW6ZRu607WRnuYVxnXjgNVIrWyzPjyq88HtO8O+j5ivm4B7hyLrgCXz/nDXktrXK+Hy95XeWC88NnD0iKZYblWK5gsMDP3XBGVgYV3j3lXeu9lQKCo4YrBsiLAsMqsoRuVnKVB9aGU9Gr3visnJyrdkSZkGu10iPsI2tEX5fmYPXC+PcY10xRVgQaU60hxbLSQKVzxGOd6ZdZCxHEW4UpZSPqCnTufdjPMAaEXdhixHbQ9jzGcVyOWvEhrk6KWjUSK30CFNcGBVV9qUuADw+LVaE+8huH0nmr8ttNcU5Hld7LT//iAh3inASn1YU4YLDBFs2zuEnzzsN/+vae/G4SN4pKCh4YrBufgFUj7DNF3wNwVm//Pd420du8H/nCrxWMnZ0q579/O/cu4QPfk2P0JGkMEmN6P6TJDSn+GkFYmqxnFSE2Q5RfmLUAAAgAElEQVQkSc4rwvE8h+YIa1iOItwqhExLbBhSLMeV8HEme/Yrtz+K//uD14FH9/l9WX0fsz3CYXv+udkQKcLdoxafxonw1Pq5E5mfbY3oFOFRpSrZ2uZ98wH6vMO61ivvxmQtFHI8ttQYZ4NIGmoUIlxwGOGNF2/D/kmDv/763as9lYKCIwLr5hcg8Qh3Htec8jQUH7r6Hv/cK8Etjdk9rmDcnCL8C399Pd72kW+o2b+pepoqwrCpNSJPRlJS460RXbGhnB8gGmqodo2UjMvjzaVDJA0+lKkvTxFOibVmCWnYvHPvJ4+lkz5swvuuugt/c+29ODBpM1aWvuf9FyxcSQdcagRF5sl14/2G55OmxbgjfnQMK+0s19c0g6vYEtpdFu1vjeyG75xV1+uDgSPzi0URLjiM8ZxTj8Hztx+Ld19558y7NQUFBQePdfMLMBWNAojISQ/lwSCvCC9/7Bwx27lvCQCwZzG9LSZ3M5WpEXAKsWz2MEQR7iuWk5tHirBY5omwyI2ISY31nlwtEo6vr838wApSI1qNVSEl3KPKDPrxycWnETRCHSUutFCfa/A+5jZYfX75x56Jk49ZcIWHbFzVGsFeW5y23uIRFOH+/cfWiGFKdl8BYHLxllumvK5fIOXfL2mNcB5hqQgXj3DB4YU3Xbwdd313H75468OrPZWCgnWP9UOEBZtwirBlqtXyxlPJkCBv2SzUIeNnfvDpNq38seb7C2OkirBFqlbmbn3HPtUwF2OceqYR/8rE4+dUz76GGs7rqr8vMjVCI1kryxHmc0yX02PNlHAJ/hmTFxvJ8VqtWI7PYRihBOKmINZaVAZ484ueCgPTFdCxcWdYIxanLbNGxOPnQNaI+VGlnkdVERZ3TTjkxVtuqZasMotY9+2rMige4YI1gR99zsk4cfN8iVIrKHgSsG5+ASTZI55Cry5XEdYaI0i/aU4xHYLcrV0f+q+QPV0Rjsex1q7QGhHIWGVMpGjyrU23LGwXj0l/9+UIW2sTpZamLBVhvnfa77IUYYXMR6RczGNUVdnPCm9PLM/xuIq/ShapR1i2ea5FakPuY7TUfRaooQZdiNB7FHmelUE40V2aNhiPqOGKXiy3d3GKX/zw9b5Yp/GKcCxzD1KElYXLbavMn2t3Cnq/flwRhuk8wjI1Yt38M1iwTjA3qvD6F5yJL9zyMHY8sne1p1NQsK6xbn4BJHGtfLGc+3u5ZFUqzECqgK1ECZZjATEx8IqwUhCWqK9JjrAbN1GEc9aIjLezo1mqrUQqwjmPcDL3zL65FQMIpEsrciQlczmKcJsQa6EIE1FuAyHPKrNsQ54prP0tEx7ca/GFh09t8J9RfceeCLcWTRvOFdlXomPruYADXI7wrGK593/1Lnzo6nvwp5//NoBwASCtEZpfl5BLBXHrs+d22LLks5ixFqX7ii+k5oUiXJnZNpeCgtXA619wJsa1KVFqBQVPMNYPERZS2MjHp+WVqd7xlF/wg1GAJaQ6SJgfL0cRjguySB+W0V75HOF0PhaOMLgh0nNnYFCZdLuwL/e3bKgh48mCEuteI6uBbF/Nx6fYrxUpwpE/N33eWBc7J+fNwSP6arFeEllnbe9FgvNyu+ezosYWvSLs1jXi/GspGNF+hTWCzqO/+Mh8oOk7QO3Lx4k1gs5tuq12ESWXyefJsoE2iOV8H2VqxNyo6n3PCwpWCyduXsDLzz4Ff/31u7F3UW+KVFBQcPBYP0RY/BpXXYvg8IO8zPGUe8zSMytv9y8Hucp5UoQ1spfYEFpFQbNpsVxOMYvb5YYxyP6g+TxN4hGWx9WtB/11mo/M96UpyzQJfsykuh6sRzhazpTLyrjjy50vfnEkia+MU9OG4Bck1oYOfbkEDQIpwk1HemnfxgCwMZHVc4Tj/dJc6Xxq0YN8PrzFsman6UuN0A6prwHNrDQIbdy+rx5fVnXFclwRLoVyBYczLr1kG3YfmOJ/XXfvak+loGDdYt0QYa2znLVWVRaHQLNGyAKvlUQ4EdpM4dTC2L0l+5cUa4QgEE4RFuNaQNylzxNhhdTYzhphEDrL8VNrTKz2ZlXPpHgsPlfy1jmRa+8RVtRoIo7LUYQ1sqYpqG3rvLc8LUNiyhVhQYTl7XXZBU6bA20y6yKNiBsVxnmPMLpklCiBIk9KCWPRUCP118evB2tEXEgYvgcp+uweQ4mrdqFptWW9xDrAdMVy3HZUCuUKDmc898xj8ZxTj8a7r7hzRelEBQUFs7FufgWSHGEfn9avtuXQl8eq+U6X+09U1hrRKVR7NSIsdtLYVBmzSDvLDUmN4ESDrBFWITOOLOpj8H31NJaL3pfgzZUeYUSPfB4HltFQQ2vQoaVlUBEaT8tIxuIe4RlEWPUICwU+KK/pHDmWmDWiaYM1wpiUcPeps36u3ZUS3TmQy2XrZboAmBvpOcLavEMLbfWQkjH839Fzm7yuNXnpJ9ZhoTFO1V6ackV43fwTWLAOYYzBpRdvxy0P7sZVd3x3tadTULAusW5+BZIc4a6kvs+ruJzxAK5Spusv92o9jk8Lf5BCpXnC5D4azSNs02K5XGpE3E0t7MOlRhhWHBi2MUDiUc0dV+51Z42IlVoac+pfT8cnYqZFy+Wg30pPx2xaeI9w7p3kn4mRsEIYcYHA70bIfdG8ammNyOyXrCCugYplxXIYlhoh3hTZYlm7mwKEz82keyRFXt4Z0d7zoTnC6TKd4PrvcQtccdsjuPG+XflB+HjseWUM6tpEVpGiCBcc7njleadiy8YxLrtix2pPpaBgXWK02hM4VEhyhEkRXmHWrx6fRo8Hf4sqIrXKD/4ejQiLv5s2JQtcafTrZearqaStDWqult1K9gH1ONjffcVy3NcqrRFNE+9Ty/xdTotlraGGNiZZFUhl1TBp88VyxjiiSDaG1qafE/leEZcOSQj6MfDjpaI+t0/jLir4sQ2wRkhFOPUIx9tNmxajKsTmWRt7qZfbWU62Bc8hlyP8+j+/Klpv6NfRwF088HGLR7jgcMfCuMZrnn8G/vzyO3DfY/tx6pYNqz2lgoJ1hXUjhyQ+x4oaari/l0temx6PsLxNy18bCr6+ZlHQiHCa0NAmipu1qTUid3taTY2wIUeYho52m3iE4zFz1ohojrDMXqIrwlDeN9pmOYpw6GDH9q9YCagjX198WnSXQDlAft75Z8/PpY33Swprn9cWQHQrv2mDt9igs8MoxD63XyDYOOqMNcJ4pdj9PW0tRnW4AKK1W+3z4feZXxar8+H5vqUpdu2f9K6nx7Hlv33xZ9egNiY6H0URLlgLeMMLtqG1Fv/zqrtWeyoFBesO6+ZXQFoZKPs/kNbl2RcmmjVCkDPt1u1Q5Bo80I+0bo2I/25EjjA9HaoIay2WW+sk4YpZBOIc4f74tJw1Qp4rmRpBy6VNICJ5K/AIS8LNx+b7a1qX62uQL5bj50vyYCPyaC3Sz1suPm1WxB+PbZs0NlyIqNYITZ2N/yYluBaElxDyjUkRthhXlT9mOd++phnasqVM6scLf+fz+A9/eyMbg42nvKatlyxjz52tJybCxSNcsBZwxnEb8cPPPAnv/+pdy7ojVlBQMBszfwWMMQvGmK8aY643xtxojPk1ZZ23GmO+ZYy5wRjzWWPMtidmunmoxXI2X5Q2C3qxXM8P/zI14VihZPvtUYTlHhIibLt4rcEtlsPrv/I338R7rtwBIPiANcIv49OSSDeh8Gr74rYBaTeZCuIaK8LucVmKsFK0paZGdIpwX3zaJGrEIKwRcP5Tvg85ijwHknBmFWG232nTMkXYMeG4+E/53GY8wJKI++XCMjFtW9R1bI3gy/vsD9q53M9SP/hZ+u7eJbFmnmD3rxVt4J9Wxh3bpCnFcgVrD5desg2P7l3C33/j/tWeSkHBusKQX4FFAC+21p4L4DwALzPGXCTWuRbABdbacwB8GMDvHtppzoa0MriiJxv9Si7HHkGEjDyjl3/7YaZiusfcLd4h0G7PA4G07FnUcoTjnTRWtFgG+TfT+/azvKN3PLIXX92x0xVjVcbfdpfrSUU4p3oaoZlG5wppsVwgV8FjK/dNFwl7l4aHy6vxaQopbtvQNY/UXHls9Jk4ftNcQvQBoQjb9JzHxYnBwjIrWYET/0kbLnRcsZyN9qPmCIvXUkVYX07nZtI4G4fx1gj53uVVaO1rQfF3G+fqZUep6Q068oNEirAxqKuiCBesTbzwaSfgrK2bcNkVd672VAoK1hVm/gpYhz3dn+PuPyvW+by1dl/351cAnH5IZzkAUhEeUYtl9tqsvNZovI71GgBf27ETb/yLr+Kb9z7ejZMqpctFTqn2ZI8pwv6HXuyvaaQi7FbRWsYOuWXettYXy3FrhHZ7mW+jjSmJopynPIepIpzOkQjMzr3BRzoLMoUC0K0ojbWRQvvm93wd72C36QGnCP/0Jdvx9f/w0uT4jIktKboiHD+XHt3c54krwg2zRjj1Wr+Q4pjlEZ51x2DatBgrivCQ1AhtGRHhDeP+QrV427xqPvRrSHc6phERLsVyBWsDFKV23d2P4fq7H1vt6RQUrBsMkkOMMbUx5joADwH4R2vtVT2r/wyAfzgUk1sOEo+wZo1Yhn2BfiwrY7CvUyBJifQd15ZdIheQUwGJ88REmPYn1rU2IQu881g8br8iTOtYdETL6IqfkakRcswMqYpzXy2kUkuPwSMs1mfjPprcQs9D9QhHhDTs3xjj7AIWuGfnftz13X18KDSt9SRSKt4GIV6M9iGJbZwjHGLQtGPm4J7aadt6S4PzM8efAe2CR77EUyc0K4gV7wUVy9Ex+89jj/2hzxpBRHhhXPd+g7T3TC2W61OE2aKqK5bjKMVyBWsJr3ruadg0V+OyK3es9lQKCtYNBv0KWGsba+15cErvhcaYs7X1jDFvAHABgN/LLH+zMeZqY8zVDz/88ErnrEIqwpWJUyOA4QruZ296EJ++8UEAZI1wr+du5y9nbEKuCUKwRjAiLJYR3HwEWbCpGiv3Ifflx7OUQ9wVg3nCE9ahZhth3Nn7kWNYpOeSxpGKMD3nBG/nvqUs4ZZoFEIWKfBkd2ndBYSBiyRrW5tcPEwb6/ODtXNcC2tEX7GcjRThdF4cnAhPmkCgjVYsp1xg5TrHAUhSFPg8aaylpsW4rvz7LqPttHn7Cz1VEXYLN8yyRijPl92pjo9iUm93sUYUrCVsXhjj1c87HR+//n48umdxtadTULAusKxfAWvtYwA+D+Blcpkx5iUAfgXAK6216jfUWvsua+0F1toLtm7dupL5ZiE9wjJHGBjuEf6Zy67GX375DgAhqxXQSRphuepwjkRr1ohcMZUslqN1fPyZWFciZ40ATPBYi/UqEzykbu6SRLnHvhxha5H6rYl8qZ3gAik94ag5NK3FrgPD7BFaQZd2EdJ0xJSaVDTWRkVVgMsRHtdBTeUg/ykfN3m/xBySznKZY5DxaaRqhvi09EIq3m/8Gp9nVZlkOe+2R/ufqyv/mZL+be2zr12AEIZaI+LvSN93b9gYBvFnFwC2Hb+pdw4FBYcb3nTxNiw1LT7wtbtXeyoFBesCQ1IjthpjtnTPNwB4KYCbxTrnA/hvcCT4oSdiorMg486cIixvgy9/3MqwH36pCLP1lqsIa+1j432kY8t9TFubqGbuFr/WXS6dg64EWq/6ylvggLsdb5Jt0jGlYCpJjbRdJIpwtI+wnxOOmgcw3B7RquOlxJHOm+k+N1IRbruLDrI/JPFpSD3CSbRcVCzHPLo570uHxUgRblmL5TT+TPWCz1CEtTsDdAyAI8Lzo4pZI+L3rM+u0Jca4Yhw/oujRd7pxXLZIXrvZrzjFc/Gv/nBp+Y3Lig4DPG0Ezfj+552PN73lTt9LUtBQcHKMUQRPgXA540xNwD4GpxH+OPGmF83xryyW+f3ABwF4K+NMdcZYz72BM03C607FrUcJqykIxz5MIFA0t51+e349b/7VvQru5yRP/z1e/C2j3wjbMtvbWs/9NBJBbXcDeO4/wxSojak2r7ptnfWiKCES0W4r6GGbJARjoHPJY01C7fjW38sfO60/tbNjginUVs6VGuEtrylHOFAYifs4Kir3MgrwmJHRssRjleRSrRsZZz1CPP4tDa2RgDhXI6qlNTK/QIxYXcpCvH60iO8NG0xNwqKsBXr6R5hWic9HrJGLCzHGtEz3tB8cOkRVqz0BQVrApdevB33PX4An7npwdWeSkHBmsfMFsvW2hsAnK+8/g72/CWHeF7LRi5HOCKKK7h45oowEYNP3OByHN/y4qep2zy6ZxGP75/grK1HqcuvuWtn9LeWjMCR+52fNnGhlLWu2C20Qe4n2Lo1wjrVN6cIz2iokWveIYmoJKi0eKqQQqfQuudDiPDj+ycYVQab5keswxkfj50XpgjXlfGWhsbayG5DxZi5YjlA6SwnlssGJpW3RmQPJUFEhBEX243rKhOfJucZnsuWw248Ogb3uNS0WBiz+LRMMxSOYakR/dfhfeNG6/WNwZ4bYeuRNomCgrWCH37WSThtywZcdsWdeNnZp6z2dAoK1jTWTaWI9Aj7PFj22kpSHozhinCeSXNydfFvfw4v/oMvZtedJJ21YoKUjp3uA3AEJT6+jjAapaHFAO8oeY5J9fXKH1tH2i5S1TOspx0DHQe/va4R00QRtsMV4XN/7dO4+Lc+243X5RJH2cHx2G49SlHokhjaOInEE+FMsZxLjYiV8r6ue60N6zc9hFJi2rSewHlF2NLc0sI3IH3va5ZuIXN13dxiRXhx2jiPcLfc36FQLjLSfabLuEe4n8SmFy/Lt0awizjEiSfSPlRQsFZQVwZvuGgbrrz9Udz64O7Vnk5BwZrGOiLC8d9BEQ6vrcQjbJgiLHlwRO66xxvve9wXN+XI2pKYLB9HVYSVorWwfhiLxpE/+EBOEU7V3NZ2hBA6+UhSI8SkvDWip6GGm7f16/MhtNzfyBpx1DBrxK4D02g/M1MjrEVdBW9009roLgNdBFGxnBQTZbGclVdhyRxsmiPce0Q0X8s6y4XX3NyqQTnC3B5Qa8VyRIQjj3Ct5AjnP5d9/mFPhOfqXltDdLeje1Rbnw+8wHVZz+HvQoQL1jJe8/wzMDeq8O4rd6z2VAoK1jTWDRFOfJCdNSAXUzYUVZQaIQisjC4D8KlvPuBfu/mBXeqYS0IRbhVipi3XfvAnUzYHOKW1YoqwVx0HKIVtGzrVxdaIsE7Vqaba3Pl+Eo4h3gdOQLWWx7FKHt67DXM1Ns7VeHTPwGI5hZBp+6NcX/JGN9ZGhShEigPZTUkUzxHWiuVia0QgYn3KqsSkaf12sgPcSCG18njdduy5UiznCwiFR5j2698h5fMh95krlqsrg3E9yxqRMmGZ5BFNSB0jPJcZ2MUZUbCWcdymObzy3FPx0WvuHZyiU1BQkGLdEGFJIqrKuNvu7LUVFcuZvPIVDdc9p4p4ALjpfv2WlSTCsyLe+qKjInXZolN0A8kisqG5OhISa90Jq6q4s1ykCCNW0nK3/9NiPX488TnlY+g5wkERro3BsRvn8Nj+2USYb5dtac1sABQ75+4k2Eh9JAI2rvLWiDRHWJ6DeL+jFSjCU6YI+9eaoAjrF1IzFOEkNSLeThbL5ZqhRGMo7yPhwKTFQpff23fMytcraZwzawwO+dktinDBWsdPX7Id+5YafPjqe1Z7KgUFaxbrhghLUlcfREMNDspqnQUis9PWYvPCCCccNYeb788owsu2Rjhot5k1Uk3FbkBIORiiFMpiOY3gVKJYrq9phJwXPx6uxMbHr6VGhPWrymBuVKm3yCUe2zfR82yV803FcuSNbkR8WvAIhwxfDmPCMn9cYh0eGWdtqugO8wjb1CPchrlpqnKaqBKTQfnZ8KkR3ctLTUeE/fL0eCTkOhwHJg02zNWzv1viYggI6R3avvQh2EWciY+98OCCtY6zTzsGzz1zC97zlTsHNxkqKCiIsY6IcKoIk84WGhcsf1xujZDQXp02Tuk7a+tR2PHoXnW71BoxQxH2q6fL+K1iC3QEKxA1UoSHWCOajrx1HZZ1MiMq77XkCbee8AhHxNZG5C9ShKmhhiiUonVqYzBXV1iaBuU9h4d2L6q5zJpNom1DRz1SkrkVZurJJinCs1IjtAuNsAwAU4TTuedsEtM2WCN8asQMa0TSWU7Ep330mnvxfb/9uWR9OneLvqEGWSNi4q57hPPkfv+k6TzH/UxUi7zTFGHpveaQth5+EVcU4YL1gEsv2Y47HtmLy297ZLWnUlCwJrGOiHD8d10RGQHmOvIiCegQuM5ymX22/AfYPU5b14Z3YVxnVcuVFsvlrBG8iMltbjzZkbffo/mLl5q2I9Kus0RQotl0k0YdA60R0W1uRhIdKQ7L9GK5cF7qymA8MoMU4Yd3L6qKsOYRbjqPcMU+N1qxHJ3PpFgORuQIp13/5Fz64rtyn7lpw4rlyKrA49OUDZNoQWGNAIB7H9uf7JvnCM/zHGGpCCsXaH3LFictNszV3Vj591G7m6N5hGm1t334Bjzll/5eXQakneWqdfOvX8GRjJeffQpOOGoe775ix2pPpaBgTWLd/BRYayMRMlgjLDbNu7jkvUvTzNZ55CwCQNzxKyhWLUaVI0UaKQH6CXlfjjCfBxGYybT1xMZ2gXGaR1gbN22tS9YISk8gMhOQeoTjMWmOSXybeH+4JYEf1+P7J9Exu+c2Io9OEZ59UfPgrgN6Awb2nHuIebFca60enyZsCQR5213GwtE+aBkQE9J0nZwiHHKEaXdBrdYbavRbI9J98IsUa61ThKPOct244njiMbpH5W06MGmwMF6eR5igpkZ08/jg1XdHf8vnMvqvKMIF6wFzowqvv/AMfO6Wh3DXo/tWezoFBWsO64YI8+IjIOTgWgCbFzoivLh8IswJoQRXdnk3rlHtorTUCnekqpa0DUhot5m9yt0EYmQtKbqB4FDcl06wpZrb6Xc91oihHuGEYliweVrWUQ5Ro5NH9yx2q3MFl6U2GJc2IFV1jmM3jgEAD+w6kMwr97xpnYWmMsHDzK0RvliOrBEDUiPkGZdRbiOFhXqSmbmImjaKNaINc8tZI2Q3Oe25nzubJxHP+VHl33eZI9z7mVXo7P5JgwWKY7PAe67cge1v/0SyXtxi2T3vU4QJi5mLJCM6y82yZhQUrBX8i4u2oTYG7/nKjtWeSkHBmsM6IsLhR93Hh1n3o+6J8NJsX6mEUwj1ZUuKIjxpLUZVtSxFWI8PC7B+vfAaEdxJ03rWabuxDGuyLLNqc/sFWGoENZagPbPVpKqmFdxpsAjkvLW8iURQe4/dOMbOfRN3TOLiIFgj0BXL5Ynwxjn3ft//OLvdz1dnY9Mwje1um3dWGKkIc2uGOxHpfmelRtBw9LpmjaBN5LZ04TNtrb+lH4rl3OO4qlQFtmmt/7zIefL38i+/dAe+cvujkTWCLjji1AiaY/r5IMzqLOeK5dyAv/upW9KVxLDeejQgPm0Pu+CVdzPkXaOCgvWAk45ewI+efTI++LW7sX8Fv3MFBUcy1hERtl6Ro1vcpAgfNb9yRVjLgyVE1ghPHpw1QoulIqRJDwGa9TXwjbBwblT7sbxSZ0OxW1CEu/g0ZVzJKVJrRPd6dHvZRGQijWAL68XHECvXQR0N45+4eQEAsHPvUqLaEsGrzGxrBG27f6lNXpPPOSEnJT0Uy4UOeJOZqRHCI6x8bqTtQVeE43UItF9ujZDHMKozxXLW+tg3ICa/POni1z/+Lbz2XV/xYzSt9efZdZYLir6bI9S5AujNRj4wcQ06gO6zn7nQjBVhh0mPh54Qfc/FRdwsW0hBwVrFpRdvx64DU/ztdfeu9lQKCtYU1g0RtpEiTHmwTuEkj/CeFRNhfZmWXDBprG8WIIuU/HaJNYKRtD4LA1s0xxRh2fK4Yo0DelMjFEXYWkqNCGkZUlXrU4RDZzmHP/7Mt7H97Z/A4rQN6R2s0Qk/v9Q++eE9i4kayMneuO5XhFtPXtMLFbecPadiudai9h310oKxaRtbIzR/aeIRFsulNaJPEZaEljdG8dYI35CDzk2mWK6xEeGNrBHKcdB4k4YR4WV3lrPRuhxeETbh4k1DtG33XLVGdMtI9c4rwkbEpxUmXLB+8Pztx+KZJ2/GZVfe2VuEWlBQEGPdEOHWhtu/pIiSIry5I8L7VkCEm1a5x90hLpYLhGpcV8tShPlq/daIsGw8CrfK51hzAk+yut94IkC6amfF3269qstPUz3CFYRHWIxJRLhb5y++dDsAYO9iEzqpCUWY/tEmIvzonqXEN+1zhM3sHGGrkKZcxBYnc1WXIxzlBxMRnlUsh1hdpUJNjqCguse+W/NW8D3ehY3OvyyWG1cxMSY0XZIJge9XI+O3P7KnG7dlRJjFp4nj6LPd5HKEF6JcYv29jHlwnljTa2Qf2bvYsGXxZ5eT36IIF6wnGGNw6SXbcdP9u3D1nTtXezoFBWsG64gIB6XLpX+F6v+D8Qg3bZ8iHHd1Axz5qrvUCNmS2W+XqFoxMZsTrWdba/G7n7wZtzy4x7/G11nobjMT86fjB0InNF0Rjv9u2qDOGTYrTj4Mi2ZzyyTpCusBcdMIEzaKfKb0/MSOCD8iFGFunyC1vd8a4R75OrE1Ij5mwJFHskaoRJhU10ovltNTI8S8WEIFHYuEdvEBxCQ7FMvRMcRqtVSTG/GZ4rFh2ufi6h3uR3TaWCw17jsTNdRAfBwaOZWkn+PAtMXCmFkjMuDj9glctGi+GzNngTKIi+VKakTBesNPnncajl4Y4bIrdqz2VAoK1gzWPBH+3M0PYvvbP4H7H9svPMKhYGluVGFcmxVaI/JRVlqxHBUm1ZVRw//ldkCaGsELmwBg31KD//KF7+Am1qmOVGAAmPdRVLYrSgs+3vFIVwlpXxxkjfDWCkZWCZWJVbVcfFpYP3hbu3hip1wr8WmxIoClufUAACAASURBVBxfHPAWy3Mj05saQdsuRYpwupzPt7FdZznEFzBNQ4pwlyPM7jpwGBOnRnD7hx9LWAlUIgwbrUsg4uj2FVsj6DBHmYQQSjIh8HnyzyKR5Yd2u+SOpabFgQnzCGesEZqi60l/siTEp5nurkOO4+a+dxK0f5p/zhoBE6vAJUe4YL1hw1yN1zz/DHzymw/gQZaaU1BQkMea/yl471fuAgDccO/j/sfe39qHIxbGGGycGx1EsZy+TCuWc53lqk4RTjecNq1CHsPzhlkd/HJlHL4OKcLWBuWVCCiRnqEtlimPmecnR7sX8Wk5jzDJh7Ru07au0A6x55fsGABw9MIYc6PKKcJCDaRzMCRHWPcIM2sEW5dWcTnH7gKCv2/U0jcownqxHDA7NULaBZajCJNa7rZzjz41ols3FEbG21K2NYETwChqTEzHKcJuuYtPI483zTF+5OgjyRNq2eytMkOsEXkERZisEYwIsw1dw5TiES5Y33jDRdvQWIv3XXXXak+loGBNYM0TYa3JBBV7oSMjxrjkCO4dHIqmTb2ehCXFIzxt2y5HWC9c6msI4I4HCRHWSCz3jHqPMDveITnCvS2WgyCcKsKMMaX2Cvdo/PpU1BcafbgUiEAKPXc2wAmb5vDInqWkxXLDyOOsYjnakp/rqEBO8whTsZyB2khDFrhJEiU7y2k5wvyY6ViyxyA2pkQNILVGtIKky/e6tfHnhdsDos+wJNDMIzwfxafFx6F3ltOPo21dNjHNJ18qJy5eetRhWjQ/0hThsJ0BijWiYN1j2/Gb8EPPOBH/86q7VtRNtaDgSMM6IMLh+ZhbIwx8m1sDg03z9RMQn5YS62nrGnuMa90jrP3DFCuUNiItAFSLxTy3RrBiOQvrc4CBoAjTMVx7105PIlWPsLXeGiFjsgBKjWBzTwrC4r+9NaJpfY6rBY8tC0SmMgZHLTjlPraLQCmW61GEfepBeqEij4e3WHYNNUx0DPQeNlIRVqwRUhFOFHdBDvXOcvG8CFuZIixTI3yxXPc5kB+7advGRJjNk3+G5T7j1Ahm+xDHoSrCgvQTolxi6Mq5hl5FWCjicbFcWM+YWA0vxXIF6xVvungbHtmziH/45v2rPZWCgsMea58Idz+4BkGtc6ol+Q+dErlxbrSiFstDi+W4NaKu8qkRi01Knq0gZok1YoYizP2jrYXrDOc9wlQsBzy0+wBe9WdX4FM3PqCO23bHavhYiEkIj2YDgI9dfx8u/cuvJnO1fn33SPm3BkbkCPP2yW58uoDhY8piuUljVcsIwFMjGPnldY1s8FAsF5RwTqDpImTKiDigWyOkIizZW9NaPL5vgr+7/r7uePNMLLFGHB2IMJ3+0FCjK5YjRVh6k4VHuIqIsO6jJuxbCsVy4X0XinCP7UaOSed2rq6iRjAa4qSPzEpse9pX7nvOLUNAaahRsH7xou/ZiqecsAnvvvLO1Z5KQcFhjzVPhInIkB2gMlwR7qwCIGvEyorlsjFokeLo4NQ3k/UIz7ZGpKkR2v75Ol4R7siXAcsRZrfL9y02sBbY1ylmmmJpAcBQZ7l0fsbEaui+pQZfvPXhaP58G1Itl6Yts1wEottwIszi1WIF13rLRd0pwkDw70rQeDznOUeq+G3+uiPqWmpEyOrNWSOAuopVU80//Xc33Iff/MRN/lgkZPtiArdGeAuQb7GMbm56Qoi8y8D3m2tHTNjXkUqts1ywlaTb8VQQDqkwW/9/KYYoxXw9Kmjc0+cRZsdePMIF6xVVZfDGi7bh63fuxDfvfXy1p1NQcFhj7RNhIlwd+au7LFinPDqSVRmyRqys9WROeVycpP7KaetusddVFSmfhCHWiPkhivAoVYRtty4p4kCcI9ww0uf2FY/pUiPY9sy+QDCCTEjI2//EDZ0SSI1OZGe5eGxrYyWRr19V4SIg53+jTflFBz8GTrK5Mu0+O/F2U1EsRyQyOQXGxDnCivu1sRYHJuEzWCvfvmA3EIqwao2I19Uyo0nlz7VYnuUhpIvHuLMc7bebs+oRTj87QDi3YxrP6tvzbWc1Bwj+fBvNOYGwrxRrRMF6xqufdzo2ztUlSq2gYAbWPBHWor18q1zAF39tmhsNjk+TP5BaW1cAWFQ6l027QqDQEjcmGioRFgplUiyncBVdEQ7H61MjmErou6gJQuz33ZKnOpw/uR75fHNICsJkfFp3gcJbG9N7WHUeTvdafE5ia0QowFPn0K1Lt+ErI8lhWDe0E6b20UIRzhTLaSQqyhFuUxLYtHEiRa3kd2nnHACO2zTnn3trhB+3I5dKZrTvyMf21VekJ7E3skbQHGPVvy81IvEI85bNNF5OEUb/cojlNHacGsE/u3F78D5rSkHBWscxG8b4p+efhr+9/j7s3Lu02tMpKDhsseaJcFTIZVxgftXdg6dCHAPXZnnfQI/wKClWyzTGUEht4xVhvYJf24aTBc0jrBXdxTnC5BEOxW7eI8waWsgc26QZhrU+bo4akvD1gdmV9nIbnhpRkTXCQuQIh3VDa2dujQjn0Vkj6m7MnDUC0fJRXalJEQAwmXJF2BFcTlbpuSyWky5hg9QjrFkj+GdJV4R1AhkrmbEi7KPdFEXYk+SMNWIWvCLMrRE+co4eNUW4Ox7xOjXoGHtrRF+L5RkMmNbrHul7kssRroxMjRg0fEHBmsWbLt6OpWmLD15992pPpaDgsMWaJ8L0Q+/SEoJPmP/GGWOwSYlPa1uL3/nkzUnw+Fj8Qmpe3xwmTfAIa9tqjSD4Gk2rdJZTFeEwR64IU7GbVIR5UwoiGHIqjqDGxYZufkxVM/0KnSRAxDsmTevtK63Vc4QjJT+jCFdMEc5aI7xHuCPClRHWiICQoEHFfKH4DOx5sGbo1ogkNQIpCWysjVRs7aIiKMLx61qRV2ioEXeWixRvT4T1YrlZoO/M/KgGfat4O/Fo0gx5Rdj97awWlBoxwxoxY460PZ3bnAVKvkfFI1yw3vGMkzfjorOOw3uuvDNb61JQcKRjzRPh4HcNsV/GxLdAjQE2jGssNS3+yxduw2/9gytW+tqO7+LPvvAdvOyd/xtn/8dP4ZE9rqOWVIT7oroIvvirU4R9pmsjiYBmjYjVzyE5wnOZ+DQAXbEbumMJBVVeyctEW9H83S3kQB5lwZH2z6m1MTkKdofugqDpiuXgiJSWGmE6Jd/5hm00dqwIdx7hzPtCmxIxGlUmGY9A7y3//PC4OhojiU9T9ssVYW75ILStjX6MRgoh1TzCG+fqSD0ml0OwRsTj8c9LUItX9lWnuyjz3Bph5Wc2Vb79S+LDwht0eGtEZt/SgjGLt9J7ye/8RP52mIj8lhzhgiMBP33Jdtz72H589qYHV3sqBQWHJdYNEaZ4rqoreOJNHwyMJxJfvu0RfPm2R/w2ALBz3wR7FqdeGZa/j0OupOlHe9K0GFUV6m6HgxRhtoqzRtTRcq1Yj9/qJmuEL1BjHuE5rxKm1gjXTS0ed9KGdActHsspwul8ZGFTmEuwRhjAdfyz4ZjIvkLrVt34VoztiXDXWQ7IK8LSIzyuq0hhpeejKrRqtha+WI6/Z7TfJD5NSY2oIiKcKuetjZMu+ry6tN93vuY8fP1XXxrtT7ZYDsVyqUeYzrNs2z0Ue3ixXLc/uvPAj4sjXhYv5O8J0K/2SkV4QXwv5Hoy6s5tG392S7FcwZGGlzzrJJxyzEKJUisoyGDNE2HildbyCLVUETaMkAWPY/wz7AujxC/7EGsEV4RHXBEelBrBiJcdFp/mmj+456QIe2UVYRlvscxVWHqUftHGF7UFSB+2didb3tKXxWWTpo0SJ/h6sTXCJCSSK8RVlyNMY2qgsX2jCeERpqfzoypSfKuOqXNF1VsnGBGnuXIYIzvLKR7s1kZqs1YsRyeSNt00P8KGuTpSL2k3QRGOyW7cEKRbpu1rAB7fP8Go6i4w/RRj/7P8OMR+7HgZj0+jwsmhxXIL4/gY6M7AzQ/sBhDeq2kmLYQKaQlFES44EjCqK7zhom340m2P4LaHdq/2dAoKDjuseSIcp0YYryryn7iKKUHTpo3sFBxUbCN/l3PFctE8usdJazGqK78/SdY08jZpLH7u/dfilgd2o1WK5TRrhDHGK4AUnxYsBoH4j1iLZenbbG1KBqYNWSO4ysv2i/64LKk6E/GYNLHSzJMr4mI595oVY6s5wqq6ns5tVBtBrN0f8+Mak46YNdai6i4u+PstVUbvz1XMEd9z0uZoH6lSOqBYTjSroPPHL1hkhJssiItSI0QhncQF245VXyc88PgBbOg+XzzlIVd8KP+Wn5UlrwibmVYH2cKZN44BXKTcT5x3Kv7qijvw0K4D/j2KOwoG8DQVoBDhgiMHr3n+GZirK7ynqMIFBQnWPBHmP/pVhY7MSEU4KEHT1qoEDwh+UPm61uJYgntkl6sIP7x7ER+7/j589Y5H0dg0R1hThA2C7UEqwpTOAMQqIc/uBZzKKbnAtG19hzet4Kky6flxxx+fh9QjHJRmaY3gBJ72G3lQma2jqgLh05pBaHMb11U0Hj2fqytmjQhpG/x0+4YanY0kWywH4LlnHovPvPUH3HjQlVIexacWy7E7C3wd+Xnmy+gCjj5z9+7cj1/40PVYnDb+vMm23YT3/ewL8K9f+BR1GQDcs3M/jt4wjvbXChU39ULrz4FMQ40M5PdUEmFjgNddeCYOTFp8877QNCB3B8dAWCPW/L9+BQXDcMJR83jFOafgw1+/B7sPTFZ7OgUFhxXW/E9BTNKMJ4HSw8m9qllFWGTG+tcHVttSUdeoDmpt4hHORK7Rutamfk6tqK0yoYGDJ8Ld0NyCwFVCWSRHhX3yWL0iTMfFlvOOc9ocG0Zw3Tzd46RtQxEeYkWYk2ZSohNrRFQsF97L3Dw4XLFc+Juezo+rqFiuNmmzkClLjdCK2wi0WSgoU4rlbKw2ayqtZesCLLe4Jz6NhqT3+u0fvQEfueYeXPGdR32xZm7u86Max3REl4Ne2z9pwnI6NghFWHykc4WJgMgRRn8CiVwkLxANDI6aHwEAdh8IBXKRIswGkZ3liiJccCTh0ku2Y+9Sg49ec+9qT6Wg4LDCmiXCexan+JE/+iK+8/Be/1qUI8zAi2QmzBqRFPKQNUIqwplWvhx/+eU7fPOBPkV4UbmdT2RZqmUEVRE2wf9LxXJv+8gNfrn3CDMiHJpYdOP2eoRNou7ycXMIiVqk8gbSSvFoDWs24fy/NDYjwkmL5Y4IVwZztTtePZM5nVOSI9ytND+qsDRtYTsbAxVayvNBj3F73ng9skrwAjb5OWpaG10Y6YrwbGuEfA9k++dd+x0p3DCu/Wd3PMp/1bU4tS0bAzkmIuw9whbiwqLPGhGDSOrcqJodAyFOoFSE+WtEhOvKiDs44bn7dyAsKcVyBUcSzj1jC849Ywsuu3LH4IzugoIjAWuWCD+46wBufXBP9FroLJcqPzzGK5CwvCL8s9//FPzuPzsHQL6DGcd7v3IX/vDTtwJA5BGWJHqikDdSCYNaFv/ga0S4MkE5JqWMJxsQIYsaajBSBzgCVQtV0lkYYkU4TpfQi+U0/zGAqLGIm5OJihFbG5RqskZYOBLJ48CiYjmvCPcXHhLGdawI0/P5UY0J+zzQ54eDF9NpTS0koogxZS6TqFiuRxEW1gi+Km3nc4SF/WEpSsvolvWwPu1YNi+M/Ht+jLBGQBTLJV5obo3IKMI5q0Y0jrBGJIqwCQV0RIQ3MvLPtwWoKyK/mClMuODIwqUXb8PtD+/Fl297dLWnUlBw2GDNEmGNBBk43x8VZcWva9aIeHsipBauot83xRhQLAcAD+zaD6BfEdbi02SRzzBFOFgjNO8kHX5QhFOyOm0zijDidAhrgyLpGl6k8/EXF8J+UYn3wRgRb8XUxeARdq/Te5bkCPekRmgkva5MpIB4a0SXGkFEsjZpCRw1q5gKIpxwKEMPdOGhq9NL09Dsoc9qwVVyICaOPj7Nz5GIsCTxbZSckYO2qK4qbO5sB14RNmFulp361AISv79yTgClRvTD35HIFMsZwBfyke9xYa7GpLFsW7Z+d8eIUKwRBUcafux7T8Hxm+Zw2ZU7VnsqBQWHDdYsEdZui1fMGsF/4owJRM5ZI9zrqTUiWAG4nWKoR3gfs0bUjHhzaKSWXiPbhCQmmu8VCJFYqXeSEyiuCNN4YVx5W3za8qK2oO5yX2q/Ity94MltzISdNaKNtiOiE4rV3GvhYiKonjWLT9OK5bRzlcanuedzoyqyyjhrhK4IuxbMjAgrLZbd8YZ90C43z49w2pYNyZz7iuXkhcQ8iw6j10KxXGeNENVfk6b1dzn6Gmpo86gMsHnBEeBjNpI1gi5M+smuVIsf3z/Bt+7bBSAcP2/ZnIP/KHVPZHyaey22Rmycc3/nsr+1GLqCgiMFC+Mar73wDHz2pgdx93f3rfZ0CgoOC6xJIvyv/upragyMt0UI+yEV0QGONOSK5ci2YG3clGJIagQQiHBdVyG/V+xDa44hrRGSmGiCdGUCIZRKGZ0DIC6WS1sspwVg07YNjS261yysV9mzneXo+DIeV5qXgYmSE7iXliwdpBIT8eT2icr0x6dp/EdaI/it9qWm9bfy+eeEwO0koz5FmI6xSsni1371JXjDRdsAAIuTWcVy8fmjc8CbSchiOekRJixNA8nva6ihWQQqY3xaRLBGdHO00hohFWF2PNbir768A69515VuTqQID7BG+DsS3d/pnQ/DiLBThEkhnorPuls/ToooinDBkYh/8YJtMMbgfVfdtdpTKSg4LLAmifCV33kUX79rZ/J6ZTpVr/tftIwVy1kL/Pnlt/sOcwTyFrbWdk0pdJ9vDvs7Ijyugm1BbqtlAstiOekd1XOEkaRG8GV0/NyiIS8AqBtftK8mWCNCZ7lAgo2BKglbrzbHt6SjAjNQMRNXhGPSTJ3lwIiwZWr2rM5yao5wVanEbX5UY2naMrU5bZ0sWzDzY+GgRfS6lmYCxNYYlYh5RZjG7Swp7Lh9hFu3icwR5nMfZI1QplEbg80LzhpxtEiNkLaPxCPMSLwFsHdpir1dh7rJNMxHy2LWxqH3VHaWo8/TXF15RZiI8YTZnKLjiuLTChEuOPJw6pYN+JFnn4QPfO0uHJg0szcoKFjnWHNE2FqLA9PG/7ByEOHgObpA7A2kW+G/+Ymb8Ndfvyfa3ucIg5pwxK/Pwr6lULme8whriiWtM2l0IqypyJUJt7slEQbC8cdkMrYvtF3UG4drsWyiaCvK2KViMu0YvJIpCLH01VZVfD4pscEtp4YayFojKhMSEKQi/If/eCt+8v/7cjK3cdJQwz0m1giTJo7kFWFpjTB+DHc+EBUB0nng5H1YsVxYRkq47yzXPTbKuQacDWFWQw1AJ4TGAEcvxIqwt0ZA5DyLiw+e8EFFmhSBt9Q0qDvr0FBBNijC4nNuwuvSGjFVMsH5xS1QrBEFRy7edPF2PLZvgo9df99qT6WgYNWx5ojw4tQpunsOKES4ihMTCAbhlui0sckPN8EXy1kA5iAU4Sg1IrVGVAZ4y4ufhrNPOzpaJ2eN0DzKBgbjrhOa9IZyQld1pIMnLxCJyRbLGQAmjjij85rjDjRFGdEWVenDYFRVERlMFeGQGiGtEUTEQ7FcfF7+5LPfjuL0CFIRJtJOxXJRQoM4wCkjwpUg9RpoFd4djxR2YAAR9uc8JbeBCNNr4WKrrkzyXk6acPHT12JZs0bUlcHRC3GxnJ+K1S/oAODfvu8aXHbFDgDu/LbMktNYi0mTthDPQX5Ntfg0en3XUGtEKZYrKMBFZx2Hp590FC67okSpFRSsPSLceSwps5cjRKfFP+6R37dNW98SXEOLQMpCW+Zh/1DQnJwi3HlzZbFcV3T1Cz/yDPzxa893rxERXoYiTNaI+VGdZtqauKCqNsapqiLRIVssh5gkWNuRue517R9OmTkcfL9hnboj7vw4tYYabZdwQeeQWjLTeSG/q1Ysp0G2WA7FV67FMo9601pOA4j2D+StEdw+wL3PJMgustQISVw5ZGoEEI47vLfdHLuLBPm5WWKKMEXOaeDz4GRbeoRDRnJs1+EXGZ/4xv34b//7dgCOCFvEqvrStPXHMYuG0qUE93Rz0PYb5mqvCG8gRThz8RqlmBQeXHCEwhiDN128HTfetwvX3PXYak+noGBVseaI8P4eTxMViaWpESYiCbkrYBetxsYSVfkz5+YVYcMU4fgHubXBa0ozonWWpuntcCDnEXbFcjJqjcYNbXidGq61WNaK5awN548TZu4R1s5GSKIIx+n2H8ZfGDulnFtA2tZGxWrkTbYIKn5QhI0/9rm6Uj3CGuYyDTXmumK50M44JWe8s1xs8xB3HdiFBwCAKfDcGrE40xoRSCMfFwg+39Dm2fjj4d0ECdwjLO8acET2C7aPzQt6fJqFjbOgM2/DXF3BMtLctBZLTYu5zus7i4j6t4yIsFIsBzjvcFIsx2xOHEOyoNcLjDG1MeZaY8zHlWXzxpgPGmNuM8ZcZYzZ/uTPsGA18U/PPw2bF0b4792Fa0HBkYo1R4T7zP2k/Pqirg7ylmhWEWZ+UWenCLeehyAonRUrltOsEURS4/GJIOZ8qhzOEmEwr8RQVZVhqiEpwmmxnCwAIxgTx6S5ODnjzy+1teWQBM5bI9g686MaNbNGzHdEiRPGkEwAlryRElHqCpcDP6xEEWZjpPFp8ThRQ42eYjkCnU/nc6a5BLsOn7Mx6f74OQf6rRG0pOkUYcl1J03rCWtfagS/K8BV5+M2zcEY4NiNMjUC4nzq34/5ce084Kw4c2naYo4U4RlElD723MrCQVsvzNXYvRgrwt4/buP1tVbV6xj/DsBNmWU/A2CntfZpAP4IwO88abMqOCywaX6Ef/3Cs/DJGx/A315X2i4XHLlYe0R4OkMR7siFrPCPiXBGEW5ZlFcVF9gtB6O6v1gudAbr9tvEHuFB1gjAK8KaJ9owsl1VpotPc8t5fJqmStL587emQQWITil+1XNPx2/8xHNwyVOP99tIb6tMjwBcFm5dheMc1U795Sq8AVOETRhTtoOeG1WRzUCCK911T2pEa+MLkPQiRFeEkzbe1GKZ7YPyqN0cUkVYU6Ct3z7dj1dru5d8sVx3QSOtFovToYowJ8JuvdoYvPp5p+Ov/uWF2LJxzh8lHVtfZzk+39am1gjtLoYGaY3QPqsAsDCq/DrBGpF+/lKP8KBprEkYY04H8OMA/jyzyk8AuKx7/mEAP2xKq70jDv/2h56KC7Ydi1/5m29ixyNpfUVBwZGANUeE9yveYAIpiqkiHFsjNGIJxIowEBfYLQcj5hGW2/piNATyNMsjrFkjqspgy8axU+3kQsMVYVa970kqjauTC1Iquc2BXqNz+caLt0fFS0HJ7B4RtiXMjyrUxogWwIGYV5X7z1o3EJ1Day2mTZxwMT+qej3CtVA5NeJGhOxA5zuvFYU212JZ82UDLDXCBm+1e909SkVYEmp/kcK86oREESYi3F0kSL/3pGk9ke9NjVD2YYzB0Qtj/MDTtybHaCEbaujfj7lRFX3upm2LSbMMIiw+S7kjIPILBGuEj08Tc4tU/fXN+94J4BcB5L4kpwG4GwCstVMAjwM4PrNuwTrFqK7wx687H3Vl8HMfuHaw3aygYD1hzRHhA5P8F7UyBqcduxGnbFlIOsvxH/tckSy3MXCFbahHmDCqKtR1THJvfmCX88Oyoiv6HZYtlodYIwyAt7/8Wfivb3heao0wLM6rYtYIUSzXtK1OhLv/5+RWu8Dgm0rbhRWkGyBrhPH/2FLHt0gRNkElprk1rSNRPAt3flzH6RPiHPGUBJcawRZ2Si3ZAMhuUynFcjzxIKdIRmAqtkVqg4lzhFNC/cLf+Tx2PLI3KiD0x1RLIhw+Y8YExfg5px6NUWXwpW8/gt//9K0A+htYaPYLbXVaazJt8R8/dqN/Pff1oGI5/7lr0RXLEdnOTikCnQtJXGm3PF9YxqdF8+/5/K4nGGNeAeAha+3XD8FYbzbGXG2Mufrhhx8+BLMrONxw2pYN+J1Xn4Mb7nkcv/epm1d7OgUFTzrWIBHu9wi/8zXn4ff+2bnRr6xBrJZlrRHcI2yCn3BofBqBWyOmrcW9j+3Hy//4cnzu5occERaKnuwslyjCGtMwBsdtmsOpXeveaBFCZ7mqO45csVyfIkxUw61vkgsKTkykAuz9nexcU7FcIMLURS6onwadmoowt9ZaLE1tRITn6tgasXcpjtPjCigRXtkymkgfjVMZJGplvlgOKuKLgzTPOV43bfwCAPc+tp8V8PVYI7rXKdrt9GM34A9/6lx84M0XYW5U4eo7d+K2h/Z052NYfNqcINtyvgDwhVsfxuXfDs1o+hRhay2Ik07btiuWG/bPDhHo4LWOl9N+NUWY3rfIGoH4fRh0YbM28X0AXmmM2QHgAwBebIx5r1jnXgBnAIAxZgTgGACPyoGste+y1l5grb1g69atcnHBOsHLzj4Zb7xoG/775Xfg87c8tNrTKSh4UrHOiLAJYf3sdWPiW6L5Yjne7pdFrq3AGsFTI3buXYK1wM59S2jaQDzocVaOsGqN4IRUkCmyMNBYsliOCIYsAONjV4Yr5zYowmxfEekTRXKBdHJrRKwI063zlil+1MLZMkW4tRaTpo0KvubHsTVit8iVrrkiXNM48OMZBNLnrRGVSYlwrlhOeoSF8kuto+Xr8TY6oeYRf5EiTNYIkRpBHmFjDF713NOxeWGcdJLrs0bw45LJFHK+gOucyJH1CI8q30iD5hkpwjMC1HY8ug/f/7uf895FubZXhFmjjdBZLv48Eo6EHGFr7S9Za0+31m4H8FoAn7PWvkGs9jEAl3bP/1m3zvL+oStYV/iVH38WnnnyZvz7D12Ph3YdWO3pFBQ8aVhzRLgvPi1q3iBugfIfPY1YAo60htSIQGaXXSxXVUERfSw7FwAAIABJREFUbqwn70tNC2utv+1MfGJFOcLQj5XGNf5511BDyRHOKsLd+IE4UyFZnEygEQnpQ+anen7kFOHF7jip0YVlpK8ygUR6Itza1BohUiMkER7XKbnjBXyVMf71oAgbzLN9GBM31IiL5ZJD99vQ8duOcAM6sZRWE0LT8iSL8PqctxR0RNivn6Z/SELfZ43gUwupEdrnwr22MBfHmGUV4bqKOhpOu/i0+dEwa8RHrrkHd393P75wS/8tee5V3yA7y4lEi/iORv/+1xuMMb9ujHll9+dfADjeGHMbgLcCePvqzazgcMDCuMafvv587Ftq8PMfvG5wWlJBwVrHTCJsjFkwxnzVGHO9MeZGY8yvKes8aZmU/R5hNidOFJlVAAhWBIlJY6PbsCv2CNfG34puWuvJOzU4CPFXMdEO1oh4PG268bHG4K1kKVt5OcVyMi+YcoRPPmYBJx29EK0Htg5/VBXhcYXKMGsEKYbcGmFCaoT3CFuXsTyKiHAtFOFJdAxxsVxMhNuuim0siuW4XQJwRC5HhHNqZlQsB15Ep6+rEc6k2x3Np2uKIa01jbXJ+JL49inCkTVCtHGO13OP8sIs9+0gRZjnCDtlf3nX3ycePR9PQOw4IsKkCJM1QngjKAYQWL+KMIe19gvW2ld0z99hrf1Y9/yAtfafW2ufZq290FpbwmQL8LQTN+M/vfLZuOI7j+K/fvE7qz2dgoInBWkgbIpFAC+21u4xxowBfMkY8w/W2q+wdXwmpTHmtXCZlK95AuY70xpBiH7jBlsjWtiWtg/keblXxi41IpBoIlpL07YjLYLICEU4F+HFIY8vXhiITFUhKZYL8Wl6sRw6BZ1X7VfG4CP/5pIohovPwcenCYuELJbjsWZ0i903j+h8yLZTU2ld21kj5uqYsD22f8n/TTmyBNrWmJic0gFVBn48+kzVVaykzo0qf9GUdJabqQi7+DTad86Cog3TMGuERuglkWsHKMJ98WnaPrT50kuytbW8+CHMd0Vs0hrhle3sjGLQhVPOGrFBIcJqsRwVkHYXW0cCES4oWC5+6oIz8KXbHsUf/uOtuOis4/C8bcet9pQKCp5QzJRmrMOe7s9x95/8lXnSMin7O8uF5yZ6PY2W0jBhPlppp5DoK7QZ1RVLPGgjRdjacLvbF8sJj3BijVBr5fLqJFca6bkrlnPLg0LXfwtcqrzk8dXmIIvkNHJE1gjCmKnmbjy3b4oe4x7h5VojRozQ0TQtmxu3RtCFimGv0fHmFeEYPtkAgby3NqynfR14ow0OWbRJCNaIeA7UYplDNtDobaih7ENjqXRsMvmCrtPkBSOR8Qk7hzwGb+i/ENM2PRdAOOfcI0xtoekCRruEpX8L1m+tXEHBymGMwf/7T8/GqVsW8HPvvw6P75vM3qigYA1j0D3KrlXndQAeAvCP1tqrxCpPWibl4kCPcF9DjRymTet/OMlbm4NsTSyXcVsF9wjzoiuZI+xbDw8olosEYTGVljEwugjQrBFtm7ZYdtt0Y3IFdcYnhdso4r/DOvOjSi3+8pm5FfMIs9SIpgUm0/4cYWmNoOPiXfY4WTcAI8KdIiysEfNcEZaFhZm3n/ZFZN4rwn3nWaCJLsjS8+UVWXZHQRJqqQj32REq5eJEX8898guQujLqew2ETnB0DqciPnAWzjn9mGh7A4Ov/cpL8LPf/xQAvFjOqcDP336sJ8VEvjUm7FM3iiJcUKDi6IUx/vPrnosHdx3A2z96Q7YOoKBgPWAQEbbWNtba8wCcDuBCY8zZK9nZocik7FeEdaJizDD1Z9rYSInrI899hGFUG0/AomK5qVP6QtU/7df90BPBkep1o9zm7ZvbYtMyRdgRvL//xgP4zsNO2KdjnOZyhEmZ7f52KQvpevwfR96tju8j9gjXcayZsEaQCk8kkshsa12RVRSfNqqwOMkrwvwWP09ycHOVxXIsNUKozvliOf388xbL5EV2Y+vrBsU4vD5lHuGom15NRDgQQ21u/PgJ/Q012D5G1MQkXU962oGOCCvvNcCJcLDiRNagHnPEzb/xMvy3Nz6v21/4Tm7dPO996rS7ux7dBwC46KzjWSMbUoTDnOgwa+YTLigo0HHeGVvw//zoM/AP33wA77vqrtWeTkHBE4ZlVa1Yax8D8HkALxOLnrRMyuHFcuy5GZYZylssy250Er1xVN12o6qKFOFFIsLMtgCkxXhDOsv1iZOTacs8wgY/9MwTAQCfuOF+AHGerp4aYbr4tKDuaqeCzyrEpsWP/NAWxkIRrmOFk4r0KFJNpkbMCdsCv0W/b1HGpwUl1jBy6h67BhSjWBHmrwHkER5qjYAfg/bB19PUR4o8k+Pl4tNobt4zS4rwgGK5cW+LZbZeHbr5SQSPcDjvI9a+Om+N6BThxvoEEj6ehoVx7UltIP40X/eMSO4rzj0V86MKr73wTP+9nGbi0wCojVMKCgpS/Oz3n4UXPX0rfuPj38LND+xa7ekUFDwhGJIasdUYs6V7vgHASwHI9jNPWiZlb7Fcxr+aq86XmHbxZoD70dUKhgh91ojN886nOKqN8wgvdcVyiTXCQRIIOVctPi0uDIzXn0SKsMHbX/5MnESV9wiezpwiXFXoCGm3vtWJHH+LU29wuo70GI98971A7IzpcoTBPcIZawT7LCwJ1Zw8sTWzRnhib52VwOcITxu/7rwkwtRQIymW099/40ma2w99JnPFZ1qCQRTjx76hdEwyb3pIsdx4lP+qR6rzKMxfgua6NA1LXftu91wqwnPCA950FzhD1Vj6jnEPOX8knHfGFtzymy/HaVs2hIsrrcDUH8ewmoGCgiMdVWXwB//8XGxeGOP/+p/XYv9S/ve3oGCtYogifAqAzxtjbgDwNTiP8MdXK5OyP0eYPRfLhhFhG3mE+zbJVeH/xk+e7bNM68o4RXjKrRHpD7rMKZbkVItv6zvWpWmbxHbxW+XeK5wplgNMR0g5cUzX4lyD1g0+ZMUaIYrliChNvTWCKdHMX9vYNHZLNtSQkXj0/tQVt0bQXN15GXekjywW0iPcH5+WhzFpsZzaqY2p1RERZhad2BpBzSJiRXja6u8PR9+FGyf1QRFW14z2T7Ds8xTGDAWLE6aq57KrNVCbcplSkaSAMPD8bkAn9PziqKCgoB9bN8/jna85D995eA9+/eM3zt6goGCNYWZ8mrX2BgDnK6+/gz0/AOCfH9qp6ei3RnDFDux5v82BMGnjav3lWiOu/Q8vxbGb5sI6lYsto6vopWmLlhEBWSxHkOphriU0QZKgSdMmBIuTSGvdnJpMsZxTZmObg0bk+LyIBPHINf460CnCCulqEmsEZfAaTyonwhoxV7tEByJWkpyNWGOItFguTo2ghhpOJQ5RXPPjCrv2O8tF2lkuPhfci+p9zmA2GNUjzAu3wutOEQ5j+fPVEXdSv/kdhTRyL/7M9BFh7eJE+8xpF277l5oojcPPtar8scXFcoF4zypWCxGEsSeaNlOJsPBR8zsS/OKwWCMKCobjhd9zAv7PH3gq/uwL38ElTz0B/+TcU1d7SgUFhwxrrrNcnzWC/7RJxXRIhr9ssbzc1AhOggHX5nfaWk+0qFjOE+FuTlLxJcJD0EhJnIoRr++K5br1KiLCYZ3P3PQQnvWOT+KBXQcyHc9CjBntXzsTfNpEBKUSLBtqaNaIoAgTiaQMXndR0FqbWiPGsV92Ii8mvEc4EK4oNcIE0kcXKrLFMleEW6kI9xApSr7g6r+qCBsD+tTG1gib2AFoPvyYvUdYUVlljm4f8Ys8wlQsp6xHq3EivMSi3riXfVwHJd5bIxryfmenEiF0diQDdvfQcyxj8ZnSLR7FI1xQsFy89aVPx/lnbsEvf/QbuPu7+1Z7OgUFhwzrigjnyCEvSupDlN+K/h/L0YBf81FlMG3aoAg3LRquiHXrSdIyV1f4oWeEYkK1OUDP4UymvCCP5qLPVzsM39jCWyN0IhWnRrhHIj2yaA4YkiNMmcch4qwyrj20liMMBDV3Ms1YIxjp4XPimcGL3nMbXzDMdT7kt334Btz3+IFBDTUAdO2pw36ATFGiiVVKgrsgs8l2vvjME9GOaNo0Pk16ZHvnq6j0euxYp0iLc+3Vf06ER1VQkFuuCKce+RzGmWI5etTKEJLUCLYKb6hReHBBwfIwriv8yWvPBwzwlvdfm9yFKyhYq1h7RHjaZG/zRqkR4nlf4RthGqVG9Eeu9d1qJniP8ITi0VwxHnEtIiBSEZ4bVfgf//JCfOatLwIwWxGWjGLStIk3NddQQSPIdOyRIqxs3ipEmE/VWhsrwqJYTqp33MJgYX13P2tdRzIZnwYEEiv/UaaxnQ9XztFGpHdfRhEe1xVuf2QvPnj13X65P0fiXESEq7uIsExJzynCerFcaH7CX3/GSZsBAM859ehumXu9VRpqyIunvgvB2BrRXTQoTJiGIGvGT1+yHT9+zilBaWdvwaiq/D7J+uKaugTSPusrSe+dzxEWdh9N7R1nfMV8f9w3XlBQMBxnHLcRv/2qc3Dd3Y/hDz5962pPp6DgkGDNEeH9S43vHiWRreTHsPg0lxrhns9sqNETn0YY151HmMWncT8nDS9bKBMZ80RCK5bjz8VUXLFcTBpyucc5guZUzXB7WU2NYM+1CC2fpdshrwiHFITKsUhQBK9ThC0mjRXNLmp/rEDeGjGqdI+wgcFRC84if99j+wG4rmTzgghrYyJzPgg8C1lTfPl6/s4FW94wrzrf7gVnHY/P/sIP4DXPPyOaw1TxCC9HreH7oHOshC4kOcJvf/kzsTCq/br8vZ6rw3mn94YuNIcWywHu/aPtvSI8Q912F6Atey1ep6qG5YoXFBSk+PFzTsHrLjwT//WL38H/vnVl/QAKCg4nrEkivGWjToQ5GZDxYkMEoIloqNHvEZ596urKJA01mpY11ICuCM93BVu0d0XcioiYnOVLnn1Sao3IKcK1wcff8kK892deIMZnjeUGeYTpNa4S20gpXRjrxXI8NcIVy3XbGfIIO2sEV+FnWSNobJ7MEDfUcGR641yNuzq/2zEbxlGxnHz/ZbfCHIxxKi0vljtO+MfdeLqHmKwRzjoR7+mpW49Ss4eTJixaX+4MtGznvmI5uvgYVaGYUe5zPKqY9Ye689GFIJL551BXxivK8qIiV0M66r53uQTH4hEuKDg4vOMVz8bTTzoKb/3Q9Xh49+JqT6eg4KCw5ojwvkmDYzKKcM4aMaqGpUZM27jFcl/3uJzVIF6nwqRpcaAjDktTpzh7j2Q3hLyNTcVynsDNik9jf9zxWz+Glz77pIRg9SnCZ592DL73tGOi1wwC0Zg0Vj3eOEc4FKKF12YpwrHiTZYMnphAxNjNIfUIk+1EXkwQadY8wjy5YMuGsbdXHLNhLKwR8TFzIi6JFOdclXFZyDw+7dQtGyBh2Nwij3Brow5sOfDFiTUiQ4SPVwl5SoTVYjlSeKkVeKe2hwumsO6oClm9NBeKhQudFQd8h6oqUnf5dpp9g45hwgpfvZLcPdam5AgXFBwMNszV+M+vey52H5jgrR+6Tv2NKihYK1hzRHj/UoMtHRF+6tZN0bKooQbTm+raDPMIH2JFeG7UEeGoWM76KC2aUtKRi9TMzHK3TFcnjSBWs4gwvcwPhyuR1uazXzWPcPQabESMZWqEpgg7Eml9UkVVGU9UZdc3AL67XC4+jWfGBqtH8Dwfs9ERw6PmRxjXVbQPecx9xXKylS+p2qbn/PPhY0XYxafN+szKglAOzRrxF5degL97ywvTeShNOzQxlY7FZTobVtyopUYEjzDvztcOIPgcdW1CXJxQkrOKcG2ii1p2AP6h8OCCgoPDM07ejHf8k2fj8m8/gnddfvtqT6egYMVYU0S4aS0Wpy22dOTl+79nK278tR/FsZ1VQhbIEUbV8NSIUCxnegvihniER5XBpJENNQIRCNaIQFoqExIpcjnDbhl7rkxF5tfmjqVmjSf42LzAbNq2akpG1FBDiUuzVvpGZXyazBF2Ow+JC44Mkv0htkY4CwM1w5BJBrQffhvcN9Sw4fzQRRXdZeBEWF7sDPW2GgB/dcUOfOSae3otOa7DWbcNJ8IdYZz1ke1VhBU/zQ8/6yRVmeaEOxyzdvHlHhenrV/PMAU/eq+5NYJ154s6Kw44nZS8AsSpD/oMwzFcdft38WN/fLm6vBTLFRQcGrz+wjPxY997Mn7/U7fg2rt2rvZ0CgpWhDVFhKnojEjLuDbYND9it5d1lbSeUfhGcMU8oTCnrxFHn22Cr7PE49O6hhqVIAKc53Ii5hVjLTVixu6HFsvVyrmrqlDEddYv/z2Wpq1KpGP1t3utjYlwdLu8rqL9zCWpEaxYDraLcTOe7MrOckDwCKfWiEDww3lmxXLda+Q390S45kR4uCLMUVWGXVCF108+eiFezxhG7sLr08aqbZMlcvYYAPiZFz6ld9t4HH5xEl80ROuxYjlazxiWN802GjGyybvztRbMIz8b5LN3+0L02OcRvuXB3bjlwd1i/g7FI1xQcGhgjMFvveocnHT0At7y/mux68BktadUULBsrC0ivCSJcCiIAvIe4brSrRHP334sLth2rP9btlimbTUMIdbeGjGJrRFcrZTgZC9nnQDi2+KyoQbfdlZ8mrdGiIsIfnj7J616vFFqBDWeiDzCcXwaEJNLmRoRPMLOVlBVbn5Edsei2QXAUiN64tO8Iswyjr0i3BFheuTnSargfU1M4rxafZuTj4mJsLOghOcE6sA26zMW2X/EZ+lnX3QWdvz2j/du77dl+6HnWqEZ97SP2Hcu3A0I647r0FmOhlpiec1DMaqqJBM5nFOdCecSM/h3ovDggoJDg2M2jPEnrzsf9z9+AL/00W9ki1QLCg5XrA8iLEifQ6xyGeVI/9X3PQVP3XqU/1u2WAaAcVYRnv1LOq6rLjUi3L5v2zBPbYT5SBEmdU6V59h66eJKEKxcAxDNGgETj7k4aVRFOVJ/ATy0+wD2LE5x4uZ5P2+a+wlHude4j3skPMLGhNg295IjseQR5u/FgleEiQgLwt2NzZVJfgvfe4Q3zHWPZK/hNoH4xMbFcsnpYMv0OxPPOmVzsp60yQDUYnmZ1ojMN/nVzz0dzzx5s77QzyM8J0LdVyw3ba0/v5FHmH0e5lhDDQK9R7XG/jMY1VwRHmapeHTvkph3quwXRbig4NDheduOxVtf+nR84ob78cGv3b3a0ykoWBbWFBHeN5kCcFFU49pgc5cDG6wG+q3ruqpURdg1bAivWxu8lfRyTpUbUiw3rg0OTBpf0BU8wukcCfzWPC2eVSynQSrasxXheFt+Lg9MmpnFcq21uObOxwAAz99+HAD45ITXXXgmrv7Vl0Tz4XPi5ySkENhOIc5YI8gjzBRhPkWeGkGb8fg0WvVYoQhzSB94VIzZc/75Mv4+veMVz8E7XvFsth6/XR+2n7a2K0jr/4wZ8Z5p+IOfOhef/PkX9Y7Dtw2KsLI/RtbposRAj87TfPmk1C7XGpF0ljP5OQ5BZUqxXEHBoca/+YGn4oVPOwH/6e9uxLeFLamg4HDGmiLCXBH+0P9xsW8soEVQ8d+5nEe4MmmL4UVfdEUEUj9FwxpqVNh9wJH3+ZHzC/MEBo1McY8wHddKiuUGe4R90VNIV5DDHZi2KpGOpmWBa+7aiblRhbO7KDbbhi5uhJE/9kC6lqbxOfHNKOCUTs0akeQINy0WxmkGcFWFc9G0Ft+893F8/paHEmsEKcMcfYqwPBv8VGRuTGDDXI1/xby7/BZ9VCzXtDgwabEw6v96yqzslSJW6fN3IfgufEGnMcwWE6dG5JIslqPGjirD7hh08+0eZ/Hg04+NCwO9H7sowgUFhxxVZfCHP3UuNs2N8Jb3X+stgQUFhzvWJBHeMFfj/DOPxeYFR2Iqr2rqShwvmOKoTOodDj/WYVsN4yHxaXWFvYuOCNNcF6etIDDxNqOBHuE+vyrg0hA2L4z8ceQUbM5va0ZS+fhOEU63j3OEgWvu3InvPe0Yb1ug+DTtfeGWhcVp45VwXtjmFWHFGrFhzpHefYtdsZzoPEfEn/vDW2vxiv/8Jexbavy+pTWCQ9pJBhfLZS7ItPVO3Ox8w/z0Nq1rwsKJvQY+9sEonLkYt779EWGmLnqA8Aiz1AiC9AgP4aJ1VfnUiDAPUoT7qfDxnR3Hb0ff6eIRLih4QnDi0Qv4g586Fzc/sBu/+YlvrfZ0CgoGYU0R4X1EhAVBUBXhSL3Si+WqCkmwfvixDoRNQz0kPq022LNERNjZOPZPGpF3LOakkKhZ6pz2o/76F5yJT/78i1iObeY4lEIpWUy0f9JkUiPCcwuLR/Ys4rQtGyJPbisVYSXfd3HaerWXq+B0oeKJMCOmm+Ycyd/dVSkvNW3kr+bHQs/5fOn4ZLEcR39qRP7952SyL2feGIPzztwCAJhMw4qTzlc+P4sI84uYg2B2PM5MNh/hiBpvRPFpqUd4XJnEt7zEGnEA+gWcxLg23lssEzZmKcKkqKffsWEpMgUFBcvHDz7jRLz5RWfhvV+5C5/85v2rPZ2CgplYU0SY4tM2zulEOO8Rzlkj8orwrCKzXBFdtE5deUJBRPjApBGEXZItrVguHXsW71kY1ziNZcbOskYArqkE0EXHsXWs1S8IeBMJR3odSeGqbtta1TNbMx8yL8ajNZvW+jzjxUlqjagqg6PmR9jVWU+mTet9wwDzCLO7AZyo0fviiDtw5nEbk+PrI8KAeA8yqREy31jivDMcEX5g1wH/msvLbryynsehsUZwG03I6J1ljQiklNa0whohiS6di6FFb0AXn9bG30l6zAnCn3nrD+Dz//4HvaIu91NVy7NnFBQULA///keegXNPPwa/+OEbcM/Ofas9nYKCXqwtIsysERzBO6jbBTjp4uDKECmmQxXhHEHm4OSTSObipI3ItxyeD0uryRazgIxPm42cp5nvj+wbvPVv3/ayoUbTuva5vJjJCmtEzQgqnfN9S8EaUTH1lnKcNWsEzZdyKyeNjRRhb42IFOHAnPZ0lpUzjtuIL7/9xbjkqccnx9cXnwbkzzv/rC0pUV7nnB7aWRMR5pi2LnJvfoZHOC6W6121F94GxHziykcu+szNSo0Y1VpqRKcIL2Ou1JTG7b+bh/986Uz4aScehaecsCm5kKDtZTFoQUHBocXcqMKfvO58tBb4dx+4LrE3FRQcTlhTRHhfZzMYYo3gLIWTL7kdES9SEz1x8YpwhggPYB7cs0qK8FLTZltBA/Etbm+NUP4N6VOVNeQUYU7uvCJsUhVtSIvl1rquYca/ZhNrBI0zqivM1e6c712csgYNbj1qIlIZ44suxoIYHr1hjF373WdiqWl9kw03fnjPvUeYETUe/H7KMRvUc9hXLOfmGv6WLZYJmiL83n/9Anzy578fAHDS0Qu4cPtxeNX5p/nl08Z1UJzlEc554pcL7g2nN0+14/BWzKyyMsTSheVzdXox9f+3d95hclNX//8eTdnm9a7LuveKjQE3cMXYxgQDAYduILQAJkAogUBI8qa/+ZGEhEBCKMYJJHQCpEACLybBxqGYYLBNBxvTm8GhuW27vz+kK11prmakmdmRZuZ8nmefmdWoHGmKvjr63nPs8mn5Vo1wNhkI7++E7RFWbDkMw3QNQ3s14CeHTsCa1/+Lyx98JepwGMaXshLC260yWvXppGu6rjuaep6zRZHnxGyQM01m37wZYd0gMXWd2VB9ud1qHA+qn2AH9KW3tBlhn30NEouKuh9SrOu8m7rBdqpWEhB2RQy1Ba7aSQxwv1cypq2tqjXCEa0Zg+U8Yr6xNml7hLNbI8znHZqMcDbCZITVY6HOpxPC3WtT2KVfd/v/O746A4tUISwHyyWDD5YrxPNKpBwrOO9dtu2pg+VgC2GPNcITUj7WCFmLW40gV4tlid+FhO6OB8MwxWfRxIE4auog/HbFBjy64cOow2EYLeUlhK2McMYtT3I/ms8zxZf33Ke2UJZC2Hsb1tcaEaiOcGZGGHCLFu/qXSW67Oxc5rr9Bgb64RevLiNs+nyzZ0PNuBSPcKdjZ7Bvr1sZYa9fW65PZsy37mzPaI7SIYSVsXcabnhj6F6bwqc72u3Wve7Bco7VwrZGuKwc2sORdZ+9Hc78jru6mF+XMy/qqkxrRGdOjzCF/Az4oQ4snDy0GQvG9cVPvjRBsz1nI+r7pbaulnjbaQOZg+WCxubnEc6lhP2E8PQRPbVWGIZhis8PDtkVI3o34Lzb1+Kjz3dGHQ7DZFBWQnhbawfqUokMkeZYI/RZUj9rhDqITp40W626tLZH2CeTGrSznEQVwu5srmdfNJUJOjVK2E/0+8eSW9B3q3WsERkCXddZTrVGWP8nDLeAz+4RdjrLpRPu99CuI6wsm/ZaI6yMsBSbbo+wcxfA21AjF7LJhve9b2v3CmH9MXXVBM5WNsK1jPPc7EYYpHxa5nHNB7moQaZFaNmJUzG6b2Y3Olf1D0U8azvLJTLtBxnl0wLcy9B5hINmhKVVps3Tme7MuaPwrQPH5dw2wzCFU59O4jfHTMbH29twwZ/Wac9nDBMlZSWEt7d1ZFSMAJQ6wpqBZoAj9rwZKoMcAZH2ZoRJLqs/WftZJlRU8dlQo2SEVSHsWb3uNb2YCid8/KtGOM8bbY8wZdQu1leNcOi0BsupAxOzeYQTBmkHt3mznK6SXZ59MD3CqhBWrRFOHWGdNSIbj31rXzz/o/0z9tk78M1ljVCn56FJ3S2WAwphZTuF3Oq3LyRziGm38HaqfMiPijy8uw7ojr3HtGR8RzIaagQIWSfw5eK56gjnspYwDFMaxg/ojv85aBxWvLQZv39kU9ThMIyL8hLCrR0ZFSMAP4+wctJWSj2pqF5BWbO11Vs+zUfwhs0Iu4SwyxqRmaV29sFEdwUdNgHoV+XCZY2wMsIdncKu0KGLS9Ko7JMQZpyGoQ6L6UgYAAAgAElEQVSWk3WEM/c3qWSEAVUIuzPiuiykvf3aJD7f2W5nGtXBcq46wtY6c5Uyk9SmEqhPJzPe+7YO9/vgb40IL0rVRTo6BXa0uwf/5VqmGEI4Vy1i9eWUPbjRueiRF0+XHLYb9hzWMyODn19nucxygvY2cyyr+61gGCYajp8+FF8Y3xc/u/9FPPPWJ1GHwzA2ZSWEpTXCi740mvNcnuAzq0Y4GVGnZa+nfJpH8ObKFKu4y6c5cbuynt64NSJZl8kMW/4pyGA5OaBv6852u2ZztuWvOX4KDp88CICa/VUGywnHN2xvTwpUxSMMQGmo4ayfyP2eZVojUugUwMfbzQFzOmuE2sHOK+5z4W2a4hXSfrf285Gk6jKt7Z1obXcP/tMu47rAyGOjnmVzfaZcd1kS7rss8qJHnZb2XHzJ4xemaoT6+XSsEbC3mQ3dbwXDMNFARPj5EbujpVsNvnbrU/ZAZ4aJmrISwr7WCNvjqFeYanbQvZySEbatEZ2uxTN8xbZADtZiWdKQ1lsjvGog4RGCgF+L5ZybdxGkfJr0MX+6ox072tyiT2cF6d9UhyVzRgAws3MdwqoaYc3qlEBT1+MIVFXYSo+wulsEd71XXdUIANiytRWAxxqh1CWW8QSpFKHibaPtHfhWzIywylafQaFe1K0U4hF27CrZ51OFvyyf5vjBRYYQ9ma0Wz22oyAXc7q21kG8xYD/oFqGYaKhuT6NK46ZhDe3bMN3//JsTnsTw5SCZO5Z4sM5+46yM7Yq+sFy+iykijpYzq4jLDPCtmAzMpZp7xQZGVLdSTaVdCaq1ghd1lddv3cftDVdQ+Yd/b3OmUL4853tGfvjt7xLCHVazQqs2Do6M2+FG8pFic4aYXiEj3rR4M3OywYgH30uhbBSR1gRd3Ift4YUwl5x6fXsqvul/qDnJbiUZWScOcunuS6a8ld5pPn+5NqefC/k+9bWIWyvsO27z8gId7heD4KuioqdEc5hjvC+X2G/MwzDFJ89h/XEeQvG4LLlL2P26BYcMWVQ1CExVU5ZCeEpQ3tqpzviypnm8k9KUeQ50au33p0R5u6MsK6pwk5kZkh1IsLPI+wS7J7FdK/pMsJhdY//YDnVGmEJ4R1tGV3c/KpnqLepOzxVI2QCVefzTSb8Bsu5b4Wrh9krrLrXeTLCGo9wQsn6bw1pjZAXO7sNbMIhewzA8TOGul73ewvyEaWqSJNe5HBVI0Jv0oVBAYSw8lxm3Bssy8+21nb7cyrfb29nPLuhhvT6BojL1XJcLhHQGpHLWsIwTDScNW8UHt34Ib77l2cxaUgzRrZ0izokpoopK2uEHzqPo+4k601E6awR3qL/XgEoRbU3I6wbaOQqTaZ4hF1tlD3L6G4F64pGeDf3/YPH293KdPgKYXWwXI2TEd7R7haN/p5oJ2vd0Slc7Wud+q+ZGXC1fBrgHGeXNcIjzjKtEWZGeMtWszalKnzk+2MoHmGZaT1vwWjce/Zsn/1xkGKvLpXAaXNGZArTIiYYdRo0lzVCfUsKtWOolhb/7WVaI6QPd1trh2ON8FRikTgd4ty2imy4vmd2Rth8EnawHFsjGCYeJAzC5UdPQm3KwNm3PG13D2WYKKgQIazLCGee9bzWCIM01oiMqhGZGWHzMbf/MK1YI+rS+oxwhm9ZY43Q4V3u5FnDXd3KvPi2ilYHy9nWiI5AHmEzDvNRZucMpdJDZ6czzbu9hHXsvbfRvcfGUEST95a6XEZmet3WCMO1HcARwnsN64kJA5u0+6Mi32vV4qKiTnWVksujTqZuC7kywupChQphUqpr+M/jPHcywuZnZmtre+ZgOR8hHLahRkYcAZetTVbEzxvDVCT9mmrxiyP3wPPvfoqf3vdi1OEwVUxFnCm0HmHN2TKzoYZijQjYYjlh16fVx6AiM5jppOGyGmSLM+ESgpn74LdcLvyqRqixDGyuAwDsu0ufjAoLfkLayf5at70NxyPc3uluoKBuL+HJrGvrCCvL6jLach07LdGednWWc7Yj1yEFc23AslpeH6wX9WJLvU0ftHGH37okuQfL+V9QhSWQNUJzQSMHr25r7bAvfORn2Gtl2eltqBEgZJdHWMbqmISzkukRZhgmTuw7ri++Mms4bnj0NSx//v2ow2GqlIoQwvYgmiyWAyDzRK9mweRJ0+sR9grIpJXFzOxul7k9KaBqkoaryoQ7Dn3GWcbnR1jhE8Qj3FyfxrrvfQHn7zcG3z5wnC2MvXG54zAf25Vsn50R9mQI1e15M8HeFssAAMrMGLtjNx+ljcNtjXAaasjtb7MywkHLatkZYd+KG/rl8hHCOsIMlivUI5wgytlQQ92mrKhRb93p2Lazw64SIufxrSNsl0/Lvb2Epo6w9/PlR0tjTc71MwwTLd88YCx2HdAdF965Dm9u2RZ1OEwVUhFC2K5LmiMjrPMI584Ie7PIptDz3kbOlhGuSRouQe3yCHtj0mTAdITNbvk1BvHuX1N9CoZB2G1QE/5xjuM59isX52R/HdFrZ4k95bLMODxC2Dru0n7g9Xn7ebXltgAnI+yyRiSc7djWiNawQtjJ6OtwZYSV6fl0ENV9XnM21FCeF54RzmyJnA1vRnhra7tdOcP73kq8361A29F+H4ItP6C5Dn8/ZzZOsAY5skeYYeJHTTKBK4+djI5OgcVLH8drH26NOiSmyqgIIay3RmSe9XQNNaTwlKLD6xH2ZgMNw1y3V1fqsmkyi5lOGK71uMuneffFeZ7txF0sa0Q2v6YqPnOVT1MrBmROy7yFL9enHiMgU9zJCw5dVtYWwjIjrAhHu2SeWjVipzlf0I5j2bLR3lhV8rJGaKblqnqgK0uXL+qgwmzIXUt5PMLbWzuUKiH642aXVwsxWM7VUEPe+QnmjAAA7Dqgyff9YxgmHgzv3YBbT5uOba3tOOrax/DK+59FHRJTRVTEGcLwnCCBYNYIVWhJ0dHW7s5ieoVi0jBcJbkkOkEps5w1qYRLSOrqHfdqSAMwayzar2mzzNlv1/vhN3+2DnlJVxY7uxBWPcLejnjqol5rhOwo51gj3MJHXnDorRHmvHJgn2olcHmErUWD1ueVpGzbRvZ996Ird5cLfdWI4NaIAnWweZcjj/q+akbYqRoh49OvL1TmWTN4VG3rHAQ5F9cRZpj4MmFgE25bMgMCwNFLH8dz73AbZqY0VIQQ1jUE0Ldd9ghhgzJbLHdkt0aYg4p0ojozLvvWesJwrUfXJGLqsB749zfnZRQX9+7GvLF9cMXiiRjSsz5zg1nwrwOcRQgrae9cg+3aleMm57TtEuq+5/IIe3zeWa0RcrCclRFWb8XXJBNoqkuhpbHGPt52x7Z0sI+9M6DPb34lJkWU5VM1QnfpFmawXK6KD7kwNBd32ZDfF3uw3M4OrSdcu61QVSNUj7CM1XwMe5TZGsEw8WZsv0bccfoM1CYNHLP0cTz9xn+jDompAipCCOtuterOeXpRKzPCVkONdrc1wnsrP2kY2tvIOuFt3/ZPGr7+ZVkLN2EQBvWo1wzCc/9fk0pg0cSBoZs2BBksp3vNyYz7+WTNR1dG2JpVCkI11gwhnHRnXVVxR4Ss1gj5ms4jnEoQVl44F0dOGeSyRhjkb3XwImPy8wj7D5YLtHoX+WSE1Q95IZ3lAMsaEUKgzhnTAsAZLLdVaaihE+W6ih5BYtZ5hDmzyzCVy/DeDbjjqzPQXJ/Gl5etxupXP4o6JKbCqQghLIVXrvJp2QbLyRN1qycjbNsuXOW4NB5hzfZsa4RHSKnis7tVu9cvi+admm9v9nyEMOBUB/CvGmHZIFweYStL3KmxRtiZdncmWB5/8oi7bOXT5HvgeIQd4UhEaK5PI5kw7PduW2s7apKJwKIxp0fYVwiXpo6wu2pEoRnhcJaFEVYnqIRhdgjc3tphXwDoPss1rqop5mOQzenuBMjVBz3MRSriwTBMiRjUox53nD4D/ZpqceL1T+DhlzdHHRJTwVSEENZaIzSnWW/Gy/SgOgI3aVBG+TQ7e2X9n7IqQOj8xl68Ik83r5oR1u+b+/98z+l+688loOTrvoPlrEfVBiGn6QbLJRLu9cljI20YmR5hmRHWvJ/2YDmrjrBGbKnPO4W/RURHymPb8EJwMubuqhHmf0vmjMB95/p3+8tGfTqBhhyD+tyDEPPajE2CcjfUAICLFo7FXWfMdE1rqEmaHuFOt0dYRf0O5Fs1wttZLigi728NwzBR0a+pFrefPgPDe3fDqX94kusMM11GRQhhfWe5zPl0Jc/sDCURkgnKaLFsD8yxljlp5lD85NDdQglhb0ZYFeTd65La2Jz9KI4S9hO8uUSFFI45G2rIOsKK11QKYXXJhP1eubOttjXCpXucdWWzRsgsvtrJT1e7GMg+ONBLKmEgnTDsjnteiPTHT+735CE9MK6/f7c/97rc6/nSpNz2F/XVQqtGUECP8JlzR2HK0B6uafXpRE6PsKtqSoiqES6PMIIvp6NQ+wjDMKWld7ca3HraNIzr34gzblqDe9a9E3VITAVSEUJYarRcJ7qMSg9Errq2KcNwMsJ29sm9jtF9GrH/rv0yO8vpsmA+t/1V0dvdygj7CRnv1HybNdSlEiBCxiC7XOuTAsavDrHclQ41I+ydphGl3oywfPRmhB2PsH+GX75nqthyV1RQtx/8I58wCLefPh3HThuifZ2gv4CR+x1GdKtz9mpI48y5I3Mv47OP+WAY+s9wEOrTCWxrdRpq6C661NJ2zl2W3DG7G8xYsbKgZZiqobk+jZtOnYbJQ3rg3Nuexp1r3oo6JKbCqAghHDQjnOnrJbc1IkFKG1gnU6wi//WKbp0gcgZbJVzzqHFKj7DfqT0jIVxARnjTJQfh+OlDXdNzDezyNsDwYleNUERvRkZYWVSux2t5sFsse7Yt37NsLZZlyTtd1tH7PIw4BYBJQ3rYFyteiBRrhHIc5fMwNgy5nuG9G7Dmu/thUI/cVUGKWTVCVxIwKPXppNVQw4pLsxq1ZF2YzbjqCOexPOC8HyyfGaY8aaxN4Yav7IlZo3rjG39ahxsffz3qkJgKoiKEcFCPsFfMkeEWZsmEkeERtjO1noFAQawR3q519rIua4Qpsra3dWj3zbveQv2OcnUtjTWYObIX+uZoQ2tXy/CzRliPdsUAQ8kS61oskzcjbAokWwh7ssfy/2zWCPme6ZovyPXonheKao1Q3xeZZffLomvXlcdtf/fAwuDL6VAHjoalocbKCGepGqHNCAeyRqjvaeaFZBg4kcww5Ut9OonrTpiKBeP64Lt/eRbXPfxq1CExFUJFCGG7soOaBdTsmU68Du1ZjxEtDRjZuxtSBvm2WJZCR2dvUKe7pxHSCcO57a+pbiGzjbLZQ8Y6PP8XOgJebnva8J645bTpvq2TJckc1gjDI0bNOsLujLC6qBwsl8jICGeKnIRBOawRcG07ZfhlhNX9Ka4Q1onHbBaBbOsCwmV2/cR+PhjKRUdY6lJJbGvN7hFWffJ2ucMA605q6ghzbpdhqpPaVAJXf3kKDtqtP37yjxdwxYOv5F1JiWEk+lFAZYZu8I22akSGEAb6dK/Fvy6YC8AUfZ2e27t2QtiTEfae6/1uK9cpo/+d0mHO642WNUK2//XiFSeFfuV1Fw3ZSHqqPGQgs7+uznLmtPuffS9jWwnPBYYUSGmfjLBcl06wy/XKwXIpRWyp0UoLgxCFWwhU1KoRKp15WCMkocSzspcFe4TJ8dqHxcwIq1UjdELYXdouKK6MsBIrwzDVSSph4IrFE1GbSuBXD76MbW3tuHjhLjwYlsmbyhDC0t7gGWjlJbOhhvt/3a11vw5yQawRAHD1lydjeO8G1zyujLBljZBdz7wUyyMsCXNbGnCyrP4eYfOxrcO5LS5/kFZatR/VHyiZ4EvaGWGraoRtH3HWnVSaPGhbLNvZaOkR9heGCSK0C1F0a4R9x0B5Xzo7w2eEnXXmlxEutGpE2M5yKvXpBLbuNOsIe8NIGISOTuHqkhfmM6i7ExD2hCczRnyaZJjKIJkwcOkRu6MubeDala9iR2sHvn/wrgX/DjLVSWUIYSlOPe15/eZz/ndP0N1a9+sglyGqfb6AM0f2zpinMGtEoR7h4LelgdxthuX6OjqdsnOZx9l5btcL9ghcu8WyZ5CbUz5NZ41wWzBS2tvoyrydxRXCfuJRWiNSITzC8m0N2PQOgKd8WoG7Nb5/d/Rrqs1r2ZpkAq3tZtUI7/GVQlhtduLMkjvopKYSSNh9ld8YzhgxTOVgGIQfL5qAulQC163ahO1tHbjksN2L+hvPVAcVIoSluMueEfbL7kr0GWHz0T6Z+iwb5LvnbS8MOHWEPw9ojSgUx9oRbL2pRGbM7vWZj2qL5Wxtog0yj21KCmxPi2VVGyUTzgAuPy+zFFoGuS9GMmNw1lksTpgxFG9s2YbrH3nNNd0RtcG3lc1f64fLRlLg5+SyoyfmvWw6aaC1oxOdQmTeZTEIrXB7hMPUEXZfAOkvThmGqU6ICN8+cBzq0kn8+p+vYEdbJ3551B6+iRuG0VEhQtj9aKLJIOawMyQ15bf87BTZxJ5/nPJE7kxrzJER9mqpYnmEg2oJZ7CcfgHvwDi1aoR3m+Z2Cb88cg/sOawngOwZ4YRh2Ovya3OcIEIHhNKZTr8f3tbOxeDkWcPx6MYPcf0jr2ktK2FEt8wih7nwUfc1SnGYShDaOgQ6OzOFsOMFVzLCIS4Q0j61ocPFl93ewzBM+UJEOH+/MahLJfCz+1/EjrYO/ObYSa7fHIbJRk5VQESDieghInqeiJ4jonM18zQR0T1EtM6a5+SuCVePPLGqIiKIR9g7T0rNKNrzmM+EJ2PnXdfYfo2547QFu7OsHCzXUKP/0nqFUb4NNZwYMrPn2chZPs2aLH26OruAdx8OmzwIg63GHumkVwi7t53NGgEolTiUihw6UWjXiy6yFsp2HMMIL/n5ChOf23sdnchLJxLo6BRo68i0RiRtIRy8aoTajc/VJMV6DLuv5y0YjdP2Ho4jpw4KtRzDMOXDGXNH4gcHj8cDz7+PJX9cg+2t+rusDOMlSEa4HcAFQoiniKgRwBoiWi6EeF6Z5ywAzwshDiaiFgAvEdHNQojWrgjai7ahhm4+ZQbzFr17LndnMv1tWJ1P8dbTpmPSkOacceqsEamEgUsO2w3ThvfULuPdj4IrxYTOCFtCOGdnObV8mpts2cqMFsueSgh23WGfjLC38YlhkDZtLucLU9s3CNmOY5jmHVbhi7wzu1EmO1NWa+ud7R0Zx0O+b7Uaj7Au+33DyXti7tg+yvKZdpewu9pYm8J3DhofcimGYcqNk2YNR106gYvvfgYn3/AElp24J7rVVMSNb6YLyfkJEUK8C+Bd6/lnRPQCgIEAVCEsADSSeabqBmALTAFdEnSZVt1JNtetZJ1HWOqvGSN7YUjPBvS3BhSpy88Y2StgnHpxfcxe+ha+ZhzueQvVwWFH0Evh6JcR9naWSxiZGdlsIk02Wkj7ZoTN536eL5cAtpbv1M2nuQgpBnJtuvclzLY6spQeC0KUt/3le7ejrdM3I5xWK3pkidX7eU/rMsLsEWYYxoej9xyC2lQC59+xDsf/bjVuOHkvNNXpu4MyDBDSI0xEwwBMArDa89KVAP4G4B0AjQCOFkJk6BEiWgJgCQAMGeIv/sKiE5g6L2zC9XrmyVTXoleemPt1r8Mlh+3mvJ6H8HBu4wdfVoZZmzKwo63TN3McFG895FzYg+V85pdTO5QWy5keYf9t7b9rP2xr7UCL7HCnvl8JtXyanzXCLXDNbWXK0q4YLAdk9/SGGbDhWCPyiy/KigjS3rKjrcPXI6x+KLJZI7zTdMeQdTDDMNlYNHEgapIJnH3rUzj2usdx4ynT0LMhHXVYTEwJfKYmom4A7gJwnhDiU8/L+wNYC2AAgIkAriSi7p55IIRYKoSYKoSY2tLSUkDYbhyPsBKvZoS5YZBzC15zMlVvZctn8qTtLVuWTwLOzl6GWFbOO7pPI1ZdNA9n7DMy/IYVvA1DciGbWviJdzsj3KFWjXDPk21bfbvX4qv7jNRaUVSPcLaqEXJewDzGWo+wj7e7UOSmdGXtwlWNMB/zdW4Us1FIWJyMcKYQThqZn/lsVSOC1PZmIcwwTC4WTuiHpSdMxYYPPsfipY/hg093RB0SE1MCnXaJKAVTBN8shLhbM8vJAO4WJhsAbAKwS/HCzI5TR1hjbfBkid2ZQze6jLAUJh0ZQjiPjLDHzxoER9ADg3vWFzwoymkVHWw9yYSRtQ2zXE277RHOXZ0jG94udLnqGHvvBpBm+0CmYC4W2dYWyiOcR/k0lUg9woo1whvHVcdNwWGTBmJESzd7Wjax7919tzWCXI8MwzDZmDe2D64/eU+89d/tOHrp43jn4+1Rh8TEkCBVIwjA7wC8IIS4zGe2NwDsa83fF8BYAK8WK8hc6AbLSdQTa3N9Gj3qzdsjumydK/ukVCEAnFv/3m2GilOK8DysEcW69S1CZoRTBmUVdOQ5Prp9C1cb13muZpf9rBFSJ6keYN2cfv7sQsm2uq6uI6wSadUIaY1o78jY5/EDuuOyoycGriOczRqhG6jKMAyTjZkje+PGU6bhw8934shrHsPrH22NOiQmZgTJCM8CcDyA+US01vo7kIi+SkRfteb5MYCZRPQMgH8C+KYQ4sMuijkD3W11Z7CbM+3s+aNw86nTXK+rqBUFbGuEIa0R7nkLs0aEz5AW6+QferBcwsgp6IiUwXJE2OYpW5OPFcTcdm5rhFfg6jzKgJOFLLZHONuRDFOhotP2WOcXRbR1hNWMsD4Otfyd3WJZd+w8k1LJzGPIg+UYhgnDlKE9cOtp07GttR1HXfsYNnzwedQhMTEiSNWIfyOHbhJCvAPgC8UKKiy6gXF2K2RlYmNtyi6yrbdGqDYK8/m+u/TFfuP74uID3E6PfLym3koU4ZYtUkbYegwqJpIG5Rz0ZRC5PMJbtrmr5oWJ3dUtzTDs9y+XNcIWV0Ta7GhXNNQwt2c+FtpQY/bo3jh4jwG4aP+xecURZSMlme3d2dbha3tIZml/DTiDQb3i2OXbZ48wwzB5MmFgE25bMgPHLVuNo699DDeeMg3jB2QMZWKqkIroQ6jLtMpnXlGk70JnohuYU5dO4LoTptoNIJzXw5+NVbEWlGLfDpaZx6AhLBjXF0dNHZx1HoLqESbMHNkLX5o4wH49TOzqrAklu+vXUCNjsJzhU0Oa3PMVi2xrC+MFr0km8JtjJmV8zgLHEYuMcOZgOe88gHJclFnrUvIC1X852yPMSphhmDwY268Rd5w+HemkgcVLH8PaNz+OOiQmBlSEEHZqyGYKWe+JNZtXVM1a5cqY5mWNyDJQz387xT35y8Rl0LUt0GTDvRhEikfYFHWXL57kvB7iYLlbLOceLGcfU+XYZu0sV/SqEf7rK6VvN9KqEbZHuNM3Dt3dFnVOv895gjPCDMMUkREt3XDH6TPQXJ/Gl5etxhObtkQdEhMxFSGEdaJXZo/82irrRIpf1lFHftYIc5kwbZKLnRF2BssVUU14PMJewnmEnZlVj7C/NQKu7Zoe4SzWiGIPlivq2vInWo+wue0dbZmd5SSqx1v33QsjctkjzDBMIQzuWY87Tp+Bvt1rcMLvV2PVK5ujDomJkIoQwtmyppktkskqsZW5Hukf1i2Xa71BkDq7szOEEC5gezrsjHARtYRBTh1hvcgJbwUBrBrGdkY4uzXCqciRvXpIosiD5eT7Igru+VdgHBF+k9WGGn4XiDpLivxcNKQTgF0azR+No4JhGCYv+jXV4vbTZ2BYrwaccsOTePD596MOiYmIihDCSc1t72zlqPxun6slnnJpt3yEpIzPW4otG9lKTeWDUzWieHKCQDkywnkKYcrdYtkZLOf8r9tcl9URjokqizJLKmv9dgr/ONKa6g9yzh4NaWXAq/9+6JrkMAzD5EvvbjW4bcl0jOvfiK/etAb3rn8n6pCYCKgIIXzwHgPws8N3Q7capwiGI4Qz59c1fQCAmlRwj3A+t9jlOkPoYFstFC0jbLdYLsrq7HV1KIPlJFL85GuNSBhBrBFSCJuv+3WWS2gulrqK5vrS97UvxX75kdbUCPaS7QKkR306kDWCPcIMwxSb5vo0bjp1GiYNacY5tz6NO9e8FXVITImpCCHct3stjt5ziGuapct8b9XrbiW7rRHZt1lIZ7lQHmH5WDRrRLiqEUEgcjLC6nGVmcJ8M8LJBCnl07JbIxKKSNJtTx6/rsoIq2/pA1+fg3u+Nruo28lFHDrLAf4WDd2FjLwz0lyfUrrG+eN8F/KJkmEYRk9jbQp/+MpemDmyN77xp3X442OvRR0SU0IqQgjrkGJTlynzyxq6rRHZz7altkbEebAcKR5h9XhL8RpmU96MsFM+zScj7Mn0+lojpEe42HWEIT3CDn0aa7HboKaibicXkVojlO+N350SXU3lj7eb9aZ7BrVGFPnuCMMwjKQ+ncSyE6diwbg++N5fn8O5tz2NT3e0RR0WUwIqVgh3qzVtEtOH98p4LYg1Ihf5CMlzF4xGU10KewxqDrxMsVsDy9v2vbuli7I+wMzUdWg8wo41IkRGWHmeoADl0zziSBXPrvmUOsPFJC6aLA6d5QD/74Xu/duy1TzJmNaI3BnhIAPqGIZh8qU2lcC1x0/FBfuNwb3r38WBV6zCmte5vFqlk7OzXLnSu1sNHjx/Dob0bMh4zS9rqFojuoI9h/XEuu+Ha8BXmzZjKtbJ/8gpg2EQ4dBJA4u0RjMrKxtqqEIolZc1wpk3aRhorE2BCGis1X9UtZ3lslgjuq6zXLRVI6L0CKt3UvziSGmOe68G82Js1wHdsdwasZ3ts8IZYYZhupqEQTh739GYNbo3zr3taRx17eM4e/4ofG3eKFcZSKZyqOh3dVSfRv1odb+MsGbeqGmQQrhIJ3/DIBw5dcI+laMAAB3kSURBVHBRv9AEZwCgKoSkRzicNcJ5nkgQ5u/SB/d8bTYGNNfp58+wRujfW5mpLn5nuXiIsii1ocsj7BOHzhqxaOIA3HradBwxZZDtLQ6yH6yDs0NEtUT0BBGtI6LniOiHmnlOIqLNRLTW+js1ilgZJq5MHtID/zhnbyzaYwAuf/AVLF76ON7csi3qsJguIH7KrwQYBmm9jF2dEc6H+rSZCY1yMFQuXL5eTUY4jCeaPOtKGIQJA/39tglPRjhh6KWpFFrF7yxX1NXlTZQZYbUDoG/VCI0QJiLMGNnLrO0d4IKi2ANHK5idAOYLIfYAMBHAQiKarpnvdiHEROtvWWlDZJj401ibwmVHT8QViyfipfc+w4FXrMJf174ddVhMkalOIexnjQjhES4VDTWmOI/z7WA1NFfVCCvD3treGXhdroxwAHHnVI1wrBG6Q2V0WUbYJFpjRPSfDzkw0i8OnTVCJVj5tPh+B+KEMPnc+jdl/UX9EWWYsmXRxIH4x7l7Y0y/Rpx721qcf8dafL6zPeqwmCIRP+VXAnwHy8XQGmFnhOMXmo0qUAxXRth83tYRRgirHuHcwsfpKCcFsY81oovqCMdFm0UthKUNxu/46soYul63B8tl8QjnGVs1QkQJIloL4AMAy4UQqzWzHU5E64noTiIaXOIQGaasGNyzHrcvmY7zFozGX55+GwdesQpPv/HfqMNiikCM5VXXEaSOcFyotwfLxVcGuCo9KIKnb/da8/U8RVou8QQoZdGU6hp+3QS98RWHeLwvUVtnZPY/Xz0uF8t2wReXi45yQAjRIYSYCGAQgL2IaIJnlnsADBNC7A5gOYA/6NZDREuI6EkienLz5s1dGzTDxJxkwsB5C8bgjtNnoKNT4IhrHsOV/3ollP2PiR9VKYTLKSMsB8vFGcMnI/zTw3fHDw/ZFZOHhCgXF1LR2QJXuTVfSmuETcS/g3HPCOdEWiOyZoRZCYdFCPExgIcALPRM/0gIsdP6dxmAKT7LLxVCTBVCTG1paenaYBmmTJg6rCfuO29vHLRbf/zigZdxzHWP4+2Pt0cdFpMn8VN+JcAvaxhHj3C91TZ6ZwifbalRD6UqhJrqUjhx5rBQGeGwOsrweIQNQ5+BTtiD5bqmfFrURDlYDgBS1kWk2uY8DLY1IqtHOK9VVx1E1EJEzdbzOgD7AXjRM09/5d9DALxQuggZpvzpXpvCFYsn4rKj9sDz73yKAy5/GPeufyfqsJg8iJ/yKwGmEM6cHkdrhMwI72jriDgSf9wZ4cLWFTbr560a4ffedvVguaiJWiTK4z+4Z31ey5PnkSmI/gAeIqL1AP4D0yN8LxH9iIgOseY5xyqttg7AOQBOiihWhilbiAiHTR6Ef5yzN0a0dMPXbnkaF/5pHbbyQLqyomIbamSjWHWE54xpwT5juvZ2oRwstz3GQlhCVPjI/rA61Vu2y9cj3EWD5SRRO8Sizgh/vM3sEjeoh77ecy6yVY1IGMQevBAIIdYDmKSZ/j3l+bcAfKuUcTFMpTKkVz3+9NUZ+PU/X8GVD23Af17bgisWT8Ieg4PbApnoqOKMcOFC+I9f2QunzB5erLC0yPJp21rjK4TtGr3FSEvmaY1IKtUjtGKqywbLmUTdWS5qj/CWra0AgME98ssIO9aIzP2Q723UWW+GYRg/UgkDF3xhLG47bTpa2ztx+NWP4qoVG/givgyoUiGsP6nGsX2izAjH2Roh7QxhB7rpCCvoZNUIue0F4/pg/137adZrzV/08mnxUGdRC2FJvhlhiW4vHCEcj31kGIbxY9qIXrjv3DnYf0I//Pz+l3Dcssfx7ic8kC7OxE/5lQCDKPJbyUGRGeHtcc4Ie0qYFbauPKtGWJ/kE2YMw1nzRmXO58kcVxpx2a2BeVsjsmSEZavu/MNiGIYpGU31KVx5zCT8/Ijdsf6tT7Dw8lW4/9l3ow6L8aEqhbCfRziO2Bnh9vgKYbL9uUVYV8j5vVUj/Ohya0SXrDU4cbmwy3fAqQxftxeyMYv6Fn99wRjcctq0vLbFMAzT1RARjpo6GH8/Z28M7VWPr970FC6+az22tfJAurhRlYPl/GrNxpEGOVguxhlheSyjsUYEs2XYVSMSxX3j4/Ixito28D8HjcMbW7blvXyuwXIAoNqwz10wOu9tMQzDlIrhvRtw1xkz8avlL+PqlRvxxKYt+PUxkzBhYFPUoTEWVZkR9hssF0fqLWtErOsIW4/FyEpSyE9kUMuDUzWiMj/yUWeET917BH60yNu8LDi2z1zzvUxZ1ggedMIwTDmSShi4aOEuuOXU6djW2oFDr3oE167ciE7+TYsFlakKcmBaI6KOIhj1ZdRZrhge4bBrkN7g3Blha/4uugCKuGhE2Xye/cgWvxTCrR3xvRhkGIbJxYyRvXD/eXtjwbi+uOS+F3H871fj/U93RB1W1VOVQjhhUNlkBmtj2OTDi3NbOzprRE6PcBfVEY7LjYVyuMPxyMXzseqiefoXs3SWk9l+zggzDFPuNNencdVxk/Gzw3fDU69/jIWXP4wHnnsv6rCqmvJQg0Xmwv3H4oy5I6IOIxDF8N12Nd7KDcVYV1Ao4CC4rvIIx4VyEMIDm+t8O8/ZneU0+3HU1MEAgN7daroqNIZhmJJBRDh6zyG495zZGNijDktuXIPv/PmZWI8FqmSqcrDc3LF9og4hFL89djLG9usWdRg5KYo1osDOcn4YAQVzvoiI60ZE7REuFBm+bjdO3Xs4Tpo1zLZIMAzDVAIjW7rh7jNm4ZfLX8K1K1/F6k1bcMXiidh1AA+kKyV8ZikDDtq9P0b1aYw6DF+MgJUbgpCvEM41WE5qqGLXEaaY1I0ocx3s1BHWHE8iYhHMMExFkk4a+NYB43DTKdPw6fY2HPrbR7Fs1as8kK6E8NmFKRgpXotSNSKksAwqwo2AmeOwyNVFPVgu6vJpheJYIyINg2EYJhJmj+6N+8+bg33GtuB///4CTrz+CXzwGQ+kKwUshD0MbC6sRWw1YovRogyWCze/TBRG7RGOWgiXO4adEWYYhqlOejaksfT4KfjJoRPwn9e2YOHlq/DXtW9zdriLYSHsYfn5c7D2e/tFHUZZQVn8nWHpsqoRFMxCwUQEeR4ZhmGqECLCcdOG4t6zZ2Ngcx3OvW0tDr7y31j58mYIzrh0CSyEPdSnk2iuT0cdRlkRtHJDsHXlt+2g1ohyKZtXbch3rxyqXzAMw3Q1o/o04q9nzcIViyfi0x1tOPH3T+C4Zaux7s2Pow6t4mBVwBRMMUVMWK9r0MFy8mXOCMcTtkYwDMO4MQzCookD8c/z5+IHB4/HS+99hkW/fQRn3fwUNn24NerwKgYWwkzBOKWvSi9j7PJpuapGdHH5NKYwitmUhWEYppJIJw2cNGs4Vl40D+fuOxoPvfQBFly2Et/58zP4gDvTFQwLYaZgimmNCEvQ9s6ONaK4MTbVpwAACyf0K+p6qw1bCEcbBsMwTGzpVpPE1/cbg5UXzsOXpw3B7f95E/tcugK/+L+X8OmOtqjDK1uqsqEGU1zsjHAEQtipGpF9vq5qqNG9NoWnvrsfmupSRV1vtWFbI1gJMwzDZKWlsQY/XDQBX5k9HL984GVc+dAG3Lz6dZw1bxSOnzEUNclE1CGWFZwRZgpG1v6Nontx0NJtXdVQAzBL3rDlojiwNYJhGCYYQ3s14NfHTMK9Z8/GhIFN+N+/v4D5v1iJu9a8hQ4uuRYYFsJMwVCEHuGg9YH7N9WhuT6FujRfKccR4owwwzBMXkwY2IQbT5mGm0+dhp4NaVzwp3U48IpV+NeL73PJtQCwEGYKRoqXKERmImDHuC/u3h+rv70v3zKKKQZ7hBmGYQpi1qje+OtZs3DlsZOws70DX7nhSRx97eNY8/p/ow4t1rAQZgpGitD6CIRw0EFwRMQiOMY4LZZZCjMMw+SLYRC+uPsALD9/H/z4SxPw6odbcfjVj2LJH5/Ehg8+izq8WMJCmCkYqV0a0qUfexm0sxwTb4jrCDMMwxSNVMLA8dOHYuWFc3HBfmPw6MaP8IVfPYxv3rke736yPerwYgULYaZg7IxwTRTWCCsGHqxW1kRZi5phGKZSaahJ4ux9R+Phi+bhpJnD8een38bcS1fgkvtewCfbuOQawOXTmCJSrIzwb46ZhHH9GwPNKzOJ3DGu3OHBcgzDMF1Fz4Y0vnfweJw8axh+tfxlLH34Vdy6+g2cOW8UTpo5DLWp6rUOckaYKRr1RRLCB+8xAKP6BBPC0hLBGeHyhgUwwzBM1zO4Zz0uO3oi/nHO3pgytAd+et+LmHvpCtz+nzfQ3tEZdXiRwEKYKZjtrR0AgIZIrBHsEa4E7KoR/DYyDMN0OeP6d8f1J++F25ZMR7+mWnzzrmew8IpV+L/n3qu6kms5hTARDSaih4joeSJ6jojO9ZlvLhGtteZZWfxQmbiyvc0UwsXKCIehq1onM6VFNmUhHi7HMAxTMqaP6IU/nzkT13x5CjqFwOk3rsHhVz+KJzZtiTq0khFEubQDuEAI8RQRNQJYQ0TLhRDPyxmIqBnAVQAWCiHeIKI+XRQvE0NkRjiK8mmJLmqdzJQWpylLtHEwDMNUG0SEhRP6YcG4PrhzzVv41YMv46hrH8P8XfrgooVjsUu/7lGH2KXkzAgLId4VQjxlPf8MwAsABnpmOxbA3UKIN6z5Pih2oEx8cTLC0VWNYCFc3hh2Zzl+HxmGYaIgmTCweK8hWPGNefjmwl3wn9e24IArVuGCO9bhrf9uizq8LiOUR5iIhgGYBGC156UxAHoQ0QoiWkNEJ/gsv4SIniSiJzdv3pxPvEwM2WZ7hEtvjZDCictulTncWY5hGCYW1KUTOGPuSKy6aB6W7D0C96x/B/N/sRIX37Uemz7cGnV4RSewECaibgDuAnCeEOJTz8tJAFMAHARgfwDfJaIx3nUIIZYKIaYKIaa2tLQUEDYTJ6LMCKetlLB8ZMoTp7NcpGEwDMMwFs31aXzrwHFY8Y25OGrPQbj76bex7y9X4KxbnsKzb38SdXhFI1AKj4hSMEXwzUKIuzWzvAXgIyHEVgBbiehhAHsAeLlokTKxpbXdLLkSRUZ45qhe+PGXJmDXAZXtYfLjltOmYYd1IVLOsDWCYRgmngxorsP/fmk3nLPvaPz+36/hpsdfx9/Xv4t9xrTgzLkjsdfwnmX92x2kagQB+B2AF4QQl/nM9lcAs4koSUT1AKbB9BIzVUQUGeGaZALHTx9atXWEZ47sjfm79I06jIIh4mwwwzBMnOnTWIuLD9gFj1w8HxfuPxbPvv0Jjl76OI645jE8+Pz7ZVt2LUgKbxaA4wE8Q0RrrWnfBjAEAIQQ1wghXiCi+wGsB9AJYJkQ4tmuCJiJL8XqLMdUHwT2BzMMw5QDTXUpnDVvFE6ZPRx3PPkmrl35Kk7945MY27cRZ8wdiS/u3h/JMrIr5lQuQoh/I8A5SghxKYBLixEUU57URZARZioDg6isb60xDMNUG7WpBE6YMQzH7DUE96x7B1ev2Ijzbl+LXy5/CUvmjMSRUwaVRevm8pHsTOypSfLHickT4owwwzBMOZJKGDhs8iD833lzsPT4KejVUIPv/uVZzP7ZQ7h6xUZ8tqMt6hCzwveymaLBGT0mXwjEJfAYhmHKGMMgfGHXfthvfF889upHuHrFRvzs/hdx1YoNOGHGUJw8azh6d6uJOswMWAgzDBM5BpuEGYZhKgIiwsyRvTFzZG8889YnuHrlBly1YiOWrdqExXsOxmlzRmBQj/qow7RhIcwUzLITpmLj5s+jDoMpY4itEQzDMBXHboOacNVxU7Bx8+e4duVG3PLEG7hp9RtYtMcAnDF3JEb3bYw6RBbCTOEsGN8XC1D+JbyY6CAQl09jGIapUEa2dMPPj9gD5y0Yg2WrNuHWJ97A3U+/jf3G98WZc0di0pAekcXGQphhmMgxDFMMMwzDMJXLgOY6fO/g8fja/FG44dHX8IdHX8Py59/HjBG9cOa8kZg9qnfJxxvxMH+GYWIAoUp7ojAMw1QdPRvSOH+/MXjk4vn4zoHj8OqHn+P43z2BQ658BPc98y46O0vXnIOFMMMwkWN2lmMlzDAMU010q0nitDkj8PBF83DJYbvhsx1tOOPmp7DgVytxx5NvorW9s8tjYCHMMEzkGDxYjmEYpmqpSSZwzF5D8M8L5uLKYyehNpnARXeuxz6XPoTf/3sTtrW2d9m2WQgzDBM5xB01GIZhqp6EQfji7gPw93Nm44aT98TgnvX40b3PY9ZP/4UrHnwFH29rLfo2ebAcwzCRw+XTGIZhGAkRYe7YPpg7tg/WvL4FVz20Eb968GUsfXgjvjJ7OC74wtiibYuFMMMwkbPf+L7oXpuKOgyGYRgmZkwZ2hO/O6knXnzvU1yzYiO27uwo6vpZCDMMEzl7j27B3qNbog6DYRiGiSm79OuOyxdPghDFrSjBHmGGYRiGYRimLCh2hSEWwgzDMAzDMExVwkKYYRiGYRiGqUpYCDMMwzAMwzBVCQthhmEYhmEYpiphIcwwDMMwDMNUJSyEGYZhGIZhmKqEhTDDMAzDMAxTlbAQZhiGYRiGYaoSFsIMwzAMwzBMVcJCmGEYhmEYhqlKWAgzDMMwDMMwVQkLYYZhGIZhGKYqYSHMMAzDMAzDVCUshBmGYRiGYZiqhIUwwzAMwzAMU5WwEGYYhmEYhmGqEhbCDMMwDMMwTFVCQohoNky0GcDreSzaG8CHRQ6nq+GYu55yixfgmEtFV8U8VAjR0gXrjSUV8psdl1jiEgfAseiISxxAfGKJSxxA/rFof7MjE8L5QkRPCiGmRh1HGDjmrqfc4gU45lJRjjFXEnE6/nGJJS5xABxLnOMA4hNLXOIAih8LWyMYhmEYhmGYqoSFMMMwDMMwDFOVlKMQXhp1AHnAMXc95RYvwDGXinKMuZKI0/GPSyxxiQPgWHTEJQ4gPrHEJQ6gyLGUnUeYYRiGYRiGYYpBOWaEGYZhGIZhGKZgykoIE9FCInqJiDYQ0cVRx6ODiF4jomeIaC0RPWlN60lEy4noFeuxR8Qx/p6IPiCiZ5Vp2hjJ5NfWMV9PRJNjFPMPiOht61ivJaIDlde+ZcX8EhHtH0G8g4noISJ6noieI6JzremxPc5ZYo7zca4loieIaJ0V8w+t6cOJaLUV2+1ElLam11j/b7BeH1bqmKuJuPxm634/IopD+x2LKBbtdyfCeBJE9DQR3RtxHBnn8IjiaCaiO4noRSJ6gYhmRBTHWOW3fy0RfUpE50UUy9etz+qzRHQrEdUWZcVCiLL4A5AAsBHACABpAOsAjI86Lk2crwHo7Zn2cwAXW88vBvCziGOcA2AygGdzxQjgQAD3ASAA0wGsjlHMPwDwDc28463PRw2A4dbnJlHiePsDmGw9bwTwshVXbI9zlpjjfJwJQDfreQrAauv43QFgsTX9GgBnWM/PBHCN9XwxgNtLfZyr5S9Ov9m634+I4tB+xyKKRfvdifDYnA/gFgD3RvweZZzDI4rjDwBOtZ6nATTHIKYEgPdg1uMt9bYHAtgEoM76/w4AJxVj3eWUEd4LwAYhxKtCiFYAtwFYFHFMQVkE80MN6/FLEcYCIcTDALZ4JvvFuAjAH4XJ4wCaiah/aSJ18InZj0UAbhNC7BRCbAKwAebnp2QIId4VQjxlPf8MwAswv8ixPc5ZYvYjDsdZCCE+t/5NWX8CwHwAd1rTvcdZHv87AexLRFSicKuN2Pxmh/z96Mo4wn7HujIWv+9OySGiQQAOArAsiu3HDSJqgnnx9jsAEEK0CiE+jjYqAMC+ADYKIfJprFMMkgDqiCgJoB7AO8VYaTkJ4YEA3lT+fwsR/YDkQAB4gIjWENESa1pfIcS71vP3APSNJrSs+MUY9+P+NctK8HvFchKrmK3b75NgZlzK4jh7YgZifJytW6prAXwAYDnMLOTHQoh2TVx2zNbrnwDoVdqIq4ZYfD7iiuY7FkUMru+OECKqWC4HcBGAzoi2r6I7h5ea4QA2A7jesossI6KGiGJRWQzg1ig2LIR4G8AvALwB4F0AnwghHijGustJCJcLs4UQkwEcAOAsIpqjvijMnH6sS3WUQ4wWVwMYCWAizC/GL6MNJxMi6gbgLgDnCSE+VV+L63HWxBzr4yyE6BBCTAQwCGYWcpeIQ2KYrGT7XSgl3u8OEU0odQxE9EUAHwgh1pR62z5kPYeXiCRMK8/VQohJALbCtNJFhjXO4hAAf4po+z1g3lEaDmAAgAYi+nIx1l1OQvhtAIOV/wdZ02KFddUCIcQHAP4M88T8vrzNbT1+EF2EvvjFGNvjLoR43/oh7wRwHZzb8rGImYhSME92Nwsh7rYmx/o462KO+3GWWLcOHwIwA6a1JKmJy47Zer0JwEclDrVaiNXnIy74/C5EivLdWRjB5mcBOISIXoNpn5lPRDdFEAcA33N4qXkLwFtKhv5OmMI4Sg4A8JQQ4v2Itr8AwCYhxGYhRBuAuwHMLMaKy0kI/wfAaGs0eBpmiv5vEcfkgogaiKhRPgfwBQDPwozzRGu2EwH8NZoIs+IX498AnEAm02HejnhXt4JS4/HQHgrzWANmzIutCgHDAYwG8ESJYyOY/q4XhBCXKS/F9jj7xRzz49xCRM3W8zoA+8H0XT4E4AhrNu9xlsf/CAD/sjLzTPGJ/W92qcnyuxBFLLrvzouljkMI8S0hxCAhxDCYn5F/CSGKkukLS5ZzeEkRQrwH4E0iGmtN2hfA86WOw8MxiMgWYfEGgOlEVG99j/aF+VtfOMUYcVeqP5gj61+G6QH8TtTxaOIbAXNk9DoAz8kYYXoQ/wngFQAPAugZcZy3wrzF3QbzyvMUvxhhjiz+rXXMnwEwNUYx32jFtB7mCba/Mv93rJhfAnBABPHOhml7WA9grfV3YJyPc5aY43ycdwfwtBXbswC+Z00fAVOUb4B5K6/Gml5r/b/Ben1EFJ/navmLy2+27vcjoji037GIYtF+dyL+vMxFhFUj/M7hEcUyEcCT1vvzFwA9IoylAeads6aIPx8/hHmx9qx1Xqopxnq5sxzDMAzDMAxTlZSTNYJhGIZhGIZhigYLYYZhGIZhGKYqYSHMMAzDMAzDVCUshBmGYRiGYZiqhIUwwzAMwzAMU5WwEGZiDxF1ENFaIlpHRE8RUdYi2kTUTERnBljvCiKaWrxIGYZhqhvl91r+Fa0jGhENI6KS1/VlKptk7lkYJnK2C7MNKIhofwCXANgny/zNAM4EcFUJYmMYhmEc7N9rhikHOCPMlBvdAfwXAIioGxH908oSP0NEi6x5fgpgpJWNuNSa95vWPOuI6KfK+o4koieI6GUi2ru0u8IwDFMdENFrRPRz63f4CSIaZU0fRkT/IqL11u/5EGt6XyL6s/WbvU65E5ggouuI6DkiesDqiscwecMZYaYcqCOitTC7gvUHMN+avgPAoUKIT4moN4DHiehvAC4GMEHJIh8AYBGAaUKIbUTUU1l3UgixFxEdCOD7MPuZMwzDMPkhf68llwghbreefyKE2I2ITgBwOYAvAvgNgD8IIf5ARF8B8GsAX7IeVwohDiWiBIBuAHrAbOV+jBDiNCK6A8DhAG4qza4xlQgLYaYcUK0RMwD8kYgmwGxL/P+IaA6ATgADAfTVLL8AwPVCiG0AIITYorx2t/W4BsCwrgmfYRimashmjbhVefyV9XwGgMOs5zcC+Ln1fD6AEwBACNEB4BMi6gFgkxBCCm3+3WYKhoUwU1YIIR6zsr8tAA60HqcIIdqI6DWYWeMw7LQeO8DfB4ZhmK5E+DwPw07leQcAtkYwBcEeYaasIKJdACQAfASgCcAHlgieB2CoNdtnABqVxZYDOJmI6q11qNYIhmEYpjQcrTw+Zj1/FMBi6/lxAFZZz/8J4AwAIKIEETWVKkimuuAMGFMOqJ4zAnCiEKKDiG4GcA8RPQPgSQAvAoAQ4iMiesQqs3OfEOJCIpoI4EkiagXwDwDfjmA/GIZhKh2vR/h+IYQsodaDiNbDzOoeY007G8D1RHQhgM0ATramnwtgKRGdAjPzewaAd7s8eqbqICHyvTvBMAzDMAyTG8u6NlUI8WHUsTCMClsjGIZhGIZhmKqEM8IMwzAMwzBMVcIZYYZhGIZhGKYqYSHMMAzDMAzDVCUshBmGYRiGYZiqhIUwwzAMwzAMU5WwEGYYhmEYhmGqEhbCDMMwDMMwTFXy/wHf/BoZGvkuPAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 10 | Time: 1m 15s\n", - "\tTrain Loss: 3.074 | Train PPL: 21.637\n", - "\t Val. Loss: 4.600 | Val. PPL: 99.469\n" - ] - } - ], - "source": [ - "for epoch in range(N_EPOCHS):\n", - " \n", - " start_time = time.time()\n", - " \n", - " train_loss = train(model, train_iterator, optimizer, criterion, CLIP, train_history, valid_history)\n", - " valid_loss = evaluate(model, valid_iterator, criterion)\n", - " \n", - " end_time = time.time()\n", - " \n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut1-model.pt')\n", - " \n", - " train_history.append(train_loss)\n", - " valid_history.append(valid_loss)\n", - " print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Let's take a look at our network quality__:" - ] - }, - { - "cell_type": "code", - "execution_count": 136, - "metadata": {}, - "outputs": [], - "source": [ - "import utils\n", - "import imp\n", - "imp.reload(utils)\n", - "generate_translation = utils.generate_translation\n", - "remove_tech_tokens = utils.remove_tech_tokens\n", - "get_text = utils.get_text\n", - "flatten = utils.flatten" - ] - }, - { - "cell_type": "code", - "execution_count": 137, - "metadata": {}, - "outputs": [], - "source": [ - "batch = next(iter(test_iterator))" - ] - }, - { - "cell_type": "code", - "execution_count": 138, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Original: each room has a tv .\n", - "Generated: each room is equipped with a tv . .\n", - "\n", - "Original: you will find a 24 - hour front desk at the property .\n", - "Generated: the hotel offers a 24 - hour front desk . property .\n", - "\n" - ] - } - ], - "source": [ - "for idx in [1,2]:\n", - " src = batch.src[:, idx:idx+1]\n", - " trg = batch.trg[:, idx:idx+1]\n", - " generate_translation(src, trg, model, TRG.vocab)" - ] - }, - { - "cell_type": "code", - "execution_count": 139, - "metadata": {}, - "outputs": [], - "source": [ - "from nltk.translate.bleu_score import corpus_bleu\n", - "\n", - "# \"\"\" Estimates corpora-level BLEU score of model's translations given inp and reference out \"\"\"\n", - "# translations, _ = model.translate_lines(inp_lines, **flags)\n", - "# # Note: if you experience out-of-memory error, split input lines into batches and translate separately\n", - "# return corpus_bleu([[ref] for ref in out_lines], translations) * 100" - ] - }, - { - "cell_type": "code", - "execution_count": 140, - "metadata": {}, - "outputs": [], - "source": [ - "import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 149, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "59it [00:03, 18.95it/s]\n" - ] - } - ], - "source": [ - "original_text = []\n", - "generated_text = []\n", - "model.eval()\n", - "with torch.no_grad():\n", - "\n", - " for i, batch in tqdm.tqdm(enumerate(test_iterator)):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output.argmax(dim=-1)\n", - " \n", - " original_text.extend([get_text(x, TRG.vocab) for x in trg.cpu().numpy().T])\n", - " generated_text.extend([get_text(x, TRG.vocab) for x in output.detach().cpu().numpy().T])\n", - "\n", - "# original_text = flatten(original_text)\n", - "# generated_text = flatten(generated_text)" - ] - }, - { - "cell_type": "code", - "execution_count": 150, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "14.449864542777785" - ] - }, - "execution_count": 150, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "corpus_bleu([[text] for text in original_text], generated_text) * 100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Baseline solution BLEU score is quite low. Try to achieve at least __18__ BLEU on the test set. \n", - "The checkpoints are:\n", - "\n", - "* __18__ - minimal score to submit the homework, 30% of points\n", - "\n", - "* __20__ - good score, 70% of points\n", - "\n", - "* __25__ - excellent score, 100% of points" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "homework.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Py3 research env", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/lab01_nlp/.ipynb_checkpoints/lab1_01_nlp_part1_embedding_based_mt-checkpoint.ipynb b/homeworks/lab01_nlp/.ipynb_checkpoints/lab1_01_nlp_part1_embedding_based_mt-checkpoint.ipynb deleted file mode 100644 index 2bcd322..0000000 --- a/homeworks/lab01_nlp/.ipynb_checkpoints/lab1_01_nlp_part1_embedding_based_mt-checkpoint.ipynb +++ /dev/null @@ -1,753 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eulvfJWl7ueY" - }, - "source": [ - "# Lab 1\n", - "\n", - "\n", - "## Part 1: Bilingual dictionary induction and unsupervised embedding-based MT (30%)\n", - "*Note: this homework is based on materials from yandexdataschool [NLP course](https://github.com/yandexdataschool/nlp_course/). Feel free to check this awesome course if you wish to dig deeper.*\n", - "\n", - "*Refined by [Nikolay Karpachev](https://www.linkedin.com/in/nikolay-karpachev-b0146a104/)*" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fV4rIjxa7uei" - }, - "source": [ - "**In this homework** **YOU** will make machine translation system without using parallel corpora, alignment, attention, 100500 depth super-cool recurrent neural network and all that kind superstuff.\n", - "\n", - "But even without parallel corpora this system can be good enough (hopefully), in particular for similar languages, e.g. Ukrainian and Russian. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idSYq2GU7uew" - }, - "source": [ - "### Frament of the Swadesh list for some slavic languages\n", - "\n", - "The Swadesh list is a lexicostatistical stuff. It's named after American linguist Morris Swadesh and contains basic lexis. This list are used to define subgroupings of languages, its relatedness.\n", - "\n", - "So we can see some kind of word invariance for different Slavic languages.\n", - "\n", - "\n", - "| Russian | Belorussian | Ukrainian | Polish | Czech | Bulgarian |\n", - "|-----------------|--------------------------|-------------------------|--------------------|-------------------------------|-----------------------|\n", - "| женщина | жанчына, кабета, баба | жінка | kobieta | žena | жена |\n", - "| мужчина | мужчына | чоловік, мужчина | mężczyzna | muž | мъж |\n", - "| человек | чалавек | людина, чоловік | człowiek | člověk | човек |\n", - "| ребёнок, дитя | дзіця, дзіцёнак, немаўля | дитина, дитя | dziecko | dítě | дете |\n", - "| жена | жонка | дружина, жінка | żona | žena, manželka, choť | съпруга, жена |\n", - "| муж | муж, гаспадар | чоловiк, муж | mąż | muž, manžel, choť | съпруг, мъж |\n", - "| мать, мама | маці, матка | мати, матір, неня, мама | matka | matka, máma, 'стар.' mateř | майка |\n", - "| отец, тятя | бацька, тата | батько, тато, татусь | ojciec | otec | баща, татко |\n", - "| много | шмат, багата | багато | wiele | mnoho, hodně | много |\n", - "| несколько | некалькі, колькі | декілька, кілька | kilka | několik, pár, trocha | няколко |\n", - "| другой, иной | іншы | інший | inny | druhý, jiný | друг |\n", - "| зверь, животное | жывёла, звер, істота | тварина, звір | zwierzę | zvíře | животно |\n", - "| рыба | рыба | риба | ryba | ryba | риба |\n", - "| птица | птушка | птах, птиця | ptak | pták | птица |\n", - "| собака, пёс | сабака | собака, пес | pies | pes | куче, пес |\n", - "| вошь | вош | воша | wesz | veš | въшка |\n", - "| змея, гад | змяя | змія, гад | wąż | had | змия |\n", - "| червь, червяк | чарвяк | хробак, черв'як | robak | červ | червей |\n", - "| дерево | дрэва | дерево | drzewo | strom, dřevo | дърво |\n", - "| лес | лес | ліс | las | les | гора, лес |\n", - "| палка | кій, палка | палиця | patyk, pręt, pałka | hůl, klacek, prut, kůl, pálka | палка, пръчка, бастун |" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cNM3_fjr7ue2" - }, - "source": [ - "But the context distribution of these languages demonstrates even more invariance. And we can use this fact for our for our purposes." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YLppwa527ue6" - }, - "source": [ - "## Data" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lYBGKAUn7ue_" - }, - "outputs": [], - "source": [ - "import gensim\n", - "import numpy as np\n", - "from gensim.models import KeyedVectors" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MwGoVhRA7ufP" - }, - "source": [ - "In this notebook we're going to use pretrained word vectors - FastText (original paper - https://arxiv.org/abs/1607.04606).\n", - "\n", - "You can download them from the official [website](https://fasttext.cc/docs/en/crawl-vectors.html). We're going to need embeddings for Russian and Ukrainian languages. Please use word2vec-compatible format (.text)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "u1JjQv_97ufT" - }, - "outputs": [], - "source": [ - "uk_emb = KeyedVectors.load_word2vec_format(\"cc.uk.300.vec\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ffzuept_7ufd" - }, - "outputs": [], - "source": [ - "ru_emb = KeyedVectors.load_word2vec_format(\"cc.ru.300.vec\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nTkXfT0W7ufk" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([ru_emb[\"август\"]], topn=10)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vdBA8lcg7ufs" - }, - "outputs": [], - "source": [ - "uk_emb.most_similar([uk_emb[\"серпень\"]])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_yJvcKXO7uf0" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([uk_emb[\"серпень\"]])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pNdYAR1q7uf6" - }, - "source": [ - "Load small dictionaries for correspoinding words pairs as trainset and testset." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "35d_DAK67uf8" - }, - "outputs": [], - "source": [ - "def load_word_pairs(filename):\n", - " uk_ru_pairs = []\n", - " uk_vectors = []\n", - " ru_vectors = []\n", - " with open(filename, \"r\") as inpf:\n", - " for line in inpf:\n", - " uk, ru = line.rstrip().split(\"\\t\")\n", - " if uk not in uk_emb or ru not in ru_emb:\n", - " continue\n", - " uk_ru_pairs.append((uk, ru))\n", - " uk_vectors.append(uk_emb[uk])\n", - " ru_vectors.append(ru_emb[ru])\n", - " return uk_ru_pairs, np.array(uk_vectors), np.array(ru_vectors)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wkNL602WHJyO" - }, - "outputs": [], - "source": [ - "!wget -O ukr_rus.train.txt http://tiny.cc/jfgecz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uoclU6JcHCcn" - }, - "outputs": [], - "source": [ - "!wget -O ukr_rus.test.txt http://tiny.cc/6zoeez" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "05BqsdSK7ugD" - }, - "outputs": [], - "source": [ - "uk_ru_train, X_train, Y_train = load_word_pairs(\"ukr_rus.train.txt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zQOZw51r7ugL" - }, - "outputs": [], - "source": [ - "uk_ru_test, X_test, Y_test = load_word_pairs(\"ukr_rus.test.txt\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ZBBNvpz7ugQ" - }, - "source": [ - "## Embedding space mapping (0.3 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_Dhk5gL7ugS" - }, - "source": [ - "Let $x_i \\in \\mathrm{R}^d$ be the distributed representation of word $i$ in the source language, and $y_i \\in \\mathrm{R}^d$ is the vector representation of its translation. Our purpose is to learn such linear transform $W$ that minimizes euclidian distance between $Wx_i$ and $y_i$ for some subset of word embeddings. Thus we can formulate so-called Procrustes problem:\n", - "\n", - "$$W^*= \\arg\\min_W \\sum_{i=1}^n||Wx_i - y_i||_2$$\n", - "or\n", - "$$W^*= \\arg\\min_W ||WX - Y||_F$$\n", - "\n", - "where $||*||_F$ - Frobenius norm." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "acOjDdtL7ugY" - }, - "source": [ - "$W^*= \\arg\\min_W \\sum_{i=1}^n||Wx_i - y_i||_2$ looks like simple multiple linear regression (without intercept fit). So let's code." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Lb-KN1be7uga" - }, - "outputs": [], - "source": [ - "from sklearn.linear_model import LinearRegression\n", - "\n", - "# YOUR CODE HERE\n", - "# mapping = ...\n", - "# -------" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "X7tqJwoY7ugf" - }, - "source": [ - "Let's take a look at neigbours of the vector of word _\"серпень\"_ (_\"август\"_ in Russian) after linear transform." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "31SrFSbn7ugi" - }, - "outputs": [], - "source": [ - "august = mapping.predict(uk_emb[\"серпень\"].reshape(1, -1))\n", - "ru_emb.most_similar(august)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "okSkjk597ugo" - }, - "source": [ - "We can see that neighbourhood of this embedding cosists of different months, but right variant is on the ninth place." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o2uY6Y9B7ugt" - }, - "source": [ - "As quality measure we will use precision top-1, top-5 and top-10 (for each transformed Ukrainian embedding we count how many right target pairs are found in top N nearest neighbours in Russian embedding space)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zptuho8LAfIE" - }, - "outputs": [], - "source": [ - "def precision(pairs, mapped_vectors, topn=1):\n", - " \"\"\"\n", - " :args:\n", - " pairs = list of right word pairs [(uk_word_0, ru_word_0), ...]\n", - " mapped_vectors = list of embeddings after mapping from source embedding space to destination embedding space\n", - " topn = the number of nearest neighbours in destination embedding space to choose from\n", - " :returns:\n", - " precision_val, float number, total number of words for those we can find right translation at top K.\n", - " \"\"\"\n", - " assert len(pairs) == len(mapped_vectors)\n", - " num_matches = 0\n", - " for i, (_, ru) in enumerate(pairs):\n", - " # YOUR CODE HERE\n", - " precision_val = num_matches / len(pairs)\n", - " return precision_val" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "duhj9hpv7ugy" - }, - "outputs": [], - "source": [ - "assert precision([(\"серпень\", \"август\")], august, topn=5) == 0.0\n", - "assert precision([(\"серпень\", \"август\")], august, topn=9) == 1.0\n", - "assert precision([(\"серпень\", \"август\")], august, topn=10) == 1.0" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0-iyd5gP7ug5" - }, - "outputs": [], - "source": [ - "assert precision(uk_ru_test, X_test) == 0.0\n", - "assert precision(uk_ru_test, Y_test) == 1.0" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U-ssEJ3x7uhA" - }, - "outputs": [], - "source": [ - "precision_top1 = precision(uk_ru_test, mapping.predict(X_test), 1)\n", - "precision_top5 = precision(uk_ru_test, mapping.predict(X_test), 5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7K-hy7a6Ksn2" - }, - "outputs": [], - "source": [ - "print(precision_top1)\n", - "print(precision_top5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hf6Ou8bx7uhH" - }, - "source": [ - "## Making it better (orthogonal Procrustean problem) (0.3 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4oLs-drN7uhK" - }, - "source": [ - "It can be shown (see original paper) that a self-consistent linear mapping between semantic spaces should be orthogonal. \n", - "We can restrict transform $W$ to be orthogonal. Then we will solve next problem:\n", - "\n", - "$$W^*= \\arg\\min_W ||WX - Y||_F \\text{, where: } W^TW = I$$\n", - "\n", - "$$I \\text{- identity matrix}$$\n", - "\n", - "Instead of making yet another regression problem we can find optimal orthogonal transformation using singular value decomposition. It turns out that optimal transformation $W^*$ can be expressed via SVD components:\n", - "$$X^TY=U\\Sigma V^T\\text{, singular value decompostion}$$\n", - "$$W^*=UV^T$$" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_KSaRJFGMFiJ" - }, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DdFQ7qti7uhL" - }, - "outputs": [], - "source": [ - "def learn_transform(X_train, Y_train):\n", - " \"\"\" \n", - " :returns: W* : float matrix[emb_dim x emb_dim] as defined in formulae above\n", - " \"\"\"\n", - " # YOUR CODE GOES HERE\n", - " # compute orthogonal embedding space mapping\n", - " # mapping = ...\n", - "\n", - " return mapping" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7X7QfYDd7uhQ" - }, - "outputs": [], - "source": [ - "W = learn_transform(X_train, Y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OVOFYYa37uhX" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([np.matmul(uk_emb[\"серпень\"], W)])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r297sYP37uhb" - }, - "outputs": [], - "source": [ - "print(precision(uk_ru_test, np.matmul(X_test, W)))\n", - "print(precision(uk_ru_test, np.matmul(X_test, W), 5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hvUZ72U5AfJg" - }, - "source": [ - "## Unsupervised embedding-based MT (0.4 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLyuVfHBLrJn" - }, - "source": [ - "Now, let's build our word embeddings-based translator!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tPAURW1CMuP7" - }, - "source": [ - "Firstly, download OPUS Tatoeba corpus." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "F80kUKzQMsDu" - }, - "outputs": [], - "source": [ - "!wget https://object.pouta.csc.fi/OPUS-Tatoeba/v20190709/mono/uk.txt.gz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0CGFZoxCUVf1" - }, - "outputs": [], - "source": [ - "!gzip -d ./uk.txt.gz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2MV3VvoVUX5U" - }, - "outputs": [], - "source": [ - "with open('./uk.txt', 'r') as f:\n", - " uk_corpus = f.readlines()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tU7nPVf0UhbI" - }, - "outputs": [], - "source": [ - "# To save your time and CPU, feel free to use first 1000 sentences of the corpus\n", - "uk_corpus = uk_corpus[:1000]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FLN8dBOXAfJ1" - }, - "outputs": [], - "source": [ - "# Any necessary preprocessing if needed\n", - "# YOUR CODE HERE" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FGksC7l_NMi9" - }, - "outputs": [], - "source": [ - "def translate(sentence):\n", - " \"\"\"\n", - " :args:\n", - " sentence - sentence in Ukrainian (str)\n", - " :returns:\n", - " translation - sentence in Russian (str)\n", - "\n", - " * find ukrainian embedding for each word in sentence\n", - " * transform ukrainian embedding vector\n", - " * find nearest russian word and replace\n", - " \"\"\"\n", - " # YOUR CODE GOES HERE\n", - "\n", - " return \" \".join(translated)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4hbbMy-tNxlf" - }, - "outputs": [], - "source": [ - "assert translate(\".\") == \".\"\n", - "assert translate(\"1 , 3\") == \"1 , 3\"\n", - "assert translate(\"кіт зловив мишу\") == \"кот поймал мышку\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ia6I2ce7O_HI" - }, - "source": [ - "Now you can play with your model and try to get as accurate translations as possible. **Note**: one big issue is out-of-vocabulary words. Try to think of various ways of handling it (you can start with translating each of them to a special **UNK** token and then move to more sophisticated approaches). Good luck!" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ap1W7ZCeOAVU" - }, - "outputs": [], - "source": [ - "for sent in uk_corpus[::10]:\n", - " print(translate(sent))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Great! \n", - "See second notebook for the Neural Machine Translation assignment." - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "homework.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Py3 Research", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/lab01_nlp/.ipynb_checkpoints/lab1_02_nlp_part2_nmt-checkpoint.ipynb b/homeworks/lab01_nlp/.ipynb_checkpoints/lab1_02_nlp_part2_nmt-checkpoint.ipynb deleted file mode 100644 index eb346fa..0000000 --- a/homeworks/lab01_nlp/.ipynb_checkpoints/lab1_02_nlp_part2_nmt-checkpoint.ipynb +++ /dev/null @@ -1,941 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Lab 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Part 2: Neural Machine Translation in the wild\n", - "In the third homework you are supposed to get the best translation you can for the EN-RU translation task.\n", - "\n", - "Basic approach using RNNs as encoder and decoder is implemented for you. \n", - "\n", - "Your ultimate task is to use the techniques we've covered, e.g.\n", - "\n", - "* Optimization enhancements (e.g. learning rate decay)\n", - "\n", - "* CNN encoder (with or without positional encoding)\n", - "\n", - "* attention/self-attention mechanism\n", - "\n", - "* pretraining the language model\n", - "\n", - "* [Byte Pair Encoding](https://github.com/rsennrich/subword-nmt)\n", - "\n", - "* or just fine-tunning BERT ;)\n", - "\n", - "to improve the translation quality. \n", - "\n", - "__Please use at least three different approaches/models and compare them (translation quality/complexity/training and evaluation time).__\n", - "\n", - "Write down some summary on your experiments and illustrate it with convergence plots/metrics and your thoughts. Just like you would approach a real problem." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# You might need to install the libraries below. Do it in the desired environment\n", - "# if you are working locally.\n", - "\n", - "# ! pip install subword-nmt\n", - "# ! pip install nltk\n", - "# ! pip install torchtext" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Thanks to YSDA NLP course team for the data\n", - "# (who thanks tilda and deephack teams for the data in their turn)\n", - "\n", - "import os\n", - "path_do_data = '../../datasets/Machine_translation_EN_RU/data.txt'\n", - "if not os.path.exists(path_do_data):\n", - " print(\"Dataset not found locally. Downloading from github. Loading special files as well\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/master/datasets/Machine_translation_EN_RU/data.txt -nc\n", - " path_do_data = './data.txt'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists('./utils.py'):\n", - " print(\"utils file not found locally. Downloading from github.\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/master/homeworks_advanced/Lab1_NLP/utils.py -nc\n", - "\n", - "if not os.path.exists('./my_network.py'):\n", - " print(\"network file not found locally. Downloading from github.\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/master/homeworks_advanced/Lab1_NLP/my_network.py -nc" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "\n", - "import torchtext\n", - "from torchtext.legacy.datasets import TranslationDataset, Multi30k\n", - "from torchtext.legacy.data import Field, BucketIterator, TabularDataset\n", - "\n", - "import spacy\n", - "\n", - "import random\n", - "import math\n", - "import time\n", - "\n", - "import matplotlib\n", - "matplotlib.rcParams.update({'figure.figsize': (16, 12), 'font.size': 14})\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "from IPython.display import clear_output\n", - "\n", - "from nltk.tokenize import WordPunctTokenizer\n", - "from subword_nmt.learn_bpe import learn_bpe\n", - "from subword_nmt.apply_bpe import BPE" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Main part\n", - "__Here comes the preprocessing. Do not hesitate to use BPE or more complex preprocessing ;)__" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer_W = WordPunctTokenizer()\n", - "def tokenize(x, tokenizer=tokenizer_W):\n", - " return tokenizer.tokenize(x.lower())" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "SRC = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "TRG = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "dataset = TabularDataset(\n", - " path=path_do_data,\n", - " format='tsv',\n", - " fields=[('trg', TRG), ('src', SRC)]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "train_data, valid_data, test_data = dataset.split(split_ratio=[0.8, 0.15, 0.05])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training examples: 40000\n", - "Number of validation examples: 2500\n", - "Number of testing examples: 7500\n" - ] - } - ], - "source": [ - "print(f\"Number of training examples: {len(train_data.examples)}\")\n", - "print(f\"Number of validation examples: {len(valid_data.examples)}\")\n", - "print(f\"Number of testing examples: {len(test_data.examples)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "SRC.build_vocab(train_data, min_freq = 3)\n", - "TRG.build_vocab(train_data, min_freq = 3)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unique tokens in source (ru) vocabulary: 9267\n", - "Unique tokens in target (en) vocabulary: 6699\n" - ] - } - ], - "source": [ - "print(f\"Unique tokens in source (ru) vocabulary: {len(SRC.vocab)}\")\n", - "print(f\"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here are tokens from original (RU) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['',\n", - " '29',\n", - " 'соль',\n", - " 'комо',\n", - " '―',\n", - " 'электрическая',\n", - " 'ming',\n", - " 'утренний',\n", - " 'детском',\n", - " 'таунус']" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "SRC.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And from target (EN) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['', 'king', 'buffets', 'catch', 'media', 'schedule', 'maraunenhof']" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "TRG.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And here is example from train dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'trg': ['laundry', 'service', 'is', 'provided', '.'], 'src': ['помимо', 'этого', ',', 'гостям', 'предоставляются', 'услуги', 'прачечной', '.']}\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[9]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check the length distributions:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Train data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfUAAAEICAYAAABGRG3WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAer0lEQVR4nO3df7ReVX3n8fdHIr+0kCApxQRNKhlbZC0rzUhcOB3HOPyybVhr1MFxDdGmTVdLW9vpTAvTrmFGZQbXOEVYKpURSrAWpFRLRqw0RRlXp8OPoA7yQ0rKryQFuZIAVqs19jt/nH3hId6be3Pvzb3PPff9WutZ95y99zlnn3Of/XzP2c9+zklVIUmS5r8XzHUFJEnSzDCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST1hUNcBkWRFkkqyaA62/c4kfznb25XmQpKrkrxvGsv/XZIfnck6tfU+nORNM73eSWx3zj57hoFBXfPaQm/AGg5zFcD2V5Jbkvz8YFpVvbiqHpyrOk3XfDn2s8WgrudJctBc10HqG086NVsM6vNIkt9OsjPJN5Pcn2RtSz8kyQeT/G17fTDJIS3vB7qi25Xt8W36qiSXJflskm8B/yLJcUk+lWQkyZNJPjSw7M8luS/J7iQ3JXn5JOt+ZJIrkjzW9uF9oycQo3VM8oG23oeSnDGw7MokX2z7/RdJPpzkD1v2F9vfp1o34usGlhtzfdJMSvJx4GXA/2rvwd8a6EHakORR4POt7B8neTzJ0+09/aqB9VzV3ts3tvf6bUle0fKS5OIkTyR5JslXk5w4Rl2WJPlMa7u72/Tylnch8M+AD7V6fqilD34eHJnk6rb8I0l+N8kLWt4+2+kEx+gFSc5L8jftM+W6JEe1vNFjtT7Jo0m+keR3BpY9LMmmts372vHdMd6xH9jsO8ZaX+9Vla958AJeCWwHXtrmVwCvaNPvAW4FfhhYCvwV8N6W907gL/daVwHHt+mrgKeBU+hO8l4E/D/g4jZ9KPD6VnYdsA34cWAR8LvAX41T3xVtO4va/KeBj7Z1/jBwO/CLA3X8HvALwEHALwF/C6Tl/1/gA8DBwOuBZ4A/HGs7k1mfL18z/QIeBt40MD/6vry6vecPa+k/B/wQcAjwQeArA8tcBTwJvLa1r08A17a804A7gcVAWhs8dmC597XplwD/Cji8beePgT8d2MYtwM/vVffBz4OrgRvasiuAvwY2tLz9aleDxwR4N91n1PK27x8FrtnrWP1P4DDg1cB3gR9v+RcB/xtY0pa/C9gxiWM/5vr6/przCvia5D8KjgeeAN4EvHCvvL8BzhyYPw14uE2/k4mD+tUDea8DRhgIkgN5fzbawNv8C4BvAy8fo+xow1oEHNMa1WED+W8HvjBQx20DeYe3ZX+E7ix8D3D4QP4fMnFQH3N9c/1/9NXP1z4Cy4/uY5nFrcyRbf4q4GMD+WcCX2vTb6QLsGuAF+y1nqtoQX2MbfwEsHtg/hbGCep0gfofgBMG8n4RuKVN71e74vlB/T5g7UDesXQnCIsGjtXygfzbgbPb9IPAaQN5P8/kgvqY6+v7y+73eaKqtgG/Dvxn4Ikk1yZ5act+KfDIQPFHWtpkbR+YPg54pKr2jFHu5cAlSZ5K8hSwi+6qYdkE63858ELgsYFlP0p3xT7q8dGJqvp2m3xx249dA2l713c8461Pmk3PvleTHJTkotYF/QxdMAI4eqD84wPT36a9Z6vq88CHgA/Ttf/Lkxyx98aSHJ7ko63r/Bm6r6cWZ3JjZY6ma6d7f5YMtu+ptquXA58eaP/3Ad+nO+H/gXUzsO90nwGDbX4y7X9f6+s1g/o8UlV/VFWvp2sgBby/Zf1tSxv1spYG8C26M2oAkvzIWKsemN4OvCxjD+zZTtdlvnjgdVhV/dUEVd9Od6V+9MByR1TVqyZYDuAx4Kgkhw+kHTdO3aW5Mt77cDD939B9hfUm4Ei6K0roTown3kDVpVX1k8AJwD8B/sMYxX6T7qu6k6vqCOCn9trGvtrLN+iunvf+LNk5mfpNYDtwxl6fHYdW1WTW/Rhdt/uo4/bK9zNggEF9nkjyyiRvTDcA7jvA3wP/2LKvAX43ydIkRwP/ia6LGrrvx1+V5CeSHEp3pb8vt9M1oouSvCjJoUlOaXm/D5w/OrinDap560R1r6rHgD8H/keSI9qgmVck+eeTWPYRYCvwn5Mc3AbC/cxAkRG64zDjv7OV9sPXmfg9+EN0J7dP0p1o/9fJrjzJP01ycpIX0p2of4fn2v/e2/h7uoGjRwEXTLaeVfV94DrgwiQ/lG4Q7L/juc+S6fj9tt6Xt/1ZmmTdJJe9ju5zZ0mSZcCv7JU/mWO/YBjU549D6AaMfIOuW+mHgfNb3vvoAt9dwFeBL7U0quqv6QbS/QXwALDPm7K0hv0zdN+xPQrsAP51y/s0Xe/Ata1r725gsqPKz6Eb6HYvsBu4nu57tcl4B913/U+2/fok3YfjaBfghcD/aV17aya5Tmkm/Te6E+unkvz7ccpcTdedvZOuHdy6H+s/gm7g1+62jieB/z5GuQ/SDQ77Rlv/5/bKvwR4SxtJfukYy/8q3UnDg3SfFX8EXLkf9RzPJcBm4M+TfLPV7eRJLvseus+hh+g+x66ntf9mMsd+wRgdXSzNG0k+STeAaO+rEEk9l+SX6Aa9TdjTtxB5pa6h17oeX9G67U+n+17yT+e6XpIOvCTHJjmltf9X0o0b+PRc12tYeZcjzQc/AnyK7je4O4Bfqqovz22VJM2Sg+l+LbMSeAq4FvjInNZoiNn9LklST9j9LklST8zb7vejjz66VqxYMdfVkIbanXfe+Y2qWjrX9dgX27I0OZNpzxMG9SRXAj8NPFFVJ7a0o+h+VrSC7q5Ib6uq3UlC99OFM+nu4PPOqvpSW2Y93b3Cobul4aaW/pN0tzk8DPgs8O6axHcCK1asYOvWrRMVkxa0JI9MXGpu2ZalyZlMe55M9/tVwOl7pZ0H3FxVq4Cb2zx0v1le1V4bgctaRUZvgnAy3cMKLkiypC1zGd0DAkaX23tbkiRpEiYM6lX1Rbp7fA9aB2xq05uAswbSr67OrXT3HD6W7gEjW6pqV1XtBrYAp7e8I6rq1nZ1fvXAuiRJ0n6Y6kC5Y9qtP6G7u9noTfmX8fyb7e9oaftK3zFG+piSbEyyNcnWkZGRKVZdkqR+mvbo93aFPSu/i6uqy6tqdVWtXrp0qMf+SJI066Ya1L/eus5pf59o6Tt5/hN0lre0faUvHyNdkiTtp6kG9c3A+ja9HrhhIP2cdNYAT7du+puAU9tTdpYApwI3tbxnkqxpI+fPGViXJEnaD5P5Sds1wBuAo5PsoBvFfhFwXZINdE8Melsr/lm6n7Nto/tJ27sAqmpXkvcCd7Ry76mq0cF3v8xzP2n7s/aSJEn7acKgXlVvHydr7RhlCzh3nPVcyRiP8KuqrcCJE9VDkiTtm7eJlSSpJ+btbWJn04rzbpywzMMXvXkWaiJpumzP6jOv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1KUFJMmVSZ5IcvdA2lFJtiR5oP1d0tKT5NIk25LcleSkgWXWt/IPJFk/kP6TSb7alrk0SWZ3D6WFzaAuLSxXAafvlXYecHNVrQJubvMAZwCr2msjcBl0JwHABcDJwGuBC0ZPBFqZXxhYbu9tSTqADOrSAlJVXwR27ZW8DtjUpjcBZw2kX12dW4HFSY4FTgO2VNWuqtoNbAFOb3lHVNWtVVXA1QPrkjQLDOqSjqmqx9r048AxbXoZsH2g3I6Wtq/0HWOk/4AkG5NsTbJ1ZGRk+nsgCTCoSxrQrrBrFrZzeVWtrqrVS5cuPdCbkxYMg7qkr7euc9rfJ1r6TuC4gXLLW9q+0pePkS5plhjUJW0GRkewrwduGEg/p42CXwM83brpbwJOTbKkDZA7Fbip5T2TZE0b9X7OwLokzYJFc10BSbMnyTXAG4Cjk+ygG8V+EXBdkg3AI8DbWvHPAmcC24BvA+8CqKpdSd4L3NHKvaeqRgff/TLdCPvDgD9rL0mzxKAuLSBV9fZxstaOUbaAc8dZz5XAlWOkbwVOnE4dJU2d3e+SJPWEQV2SpJ4wqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9Ma2gnuQ3ktyT5O4k1yQ5NMnKJLe15yl/MsnBrewhbX5by18xsJ7zW/r9SU6b3i5JkrQwTTmoJ1kG/BqwuqpOBA4CzgbeD1xcVccDu4ENbZENwO6WfnErR5IT2nKvonv28keSHDTVekmStFBNt/t9EXBYkkXA4cBjwBuB61v+3s9mHn1m8/XA2nZ/6HXAtVX13ap6iO6WlK+dZr0kSVpwphzUq2on8AHgUbpg/jRwJ/BUVe1pxQafp/zsM5hb/tPASxj/2cw/wGcwS5I0vul0vy+hu8peCbwUeBFd9/kB4zOYJUka33S6398EPFRVI1X1PeBTwCnA4tYdD89/nvKzz2Bu+UcCTzL+s5klSdJ+mE5QfxRYk+Tw9t34WuBe4AvAW1qZvZ/NPPrM5rcAn29PgdoMnN1Gx68EVgG3T6NekiQtSFN+9GpV3ZbkeuBLwB7gy8DlwI3AtUne19KuaItcAXw8yTZgF92Id6rqniTX0Z0Q7AHOrarvT7VekiQtVNN6nnpVXQBcsFfyg4wxer2qvgO8dZz1XAhcOJ26SJK00E0rqEtSH60478YJyzx80ZtnoSbS/jGoz5DJfAiAHwSSpAPHe79LktQTBnVJknrC7ndJvTHZr8GkvvJKXZKknjCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST1hUJckqScM6pIASPIbSe5JcneSa5IcmmRlktuSbEvyySQHt7KHtPltLX/FwHrOb+n3JzltrvZHWogM6pJIsgz4NWB1VZ0IHET3eOT3AxdX1fHAbmBDW2QDsLulX9zKkeSEttyrgNOBjyQ5aDb3RVrIDOqSRi0CDkuyCDgceAx4I3B9y98EnNWm17V5Wv7aJGnp11bVd6vqIWAbYzyKWdKBYVCXRFXtBD4APEoXzJ8G7gSeqqo9rdgOYFmbXgZsb8vuaeVfMpg+xjLPSrIxydYkW0dGRmZ+h6QFyqAuiSRL6K6yVwIvBV5E131+QFTV5VW1uqpWL1269EBtRlpwDOqSAN4EPFRVI1X1PeBTwCnA4tYdD7Ac2NmmdwLHAbT8I4EnB9PHWEbSAWZQlwRdt/uaJIe378bXAvcCXwDe0sqsB25o05vbPC3/81VVLf3sNjp+JbAKuH2W9kFa8Hz0qiSq6rYk1wNfAvYAXwYuB24Erk3yvpZ2RVvkCuDjSbYBu+hGvFNV9yS5ju6EYA9wblV9f1Z3RlrADOqSAKiqC4AL9kp+kDFGr1fVd4C3jrOeC4ELZ7yCkiZk97skST1hUJckqScM6pIk9YRBXZKknjCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST3hHeVm2YrzbpywzMMXvXkWaiJJ6huv1CVJ6gmDuiRJPTGtoJ5kcZLrk3wtyX1JXpfkqCRbkjzQ/i5pZZPk0iTbktyV5KSB9axv5R9Isn78LUqSpPFM90r9EuBzVfVjwKuB+4DzgJurahVwc5sHOIPu2cqrgI3AZQBJjqJ7MtTJdE+DumD0RECSJE3elIN6kiOBn6I9X7mq/qGqngLWAZtasU3AWW16HXB1dW4FFic5FjgN2FJVu6pqN7AFOH2q9ZIkaaGazuj3lcAI8AdJXg3cCbwbOKaqHmtlHgeOadPLgO0Dy+9oaeOlz4rJjEaXJGk+mE73+yLgJOCyqnoN8C2e62oHoKoKqGls43mSbEyyNcnWkZGRmVqtJEm9MJ2gvgPYUVW3tfnr6YL811u3Ou3vEy1/J3DcwPLLW9p46T+gqi6vqtVVtXrp0qXTqLokSf0z5aBeVY8D25O8siWtBe4FNgOjI9jXAze06c3AOW0U/Brg6dZNfxNwapIlbYDcqS1NkiTth+neUe5XgU8kORh4EHgX3YnCdUk2AI8Ab2tlPwucCWwDvt3KUlW7krwXuKOVe09V7ZpmvSRJWnCmFdSr6ivA6jGy1o5RtoBzx1nPlcCV06mLJEkLnXeUkySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiQAkixOcn2SryW5L8nrkhyVZEuSB9rfJa1sklyaZFuSu5KcNLCe9a38A0nWj79FSTPNoC5p1CXA56rqx4BXA/fRPaTp5qpaBdzMcw9tOgNY1V4bgcsAkhwFXACcDLwWuGD0REDSgWdQl0SSI4GfAq4AqKp/qKqngHXAplZsE3BWm14HXF2dW4HF7QFOpwFbqmpXVe0GtgCnz+KuSAuaQV0SwEpgBPiDJF9O8rEkLwKOaQ9eAngcOKZNLwO2Dyy/o6WNly5pFhjUJUH3HIiTgMuq6jXAt3iuqx149vkNNRMbS7IxydYkW0dGRmZilZIwqEvq7AB2VNVtbf56uiD/9datTvv7RMvfCRw3sPzyljZe+vNU1eVVtbqqVi9dunRGd0RayAzqkqiqx4HtSV7ZktYC9wKbgdER7OuBG9r0ZuCcNgp+DfB066a/CTg1yZI2QO7UliZpFkz3eeqS+uNXgU8kORh4EHgX3Yn/dUk2AI8Ab2tlPwucCWwDvt3KUlW7krwXuKOVe09V7Zq9XZAWNoO6JACq6ivA6jGy1o5RtoBzx1nPlcCVM1s7SZNh97skST1hUJckqSfsfpekKVhx3o0Tlnn4ojfPQk2k53ilLklSTxjUJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ4wqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ7w0atDyEc6SpKmYtpX6kkOSvLlJJ9p8yuT3JZkW5JPJjm4pR/S5re1/BUD6zi/pd+f5LTp1kmSpIVoJrrf3w3cNzD/fuDiqjoe2A1saOkbgN0t/eJWjiQnAGcDrwJOBz6S5KAZqJckSQvKtIJ6kuXAm4GPtfkAbwSub0U2AWe16XVtnpa/tpVfB1xbVd+tqoeAbcBrp1MvSZIWouleqX8Q+C3gH9v8S4CnqmpPm98BLGvTy4DtAC3/6Vb+2fQxlnmeJBuTbE2ydWRkZJpVlySpX6Yc1JP8NPBEVd05g/XZp6q6vKpWV9XqpUuXztZmJUmaF6Yz+v0U4GeTnAkcChwBXAIsTrKoXY0vB3a28juB44AdSRYBRwJPDqSPGlxGkiRN0pSv1Kvq/KpaXlUr6Aa6fb6q3gF8AXhLK7YeuKFNb27ztPzPV1W19LPb6PiVwCrg9qnWS5KkhepA/E79t4Frk7wP+DJwRUu/Avh4km3ALroTAarqniTXAfcCe4Bzq+r7B6BekiT12owE9aq6BbilTT/IGKPXq+o7wFvHWf5C4MKZqIskSQuVt4mVJKknDOqSJPWEQV3Ss7ztszS/GdQlDfK2z9I8ZlCXBHjbZ6kPDOqSRs3abZ+95bN0YBjUJc36bZ+95bN0YByIm89Imn+87bPUA16pS/K2z1JPeKUuaV+87bM0jxjUJT2Pt32W5i+73yVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTPnp1nlpx3o0Tlnn4ojfPQk0kScPCoC5JB8hkTr7BE3DNHLvfJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ4wqEuS1BMGdUmSemLKQT3JcUm+kOTeJPckeXdLPyrJliQPtL9LWnqSXJpkW5K7kpw0sK71rfwDSdZPf7ckSVp4pnOlvgf4zao6AVgDnJvkBOA84OaqWgXc3OYBzgBWtddG4DLoTgKAC4CTgdcCF4yeCEiSpMmbclCvqseq6ktt+pvAfcAyYB2wqRXbBJzVptcBV1fnVmBxkmOB04AtVbWrqnYDW4DTp1ovSZIWqhn5Tj3JCuA1wG3AMVX1WMt6HDimTS8Dtg8stqOljZc+1nY2JtmaZOvIyMhMVF2SpN6YdlBP8mLgT4Bfr6pnBvOqqoCa7jYG1nd5Va2uqtVLly6dqdVKktQL0wrqSV5IF9A/UVWfaslfb93qtL9PtPSdwHEDiy9vaeOlS5olDnyV+mE6o98DXAHcV1W/N5C1GRhtyOuBGwbSz2kfBmuAp1s3/U3AqUmWtA+MU1uapNnjwFepB6bz6NVTgH8LfDXJV1rafwQuAq5LsgF4BHhby/sscCawDfg28C6AqtqV5L3AHa3ce6pq1zTqJWk/tRPsx9r0N5MMDnx9Qyu2CbgF+G0GBr4CtyYZHfj6BtrAV4AkowNfr5m1nZEWsCkH9ar6SyDjZK8do3wB546zriuBK6daF0kzZzYGvibZSHeFz8te9rKZq7y0wHlHOUnPmq2Brw56lQ4Mg7okwIGvUh8Y1CU58FXqiekMlJPUHw58lXrAoN5jK867cVLlHr7ozQe4Jhp2DnyV+sHud0mSesKgLklSTxjUJUnqCYO6JEk9YVCXJKknHP0uSXNsMr9U8Vcqmgyv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk948xl54wtJ6gmv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hAPlJGkecECrJsMrdUmSesKgLklST9j9rkmx60+Shp9BXdLQm8xJpSS73yVJ6g2DuiRJPWFQlySpJ3r9nbrfw80uB9NJc8s2KK/UJUnqiaG5Uk9yOnAJcBDwsaq6aI6rpAPAK4n+sy1Lc2cognqSg4APA/8S2AHckWRzVd07tzWTtD9sy8Nvsl9LenI9Pw1FUAdeC2yrqgcBklwLrAP8IFiAZnIshB9Ms8623BMz1Q5tg7NrWIL6MmD7wPwO4OS9CyXZCGxss3+X5P4x1nU08I0Zr+HssO4zLO+fVLGhrPskTKbeL5+NigyYybYMw/u/sV6TlPcPX52a+VivCdvzsAT1Samqy4HL91UmydaqWj1LVZpR1n1uzNe6z9d6w+TaMgzvPlqvyRvGOkF/6zUso993AscNzC9vaZLmF9uyNIeGJajfAaxKsjLJwcDZwOY5rpOk/WdblubQUHS/V9WeJL8C3ET3M5grq+qeKa5uwi69IWbd58Z8rfvQ1XuG2zIM4T421mvyhrFO0NN6papmqiKSJGkODUv3uyRJmiaDuiRJPdGroJ7k9CT3J9mW5Ly5rs++JDkuyReS3JvkniTvbulHJdmS5IH2d8lc13UsSQ5K8uUkn2nzK5Pc1o79J9sgqaGTZHGS65N8Lcl9SV43j475b7T3yt1Jrkly6Hw57lMxDO152NvpMLbDYWxjw9J2klyZ5Ikkdw+kjXls0rm01e+uJCdNZhu9Cep57vaUZwAnAG9PcsLc1mqf9gC/WVUnAGuAc1t9zwNurqpVwM1tfhi9G7hvYP79wMVVdTywG9gwJ7Wa2CXA56rqx4BX0+3D0B/zJMuAXwNWV9WJdIPQzmb+HPf9MkTtedjb6TC2w6FqY0PWdq4CTt8rbbxjcwawqr02ApdNagtV1YsX8DrgpoH584Hz57pe+1H/G+jul30/cGxLOxa4f67rNkZdl7c33xuBzwChuwPSorH+F8PyAo4EHqINEB1Inw/HfPRObUfR/WrlM8Bp8+G4T3F/h7I9D1M7HcZ2OIxtbNjaDrACuHuiYwN8FHj7WOX29erNlTpj355y2RzVZb8kWQG8BrgNOKaqHmtZjwPHzFG19uWDwG8B/9jmXwI8VVV72vywHvuVwAjwB63L8mNJXsQ8OOZVtRP4APAo8BjwNHAn8+O4T8XQtechbKfD2A6Hro3Ng7Yz3rGZUhvoU1Cfl5K8GPgT4Ner6pnBvOpOz4bqN4dJfhp4oqrunOu6TMEi4CTgsqp6DfAt9uoGHMZjDtC+Z1tH96H5UuBF/GA3ng6QYWunQ9wOh66Nzae2MxPHpk9Bfd7dnjLJC+k+KD5RVZ9qyV9PcmzLPxZ4Yq7qN45TgJ9N8jBwLV3X3yXA4iSjNzMa1mO/A9hRVbe1+evpPoCG/ZgDvAl4qKpGqup7wKfo/hfz4bhPxdC05yFtp8PaDoexjQ172xnv2EypDfQpqM+r21MmCXAFcF9V/d5A1mZgfZteT/cd3tCoqvOranlVraA7xp+vqncAXwDe0ooNXb0BqupxYHuSV7aktXSPBB3qY948CqxJcnh774zWfeiP+xQNRXse1nY6rO1wSNvYsLed8Y7NZuCcNgp+DfD0QDf9+GZrsMIsDUA4E/hr4G+A35nr+kxQ19fTdbPcBXylvc6k+17sZuAB4C+Ao+a6rvvYhzcAn2nTPwrcDmwD/hg4ZK7rN06dfwLY2o77nwJL5ssxB/4L8DXgbuDjwCHz5bhPcX/nvD3Ph3Y6bO1wGNvYsLQd4Bq67/W/R9ersWG8Y0M38PHD7f3/VbrR+xNuw9vESpLUE33qfpckaUEzqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9YVCXJKkn/j90WDhb6Ns32gAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in train_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in train_data.examples])\n", - "\n", - "print('Length distribution in Train data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Test data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe0AAAEICAYAAAByPazKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAfXElEQVR4nO3df7RdZX3n8fdHIij+4GdETMCkktqiq1YmBVzYjiVWAa241qDFsWPUdNJatLbaarBdpctKJ06dIi4tNRUKTK1AqdaMUjFFrdNpQYNaFFBJEUjSYCK/bKVq0e/8sZ8Lh8tN7k3uz33O+7XWWWfv53n23s++5z7nu/ezn7N3qgpJkrTwPWq+KyBJkqbGoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFb05JkWZJKsmgetv3qJH8/19uV5kOSi5O8YxrL/1uSH5nJOrX13pbk+TO93ilsd96+e+aTQVu9MKoNVAvLfAWovZXkM0l+aTCtqh5fVbfOV52mqy9/+9lm0B5RSfab7zpIw8aDSs02g/YClOStSbYn+dckX0uyqqUfkOTdSf6lvd6d5ICW94iu4nZmekybvjjJBUmuSvId4GeTHJXkw0l2JbkryXsHln1tkpuT3JPk6iRPnWLdD0pyYZIdbR/eMXaAMFbHJO9q6/1GklMHll2e5LNtv/82yfuS/HnL/mx7v7d18z1nYLkJ1yfNpCT/Gzga+D/tf/AtAz1Aa5LcAXyqlf3LJHcmua/9Tz9jYD0Xt//tj7f/9euSPK3lJcl5SXYm+XaSLyd55gR1OSTJx1rbvadNL2155wI/Dby31fO9LX3w++CgJJe25W9P8jtJHtXy9thOJ/kbPSrJuiT/3L5TrkhyaMsb+1utTnJHkm8l+e2BZR+b5JK2zZvb33fb7v72A5t95UTrG1pV5WsBvYCnA1uBp7T5ZcDT2vTbgWuBJwGLgX8Afr/lvRr4+3HrKuCYNn0xcB9wEt3B2uOAfwLOa9OPAZ7byp4ObAF+HFgE/A7wD7up77K2nUVt/iPA+9s6nwR8DvjlgTr+B/Dfgf2A1wH/AqTl/yPwLmB/4LnAt4E/n2g7U1mfL18z/QJuA54/MD/2f3lp+59/bEt/LfAE4ADg3cCXBpa5GLgLOL61rw8Cl7W8FwLXAwcDaW3wyIHl3tGmDwP+C3Bg285fAn89sI3PAL80ru6D3weXAh9tyy4Dvg6saXl71a4G/ybAG+m+o5a2fX8/8KFxf6s/BR4LPAv4HvDjLX898HfAIW35G4BtU/jbT7i+YX3NewV8jftA4BhgJ/B84NHj8v4ZOG1g/oXAbW361UwetC8dyHsOsIuBIDiQ9zdjDbjNPwq4H3jqBGXHGs4i4IjWaB47kP8K4NMDddwykHdgW/bJdEfRDwAHDuT/OZMH7QnXN9+fo6/hfO0hcPzIHpY5uJU5qM1fDHxgIP804Ktt+mS6AHoi8Khx67mYFrQn2MZPAvcMzH+G3QRtukD8feDYgbxfBj7TpveqXfHwoH0zsGog70i6A4BFA3+rpQP5nwPObNO3Ai8cyPslpha0J1zfsL7sHl9gqmoL8OvA7wE7k1yW5Ckt+ynA7QPFb29pU7V1YPoo4PaqemCCck8Fzk9yb5J7gbvpjvqXTLL+pwKPBnYMLPt+ujPuMXeOTVTV/W3y8W0/7h5IG1/f3dnd+qS59OD/apL9kqxvXcTfpgs2AIcPlL9zYPp+2v9sVX0KeC/wPrr2vyHJE8dvLMmBSd7fura/TXf56OBMbazK4XTtdPx3yWD73td29VTgIwPt/2bgB3QH9I9YNwP7TvcdMNjmp9L+97S+oWTQXoCq6i+q6rl0DaCAd7asf2lpY45uaQDfoTsiBiDJkyda9cD0VuDoTDxwZitdl/bBA6/HVtU/TFL1rXRn2ocPLPfEqnrGJMsB7AAOTXLgQNpRu6m7NF929384mP5f6S4xPR84iO6MELoD38k3UPWeqvpPwLHAjwK/NUGxN9NdSjuhqp4I/My4beypvXyL7ux3/HfJ9qnUbxJbgVPHfXc8pqqmsu4ddN3iY44al+93AAbtBSfJ05OcnG6A2XeBfwd+2LI/BPxOksVJDgd+l64LGbrr089I8pNJHkN3pr4nn6NrJOuTPC7JY5Kc1PL+BDh7bPBMG7TyssnqXlU7gE8C/yvJE9uglKcl+c9TWPZ2YDPwe0n2bwPNfn6gyC66v8OM/85U2gvfZPL/wSfQHbzeRXcg/QdTXXmSn0pyQpJH0x2If5eH2v/4bfw73cDMQ4FzplrPqvoBcAVwbpInpBtk+iYe+i6Zjj9p631q25/FSU6f4rJX0H3vHJJkCfD6cflT+dsPPYP2wnMA3YCMb9F1+zwJOLvlvYMusN0AfBn4Qkujqr5ON1Dtb4FbgD3edKQ13J+nu8Z1B7AN+IWW9xG6s/vLWtfbV4Cpjsp+Fd1AspuAe4Ar6a5rTcUr6a6139X263K6L7+xLrpzgf/Xut5OnOI6pZn0P+gOnO9N8pu7KXMpXXfzdrp2cO1erP+JdAOr7mnruAv4wwnKvZtu8NW32vo/MS7/fOCMNhL7PRMs/wa6g4Jb6b4r/gK4aC/quTvnAxuBTyb511a3E6a47Nvpvoe+Qfc9diWt/TdT+dsPvbFRu9KCk+RyugE6488iJA25JK+jG1Q2aU/dKPFMWwtG6xp8WutWP4XuuuBfz3e9JM2+JEcmOam1/6fTXbf/yHzXa6Hx7j1aSJ4MfJjuN6jbgNdV1Rfnt0qS5sj+dL82WQ7cC1wG/PG81mgBsntckqSesHtckqSeWNDd44cffngtW7ZsvqshLXjXX3/9t6pq8XzXY09sz9LU7Kk9L+igvWzZMjZv3jzf1ZAWvCS3T15qftmepanZU3u2e1ySpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ5Y0HdEm0nL1n18RtZz2/oXzch6JM2uqbR527P6xjNtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoSyMkyUVJdib5yrj0NyT5apIbk/zPgfSzk2xJ8rUkLxxIP6WlbUmybi73QRplI3NzFUkAXAy8F7h0LCHJzwKnA8+qqu8leVJLPxY4E3gG8BTgb5P8aFvsfcDPAduAzyfZWFU3zdleSCPKoC2NkKr6bJJl45JfB6yvqu+1Mjtb+unAZS39G0m2AMe3vC1VdStAkstaWYO2NMvsHpf0o8BPJ7kuyd8l+amWvgTYOlBuW0vbXfojJFmbZHOSzbt27ZqFqkujxaAtaRFwKHAi8FvAFUkyEyuuqg1VtbKqVi5evHgmVimNtEmD9kQDV5L8YRu0ckOSjyQ5eCDPgStSv2wDPlydzwE/BA4HtgNHDZRb2tJ2ly5plk3lTPti4JRxaZuAZ1bVTwBfB86GRwxcOQX44yT7JdmPbuDKqcCxwCtaWUnz76+BnwVoA832B74FbATOTHJAkuXACuBzwOeBFUmWJ9mfrs1vnJeaSyNm0oFoEw1cqapPDsxeC5zRph24Ii1gST4EPA84PMk24BzgIuCi1pv2fWB1VRVwY5Ir6NrpA8BZVfWDtp7XA1cD+wEXVdWNc74z0giaidHjrwUub9NL6IL4mMEBKuMHrpww0cqSrAXWAhx99NEzUD1JY6rqFbvJ+sXdlD8XOHeC9KuAq2awapKmYFoD0ZL8Nt0R+AdnpjoOXJEkaXf2+Uw7yauBFwOrWlca7HmAigNXJEmahn06005yCvAW4CVVdf9AlgNXJEmaJZOeae9m4MrZwAHApvZzzmur6leqyoErkiTNkqmMHp9o4MqFeyg/1ANXlq37+JTK3bb+RbNcE0nSqPGOaJIk9YRBW5KknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoSyMkyUVJdib5ygR5b05SSQ5v80nyniRbktyQ5LiBsquT3NJeq+dyH6RRZtCWRsvFwCnjE5McBbwAuGMg+VS6J/WtANYCF7Syh9I9OOgE4HjgnCSHzGqtJQEGbWmkVNVngbsnyDqP7nG7NZB2OnBpda4FDk5yJPBCYFNV3V1V9wCbmOBAQNLMM2hLIy7J6cD2qvqncVlLgK0D89ta2u7SJ1r32iSbk2zetWvXDNZaGk0GbWmEJTkQeBvwu7Ox/qraUFUrq2rl4sWLZ2MT0kgxaEuj7WnAcuCfktwGLAW+kOTJwHbgqIGyS1va7tIlzTKDtjTCqurLVfWkqlpWVcvourqPq6o7gY3Aq9oo8hOB+6pqB3A18IIkh7QBaC9oaZJmmUFbGiFJPgT8I/D0JNuSrNlD8auAW4EtwJ8CvwpQVXcDvw98vr3e3tIkzbJF810BSXOnql4xSf6ygekCztpNuYuAi2a0cpIm5Zm2JEk9YdCWJKknDNqSJPWEQVuSpJ6YNGhP9ICBJIcm2dQeFrBp7L7DPmBAkqTZM5Uz7Yt55H2F1wHXVNUK4Jo2Dz5gQJKkWTNp0N7NAwZOBy5p05cALx1I9wEDkiTNgn29pn1EuzMSwJ3AEW3aBwxIkjRLpj0Qrd2AoSYtOPX1+YABSZImsK9B+5ut25v2vrOl+4ABSZJmyb7exnQjsBpY394/OpD++iSX0Q06u6+qdiS5GviDgcFnLwDO3vdqD4dl6z4+aZnb1r9oDmoiSeqDSYN2e8DA84DDk2yjGwW+HriiPWzgduDlrfhVwGl0Dxi4H3gNdA8YSDL2gAHwAQOS9tFUDnalYTVp0N7DAwZWTVDWBwxIkjRLvCOaJEk9YdCWJKknDNqSJPXEvo4e1yQcLKOFKMlFwIuBnVX1zJb2h8DPA98H/hl4TVXd2/LOBtYAPwB+raqubumnAOcD+wEfqKr1c70v0ijyTFsaLRfzyFsIbwKeWVU/AXyd9nPMJMcCZwLPaMv8cZL9kuwHvI/uWQPHAq9oZSXNMoO2NEImepZAVX2yqh5os9fS3fwIumcJXFZV36uqb9D9lPP49tpSVbdW1feBy1pZSbPMoC1p0GuBv2nTPktAWmC8pi0JgCS/DTwAfHCm1llVG4ANACtXrpyxZxTMJe9cqIXEoC2JJK+mG6C2qt0kCfb8zACfJSDNA7vHpRHXRoK/BXhJVd0/kLURODPJAUmWAyuAz9HdjnhFkuVJ9qcbrLZxrustjSLPtKURsptnCZwNHABsSgJwbVX9SlXdmOQK4Ca6bvOzquoHbT2vB66m+8nXRVV145zvjDSCDNrSCNnNswQu3EP5c4FzJ0i/iu4BQZLmkN3jkiT1hEFbkqSeMGhLktQTBm1JknrCoC1JUk84elzSyPJpfOobz7QlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknphW0E7yG0luTPKVJB9K8pj25J/rkmxJcnl7ChDtSUGXt/TrkiybiR2QJGlU7HPQTrIE+DVgZVU9k+5pP2cC7wTOq6pjgHuANW2RNcA9Lf28Vk6SJE3RdLvHFwGPTbIIOBDYAZwMXNnyLwFe2qZPb/O0/FVpzwGUJEmT2+egXVXbgXcBd9AF6/uA64F7q+qBVmwbsKRNLwG2tmUfaOUPG7/eJGuTbE6yedeuXftaPUmShs50uscPoTt7Xg48BXgccMp0K1RVG6pqZVWtXLx48XRXJ2lAkouS7EzylYG0Q5NsSnJLez+kpSfJe9o4lBuSHDewzOpW/pYkq+djX6RRNJ3u8ecD36iqXVX1H8CHgZOAg1t3OcBSYHub3g4cBdDyDwLumsb2Je29i3nkwfU64JqqWgFc0+YBTgVWtNda4ALogjxwDnACcDxwzliglzS7phO07wBOTHJguza9CrgJ+DRwRiuzGvhom97Y5mn5n6qqmsb2Je2lqvoscPe45MHxJuPHoVxanWvpDsiPBF4IbKqqu6vqHmATM9DLJmly07mmfR3dgLIvAF9u69oAvBV4U5ItdNesL2yLXAgc1tLfxENH85Lm1xFVtaNN3wkc0aYfHIfSjI1R2V36IzhGRZpZ03rKV1WdQ9dNNuhWui6z8WW/C7xsOtuTNLuqqpLMWA9YVW2gO5hn5cqV9qxJ0+Qd0SR9s3V70953tvQHx6E0Y2NUdpcuaZYZtCUNjjcZPw7lVW0U+YnAfa0b/WrgBUkOaQPQXtDSJM2yaXWPS+qXJB8CngccnmQb3eWt9cAVSdYAtwMvb8WvAk4DtgD3A68BqKq7k/w+8PlW7u1VNX5wm6RZYNCWRkhVvWI3WasmKFvAWbtZz0XARTNYNUlTYPe4JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JACS/EaSG5N8JcmHkjwmyfIk1yXZkuTyJPu3sge0+S0tf9n81l4aDQZtSSRZAvwasLKqngnsB5wJvBM4r6qOAe4B1rRF1gD3tPTzWjlJs2xaQTvJwUmuTPLVJDcneU6SQ5NsSnJLez+klU2S97Qj8xuSHDczuyBphiwCHptkEXAgsAM4Gbiy5V8CvLRNn97mafmrkmQO6yqNpOmeaZ8PfKKqfgx4FnAzsA64pqpWANe0eYBTgRXttRa4YJrbljRDqmo78C7gDrpgfR9wPXBvVT3Qim0DlrTpJcDWtuwDrfxh49ebZG2SzUk279q1a3Z3QhoB+xy0kxwE/AxwIUBVfb+q7uXhR+Djj8wvrc61wMFJjtznmkuaMa1H7HRgOfAU4HHAKdNdb1VtqKqVVbVy8eLF012dNPKmc6a9HNgF/FmSLyb5QJLHAUdU1Y5W5k7giDb94JF5M3jULml+PR/4RlXtqqr/AD4MnER3cL2olVkKbG/T24GjAFr+QcBdc1tlafRMJ2gvAo4DLqiqZwPf4aGucACqqoDam5XanSbNizuAE5Mc2K5NrwJuAj4NnNHKrAY+2qY3tnla/qdae5c0i6YTtLcB26rqujZ/JV0Q/+ZYt3d739nyHzwybwaP2h9kd5o091o7vhL4AvBluu+GDcBbgTcl2UJ3zfrCtsiFwGEt/U2MO2CXNDsWTV5kYlV1Z5KtSZ5eVV/joSPzm+iOwNfzyCPz1ye5DDgBuG+gG13SPKuqc4BzxiXfChw/QdnvAi+bi3pJesg+B+3mDcAH2w0XbgVeQ3eEfkWSNcDtwMtb2auA04AtwP2trCRJmqJpBe2q+hKwcoKsVROULeCs6WxPkqRR5h3RJEnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEsCIMnBSa5M8tUkNyd5TpJDk2xKckt7P6SVTZL3JNmS5IYkx813/aVRMN3naWuWLVv38UnL3Lb+RXNQE42A84FPVNUZSfYHDgTeBlxTVeuTrAPWAW8FTgVWtNcJwAXtXdIs8kxbEkkOAn4GuBCgqr5fVfcCpwOXtGKXAC9t06cDl1bnWuDgJEfOcbWlkeOZtiSA5cAu4M+SPAu4HngjcERV7Whl7gSOaNNLgK0Dy29raTsG0kiyFlgLcPTRR89a5efbVHrEwF4xTZ9n2pKgO4A/Drigqp4NfIeuK/xBVVVA7c1Kq2pDVa2sqpWLFy+escpKo8qgLQm6M+VtVXVdm7+SLoh/c6zbu73vbPnbgaMGll/a0iTNIoO2JKrqTmBrkqe3pFXATcBGYHVLWw18tE1vBF7VRpGfCNw30I0uaZZ4TVvSmDcAH2wjx28FXkN3YH9FkjXA7cDLW9mrgNOALcD9raykWWbQlgRAVX0JWDlB1qoJyhZw1qxXStLD2D0uSVJPTDtoJ9kvyReTfKzNL09yXbtT0uWtq40kB7T5LS1/2XS3LUnSKJmJ7vE3AjcDT2zz7wTOq6rLkvwJsIbubklrgHuq6pgkZ7ZyvzAD25ekXvAOh5quaZ1pJ1kKvAj4QJsPcDLdz0XgkXdQGruz0pXAqlZekiRNwXS7x98NvAX4YZs/DLi3qh5o82N3SYKBOyi1/Pta+YdJsjbJ5iSbd+3aNc3qSZI0PPY5aCd5MbCzqq6fwfp4ByVJknZjOte0TwJekuQ04DF017TPp3twwKJ2Nj14l6SxOyhtS7IIOAi4axrblyRppOzzmXZVnV1VS6tqGXAm8KmqeiXwaeCMVmz8HZTG7qx0Riu/V/cxliRplM3G77TfCrwpyRa6a9YXtvQLgcNa+psY9zACSZK0ZzNyR7Sq+gzwmTZ9K3D8BGW+C7xsJrY33lQfizes/BmJhsWot2VpMt4RTZKknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0Jb0IB+1Ky1sBm1Jg8YetTtm7FG7xwD30D1iFwYetQuc18pJmmUGbUmAj9qV+sCgLWmMj9qVFjiDtiQftSv1xIzce1xS7/moXakHPNOW5KN2pZ4waEvaEx+1Ky0gdo9Lepj5ftSupN3zTFuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPXEPgftJEcl+XSSm5LcmOSNLf3QJJuS3NLeD2npSfKeJFuS3JDkuJnaCUmSRsF0zrQfAN5cVccCJwJnJTmW7naG11TVCuAaHrq94anAivZaC1wwjW1LkjRy9jloV9WOqvpCm/5X4Ga6Z+yeDlzSil0CvLRNnw5cWp1r6Z4edOQ+11ySpBEzI9e0kywDng1cBxxRVTta1p3AEW16CbB1YLFtLW38utYm2Zxk865du2aiepIkDYVpPzAkyeOBvwJ+vaq+neTBvKqqJHv1uL6q2gBsAFi5cqWP+pshy9Z9fErlblv/olmuiSRpX03rTDvJo+kC9ger6sMt+Ztj3d7tfWdL3w4cNbD40pYmSZKmYDqjx0P3TN2bq+qPBrI2Aqvb9GrgowPpr2qjyE8E7hvoRpckSZOYzpn2ScB/A05O8qX2Og1YD/xckluA57d5gKuAW4EtwJ8CvzqNbUuaQf6EU+qHfb6mXVV/D2Q32asmKF/AWfu6PUmzauwnnF9I8gTg+iSbgFfT/YRzfZJ1dD/hfCsP/wnnCXQ/4TxhXmo+ZKYy/sSxJ6PLO6JJ8iecUk8YtCU9jD/hlBYug7akB43/CedgXrvEtdc/4ayqlVW1cvHixTNYU2k0Tft32pKGw55+wllVO/wJ58Lhde/R5Zm2JH/CKfWEZ9raax7lD6Wxn3B+OcmXWtrb6H6yeUWSNcDtwMtb3lXAaXQ/4bwfeM3cVlcaTQZtPcxUb3eq4eJPOKV+sHtckqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFbkqSe8I5okjSEvN3wcDJoS9KImuptiw3uC4dBW7PCo3xJmnle05YkqScM2pIk9YTd45o3Xk+TpL1j0NaC5/VxaX7ZBheOOe8eT3JKkq8l2ZJk3VxvX9LMsC1Lc29Oz7ST7Ae8D/g5YBvw+SQbq+qmuayHho9nAnPLtqzxpnq5azK20z2b6+7x44EtVXUrQJLLgNMBG7pm3UIM7AuxTlNkW9ascKzLns110F4CbB2Y3wacMFggyVpgbZv9tyR3Ad+am+rNq8MZ/v1c8PuYd87IamZ0P6dYp6fO1PamaNK2DBO256/NQd3myoL/f94HvdmnvWyrvdmvZrftecENRKuqDcCGsfkkm6tq5TxWaU6Mwn6Owj7C6OznVIxvz8NkGD/nYdwnGK79muuBaNuBowbml7Y0Sf1iW5bmwVwH7c8DK5IsT7I/cCawcY7rIGn6bMvSPJjT7vGqeiDJ64Grgf2Ai6rqxkkWG8qutQmMwn6Owj7CCOznPrblYTOMn/Mw7hMM0X6lqua7DpIkaQq897gkST1h0JYkqScWdNAextskJjkqyaeT3JTkxiRvbOmHJtmU5Jb2fsh813UmJNkvyReTfKzNL09yXftML2+DmHorycFJrkzy1SQ3J3nOsH6Wo2rY2+ywtdFhb5MLNmgP3CbxVOBY4BVJjp3fWs2IB4A3V9WxwInAWW2/1gHXVNUK4Jo2PwzeCNw8MP9O4LyqOga4B1gzL7WaOecDn6iqHwOeRbevw/pZjqphb7PD1kaHu01W1YJ8Ac8Brh6YPxs4e77rNQv7+VG6+zd/DTiypR0JfG2+6zYD+7aUroGcDHwMCN1diRZN9Bn37QUcBHyDNqBzIH3oPktfD/t8h6bNDlsbHYU2uWDPtJn4NolL5qkusyLJMuDZwHXAEVW1o2XdCRwxT9WaSe8G3gL8sM0fBtxbVQ+0+b5/psuBXcCfte7FDyR5HMP5WYqhbLPD1kaHvk0u5KA91JI8Hvgr4Ner6tuDedUdDvb6t3hJXgzsrKrr57sus2gRcBxwQVU9G/gO47rdhuGzVGfY2uyQttGhb5MLOWgP7W0SkzyarvF/sKo+3JK/meTIln8ksHO+6jdDTgJekuQ24DK67rfzgYOTjN3Up++f6TZgW1Vd1+avpPvCGLbPcuQNaZsdxjY69G1yIQftobxNYpIAFwI3V9UfDWRtBFa36dV01816q6rOrqqlVbWM7rP7VFW9Evg0cEYr1uv9rKo7ga1Jnt6SVtE9mnKoPstRN6xtdhjb6Ci0yQV9R7Qkp9Fdcxm7TeK581ylaUvyXOD/Al/moetIb6O7RnYFcDRwO/Dyqrp7Xio5w5I8D/jNqnpxkh+hO6o/FPgi8ItV9b35rN90JPlJ4APA/sCtwGvoDoaH8rMcRaPQZoepjQ57m1zQQVuSJD1kIXePS5KkAQZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9cT/B/odrf8G1COhAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in test_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in test_data.examples])\n", - "\n", - "print('Length distribution in Test data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model side\n", - "__Here comes simple pipeline of NMT model learning. It almost copies the week03 practice__" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "device(type='cuda', index=1)" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "device" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def _len_sort_key(x):\n", - " return len(x.src)\n", - "\n", - "BATCH_SIZE = 128\n", - "\n", - "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE, \n", - " device = device,\n", - " sort_key=_len_sort_key\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "[torchtext.data.batch.Batch of size 128]\n", - "\t[.trg]:[torch.cuda.LongTensor of size 55x128 (GPU 1)]\n", - "\t[.src]:[torch.cuda.LongTensor of size 59x128 (GPU 1)]\n", - "torch.Size([59, 128]) torch.Size([55, 128])\n" - ] - } - ], - "source": [ - "for x in train_iterator:\n", - " break\n", - "print(x)\n", - "print(x.src.shape, x.trg.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "import my_network\n", - "Encoder = my_network.Encoder\n", - "Decoder = my_network.Decoder\n", - "Seq2Seq = my_network.Seq2Seq" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(SRC.vocab)\n", - "OUTPUT_DIM = len(TRG.vocab)\n", - "ENC_EMB_DIM = 256\n", - "DEC_EMB_DIM = 256\n", - "HID_DIM = 512\n", - "N_LAYERS = 2\n", - "ENC_DROPOUT = 0.5\n", - "DEC_DROPOUT = 0.5\n", - "\n", - "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n", - "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n", - "\n", - "# dont forget to put the model to the right device\n", - "model = Seq2Seq(enc, dec, device).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Seq2Seq(\n", - " (encoder): Encoder(\n", - " (embedding): Embedding(9267, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - " (decoder): Decoder(\n", - " (embedding): Embedding(6699, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (out): Linear(in_features=512, out_features=6699, bias=True)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - ")" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def init_weights(m):\n", - " # \n", - " for name, param in m.named_parameters():\n", - " nn.init.uniform_(param, -0.08, 0.08)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The model has 14,880,299 trainable parameters\n" - ] - } - ], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [], - "source": [ - "PAD_IDX = TRG.vocab.stoi['']\n", - "optimizer = optim.Adam(model.parameters())\n", - "criterion = nn.CrossEntropyLoss(ignore_index = PAD_IDX)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, clip, train_history=None, valid_history=None):\n", - " model.train()\n", - " \n", - " epoch_loss = 0\n", - " history = []\n", - " for i, batch in enumerate(iterator):\n", - " \n", - " src = batch.src\n", - " trg = batch.trg\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " output = model(src, trg)\n", - " \n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - " \n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - " \n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - " \n", - " loss = criterion(output, trg)\n", - " \n", - " loss.backward()\n", - " \n", - " # Let's clip the gradient\n", - " torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " history.append(loss.cpu().data.numpy())\n", - " if (i+1)%10==0:\n", - " fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8))\n", - "\n", - " clear_output(True)\n", - " ax[0].plot(history, label='train loss')\n", - " ax[0].set_xlabel('Batch')\n", - " ax[0].set_title('Train loss')\n", - " if train_history is not None:\n", - " ax[1].plot(train_history, label='general train history')\n", - " ax[1].set_xlabel('Epoch')\n", - " if valid_history is not None:\n", - " ax[1].plot(valid_history, label='general valid history')\n", - " plt.legend()\n", - " \n", - " plt.show()\n", - "\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "def evaluate(model, iterator, criterion):\n", - " \n", - " model.eval()\n", - " \n", - " epoch_loss = 0\n", - " \n", - " history = []\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for i, batch in enumerate(iterator):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - "\n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - "\n", - " loss = criterion(output, trg)\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "def epoch_time(start_time, end_time):\n", - " elapsed_time = end_time - start_time\n", - " elapsed_mins = int(elapsed_time / 60)\n", - " elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n", - " return elapsed_mins, elapsed_secs" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "train_history = []\n", - "valid_history = []\n", - "\n", - "N_EPOCHS = 10\n", - "CLIP = 1\n", - "\n", - "best_valid_loss = float('inf')" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHwCAYAAABUqPIVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdeXxU1f3/8dfJAmFPAggikOCGInuCG7hiFTfcW6y20tri1tp+269V237VWv3Vqm1ta9WqdWndt7buxVYUFyqbgKIoLkEWhbAJAQJJ5vz+uDOTmclMZu5kMnfuzPvpAzOZucuZJclnPvM5n2OstYiIiIiISKsirwcgIiIiIpJrFCSLiIiIiMRQkCwiIiIiEkNBsoiIiIhIDAXJIiIiIiIxFCSLiIiIiMRQkCx5xRjzgjHmvDT3rTPGHJPpMYmIiIj/lHg9ABFjTEPEt92BnUBL8PsLrLUPpnosa+3xmRybiIiIFCYFyeI5a23P0GVjTB3wHWvtv2O3M8aUWGubszk2ERERKUwqt5CcZYw50hizyhhzuTHmC+BeY0yFMeZZY0y9MWZT8PLgiH1eMcZ8J3h5ujHmdWPMzcFtPzXGpJRpNsZ0NcbcYoxZE/x3izGma/C2fsHzbjbGbDTGvGaMKQredrkxZrUxZqsx5gNjzOROeGhERESkkylIllw3EKgEqoAZOK/Ze4PfDwV2ALe2s/9BwAdAP+BG4C/GGJPCeX8GHAyMBcYABwI/D972Y2AV0B8YAPwUsMaY4cD3gAnW2l7AcUBdivdTREREcoiCZMl1AeBqa+1Oa+0Oa+0Ga+2T1trt1tqtwPXAEe3sv8Jae5e1tgW4H9gdJ7BN5hzgWmvtOmttPfAL4BvB25qCx6my1jZZa1+z1lqcOuquwAhjTKm1ts5a+3Fa91pEREQ8pSBZcl29tbYx9I0xprsx5s/GmBXGmC3AbKDcGFOcYP8vQhestduDF3sm2DbSIGBFxPcrgtcB3AR8BMw0xnxijLkiePyPgB8C1wDrjDGPGGMGISIiIr6jIFlynY35/sfAcOAga21v4PDg9amUULixBqekI2Ro8DqstVuttT+21u4JTAV+FKo9ttY+ZK2dFNzXAr/O8LhEREQkCxQki9/0wqlD3myMqQSu7qTzPAz83BjT3xjTD7gKeADAGHOSMWbvYG3zlzhlFgFjzHBjzNHBCX6NwXEGOml8IiIi0okUJIvf3AJ0A9YD/wVe7KTzXAfMB5YA7wALg9cB7AP8G2gA5gC3WWtn4dQj3xAc2xfAbsCVnTQ+ERER6UTGmW8kIiIiIiIhyiSLiIiIiMRQkCwiIiIiEkNBsoiIiIhIDAXJIiIiIiIxFCSLiIiIiMQo8XoA8fTr189WV1d7PQwREdcWLFiw3lrb3+txZJN+Z4uIX7X3Ozsng+Tq6mrmz5/v9TBERFwzxqxIvlV+0e9sEfGr9n5nq9xCRERERCSGgmQRERERkRg5WW4hIiKdwxhTB2wFWoBma21tzO1HAv8EPg1e9ZS19tpsjlFEJBcoSBYRKTxHWWvXt3P7a9bak7I2GpEOaGpqYtWqVTQ2Nno9FMlhZWVlDB48mNLS0pT3UZAsIiIivrVq1Sp69epFdXU1xhivhyM5yFrLhg0bWLVqFcOGDUt5P9Uki4gUFgvMNMYsMMbMSLDNIcaYxcaYF4wxB2RzcCJuNTY20rdvXwXIkpAxhr59+7r+tEGZZBGRwjLJWrvaGLMb8JIxZpm1dnbE7QuBKmttgzHmBOAfwD6xBwkG2DMAhg4dmo1xiySkAFmSSec1okyyiEgBsdauDn5dB/wdODDm9i3W2obg5eeBUmNMvzjHudNaW2utre3fv6DWThHJaUceeWTcvuW33HIL27dvd328q666in//+98pb3/ffffxve99L+5tJ5xwAps3b064b7pj7CwKkkVECoQxpocxplfoMnAs8G7MNgNNMOVijDkQ5+/EhmyPVUTis9YSCARc79deANrS0pJwv2uvvZZjjjnG9fnief755ykvL094ezpBcntj7ygFySIihWMA8LoxZjEwF3jOWvuiMeZCY8yFwW3OBN4NbvMHYJq11no0XhFf+OUvf8nw4cOZNGkSZ599NjfffDMAH3/8MVOmTKGmpobDDjuMZcuWATB9+nQuvfRSDj30UPbcc0+eeOKJ8LFuuukmJkyYwOjRo7n66qsBqKurY/jw4Xzzm99k5MiRrFy5kosuuoja2loOOOCA8HaJ/OEPf2DNmjUcddRRHHXUUQD07NmTH//4x4wZM4Y5c+Zw7bXXMmHCBEaOHMmMGTMI/dhPnz49PL7q6mquvvpqxo8fz6hRo8L3J9aaNWuYMmUK++yzDz/5yU/C11dXV7N+/Xq2bdvGiSeeyJgxYxg5ciSPPvpo3DE+/PDDjBo1ipEjR3L55ZeHjxM59uuvv55TTz01fNtLL73EaaedlsKzlpxqkkVECoS19hNgTJzr74i4fCtwazbHJZIpv3hmKe+t2ZLRY44Y1JurT048f3XevHk8+eSTLF68mKamJsaPH09NTQ0AM2bM4I477mCfffbhrbfe4uKLL+bll18G4PPPP+f1119n2bJlTJ06lTPPPJOZM2eyfPly5s6di7WWqVOnMnv2bIYOHcry5cu5//77OfjggwG4/vrrqayspKWlhcmTJ7NkyRJGjx4dd4yXXnopv/3tb5k1axb9+jnVU9u2beOggw7iN7/5jXM/R4zgqquuAuAb3/gGzz77LCeffHKbY/Xr14+FCxdy2223cfPNN3P33Xe32WbRokW8/fbbdO3aleHDh/P973+fIUOGhG9/8cUXGTRoEM899xwAX375JX369Ika45o1a7j88stZsGABFRUVHHvssfzjH//g1FNPjRq7tZb999+f+vp6+vfvz7333su3v/3t9p/UFCmTLCIiIpKmN954g1NOOYWysjJ69eoVDiwbGhp48803Oeussxg7diwXXHABn3/+eXi/U089laKiIkaMGMHatWsBmDlzJjNnzmTcuHGMHz+eZcuWsXz5cgCqqqrCATLAY489xvjx4xk3bhxLly7lvffeczXu4uJizjjjjPD3s2bN4qCDDmLUqFG8/PLLLF26NO5+p59+OgA1NTXU1dXF3Wby5Mn06dOHsrIyRowYwYoVK6JuHzVqFC+99BKXX345r732Gn369GlzjHnz5nHkkUfSv39/SkpKOOecc5g9e3absRtj+MY3vsEDDzzA5s2bmTNnDscff7yrxyIRZZJFREQkL7SX8c22QCBAeXk5ixYtint7165dw5dDpQ3WWq688kouuOCCqG3r6uro0aNH+PtPP/2Um2++mXnz5lFRUcH06dNdtzcrKyujuLgYcNroXXzxxcyfP58hQ4ZwzTXXJDxeaNzFxcU0NzcnvW/xttt3331ZuHAhzz//PD//+c+ZPHlyOIvtduwA3/rWtzj55JMpKyvjrLPOoqQkM+GtMskiIiIiaZo4cSLPPPMMjY2NNDQ08OyzzwLQu3dvhg0bxuOPPw44AfDixYvbPdZxxx3HPffcQ0NDAwCrV69m3bp1bbbbsmULPXr0oE+fPqxdu5YXXngh6Th79erF1q1b494WCoj79etHQ0NDVI10Z1izZg3du3fn3HPP5bLLLmPhwoVtxnjggQfy6quvsn79elpaWnj44Yc54ogj4h5v0KBBDBo0iOuuu45vfetbGRunMskiIiIiaZowYQJTp05l9OjRDBgwgFGjRoXLBx588EEuuugirrvuOpqampg2bRpjxrSZFhB27LHH8v7773PIIYcAzgS1Bx54ICprCjBmzBjGjRvHfvvtx5AhQ5g4cWLScc6YMYMpU6YwaNAgZs2aFXVbeXk53/3udxk5ciQDBw5kwoQJbh8GV9555x0uu+wyioqKKC0t5fbbb487xhtuuIGjjjoKay0nnngip5xySsJjnnPOOdTX17P//vtnbJwmFyct19bW2ng9/kREcp0xZoG1ttbrcWSTfmeLl95///2MBkbpaGhooGfPnmzfvp3DDz+cO++8k/Hjx3s6pkLzve99j3HjxnH++ecn3Cbea6W939nKJIuISPZt+Bj67uX1KEQyYsaMGbz33ns0NjZy3nnnKUDOspqaGnr06BHu1JEpCpJFJKf86LFF9Opawi9OGen1UKSz1H8Id0yCkafD8TdCWW+vRyTSIQ899JDXQyhoCxYs6JTjauKeiOSUj+u38cn6bV4PQzpT5TCY9ENY8qgTLH/2ltcjEhFpQ0GyiIhkV3EpHPVT+NYLgIV7p8CsX0FL/HZSIiJeUJAsIrnFWnJwPrF0hqEHw4VvwOivwas3OMHyxk+8HpWICKAgWURyjAUsipILRllvOO0OOPOeYK3yYfD2g+idkoh4TUGyiOQUaxUfFaSRZ8BFb8DuY+GfF8Pj02H7Rq9HJeI7Rx55JJloyRh5nBNOOIHNmze32eaaa67h5ptvbnP99OnT4y5IsmbNGs4888yE59y8eTO33XZbB0adWQqSRSSnWFRuUbDKh8B5T8Pkq2HZs3D7RPh0ttej6nyBACx+FP50MLx5q9ejkRxnrSUQCGT1nM8//zzl5eUdPs6gQYPaXc0vnSA50dLYmaAgWURyigLkAldUDIf9CL7zb+jSHe6fCjP/D5p3eT2yzLMWPvwX/Pkw+PsM2FQHr/xKGXQf+uUvf8nw4cOZNGkSZ599dji7+vHHHzNlyhRqamo47LDDWLZsGeBkWi+99FIOPfRQ9txzz6jA8aabbmLChAmMHj2aq6++GoC6ujqGDx/ON7/5TUaOHMnKlSu56KKLqK2t5YADDghvl8iLL77IWWedFf7+lVde4aSTTgJI6TjV1dWsX78egOuvv559992XSZMm8cEHHyQ85+zZs9vcv7q6OkaOdNp7Ll26lAMPPJCxY8cyevRoli9fzhVXXMHHH3/M2LFjueyyy7DWctlllzFy5EhGjRrFo48+Gh7/YYcdxtSpUxkxYgRXXXUVt9xyS/jcP/vZz/j973/f7mOSCvVJFpGco5pkYdA4uGA2/Otn8OYf4JNX4Iy7of9wr0eWGZ/9F/59DXw2Byr3dGqy++8Ptx/q3N9jrvF4gD71whXwxTuZPebAUXD8DQlvnjdvHk8++SSLFy+mqamJ8ePHU1NTAziLjNxxxx3ss88+vPXWW1x88cW8/PLLAHz++ee8/vrrLFu2jKlTp3LmmWcyc+ZMli9fzty5c7HWMnXqVGbPns3QoUNZvnw5999/PwcffDDgBKuVlZW0tLQwefJklixZwujRo+OO8ZhjjmHGjBls27aNHj168OijjzJt2jTXx1mwYAGPPPIIixYtorm5Oeq+xop3/yLdcccd/OAHP+Ccc85h165dtLS0cMMNN/Duu++yaNEiAJ588kkWLVrE4sWLWb9+PRMmTODwww8HYOHChbz77rsMGzaMuro6Tj/9dH74wx8SCAR45JFHmDt3bsLnLFUKkkUkp6gmWcK69ICTb4F9vgJPfx/+fAQcdx3Ung/GeD269KxdCv/5JXz4AvQcACf+FsZ/02mLB05t9lt3wsGXQM/+3o5VUvLGG29wyimnUFZWRllZGSeffDLgLFX95ptvRmVwd+7cGb586qmnUlRUxIgRI1i7di0AM2fOZObMmYwbNy58jOXLlzN06FCqqqrCATLAY489xp133klzczOff/457733XsLgtqSkhClTpvDMM89w5pln8txzz3HjjTe6Ps5rr73GaaedRvfu3QGYOnVqwscl3v2LdMghh3D99dezatUqTj/9dPbZZ58227z++uucffbZFBcXM2DAAI444gjmzZtH7969OfDAAxk2bBjgZLr79u3L22+/zdq1axk3bhx9+/ZNOLZUKUgWkZxig/9EwvY7EfaogX9cDM/9GJa/BFNv9VcQuWmFU0qx+BHo2hsmXwUHXei8EYh0xOWw9Cl48/dw7HXejNXP2sn4ZlsgEKC8vDycFY3VtWvX8GUbzAxYa7nyyiu54IILoratq6ujR4/W18qnn37KzTffzLx586ioqGD69Ok0Nja2O55p06Zx6623UllZSW1tLb169UrrOKmKd/8iff3rX+eggw7iueee44QTTuDPf/4ze+65Z8rHj3w8AL7zne9w33338cUXX/Dtb387/YFHUE2yiOQUa62iZGmr10A45wmY8mv4eBbcfogTLOe6hnp44XL4Yw0s/TtMvBR+sAgO+3HbABmg/74w6qsw927Y2jb7Jrln4sSJPPPMMzQ2NtLQ0MCzzz4LQO/evRk2bBiPP/444PxuW7x4cbvHOu6447jnnntoaGgAYPXq1axbt67Ndlu2bKFHjx706dOHtWvX8sILLyQd5xFHHMHChQu56667wqUWbo9z+OGH849//IMdO3awdetWnnnmmaTnTeSTTz5hzz335NJLL+WUU05hyZIl9OrVi61bt4a3Oeyww3j00UdpaWmhvr6e2bNnc+CBB8Y93mmnncaLL77IvHnzOO6449IeVyRlkkVExB+KiuDgC2HYYfDkd+HBM+HAGfCVa6G0m9eji9a4Beb8CebcCk07YNy5Tpa4zx7J9z3iJ/DO4/DGLTDlV50/VumQCRMmMHXqVEaPHs2AAQMYNWoUffr0AeDBBx/koosu4rrrrqOpqYlp06YxZsyYhMc69thjef/99znkkEMA6NmzJw888ADFxcVR240ZM4Zx48ax3377MWTIECZOnJh0nMXFxZx00kncd9993H///WkdZ/z48Xzta19jzJgx7LbbbkyYMCHpeRN57LHH+Nvf/kZpaSkDBw7kpz/9KZWVlUycOJGRI0dy/PHHc+ONNzJnzhzGjBmDMYYbb7yRgQMHhidARurSpQtHHXUU5eXlbR6vdJl4KXCv1dbW2kz0+BMR/znud7Pp3a2Exy881OuhpMUYs8BaW+v1OLLJk9/ZTY3wn1/Af2+D/vs5k/oGjsruGOJp3gnz74HZN8H2DTDiFDj6/6Bf23rLdv3jEidQ/sFi6L1754w1T7z//vvsv//+no6hoaGBnj17sn37dg4//HDuvPNOxo8f7+mYCk0gEGD8+PE8/vjjceubIf5rpb3f2Sq3EJGck4Pv3SXXlJY5WdZzn4Idm+Cuo50ew1nuHxsWaIFFDzllFS9eAQNGwndfhq/+1X2ADHD4/4Jtgdd/m/mxSsbNmDGDsWPHMn78eM444wwFyFn23nvvsffeezN58uSEAXI6VG4hIjnFqgGcuLH3ZLhojtP9YubP4KOX4NTbofeg7JzfWvjgBfjPtVD/vrNi4NQ/wl5Hdey4lcNg7Dmw4D6Y+APoMzgjw5XO8dBDD3k9hII2YsQIPvnkk4wfV5lkEckpTgs4hcniQo++MO1BOPn3sHKu02v4vX92/nlXvAn3HAePnA0tu+Cs++C7szoeIIccfpnzA/HabzJzPBFxRUGyiOQUhceSFmOgZjpc8BqUV8Fj34R/XgI7GzJ/ri/egQfPgnuPh82fOcH5JW/BAac5kwszpXwI1JwHC//mtJCThPTGWpJJ5zWiIFlEcoq1KreQDui3N5z/Ekz6Ebz9oLPk86oFmTn2xk+drhp3HAYr34JjfgHfX+gE56HFQDJt0o/AFDkTASWusrIyNmzYoEBZErLWsmHDBsrKylztp5pkEckpFk3ckw4q6QLHXA17HwN/vwD+8hU48ko47EdQlEZrqIZ18OqNTn1wUQlM+qFTJ9ytIuNDb6PPHlD7LZh7lzP+ytQXWygUgwcPZtWqVdTX13s9FMlhZWVlDB7srrZfQbKI5BzFyPmtsamFOZ9sYPiAXgwq78T+xtUT4cLXnVX6Zl0HH/0bTr8TKqpSHOiX8OYfYc5t0NzolD4c/pPst2Sb9D9OgP7qTXDa7dk9tw+UlpaGlycWySSVW4hIblEqOe9t2LaLb907j5lLv+j8k3UrhzP/AqffBevegzsmwZLH2t+nqdEJjn8/xilz2Pc4+N48OOl33vQs7jUQJnwHljwC6z/K/vlFCpSCZBHJKQqP898e5d0Y1KeM+Ss2Ze+ko7/qZJV3GwFPfReeOB92bI7epqXZmST3x/Ew8+cwaBzMeAXOuhf67pW9scYz8QdQUgav/trbcYgUEAXJIpJTNHGvMNRUVzK/blN2J1tVVMH05+Con8PSvztZ5bo3nE8u3n/GaR339PeczO03n4Zv/N0JlHNBz93gwO86q/DVf+D1aEQKgoJkEckpqrYoDBOqK/hiSyOrN+/I7omLS+CIy+D8mc4kvPtOhNsOhkfPBRuAr/4NvvMf2POI7I4rFYf+ALr0gFdu8HokIgVBQbKI5BRr0Zp7BaCmyukMMb8uiyUXkQbXOuUX478BzTudVfIu/i+MmOr0XM5FPfrCQRc4WfC1S70ejUjeU5AsIjlHmeT8t9/A3vTsWsL8FRu9G0TXnk5w/INFMP6bTpY51x3yPejaS9lkkSxQkCwiOcViFSQXgOIiw7ih5d5lkv2qeyUcfDG8/zR8vsTr0YjkNQXJIpJTFCAXjtqqSj5Yu5UvdzR5PRR/OfgiKOujbLJIJ0s5SDbGFBtj3jbGPBvntq7GmEeNMR8ZY94yxlRH3HZl8PoPjDHHZWbYIpKvnJpkKQQTqiuwFt7+TNlkV7qVwyHfhw+egzVvez0akbzlJpP8A+D9BLedD2yy1u4N/A74NYAxZgQwDTgAmALcZoxJY01QESkkWW0LJp4ZO7Sc4iKjkot0HHSBsyz2rP/n9UhE8lZKQbIxZjBwInB3gk1OAe4PXn4CmGyMMcHrH7HW7rTWfgp8BBzYsSGLSD5TgFw4uncp4YBBvb2dvOdXZb3h0Eth+UxYOc/r0YjkpVQzybcAPwECCW7fA1gJYK1tBr4E+kZeH7QqeJ0ErdvSyPqGnV4PQySnKE4uHDVVFSxauZmmlkR/XiShA2dA977wirLJIp0haZBsjDkJWGetXdCZAzHGzDDGzDfGzK+vr+/MU+WUHz++mKv++a7XwxDJGYqPC0ttVSWNTQGWrtni9VD8p2tPmPhD+PhlWDHH69GI5J1UMskTganGmDrgEeBoY8wDMdusBoYAGGNKgD7AhsjrgwYHr2vDWnuntbbWWlvbv39/V3fCz7Y0NrO1sdnrYYjkDC0mUlhqq0OLiqjkIi0TvgM9dlM2WaQTJA2SrbVXWmsHW2urcSbhvWytPTdms6eB84KXzwxuY4PXTwt2vxgG7APMzdjo84FVT1iRSOqTXFgG9C5jSGU3Td5LV5fuMOl/4NPZ8OlrXo9GJK+k3SfZGHOtMWZq8Nu/AH2NMR8BPwKuALDWLgUeA94DXgQusda2dGzI+cUCAUUEImFqAVd4JlRVMn/FJk3aTFftt6DnQHjlVyroF8kgV0GytfYVa+1JwctXWWufDl5utNaeZa3d21p7oLX2k4h9rrfW7mWtHW6tfSGzw/c/a/U7TSSSRR0uCk1NdQXrG3ayYsN2r4fiT6Xd4LAfw4o34NNXvR6NSN7Qinses8H/REQK1YTqSgDmr1DJRdrGfxN67+H0TdabTJGMUJDsMWWSRaKp3KLw7N2/J73LSjR5ryNKy+Dw/4WVb8HH//F6NCJ5QUFyDlBAIBJJUXKhKSoy1FRVKJPcUWPPhT5DlU0WyRAFyR6zFgUEIhGUSS5MtdWVfLSugU3bdnk9FP8q6QJHXAarFzgr8YlIhyhI9pi6W4hE08S9wlRb5fRLXqBscseMORsqqmHW9comi3SQgmSPWatpeyKRFCAXpjFDyiktNiq56KjiUjjicvh8MSx7zuvRiPiaguQcoKBAJJp+IgpPWWkxI/foo8l7mTDqq1C5l9M3ORDwejQivqUg2WOqvxSJ5pRbeD0K8cKE6kqWrPqSxiatOdUhxSVw5BWw9l14/2mvRyPiWwqSPaYleEWiOW8c9UNRiGqqKtjVEuDd1V96PRT/G3kG9Ns3mE3Wmw6RdChI9pjTJ1kBgUiItXrjWKhqgpP3VJecAUXFTja5fhks/bvXoxHxJQXJHlMHOJFo+nkoXP16dmXPfj2YX6cgOSNGnAb994dXblA2WSQNCpI9pqyZSAytQlnQaqoqWLBioz5hy4SiIjjqStiwHN55wuvRiPiOgmSPOZlk/TEQEQFn8t6m7U18XL/N66Hkh/1OhgGj4NUboKXZ69G4Z606dIhnFCTnACVMRFppMZHCVlMdrEtWK7jMKCqCo34KGz+BJY96PRp3Vs6DPx0E950Au/SmSbJPQbLX9NGySBQtsFPY9uzXg8oeXTR5L5OGHw+7j4VXfw0tTV6PJrmmRnjpKrjnWNi5BVa+BY+d54+xS15RkOwxLUstEk0/DYXNGENNVYUyyZlkDBz1M9i8AhY95PVo2rdqAfz5cHjj9zDuG3DJXDjpd/DRS/D0pcoqSVYpSPaYPlYWiWb16UrBq62qoG7Dduq37vR6KPljn6/AHrUw+yZo3uX1aNpq3gn/vgb+coxTWnHuUzD1D1DWG2qmw5E/hcUPOduIZImCZI9pdTGRaDb4nxSu2upKABao5CJzjHFqk79cCW//1evRRFsdzB6//jsYew5c/CbsPTl6myN+ArXfhjdugf/e7s04peAoSPaYVhcTaUtvHAvbyD1606WkSCUXmbbX0TDkYJj9G6fu12vNO+E/18LdX4HGLXDOE3DKrVDWp+22xsAJN8P+J8OLV6ilnWSFgmSPaVlqyWefbdjuuqTIeeMohaxrSTFjB5dr8l6mhbLJW9fAwvu9Hcuat+HOI+G138CYs+HiOU5JSHuKiuH0u6FqEvz9Qvj45awMVQqXgmSPWauJe5KfPqlv4PCbZrHws82u9tNPg4DTCu7d1V+yY5dWisuoYYc7QeZrv4GmHdk/f/MuePk6uGsy7NgEX38cTv0TdCtPbf/SMpj2IPTbFx79hhNsi3QSBckeU9ZM8tXmHU67pi93uJwkpIl7gjN5rzlgWbzK3ZssScIYZxW+hrUw/57snnvNIid7PPsmGP1VJ3u877Huj9OtHM59ErpVwoNnOT2gRTqBguRcoIBA8lC6ga4NrkMpha2myllURJP3OkH1JBh2hDNRLhuLdDTvgln/D+6eDNs3wNmPwGl3QLeK9I/Ze3f4xlMQaIG/nQYN6zI3XtFd3JIAACAASURBVJEgBcke08IJkr+cV7bbYFkt4ASgvHsX9tmtJ/M0ea9zHPVT2FYP8+7u3PN88Q7cdbSzkMnIM5zs8fDjM3PsfvvAOY87AfIDZziT/0QySEGyx7QEr+S7dF7e+okQcFrBLVixiUBAr4iMG3ow7DUZXr8Fdm7N/PFbmuCVG5zyioa1MO0hOP1O6F6Z2fMMroWv/hXWvQePnut0zBDJEAXJOUC//yUfhYJjty9vvXGUkNqqCrY2NvPhuk4I4sRZhW/HRph7Z2aP+8W7Tvb4lV/BAafBJW/Bfidm9hyR9vkKTL0VPn3V6XoRCHTeuaSgKEj2mPokS74Kvardt4DTz4M4JgQXFZlfp7rkTjG4BvadAm/8ITOlCi1N8OpNTvZ46+fwtQfgjLsznz2OZ+zZcMwvYOlT8K+fqmZLMkJBssfUJ1nyVfoT91RuIY4hld3o36urFhXpTEdeCY2b4a07Onacte85E/NmXQcjpsLFbzkLf2TTxB/AwZfAW7c7K/OJdFCJ1wModJqkJPkqlBF2XW6hnwkJMsZQW1WhRUU606CxsN9J8OatcOCM1PsVh7Q0OwHpKzc4K+V99a8w4pTOGWsyxsCx18G2dfDva6DHbjDuHG/GInlBmWSPKRaQfNVabpHGvoqSJai2upJVm3bwxZc5sIxyvjryCtj5Jcz5k7v91i2DvxwDL//SqTm+5C3vAuSQoiI45TbY8yh4+vvw4b+8HY/4moJkjzlZMwUEhWLjtl1c+dQ7NDbl/ypirS9r969v/URISG2wX/L8FSq56DQDRznB7X9vh+0pPM4tzU6P5T8fBps/gzPvha/eDz36df5YU1HSBb72N+d+PXYerJzn9YjEpxQke86qu0UB+fULy3h47mc8vXiN10PJSXrD2PmMMXXGmHeMMYuMMfPj3G6MMX8wxnxkjFlijBnvxThDRgzqTbfSYk3e62xHXAG7GuDNP7a/Xf0HcM+xTjnDvlOc2uORp2dliK507QXnPAG9BsJDZ0H9h16PSHxIQbLH1N2isARCdboFEAzaNBYTCW+b/w+P146y1o611tbGue14YJ/gvxnA7VkdWYzS4iLGDilXJrmzDRjhBLtv/Rm2rW97e6AF3vg93HGYswz0GX9x6o979s/+WFPVs7+zKl9RKTxwOmxRckLcUZDsMacnrNejkGwxxvlaEM95Gn2SFSPnhFOAv1rHf4FyY8zuXg5oQnUF763ZQsPOZi+Hkf+OuAKadzjBcKT1y+Ge4+Clq5yexJfMhVFntv5Cy2WVe8K5T8COzc6qfDv0iYSkTkGyx7QsdWExOH9UCuE5T2fini2gTLuHLDDTGLPAGDMjzu17ACsjvl8VvM4zNdWVBCws+myzl8PIf/33hVFnwdy7nKWeAy1O+cUdk5xA+fS7nd7HPXfzeqTu7D4Gpj3o3IeHvw5NO7wekfiEgmSPKZNcWAopk9yR5agL4OHx0iRr7XicsopLjDGHp3MQY8wMY8x8Y8z8+vr6zI4wxrih5RijyXtZccTl0LILXrwS7j0eZv4c9jra6Vwx+ix/ZI/j2fMIOP3P8NkcePI7zhsAkSQUJHtM3S0KSzhILoAwMFyTXAD31U+stauDX9cBfwcOjNlkNTAk4vvBwetij3OntbbWWlvbv3/n1qX2Litlv4G9WaB+yZ2v714wZhq8+wTUL4PT7oRpDzkT4Pxu5Bkw5QZY9iw89+PCyFZIh2gxkRygH9NCEiy3KIAnPXQf05m4VwiPjxeMMT2AImvt1uDlY4FrYzZ7GvieMeYR4CDgS2vt51keahu1VRU8tXAVzS0BSoqV3+lUx1wDvfeA2m9Db0/L0TPv4AuhYS28/lsn8D/yCq9HJDlMv2k8Zq1VJrmAtGaSC4e7iXvKPneyAcDrxpjFwFzgOWvti8aYC40xFwa3eR74BPgIuAu42JuhRqutrmDbrhaWfbHV66Hkv567wdE/y78AOWTyVTD2XHjlVzDvL16PRnKYMskesxRWwFTowtV8BfDGqHXiXur3VZnkzmWt/QQYE+f6OyIuW+CSbI4rFbXVlQDMr9vIyD36eDwa8TVj4OTfw7Z6eP5/nTcF+5/s9agkBymT7DWrgKCQFFImuSOfkBTC4yPu7FHejd37lDFfdcmSCcUlcNZ9sEcNPHE+1L3h9YgkBylI9pjT3UIhQaEwhVSTnM4+BfC4SPpqqyuZX7dJvzMlM7p0h68/BhVV8PDZsHap1yOSHKMg2WNOTbLXo5BsaW0BVwBPekdKJwrg4RH3aqsq+GJLI6s3q8+tZEj3Sjj3KejSw1lsZPNnXo9IcoiCZI+pJrmwhGqSC+E5T2cSnibuSXtqqysA1ApOMqt8CJz7JDRth7+dDts2eD0iyREKkj2mPsmFxZgCKrdIZzERTdyTduw3sDc9u5Ywr06LikiGDRgBZz/iZJIf+irs2ub1iCQHKEj2mFXOrCAV0nPuqk9yzFeRSMVFhnFDy5lfp0yydIKqQ+HMe2DNQnh8OrQ0eT0i8ZiCZI9ZdbcoKIVUk5zeYiL5/7hIx9RWVfLB2q18uUMBjHSC/U+CE38Ly2fC05fqD3SBU5DsMQsE9ENYMExrp+S8l05WOJ3eylJYaqsrsBbe/kzZZOkktd+CI38Kix+Cf1/j9WjEQwqSvWb10XIhac0kezuObFCfZOkMY4eUU1xkNHlPOtcRP3GW5X7jFvjv7V6PRjyiFfdygSKCgtHa3SL/n3StuCedoUfXEkbs3luT96RzGQMn3OysyvfiFdCjP4w60+tRSZYpk+wxq6l7BaWwMsnBr6526oyRSL6pra5g0crNNLUEvB6K5LOiYjj9bqiaCH+/ED6e5fWIJMuSBsnGmDJjzFxjzGJjzFJjzC/ibPM7Y8yi4L8PjTGbI25ribjt6UzfAb/TxL3CEm4B5/E4ssN9lKw3jJKK2qpKGpsCLF2zxeuhSL4rLYNpD0G/feHRc2HJYxDQm7NCkUomeSdwtLV2DDAWmGKMOThyA2vt/1hrx1prxwJ/BJ6KuHlH6DZr7dSMjTxPaOJeYQmXWxTQU+5qMREbebmAHiRxJbSoyHyVXEg2dCt3Fhvpuzc89V246yj4dLbXo5IsSBokW0dD8NvS4L/2/nqdDTycgbEVBGuVOysooXKLAnjW01pMpIP7S2EY0LuMIZXd1C9Zsqf37vDdWXDanbBtPdx/Mjz0Naj/wOuRSSdKqSbZGFNsjFkErANesta+lWC7KmAY8HLE1WXGmPnGmP8aY07t8IjzjEXBQCEJtYArhOe8deJex/YXiae2qpL5KzbpEwfJnqIiGPM1+P58OOYaWPEm3HYIPPND2LrW69FJJ0gpSLbWtgRLKQYDBxpjRibYdBrwhLW2JeK6KmttLfB14BZjzF7xdjTGzAgG0/Pr6+td3AV/0+/3wmIKp01yWhP3IgMeBT/SntrqCtY37OSzjdu9HooUmtJuMOl/4NJFMOE78Pbf4A/j4NUbtZx1nnHV3cJauxmYBUxJsMk0YkotrLWrg18/AV4BxiU49p3W2lprbW3//v3dDCsvKCDwr+ornuOap5emtG1rTXL+P9+hkpJ0lqUWSaa2qhKAeSq5EK/06Asn3AiXzIW9j4ZZ18Mfa2DhXyHQknx/yXmpdLfob4wpD17uBnwFWBZnu/2ACmBOxHUVxpiuwcv9gInAe5kZuv9FZ808HIh02H1v1qW0XSG2gEt3nwJ4iKQD9tmtJ73LSliwQpP3xGN994KvPQDf/hf0GQxPfx/uOAw++rfXI5MOSiWTvDswyxizBJiHU5P8rDHmWmNMZLeKacAjNjpFtj8w3xizGCcDfYO1VkFyUOQjpQ4XhSFck+zxOLKhdVlqF90t0BtHSU1RkaGmqkKZZMkdQw+G81+Cs+6Dpm3wwBnw11Phi3e8HpmkKemKe9baJcQpkbDWXhXz/TVxtnkTGNWB8eU1m+Cy5K9CyiSHuLqvUZnkAnqQJC211ZXM+uADNm3bRUWPLl4PR8T5JX/AaTD8RJj/F3j1105WeezX4aifQZ89vB6huKAV9zykcovCU1DLUgdf1OneU/1MSDK1VU6/5AUrlE2WHFPSBQ6+CC59Gw79PrzzuFOv/J9fQqMWwfELBck5ohCCJiGcSlYAGJ8eFnFjzJBySosN8xUkS67qVgHH/hK+Nx/2Pwleu9nphDH3Lmhp8np0koSCZA9p4YTC05pJzn/h17SLF7d+DsSNstJiRu7RR5P3JPdVVMEZdzsLkvTfD57/X6fH8rLn9IsvhylI9lD0ErzejUM8UABPeLgFXBr7QEE8RJIBtVUVLF71JTub1XJLfGCP8TD9WTj7EeeTxUe+DvedCKsXeD0yiUNBsoeiAoKCyC3mH7f9jkPbBwrg6Q4vJuKmT7Im7olLtdWV7GoO8O7qL70eikhqjIHhx8NFc+DE38L6D+Guo+GJb8OmOq9HJxEUJHtImWT/cxvshjZvKYAnvDVIdtMCru3+Iu2pCU7eUys48Z3iEphwvjO57/DLYNnzcOsE+NfPYIdez7lAQXKOUDzgT277WwfCmWQ948noEZJU9OvZlWH9ejBfQbL4VddecPTP4dKFMOqrMOdP8PuxztfmnV6PrqApSPZQdCZZIYEfuQ12Q5sHCqDewsZ8TWkf/RxIGmqrKliwYqNeP+JvvQfBqX+CC1+HPWrgXz91MsvvPqWP1jyiINlD0TXJ4kduf2+FYuMCiJFb+ySnW5OsPwqSotrqCjZtb+Lj+m1eD0Wk4waOhG88Bec+5WSZn/gW3H0MrJjj9cgKjoJkD0UFBAHvxiHpcxvHhd4YtRRAlJxOJjne/iLJ1FZXAqgVnOSXvSfDBbPhlNtgy2q4dwo8cg6s/8jrkRUMBckeil6WWiGBH6VbblEQWdI07qIms0o69uzXg8oeXTR5T/JPUTGMOwe+v9CpW/7kFbjtIPj7RbDkcdiyxusR5rUSrwdQyLQstf+5D5KDmeQCeMLDfZJddbeI6gEnkhJjDDVVFVqeWvJXl+5OB4zx58ErNzjLXC9+yLmtYhhUTYTqiVB1KJRXhVd3lY5RkOwhm+Cy+Ifb562wapK9HoEUktqqCl56by31W3fSv1dXr4cj0jl67gYn/RZOuAm+eAdWvAEr3oQPnoNFDzjb9B7sBMtVh0L1JOi7t4LmNClIzhEF8fF7HnJbS15I3S1CtJiIZENttdMvecGKTUwZOdDj0Yh0sqJiGDTW+XfIJRAIQP2yYND8hlOW8c5jzrY9dgsGzcFsc//9oUjVtqlQkOyhyICggGKmvKI+yYmlcw+1mIika+QefehSUsSCFRsVJEvhKSqCASOcfwd+1/kFuuFjWPG6k2muewPe+4ezbbcKGBrKNE+EAaOchU2kDT0qXlLWzPfSDXZbCqCbSXiSoovXdlSdfqYHJHmta0kxYwb30eQ9EXDKK/rt7fyrme5ct2mFEzCHAucPnnOu79ILhh7cWp6x+1go6eLZ0HOJgmQPaZKS/7mvSXY/mc2vWifuudkn4nIBPEaSWbXVldz92ifs2NVCty7FXg9HJLdUVDn/xp7tfL9lTTBoDtY1/+cXzvWl3WHwhNbyjD1qobTMu3F7SEGyh6xiZN9Le8W9AggAWzPJ7vcRSUdtVQW3v2JZvGozB+/Z1+vhiOS23oNg1JnOP4Bt6yOC5jfglV8BFoq7OIFyqDxj8IHQtaenQ8+WvAmSv9zRxOKVmzl83/5eDyVlqr/0P/cr7oVawHXCYHJMeDGRNO9rATxEkmE1Va2T9xQki7jUox+MmOr8A9ixCT57qzVofv138NrNUFTilGSEyjMGjYee/om93MibIPn7D7/N7A/rmf/zY+jX0x/tfyI/Ti6EzGI+cp1JTnM/X0rrPqp3uKSvvHsX9tmtJ/PqtPKeSId1q4DhU5x/ADsbYOVbrdnmt+6AN//g3NZzAAwcBQNGtn7tu7fvJwT6e/QRPl3fAMD2nS3gk08B1CfZ/1wvSx3qblFA7UzcTdxLbz+RkNrqCp5b8jmBgKWoSL1hRTKma09nqey9JzvfN+2A1Qvg88Xwxbuw9h345FUINDm3l5TBbvtHB84DR0JZH+/ug0t5EySXBnv+7fJR24DoJXgVEPiRapITS1ZuYa3l8fmrOGnM7nTvUhK1T9tvRFJTW1XJw3NXsnxdA8MH9vJ6OCL5q7SbU25RPan1uuZdsP5DWPuus9jJ2nfhg+fh7b+1blM+1Gk7N3BkawBdXpWTvZvzJ0gudh7c5oCPgmR9tOx7adck++dlmrZkj03dhu385MkldO9azEmjB6W0j0gyoUVF5tVtVJAskm0lXZzgd+BIGDPNuc5a2PpFa+AcCp4/fKF1Ra4uvWDAARGB82gnC92lu3f3hXwKkkucj9Wamn30V9ZHQ5X40s0kF8InB8nuY3PwnUJzxCzGqDeOnTMsyXNDK7vTv1dXFqzYxLkHV3k9HBExBnrv7vzb5yut1+/aDvXvBwPnd53AefGjsOvu4H5FULlXdOA8cCT02j1ry2znT5Bc7MNyi8jLigh8yX0m2fnaUgBPeGu5Rfz7GkhSelIAD5F0AmMMtVUVmrwnkuu6dIc9apx/IYEAbF4RzDoHA+fVC2Dp31u36VYZzFaPbq1z7je8UxZAybsguclHQXKkQqhRzUfuu1uElqXujNF0nuaWAC3W0rUk9QUaWrPm8W9vLT2JX3akiXuSrpqqCl549wu++LKRgX0KcxEEEV8qKoLKYc6//U9uvb7xS1i71Amcv1jiBM/z7obmxuB+pdB/OHz9MeizR8aGk0dBcrDcwkdBci4tJrJy43YAhlR6W//jN26D3fDEPZ9FyTf+6wMWfbaZxy48JOV9bMzXWK2rD0bsk+CyiBsTqisBmL9iY7jeXUR8rKyP05e56tDW61qaYePHrXXO696DHpnt15xHQXJw4p6PVmmInrjn7bgPu3EWAHU3nOjpOPzG7fMWbgHnswjwiy8b+WJLo6t9kj028Tp9KHssmTBiUG+6lRYzv26TgmSRfFVc4mSP+w9vXTUww3Kv30aafFmTnEOZZEmP2+ctXJPss0yyJf0ANlm5RSBRJjmts4k4fw/GDilnwYpNXg9FRHwsb4LkLj6sSdbEPf9Ld8U9vz3fAWs7sLx0uhP3fPYgSU6pra7gvc+3sG1ns9dDERGfypsg2Z81yblTbiHpCbXlTrUbTXiymt+eb5vO6oLRX2O11iSru4VkXm11JS0By6KVm70eioj4VB4FycFyi2Y/BckRl70bhnRAKNBLuWOjT1fcczLJ6XXySLSXTVJuIdIR44aWYwxqBSciacubILnEh0FyJAUHhSFch+u3mmTr/o1cstd0vHILTdyTTOldVsrwAb1UlywiacubILlLsNxip4+CZPWE9b9wJjnFeovWjg6dNaLOYXFfkxzePNFiIoEkE/d89hhJ7plQXcnCFZvCqzuKiLiRN0FyiR+7W0S1gPNwIJK2UICXarlFvAU0/CBg3b+RC9ckt3NMiM6qR01m1RtH6aDa6gq27Wph2RdbvR6KiPhQHgXJTpjip3ILt1mzloDlhheWsW6ru3610nlaM8mpbR96mv1Wk2zTmLgXuW/869v2jI6ezJre+URCaoOLiqjkQkTSkTdBcpHxYZAccTmVoGnlxu3c8erHvPbh+s4blLjivuODPxcTAet+dcEkmeDWmuRE+4t0zB7l3di9T5km74lIWvImSA7FHH4Kkt0Kt8zyeBzSKhT0mhQLLvxak+yMN91yi0R9kuNkkqP299mDJDmptrqS+XWb9HoSEdfyJ0gO/nn1VU2yy4+Wky2+INkXDnZd9kn2X3eLDiwm4qJPsl7akmm1VRV8saWR1Zt3eD0UEfGZ/AmSfZhJdjtJycYJKsRbbvsk+7YmmXRawCXrk+x8jX6/ED+rLJKumqoKQHXJIuJeHgXJfswkx7+cSLIaTsk+t7Fu6Lnz24p7Aev+zVnyPsnxJu6lvr9IKvYb2IueXUuYX6cgWUTcyaMg2fnqp0xyZK4slcxieAUzBQ85I1yTnGp3i3C5RWeNqHNY674hW2j7xOUW0V8j94n3nUg6SoqLGDe0XJP3RMS1/AmSg1/9FCS7XZY6FFj57aP6fNbaJ9ntxD1/PYfWpl9HnXTiXoLj+uwhkhxWW1XJB2u3sqWxyeuhiIiP5E+QHMok+6ncIvJySuUWqknONe77JPuzBZxNY2mP8F101SfZ/dhEkqmtrsBaWKi6ZBFxIX+C5FB3C59mklPJJfu1fVg+c/tUhD4N8NF7OSD42kuzT3Ki3eKWWyRoByfSEWOHlFNcZDR5T0RcyZ8g2Yc1yW6Xpc50FnLHrhYWrdyckWMVKvfdLfz5aUAgnZrkFCfuJQqMffYQSQ7r0bWEEbv3Vl2yiLiSR0Gyv7tbpJIdznR3ix89tohT//QGG7ftyswBC1DrxL3UwmS/drew6XS3CO+bqCY59DVBdwvlkiWDaqoqWLRyM00++hshIt7KnyA5+NVXmeQEHzMnkuma5Lc/c7LIO5tbMnK8QhQqn0g1kxx6ofpuMRHSeHMWfr0mujn0yUjkeVSfLJ1jQnUljU0B3luzxeuhiIhP5E+Q7MNyi0ipxAM2SdDhViibWZTqrDNpI/xUuF1xz2cBoNMCLs1McsJjOl/9NolR/Km22llURCUXIpKq/AmSg3+Kd/ooSHabNcv0stShbKaC5PQVzIp7NvOZ3UC8N31Rn65k9nxS2Ab0LmNIZTdN3hORlOVPkOzHFnBJ6i/nfLyBv82pa7N9prKQbtuXSVvua5Kd7Vt8lkpOo7lF+PWabDGRyMciOl7212Mkua+2qpJ5dZt8N3FWRLyRNEg2xpQZY+YaYxYbY5YaY34RZ5vpxph6Y8yi4L/vRNx2njFmefDfeZm+AyGhv7N+LbeIFw+cfdd/+b9/Lg1/H28Z344IJAliJLnwYiIpr7gX/dUvAtZ2oAVcool77fdJ9ttjJLmvpqqC9Q07+Wzjdq+HIiI+UJLCNjuBo621DcaYUuB1Y8wL1tr/xmz3qLX2e5FXGGMqgauBWpw/sQuMMU9bazvh8y7nL6qfZi67726R2Yl7oXILZezS5/YNS+i582V3C7c1yUneECSbuCeSaROqKwGYV7eJqr49PB6NiOS6pJlk62gIflsa/JfqX7LjgJestRuDgfFLwJS0RppE6A9xs48+xo6qSXaxmEim4qtwgOefhyznhB7C/K9Jtq7LfJJt3vpJhjpaSHbss1tPepeVsGCFJu+JSHIp1SQbY4qNMYuAdThB71txNjvDGLPEGPOEMWZI8Lo9gJUR26wKXhfvHDOMMfONMfPr6+td3AVH6I+rn2o93X60nOnOCC0ZPl4haq3rdleTnE7fYS9Z0uiTnGTzZOVDoat3Nrfw8NzPfPV4SW4qKjLUVFUwv06T90QkuZSCZGtti7V2LDAYONAYMzJmk2eAamvtaJxs8f1uB2KtvdNaW2utre3fv7/b3cOZ2JaA9c0f0wST+hNvn/HuFqFz++PxykWuM8kuS2xyRRolyUlXF4y7LHWc/d/4aD1XPvUO732u/raZEEx6vG2MeTbObQnnl+SL2upKlq9rYPN2LaIkIu1z1d3CWrsZmEVMyYS1doO1dmfw27uBmuDl1cCQiE0HB6/LuMg/tH4puYj+mDn5mDNek6xMcoe57RAS+Vj76VOPgLVpl0Ik2s3GySTHK71oavFnR5Ac9gPg/XZuf9RaOzb47+5sDSpbaqucfslqBSciyaTS3aK/MaY8eLkb8BVgWcw2u0d8O5XWX8D/Ao41xlQYYyqAY4PXZZz1YfARlTVLYciZbgEXLrfwyePVWV75YB1bGpvS2tdt4BgZBPqpLrm1Ht7FmJPU0IcnjibMJEfvX+Av04wwxgwGTsRJZhSkMUPKKS02zFPJhYgkkUomeXdgljFmCTAPpyb5WWPMtcaYqcFtLg22h1sMXApMB7DWbgR+GdxvHnBt8LqMiywZ8E8mOeJyCh9mhzPJGSqP8GMdd6at29LI9Hvn8f2H3k5r/9ZAN3Eq+f3Pt7BuSyMQW27hn8c9NmB1s08icRfHibOTzfAnKAXuFuAnQHttgOLNL2mjo/NIvFJWWswBg/po8p6IJJW0BZy1dgkwLs71V0VcvhK4MsH+9wD3dGCMqYnMJLf45Y+pu1n9nZVR88ubis7Q2OTECp+sb0iyZXzhmuR2yi2O//1rdC0p4oPrjo96g+Onhz0cqKa1T6Ka5LZlFNGrUEaXA/no4cpJxpiTgHXW2gXGmCMTbPYM8LC1dqcx5gKc+SVHx9vQWnsncCdAbW2tr56eCdUV3D9nBTubW+haUuz1cEQkR+XPinsRl5sD/umVHOKuu0Vm/x4VciY5FNym+5Cmuix1aLl0v9Ykp1NukbxPcpLbw1/blmVIWiYCU40xdcAjwNHGmAciN2hnfkleqamqZFdzgHdXf+n1UEQkh+VPkBzxF9QvmdEknzK30Vkr5PkpWOssmZ6Ulvg87iZr5opAOpnkmK+Jjplsxb3Wl6d/Hq9cZK290lo72FpbDUwDXrbWnhu5TTvzS/JKbbUzeU+t4ESkPXkTJPuyu0Xk5RQCpvDHzxm+f36qjc20VLtSJOK2u4UfJ5hCmjXJSbaNV5McvU90LXIBv0w7VSrzS/JNv55dGdavhybviUi7UlmW2hci/376pSbZ/bLUqW/rhl/eVHSmdLO6oYfOpNgpOfIsTT55nULy+uL2941/fbwWhPE21cKQmWetfQV4JXg5pfkl+aa2qoJ/v78Wa23KiwGJSGHJm0xydLmFP2qSowMz285toS0y290ixE8ZzUwL/XFM9xGwLjPJkVnTxqaWNM+afeksiW6TFFzE61oRr09y6DiF3qpQMqu2uoJN25v4uH6b10MRkRyVP0FyxGW/BH3R5RbRt+1qaRvou6lJ3rGrhVc/TK0tk18er84Qim1TDf5e/bCew2+cxc7mFlf7WprMDwAAIABJREFUhVgLXUucH7vGZh8FyaGvaZRbJM4kR3+NPE/k5daVIUUyp6aqEkCt4EQkobwJkiP/gvrlY+z2Ju7tam4bJMdboSyR59/5nPPumUv91p1Jt/VL5r0zpZqdv/aZpXy2cTsrN24HUu9uEXmebl2cllOh9nN+0JFyi0SSdWuJXYZdNcmSSXv170FF91JN3hORhPImSI784+2XzGh0T9jo2+IFyW5awIWylPEy0m2O659YLePcvlJKi50fmV3N0fW0qdY0BgLQvdQJknfs8k8mOZ3OKskm3IWOGdUnOaq7RXRHjUyXGUlhM8ZQU1XJfC1PLSIJ5E2QHBno+SYzGjVxLzoAiFtuEQhtm/zQrUFN8o1bPE7RvbPqS6bfO5emFAL6TIu3NHJ7ugRLJUJjTfb4xrs9nEn2VblFdMCa2j7R+7a5PW4QHacWXzP3pJNMqK7g0/XbWN+Q/BM3ESk8eRMk+zOTHP8yJCi3CH1N4e65aZvV4vGbiv99fDGvfFDPx/XprXqXCam+ZEKZ5NYguf3tY1+LAWvp3sVpKrPThxP33LQLTF6T3PaTkXglSIqRpbOoX7KItCd/guSIv6B+aWkW76PlkJ0RQXJsLXJqPZWJ2qc9HiRwo3j5MbrbJHppsVNWsSu8gp5zgKIEP0mxWXproVupH2uSo7+62jfB9XH7JMc5Z7ztRDJh5B596FJSpMl7IhJX/gTJEZebszxx78O1W9m2s9n1fu0Fh7uiguTQ19Rrkt1MdorMJHu5ClyqvYYzqfWxTO1+h2uSW6KXmU409tgkfcC2Ttzb4atMsvt0brI3P3H7JEdlkmPfHKZ+7vbMq9vIA/9dkZmDia91LSlmzOA+WlREROLKnyA5KpOcvQxdS8By7O9mc/GDCzt0nNgAICqTHPzqZjERN9m35gQTp7LFywShmxIWgC7BIDmUBU624l6bTDLQPdzdwj9BcrjG3UWUnCz7HPvmr+0GUV8y9nnDkwtW8fv/LM/Q0cTvDhxWyburv2TdlkavhyIiOSZvgmQ8qknesqMJgIWfuc9ERK+4FzNxLyJIjq3dTCXwje0M0J6o7gIpbJ9poXN6seiVm8cJWifu7Uxx0l3s6o/W+rQFXHiJaDf7RO8bKzRpMrpPctvXYrxFRzoiYK3ayUnYWTVDaLGWv+nTBRGJkTdBcsBCUTDIymZN8pfBILl3WanrfePVX4ZEBmGxGblU/sC7qV+ODJK9rPsMxciPzP2Ml5etzco53XQBgdZyi1D7tlCglyi+j1eTXFbqx3KL4Nc09km0U9ya5DjbZnrinnNeRcniqO7Xg2P2H8AD/13hq093RKTz5U2QbK2lJBjAZDOTHAqSe5WVuN43agnemNviZZLdZNTclGYk6lObLbH35y+vf8pj81Zl6+wR/0+uNFxuEQySQzXJCdLg8bpbFBtD15IiX3W3SG/yXIo1yQk+yWgz+TRDr82AtSl3M5HCcP6kYWza3sRTC1d7PRQRySH5EyQDpcFUcjb77XYoSI68nEqfZFc1yW4m7uVIJtm0jiFbHS/cdm3oUuIMckdsTXKC7WMfT4vziUdZabHPslZplFskiW1bJ6K2vc7ZL/r2TL0mrPV2gqrknoOGVXLAoN785fVPot60iUhhy58g2UJpiZeZZPflFpF/81PJJLurSY7etz1et8yLPbu1qfct7ii35RYlRdGZ5GSTytpkkgMWYwxlpUUdrkm21mbtD7qNCVS372pm+r1z+WzD9hT2TVCTnOQ12qarS4be+yqTLLGMMXznsGF8XL+NV5fXez0cEckR+RMk0xrAeFGTnF4mOXGUvDNuC7jor+0eO412cakeO+PC53Tysc7EqiwFfy7LLUKPVThITlICExskW5yMeVlpcYdrkh+dt5LDbpzVoWOkKrbkYeXGHbzyQT2LV21OuE+yTHK8FnBR+6d4HLcCyiRLHCeOGsRuvbpyz+ufej0UEckR+RMkW0uX4EIP/qlJbr2c6e4WrRnS5OPIvXKL7AXrbs8TGySHy1qSbB95PoOhWwbKLVZv3sHqzTuyEvDFZsxTeXORvE9ycLtkE/fCpR6ZuZ/qbiHxdCkp4rxDq3lt+XqWfbHF6+GISA7IoyCZ8MS9bGaStzQ6QXKX4mLX+0YvnBBtV5w+yel1t0i+ba60gAtxPg7PzkjcTgoLLVSzI2biXqLhtskkW0uRga6lxTTGWXrcjUwvstGe2PKdUOlDe4FrstdrvE87olrAxWSaM3U3rfVyjUfJZeccNJSy0iJlk0UEyKcgGUtJMJPcnMWJe6E+yekEdZF7xO4eOXHPBi+mlUlOIRzImUxy8Gs2a5LdfpQfaukWqieOrdWNFft4Bmyw3KKkqMOZ5FSCx4adzSnVDSc/V3RAnkrf5Njsc6JjJl5xL/65OyoQ0BLXEl959y6cWTOYf7y9hvqtO70ejoh4LH+CZNu6GpoX5RbpiDeTPyR6xb3oYCKVuxevc0Ai3q+4FxtIZi+TnGgMiYQmyu1oU5Mcf/vY92sWS5ExdOvS8XKLVCZnfu3Pczj8plkdOo9zsqgvKZ07duJdrKR9kttkolVuIZ3vWxOHsasloKXLRSS/guRwJtmDIDnTmeRAVHY3tH3qtZnJss6Rx4/uk5z96CFeuUW2JKspjhV6bbWpSU6x3CJgAQNlJZkIkpNnWJeuyUxtZdte3UR9jTu+JHXL8R676Bg5+pyZ+rEOWGWSJbG9+vdk8n67aXEREcmjIBkbXujBi0xyR//oxsumxt6WSvaudf/QvvFvj1wJzuvFRMLnDn7NZhDjps4b4k3cS2371hNCUYZawLkpv+moRBnkjpw7tGuyn1e3z1Hy86omWdp3/qRhbNi2i38u0uIiIoUsb4LkgIXSUAu4Fi+CZPf7tjdxL/L78MfSgdQzask+6k5Uh+xFhi02CHL6/2bp3OGvqd3vlnAmOaYmOcHjFvupRsBaDJlZTCT2dbBk1WbWbW3s0DETiRPrR32Nv1P7G8ULtON1usj0YiIBa72ZoSq+cchefdl/99785fVP1S5QpIDlTZCMhaIiZ1JUc7YiLCKDpXR+kSbO4Mab8e9mln+yj6gjg7eomuQUjt15Wsec7e4WqZ4uVGO8I8UWcPH6JDuZ5I73SY4NMqfe+gbH/m52h46Z6rliyy/iSfYGJN6nHVHlFjHBccYm7qncQpIwxnD+pGF8uLaB15av93o4IuKRvAmSLRaDoaTIZLUmOZzdTSMujwoOEqXqoM1H3G5qkhNmkiOy7YEEWeVsiQ2CAln8ONx1d4vgE912MZH427ftbmHDi4ns7PCKe6Fjtl63eXv6E0nbPVfMOVOqSY55bN74aH3UZKi4JRtxLma6BVw2X1/iXyeP2Z1+PbvyF7WDEylY+RMkWyeLXFJUlNWa5JZ4f+hTZBNcjj1ebCDnpiY5cSY5EHG5nbqPLArfv0D2Vtxze4dD7y1CnyAkKwVo2yfZyVKVFJmouvB0JFvtL6NiXoOpTKaLvemcu9/i5/94t/X2eOUWcfokZ/p+WmWSJQVdS4o575AqXv2wnuVrt3o9HBHxQP4EyYSCZJPVmuRQEJRWsUWcbHG872OzwqlkrVvLAJIHb9H1ycmPnWnxspPZGkf4PCmeLxCuSQ5mkpOUAkRm6UPPn8F5rXY0UHPTErCj2nS3CF7fXk42WbY5EO7/3XafeMfJFLWAk1Sdc3AVXUuKuOcNZZNFClH+BMnW6T9bXGzCH4ln57zO1/QyyYnLHOIld910FEgWoDQnCJKTfRBdt34bm7fvSnp+N2JrT7PZJ9nt8xd6rHY2x6y4l2j7OJ8IFBmDMabDgZrb1QI7Iva9RCqTSJMF0slKgmJf94/MXcnhN85KccSJuSlbksJW2aMLp48fzJMLV7OhQYuLiBSavAmSQ3+sS4oMTdkstwhlktM4Zbv1nJEBdExAksq5kgXUUZnkOIFcItPvncsf/vNR8gGkIbLGNlvxi9tAKfS4NbU4JSHJFxNp+0bIGCgy6Z0/UmwQ2ZnarrgXGkTqb9jaXB8+dtvr4p3rs43b+WxjJlYPbH9cIpHOn1TNruYAD771mddDEZEsy5sg2Sm3MBQXmahJaZ0tYzXJ7ZRbxF6XyrSjZF0bojLJLW0DuUQadjazpbGTJodFZHWzleVzOykstr90sg8touttHUXGySZHnj8dsaUPqWyb/rmijxNvSWm35wzdnrhPd+y5MpQB7sCnP1J49t6tF0cO789f56wIf4IkIoUhb4JkrNN/tqSoyJvuFmllkhOXOUTe1qYmOYVzJSsjiCxJcZNJDlhojl1rOUMi63uz9RS6WcUQogO65oCNeN4SPc6tl1szyQYTc106Wmt608/mpsLGCfRbl4xOfOD2H5nImur4PwexJUNu2/Ul4naVRZHzJw1jfcNOnl60xuuhiEgW5U2Q7GSSnaWps1mTnKlVz2J3jzeZyc25kq64F4i87CZItp1WzhJ5P7OW5XOZSY4cV1NLIIXHue1jawwUBestOnI3I4PGQJLnpCOPpo37Woz+2t5JE07cixP0/n/23jzejqO6Fl7Vfc69V7ItyYM84HnAgDFgg/GADWFyAiExCYRA+GwMmCFfSML3SMiD5D14EJKQkADJBwmTAQdihzkhgAFjTIzxIAvbeMajLHmQLVuzdIdzuuv90b2rdlXt6j7n3itd6aiWf/6doburqvv0Va9atfbe8goKKc5Os7PGzqxUmDAaOOu4A/CUg/ZJxUUSEvYwjA5J1lXWgHwn5knWWjOSNAu7BTvEH7JUBW+YbAZ2KV7eOZYCrs3KUZZ63u0sUtntnfUcGtafyjOn9AvdqkpKqfwUFGq3xdyUZHbvtbUzp/LRznvtvTYd16wly0py2K+/31xJSvIkJwwLKi5y59otuPreJxZ6OAkJCTsJo0OSUWW32Jkp4NpSVw2DwG4hvJ9NMZGYqO4ElA2RAk7rHVfRkFcW3GnZLYbUJR0luSwDr64PKXMI9yTPyQbBiGpbzuV5s1t4E7VGu0ULGZXyJDcdP5d0i1K/iSQnDINzTnoSDth7LBUXSUjYgzAyJLksq2XsfCd6kqXMBcNA8l/azzp430Y6ONoC0qIp4AZQJHvzrSSbvnc+gRm2H8eTXLDsFpH9eUYLOlQpzIsneRjlfS415vifU5AvuaHZQaw7fvv8IJ4SUHp9x8U34Fs3Ptg6/rBft/2EhEEw0c1x7ulH4sd3PoZ7Htu60MNJSEjYCRgZklxTEXR3oifZtUQMf/ygBMNXKwfzJDfv6weg2ePa2t2RSrK8BN+GstS4ZpZLoMOSVJ8kty3d06pGppT5/TKlrJI85Hg5OMlsqzI5H4o1b8f3yTcdF51AkMc4Zrfw73sv3eJ3b34E/+Mrv2g/Ab/fpCQnzBLnnn4kxjoZvpCKiyQk7BEYHZKsda0k7zxPsus3nYWS7BBh93iJgA/nSW4eVz+a9q1dSZ5vOwsf62wyD1x73xP4vc9eizvXbh6+7yH3L+r7DKjsFm0WGLJBKLi/2872JM/NbiH1PfjvFLs2kpIsBu6Z/QfobADMZiKWkAAAB+w9jt8+6VB844YHsWHb/BZVSkhI2PUwMiQZQJ0CTrWqavOFuZZz5of4xzsE2lt2HoSQD1PkYjglWc/7JIQrjrPJPLB1ug8A2D4zixyms7BbjHeqP5vKbhFvZs367fjl2i0Aag9yvZOjJM9BlOe/8TD5mofvJ3w/kJLccG349pjdx5Lj6t1ccpK7/Q4/EUtIILz5rKMx1Stx8YpUXCQhYdQxMiRZa1gleWcF7gk5cIeBFBBl2wv7GUYBa1OduWWC21PafaQ7ME+yHs53bcfUPHnYPtPHn3/rFrEIyrC/W1lqjHdyAFUKOGNFEJp5/t9dYYN8lOtPng8lmV+rViV51r14dotgwjb7PqWVGNdu4fYxX351M/adlykyYYTwlIP3wfOffAAuunoVZvrpJkpIGGWMDkk22S2yHeaZ9TFnT7LzXnvbQmICQ1AGH9tAZamHCEDcIYF7pjk9KyWZlO3Ydbno6gdw8XWr8dkr72voe/C+JrqZeV8OmLs3s0LyDvAk6wGyW8zd1lG1U78O0G7bhGfQvx/axrNbzMf5pMC9hNnigrOOxmNbpvGdm1NxkYSEUcbIkORS82IiO/bhd8PqDfjSNau8SnVzY8n+4dIStxToFG26ZanbsViwOUVzMGFlL5jL9b30lkdwxS8fi7Q/uxy2NJ7YuKZ6lQ1DkXzLMLSSrF0leVALjIJylORsXjzJljS2FROZj/LXTp8DqbrNtgY5awbv1+2fe7Dncg8OUlI7IaEJv3L8cjz5wL3xuZ+m4iIJCaOMkSHJWmso1HmSd/DT71X/fDX+93/e5hCT2XFkrha7kIpQDBOV30be+Ni58t6krlFTvTko9Z+68j583sszynXy2SjJRdl8TK+2h3SzkCQP+7NxT7JTca/lOKXs9VNKGcI+H4F7VYXClp3nQpKF9/aebDiOkVrpPhTJd0M7dl+If+Nr1m/H534arhY0jSshYTZQSuHNZx2N2x/ZjGvvW7/Qw0lISNhBGB2SDAC1J7lNZdo63cemydCfOiwK4SE/DJxDvOP5x7DiXntfbYSaj527J5qapjbn4vkuyjL4ffiyvC7d7wYBEabYMbS92wlv92F/tqLUGO/awD2jlra0w1PAKVhP8mx42oZtM3jvN28xCrnWAxQTmQNL1sJKwyCWBb5FsuiINg4ncC8y+YmsZlx66yP40HfvwBbBe+72m5TkhLnjt08+FPvtlYqLJCSMMkaGJENTdovMKIcxnPKhy/CsD/xwzl1Ky8XDgB/RVJbaKq2DP9zbCDVvY9DAPTpmLoF7RRm3RfBUZsPmSW46hoJrurlEkof73YpSY6K2W/TLciCyCFT3Ju3heJJnQdR+/sAGXLJiNe54ZEvdht6xdgtBSx7mXgTk3NpS6XWn34hKryFnWCm8ANcYrGqdWHLC7DHRzXHuaUfg8jsfxf2Pb1vo4SQkJOwAjAxJ1qiWwLJMtT4kp3rzE9g3TDlnCY7/sqEsta8KD0IgfR9nbDvQlDPZhVGS5+IHLSU/qV1qn02hh7bAPSJoY/k82C20xkSXPMl64PEqL7vFXDzJJiVafV6DlPGei7VA9MdrYWOkT62BXr9ZSZa86LH7PVY8xZZib5kwzGK1IiFBwrlnHIluloqLJCSMKkaHJGuNTAG5mh3xmA2GKefchjBwL2x7GALZ5ruU7BxAM2mk3eZEknU8E4PWw1lKeJtNxxBBk5Xkgbup+ijh5Eke3JOsTF+Zwpw8yaX3O2i0e5LnoiRLWSjaJmHVuOhVY0ZYfZA8yeLxwt+GpExTe4Nm+kgkOWGuOHCfCZxz0pPwtZUPYuP2VFwkIWHUMDIkudTVknamdmIxEYE8DIOmwL0m9W4QQt5GqHnfg6aAo21tdpYmFDpUku2yuh6IfPkgJTymINJ4OwJJHrostWae5LK0v0VLMxlXkqFAmvZsiJohg6SgN1hYzDFz8SQLffPfLD5O+yrdMw75ZoQ/7Mu7XyCfr7VbtCjJs5iIJSTE8OYzj8Zkr8AlK9Ys9FASEhLmGa0kWSk1oZRaoZT6hVLqNqXUB4R93qWUul0pdbNS6nKl1JFsW6GUuqn+/9vzfQIEDW3sFvzZd//j23DUe76LH93+6Pz32aKEtR8vv/fb8x/q8+JJZpzFVcSb2qw2ziVwT7JbaPZmmImAP67YdZFUzKDvAUCpxyY61m4xCFm0x1evld1i9p5kOoQrya3Xaw580LUF0evgSjLg3jPSRKgp9Z/k15fuQWu3iI+J75cocsJ84IQnLcGZx+2Pi65eNScBISEhYdfDIEryNIAXa62fBeAkAC9TSp3u7XMjgFO01s8E8HUAf8e2TWqtT6r/P2deRi1AGyXZJYY3rt4AADsk6XsxYH7hGCSFTmrPVyuHym4xQN/9MiQwYpv1+c5FqZeUZD6m2RAY60luVpIlpXk4Ml692uwWPE9y+7GWJCtkGX0//LW0k5XS9N1mMZivPMlmVWMAXy9XnflERZrASb+79l75Buke8m1JMcxmIpaQ0IQLzjoaazdP4Xu3PLLQQ0lISJhHtJJkXWFr/bFb/6+9fa7QWm+vP14L4LB5HeUA0BqACu0W9DYTCknMFXP2JDd4gSWVbRglORb0ZLeHS93SONwxVVvnkidZsgZwMjWXPMmx34BUR4lIDvOzUT+mmEipHXK3YdsM3nHxDdHy16S+ZkxJnosnmauvbT/JvNktvAIhAxW2gXYUNskTLE42IvewhuyLp9+31XqSPMkJ84wXHn8gjlm+Fy68KhUXSUgYJQzkSVZK5UqpmwA8BuAyrfV1DbtfAOBS9nlCKbVSKXWtUuq35jDW9nEizG5hctPuAJI8r2Wpg39YOXF1ieMwnuR4dgv73q2+V71f9fg2nPKhH+HhjZNBmzqi5A2CslFJ1kypHJ4kt9ktRPVxCPJI528D90onQPJTV96L7978CL587QPhwdqOj9+Ks8uK4h5UFRNpI4bD98Pbtw253w3SrO9JlpRkybZi3nudxO6/Qb3Gs8mgkpDQhCxTePOZR+PmBzfh+lUbFno4CQkJ84SBSLLWutBan4RKIT5VKXWitJ9S6lwApwD4CPv6SK31KQBeD+DjSqljI8e+rSbTK9etWzfUSQDVgy+rU2tJy8NCsbU5oy3PaxuaPcnhtkEzKTjjiSnJkMdO7y65fjUe3zqN/7jpIXFMs/XeFWWY3cKqkrMjMAPbLeaoJFM/43UKuD7zJAMw4Xiyr9b6hnme5Nk4Y/32NdonLXMJUnMVX9Ypmicz2rlf4pO+2PhiqyGllrNbDLrSkgL3EnYEXv3sw7BscRcXXtVe9TEhIWH3wFDZLbTWGwFcAeBl/jal1EsB/AWAc7TW0+yYh+rX+wD8BMDJkbY/o7U+RWt9yvLly4cZVn28DYiSKuHtSLtFN7fq9fptM/jaysGinLVATgmiyjaEFcFfko9tB+TAPbIUTPfCZXL/mGEQC7oCyJNs3w/cZtm8zE4ErUl9HATWblGXpS6ZJxmuQuyDnxvA7RaD90+QSOOOVJK18969B5ua5dYM125RvXKe2xy4F06qZE9y9Tqw3aJxr4SE4bBoLMf/c9oR+OHtj+KBJ1JxkYSEUcAg2S2WK6WW1e8XATgbwJ3ePicD+DQqgvwY+35fpdR4/f4AAGcCuH3+hm+hoSu7hVJikY8dwJHNw7uTZebB+45/uwHv/vrNA/0j6ZAPYUnZ72coK4JHrIPN7GspcI+IoBRwBcw+w0VRhoSOTwJm40luK0vda7JbDNFP6ZHkvldMpOkWq/bjSjL/fjj4p6EHIMlzQSlMouibpn7NJi2T5Jgv3uznt2M+24p7fIWozZvu95+U5IT5xhvOOAqdTOELP1u10ENJSEiYBwyiJB8C4Aql1M0ArkflSf6OUuqDSinKVvERAHsD+JqX6u1pAFYqpX6BSoH+sNZ6x5BkpiRLBLPNk7xuyzTe+81bMN0vBu6TFMpOpkw/j26ecra1jdkfp/TZJySDPNtbPcmRPMn0joggV5L5mGYbvFeUYUlhruwNo5b744rbLeLbh6FJNO6xDs9uEe4nkbTKSlK9V8pO2mZzGaVMKO0Wg3CHlavW4yX/8BNMzgx+zw9zL3KfsVTVsdTV347bXkjIw0mBvWf5CpGpRJg8yQkLhIOWTOA3n/kkfHXlGmyaDAN4ExISdi902nbQWt8MwSKhtX4fe//SyLFXA3jGXAY4KGi5O1PuQ5LetXmS3/eft+LSW9fihU9Zjl97+sED9UkP6pzZLai/QZTrtme0Uq5PdxgvZRtxjNkt/OA0Pmngbc1WSS51gxKu+TkO3qYpJtKqJAtdzoKM55lCN1folW7YH/3mUpNac3+8MpO22WSdkPzrs7Fb/OV3bse967bhzrWbcfIR+w50bFBMpElJZufGVyQ40e7kCv2SVS7U0vHhpIB+c06SJRuHhJQCLmFH4s1nHY1v3vgQvnL9arztBWIITkJCwm6Ckam4VynJCjlTdavvw4ephK3TfQDARB2UNQhMJTdmtxjGA23HFj6wS62RG9Zlv2MfG9FKMiP+Yvqa1NKZfonP/fQ+vOZTV3sWjdkpyaUWlGTzymwDQ/CXdiV5fgL3zKRIKXSyrM5uMZg6zX3DVBly2P55W05/A9gtpO2ksEvlumPH+uS4qVe7r5/dgivJmfOdFo4PlGTYDClKsFsMnN2ica+EhNnhxEOX4vRj9sMXf7bK5DJPSEjYPTFCJFlDoSLKTo5hwbsoYXu95Lx4bHCS3DckQzmEAGj2p/rIlAoe2FpXaYWAZitGDG22BUdJ5t5QoyTXgXv9Eveu24Z7122bFyW5KBtSwGmrAg7nSY4XCwGAXr8pBdzgMCQ5U+jkCr1CJqeiksy+V8qWpZ5VVhTvc6nb1VOpF34+gx7r50ke6F5E3JNMfTcWsQnuf/ubi3aL1sC9wceekDAbXHDWMXh40xQuvXXtQg8lISFhDhgdkgxrt5AD95qJAJFkKjk8CEgR7eSKqVOhwtUG30cNuEqy/1AfxIrQZluQvJ+AJT+dvOp7pl8apdIJ3JuDkhyQGDbBmEsxkXie5Dh5mk0/ld0iQ78sRXIqWSgcJVmBVdwbuPvomB0FPgJpM/2GnTaSLPmEW4Ilq3HZ43tiWWrNPMnhOO3x4XgKYfKb8iQn7Cp4yVMPxFH7L8bnUnGRhITdGqNDknWV3cK3W3Bi0oTJmcpuMYxHlB7UnSxrTGEFVHYOX+m0yqK0hG4VPkMcTSng9jG2EYE28kzHTfeLiuB5CvAggYkSGpVk8AnBcG1Wx8gHka96znYL5knuZKrKkyzsR21y8sn7cTzJs3iAioFsrSQ53E6TvLYJpJSzWwvbwk7ti1xMJLzHnSsamTBpNnbXkzzYBGs2E7GEhGGQZQpvPuto/GLNRtywOhUXSUjYXTF4X9o5AAAgAElEQVQ6JBmUNaCyW/ikoM0jTEryUAFjTInzSSkngttn+jjx/T/A337fyZzHyhRH7BauJXkoJbltSTnWhH/cTFGiKBFcU35+26b7g6W8q9XoGKHz8/0OSiBtCrgISe4NZrdo669kxKybZ+gVWjyGvvFtDJIneVbppoUJlZhCjV9LoRmyzLRf57Ad2138WE54yfLC+3OUZDMBlHr12tVo8SRHh+RsTxw5YUfid55zGJYu6uLCq+5f6KEkJCTMEqNDkjWRD/sZ4IF0zcdPzsTVxhhMCrg8CwKa+IN682SlUvPqdXyMmQof2KXg1xzGS9m+/B4jqu7YYnYLrgye//kV+JWP/KR1THwCIRE4brfg+7ehbCBHRamby1I7qw7N/RC57xhPcilf5/q7MS8gju6XLLOe9flQkv3fRtpP6mZYUlm14xLrJteNZvdSrwivs9ZVZhg6ByAWuOdPClieZPaHTe02eZJjNqOEhPnG4rEOfu/UI/D9W9dizfrtCz2chISEWWCESLKuslsoz6JgiGgzS95GdouhPKoVQ+g6nmQ7HrMfLdN7Y6A9qrHpYFteG1ftErdLlpvge0dj28PvrcoHVIF7hSHJ9iCeoWLlA4MtJ7oBgmLvbtDlkEqytD9PYddUpS223enHkNzablFG8iTXr+TrJsz06X7J5r3injR2KbUfhwl4bJ1Q8b7c75rsSZq98rzavGqfn91CujF9Iq5h//ZEu0XDRXUIPzT+8ju34we3peCqhB2D8593JDKl8MWrVy30UBISEmaBESLJFBBVPTQLj+y1pZuYzRIskbM8U4HqJWbY8ORsM7SokuyOTVL0YhimmIh7nPs606+IoJ+LtzeL1EactPHAP66UD6Ps2v100D6BF0ORbB5uf4Nd0zxit/AP972+PGWgXfGYu5KsI+20qfJ0/7Zmg/DyW/Dvmg7VjPj2+uFYpMA96Xj/XtWa50mu/P7v/ebN2DzVax2Tf02+ccOD+O+71sUPSEiYAw5ZugiveOYh+Mr1a7BlKhUXSUjY3TA6JBkwZakBd6kXGCxvMdBOzDgZ6QsV97ilwOxX2v3cMVuSLQbueQUnyiEIpBgI5WyPHRkqyaWuyGCb6joocQdcdZArjr7SNwistzbcNsWLoYjZLVh/Ld0F2S2KUjwmZhOwSrIN3JuNkhxW3NPiBKBNSS4arhuH81t5k6hBOb6UAk4O3GN9we7njEfzPMkKn/vpfbhkxRr86I7HADQHMfpBvUWhzXVISNgRuOCso7F1uo+vXL9moYeSkJAwJEaHJGttUsABPLuDVZwGQZua6GZ4sMqgIQ1CO2ZpOKIkSyngtNYsTzKc10HG6ZOZ2PbY93TdpnsFylIHSrKUJ7nV0xtRknnfs/EkN2W3mOm3KMns/cBKcu1J7peR7BaQFVpzv+QZq843PEEL75WIH7ul7d6gdgshcM+/T+Tj7KuUi1trbSwpvu/eeS+cr81uAUz23LLaTefjtI+qHb+4TULCfOKZhy3DqUfthy+k4iIJCbsdRockw80a4AclDa4kNz8w+QPVpIDLVfCQ583M1EvNvieZoKTsFoAQuNesDErnEQ/Qa1aYaTMpyU2Be4S2ZXvOiyV10s/3O6gn2Vprwm08YKytLHXrb18wJTnLooF7MXvtjJlUsRUPbyxf/Nn9rcuyUnENMbuFcI056PdqTx8Xvh+kah2/Ds7kh407N55k2ibt5/bCK+5lSjmWGqDNk8wnYVU7s835nZAwKN581tF4aOMkfnj7ows9lISEhCEwOiRZu57k0nuYt+WC5e00oe+oodX7bp6xh29IPIhQ+inBuMotBWP5xURElS2C2aa58r2mM/0qBZzWsoXEH3PzmNqV5GHOkdCkJPN+5DRpbHwtXKkwv1etJBdaHGOM3E3XqvZYJwtWPADgmnufwP/5r9vx/v+8rXEcwYRKy8VEJPWWoy11nm3fvvcnX42qrXnVzqBL9nuRBakwYwnbkaxIPE/ydN/94Zoma769pl+WSUlO2OE4+4SDcMR+i1M6uISE3QyjQ5KhkSkVBETR86+JIruFDtrUxJK9t8qiT0qlIDdfzaZdeOAf30aEX/IktxFI6Ri3/WYl2XiSi1IkRLMhyZy08fe81PGslOQGssdtIbLdYvD+iERXdosMvbK5LLVP1sj60clkTzLl6t44ObyS3Jbd4qY1G3HzgxvFcbb68BvtFu6+L/v4lTj3c9cFG6U0blq3eJIjanU1YbN/hzMeSR40cK+obUTJk5ywo5FnCm868yj8/IENuDEVF0lI2G0wMiS51ABUaLegB22TkLxtuu+20wBXSQ5TwEk2B1pmzyJXO1Nh4F6prT2D+MAwnmRTnCG2PbLBt3bM1CngAHcyIXnr2u0WLkGxfdq+3XNsbM6OxWRpiG/z+/f7rvprmSCVdkWgm6l44F4kVR9PAacEJZneta15hH3KijZv+6+/dwf+7vu/FNtrsidU7fC+3cmnf83uXLsFV93zeD0qO17/OtNnyiU9qPpbtcvzJLtp/qQxOceye4TaSEpyws7Aa045HPtMdJKanJCwG2FkSDJ0nd0iYrcwqpnwQNzqkOQ2JZkrxKQkZ4YR0FZO2Mx+kTzJXIk227RmSrL9btBxti2H09c0JBtIRu3bfek9P3cxcG9AuwIQJ0WzqbjHl++DPtmgJCV5GFIeBO7FFMgIgbSBe9aT7FZhbp/QAeE9XGr53Pg1nilKM1kL9hvwXnK+Y+p//Dj76tsc6NpQ4J5kt7AOJl+FtuqvZLcotcbF163G11aG2QT4b2KLzCRPcsKOx97jVXGRS29di4c2Ti70cBISEgbAyJBkDTe7BT3YiReYzwIJ2zZd8IYawT2uppiIkALOsVv027JbyL5Lmyc5JIHthM7tI9zuEvdOg7WjNEotOyeBWAyTGcRRkulVu4RsQI7c6K11A/fiZA8YZPzVa67IbhEJ3KP9tcbZJxyE804/EgC3W8ieZPuumSX7Pbop0dzvzTFavj60bdD+/EnUIBMZjfA60ye/mIhr7XBVa348/eYKED3JX/v5GnzzBrfCJe8HsBO9pCQn7Cyc/7yjAAAXpeIiCQm7BUaHJGsvu4UhyZTmCvXnkGhStb3qu8GVZF5MxFeuJU9yqCTXalimAubDA/eMSi2ojjG0ZbegbzNDkmuyIlg7aDlbyuzBMUyWhBhhdTJgDKoka5lM+f00+YcH6c/Jk5w1BO5pm1f6hEOW4NjlewGw90GXKcm+wgoMoCQLEyqjQvP9vHnMIOq9BLeEuHszNh1qbSfuddKsz65RktnGlvFpMOUZVZpCd7zVxK5t5YB+j+iKQELCPOPQZYvw8hMPxiXXrXZWMBMSEnZNjA5JRkUuco98+CWLJVLEH5JDeZKpmAjPbkFkmREU60mOBO4JnmQNnqkjJN5t/NGSdnk75ZUmVtVpsHaQ0s6XpXtinuTZKcmcd/EW+qXGRy+7C09snW5sl34HqX/uo24rgNL223OSnCkFXf/nnYbTVqasBWi6sJ5ku5+j0wJo9yRLyiovrmHG612P2Pm1VtzjfzPeJKopMDQ2qePbOp4nWVKt/S60dv+uJSW50HKpbj6OnrFbJJKcsPNwwVlHY8t0X7QDJSQk7FoYHZKsq+wWJiCKHqLmYVp9L6XFalMbObjdggfueRzZU5JrctXoSfYJDUsBZ87R3d6EQTzJjCMjz+OEnIo1uBMEwW4xjCdZUCe5VxUAfnr34/iny+/G+1pSovk5sTn6rXmS7fu2ADb6vTu5AlR1vtIh/DwyZYkr5fPlnmSXSFavrdkKGxRSfqhP/tr86dHuhPex4ETCZK/w7lf3vVGSvYmg1K+UGYMmbFoLdgutUZSx1QqLnrFbJE9yws7DyUfsi+ccuS8+/7P70wQtIWEXx8iQ5FK7dgujEnt+1VhWBftdC1ESgteqintuP6Ldwi/7R0vkKgzcg25WkgcNMoudjk2Z53qSbdCZ3XdyhpRkdu6zsFuUZXjt3DG51397bYOJBZz5/TblApYmItUx/H3z+IlUdbMMCiHJJWjWVpYpQ1yN3SLLTKYT18pQQbVoyZKSbPOBx88ndn7tmVLCvw9ppYKfy5apvus154q7dgvxAG42Gj+I1IcG/7sGZvq+3UJXdgvhHnU9yXumkqyUypVSNyqlviNsG1dKfUUpdY9S6jql1FE7f4SjjwvOOhpr1k/islRcJCFhl8bIkGQAgFIs72r1lV9ogj/wjZI8BPmMVdzzfbH8Ad+L2S0AE2zok3OxmAg/tk1J9vZ7fOs0Nm23+XdLXfVNhMTPVysV/mjPbtE8plhxCz5ZcbzQtfI6ljffpuY3FvqnsY93sgEC9xq7MeplpQTTOYTnVGrrreYTkZl+iawueDMbT/KL/v4nuGTFatGTLE8A3O9ic41Wu4Xw3jbNJ4OcJPdsOkH4qf0sgR3reIF72k50Y7m+OcmW7RaI2i0cT3L9QbIOjTjeCeCOyLYLAGzQWh8H4GMA/nanjWoPwq+ecBAO23cRPp/SwSUk7NIYCZLMg5b8/LPWu1h9XwjkbCi7BWMa9JDt8GIiVHGPp9/qU+CeP26rfvvPc42wLPVQSrLn8TzlQz/Csz90mdu3spolBe6FJMiSCFdJnlt2C0mJ5goswMo4+xcu0m5T4N5YJxPHN4yFpVfY31spIqfseOE+yjN7T/aK0nhwlbcfYO8diST3ihL3P74N9z++TVCSLQHmKnRgt4gG7olf23EJLNlXlAE3X/HmKTcoybeVmN8lzwHYSZeGzVAj9l/33eRJLpuUZPYdZZ3Zk5RkpdRhAF4B4HORXV4J4KL6/dcBvEQNWq40YWB08gxvfN5RWLFqfVDkJyEhYdfBiJDk6lWxYiJ+2rImJXkY8in5cjuMzFJToifZqyZSpa1TYgq4Uod5kofJ/CCNw7WaaMeeYgm53W7HX3Xcaw3caxxS1P/LRUmXdNmUaU1oKktN44wqyQNYbW55cBO2Tvft751nNnCP2QNsukHuSbZVIGeK0nhwlXJ/WzoOCO0WazdN4YEntgOoMjlouON0ylJzu4U3j5HS+jWdt7Tdz4/Mt3GyuoWRZK39AEmbeYKUZPpMkzd674+7+mzzJJc6zG5B7cdKnxPMCsme5Un+OIA/AxA76UMBrAEArXUfwCYA+++coe1ZeO1zD8fe46m4SELCrozRIMn1q0LcbmFUY4EQ82XooTzJxu+amT7sg90eYz3J3rhrJRmSkqyt8mxUYcG/GkNTWjQ6XspuIR3Xl5TkWVTca1eStUjOuwMqydJvRxaJsajdgu8btl2WGr/5iatw3oXXmTF38kpJLjUpn2HwGfGuSq23dotuTQr9fN7OWLzTPf1vLsdLP/rfACoiGiqr9h5xAvd8u4Wg0kv7+eBbjRdZUO9nHJLcc2xCfhd0fcZyd1KroQNHdnC+0E6sge9ZL2oVWbodpcnrnqIkK6V+A8BjWuufz1N7b1NKrVRKrVy3bt18NLlHYZ+JLl773MPx3ZsfwSObUnGRhIRdEaNBko1qZ8kHPfh8ldFXU8Pvmvtyi4lop8+qSEKoavZaAs8kT7LWTN0Ftc+8wy0Pdskb7bZfZwOpP4d9hYRKmiC4fbaQ5BYl0/fWmuIbLSTZlKVuVJLzWdktqO0bV28077tZBkAZckrecU7WbbEWa5+Y7pdGFZeItZTr2MdMvxQq7llltilwj3/kxHLQIFDehlGS2X5cSd461XfyJLurNXa8lA6P/3bKm0AEw9Nu/vPQblH9Lm2WoD2wmMiZAM5RSq0C8O8AXqyU+rK3z0MADgcApVQHwFIAT0iNaa0/o7U+RWt9yvLly3fcqEcYb3zeUSi1xkVXP7DQQ0lISBAwEiSZnnGKpduynuTS+eySItTfuQ/wJrhlqcsqZy5Tr6UlYlv+1m2L1NxMqVBpq0msM06EwXwxtBUTKWsVm8ZOXlnJa0qQ0t9JfUbHFFGSOeHizZLHtdsauOcWjHHGXNjgvzZi3xb81mceaZrY0G/I99XsPQ/Sm+mXRhW3qQqrV+2QxGrj11auwRu/sMIZz3S/DEijBr+X2di985V88n7fEvhmfxLo2i2s7WHLVN87zm2PxkbKul0tYYF7wt9S9dnNkxyq1BUJl1wU0grPnlJMRGv9Xq31YVrrowC8DsCPtdbnert9G8D59fvfqffZMy7QAuDw/RbjZScejIuvewDbUnGRhIRdDiNBkm3AkwpIpB/UJQXpzTa7Rb/wcjNr7ai+hF4/VJdpjAqyJ1lrq6By4upnoYghRjD4dq4kdxo8yeZ8vXP34S9b94sSX71+TaDqA7LqG1OS20lyfEJAYx7vZqL67pO3WNuAELhXN+CrwlqDKbvKC9yrPtgMDhVe/o8/xed+en91TP3du79+M37yS3cZe7oflsKmQDX/fJryJLcVWeGQrpH/Cgh2C3a8Pxmh32XMKyYChNdGslvQ/jN9ebJWlPJEzrHzGE/yns0BlVIfVEqdU3+8EMD+Sql7ALwLwHsWbmR7Bi446xhsnurjGzc8uNBDSUhI8NBZ6AHMB/hDlGK8iKQEeZIF5XC2xUSKUpvqazQOKUAwVtlLo5JzlVBxjyvJnLj63uEY2oo9lNr2DTSngDPn66jogurqcZKvrFyDv/jWrdgy3ccFZx3tTBx4W3YJX0Nr6xewJHnA7BaCcki//0Qnx5bpXrCdn2ZrXt2yWjlQdVo3yk9M185WjbPqZs5TwBVlbdUI1efV67eb37Ypl8B0vwhJo+YEPT4R4fckJ5dD2S1Mn+F9wm0Pm6f6diwaDtPWsNeqawL3bLv++UsTSPpdt8+4QXtVW7W9owwvpDR5LfaswD0AgNb6JwB+Ur9/H/t+CsBrFmZUeyaec+S+OOnwZfj8Vffj3NOODFKFJiQkLBxGQkkmONktPAJsyukK/mPJFxqDnwaNq7F86VdS7QLiqSkFXEhUNLhP2CrJfsaLGEyAVcOOPPOCryTLdgtGwAawW9C1uuexrcH2gZRkkz2kRUluIvakWHaywO5y/+Pb8DgreS3aLdgxmyZ7lsjW49VAkN2iKJndgnmSe30dKslsMkfXt+kRWQXu+aRRs2sgj93f5pLklruJE1xmKak+y21OzhRRJZl7lMPAPXv+klpN+zSp35T+re0e7SclOWEXwQVnHY1VT2zH5Xc+ttBDSUhIYBgpJZkXbvCD8mRrBe0D9l1zXz0veC1T4fKw385MVEm2xN5/oGtt/cd//b07ce196x3Vso3MW3+svF9pFDvyJA+gJJP6lyuTI9rZ7h2z11h1exERdewWEesD/5qKibRlaW3Kk0wTlLGOa7d4bPMUXvT3P3H2lY7n5/Tghklj/VBKgVL++RMznqO3suNYJXnxWO6ck2MDqt83paWd6ZfihEqyyQSTFmEC4h8jwa+Wx9t2lWSr6hYeKeY9aM3vpTAFXOZNDqVCO03ElgL3tHAZpcnrnpLdImHXxctPPBiHLluEC6+6D2efcNBCDychIaHGSCjJ9ODjeX+JAwTFRCQleQi7BSezvULXy+/hsa7dwiXstn9de5LlFHB82e3Hdz7mZLxoeq7zfM2x06ndFmbsvlorHcd9pFIKOJ/MbK/z1xqS7Kh4Qgdae6SrVuBbSExTnmTKQNLNldP/Ry+7q3X8vG2gIskdFnhHyrdvVSnY9edlqad7hQmQ5FYarStSTcS1TUkOrDmMlJcamOoV+JtL7wgCgfj5cdW3JfmKo0jzTCSAOzGkSU3VpnY2lt57v+Iev86+khz8/LrZIlHUgXtyXmz7nv4u95TAvYRdF508w/nPOxLX3rcetz60aaGHk5CQUGMkSDI94pSynmTfbmE8lEJBDinFVQxcSS5qj6ohPJFiH1TZSwzcq5fjJU9y7imKpZY9yd+68UF8+xcPi+cQDdyDduwWgSdZUopZzmE5cM/9vL0maUSSncmI6P91idxkTbKb8vjyUtaxYiKdrCr+wfu8d91WYfzSmDhJ3m4mEwrKWG39tIOl5koymCdZs2Iitn2/2yYlebpfiEoyH+e/Xbcan/7v+/CJK+6Jnt8wdgstvKemHOLNboB+qR27hV+0hfrseoF7VGCH9xUUT4FuJLY06WjLZmKyW+yBnuSEXQ+vfe4R2Hu8g7/41i3YmjJdJCTsEhgNkmyUZBUsffNUUUCkLLX3AG+CW1DDVswDgA9//w6zTUo1JdotQIF7CLb5ARyOksye6//jK7/AH19yI+u7nfSXRNDJbjGAJ5kmCGMdOZ2af37b6qCqdVumDXEhiMVEPMI4WR/fpHS6QZfh9n6dUSLPXCVZJsTh8fxaTvVKc51MRhIdph0sS0sCnYp7/UL0JPskrcleUpFbf0IlT0A2TfaC/Qg9J3CyhSSL95P2PlsleVE3r+wOTHV2J26IZregFY7YuKvza7ZIUOCenNLPvu+zqn1tqxUJCTsaSxd18bHXnoRbH96Mt160ElO9MCg1ISFh52I0SHL96palRv1qH4SAnMmibCFaHD1PLcuVzZN8yYo1Zpvk//TFL10TLKmYSFnqoEKftLQvwV3ajijJ2k1P5ts4JC8zXyJvS68FAJMzlRoy1Svx+NYZL6hMIMneeI2S3KD09YXf09+eZ1VqwDZC3Wa3ACDbLbxiIoW27WcZTwGnraeZjcPvo9Vu4efb9iYXlA3Ef8jya75281T0HH3wzUZJpr8vwZO8eCxHvywdJdjxNcOmrCO7Es8xTasrMCTbHd8Nqzdg5QMbGsfrK8mrn9iOX/vYlVi3xQZqOmnw2paQEhJ2As4+4SD8/WueiWvuewJ/dMmNoq0tISFh52E0SDJ7vuWe3cJXkt3Apup1mBRwfmlmSgkWjslVnIGQGFIJXkon5m5DYLfQ2v3u5gc34gs/u998Puo938Xfff/OQEmWyF/lh7aTCpOTuSF1nKk4l2diCjifaGxj6bk2T/W8zALSmFxCNqySLOdJLtHNM2SZavWei0qy13eXeYopL7a1W9i23ewWNnDPKtFcSfZIclMKuF4h2nb4tSff86SXHo0f97N7HmfWnXh/dQ+sLyKz2ttiPeSLxnIUpaceO75m+7v5E5jqGM9u4Y3vn39yb+NoKyW/vp/qdu96dAt++egWrHp8m9lvmFzRCQk7C7998mH4wDlPx2W3P4o/+/rNaZUjIWEBMRIkmZ6mPJOA9SRXD0JjrRCIEud7bf8eOQU1So0sk5W/gfIkawCKylIj2ObbLXwl+ZxP/Awf+K/bnX18AsELL/jtczKWZ5n5ntr3YTzJkcA9v5vtzFc33Svbi4lA9iQ3TVzcUtfC9rLycedKiZlNYm3FviNiCWXHa7IxCHaLnKUILErtkOxqHG7O6KrphuwWRVhxjxcTAayFYdJTkm0GCY2r7nkcZz35AHN8ExwlWbvf8W1Ekvca66AoS2fCxZXkUtuUdVm9EmOvs65tQM33YhOcuAEzWa7G1ivdlSD+fqpX4Gsr17TaTxISdjTOf95ReNfZx+ObNz6ED37n9nRPJiQsEEaCJJvsFkKe5GZPcqjwtv1jxAOeqhRw1gftjsm+j+ZJriF5kqXAPceT3DDM0iOOsQAmXi2w4xM9SZWtycd4J2P5p3lf7jG80MNM4WZlKIoS///ld2PtJrvszwO6AG63aCDJAiHiqAL3aiLGSLSkTlPf37n5Ybz3m7eIfXdyG7iHWvk2GVXMxMyWRFbe/WHsFmzFYygluV8KKxLu79XtyPcIfV71xHas2zKNs44bjCTzzcFKg2O3qE56Yix3Jp7+CkGl8FbvSUnm50Q+fUmtHgSSQkzXmAf8OX/LRYnL73gM7/76zSavd0LCQuKPXnwcLjjraHzx6lX42I/uXujhJCTskRgJkkyPPQVrR6CHsJ8ezM1uUe/TojByOJ7koqwrqoX7uUUx4sSzsjxUKu0H/us23PXoluqcIkqyn8/YR565hFtDDmDSpm9ljqN+q/bDtiW7RVMwnEOS+66SfOvDm/EPl93lBBz6/RKJeWD9drz9Syvx3ZsfCcbUFnRZBe5lyDM/oFFS16vv/vDiG3HJitXifl0TeGfbC7NbuNt4dj3jaTZ9Cp7kBpKsNTDtMfwqm4b9LE3agDCg75Cli8x4myDlSeal0gkz/RJjnQydrMr77ZBr9p4ryXmm6v3d/fgZSMvNpx61H/70V48Xx+tXxQQsOe4XspL8g9vWmiwsvgKfkLAQUErhf73iaXjNcw7DP11+Ny686v6FHlJCwh6HESkmQkpymLPYLzQhZbdwlNcWwuAH7mVZTEnWwTFinuRaaXxsyzS+8LNV+MLPVuGev3o5SuMZ5kvbVl2OKd7dXAXnE/P/8uvVzV31sSmIbayTYfNULzgnn4xvm+ljopthqlcGqcsoqIwTEq1l8n/dfU9gul/iB7c9ihc/9WVYVBfkCPsPDkWf2y243UNS1z11mfLtctBkQjkk2ctuwRTxPFOOfYJSyNkCNGEfRBGlSoxA5Uvm0Fr2wAfnV+9D154KmwxltwheuZJcYDzPqkwipXYItZMhA0BZX+zcqPzWFkWBe/b4cExZFk+VN9MPVxfob3CGXRv+t/w/v3ELnnX4suD7hISFhFIKf/OqZ2DLVB9/+Z3bsc9EB797yuELPayEhD0Go6UkC3aLME9ySOqkUtUx+HaLnBFNDtluEY7bVNxjD++f3v14vc0l4BqsGllknN08c/I1a60Dzyt9T95PQCiBLfCEvpAnuSm7xORMgX0XjwGoK8UJPm2ulmto8bx4/l1+/YH2oMt+UanvgwTu+WR161Q/INPdjAfuVd/ZstT2njPZLbz7o+ulgBM9yQrOPj6meu414GWp+Th8lDWZJpJMk422wCCpip+0MjPdLzHezYIJCeBaJjRTvnPlpuerglm9wD1hTDGbE+ApyV4hH64k+8Gn6+qMH9P9RJITdh108gz/+Hsn4flPPgDv+cbN+P6t4YpaQkLCjsFokGS2ROt7dpsq7lyqBxQAACAASURBVMllqZsJA1ei+kXpEHMO3g8VE+HffeQHd+Jfr3mgogPKDSha9cQ2Q2L982zLSDCWZ0F2C1FJhks0cuW221SWeiy3eZIdT7DHLbbN9LGMkWQnLV59TXJ2jpVXVVa9Cd+/7RF8+doHgjHxsXNUSnImBO6F+/rq+eapXjBZ8O0SgEt4q1cbLKmUq3jyFHK0b5AnmdqVfDwI7QC+57epZLPWlgRaJTm6u3g8P4YfOtMvMd7J0clJSdZmH/+eNMVWMirLbrdR/m5ppYfA80/78Fd7AJgy6vza+Irx+u0z5jwSEnYljHdyfOrc5+BZhy/DH19yE666+/GFHlJCwh6B0SDJIEKiAn+o70meazER/mAtNS0XC2MSPMmc0H3yinvZmF3l7eGNk5UnWYUEoS1P8ljHJcnxogo1CSfVkvL5Cl5TglNMRKhW5vezfbrAvou79TVwPcnUVu4oye2E7X9+4xb8r/+41XzmpCeWAs4UE2lTkutTmehWP+jmqV6Y3YJyDDLia/3c9l6j91VFRnu8CdxTNOa48usHblLWiiD/sTe5aCzZzJTkwe0W8koEfwUq8j3Wycz9bJRgHQadcjtKntnxa8BMHJvSEcYmp4BrtzABvML96mdoIYU+keSEXRF7jXfwxTeeimOW74W3fWklblgdzxWekJAwPxgNksyWvf0UcH3P6tBmt2gjaf4DNI96ku37tuwWmXKXfh/aOFl7ksOsF7bintxWJ3ePafQkg9kt6lRl3JPczd3z4p7knjcJkc5v20zf2C2me252C7JQcJU15kluQpvPmFLAZZnv1Rb2rb/be7yy6m+Z6gf7UVlpTnzpHKJ2C6Y7+yngpDzJ9JFPIP7k7OPx/539ZAASSXYnQlIOa74vVcab6A6mJPNLwKvo+dtm+gXG8jpwTzNPch08yv30Jk+yCtPzKeUmwYspybEAx76Q5s0P4APi12kmeZITdlEsXdzFv15wKpbvM443fn4F7ly7eaGHlJAw0hgtkgwVzdIgRePzYL5Oi0JL8JdoY2oWJ2xTDanMekUZtPHQhkmnSAWHyWccGV83z7zqZqHnlb7ndgsKQDSeZK1NkBmByMe4oyTLxLMoNaZ6JZbVSvK0pyTTZIOfY+VJnj1Jln67XlFauwXfV8ypTDaEiiRvnuwFkxFrt7ADN6sXZiIGx27Bz9EvJvLwpimsXr/d6YOuAT/uecftj+MP3AeA4ElG+2SBUJbAVF0Zb6KbV6sVLSzZCdxjxLfaZjdy/7cUPJiZtHduMRHuF+cTXpi+QmQNSrKz2kN2i8J99feLHZ+QsKvhwH0m8OULTsPisQ7Ou3CFUyAnISFhfjEaJBkhIbH5UV0VV0r35hbpkPuY6hX45BX3YNtM4SissYc1EZ1+URoPqETipvulo4gt6uZ4qLZbSFJZG5kfy7NA+ZNKSJe1lMyDxBTLpqBZXwTRk+yQM7sv+WadwD02Lpo4ZIGSLJ5WFK2eZB64pwfLA71XrSRvFgL3bHYK+52v7jt2C6U8T7Jblvri61bjT7/6C6cPbkUgjOU5xjqy3cLPL91rsFuUzG4x0c3rlIHNF93PTFGdq/sZqP62KKVbVY3QEl8377d2SLIfuOf3JU2cfE8y/1PhRNjep6HdIkaGk90iYVfH4fstxpffcir6RYlzL7zOyTefkJAwfxgNkmyU5Hh2C3o2SoVDytIW1ogpmdfdvx4f+cEvseL+JzDesSnI2uwW2xmhkRS+mb6rJB++3yI8vrUKIJKUZPKlxtTCQT3J0K4VIK9TanEFvuPZLXieZDG7BXtP1faMktwvnO1E1PxLNxe7hexJ1qZgBd+fH0fXmfreq/bqbmGeZFN0xQu8q95795xmFeUyL7tFFh6/hVUmrNqpXjlJ7nYUxjtyJT1/ciGtHJhtWhsleqKTiYVsfPDNpUdm/XSDVEGvX3K7BZy836W2k9VMeX5xzQP3tNnfh6r7IZCNBZCLifRMnmRX+ZaQSHLC7oDjDtwHF735VGzc3sN5F16HDdtmFnpICQkjh9EgyfWrUmGKtKDinkOqUH9nyW6MpE3WhTGmeqVR9IB4lD2RgO3T1XG8Sp0Pfvzh+y522vaxZFHHGbuPbp4Fy+OxinsKnpIMV2nt5O7tQePvdjKjyBWRwD0qJLKvkN2ikylD9FwlWbcSNh9tGSv6ZYlunhlyWwjEi86TrtNiUpIn+2Zytaj275KSzNXhoJiI40l2z9EoyQ2eWltBkivJGcbrMUieZH5vNWa3KKvj80yhk2dicGhwjA7/ZkwX7NBqsom6OIgbuMdXa7S2EyqawJhUjZDyJEtKsnt9xltIsilLzdMJRpTklAIuYXfBMw9bhs++4RQ8sH473viFFdjqTbgTEhLmhtEgyYxUWH+oRsnVLG2/J5hgPq1rMhMnn9N9S0zGGUnOM3c53bZdNbRtpvpHa5+JbpQAchK1315j5r3EofaZ6Drt++jmYT5gSTHT2u3XKOlmuzaqJ6FXVNepmyn0Ch0EnTkp3gobHNbJFK68ax3+6fK76zFmZtLhj2l4T3I8uwZQ2y2Y2m9WFNi+dJ4mLVl92punemY/CnLrCkpy7inJPHtDLE8ybZMgXYJunpn7zr+PtHc+3FKwmBVeAarfaLpfYqJuyy8J3TYe7X3n+9A56XVtP+514hX3uJKsdRWwqgDcsHoDNmybEf9u/Mlpl/1N9gt3TAArSz3AZCIF7iXsTjjj2P3xz69/Nm59eDPeetHKYBKdkJAwe4wISa5eFeQHMX0G5OwW9HBvWnrm//AQYQLiSjJ1TUrykokOilKjV5T4+QPrnX1V5GEv5cldUpPkmCrdDTzJkepy2s3DTEq6taBAUJIrawh9X3ptSxk9OrnCWCfDDas3GnV5rJOZJX8nuwVmY7eg85bJXr/UdQq4en/Bk9zxLCz0yu0WRkkWA/eE7BaGcMsp4Ko2ZEirHmOdzFnB8Pd3SbJ9f/CSiWDfqV5h7uGs4Z4nOJvJbmFsFxaF1p59wsrN3G6h4SrJ/goO3RI3rN6I8z5/nTimLHN/gzF2XWckJdkE7oXbfPT6w92DCQkLjZeecBD+4TXPwrX3P4E/vPjGFHyakDBPGC2SzIKkQgJXEw+BOFPAUdbgSeYZBcY9Iit6kktfSe6g0BqX3f4oXv0v1zj7+svqTSC7RYzYSMVEwrLHoSKYKTgp4Cq7RehJJhIEVITDzRgRKnjdXDnXq/ousyng+Jj04IF7l9/xKG5/eLNRTX2bCR9HJ8tEImvH426j61fZLUhJrs5BCtyzhUFgjrd2C+WcZWcAJZmO5Xl8x5iSHEC72Tq4J/kgnySXlSeZSLIawG7h3E/eq3uvaUcZ5qs4GjaDDF+ByJWnJNdtTdWWh1sfklNcqUBJth8c3zEF7BWh3SKGmSIpcQm7H37r5EPxwXOejh/d8Sj+7Os3t64QJSQktKOz0AOYD5jsFmBBWKX20pPZ781x7DtKhxYjDFxJduwWCqK31HiSa5K890QHWmtsmuwF+/KHve939kF2ixiZV0oFgVaSYkZ2i7625aEzL3Cvm4WeZKUsqex711guFhIqoPz68VOkfLqD4IKLVgIAvnTBqQBgMir4qFLAqSADBd+ViK+fsWPLdM+8pxLORPR8FR5wFWBjt8iAjPEyR0lu8SQ7RL6TYayMK8l8IsR/k+X7jHv7VingxmvSP1h2i3BsJoCPbStKjfEOI8l0PGxQnxkvKe11CripXokHN2w3+bt59gvp/vXLUo9FPMkmX7qnKDchBe4l7K4474yjsGmyh7//4V1YMtHB/znn6aIdMCEhYTC0KslKqQml1Aql1C+UUrcppT4g7DOulPqKUuoepdR1Sqmj2Lb31t//Uin1a/M7/ApWSXYJC1fU5DzJ9oFvSbLch6sk8+wWmUhmaUzbarvFPuNdFKUWH8D8eO5Zlf5tWzLRrCT7S+8aMjGgstcmk0KVUoCRH0FJLijFV00qfSWZE7WaqHQzFZBkUmWBUEke0m1hyE+V1cPd9up/uRoPbpg0FfcAO3nh4+74SnL9E22atJ5ka7dwi4Hw91yJdu0WLMCsZRIEsPSB3G6RZ0HeakKp3UkT9yRTdhFCoTWmewUmOtxu0UKSnbHJr1XbFemlYiL8fEhlpmOcinsKuOa+J3DW316B7TOFmzJP8jKBAvfsZze7Bb8PXXLci/3hMCSSnLA74x0vOg5vff7RuOiaB/Cxy+5a6OEkJOzWGERJngbwYq31VqVUF8BVSqlLtdbXsn0uALBBa32cUup1AP4WwGuVUicAeB2ApwN4EoAfKaWO11rP63omPfYUi3gvtW+tqF4lDy1lt2haep7igXuM5FWp08L9qZ3tzG5RavkB7JJkTqLCdpcsqj3JkXHyHL30WSwrXPfrlk9WznF0Tejwfu1JdpTkwu2LwJVAPqkAbLEOYO6e5Ec2VvlBKy+2e+zPH9hQj8FOZHguY0LX9yTX2yqSXO3jB+5xGLuFadtN4xYjc1Elub5FfEuIX6aa4PvO+fuli1ySTHYLuoeVUmhzIIh5kj1vMrWdq4ooF4WGYj97qd2y1NQn2S0I22f6zsSpm2ditommyYfjO9ZEjl3bRRNS4F7C7gylFP7815+GzZN9/NOP78GSRV285fnHLPSwEhJ2S7QqybrC1vpjt/7fZzKvBHBR/f7rAF6iKvbzSgD/rrWe1lrfD+AeAKfOy8jdMQJwg6TKUovEQSpNXGpdBwLFlcyY3aITUZKpP6MkT9h8wT6iJEoI7aLAvV5E7eIlkYGKkHBFkqd449ksKCUZz9CglEvMyG6RGyXZv8a2X1LsOnkW+KwXscBHhxRHCH0TblpTEeFuHk+x1xWUZN4PbfPtFhu3W7vFREMKOEmlttktmiZBMU+ymzmkm1de+1wg6LS/GzTZQJK1xnTfKsl51p5RxPuJvPbctnOmJHO7hYa9z3nxkyxzr8NUr3CWF/KIklyle7SfOy0p4KSy1DHMpMC9hN0cSin89auegV9/xsH40HfvwFevX7PQQ0pI2C0xUOCeUipXSt0E4DEAl2mt/ZDzQwGsAQCtdR/AJgD78+9rPFh/N6/gD2put5AIsVR8oiiramBZpqKEIW63iBcTue6+J/DD29cCqJRkICwEAQxntyBv7JRAtqt+dUA83TRp5mtnUpAreJ5k7RRnAWzgHtkTekXpLO2LdotcsFuwtGTO7wH3twTigYxU8OOWOrCrk1e2gS9f+wD+8OIbnH2pLDXvj/eT1TYdX2XeMtU3yr+f3YJzN2u3sNdBynUMuJ7zuCfZHR9dg5iSzMfMzxEAzjzugKDtKnDP2kaGC9xzrxGvkFdNooRiIroO6lPWbkFj7GSZY+vZPlM4U8O43UJF/27EFHDGbuFOLqX2k5KcMArIM4WPvfYkPP/JB+A937wZ37vlkYUeUkLCboeBSLLWutBanwTgMACnKqVOnO+BKKXeppRaqZRauW7duiGPtoTEBgf5kffVaxH5jgLXYpbF6VjgXhZLAafx2s9ci2vvW18Xgqirpc1IS8f2fZvSSESDk3aOsnQzHZRaJg2VkuzaLRTcgKzMU+sKypPM7Bax7BYme4HgSV7E7CrcfkJkioNbWziozS1TVSAkleO++cGNuO5+N8VevyxttTfByqBQKc1+4RkA2LB9ph6zlyeZtU+/H6/gaDy3DWSuyZPMJx+UFjBiSQ6CM+k87v3rX8eTli5y9i3KMAVcu92Cf3C/C5RkVZelLq3tp/Ik2wmshlXJfaXdn0T6vniCX0xkjE1cxRRwxm7h3V9CxpCZyAQ0IWF3w3gnx6fPew5OPmJfvPPfb8SVdw37bE1I2LMxVAo4rfVGAFcAeJm36SEAhwOAUqoDYCmAJ/j3NQ6rv5Pa/ozW+hSt9SnLly8fZlhOnmSbjssjcIzA2O9gvstrq8bwnmS5mAjve9FYbshtu5LcrDTSd7GE8UUQuCcr6tSWa7dQbIm8TvcmKckZeXhLMYMIYMlIN88CBZR7kh0rCDQ8oc+o9j6ZocPIr0pKsh/EBgCPbp5y8iT725WiKnFURdBue6Iu9UoKPllNeA5royTTyoS251GVaebn064kF96926Yka095pvPIFKAyf1+NqX7hpIBrtVuw923ZLaiYSL/UbsBffT9Vx1JGmdrS43iS/cA9+Z+oIP90RHH2lWTfkzzezYNjUuBewihh8VgHnz//uTh2+d54+5d+HuTpT0hIiGOQ7BbLlVLL6veLAJwN4E5vt28DOL9+/zsAfqyrJ++3Abyuzn5xNIAnA1gxX4MnWKLHiomU2lNUyVoRfmeD1GaR3ULJ5YVdj2hpiIBIbtnxbSngqJ1Y6VxeZbD6DI/IcqXYLrXbID1tjsvqpXMCLafTEnXP9yRzu0V98SuF1h0rL8bC/aNah5MUsgWEJLnajxR+ypMspbx7dPO0k4HC307ea6Mkl9r0u35rRZJpYiQryaGVg65FlTRkWE+y+5vRMTF/Ls+mAVS/CwWx+kcUWjt2i8FSwIWrL/bV3cY96ybdHqr7iWe3oNzkgEv+J327RURJ5qtGgFx4B7C/Sc97JYhKcrJbJIwYli7u4ksXnIaDlozjTV+4Hnc8IucfT0hIcDGIknwIgCuUUjcDuB6VJ/k7SqkPKqXOqfe5EMD+Sql7ALwLwHsAQGt9G4CvArgdwPcBvGO+M1tU/VSv3Kc4mN1C49r7nsCGbT1TnCDuSY4oybnsSebtbJ8pzD4SSY7le5U4FO073aAkk4ZHuYOljB7kObaeZOV8psA+Tsx6ZYlM2SCpfuHlouZL/qQkZxlmvCVuHrjn2C2AYJJCJGaRV16Z+iJCQ0VU/IBNoFKSrfqtgz4Uqt+R2iy0xv57VfmFjZLsBe7BIWjumLjVwLfjtGUvAap7h6cvpImTX0DD7I/qtxljWTqIePqrHGVZ3YPjTgo4eRx2PG5f/Dt+KBFfOkVzb+g6mwqPFyitspw5SnLf+XtoSgHHN8VUdhuMKWe3mBCU5FRxL2EUsXyfcXz5Ladh8VgH5124Avc/vm2hh5SQsMtjkOwWN2utT9ZaP1NrfaLW+oP19+/TWn+7fj+ltX6N1vo4rfWpWuv72PF/pbU+Vmv9FK31pTviJEyQFCx3KXR7WeqZQuN1n7kWK1atR541BzE5JJkRnU7Ek1xojcVjboAfELNb2Peu3SJsmPaN2S140BeRD+7DfOclN+K6+54wSrKxqij3/Gk7Jx/0nQncK0svKLA69p7HtmB9TS47uQoycSwas+foq9yUv5lAJMYnM77doptntRfYzZsNAO/+taewDBZl8BtnWfU7ciV5/73HAADrt01XYx4kcI/ZLQpmt+Dn4wbuxZVhGssBe4/jmYctNdskNVnrKhUf94rTb+/vXmqNaScFXDydoGm/psLVJNL9zg8YrFK6uSn1NChwD+YzWTMAl+CWGlg8Hv7d+MiUa3NqUtkBm/GjN4AneTopyQkjisP2XYwvv+VUlFrj3M9dh0c2TS70kBISdmmMWFlqOIowEeJurhwFlcDJGwVYDWS3YITNf1gTSg0cuswGTRFpmZwZIruFMA5qp8luQedolGR2zpff+Riuu399tQ9Tzv2cyJQWz1/GzpStxFeUbp5k8vu+9KNX4q++d4cZg18KmCvJjrJXq/9cPSQSM9HxSbIl84D9jfn5ZkrhHS86Fq969mFO6Wk61lonVB3AxpXkiiT7nmR+jL0mdvXC7yMIMHNS/MkoS0sw/+RXj8c/vu7koC9pfyLg/aI05Ng3XPRLjZmitCngVDyji2mfVhsyxchx9Z1r7anIeaAk1/tZu4V27RbePUbpEqtzkcfmB/zF7Ba+F9m/FzlJPuGQJTjhkCXJk5ww0jjuwH1w0ZtOxabJHs67cIURNBISEkKMBkm2rmQAMISnMGQxs55kwTcLuEvBHFO9Ake957u4nXm43DzJoZJMS/+Lx22Ams1KIeVJZiS5xZMcs2284Ywj8bRDllTlgBnxLTUC+wGVDFawy+WUyo4+UxVCfxm7IkHkSXZVWa0RlN3u5FlIklngHlf2yG7BSRMpyH6WC5/Xdeprrtn5am2X9KnN+9Ztw6W3VGn5bM5jCtyjpXmNZYvHkCnrSfbzJLtKcvXKS16b3N2ZS1MHSQHXL0vcsXazM25znpKSDI1eUZpViL5jt3D3pUkaz27hB0sG7ZtJqF154Pm2CaWmdHr+b6XN/QQAazdNYf22GXNuPsGldIlAfDLoW08i1uWgwmLf+1sg28nLnn4wvvfO5+PoA/ZK2S0SRh7POGwpLjz/FKxZvx3nf36FyRKUkJDgYjRIsre8ToowPfw7Gcv/K6TKAmolOUNQJmW7oPxyJZnnSR7LM6z68Ctw5P6L68DBqrG/fOXTjbrWarfI4iTq5ScezOwWLnl4xqFLcdi+i5zCEnmd99knBqWuTKKu3cKdJJQ6zDxQjZVV3PM8yUWp8cimKWf/Tq6CJe5F0cA9XSvJ9hrElGQ/GHCMBe5ZNdyeF7X5xatX4U++9gszNsB6km1Z6krlXLqoiy3TVcVES5JD8pl5144HB4b5fO25xQTc61dtwJu+cL3Tn+lLIMkU6McrB2Yxktyrzme8M7zdgqvO5mdnhxpPsvKPr/6ne+mff3Iv/vOmhw2R989xyQAk2b+urYF7hUuWCePdDFe++0X4+OtOAlBNYlLgXsKegNOO2R//cu6zcccjm/GWi1ZGLXwJCXsyRoIkE+gxmWVuVS9KDwa42S34Uq7NkywQSg/ck8xJMnE7aqfUGi992oE474yjmALcVpaaL+Pbfa5894vwydc/25A+v3Ifpd7imQ5oclB6xKBf2wGUsufnTxIqFTbMzesE7pVlUNVwrU+SMxWQDu5J9rNbaO2eN5HTsU7mED7fFtPNlfnNeZo3aosIMZUJB6z1Qdfnb/MkV8ctWzxmx1GTSjp3rg/zUugApeGD6T/2+7YFzAGhkix7kis7wTi3WxCZ9+wW0/X9RysWNJFqgmbn4nNjR0muyXmeh6o/ZZDhsL7puN0ipupWNhb+uZkk+5MqQidTOGL/xfY+y7MUuJewx+DFTz0I//C7z8KKVevxjn+7IVj1S0jY0zESJNk+xD27BZHFPGMKKfMks38QiMz4xMUnmIC79N/JbGBWh+XQpQA6pVwiIGe3sO9jgV0T3cyxOvhkO6+38ewNZJ8IlOTabsHtFXlWESquJIt2C2VTwPULbRQ6VfubAyU5E+wWjifZt1top8QwkZc8U47K7qNb/8ZFieAaVOMIrxsRZyJwPPAuzxT2ZnaZJXV550UsvzDBLwroll2OB+4NUoLbzxMsZXGg1QLHbhEJ3KMJS4cR1PbsFlYV9+0W/FCrJLud6jq9RUD4jRXG7W8fdt1jqi5Nav22fMRsFuY47/omJTlhT8MrTzoUf/nKE3H5nY/hT7/2C/GZl5Cwp2IkSLItAVx9NnYLCtDKLBGQKpMBNnDNJy7SUjTPk8xJECcmulYzM2+bZLdwPMmRwC6f9PhlqSmnMffDUg7cwlPRKLCMB+qRtYI4a2mUZJd8KGXJZb/UJiBqvJOh0BprvWjpTqbw9hcc63zneJJLV0n2FccJlp84lsEAqCZC3I9t7Q52O+BOUuhal2VFRintGimiXPU96fBl+OTrn40zjt3fXC+Cr2K6gXsuSeZE31/6lzCQkoxqstHtKNOuCdzzxkZBafx+ahsHz5ZC1gse4Gn2K7UziXLb0AGRjQfuMZIc9SQPFrjnFxPx0fW8Id08S4F7CXsczj39SLz7156C/7zpYbz/27e1ri4lJOwpGAmSTH/OliTD2B2Aym9qyBMP3HOUZOUoZQSJQPhKsh8cRko2D1aih7gcuGffx4qJ2ICxuJKcKcqTbL/j6cTMeZe6LvzAylLXJNnk+tUAEFOSq7H0CltxbyzPUJRwlGQi2f/vC4/Fqg+/AicfsQx7jeUNSnKlALvZLaySHCssUfWvnGIk9LtZT3J4/bvMbpFl1m5BimjHs9W84pmHiAptkItYW1tP7mU/4WRuMCW5nSSXWjt5kntFaT3J3r6k/OfsvmwvJmL79smxk92CVh+8MdYW+IDI8r8XDm63iPF3v+Ie/VQ+6eXBmFLfnVxQkhNJTtgD8QcvPBZvf8Ex+NK1D+AffnjXQg8nIWGXQKd9l10fRPTIf0kPfiIq3SzDtrqGCS/SwAPKiGQGnmThecmzW/Bl38whHm4AFZES3ufv/8qxeNsLjsFXV64x38XKUpsgs/o735NM1giuYhr7hRC4p7UOsltUvtzqhEkFbw3cq89nrJNDa421my1J9gnIt/7gTADALQ9uMt/5acJ4RgrAksROljnXxgdZaqg5Uqh9lX+KESBqW+vKqmDLUldj4Ona/MlCk90CcFc3YgL4IGKNdP19FPWkhwfu0XH+7mR9sfmew4lhME6WJ9lP/caPrfoNx0w2mtkoyTFUnuTQblHZe+zfBt1fvuUnVwoFdFDOmuwWVb7u+KQsIWHUoJTCe17+VGye6uETV9yDpYu6eOsLjlnoYSUkLChGSkkm2cwEsAmBe9yvyYN5bJ7k4ewWPAWc9XnaLAuWpIXj3ndxF/vtNeaQKK4cOp5LIhQUuCcpybXSR3kvSfnzlWSTAk6xCYaypBqwhUN8UlbZLWzgXr8soVSl4BWlxqOcJEfYIQ/c81FqtxQxJ1Kx9siPzS02NBmi8ROx5yohnYeu/bLGalKSkmz78/mSlCeZg0hZplQQPEeYL7sFnZPJk+xkt3D3p3HNxm5RtUUrDeRJtscWWju+eQPt5kkm0EdJSf7r334GTjx0SXRMvpJMKrW/2sD/7jnyyP40AfYzsiQk7AlQSuFDv/UMvOKZh+CvvncHPvXf9y70kBISFhQjoiRXr/S4U+RJLklRzcz7fr0sPVkWQXYLIFzeFe0WHb4MnxkiwpePiaTHlpT9/QkxYtbx9vVtG0TWH9o4iXf++00AKgV9IaRjsAAAIABJREFUUhdynmTtBm0RKTRp0HRVTERSMomoF2VFPshyUmo3ZVeM1EqlgAEb8Mb7NOSnhSSTn5zuBVKSrSc8JOZEnDV0nSe5VpLrMZAy66uWQHtmBU5GY0rybOwW0jWgvkj55v5kGit1Rb9Px7tXG2FWakL1m99atAoQy8ARs1tISvLrTzsCD6zfhlsf2gwJqvbgm7bMZMj9nf1iIn7fgd2i/jzZKxzrU0LCnoI8U/jY756ETCl8+NI7sWHbDN7z8qemlZWEPRIj8hRwVcNMVQSZVOBubpeU+4V2FDcCKcmcBKzbMh0UxwBc33DOltOJh+WqKtBAHk1AVgBtzl1ZlZTK7lq7hfvQ9xW85x61L048dGlUSS7JbuF5kvuMJPtEhM6RzrPUVeBeJ8uQZfTZ9hWzRywek+dm1bK8SwTNMnquAkLD96Hfzg/U8rNbcJjAvZrA9U3gnmsrkX67psA9t/94+elZ2S1Ekkz3eagk++Oz5N163NvtFjblWuhJZkpyqc195B6vnbLU/rjCPMmVJ3mswV6Tsb+76nzktujfAP9vgHbz7RbHH7wPAODnD6yP9p2QMOoY62T4+GtPwnmnH4lPX3kf3vONW4KJZkLCnoCRIMl2Obh6tVkdartFxu0WrDIZD9yryR9/6D/3r36EV//L1QCA5xy5L/7i158GwCV/eZ6ZBWdewY2UZBNMKJFkplSacXCSzPYlokWVxvzgIiKKhD940XEY67h5kwmF5oF7tn1edY6UZp/Y8H4oKLCTWz+0Y2GJ2S2iSnLoSabrlmdZNHCPVPSCWWyCwD3hWCJUVAqbpwnMFJiSHB7r/mbhmGxqPBVYNQgDKcneuKVUZ5SyjCZvRVk6+/EjZnwlOWsfh5kwqbAstfUoazPZCFLAaTsRkc7Nn0SQJ9lPf8fhl4Ontv2JGa0O8MnbWJ5F9z/9mP2w93gHP7zt0WjfCQl7AvJM4YOvfDr++CVPxldWrsE7Lr4hFRxJ2OMwEiTZ2i34ErL9vipLXb3vFxpj9cO5x8gj+W/Xb5/BbQ/bwDLC215wDJ5//AF43rH745ClE+b7SjW2gXK2f2tpoP18+BaK6j3Ye5ldZUo5qdOAmuQHAW/xPMkmBRydRxYqyZkKia5y7BaVctyp/dB8YgLEleTxyDK2RtVeVwiY62TxPMm8EIxZMSCSzM7Ph+mn9sv62S1ou6gkIyRoHP2ybLTaUD9t8G0iUl9EfB0lOeJtDz3Jqr3inq6VZPCAPc22syqPSiH3iL1G9dsGgXtsssWxd02SuWXER6UkyysOHDQP5pO3sU5mVWzvHh3v5PiVpyzHj+98LNp3QsKeAqUU3nX28Xj/b56AH9z2KN70hetTCeuEPQojQpJt8Bm9lk4xEasS9kpmtyhc1VMphRtXb8TvfebaoI9cKTz14CW4+K2nO57aDiNXtjgCz25h2/chKcl8v5gCKS2R+8vcnSxDpqpr43uSOYG0OZXrXMEsBZwUuJcpS9RIOe7kmSGpfu5pcfyZwiVvPR2vOvlQ53uyS3SdwD3bVqw98kRrzZbXC9eTLBHsjvEk26BFzSY3XZYBIjiHlslMr2C5isVRD2a3CFPAhfsYT3JNKvusb38AYZ7k6p7/5BX3RMdQ6mpSQF5/f+yVUmztJdKEsBQC98AmW4RF3dyQ/abiMbSiQqD3gd2iLOtJof2um9spjp8yDgCO3G+xCX5NSEgA3nTm0fj4a0/C9avW4/WfvQ5PbJ1e6CElJOwUjAZJrl+5akjliYGKjNJDvShLRpJ9T3L1fuu0LV1stjcEjRWeksztFk1qonmgR3zIsYAviTzz8thARQCrcYQleWlsPHCPAq5cT7IcuEdfaV0HidVL7GXpXtOmvMZnHLs/li8Zd77TqEguV/d44J5EaGifzCNcdB7GriEcaz3J2lhN+G/ZFSYxBCUQNI5+UQbVFmeDMLtF+CdLJNkqyaXTp2O3KNx7lZr/yA9+6bSptcYPb1uLux/dgk/9973G0mGKifB9wYq3ZGGAJU08fFsFTWTo2Fc9+1D87984wWyP/d7VuN37XbJPVBlXwpWUrmCR4hjv5OiX4eQyIWFPxm+dfCg++4ZTcPdjW/CaT12DhzZOth+UkLCbYzRIsseSrd2iDmhiftMeW8537BYZ99qGWS1iFb0kkpzVFoySEQOJZEuFKVwlOd6njyxzlTWrrtp80QTKblHZLWyAG8/wYDzJAkmm74paOc5rQl59tp01KYE0Rg4KMpTyE+cNgXs8DZ8fuOcXE+Ew2S20/R1pYlWlnIvbLdCSAq5fMnvB7DlyqCR7bWVKDtxzMoQIdgvJ6rONTQ7/46aH8LYv/RwXXLQSQFVxUCkYduxU2tNubm7/b0XD5t3mMJkn6nvm5MOX4fWnHWHPvSVwj192utY8x/JYnuG7tzyMB57Y5hzbzTNnlckHFQtKRUUSEly86KkH4ksXnIZ1W6fx6n++Gnc/umWhh5SQsEMxGiTZy25BhM1UPWMkuV+Ust3CW76Vig9IyDOFo5fvBQB4x4uOZf25xEB63kvL+XwMMW4lppMT7RZk+wjLUmu4tg0iN3Ta5EmWUp9x1bZXlOhmmanWxycXTWWkacwuqhLart2iVgjrPiRQSW7A/m5+CjhJMeRKMqnodKkypYwnVpyUcCVZ2F5VvQv3HRZteZLzTAV5krkXHnDJZOBJZu09zpZQH9pQqURr6wqKX37LaY6H3bdbmImikpRkmPuNo1e6SrJ/Hflkyb9VYkryElatb9tMgUc3T+PtX/65227HkmRJrSbPvF+wJyEhAXjuUfvhq28/A4XWeM2nr8FNazYu9JASEnYYRoMkm8C9CjkpqNoqbMTdCqZU9pw8yW4wlk+SY6Joniksmehi1YdfgZedeEi1r3ItDdV3kpLcnN0itkwftVtwkpyTKh6WpTZV+ZS9dr6SXKWIC7NbcCWZ/M6dXNXFOLRzTZuWy6sxuxeV7BuO3YIp8bH2yFoC2KV1Q7walGQizpWSnGH9thn86zWr6v6sEi79Dm22mMoXPLjdIpYD2if3oUdc2ewWgk0FiGW3CO89TpJ9W4ZNs2ftOARe6VApedLAM70QfNU/yAmdx/8W+GSNj3PJoi7a0M1tZcmY3QII0ywmJCRUeNohS/D13z8DSya6eP1nr8VVdz++0ENKSNghGC2SzEgJBWEBLECrJnE2TzJLAafcVF3+A7JJSfZBAU68LHWTJzmeJ1ns0uwz5pFJPpRuXittGkIKOAC12mhU+MzN8FCl7JLy9HIluZpoUEEVnhsYaF4ur7aHiqMfuEf7dJj9gUDnnys34wbA7BZm3GEqti67LzqZwvptM/ibS++sz0GhqQx2W+BevywNWRzEkhzL+OF7qYOUcJligXv8fpDHJwXuEdZtmQ7267BzUKjui3vXbcWWKde3zzO8cLJLzfvqNmB/I5rM+tu7uXw+tK9jU1KkJIc5uI8/cJ+gXfrTb1SSe4kkJyTEcOT+e+Hrv38GjthvMd70xRX43i2PLPSQEhLmHaNBkiFkt+BLwBmpqm6eZN8awB/S26fdpdZo4J7AgDJVkQYnBZykZLbYLWIKpLEgeLYEPpa8zm4hK8klKyZiz8PPk0y5cd1zY/7f2oPczSvFmYgVFQtptVsE3lXtTCz4NcjzUEnmxT5oP5r49I3dwh7je6T97Bb+2Lga78Pxw7JjaUzbpovWFHAc45Hc0b666reVZwo9kwKO/f58P/aWVGdz77H21221GR0MmWbXWKkqpeDvfuoap8gOD5L1/44U28cH/UZEsP0JQLdBSfatQMZuISjJfuBqN8+skixMhMaS3SIhYSAcuGQCX3nbGXjWYcvwjotvwMXXrV7oISUkzCtGgyT7dotMgac+I3JEhNHaLVwlmXOo7T1XKYsG7glKFHmgyddL7Uv7VdvC75wT8mB9zpwUu2Ps1MopzzxAIE+yUix1V6aQZ5lR98hPLeVJpvRbdI2JGE3X15NS5LXZLSTvalGruv716GRhECGRGSpLDQCFCQZzJ068LYLjSRYmA2Nmezh2bs3hhy5dNAagKg++j6BqxhBVkgfwJJPFZRi7RZOSfNejWwwJpt8iV8qkDFy/3U2PptlqRaYU3NLq9UTDm/wAPHCPCLZ7DbiSvGjMnUT4KwPUJfckX/rO5wMIV4XGmP1KWi2wnuSkJCcktGHp4i6+dMFpeOHxy/Hn37oFn7ziHiePekLC7ozRIMn1Kz00yW5RenYLKpsspYCj4wjbfCU5ogZKXtLKbmErkAEuuaG3UkW3tnLHfB+eO9gPZOrktsBGWEzELn/TFutJtp5THhBnr4PdnwfuZSyAbHFNaKR0ZU5bgZJc/Sb8OFtMJAtUP17swyrJrt3CvybS8VqHk52MKclSKjD+0/A+li6qiPHDGycNYRtESR6LkOQwu0X4mdThbsxuwT6YwD1jA7L7rdsyjV5R4lc/diW+ccOD1X7cMlKvFvjPP+5JpvvIH0epQ9tJz0sB558b/72X7+2mC/RXOawn2U5MnnbIEhy8ZCKoEsaLlIh2i27yJCckDINFYzk+84ZT8FsnPQkf+cEv8aHv3hHY/BISdkcMLnXtwrDFROyDv9QI1KIqMMzaLXgKOCq9S5icmYvdQtX922Alzu/2Gutgy3TfJSBCHzFqxYuW5EqhgA6KbXSqSEQxnR2Vb66UQdtmnnueZBWWGDYe6zpQr/IQV3moZ+rlaSLJ3WHtFvVYHZJFSnKugvaIWPIUcJYk13YLdt19wtk1kyeZjCpSkkWSLFsbli0eA7AN/VIbwtZyGQDEleRgkhJ4xDnhy8T9+BE9ISCP8PjW6YAYUlEaVSvJk0JZWg0wu4U06XNXVQjmt/JsUfZ87OcDl4zj7se22vNWXiaYus+lnt0izxSmeqHdgp+fj5TdIiFheHTzDB/93ZOwbPEYLrzqfmzc3sPfvvoZrbEpCQm7MkaEJFev9Mz0SyRbT3JttzCVyezDk1fHA4BtM57dYojAvby2MXBP8iFLF+HZRyzDYfsuxsbJHq68a10QuCcFJ0ngFo48U0AhKMmZDdybnCkw1smM0tsvw+IOWVYdQ+p7rJgI5ZClcty9QmOiWx1HiiYtjTcVE5GunYbNu2z3sfv6/9iSvSDLwhRwppiI59PmsAQptALkGZDXd5RUttklaPb9MkbSiLDF8l075xIhyb6P2r9mnNyPO0qyvDrR8zzJyiPJfm5g7WVokYLZdOnZLbJQ0RaLiXiZSMJJjG1nv71cJdm/3+m67DPhkuROrgKyy20pYp7kZLdISJgVskzh/b95AvbbawwfvewubJrs4ROvP9mpUpuQsDthNEiyCdyzD3Ne4KDDFMN+qY06xu0WfsDYwEqy8D31XzD1bK/xDr75B2cCAP74khurcXlpuKTgpMv/5FcCWwgvkGEUwcwrJpIrE7i3dbqPJRNdk+KrrDN/+JaBTCkzcSCC749pok6PRWo9BUL2ijKwW0gqHUdgVdFVUKG7XG/PNUaizIQAlnBJFgl/aZ3a04KSzFPdiYF7jifZvl+62JI0a7cIDg9AaccIR+2/GP/65tNELy616Zd6drNBhJYHQPIk222TM0WQ+rBXaidLh6wkuxNS/rMrNtHw5wp+xT3f8uIqvuHfhutJrj4smYgryWN5hpmidGwpcnaL2m6RslskJAwNpRT++CVPxr6Lu3jft2/DGz6/Ap87/5TgbzMhYXfASKyD+EpyJ1Po9bVRt3jgXq8ojYe1x6Le+6V2lqV9JTm2YhRNAVfKAWEAS2vmZbcIlGMFHLt8bzzlYDeFFZEQXmVPslsoKEOSlzKvpi0m4p5H6EkOz5uU5Kzet19oQ9aJgC3qDqoke3mSUf0OkkKYZ1nQHqmvnMzThILIntQWgchSqXVA0HhZ6lLgSlLQGFCdOymVvpJ89gkHhQ3Vx/h2i4lujiP2Xxzsyz3a/jnF7BbccGGyW2Rhhhc+0SH0i9L0qaBEkkzWIiCuJFPebf84APjfv3ECnv/kA3D60fs7251Ucl6f3JOslJ08THT91QKrJNM1HmuzW3ST3SIhYa4474yj8I+vOxk3PLABr/v0tU6KyYSE3QWjRZLrJ+nSRV1smuyBBFhT/KKsSBTlEOaqWVk2e5Jj1gfpIZtnvPRzeNxY7hIcTnQH6dPxJDtKMiPJtZKsAWyZ6jupsSiokZOWLPAka0dNJZDKRhk8+nUxkUwxkjxgCrgwu0U1sXGyW9Tn1M2VUbEJXTbZsJ5kr4qbZ0GR+i+1EBCXKeZZbgnc85Rv8iLza37te1+CT7z+5KAdAHjX2cfjTWce7Y4tVjiF/d6AO+54nmT73leSuaWgX+rAYtAr7GqIUgiC4ID6d2NZUvifBF3/amVCPCUcd+De+NIFoWruWFC8Y/nESMFO5vy/tzzLjJJM5FfKw+30WzeWylInJMwN5zzrSfjc+afg/se34TWfuhpr1m9f6CElJAyF0SDJ9Ss9NPddPIYN22eskszITlUhrgpqK7jdwgssCrJbRJ7wkqMgUypaIAGwD2ZSuBVTxDhiFNPkDmYk1i+r3anXo7UGtkz1nICmQhOB523CU5JdtY6uoeNJ1hr9ojQlsEmlXEwp4IbMbkF2mDxXOOnwZXj/b57gZAc5/3lH4fwzjjT7O0oyI7yADcp0LSiep9mUcdai3cLk026xW/jWBvLFcpJ88NKJwFJB+O1nH4qXPPVAvPzEg3HyEcvq820O5KPxuoF78pj472w8yQJJ7vVDJblXlE4fnCTz/OOmBLynJPObeABrtgO3kElot+A2pacdsgQnHroEBy+dCNqgMRP55Yq7mALOKMmJJCckzBUvfMqB+PJbTsOG7T38zqeuxi/XblnoISUkDIyRIMm+0rdscRcbJ3uG3BA5KrVGryzRrW0KPLuF70kO8iQPoSQr5u2VuLVZKo/YLairWG5mvj1nZMnJaKCUsT1M90vHD1aaPMmuFSHPsjqoz3qW/Xy6pOYqpVCU5PGur2c96Rg0cC9QkmGzW/zHO87Em848mtkLFA7fbzFe+JQDzf7ck+wrwXT9+TmGSjJLASeUf6b7Rkr56Sq17nWnqm9S9TcJnawi+f9y7nNw6lH7AXAtHBx0nkcesBhLJjrO5IdbNtzsFsxuESjJFYGc6GbolTb4ktCv82DTefIVFhqL60l21W1+GoOkwuOg33fZoq4Y1Monl089eAm+80fPx97j7jXPM2XI7rjJ3x33OgOpLHVCwnzjOUfu+3/bO/MwKarr/b+nl9mHYRvGYXPYF9kFBGQREEVRcddEoyQajcbdmK9mVZNfNO5J3GLcTeJuohITRYMYkaiogLKoGIiKC6gRFQRmub8/qm7Vrapb1dVr9fScz/PM0z3VtZyuru5669R7z8EDp06GEMDRv1+GV/77v6hDYphQlIRIdtstOleVYVdLG7bvNISuPBHue80SSxCpg9QA016g7A233cJP8Om8yvGYUqdXcxJOWplkvd1CvY2s36aSSVasF+5tdasus557M8nCYxlQ7QdtQmbrnJYDK5McM8R2S6tht1BFmSWSU9gt3PEK077hqJOseJIBZzbStq14/cqt2kyySyQrHfe87Z6Dm6E461kr74nIyiDrur/pcF/cAP6DHuU+mzG4Hqsu3h/ViigMM3DPXQJOiubaiqTWk6xePLoH7skQheJJJtO2444X8D+e/ZCCvUt1mfcui5JJDqoeoqv+UZZIkUnmEnAMk3OG7FaLh0+bgi5VSRx/64tY8taWqENimJSUhEiWhguZMetiVhj4ZJvRGUyeKP+33ewiFpeVHOwUYUur05O83V3dws8frBEzMSJtCTJJwuNJdorlVCd/tVZxTBHM7m11UUSy2mShpVV4PKKq/7ilrc0q/WVPM96PzCRbdgvTvqKKoSpr4F561S10pcA89gKNB9ddm1eN15ktdzcjMUWy8JaAU+0WOvw9yXaFBXfNXj90A9387D3u/aDO5hDJGuEN2MJTHs/y/9qKBFpahdaHq4pktWiIlUlWRLJ64aYu644jDE3dqnH8pL74wwnjNXYLSnkxCTj3oxS/6sVPoEjm6hYMk1P6dK3Cg9+bgqbu1Tj5rpfx2MoPog6JYQIpCZHszSQb4uRTs+SZt1Ob0dJWrW7R5hJK2911klOIFse8RNYAMt1itk1AjjaylzPehy1KdOhKwMViXiHftdpbsxeQHdKcA/cML6mZNW6zPcnWoEdzH8tMMpm+6xazZJu67zKtk+zOcsq41GnqdtT96F6Xzu7iV0JO20wkFiyS/e0W9gVJ2JJHzm6M5Jmmm5c0AlHNkAbt+hjZx7OaSd7V2oZdrd7sqTUozi1UrTsPzhJwuu6SRszO9V54wFD/IM11/fLQkRhQX+NZVhXJ7gucHx84DMft1ReA8xhMajzJumOUiFCWiLHdgmHyQH1tOe4/dRLG9u2Cs+97Dfcs2xh1SAzjS4nUSTZQ7RYA8Jkrkyyx7RZuT7I9T9hMsk48E+mbWUikzcKvBFzc5+RvbVOKasWLGyev3UJtwKAKNl0JuFjMFmDNisD3q5Mcj5Ftt4jFHKKsyqxukcpu4bYUtLY5/bIAUFVubK+63K7PLLHaese8WXTd/vf1JEN4RKlqP9GjF4Ixogwyyd51+Q4UdYljR2bdz27h0vrqfpciuVNFAi0au4Vjm66QrAsowFHdQmcfccf0t7OmYo+eddr3qMMrkgGK6V/77vT+SozefeIYuOdjaylXmu8wDJNbOlUkcfd3JuKMP7+Knz66Gp9ta8ZZswemfbeJYfJNSWWS1eoWAPDptl2OGqqSpFkyrEUduCecWbntruoW6WSS40Se7LZKv/pq9KyrsG7rej3J5rLaLaqeZDsudzMRAOha5eNJbhOekneqtaK5pU2Z5lyn7UkmtFrNRJzd8CrLXJnyFO9D0qKxW4zt0xl3LJiAcX27AHCKLrUttTeTLO0W9jRvnWWZCfXGEjeziX6os7tFYe+uVah1DaoLQmeN8BPJ8v3ohKuv3cJ1JKnr3tli2y3aBDwtnB3bck2XF2htbcKyYRgXVkq80D9PdWx4cW5dPQ6CTqsJTTCpOu4BhkhmTzLD5I+KZBw3Hb8nDh/XC9c+/RYueXyNVZGKYYqFksgkW4OGzP9tu8UurVdXlixzrCNFdYtUmT3HNM3tc5WDRvXEQaN6euZxCyC/TLIqpBIxwzqiWiMktRUJQ8y2CWedZCHg/ilS7RbS9hAjbwbd2XHPyCTHY+Ro4lCZDJlJVgRKr86VVnkyt+icOdSuaKGuslwpAecW83Y2XM0kuxtNyJFnGpGcIpPsEGmuQXLHTuiDA0bsFiiy/dB5jZ2vOx/VbTtLwKmxOtehvi/LblFuHB9f7XQe94CdiXZnedTj3LJbkFF1IhknNLd6B4dKUtXQ9sSgySS7L4516Kwf6ufiL5LjbLdgmDyTjMdw1ZGj0aWqDLc9vwGfb9+FK48aHWh1Y5hCUhIi2Wu3ME74W79uRlki5jnBSmGp0urKrLozyX52C52QcgiDEFpAzm+d0Mk53Y0tlIwLAD97RixG6FKVxCdf7fLYLSDct+TJujXdrJRPc2fQrY57ZNotzIF7ag1gqy11ih86dfs15Qkr+xskTnV1gd2eaMCugR1ot7CqW6Q/cC/m8xnL5brXlHsXCoFO/KpYrddddx3c+0Dnc7ZeU4ShOnAP8HrxAfvYd4ckRWqbsLtbyrhqK5L4bNsuXwGb6gLKjc6TLNfRv77adzln1z7jeVi7BYtkhsk/sRjhJ/OGoWt1Ga588k1s/boZNx63p6e5EMNEQUlcrg3qUYPT9xlgeZHLE3FLqOl8tYk4eUSIp5mIZ+CeftvudsaAU1CHyZi5M8d+HlD3/HKQlC2YvAt0NStcdHK1pW5zlYADbEGxS2npXF2WcGTeKpK2J1lmIZMxctToratMoktVEr26VAa+b7n8waN7mgMp5eCvcOJU9ST7WTfU9+hpS60M3Gtx9Z6Okb7ygcQ96NGanp728yA/S7/VuNcv43B3XHQMJnStQ413p1nSTTZA+Wqn12LgF5P0kAthN1yR+1jWK/aLKd1MsnfQIFBdnsAdCybgjm9P9F3OcbFgfpypOu4BRrZ5p6a7IMMwuYeI8P2ZA/H/DhuBZ9/agm/d9iK2ft0cdVgMUxoieVhjJ/xw7lBH9q6zaS8wbv8650/EndnleSMbccURo1zVLUKWgNNM9xu45IfXbuGc7rdNKZDt/73zSn92bYqBe2rczUpliGMn9sGDp0625lEtDlY5sTh5Mskv/mhfHDyqMfB9T+zXFT+ZNwyXHz4SgF2RIqzNIbCZSJvMhsMzv/v/NuEtfaa2pdbHoY8p24EnbsuN53WNWASM90LK2/PruAc4j095F6ZGZpJ1dgvr+HTt41a7hbnqSQbgaerhjiNV5ZOgZdVYZg7tYV0I6nAOijSep6qTDBiNR9yNVRiGyS/H7bU7rv/GOKx8/3Mc8/tl2PzFjqhDYjo4JSGSdciMp67yQdI17dpjxqCpe7XjROwRyT7CTTc96Fa3DmvgXshMsmrPiJO+PJqka3UZasoTjpisEnCaDDugNEIhQm1FEqP7dLbmsferPegrGYs5PMnxmDHoLZVgjMcIJ0/rj+ryhNnBz78Bi0RfJ9lrC2nVVLdwf1ZSQAnhHMQpYwiyi/hZatJMkHqwa2D7ve6KwxTNibhTPgddqKnC8YFTJ+OMmQNRY1YPcd9BARS7hWu63GcCsO0W5rxSdPt0qM4gk+wkbPc+3V2JVG2pAdNuwXWSGabgzBvViNsXTMC7n23HkTcvw7ufbo86JKYDU7Ii2boVrxm45741LU/YanvrVkU0yYFxOnQne11JryDIJY6D7BPq9Lg5WM+deVYXG9bYCQN71DjibG0zmokQgEdOn4Lz5wx2vJddrd4srERmkuNkt/uNu+wWmQy6INgZ7EBPsvJSuSKS/WouB3msZulPAAAgAElEQVSS1ThbXFnDOJGjCoI3Xr34Trf1shvblh782be57CRuT7Ij0+1ah2oRGtGrDj/Yf4i1L7YF2S1cK2qx7BbOOskAUJvCbpFudQvPoMGQu9k5mNG+AyPxE+tc3YJhomPaoHr86eS98MWOZhxx8wtY/cHWqENiOiilK5IT9gnRfXJPxmPaTKDf3VU/q4Xfa6mqW7ixbrG7BmP5LWk3ECGHQNRllM+aPQh//f7eDlHRJozsKRFhXN8uOHP2IMd7kSXgdBcGaobe8iTHCeVJ226RbpbQ2JadmQxaXt9MRFMnWddMxKctNWCLakkqu0XMx9qQzluv0gxMCXsXQV7DqW2sHRd+GnFqD/Lzfu1l1lwO3NtnSL2yvHNbErnPVE+y3JYcCOgn3DM5RlTCZ5K9n03Q91nCA/cYJlrG9u2CB0+djDgRDr1hKa74xzrtwGKGySclK5KlEIgReTJCsi218bp98lczySqBt/9TZZLTGrjn/N+/woHxKDPJqmgG9OLa22yjzderajfi8K7H4Uk296tR3UIpq5WpSA5R3ULnMQ4euKdmDV2eZOX/5lb3wD3vOh1xKHvZz5+ciud+OBPPnD/DuV7XXQXvdg1kET9L+MbJV4Ra2WZXO3SVMvOCYNvOVtSWJ3DntyeiUhmkqW7LjXHR5RbJSfM19Y6Mf1Y/Fe79EXY3J5TYZSRhkthcAo5homdQQy0WnjUV88f0wo3PvoM51zyHp1Z/ZP3eMEy+KVmRXGYJAu8tZNWioAoGty/Vmj/NW+jOLl+p53fHkk51C2d7XufrumUkbcIbm8ysqtUt3FjVLdSBezGyphvrycRuQdo6yW6cmWSZRfXWNG7VVLdwZ4bl3QbA+9kbdyDCiXWdIA1D95pyDKivcUyzs7Z+25UXdOb/5vREzFmyThe73Ec6gSovKrftanF4vY2YjMdenau0MQkI6y6MVd3CzCRvUwYCqltN25Psmj1dT3IiFtP61P0Y2KMGQ3arTStGhmFyT/eaclx11Gg8+L3JqClP4JR7XsFJdy1nrzJTEEpWJEsBFCPy3KIx7BbejJ1fJjndE7qqxcIN3HNmji37ha8n2Xx02y0CxLXuPXg7sZl1klu8VgVJmZW9hTWwKRl3epKztVsEVT7QDdzT+c51nmTvwD075kNG93S8lkrnx3w+4+w9ycGZZDndnblNxGOIx8iycMQ1dhApjnWfTzIhPcktyn6F43FYo140trWpJeCMabK6xZc77O+ew/pSoIF78lhKxMnjmw7i3DmDccM3x6UVY3uAiCqI6CUiWklEq4noEs08C4hoCxGtMP9OjiJWhlGZ0NQVC8+aip/MG4YX//Mp5ly7BL995m0eO8DkldIVycrAPXftV7WZiKPqg08mOV3NF3dk9FLPTy4xIv/3W1RtSy0tF0DwgD/de/Bkks0Jza1eq4K1jCK0dlkD7ZzNRDKyW0AtPRckku3n5QFtqVvbNJ5k1zzqwLwRveqw8fJ5djwpPzh99jhLq22ITLLxKDyeZDODa4pTXe3mZIDdImlO27az1RbJLlE9rLGTNiYB4alu0cnMJKuWBQq4YEmFN5Mcbjm5nUQsPZFcwuwEMEsIMRrAGABziWiSZr77hRBjzL9bCxsiw+hJxmM4eVp/PHP+Pth3eAOuWfQW5l73Lzz31paoQ2NKlJIXyaq1QqIOclLFRKuf3SLtTHJ6mUV3LFa20Gd+uy11zOHHDfKOakWfa5rtSQ4jVu2Be4kYWZ34gPRr4MpY5P4Pqnygq5Ns7APnfC2aW+tuG0hQnOpnuEdPrzjMVybZupvgc4kkp0pPsu03dopk0ohkaYEIzCTvarGtSq47HL4iWdh3Ydwl4FTUzaZbT9o9f9jlLYtJPGbFmK59qpQQBl+Z/ybNPzZ4Mu2K3eoqcMM3x+Gek4xGQifc/hK+/6dX8eHWryOOjCk1Slgk24Lx1OkDrDJngBy4ZzxXB9blym6ha2AQhPsWuzyJ+w4ktLLgzkYaQZ5kHe65rI57AXYLNWaZJUzEY6hwZJIzKwEnB+4Fe5Lt50HVLZo1Zez82lLrkDG88pN98dD3pnjj9RHG2TYTcfvLPa/H5LFh/C8/q927Gq2Zq8qdg+3UdcqKE3pPsswkt3gzyebyfk07hIAnS1tTnvTOmMW+cS/JmeTMIaI4Ea0AsBnAIiHEi5rZjiCiVUT0EBH1KXCIDBOKaYPq8Y9zpuH8OYPx9NqPMfvqJbjluXc8A7EZJlNKWCTbJ/rKsrhV5sx4zRZVziYb+nWlmx101mFNPb8t2I1H+5Z6sGiPxwjDGjthj151jjjDhqurHw3o/by6GOzMrzOTnKknublNiu7gDLZEvVvgtVuk9iQng9pfm/N2qylHpaZUm7omR4bUd43hSOVHl5PlBdRJU/vhogOG4oojRwEAqpKy9Jo3plpTuGozyUqLbtuqBM/8T54zHXcsmOBYVjanAez9puu4l4029dZJTjOTHCM0dKoAAFSVeWPrSAghWoUQYwD0BjCRiEa4ZnkcQJMQYhSARQDu0q2HiE4houVEtHzLFr7dzURDecI4vz993gxM7t8Nv3piHeb99l94acNnUYfGlAAle7ZIum4ZqyRi+oF7ObNbOKodpF7WLYyshhE+ol2N/SxF/Lu9yWG3K5EZYF0W1o3D6+sauJcJBNtnG7a6haPjnivYFm11C2eMQfsp1S15h8XCkbUNXCwlttdY/7plwzDnG92ns6MjohT0jvfmySQHd6ErS7jtFvZ8Q3ar9Xw+AnaNcbnM+KYuOGrP3qitSOL2pRucsWdA1tUt4jFcfsQozBnegOEa+0xHRAjxOREtBjAXwBvK9E+V2W4FcIXP8rcAuAUAxo8fz5YNJlL6dK3CbQsmYNGaj3HxY6tx9O+X4fBxvfCjA4ehe0151OEx7ZTSzyRrTqZxxcOq6iZ/e0PmIjlcMxFnVtsWycHxuMVKqsoI3vW4/jf3hW4A3e7dnOW/3N3T1IF7mRC2hq761uoqE6gqi6NHbYW3TnKI6hZBpHKMOAfr6QVzJtheY/165Or9FEl1uV2iz71Mp8qgTLI9rdxn4J5EbUEOAKfesxw7mlsd20rGY7jyqNHoV1/tiSMT3IuG/UranmRCXWUSh4/rnXkQJQAR1RNRZ/N5JYA5ANa55mlU/j0EwNrCRcgw2TFneAOePm8Gvj9zAB5f+QFmXfUs7lm20TcJxjBBlGwmWTZH0ImWZNwWdXkZuJdmZtGvBFybj63Kz3tsrSdknG4hJjOMVp1kRQs9fd4Mh2hX32MiTh7hlC5O+0JAJlnZbk15Ei9cOAudKpLY8tVOx3y6hii7dapAbXkCX+50lgQcqqmHmyqTrL7srCSRnUiWh6CvJ1leQPkcq5VJTac78zHIk+zIJLsuMN37olfnSvzowKGoTMbx00dX4+MvdmLT58aAGff3Tf0vm12TeSbZtluovPrTOdYA1Q5GI4C7iCgOI0nygBBiIRFdCmC5EOIxAGcR0SEAWgB8BmBBZNEyTAZUlsVxwf5DcdjY3vj5Y2/gp4+uxgPL38cvDx3huPPGMKlIKZLNQRt3A2iAkcC6RQjxG9c8FwA4TlnnMAD1QojPiGgjgC8BtAJoEUKMz134/tjdxXSv2aJOPam7RfLQ3Wqx7qMvsxTJqZe1vJ/m4wEjGrHuoy/RvVY/UEqKFndc8v+wQs09m+VJbjGzmYrE8VgVXJnf7DPJ9vNgT7LzeeeqMk88gN2WWt0Xh4zuiVnDemDUxU9Z0/5xzjQ01lV6tpPqM9dVj3DHlwnSh+5rt5AXUD5JEVknWZfprtVUvpBo7RY+xxMR4ZTpA7Dyvc+taTKT7BbU6r/ZXEC4rRphx4ZadZJdC/gNQix1hBCrAIzVTP+Z8vwiABcVMi6GyQcDe9TgjyfthcdXfYhfLlyDQ29cim9O7IsL9h9inTsYJogwp5oWAOcLIYYDmATg+0Q0XJ1BCHGlrKkJ48d1iRBCdc3PNF8viEAGUnmSyarG4By451Qeo3rLAXHpbTvzttTG45mzBmLlz/ZDj9oK7fzu2rj2epyPqXALD7k+mWEL9CQrG0nGY5awyhQ1liC7hV8lCb+21G4rRKcKZ9WFobt1Ql2ltxJDKkHnl/nOxncL2J7kVBdXwsdwIatb7GpV6xMbx0S1KZJ3aUZ+qxcm7mYifoNPR/Wuw2WHjwQA7GiWdx8CrDK+r6Qm20yyu9siwzAdAyLCIaN74pnzZ+DbU/rhvpffw6yrl+DB5e/53pFjGElKZSOE+FAI8ar5/EsY/rReAYt8A8C9uQkvc6TdQtueNx5ztFeWuDPJI3sbt2XSzSSnO5BLhmANyIsR6qo0JbRc63QLkqBmIkHblaRV3UK1G2iqS6SLw74QWCfZuV07HrdITl3GLohU78fP65xtJlleqPmtxu64p39dVrfYrjTQIRgDG2Xt4h27vB2qAu0WPm+KiNDQyRgQI7teuWdVLxqyqSHtrW4RbrmgLoMMw3QcaiuS+NnBw/H4GVPRr3s1LnhoFY7+/TKs/fCLqENjipi00n9E1ATjVp2uriaIqArGSOmHlckCwFNE9AoRnRKw7pyWE5InfZ2WUEuWOeskO+drMger6U7u35jYF3sP7KbddqZ1ksOUi3PM7y7hZmWk9cut/38H4Mbj7Fa77vlkNtGukxyQ0XVlkrPFYbcImUl2WC9cIbS2ei0j6ZCOJ9lpAclOjMljMOXAPR+VLAfubVeEsJFJJqss2w5NG1fHwL1ksN3CuZwxr6yZ7RajTruF72pS4l40rHVDrW7BMAwzvGcnPHjqZFxxxCj855NtOOh3z+MXC9fgK9dYFYYB0hi4R0Q1MMTvOUIIv0uvgwEsdVktpgohNhFRDwCLiGidEOI594K5LickO4ipt1MaOpXj4y92GnYLM5OccIhkY95L5++B+ppyy8eoy0LJ28w6dC2Bg0i7KoVPqTf3AEA3iXjM8X7dAlK+z+27jB+LoLJujlJsuRDJSixhS8A5K2I4Y2jWlIBLh3SqW/j5kzNBiOAsvtyW311CWQJOfoYyJiKg2qwP/HWKTHJX06tnVV0JIZLt6hYukaw8zy6T7Pw/3TrJbLdgGEYSixGOntAH++3RgCuefBO3L92Ahas+wE8PGo55IxuzHoDNlA6h1A0RJWEI5D8JIR4JmPVYuKwWQohN5uNmAH8BMDGzUNPD9tfaauKh703BFUeOcnSI09VJHtunCw4Y2eg7Mj4VmQ7cC1s+zB7o5yek/JdVvafu+aTQlFfU8kJCh7rtZCL7H5SwmWRdaTNAk0mWnuQ0P7tu1fqBgG78RH22meRUdZLtEnA+nuQyTSYZZIjkgEyyus/loDapm4N2oZVJbs53Jjkzu4XdeIczyQzDOOlcVYZfHTYSj5w2Bd1rynHGn1/Dt257Ce9s+Sr1wkyHIOWZg4xLqtsArBVCXBMwXx2AGQAeVaZVE1GtfA5gPyhF6/NJmSaT3KdrFY4eb3RYrUh6s8S2sILjtXSFlipEQ3khpUgOm0n2sWe0pchCul9zXy1LASpFcmVZ6o50QO7tFoGDv5RNBXmS7VrP6cXx8GlTcMkhe6R8T36iPutMMryl63Qb9q9uYXqSFZEcixmfuywBJwfZOVeriGSz8L5l3wnYiWWW3UKfSfbbRrpknUlmTzLDMD6M7dsFj50xFZfO3wMr3/8cc697Dlc9+ab2rhvTsQhjt9gbwLcAvE5EK8xpPwLQFwCEEDeb0w4D8JQQYpuybAOAv5gnxwSAPwsh/pGLwFMhRU6rj3dTZknVjJwUmWrbZyC1P9WNn2821fxh7wj7iXdbJKdeFvD6PONmANt2SruFfyZZ3UZQe+ewZFLdIqhRiK6ZSBiaulejqXt1yvnU9daUJ1AWj2FXa1vOPMl+67E9yfrlBzcYNZ+nDLD98nLgnpVJTvHDL7PpoewW5l0Eq7qFa1b1c81mz7iXDbub3d9lhmEYHfEY4YTJTThgRCMue2Itrl+8Hn9dsQkXH7wH9h3eEHV4TESkFMlCiOcR4vwmhLgTwJ2uaf8BMDrD2LJCimS/Ei8ykyzFFGBnkqUoSGSaSVZEY5jsWbqeZDmbW0ymElhAcCUGub5tZmWEQLuFmknOsd0i2JOsX4bIsBRI8diapSc5FepqiYC9B3bD4je3WIMeM8UqQ+gTtxSdfgP3+nWvxis/2ddRB9jyJJuD+nR2CxXbbpH6uJTH+o6WVsRIc7yrdwiIcOLk3fF1c/rZmYwzyeaFXy7udjAMU/rU15bjmmPG4OgJffDTv76Bk+9ejn2H9cDPD94DfbpWpV4BU1KU7JlDDtRJlUlWa8ZaItMlDtLNJOffk6wXL7Ic14K9m/xjC7JbSJG8S3qSww3cy7UAcQ/C89uu+/2r761Z1nrOsm5xmDgIhFlDewAAPvpiR1brTVUnOVUmGQC61ZS7BhOSo7pFc2vAwrAzyenYLXY0t2ovbtwXE5fMH4Erjkz/utlbAi7c52pXt+BMMsMw4ZnUvxueOHsafnTgULzwzqeYc+0SXP/Pty1rGdMxKNm21Jbdwi+TbFoJ1Exy95oybPhkm1XVQZ5Y892W2qqPHHIzdpMH5wK1FUlsvHxexrFZnuQdpic5ZCZZZqDv+s7EjIuzBzUGcc5nP/e05Y6RdaVji82MwkmNmsWOGSUBv25uxTET+ma1WpHCMrPv8AY0PvM2TprWL/Q6yVyf/Dy7BNTgBoAult3C+D/oGkjeRdjZ0pbyrkk2n4U3kxxuOa6TzDBMpiTjMZwyfQAOGtUTv1i4Blc99RYeXfEBLj9iJPbcvWvU4TEFoORFsp9ok7Vgm5VM8o3H7YnFb25G7y7O+sjp2i0yziSnWQIukxO/471oRGaMwlW3kKtJxskSRzMG16cdjxWK8jyTOsmAPuOfrUfYPw77OcEor3fK9AFZr9eqk+yTAe9eU45lF81Oa50xMvYDEeH6b47FqF6dA+e3ulWGsFuoJeB0+98pnDP/LNz7I+wgQHvgXsneNGMYJs/07FyJm47fE4vXbcaP//I6jrx5Gb41aXdcsP8Q1FYEJx2Y9k3JnjlS2S3koDRVJNfXllvVLwAlC5XmuT19kZyeGE/Xw6ySCMgkG6/HrMYQgSLZqj+bm0MobHULuxa0v11EN79KqkxqGNx2hlyRjww4kX0hc9ConujbLZyvLj2R3JbSbhFJJtn88rLdgmGYbJk5tAeeOm8GTpzchHv+/V/MueY5LFrzcdRhMXmkZEVymWW30L8u/bZBA60yHRnvEKIh9rAtesOt389uEW5Zp5fWjT3QKbjVtHtwY7bItaRaX1DWXbcoafb/kh/OxEs/Ti8b61lviu1mitWWOofCW9ot0l6OUh//aiMZ7f533LjIJpPsJG1PMtstGIbJATXlCVx8yB545LQpqKtM4rt3L8f3//QqNn+Z3XgUpjgpWZFsddxLMXAvaBBTmExa0HJhl7W8n2FP/BkOKHTHpltcDu6qCCj/BtjvqyygK186hBFk6ny62cJmkjtVJNGjtiKDKPXrzeXgwANHNjoec4EcuJeKymQcA3vUWP/L5GtwMxH1glBnt7Cf5zaTnJ7dgttSMwyTS8b27YLHz5yKH+w3GIvWfox9r16C+19+17fyENM+KVlPstVxzyeVbIvk3GeS0x+4Jx/DbccSihmojlR2i9qKBDZ/uRMVZSlEshQfOfJ6hs0kA7bH1k2q2/25xJkhzd16h+xWm3LwZboQhTsO37hkf8f/YZrpxGN26b1Ux0I2+8nT7jrkYWd1zWS7BcMwOaYsEcMZswbhgJGNuOiR1/F/D7+Ov7y2CZcdPgr9QtTbZ4qfkk2vWAP3fC7qrDrJAdUYMu2458zWhvckhxXjcv5MbiGr70WXAZWDEILKvwGK3SJH4sOupBBuf+ntFuGm5YJ8ieR8QAh3HMZjpD12U3XRk9+1Ks2FlaOZSA53VNqZZLZbMAyTJwbU1+C+707CZYePxOoPvsD+1z2HGxavD0zCMe2DkhXJ0gaQqgRcEJnaGtT5wyybqSc5k0yys06y9/VOlaZITmm3MNeXM/ER/ra4IZK903VCKH/NRPJjt8gHMaJQ3ng3YY9/2fJZVzLQcTGRfgjKepxLhz3s4jm+48EwDKMjFiN8Y2JfPHPeDMwe2gNXPvkmDv7d81jx3udRh8ZkQcmeOcK2pQ4inmmdZNWnmY9mIrHMxDuQOstdW2F6klPsH3fDlWyRqwnr4dZmjUN6knNBu8ok++yvVIS9kyL9/yktOhEM3EvwwD2GYQpIj04VuOn4PXHLt/bE/7bvwuE3LsWlj6/BNrO0KtO+KGGRbJwUU7WlDsLqOJbmyT3hEKKp5w9zW1vFtmekFRYA5y1xnW7oZIrkoEYixrbTy36nIj1PMmn3q37gXpaBBcSge16MZCqS7QuX4PnkBWlliu9UNrvJvWzYdcWtEnAl+1PHMEwRst8eu2HReTPwzb364valG7Dftc9h8Zubow6LSZOSPXPkJJMcy0yMOgRUSJUWj1HozHC6zUdUqsvtsZpau4XpSS4P6UnOld0iPU+yfr5CNhMhn+fFSCIWy+hzioe8Y1FmiWSd3SI3FxOZZ5KN+ZI8cI9hmALTqSKJXx46Eg9+bzIqkjF8+46XcfZ9r+HTr3ZGHRoTkpKtbiFFsl81lvIQpctyUic55KK/OmwEJjSFa3OZ7kA/FfV96wfuJUKtOx17RBhkLGFKyvkO3CukJ7kd2S1OndEfn361K+3lwja5kQK0UjtwT/887VhcMfDAPYZh2gsTmrriibOn4cbF7+DGZ9fjube24CfzhuPwcb1yOqCZyT0lnElOJfJSH5iZ2i3SrZMMAMdM6Iv+9TWpZ4TiB87gxO/sFOd9XVa3aAmoHw1kfgHhH5fxGCbjR6T//FK3Rc4d+eq4lw/26FmH6Rm0DA/b2dG2W3ivuZ0XE7nMJIdbrmt1GX4ybxjmjshd3WmGYZh0KU/Ece6cwfjbWdPQr3s1zn9wJU64/SW8++n2qENjAihhkZz9W4uZNWDTFYK5usXshwwn2+xY0MC9XSlK18RznKGzRXKITHJMX91CXjSE9dJmiyHW87uNKJEfRaqPxBLJZd4ZnSXgMo+l3FVtJZ2a4idP64/d6rJrHsMwDJMLBjfU4qHvTcEv5u+B1979HPtdtwS3PPeOb08HJlo6tEg+bGwvXH3U6MB5ErFw3cr8yIdQs+wWGcYVJCKlJzlVfcdsGppo15d2CThNJtlcNGlWNMj3gDq/OEqF0JnkhL8nWbe+TJg3qhF3LJiQ8fIMwzDFQixG+NbkJiw6bzqmDuyOXz2xDofeuBRvbNoadWiMi5IVyWGyv9ceMwZH7Nk7cJ4YUVaWgrxkkrOwWwBAdZmRLQ7yJKcSyZlaUXwxV1MWwm6RauCebHCSbwFLKP5Be9kQtsxf6DrJWeys6vIEZg7tkfkKGIZhiozGukr84YTxuOGb4/DR1p2Yf8NSXPb3tfh6V2vUoTEmJSuSAWBEr0644ohRWa3juL12xz5D0vdzSvJpt8hUvMsycLrQaqTdoiWV3cJ8zNnAPYMwdwDIpwRczOWTzneS168UXakQtoJJmzk6VlcnWV2Sx84xDMM4ISLMG9WIZ86bgSPH9cbvl/wH+1/3HJau/yTq0BiUuEheeOY0HD2hT1br+NnBwzFtUOYimfKwh8PeBvdDimSdaJHez+YUA/dsu0VGIfiuL5zdQv/epaiTQjvvVgifAYSlQtgmN7KrZapMcmnn3RmGYTKnriqJXx85Cvd+dxLiMcJxt76I8x9Yif9tS78yEZM7SlokFwO5yrSqZJ9JllUIvMv36FQOAJg/pmfgOnJeJ9l8DGe3CB64lyhQJrnj2C2C52sJEMmO9ZXyzmIYhskBkwd0w9/PnobT9xmAR1dswr7XLMGjKzZB+NWzZfIKi+Q8kx+7RXYD94IyyZ0qklhz6f44a9agwHXEQ/pVwyJXkwiRmo4RabObhc4kdxi7RYo3aWWSNXYL9TMo5aw7wzBMrqhIxvHDuUPx2BlT0btLJc6+bwW+c+fL2PT511GH1uFgkZxn8qELsmkmAgBVZtc9P9FSVZZIeYvdnbXNFvmekiGaifi1WZYD9mTDlLxnkn3iKBXCDtyTmWRdF8vKFG3QGYZhGD3De3bCI6fvjZ8eNBwvbvgMc65Zgtuf32AlJpj8wyI5z+SnuoXxmKlIrjaFS6rBeYExZGn5cGMP3MvCbmHu6/Jk3PF/voiRrj5I6RC2416QJ1mdVtp7i2EYJvfEY4STpvbDU+dOx8R+XXHpwjU4/KYXsO6jL6IOrUPAIjnP5LVOcpae5G27WjKOQd6Cz9ktdHM1yVB2C5+BezFnJjnvzURQ2haCsM1EWtqMiy2d3ULNLpfwrmIYhskrvbtU4Y4FE/CbY8fgvc+2Y95vn8cvFq7BFzuaow6tpGGRnGfykc3cZ3AP/GC/wejfvTqj5avLDeGyfWfmIlne7MlZxz1Iu0W4TLJOnMp9XZEsjCe51Dvuha2iIluYazPJZSySGYZhcgERYf6YXnjmvBk4enwf3L50A2ZdtQSPvPo+D+zLEyyS80yuOtKp1FUlccasQRmvWwqXbVkULJc+1Jx13JOZ5FB1kvXZTTlNlrHLvye5tA0EYUWyLBeo8yRXlbHdgmEYJpd0qS7DZYePxF9P3xu9ulTivAdW4ujfL8OaD9iCkWtYJHdAZMe9bLr6yKvWnFW3MB/DiGT/ttTOTHK+rRBU8nWSw9l6WgPsFmp2OVc1tRmGYRhgdJ/O+MtpU3D54SOxfvNXOOh3/8LFj63G1q/ZgpEr+LTVATlsbC/UlCdw2LheGa9DDtYKMc4uFHYmORd2C/8Sd7nEbwBhqSCvV7Kpk1zBA/cYhmHyRixGOHZiXyz+wT44bq/dcfeyjZh11bN4YPl7aOMqGFnDIrkD0qdrFd64ZH8MqK/JeB25tltIwtotdJv1DrX8cRYAABZJSURBVNzLcyYZJZ5JDlkCTloqyjTl+8qVabk4VIY01Ga/EoZhmBKjc1UZfnHoCDx2xlQ0da/GDx9ahSNufgFvbNoadWjtmkTqWRjGi7Rb5KqjoBTdWdktZAk405Oc73EMRJT/jURIWLvFn787CU+v+Rg15d6fE/UiIheHyqNn7I2dzZmXLmQYhillRvSqw4OnTsYjr23C5X9fi4Ovfx7fnNgXF+w/BJ2ryqIOr93BIpnJiFZTp+SqTnKrJZJD2C1ienEec3mSW/J8q8kIoXQzyfGQA/cG1NdgwIzUdyVykXWvSMa1AwQZhmEYg1iMcOSevTFneAOue/ot3L3sv3ji9Q/xw7lDccz4PnkpKFCqsN2CyYhWmUnO0ZdNlhEL25Zap7fiLk+yrN+bLwy7RV43ESmW3SLHDWMYhmGY/FNXmcTPD94DC8+cikE9anHRI6/jsBuXYsV7n0cdWruBRTKTEXJAQK5EcrOZmg7TltrPbmFnkk2R3JrfTHLpd9xzPma/vlLeWwzDMMXJsMZOuP/USbjumDH4YOsOHHbjUlz48Cp8tm1X1KEVPSySmYzoZzYyGdGzLifrk3aLslDVLfTlxOw6ydJukedMsk/nv1JBXgDlyndewruKYRimqCEiHDq2F/55/gycPLUfHnrlfcy86lnc8+//WudfxguLZCYjpg+ux1PnTsdR43vnZH3NbeHtFidOacJxe+3umS6XlSI53198P9tHqWA1E8lRKrmULygYhmHaA7UVSfx43nD8/expGN7YCT/96xs45Prn8cp//xd1aEUJi2QmYwY31OasBJpsSBHGbjF/TC8cOLLRM12KsHLTbtGcZ7sFUNo+27Ad9xiGYZj2xaCGWvz5u3vh+m+Oxadf7cIRN72AHzy4Ep98tTPq0IoKFslMUSAFbRi7hR/SblGo6gel3nFP7s8QVflCwSOqGYZhigciwkGjeuKZ82fgezMG4NEVmzDzqmdx59INaGnlUpsAi2SmSJBfyDB2Cz+sgXshstG5oMPYLXLcepxhGIYpHqrLE7jwgKH4xznTMaZPZ1z8+Boc9Lvn8dKGz6IOLXJYJDNFgVUnOQuBG3fZLfJNqQ/cy5VItqtklO6+YhiGae8MqK/B3d+ZiJuPH4cvd7Tg6N8vw7n3r8DmL3ZEHVpksEhmigJpt0hmcUve3ZY635R6JtmqbpGlTULaX0p5XzEMw5QCRIS5Ixrx9HkzcOasgfjbqg8x6+oluPVf/7FKtXYkWCTniZuOG4ezZw+KOox2Q0saA/f8GN27M6YPri+YSCaUtoXAaiaSpbq1RHLWETEMwzCFoLIsjvP3G4Knzp2OCU1d8Mu/rcW83/4Ly975NOrQCgqL5DxxwMhGnDtncNRhtBtarLbUmR+S+w5vwN3fmZjVOtKixO0W0r6SbSZZXrQ0cy1OhmGYdkVT92rcvmACbj1hPL5ubsU3/vBvnHnva/hoa8ewYLBIZooCuy119qKzrIB2i1JOj04b3B3nzRmMgT1qslpP365VAOwujQzDMEz7gYiw7/AGLDp3Bs7ZdxCeWv0RZl39LG5e8g52tZS2BYNFMlMUWB33ciBwywqUSS51u0WniiTOmj0o60zyTcfviauOGo0+plhmGIZh2h8VyTjO2Xcwnj5vBqYM6I7L/74Oc3/zHBau+qBk/coskpmiQH7BcmGVyMbXnA4xopK2W+SKrtVlOHLP3HRmZBiGYaKlT9cq3HrieNyxYAIggDP+/BqmX7EYNyxej8+27Yo6vJySiDoAhgFsT3JO7BaFyiSXttuCYRiGYXyZObQHpg+ux7NvbsadL2zElU++id888zYOHdMTJ05pwh4966IOMWtYJDNFgWwm0q7sFkQgttkyDMMwHZR4jDB7WANmD2vA2x9/ibuWbcTDr2zCA8vfx8R+XfHtKU2YM7wBiUINqM8xLJKZoiCXmeRkojD5XQLX/mUYhmEYABjUUItfHjoSF+w/FA8ufw93LduI0/70KnrWVeBbk5tw7IQ+6FJdFnWYadE+pT1TcrTkoOOepKB2C1bJDMMwDGNRV5nEydP649kfzMQfThiPfvXV+PU/1mHSZc/gwodXYe2HX0QdYmg4k8wUBVZ1ixwI3GyrMYQlRgQh2G/BMAzDMG7iMcKc4Q2YM7wBb370Je58YSP+8tr7uO/l9zCpf1csmNIPc4Y3FOycnQkpFQkR9SGixUS0hohWE9HZmnn2IaKtRLTC/PuZ8tpcInqTiNYT0YW5fgNMadCaQ7tFobK7RECM78UwDMMwTCBDdqvFZYePxL8vmo2LDhiK9z77Gt/74yuYfsVi3PLcO9i6vTnqELWEySS3ADhfCPEqEdUCeIWIFgkh1rjm+5cQ4iB1AhHFAdwAYA6A9wG8TESPaZZlOjiVyTi+bm4t6itKNzxwj2EYhmHC07mqDKfOGICTpvbD02s3484XNuBXT6zDtYvexmHjemHBlCYMbqiNOkyLlCJZCPEhgA/N518S0VoAvQCEEboTAawXQvwHAIjoPgDzQy7LdCAeO2NvLF3/Sbvy+PLAPYZhGIZJn0Q8hrkjdsPcEbth7Ydf4K4XNuLhV97Hn198F3sP7IYFU/ph1tAekSfO0vIkE1ETgLEAXtS8PJmIVgL4AMAPhBCrYYjp95R53gewV0aRMiXNoIZaDCqiq8cwGHWSWSUzDMMwTKYMa+yEy48Yhf+bOxT3vvwu7ln2X3z37uXo07USJ05uwlHj+6CuMhlJbKEdlURUA+BhAOcIIdxDE18FsLsQYjSA3wH4a7qBENEpRLSciJZv2bIl3cUZpuDEiCUywzAMw+SCLtVlOH2fgfjXD2fixuPGobFTJX75t7WY9Ktn8JO/vo71m78seEyhMslElIQhkP8khHjE/boqmoUQTxDRjUTUHcAmAH2UWXub0zwIIW4BcAsAjB8/np2eTNHDdguGYRiGyS2JeAwHjmzEgSMb8camrbjrhY14YPn7+OO/38W0Qd3x7b2bsM/gHogVwIoRproFAbgNwFohxDU+8+xmzgcimmiu91MALwMYRET9iKgMwLEAHstV8AwTJTEixFglMwzDMExeGNGrDlceNRrLLpyFC/Yfgrc//grfuXM5Zl79LG5/fgO+2JHfqhhhMsl7A/gWgNeJaIU57UcA+gKAEOJmAEcCOI2IWgB8DeBYYRSQbSGiMwA8CSAO4HbTq8wweeWFC2ehPAeNSQIhcHULhmEYhskz3WrK8f2ZA3HK9P54cvVHuGPpRly6cA2ufupNHLlnb5wwpQkD6mtyvt0w1S2eB4Ktl0KI6wFc7/PaEwCeyCg6hsmQnp0r874NtlswDMMwTOFIxmM4aFRPHDSqJ1a9/znufGEj7n3pPdy17L+YMbgelx0+Mqfnf+64xzAZcvi4XmhtizoKhmEYhul4jOrdGdccPQYXHTAM9770Lp54/UN0rS7L6TZYJDNMhhwzoW/UITAMwzBMh6a+thxnzR6EM2cNzHmvBW6qyzAMwzAMw7Rr8tGMjEUywzAMwzAMw7hgkcwwDMMwDMMwLlgkMwzDMAzDMIwLFskMwzAMwzAM44JFMsMwDMMwDMO4YJHMMAzTQSCiCiJ6iYhWEtFqIrpEM085Ed1PROuJ6EUiaip8pAzDMNHDIplhGKbjsBPALCHEaABjAMwlokmueU4C8D8hxEAA1wL4dYFjZBiGKQpYJDMMw3QQhMFX5r9J80+4ZpsP4C7z+UMAZlM+CpAyDMMUOSySGYZhOhBEFCeiFQA2A1gkhHjRNUsvAO8BgBCiBcBWAN006zmFiJYT0fItW7bkO2yGYZiCwyKZYRimAyGEaBVCjAHQG8BEIhqR4XpuEUKMF0KMr6+vz22QDMMwRQCLZIZhmA6IEOJzAIsBzHW9tAlAHwAgogSAOgCfFjY6hmGY6GGRzDAM00Egonoi6mw+rwQwB8A612yPATjRfH4kgH8KIdy+ZYZhmJInEXUADMMwTMFoBHAXEcVhJEkeEEIsJKJLASwXQjwG4DYA9xDRegCfATg2unAZhmGig0UywzBMB0EIsQrAWM30nynPdwA4qpBxMQzDFCNst2AYhmEYhmEYFyySGYZhGIZhGMYFi2SGYRiGYRiGcUHFOGiZiLYA+G+ai3UH8EkewsknHHNhaG8xt7d4AY5ZZXchRIcqHJzhbzZQPMdNscQBFE8sxRIHwLHoKJY4gOKJJdM4fH+zi1IkZwIRLRdCjI86jnTgmAtDe4u5vcULcMxMZhTLZ1AscQDFE0uxxAFwLMUcB1A8seQjDrZbMAzDMAzDMIwLFskMwzAMwzAM46KURPItUQeQARxzYWhvMbe3eAGOmcmMYvkMiiUOoHhiKZY4AI5FR7HEARRPLDmPo2Q8yQzDMAzDMAyTK0opk8wwDMMwDMMwOaEkRDIRzSWiN4loPRFdGHU8fhDRRiJ6nYhWENFyc1pXIlpERG+bj10ijvF2ItpMRG8o07QxksFvzf2+iojGFUm8FxPRJnM/ryCiA5XXLjLjfZOI9i90vGYMfYhoMRGtIaLVRHS2Ob2Y97NfzEW5r4mogoheIqKVZryXmNP7EdGLZlz3E1GZOb3c/H+9+XpTIePtaBTLb7bu9yOiOLTfr4hi0X53IownTkSvEdHCiOPwnL8jjKUzET1EROuIaC0RTY4ghiHK7/4KIvqCiM4pdBxKPOeax+sbRHQvEVXkZMVCiHb9ByAO4B0A/QGUAVgJYHjUcfnEuhFAd9e0KwBcaD6/EMCvI45xOoBxAN5IFSOAAwH8HQABmATgxSKJ92IAP9DMO9w8PsoB9DOPm3gEMTcCGGc+rwXwlhlbMe9nv5iLcl+b+6rGfJ4E8KK57x4AcKw5/WYAp5nPTwdws/n8WAD3F3ofd5S/YvrN1v1+RBSH9vsVUSza706E++Y8AH8GsDDiz8hz/o4wlrsAnGw+LwPQOeJ44gA+glFvOIrt9wKwAUCl+f8DABbkYt2lkEmeCGC9EOI/QohdAO4DMD/imNJhPowDHubjoRHGAiHEcwA+c032i3E+gLuFwb8BdCaixsJEauATrx/zAdwnhNgphNgAYD2M46egCCE+FEK8aj7/EsBaGF/yYt7PfjH7Eem+NvfVV+a/SfNPAJgF4CFzunsfy33/EIDZREQFCrejUTS/2Wn+fuQzjnS/X/mMxe+7U3CIqDeAeQBujWL7xQgR1cG4uLsNAIQQu4QQn0cbFWYDeEcIkUlDoVyRAFBJRAkAVQA+yMVKS0Ek9wLwnvL/+4joxyUEAsBTRPQKEZ1iTmsQQnxoPv8IQEM0oQXiF2Mx7/szTGvC7YqFpejiNW/rj4WRrWkX+9kVM1Ck+9q8TbsCwGYAi2BkLz8XQrRoYrLiNV/fCqBbIePtQER+bBQzmu9XFDE4vjtCiKhiuQ7ADwG0RbR9Fd35Owr6AdgC4A7ThnIrEVVHGA9g3H27N6qNCyE2AbgKwLsAPgSwVQjxVC7WXQoiuT0xVQgxDsABAL5PRNPVF4Vxn6Coy420hxgB3ARgAIAxML4wV0cbjh4iqgHwMIBzhBBfqK8V637WxFy0+1oI0SqEGAOgN4zs5dCIQ2KYQIJ+EwqJ+7tDRCMKHQMRHQRgsxDilUJv24fA83cBScCwCN0khBgLYBsMe14kmOM6DgHwYIQxdIFxN6ofgJ4Aqono+FysuxRE8iYAfZT/e5vTig7zagdCiM0A/gLjxP2xvHVuPm6OLkJf/GIsyn0vhPjY/JFvA/AH2Lf5iyZeIkrCOBn+SQjxiDm5qPezLub2sK/NW5GLAUyGYVVJaGKy4jVfrwPwaYFD7SgUzbFRTPj8JkSK8t2ZG8Hm9wZwCBFthGHJmUVEf4wgDgC+5+8oeB/A+0p2/yEYojkqDgDwqhDi4whj2BfABiHEFiFEM4BHAEzJxYpLQSS/DGCQOWq9DEba/7GIY/JARNVEVCufA9gPwBswYj3RnO1EAI9GE2EgfjE+BuAEMpgE4xbHh7oVFBKXX/cwGPsZMOI91qxk0A/AIAAvRRAfwfCTrRVCXKO8VLT72S/mYt3XRFRPRJ3N55UA5sDweS4GcKQ5m3sfy31/JIB/mtl8Jve0i9/sQhLwmxBFLLrvzrpCxyGEuEgI0VsI0QTjGPmnECIn2cF0CTh/FxwhxEcA3iOiIeak2QDWRBGLyTcQodXC5F0Ak4ioyvwuzYbxe589uRj9F/UfjNH/b8HwHP446nh8YuwPYxT3SgCrZZwwfI/PAHgbwNMAukYc570wbps3w7hiPckvRhijoG8w9/vrAMYXSbz3mPGsgnHybVTm/7EZ75sADohoH0+FYaVYBWCF+Xdgke9nv5iLcl8DGAXgNTOuNwD8zJzeH4ZYXw/j9mC5Ob3C/H+9+Xr/KI6NjvJXLL/Zut+PiOLQfr8iikX73Yn4eNkHEVa38Dt/RxjPGADLzc/orwC6RBRHNYw7bnVFcIxcAuNi7g3zvFSei/Vyxz2GYRiGYRiGcVEKdguGYRiGYRiGySkskhmGYRiGYRjGBYtkhmEYhmEYhnHBIplhGIZhGIZhXLBIZhiGYRiGYRgXLJKZdgsRtRLRCiJaSUSvElFg8XAi6kxEp4dY77NEND53kTIMwzDKb7b8y1mnOCJqIqJIahczpUsi9SwMU7R8LYzWqSCi/QFcBmBGwPydAZwO4MYCxMYwDMM4sX6zGaY9wJlkplToBOB/AEBENUT0jJldfp2I5pvzXA5ggJnBuNKc9//MeVYS0eXK+o4iopeI6C0imlbYt8IwDNNxIKKNRHSF+Vv8EhENNKc3EdE/iWiV+Zve15zeQER/MX+3Vyp3EeNE9AciWk1ET5kdAxkmYziTzLRnKoloBYxuaY0AZpnTdwA4TAjxBRF1B/BvInoMwIUARijZ5wMAzAewlxBiOxF1VdadEEJMJKIDAfwcRm94hmEYJnPkb7bkMiHE/ebzrUKIkUR0AoDrABwE4HcA7hJC3EVE3wHwWwCHmo9LhBCHEVEcQA2ALgAGAfiGEOK7RPQAgCMA/LEwb40pRVgkM+0Z1W4xGcDdRDQCRhvnXxHRdABtAHoBaNAsvy+AO4QQ2wFACPGZ8toj5uMrAJryEz7DMEyHIshuca/yeK35fDKAw83n9wC4wnw+C8AJACCEaAWwlYi6ANgghJAinH+7maxhkcyUBEKIZWbWuB7AgebjnkKIZiLaCCPbnA47zcdW8PeEYRgm3wif5+mwU3neCoDtFkxWsCeZKQmIaCiAOIBPAdQB2GwK5JkAdjdn+xJArbLYIgDfJqIqcx2q3YJhGIYpHMcoj8vM5y8AONZ8fhyAf5nPnwFwGgAQUZyI6goVJNOx4AwZ055R/W0E4EQhRCsR/QnA40T0OoDlANYBgBDiUyJaapYJ+rsQ4gIiGgNgORHtAvAEgB9F8D4YhmE6Am5P8j+EELIMXBciWgUjG/wNc9qZAO4gogsAbAHwbXP62QBuIaKTYGSMTwPwYd6jZzocJESmdzUYhmEYhmGyw7TEjRdCfBJ1LAyjwnYLhmEYhmEYhnHBmWSGYRiGYRiGccGZZIZhGIZhGIZxwSKZYRiGYRiGYVywSGYYhmEYhmEYFyySGYZhGIZhGMYFi2SGYRiGYRiGccEimWEYhmEYhmFc/H8jUJGRag2ougAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 10 | Time: 1m 10s\n", - "\tTrain Loss: 2.998 | Train PPL: 20.040\n", - "\t Val. Loss: 4.710 | Val. PPL: 111.007\n" - ] - } - ], - "source": [ - "for epoch in range(N_EPOCHS):\n", - " \n", - " start_time = time.time()\n", - " \n", - " train_loss = train(model, train_iterator, optimizer, criterion, CLIP, train_history, valid_history)\n", - " valid_loss = evaluate(model, valid_iterator, criterion)\n", - " \n", - " end_time = time.time()\n", - " \n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut1-model.pt')\n", - " \n", - " train_history.append(train_loss)\n", - " valid_history.append(valid_loss)\n", - " print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Let's take a look at our network quality__:" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [], - "source": [ - "del utils" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "import utils\n", - "import imp\n", - "imp.reload(utils)\n", - "generate_translation = utils.generate_translation\n", - "remove_tech_tokens = utils.remove_tech_tokens\n", - "get_text = utils.get_text\n", - "flatten = utils.flatten" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "batch = next(iter(test_iterator))" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Original: there is a 24 - hour front desk at the property .\n", - "Generated: the property offers a 24 - hour front desk . .\n", - "\n", - "Original: this property also features free wifi .\n", - "Generated: free wifi access . . . .\n", - "\n" - ] - } - ], - "source": [ - "for idx in [1,2]:\n", - " src = batch.src[:, idx:idx+1]\n", - " trg = batch.trg[:, idx:idx+1]\n", - " generate_translation(src, trg, model, TRG.vocab)" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [], - "source": [ - "from nltk.translate.bleu_score import corpus_bleu\n", - "\n", - "# \"\"\" Estimates corpora-level BLEU score of model's translations given inp and reference out \"\"\"\n", - "# translations, _ = model.translate_lines(inp_lines, **flags)\n", - "# # Note: if you experience out-of-memory error, split input lines into batches and translate separately\n", - "# return corpus_bleu([[ref] for ref in out_lines], translations) * 100" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "59it [00:03, 18.87it/s]\n" - ] - } - ], - "source": [ - "original_text = []\n", - "generated_text = []\n", - "model.eval()\n", - "with torch.no_grad():\n", - "\n", - " for i, batch in tqdm.tqdm(enumerate(test_iterator)):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output.argmax(dim=-1)\n", - " \n", - " original_text.extend([get_text(x, TRG.vocab) for x in trg.cpu().numpy().T])\n", - " generated_text.extend([get_text(x, TRG.vocab) for x in output[1:].detach().cpu().numpy().T])\n", - "\n", - "# original_text = flatten(original_text)\n", - "# generated_text = flatten(generated_text)" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "14.139920232081806" - ] - }, - "execution_count": 111, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "corpus_bleu([[text] for text in original_text], generated_text) * 100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Baseline solution BLEU score is quite low. Try to achieve at least __24__ BLEU on the test set. \n", - "The checkpoints are:\n", - "\n", - "* __22__ - minimal score to submit the homework, 30% of points\n", - "\n", - "* __27__ - good score, 70% of points\n", - "\n", - "* __29__ - excellent score, 100% of points" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "homework.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Py3 Research", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/lab01_nlp/README.md b/homeworks/lab01_nlp/README.md deleted file mode 100644 index 3b94c95..0000000 --- a/homeworks/lab01_nlp/README.md +++ /dev/null @@ -1,6 +0,0 @@ -Lab assignment #1 - -* Part 1: Embedding-based Machine Translation: -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/homeworks/lab01_nlp/lab1_01_nlp_part1_embedding_based_mt.ipynb) - -* Part 2: NMT: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/homeworks/lab01_nlp/lab1_02_nlp_part2_nmt.ipynb) diff --git a/homeworks/lab01_nlp/lab1_01_nlp_part1_embedding_based_mt.ipynb b/homeworks/lab01_nlp/lab1_01_nlp_part1_embedding_based_mt.ipynb deleted file mode 100644 index 2bcd322..0000000 --- a/homeworks/lab01_nlp/lab1_01_nlp_part1_embedding_based_mt.ipynb +++ /dev/null @@ -1,753 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eulvfJWl7ueY" - }, - "source": [ - "# Lab 1\n", - "\n", - "\n", - "## Part 1: Bilingual dictionary induction and unsupervised embedding-based MT (30%)\n", - "*Note: this homework is based on materials from yandexdataschool [NLP course](https://github.com/yandexdataschool/nlp_course/). Feel free to check this awesome course if you wish to dig deeper.*\n", - "\n", - "*Refined by [Nikolay Karpachev](https://www.linkedin.com/in/nikolay-karpachev-b0146a104/)*" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fV4rIjxa7uei" - }, - "source": [ - "**In this homework** **YOU** will make machine translation system without using parallel corpora, alignment, attention, 100500 depth super-cool recurrent neural network and all that kind superstuff.\n", - "\n", - "But even without parallel corpora this system can be good enough (hopefully), in particular for similar languages, e.g. Ukrainian and Russian. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idSYq2GU7uew" - }, - "source": [ - "### Frament of the Swadesh list for some slavic languages\n", - "\n", - "The Swadesh list is a lexicostatistical stuff. It's named after American linguist Morris Swadesh and contains basic lexis. This list are used to define subgroupings of languages, its relatedness.\n", - "\n", - "So we can see some kind of word invariance for different Slavic languages.\n", - "\n", - "\n", - "| Russian | Belorussian | Ukrainian | Polish | Czech | Bulgarian |\n", - "|-----------------|--------------------------|-------------------------|--------------------|-------------------------------|-----------------------|\n", - "| женщина | жанчына, кабета, баба | жінка | kobieta | žena | жена |\n", - "| мужчина | мужчына | чоловік, мужчина | mężczyzna | muž | мъж |\n", - "| человек | чалавек | людина, чоловік | człowiek | člověk | човек |\n", - "| ребёнок, дитя | дзіця, дзіцёнак, немаўля | дитина, дитя | dziecko | dítě | дете |\n", - "| жена | жонка | дружина, жінка | żona | žena, manželka, choť | съпруга, жена |\n", - "| муж | муж, гаспадар | чоловiк, муж | mąż | muž, manžel, choť | съпруг, мъж |\n", - "| мать, мама | маці, матка | мати, матір, неня, мама | matka | matka, máma, 'стар.' mateř | майка |\n", - "| отец, тятя | бацька, тата | батько, тато, татусь | ojciec | otec | баща, татко |\n", - "| много | шмат, багата | багато | wiele | mnoho, hodně | много |\n", - "| несколько | некалькі, колькі | декілька, кілька | kilka | několik, pár, trocha | няколко |\n", - "| другой, иной | іншы | інший | inny | druhý, jiný | друг |\n", - "| зверь, животное | жывёла, звер, істота | тварина, звір | zwierzę | zvíře | животно |\n", - "| рыба | рыба | риба | ryba | ryba | риба |\n", - "| птица | птушка | птах, птиця | ptak | pták | птица |\n", - "| собака, пёс | сабака | собака, пес | pies | pes | куче, пес |\n", - "| вошь | вош | воша | wesz | veš | въшка |\n", - "| змея, гад | змяя | змія, гад | wąż | had | змия |\n", - "| червь, червяк | чарвяк | хробак, черв'як | robak | červ | червей |\n", - "| дерево | дрэва | дерево | drzewo | strom, dřevo | дърво |\n", - "| лес | лес | ліс | las | les | гора, лес |\n", - "| палка | кій, палка | палиця | patyk, pręt, pałka | hůl, klacek, prut, kůl, pálka | палка, пръчка, бастун |" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cNM3_fjr7ue2" - }, - "source": [ - "But the context distribution of these languages demonstrates even more invariance. And we can use this fact for our for our purposes." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YLppwa527ue6" - }, - "source": [ - "## Data" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lYBGKAUn7ue_" - }, - "outputs": [], - "source": [ - "import gensim\n", - "import numpy as np\n", - "from gensim.models import KeyedVectors" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MwGoVhRA7ufP" - }, - "source": [ - "In this notebook we're going to use pretrained word vectors - FastText (original paper - https://arxiv.org/abs/1607.04606).\n", - "\n", - "You can download them from the official [website](https://fasttext.cc/docs/en/crawl-vectors.html). We're going to need embeddings for Russian and Ukrainian languages. Please use word2vec-compatible format (.text)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "u1JjQv_97ufT" - }, - "outputs": [], - "source": [ - "uk_emb = KeyedVectors.load_word2vec_format(\"cc.uk.300.vec\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ffzuept_7ufd" - }, - "outputs": [], - "source": [ - "ru_emb = KeyedVectors.load_word2vec_format(\"cc.ru.300.vec\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nTkXfT0W7ufk" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([ru_emb[\"август\"]], topn=10)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vdBA8lcg7ufs" - }, - "outputs": [], - "source": [ - "uk_emb.most_similar([uk_emb[\"серпень\"]])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_yJvcKXO7uf0" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([uk_emb[\"серпень\"]])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pNdYAR1q7uf6" - }, - "source": [ - "Load small dictionaries for correspoinding words pairs as trainset and testset." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "35d_DAK67uf8" - }, - "outputs": [], - "source": [ - "def load_word_pairs(filename):\n", - " uk_ru_pairs = []\n", - " uk_vectors = []\n", - " ru_vectors = []\n", - " with open(filename, \"r\") as inpf:\n", - " for line in inpf:\n", - " uk, ru = line.rstrip().split(\"\\t\")\n", - " if uk not in uk_emb or ru not in ru_emb:\n", - " continue\n", - " uk_ru_pairs.append((uk, ru))\n", - " uk_vectors.append(uk_emb[uk])\n", - " ru_vectors.append(ru_emb[ru])\n", - " return uk_ru_pairs, np.array(uk_vectors), np.array(ru_vectors)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wkNL602WHJyO" - }, - "outputs": [], - "source": [ - "!wget -O ukr_rus.train.txt http://tiny.cc/jfgecz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uoclU6JcHCcn" - }, - "outputs": [], - "source": [ - "!wget -O ukr_rus.test.txt http://tiny.cc/6zoeez" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "05BqsdSK7ugD" - }, - "outputs": [], - "source": [ - "uk_ru_train, X_train, Y_train = load_word_pairs(\"ukr_rus.train.txt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zQOZw51r7ugL" - }, - "outputs": [], - "source": [ - "uk_ru_test, X_test, Y_test = load_word_pairs(\"ukr_rus.test.txt\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ZBBNvpz7ugQ" - }, - "source": [ - "## Embedding space mapping (0.3 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_Dhk5gL7ugS" - }, - "source": [ - "Let $x_i \\in \\mathrm{R}^d$ be the distributed representation of word $i$ in the source language, and $y_i \\in \\mathrm{R}^d$ is the vector representation of its translation. Our purpose is to learn such linear transform $W$ that minimizes euclidian distance between $Wx_i$ and $y_i$ for some subset of word embeddings. Thus we can formulate so-called Procrustes problem:\n", - "\n", - "$$W^*= \\arg\\min_W \\sum_{i=1}^n||Wx_i - y_i||_2$$\n", - "or\n", - "$$W^*= \\arg\\min_W ||WX - Y||_F$$\n", - "\n", - "where $||*||_F$ - Frobenius norm." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "acOjDdtL7ugY" - }, - "source": [ - "$W^*= \\arg\\min_W \\sum_{i=1}^n||Wx_i - y_i||_2$ looks like simple multiple linear regression (without intercept fit). So let's code." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Lb-KN1be7uga" - }, - "outputs": [], - "source": [ - "from sklearn.linear_model import LinearRegression\n", - "\n", - "# YOUR CODE HERE\n", - "# mapping = ...\n", - "# -------" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "X7tqJwoY7ugf" - }, - "source": [ - "Let's take a look at neigbours of the vector of word _\"серпень\"_ (_\"август\"_ in Russian) after linear transform." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "31SrFSbn7ugi" - }, - "outputs": [], - "source": [ - "august = mapping.predict(uk_emb[\"серпень\"].reshape(1, -1))\n", - "ru_emb.most_similar(august)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "okSkjk597ugo" - }, - "source": [ - "We can see that neighbourhood of this embedding cosists of different months, but right variant is on the ninth place." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o2uY6Y9B7ugt" - }, - "source": [ - "As quality measure we will use precision top-1, top-5 and top-10 (for each transformed Ukrainian embedding we count how many right target pairs are found in top N nearest neighbours in Russian embedding space)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zptuho8LAfIE" - }, - "outputs": [], - "source": [ - "def precision(pairs, mapped_vectors, topn=1):\n", - " \"\"\"\n", - " :args:\n", - " pairs = list of right word pairs [(uk_word_0, ru_word_0), ...]\n", - " mapped_vectors = list of embeddings after mapping from source embedding space to destination embedding space\n", - " topn = the number of nearest neighbours in destination embedding space to choose from\n", - " :returns:\n", - " precision_val, float number, total number of words for those we can find right translation at top K.\n", - " \"\"\"\n", - " assert len(pairs) == len(mapped_vectors)\n", - " num_matches = 0\n", - " for i, (_, ru) in enumerate(pairs):\n", - " # YOUR CODE HERE\n", - " precision_val = num_matches / len(pairs)\n", - " return precision_val" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "duhj9hpv7ugy" - }, - "outputs": [], - "source": [ - "assert precision([(\"серпень\", \"август\")], august, topn=5) == 0.0\n", - "assert precision([(\"серпень\", \"август\")], august, topn=9) == 1.0\n", - "assert precision([(\"серпень\", \"август\")], august, topn=10) == 1.0" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0-iyd5gP7ug5" - }, - "outputs": [], - "source": [ - "assert precision(uk_ru_test, X_test) == 0.0\n", - "assert precision(uk_ru_test, Y_test) == 1.0" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U-ssEJ3x7uhA" - }, - "outputs": [], - "source": [ - "precision_top1 = precision(uk_ru_test, mapping.predict(X_test), 1)\n", - "precision_top5 = precision(uk_ru_test, mapping.predict(X_test), 5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7K-hy7a6Ksn2" - }, - "outputs": [], - "source": [ - "print(precision_top1)\n", - "print(precision_top5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hf6Ou8bx7uhH" - }, - "source": [ - "## Making it better (orthogonal Procrustean problem) (0.3 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4oLs-drN7uhK" - }, - "source": [ - "It can be shown (see original paper) that a self-consistent linear mapping between semantic spaces should be orthogonal. \n", - "We can restrict transform $W$ to be orthogonal. Then we will solve next problem:\n", - "\n", - "$$W^*= \\arg\\min_W ||WX - Y||_F \\text{, where: } W^TW = I$$\n", - "\n", - "$$I \\text{- identity matrix}$$\n", - "\n", - "Instead of making yet another regression problem we can find optimal orthogonal transformation using singular value decomposition. It turns out that optimal transformation $W^*$ can be expressed via SVD components:\n", - "$$X^TY=U\\Sigma V^T\\text{, singular value decompostion}$$\n", - "$$W^*=UV^T$$" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_KSaRJFGMFiJ" - }, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DdFQ7qti7uhL" - }, - "outputs": [], - "source": [ - "def learn_transform(X_train, Y_train):\n", - " \"\"\" \n", - " :returns: W* : float matrix[emb_dim x emb_dim] as defined in formulae above\n", - " \"\"\"\n", - " # YOUR CODE GOES HERE\n", - " # compute orthogonal embedding space mapping\n", - " # mapping = ...\n", - "\n", - " return mapping" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7X7QfYDd7uhQ" - }, - "outputs": [], - "source": [ - "W = learn_transform(X_train, Y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OVOFYYa37uhX" - }, - "outputs": [], - "source": [ - "ru_emb.most_similar([np.matmul(uk_emb[\"серпень\"], W)])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r297sYP37uhb" - }, - "outputs": [], - "source": [ - "print(precision(uk_ru_test, np.matmul(X_test, W)))\n", - "print(precision(uk_ru_test, np.matmul(X_test, W), 5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hvUZ72U5AfJg" - }, - "source": [ - "## Unsupervised embedding-based MT (0.4 pts)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLyuVfHBLrJn" - }, - "source": [ - "Now, let's build our word embeddings-based translator!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tPAURW1CMuP7" - }, - "source": [ - "Firstly, download OPUS Tatoeba corpus." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "F80kUKzQMsDu" - }, - "outputs": [], - "source": [ - "!wget https://object.pouta.csc.fi/OPUS-Tatoeba/v20190709/mono/uk.txt.gz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0CGFZoxCUVf1" - }, - "outputs": [], - "source": [ - "!gzip -d ./uk.txt.gz" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2MV3VvoVUX5U" - }, - "outputs": [], - "source": [ - "with open('./uk.txt', 'r') as f:\n", - " uk_corpus = f.readlines()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tU7nPVf0UhbI" - }, - "outputs": [], - "source": [ - "# To save your time and CPU, feel free to use first 1000 sentences of the corpus\n", - "uk_corpus = uk_corpus[:1000]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FLN8dBOXAfJ1" - }, - "outputs": [], - "source": [ - "# Any necessary preprocessing if needed\n", - "# YOUR CODE HERE" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FGksC7l_NMi9" - }, - "outputs": [], - "source": [ - "def translate(sentence):\n", - " \"\"\"\n", - " :args:\n", - " sentence - sentence in Ukrainian (str)\n", - " :returns:\n", - " translation - sentence in Russian (str)\n", - "\n", - " * find ukrainian embedding for each word in sentence\n", - " * transform ukrainian embedding vector\n", - " * find nearest russian word and replace\n", - " \"\"\"\n", - " # YOUR CODE GOES HERE\n", - "\n", - " return \" \".join(translated)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4hbbMy-tNxlf" - }, - "outputs": [], - "source": [ - "assert translate(\".\") == \".\"\n", - "assert translate(\"1 , 3\") == \"1 , 3\"\n", - "assert translate(\"кіт зловив мишу\") == \"кот поймал мышку\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ia6I2ce7O_HI" - }, - "source": [ - "Now you can play with your model and try to get as accurate translations as possible. **Note**: one big issue is out-of-vocabulary words. Try to think of various ways of handling it (you can start with translating each of them to a special **UNK** token and then move to more sophisticated approaches). Good luck!" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ap1W7ZCeOAVU" - }, - "outputs": [], - "source": [ - "for sent in uk_corpus[::10]:\n", - " print(translate(sent))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Great! \n", - "See second notebook for the Neural Machine Translation assignment." - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "homework.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Py3 Research", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/lab01_nlp/lab1_02_nlp_part2_nmt.ipynb b/homeworks/lab01_nlp/lab1_02_nlp_part2_nmt.ipynb deleted file mode 100644 index eb346fa..0000000 --- a/homeworks/lab01_nlp/lab1_02_nlp_part2_nmt.ipynb +++ /dev/null @@ -1,941 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Lab 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Part 2: Neural Machine Translation in the wild\n", - "In the third homework you are supposed to get the best translation you can for the EN-RU translation task.\n", - "\n", - "Basic approach using RNNs as encoder and decoder is implemented for you. \n", - "\n", - "Your ultimate task is to use the techniques we've covered, e.g.\n", - "\n", - "* Optimization enhancements (e.g. learning rate decay)\n", - "\n", - "* CNN encoder (with or without positional encoding)\n", - "\n", - "* attention/self-attention mechanism\n", - "\n", - "* pretraining the language model\n", - "\n", - "* [Byte Pair Encoding](https://github.com/rsennrich/subword-nmt)\n", - "\n", - "* or just fine-tunning BERT ;)\n", - "\n", - "to improve the translation quality. \n", - "\n", - "__Please use at least three different approaches/models and compare them (translation quality/complexity/training and evaluation time).__\n", - "\n", - "Write down some summary on your experiments and illustrate it with convergence plots/metrics and your thoughts. Just like you would approach a real problem." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# You might need to install the libraries below. Do it in the desired environment\n", - "# if you are working locally.\n", - "\n", - "# ! pip install subword-nmt\n", - "# ! pip install nltk\n", - "# ! pip install torchtext" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Thanks to YSDA NLP course team for the data\n", - "# (who thanks tilda and deephack teams for the data in their turn)\n", - "\n", - "import os\n", - "path_do_data = '../../datasets/Machine_translation_EN_RU/data.txt'\n", - "if not os.path.exists(path_do_data):\n", - " print(\"Dataset not found locally. Downloading from github. Loading special files as well\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/master/datasets/Machine_translation_EN_RU/data.txt -nc\n", - " path_do_data = './data.txt'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if not os.path.exists('./utils.py'):\n", - " print(\"utils file not found locally. Downloading from github.\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/master/homeworks_advanced/Lab1_NLP/utils.py -nc\n", - "\n", - "if not os.path.exists('./my_network.py'):\n", - " print(\"network file not found locally. Downloading from github.\")\n", - " !wget https://raw.githubusercontent.com/girafe-ai/ml-mipt/master/homeworks_advanced/Lab1_NLP/my_network.py -nc" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "\n", - "import torchtext\n", - "from torchtext.legacy.datasets import TranslationDataset, Multi30k\n", - "from torchtext.legacy.data import Field, BucketIterator, TabularDataset\n", - "\n", - "import spacy\n", - "\n", - "import random\n", - "import math\n", - "import time\n", - "\n", - "import matplotlib\n", - "matplotlib.rcParams.update({'figure.figsize': (16, 12), 'font.size': 14})\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "from IPython.display import clear_output\n", - "\n", - "from nltk.tokenize import WordPunctTokenizer\n", - "from subword_nmt.learn_bpe import learn_bpe\n", - "from subword_nmt.apply_bpe import BPE" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Main part\n", - "__Here comes the preprocessing. Do not hesitate to use BPE or more complex preprocessing ;)__" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer_W = WordPunctTokenizer()\n", - "def tokenize(x, tokenizer=tokenizer_W):\n", - " return tokenizer.tokenize(x.lower())" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "SRC = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "TRG = Field(tokenize=tokenize,\n", - " init_token = '', \n", - " eos_token = '', \n", - " lower = True)\n", - "\n", - "dataset = TabularDataset(\n", - " path=path_do_data,\n", - " format='tsv',\n", - " fields=[('trg', TRG), ('src', SRC)]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "train_data, valid_data, test_data = dataset.split(split_ratio=[0.8, 0.15, 0.05])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training examples: 40000\n", - "Number of validation examples: 2500\n", - "Number of testing examples: 7500\n" - ] - } - ], - "source": [ - "print(f\"Number of training examples: {len(train_data.examples)}\")\n", - "print(f\"Number of validation examples: {len(valid_data.examples)}\")\n", - "print(f\"Number of testing examples: {len(test_data.examples)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "SRC.build_vocab(train_data, min_freq = 3)\n", - "TRG.build_vocab(train_data, min_freq = 3)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unique tokens in source (ru) vocabulary: 9267\n", - "Unique tokens in target (en) vocabulary: 6699\n" - ] - } - ], - "source": [ - "print(f\"Unique tokens in source (ru) vocabulary: {len(SRC.vocab)}\")\n", - "print(f\"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here are tokens from original (RU) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['',\n", - " '29',\n", - " 'соль',\n", - " 'комо',\n", - " '―',\n", - " 'электрическая',\n", - " 'ming',\n", - " 'утренний',\n", - " 'детском',\n", - " 'таунус']" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "SRC.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And from target (EN) corpus:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['', 'king', 'buffets', 'catch', 'media', 'schedule', 'maraunenhof']" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "TRG.vocab.itos[::1000]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And here is example from train dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'trg': ['laundry', 'service', 'is', 'provided', '.'], 'src': ['помимо', 'этого', ',', 'гостям', 'предоставляются', 'услуги', 'прачечной', '.']}\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[9]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check the length distributions:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Train data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfUAAAEICAYAAABGRG3WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAer0lEQVR4nO3df7ReVX3n8fdHIr+0kCApxQRNKhlbZC0rzUhcOB3HOPyybVhr1MFxDdGmTVdLW9vpTAvTrmFGZQbXOEVYKpURSrAWpFRLRqw0RRlXp8OPoA7yQ0rKryQFuZIAVqs19jt/nH3hId6be3Pvzb3PPff9WutZ95y99zlnn3Of/XzP2c9+zklVIUmS5r8XzHUFJEnSzDCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST1hUNcBkWRFkkqyaA62/c4kfznb25XmQpKrkrxvGsv/XZIfnck6tfU+nORNM73eSWx3zj57hoFBXfPaQm/AGg5zFcD2V5Jbkvz8YFpVvbiqHpyrOk3XfDn2s8WgrudJctBc10HqG086NVsM6vNIkt9OsjPJN5Pcn2RtSz8kyQeT/G17fTDJIS3vB7qi25Xt8W36qiSXJflskm8B/yLJcUk+lWQkyZNJPjSw7M8luS/J7iQ3JXn5JOt+ZJIrkjzW9uF9oycQo3VM8oG23oeSnDGw7MokX2z7/RdJPpzkD1v2F9vfp1o34usGlhtzfdJMSvJx4GXA/2rvwd8a6EHakORR4POt7B8neTzJ0+09/aqB9VzV3ts3tvf6bUle0fKS5OIkTyR5JslXk5w4Rl2WJPlMa7u72/Tylnch8M+AD7V6fqilD34eHJnk6rb8I0l+N8kLWt4+2+kEx+gFSc5L8jftM+W6JEe1vNFjtT7Jo0m+keR3BpY9LMmmts372vHdMd6xH9jsO8ZaX+9Vla958AJeCWwHXtrmVwCvaNPvAW4FfhhYCvwV8N6W907gL/daVwHHt+mrgKeBU+hO8l4E/D/g4jZ9KPD6VnYdsA34cWAR8LvAX41T3xVtO4va/KeBj7Z1/jBwO/CLA3X8HvALwEHALwF/C6Tl/1/gA8DBwOuBZ4A/HGs7k1mfL18z/QIeBt40MD/6vry6vecPa+k/B/wQcAjwQeArA8tcBTwJvLa1r08A17a804A7gcVAWhs8dmC597XplwD/Cji8beePgT8d2MYtwM/vVffBz4OrgRvasiuAvwY2tLz9aleDxwR4N91n1PK27x8FrtnrWP1P4DDg1cB3gR9v+RcB/xtY0pa/C9gxiWM/5vr6/przCvia5D8KjgeeAN4EvHCvvL8BzhyYPw14uE2/k4mD+tUDea8DRhgIkgN5fzbawNv8C4BvAy8fo+xow1oEHNMa1WED+W8HvjBQx20DeYe3ZX+E7ix8D3D4QP4fMnFQH3N9c/1/9NXP1z4Cy4/uY5nFrcyRbf4q4GMD+WcCX2vTb6QLsGuAF+y1nqtoQX2MbfwEsHtg/hbGCep0gfofgBMG8n4RuKVN71e74vlB/T5g7UDesXQnCIsGjtXygfzbgbPb9IPAaQN5P8/kgvqY6+v7y+73eaKqtgG/Dvxn4Ikk1yZ5act+KfDIQPFHWtpkbR+YPg54pKr2jFHu5cAlSZ5K8hSwi+6qYdkE63858ELgsYFlP0p3xT7q8dGJqvp2m3xx249dA2l713c8461Pmk3PvleTHJTkotYF/QxdMAI4eqD84wPT36a9Z6vq88CHgA/Ttf/Lkxyx98aSHJ7ko63r/Bm6r6cWZ3JjZY6ma6d7f5YMtu+ptquXA58eaP/3Ad+nO+H/gXUzsO90nwGDbX4y7X9f6+s1g/o8UlV/VFWvp2sgBby/Zf1tSxv1spYG8C26M2oAkvzIWKsemN4OvCxjD+zZTtdlvnjgdVhV/dUEVd9Od6V+9MByR1TVqyZYDuAx4Kgkhw+kHTdO3aW5Mt77cDD939B9hfUm4Ei6K0roTown3kDVpVX1k8AJwD8B/sMYxX6T7qu6k6vqCOCn9trGvtrLN+iunvf+LNk5mfpNYDtwxl6fHYdW1WTW/Rhdt/uo4/bK9zNggEF9nkjyyiRvTDcA7jvA3wP/2LKvAX43ydIkRwP/ia6LGrrvx1+V5CeSHEp3pb8vt9M1oouSvCjJoUlOaXm/D5w/OrinDap560R1r6rHgD8H/keSI9qgmVck+eeTWPYRYCvwn5Mc3AbC/cxAkRG64zDjv7OV9sPXmfg9+EN0J7dP0p1o/9fJrjzJP01ycpIX0p2of4fn2v/e2/h7uoGjRwEXTLaeVfV94DrgwiQ/lG4Q7L/juc+S6fj9tt6Xt/1ZmmTdJJe9ju5zZ0mSZcCv7JU/mWO/YBjU549D6AaMfIOuW+mHgfNb3vvoAt9dwFeBL7U0quqv6QbS/QXwALDPm7K0hv0zdN+xPQrsAP51y/s0Xe/Ata1r725gsqPKz6Eb6HYvsBu4nu57tcl4B913/U+2/fok3YfjaBfghcD/aV17aya5Tmkm/Te6E+unkvz7ccpcTdedvZOuHdy6H+s/gm7g1+62jieB/z5GuQ/SDQ77Rlv/5/bKvwR4SxtJfukYy/8q3UnDg3SfFX8EXLkf9RzPJcBm4M+TfLPV7eRJLvseus+hh+g+x66ntf9mMsd+wRgdXSzNG0k+STeAaO+rEEk9l+SX6Aa9TdjTtxB5pa6h17oeX9G67U+n+17yT+e6XpIOvCTHJjmltf9X0o0b+PRc12tYeZcjzQc/AnyK7je4O4Bfqqovz22VJM2Sg+l+LbMSeAq4FvjInNZoiNn9LklST9j9LklST8zb7vejjz66VqxYMdfVkIbanXfe+Y2qWjrX9dgX27I0OZNpzxMG9SRXAj8NPFFVJ7a0o+h+VrSC7q5Ib6uq3UlC99OFM+nu4PPOqvpSW2Y93b3Cobul4aaW/pN0tzk8DPgs8O6axHcCK1asYOvWrRMVkxa0JI9MXGpu2ZalyZlMe55M9/tVwOl7pZ0H3FxVq4Cb2zx0v1le1V4bgctaRUZvgnAy3cMKLkiypC1zGd0DAkaX23tbkiRpEiYM6lX1Rbp7fA9aB2xq05uAswbSr67OrXT3HD6W7gEjW6pqV1XtBrYAp7e8I6rq1nZ1fvXAuiRJ0n6Y6kC5Y9qtP6G7u9noTfmX8fyb7e9oaftK3zFG+piSbEyyNcnWkZGRKVZdkqR+mvbo93aFPSu/i6uqy6tqdVWtXrp0qMf+SJI066Ya1L/eus5pf59o6Tt5/hN0lre0faUvHyNdkiTtp6kG9c3A+ja9HrhhIP2cdNYAT7du+puAU9tTdpYApwI3tbxnkqxpI+fPGViXJEnaD5P5Sds1wBuAo5PsoBvFfhFwXZINdE8Melsr/lm6n7Nto/tJ27sAqmpXkvcCd7Ry76mq0cF3v8xzP2n7s/aSJEn7acKgXlVvHydr7RhlCzh3nPVcyRiP8KuqrcCJE9VDkiTtm7eJlSSpJ+btbWJn04rzbpywzMMXvXkWaiJpumzP6jOv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1KUFJMmVSZ5IcvdA2lFJtiR5oP1d0tKT5NIk25LcleSkgWXWt/IPJFk/kP6TSb7alrk0SWZ3D6WFzaAuLSxXAafvlXYecHNVrQJubvMAZwCr2msjcBl0JwHABcDJwGuBC0ZPBFqZXxhYbu9tSTqADOrSAlJVXwR27ZW8DtjUpjcBZw2kX12dW4HFSY4FTgO2VNWuqtoNbAFOb3lHVNWtVVXA1QPrkjQLDOqSjqmqx9r048AxbXoZsH2g3I6Wtq/0HWOk/4AkG5NsTbJ1ZGRk+nsgCTCoSxrQrrBrFrZzeVWtrqrVS5cuPdCbkxYMg7qkr7euc9rfJ1r6TuC4gXLLW9q+0pePkS5plhjUJW0GRkewrwduGEg/p42CXwM83brpbwJOTbKkDZA7Fbip5T2TZE0b9X7OwLokzYJFc10BSbMnyTXAG4Cjk+ygG8V+EXBdkg3AI8DbWvHPAmcC24BvA+8CqKpdSd4L3NHKvaeqRgff/TLdCPvDgD9rL0mzxKAuLSBV9fZxstaOUbaAc8dZz5XAlWOkbwVOnE4dJU2d3e+SJPWEQV2SpJ4wqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9Ma2gnuQ3ktyT5O4k1yQ5NMnKJLe15yl/MsnBrewhbX5by18xsJ7zW/r9SU6b3i5JkrQwTTmoJ1kG/BqwuqpOBA4CzgbeD1xcVccDu4ENbZENwO6WfnErR5IT2nKvonv28keSHDTVekmStFBNt/t9EXBYkkXA4cBjwBuB61v+3s9mHn1m8/XA2nZ/6HXAtVX13ap6iO6WlK+dZr0kSVpwphzUq2on8AHgUbpg/jRwJ/BUVe1pxQafp/zsM5hb/tPASxj/2cw/wGcwS5I0vul0vy+hu8peCbwUeBFd9/kB4zOYJUka33S6398EPFRVI1X1PeBTwCnA4tYdD89/nvKzz2Bu+UcCTzL+s5klSdJ+mE5QfxRYk+Tw9t34WuBe4AvAW1qZvZ/NPPrM5rcAn29PgdoMnN1Gx68EVgG3T6NekiQtSFN+9GpV3ZbkeuBLwB7gy8DlwI3AtUne19KuaItcAXw8yTZgF92Id6rqniTX0Z0Q7AHOrarvT7VekiQtVNN6nnpVXQBcsFfyg4wxer2qvgO8dZz1XAhcOJ26SJK00E0rqEtSH60478YJyzx80ZtnoSbS/jGoz5DJfAiAHwSSpAPHe79LktQTBnVJknrC7ndJvTHZr8GkvvJKXZKknjCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST1hUJckqScM6pIASPIbSe5JcneSa5IcmmRlktuSbEvyySQHt7KHtPltLX/FwHrOb+n3JzltrvZHWogM6pJIsgz4NWB1VZ0IHET3eOT3AxdX1fHAbmBDW2QDsLulX9zKkeSEttyrgNOBjyQ5aDb3RVrIDOqSRi0CDkuyCDgceAx4I3B9y98EnNWm17V5Wv7aJGnp11bVd6vqIWAbYzyKWdKBYVCXRFXtBD4APEoXzJ8G7gSeqqo9rdgOYFmbXgZsb8vuaeVfMpg+xjLPSrIxydYkW0dGRmZ+h6QFyqAuiSRL6K6yVwIvBV5E131+QFTV5VW1uqpWL1269EBtRlpwDOqSAN4EPFRVI1X1PeBTwCnA4tYdD7Ac2NmmdwLHAbT8I4EnB9PHWEbSAWZQlwRdt/uaJIe378bXAvcCXwDe0sqsB25o05vbPC3/81VVLf3sNjp+JbAKuH2W9kFa8Hz0qiSq6rYk1wNfAvYAXwYuB24Erk3yvpZ2RVvkCuDjSbYBu+hGvFNV9yS5ju6EYA9wblV9f1Z3RlrADOqSAKiqC4AL9kp+kDFGr1fVd4C3jrOeC4ELZ7yCkiZk97skST1hUJckqScM6pIk9YRBXZKknjCoS5LUEwZ1SZJ6wqAuSVJPGNQlSeoJg7okST3hHeVm2YrzbpywzMMXvXkWaiJJ6huv1CVJ6gmDuiRJPTGtoJ5kcZLrk3wtyX1JXpfkqCRbkjzQ/i5pZZPk0iTbktyV5KSB9axv5R9Isn78LUqSpPFM90r9EuBzVfVjwKuB+4DzgJurahVwc5sHOIPu2cqrgI3AZQBJjqJ7MtTJdE+DumD0RECSJE3elIN6kiOBn6I9X7mq/qGqngLWAZtasU3AWW16HXB1dW4FFic5FjgN2FJVu6pqN7AFOH2q9ZIkaaGazuj3lcAI8AdJXg3cCbwbOKaqHmtlHgeOadPLgO0Dy+9oaeOlz4rJjEaXJGk+mE73+yLgJOCyqnoN8C2e62oHoKoKqGls43mSbEyyNcnWkZGRmVqtJEm9MJ2gvgPYUVW3tfnr6YL811u3Ou3vEy1/J3DcwPLLW9p46T+gqi6vqtVVtXrp0qXTqLokSf0z5aBeVY8D25O8siWtBe4FNgOjI9jXAze06c3AOW0U/Brg6dZNfxNwapIlbYDcqS1NkiTth+neUe5XgU8kORh4EHgX3YnCdUk2AI8Ab2tlPwucCWwDvt3KUlW7krwXuKOVe09V7ZpmvSRJWnCmFdSr6ivA6jGy1o5RtoBzx1nPlcCV06mLJEkLnXeUkySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiQAkixOcn2SryW5L8nrkhyVZEuSB9rfJa1sklyaZFuSu5KcNLCe9a38A0nWj79FSTPNoC5p1CXA56rqx4BXA/fRPaTp5qpaBdzMcw9tOgNY1V4bgcsAkhwFXACcDLwWuGD0REDSgWdQl0SSI4GfAq4AqKp/qKqngHXAplZsE3BWm14HXF2dW4HF7QFOpwFbqmpXVe0GtgCnz+KuSAuaQV0SwEpgBPiDJF9O8rEkLwKOaQ9eAngcOKZNLwO2Dyy/o6WNly5pFhjUJUH3HIiTgMuq6jXAt3iuqx149vkNNRMbS7IxydYkW0dGRmZilZIwqEvq7AB2VNVtbf56uiD/9datTvv7RMvfCRw3sPzyljZe+vNU1eVVtbqqVi9dunRGd0RayAzqkqiqx4HtSV7ZktYC9wKbgdER7OuBG9r0ZuCcNgp+DfB066a/CTg1yZI2QO7UliZpFkz3eeqS+uNXgU8kORh4EHgX3Yn/dUk2AI8Ab2tlPwucCWwDvt3KUlW7krwXuKOVe09V7Zq9XZAWNoO6JACq6ivA6jGy1o5RtoBzx1nPlcCVM1s7SZNh97skST1hUJckqSfsfpekKVhx3o0Tlnn4ojfPQk2k53ilLklSTxjUJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ4wqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ7w0atDyEc6SpKmYtpX6kkOSvLlJJ9p8yuT3JZkW5JPJjm4pR/S5re1/BUD6zi/pd+f5LTp1kmSpIVoJrrf3w3cNzD/fuDiqjoe2A1saOkbgN0t/eJWjiQnAGcDrwJOBz6S5KAZqJckSQvKtIJ6kuXAm4GPtfkAbwSub0U2AWe16XVtnpa/tpVfB1xbVd+tqoeAbcBrp1MvSZIWouleqX8Q+C3gH9v8S4CnqmpPm98BLGvTy4DtAC3/6Vb+2fQxlnmeJBuTbE2ydWRkZJpVlySpX6Yc1JP8NPBEVd05g/XZp6q6vKpWV9XqpUuXztZmJUmaF6Yz+v0U4GeTnAkcChwBXAIsTrKoXY0vB3a28juB44AdSRYBRwJPDqSPGlxGkiRN0pSv1Kvq/KpaXlUr6Aa6fb6q3gF8AXhLK7YeuKFNb27ztPzPV1W19LPb6PiVwCrg9qnWS5KkhepA/E79t4Frk7wP+DJwRUu/Avh4km3ALroTAarqniTXAfcCe4Bzq+r7B6BekiT12owE9aq6BbilTT/IGKPXq+o7wFvHWf5C4MKZqIskSQuVt4mVJKknDOqSJPWEQV3Ss7ztszS/GdQlDfK2z9I8ZlCXBHjbZ6kPDOqSRs3abZ+95bN0YBjUJc36bZ+95bN0YByIm89Imn+87bPUA16pS/K2z1JPeKUuaV+87bM0jxjUJT2Pt32W5i+73yVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk8Y1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTPnp1nlpx3o0Tlnn4ojfPQk0kScPCoC5JB8hkTr7BE3DNHLvfJUnqCYO6JEk9YVCXJKknDOqSJPWEQV2SpJ4wqEuS1BMGdUmSemLKQT3JcUm+kOTeJPckeXdLPyrJliQPtL9LWnqSXJpkW5K7kpw0sK71rfwDSdZPf7ckSVp4pnOlvgf4zao6AVgDnJvkBOA84OaqWgXc3OYBzgBWtddG4DLoTgKAC4CTgdcCF4yeCEiSpMmbclCvqseq6ktt+pvAfcAyYB2wqRXbBJzVptcBV1fnVmBxkmOB04AtVbWrqnYDW4DTp1ovSZIWqhn5Tj3JCuA1wG3AMVX1WMt6HDimTS8Dtg8stqOljZc+1nY2JtmaZOvIyMhMVF2SpN6YdlBP8mLgT4Bfr6pnBvOqqoCa7jYG1nd5Va2uqtVLly6dqdVKktQL0wrqSV5IF9A/UVWfaslfb93qtL9PtPSdwHEDiy9vaeOlS5olDnyV+mE6o98DXAHcV1W/N5C1GRhtyOuBGwbSz2kfBmuAp1s3/U3AqUmWtA+MU1uapNnjwFepB6bz6NVTgH8LfDXJV1rafwQuAq5LsgF4BHhby/sscCawDfg28C6AqtqV5L3AHa3ce6pq1zTqJWk/tRPsx9r0N5MMDnx9Qyu2CbgF+G0GBr4CtyYZHfj6BtrAV4AkowNfr5m1nZEWsCkH9ar6SyDjZK8do3wB546zriuBK6daF0kzZzYGvibZSHeFz8te9rKZq7y0wHlHOUnPmq2Brw56lQ4Mg7okwIGvUh8Y1CU58FXqiekMlJPUHw58lXrAoN5jK867cVLlHr7ozQe4Jhp2DnyV+sHud0mSesKgLklSTxjUJUnqCYO6JEk9YVCXJKknHP0uSXNsMr9U8Vcqmgyv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hEFdkqSeMKhLktQTBnVJknrCoC5JUk948xl54wtJ6gmv1CVJ6gmDuiRJPWFQlySpJwzqkiT1hAPlJGkecECrJsMrdUmSesKgLklST9j9rkmx60+Shp9BXdLQm8xJpSS73yVJ6g2DuiRJPWFQlySpJ3r9nbrfw80uB9NJc8s2KK/UJUnqiaG5Uk9yOnAJcBDwsaq6aI6rpAPAK4n+sy1Lc2cognqSg4APA/8S2AHckWRzVd07tzWTtD9sy8Nvsl9LenI9Pw1FUAdeC2yrqgcBklwLrAP8IFiAZnIshB9Ms8623BMz1Q5tg7NrWIL6MmD7wPwO4OS9CyXZCGxss3+X5P4x1nU08I0Zr+HssO4zLO+fVLGhrPskTKbeL5+NigyYybYMw/u/sV6TlPcPX52a+VivCdvzsAT1Samqy4HL91UmydaqWj1LVZpR1n1uzNe6z9d6w+TaMgzvPlqvyRvGOkF/6zUso993AscNzC9vaZLmF9uyNIeGJajfAaxKsjLJwcDZwOY5rpOk/WdblubQUHS/V9WeJL8C3ET3M5grq+qeKa5uwi69IWbd58Z8rfvQ1XuG2zIM4T421mvyhrFO0NN6papmqiKSJGkODUv3uyRJmiaDuiRJPdGroJ7k9CT3J9mW5Ly5rs++JDkuyReS3JvkniTvbulHJdmS5IH2d8lc13UsSQ5K8uUkn2nzK5Pc1o79J9sgqaGTZHGS65N8Lcl9SV43j475b7T3yt1Jrkly6Hw57lMxDO152NvpMLbDYWxjw9J2klyZ5Ikkdw+kjXls0rm01e+uJCdNZhu9Cep57vaUZwAnAG9PcsLc1mqf9gC/WVUnAGuAc1t9zwNurqpVwM1tfhi9G7hvYP79wMVVdTywG9gwJ7Wa2CXA56rqx4BX0+3D0B/zJMuAXwNWV9WJdIPQzmb+HPf9MkTtedjb6TC2w6FqY0PWdq4CTt8rbbxjcwawqr02ApdNagtV1YsX8DrgpoH584Hz57pe+1H/G+jul30/cGxLOxa4f67rNkZdl7c33xuBzwChuwPSorH+F8PyAo4EHqINEB1Inw/HfPRObUfR/WrlM8Bp8+G4T3F/h7I9D1M7HcZ2OIxtbNjaDrACuHuiYwN8FHj7WOX29erNlTpj355y2RzVZb8kWQG8BrgNOKaqHmtZjwPHzFG19uWDwG8B/9jmXwI8VVV72vywHvuVwAjwB63L8mNJXsQ8OOZVtRP4APAo8BjwNHAn8+O4T8XQtechbKfD2A6Hro3Ng7Yz3rGZUhvoU1Cfl5K8GPgT4Ner6pnBvOpOz4bqN4dJfhp4oqrunOu6TMEi4CTgsqp6DfAt9uoGHMZjDtC+Z1tH96H5UuBF/GA3ng6QYWunQ9wOh66Nzae2MxPHpk9Bfd7dnjLJC+k+KD5RVZ9qyV9PcmzLPxZ4Yq7qN45TgJ9N8jBwLV3X3yXA4iSjNzMa1mO/A9hRVbe1+evpPoCG/ZgDvAl4qKpGqup7wKfo/hfz4bhPxdC05yFtp8PaDoexjQ172xnv2EypDfQpqM+r21MmCXAFcF9V/d5A1mZgfZteT/cd3tCoqvOranlVraA7xp+vqncAXwDe0ooNXb0BqupxYHuSV7aktXSPBB3qY948CqxJcnh774zWfeiP+xQNRXse1nY6rO1wSNvYsLed8Y7NZuCcNgp+DfD0QDf9+GZrsMIsDUA4E/hr4G+A35nr+kxQ19fTdbPcBXylvc6k+17sZuAB4C+Ao+a6rvvYhzcAn2nTPwrcDmwD/hg4ZK7rN06dfwLY2o77nwJL5ssxB/4L8DXgbuDjwCHz5bhPcX/nvD3Ph3Y6bO1wGNvYsLQd4Bq67/W/R9ersWG8Y0M38PHD7f3/VbrR+xNuw9vESpLUE33qfpckaUEzqEuS1BMGdUmSesKgLklSTxjUJUnqCYO6JEk9YVCXJKkn/j90WDhb6Ns32gAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in train_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in train_data.examples])\n", - "\n", - "print('Length distribution in Train data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Length distribution in Test data\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe0AAAEICAYAAAByPazKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAfXElEQVR4nO3df7RdZX3n8fdHIij+4GdETMCkktqiq1YmBVzYjiVWAa241qDFsWPUdNJatLbaarBdpctKJ06dIi4tNRUKTK1AqdaMUjFFrdNpQYNaFFBJEUjSYCK/bKVq0e/8sZ8Lh8tN7k3uz33O+7XWWWfv53n23s++5z7nu/ezn7N3qgpJkrTwPWq+KyBJkqbGoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFb05JkWZJKsmgetv3qJH8/19uV5kOSi5O8YxrL/1uSH5nJOrX13pbk+TO93ilsd96+e+aTQVu9MKoNVAvLfAWovZXkM0l+aTCtqh5fVbfOV52mqy9/+9lm0B5RSfab7zpIw8aDSs02g/YClOStSbYn+dckX0uyqqUfkOTdSf6lvd6d5ICW94iu4nZmekybvjjJBUmuSvId4GeTHJXkw0l2JbkryXsHln1tkpuT3JPk6iRPnWLdD0pyYZIdbR/eMXaAMFbHJO9q6/1GklMHll2e5LNtv/82yfuS/HnL/mx7v7d18z1nYLkJ1yfNpCT/Gzga+D/tf/AtAz1Aa5LcAXyqlf3LJHcmua/9Tz9jYD0Xt//tj7f/9euSPK3lJcl5SXYm+XaSLyd55gR1OSTJx1rbvadNL2155wI/Dby31fO9LX3w++CgJJe25W9P8jtJHtXy9thOJ/kbPSrJuiT/3L5TrkhyaMsb+1utTnJHkm8l+e2BZR+b5JK2zZvb33fb7v72A5t95UTrG1pV5WsBvYCnA1uBp7T5ZcDT2vTbgWuBJwGLgX8Afr/lvRr4+3HrKuCYNn0xcB9wEt3B2uOAfwLOa9OPAZ7byp4ObAF+HFgE/A7wD7up77K2nUVt/iPA+9s6nwR8DvjlgTr+B/Dfgf2A1wH/AqTl/yPwLmB/4LnAt4E/n2g7U1mfL18z/QJuA54/MD/2f3lp+59/bEt/LfAE4ADg3cCXBpa5GLgLOL61rw8Cl7W8FwLXAwcDaW3wyIHl3tGmDwP+C3Bg285fAn89sI3PAL80ru6D3weXAh9tyy4Dvg6saXl71a4G/ybAG+m+o5a2fX8/8KFxf6s/BR4LPAv4HvDjLX898HfAIW35G4BtU/jbT7i+YX3NewV8jftA4BhgJ/B84NHj8v4ZOG1g/oXAbW361UwetC8dyHsOsIuBIDiQ9zdjDbjNPwq4H3jqBGXHGs4i4IjWaB47kP8K4NMDddwykHdgW/bJdEfRDwAHDuT/OZMH7QnXN9+fo6/hfO0hcPzIHpY5uJU5qM1fDHxgIP804Ktt+mS6AHoi8Khx67mYFrQn2MZPAvcMzH+G3QRtukD8feDYgbxfBj7TpveqXfHwoH0zsGog70i6A4BFA3+rpQP5nwPObNO3Ai8cyPslpha0J1zfsL7sHl9gqmoL8OvA7wE7k1yW5Ckt+ynA7QPFb29pU7V1YPoo4PaqemCCck8Fzk9yb5J7gbvpjvqXTLL+pwKPBnYMLPt+ujPuMXeOTVTV/W3y8W0/7h5IG1/f3dnd+qS59OD/apL9kqxvXcTfpgs2AIcPlL9zYPp+2v9sVX0KeC/wPrr2vyHJE8dvLMmBSd7fura/TXf56OBMbazK4XTtdPx3yWD73td29VTgIwPt/2bgB3QH9I9YNwP7TvcdMNjmp9L+97S+oWTQXoCq6i+q6rl0DaCAd7asf2lpY45uaQDfoTsiBiDJkyda9cD0VuDoTDxwZitdl/bBA6/HVtU/TFL1rXRn2ocPLPfEqnrGJMsB7AAOTXLgQNpRu6m7NF929384mP5f6S4xPR84iO6MELoD38k3UPWeqvpPwLHAjwK/NUGxN9NdSjuhqp4I/My4beypvXyL7ux3/HfJ9qnUbxJbgVPHfXc8pqqmsu4ddN3iY44al+93AAbtBSfJ05OcnG6A2XeBfwd+2LI/BPxOksVJDgd+l64LGbrr089I8pNJHkN3pr4nn6NrJOuTPC7JY5Kc1PL+BDh7bPBMG7TyssnqXlU7gE8C/yvJE9uglKcl+c9TWPZ2YDPwe0n2bwPNfn6gyC66v8OM/85U2gvfZPL/wSfQHbzeRXcg/QdTXXmSn0pyQpJH0x2If5eH2v/4bfw73cDMQ4FzplrPqvoBcAVwbpInpBtk+iYe+i6Zjj9p631q25/FSU6f4rJX0H3vHJJkCfD6cflT+dsPPYP2wnMA3YCMb9F1+zwJOLvlvYMusN0AfBn4Qkujqr5ON1Dtb4FbgD3edKQ13J+nu8Z1B7AN+IWW9xG6s/vLWtfbV4Cpjsp+Fd1AspuAe4Ar6a5rTcUr6a6139X263K6L7+xLrpzgf/Xut5OnOI6pZn0P+gOnO9N8pu7KXMpXXfzdrp2cO1erP+JdAOr7mnruAv4wwnKvZtu8NW32vo/MS7/fOCMNhL7PRMs/wa6g4Jb6b4r/gK4aC/quTvnAxuBTyb511a3E6a47Nvpvoe+Qfc9diWt/TdT+dsPvbFRu9KCk+RyugE6488iJA25JK+jG1Q2aU/dKPFMWwtG6xp8WutWP4XuuuBfz3e9JM2+JEcmOam1/6fTXbf/yHzXa6Hx7j1aSJ4MfJjuN6jbgNdV1Rfnt0qS5sj+dL82WQ7cC1wG/PG81mgBsntckqSesHtckqSeWNDd44cffngtW7ZsvqshLXjXX3/9t6pq8XzXY09sz9LU7Kk9L+igvWzZMjZv3jzf1ZAWvCS3T15qftmepanZU3u2e1ySpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ5Y0HdEm0nL1n18RtZz2/oXzch6JM2uqbR527P6xjNtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoSyMkyUVJdib5yrj0NyT5apIbk/zPgfSzk2xJ8rUkLxxIP6WlbUmybi73QRplI3NzFUkAXAy8F7h0LCHJzwKnA8+qqu8leVJLPxY4E3gG8BTgb5P8aFvsfcDPAduAzyfZWFU3zdleSCPKoC2NkKr6bJJl45JfB6yvqu+1Mjtb+unAZS39G0m2AMe3vC1VdStAkstaWYO2NMvsHpf0o8BPJ7kuyd8l+amWvgTYOlBuW0vbXfojJFmbZHOSzbt27ZqFqkujxaAtaRFwKHAi8FvAFUkyEyuuqg1VtbKqVi5evHgmVimNtEmD9kQDV5L8YRu0ckOSjyQ5eCDPgStSv2wDPlydzwE/BA4HtgNHDZRb2tJ2ly5plk3lTPti4JRxaZuAZ1bVTwBfB86GRwxcOQX44yT7JdmPbuDKqcCxwCtaWUnz76+BnwVoA832B74FbATOTHJAkuXACuBzwOeBFUmWJ9mfrs1vnJeaSyNm0oFoEw1cqapPDsxeC5zRph24Ii1gST4EPA84PMk24BzgIuCi1pv2fWB1VRVwY5Ir6NrpA8BZVfWDtp7XA1cD+wEXVdWNc74z0giaidHjrwUub9NL6IL4mMEBKuMHrpww0cqSrAXWAhx99NEzUD1JY6rqFbvJ+sXdlD8XOHeC9KuAq2awapKmYFoD0ZL8Nt0R+AdnpjoOXJEkaXf2+Uw7yauBFwOrWlca7HmAigNXJEmahn06005yCvAW4CVVdf9AlgNXJEmaJZOeae9m4MrZwAHApvZzzmur6leqyoErkiTNkqmMHp9o4MqFeyg/1ANXlq37+JTK3bb+RbNcE0nSqPGOaJIk9YRBW5KknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknjBoSyMkyUVJdib5ygR5b05SSQ5v80nyniRbktyQ5LiBsquT3NJeq+dyH6RRZtCWRsvFwCnjE5McBbwAuGMg+VS6J/WtANYCF7Syh9I9OOgE4HjgnCSHzGqtJQEGbWmkVNVngbsnyDqP7nG7NZB2OnBpda4FDk5yJPBCYFNV3V1V9wCbmOBAQNLMM2hLIy7J6cD2qvqncVlLgK0D89ta2u7SJ1r32iSbk2zetWvXDNZaGk0GbWmEJTkQeBvwu7Ox/qraUFUrq2rl4sWLZ2MT0kgxaEuj7WnAcuCfktwGLAW+kOTJwHbgqIGyS1va7tIlzTKDtjTCqurLVfWkqlpWVcvourqPq6o7gY3Aq9oo8hOB+6pqB3A18IIkh7QBaC9oaZJmmUFbGiFJPgT8I/D0JNuSrNlD8auAW4EtwJ8CvwpQVXcDvw98vr3e3tIkzbJF810BSXOnql4xSf6ygekCztpNuYuAi2a0cpIm5Zm2JEk9YdCWJKknDNqSJPWEQVuSpJ6YNGhP9ICBJIcm2dQeFrBp7L7DPmBAkqTZM5Uz7Yt55H2F1wHXVNUK4Jo2Dz5gQJKkWTNp0N7NAwZOBy5p05cALx1I9wEDkiTNgn29pn1EuzMSwJ3AEW3aBwxIkjRLpj0Qrd2AoSYtOPX1+YABSZImsK9B+5ut25v2vrOl+4ABSZJmyb7exnQjsBpY394/OpD++iSX0Q06u6+qdiS5GviDgcFnLwDO3vdqD4dl6z4+aZnb1r9oDmoiSeqDSYN2e8DA84DDk2yjGwW+HriiPWzgduDlrfhVwGl0Dxi4H3gNdA8YSDL2gAHwAQOS9tFUDnalYTVp0N7DAwZWTVDWBwxIkjRLvCOaJEk9YdCWJKknDNqSJPXEvo4e1yQcLKOFKMlFwIuBnVX1zJb2h8DPA98H/hl4TVXd2/LOBtYAPwB+raqubumnAOcD+wEfqKr1c70v0ijyTFsaLRfzyFsIbwKeWVU/AXyd9nPMJMcCZwLPaMv8cZL9kuwHvI/uWQPHAq9oZSXNMoO2NEImepZAVX2yqh5os9fS3fwIumcJXFZV36uqb9D9lPP49tpSVbdW1feBy1pZSbPMoC1p0GuBv2nTPktAWmC8pi0JgCS/DTwAfHCm1llVG4ANACtXrpyxZxTMJe9cqIXEoC2JJK+mG6C2qt0kCfb8zACfJSDNA7vHpRHXRoK/BXhJVd0/kLURODPJAUmWAyuAz9HdjnhFkuVJ9qcbrLZxrustjSLPtKURsptnCZwNHABsSgJwbVX9SlXdmOQK4Ca6bvOzquoHbT2vB66m+8nXRVV145zvjDSCDNrSCNnNswQu3EP5c4FzJ0i/iu4BQZLmkN3jkiT1hEFbkqSeMGhLktQTBm1JknrCoC1JUk84elzSyPJpfOobz7QlSeoJg7YkST1h0JYkqScM2pIk9YRBW5KknphW0E7yG0luTPKVJB9K8pj25J/rkmxJcnl7ChDtSUGXt/TrkiybiR2QJGlU7HPQTrIE+DVgZVU9k+5pP2cC7wTOq6pjgHuANW2RNcA9Lf28Vk6SJE3RdLvHFwGPTbIIOBDYAZwMXNnyLwFe2qZPb/O0/FVpzwGUJEmT2+egXVXbgXcBd9AF6/uA64F7q+qBVmwbsKRNLwG2tmUfaOUPG7/eJGuTbE6yedeuXftaPUmShs50uscPoTt7Xg48BXgccMp0K1RVG6pqZVWtXLx48XRXJ2lAkouS7EzylYG0Q5NsSnJLez+kpSfJe9o4lBuSHDewzOpW/pYkq+djX6RRNJ3u8ecD36iqXVX1H8CHgZOAg1t3OcBSYHub3g4cBdDyDwLumsb2Je29i3nkwfU64JqqWgFc0+YBTgVWtNda4ALogjxwDnACcDxwzliglzS7phO07wBOTHJguza9CrgJ+DRwRiuzGvhom97Y5mn5n6qqmsb2Je2lqvoscPe45MHxJuPHoVxanWvpDsiPBF4IbKqqu6vqHmATM9DLJmly07mmfR3dgLIvAF9u69oAvBV4U5ItdNesL2yLXAgc1tLfxENH85Lm1xFVtaNN3wkc0aYfHIfSjI1R2V36IzhGRZpZ03rKV1WdQ9dNNuhWui6z8WW/C7xsOtuTNLuqqpLMWA9YVW2gO5hn5cqV9qxJ0+Qd0SR9s3V70953tvQHx6E0Y2NUdpcuaZYZtCUNjjcZPw7lVW0U+YnAfa0b/WrgBUkOaQPQXtDSJM2yaXWPS+qXJB8CngccnmQb3eWt9cAVSdYAtwMvb8WvAk4DtgD3A68BqKq7k/w+8PlW7u1VNX5wm6RZYNCWRkhVvWI3WasmKFvAWbtZz0XARTNYNUlTYPe4JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JACS/EaSG5N8JcmHkjwmyfIk1yXZkuTyJPu3sge0+S0tf9n81l4aDQZtSSRZAvwasLKqngnsB5wJvBM4r6qOAe4B1rRF1gD3tPTzWjlJs2xaQTvJwUmuTPLVJDcneU6SQ5NsSnJLez+klU2S97Qj8xuSHDczuyBphiwCHptkEXAgsAM4Gbiy5V8CvLRNn97mafmrkmQO6yqNpOmeaZ8PfKKqfgx4FnAzsA64pqpWANe0eYBTgRXttRa4YJrbljRDqmo78C7gDrpgfR9wPXBvVT3Qim0DlrTpJcDWtuwDrfxh49ebZG2SzUk279q1a3Z3QhoB+xy0kxwE/AxwIUBVfb+q7uXhR+Djj8wvrc61wMFJjtznmkuaMa1H7HRgOfAU4HHAKdNdb1VtqKqVVbVy8eLF012dNPKmc6a9HNgF/FmSLyb5QJLHAUdU1Y5W5k7giDb94JF5M3jULml+PR/4RlXtqqr/AD4MnER3cL2olVkKbG/T24GjAFr+QcBdc1tlafRMJ2gvAo4DLqiqZwPf4aGucACqqoDam5XanSbNizuAE5Mc2K5NrwJuAj4NnNHKrAY+2qY3tnla/qdae5c0i6YTtLcB26rqujZ/JV0Q/+ZYt3d739nyHzwybwaP2h9kd5o091o7vhL4AvBluu+GDcBbgTcl2UJ3zfrCtsiFwGEt/U2MO2CXNDsWTV5kYlV1Z5KtSZ5eVV/joSPzm+iOwNfzyCPz1ye5DDgBuG+gG13SPKuqc4BzxiXfChw/QdnvAi+bi3pJesg+B+3mDcAH2w0XbgVeQ3eEfkWSNcDtwMtb2auA04AtwP2trCRJmqJpBe2q+hKwcoKsVROULeCs6WxPkqRR5h3RJEnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPWEQVuSpJ4waEsCIMnBSa5M8tUkNyd5TpJDk2xKckt7P6SVTZL3JNmS5IYkx813/aVRMN3naWuWLVv38UnL3Lb+RXNQE42A84FPVNUZSfYHDgTeBlxTVeuTrAPWAW8FTgVWtNcJwAXtXdIs8kxbEkkOAn4GuBCgqr5fVfcCpwOXtGKXAC9t06cDl1bnWuDgJEfOcbWlkeOZtiSA5cAu4M+SPAu4HngjcERV7Whl7gSOaNNLgK0Dy29raTsG0kiyFlgLcPTRR89a5efbVHrEwF4xTZ9n2pKgO4A/Drigqp4NfIeuK/xBVVVA7c1Kq2pDVa2sqpWLFy+escpKo8qgLQm6M+VtVXVdm7+SLoh/c6zbu73vbPnbgaMGll/a0iTNIoO2JKrqTmBrkqe3pFXATcBGYHVLWw18tE1vBF7VRpGfCNw30I0uaZZ4TVvSmDcAH2wjx28FXkN3YH9FkjXA7cDLW9mrgNOALcD9raykWWbQlgRAVX0JWDlB1qoJyhZw1qxXStLD2D0uSVJPTDtoJ9kvyReTfKzNL09yXbtT0uWtq40kB7T5LS1/2XS3LUnSKJmJ7vE3AjcDT2zz7wTOq6rLkvwJsIbubklrgHuq6pgkZ7ZyvzAD25ekXvAOh5quaZ1pJ1kKvAj4QJsPcDLdz0XgkXdQGruz0pXAqlZekiRNwXS7x98NvAX4YZs/DLi3qh5o82N3SYKBOyi1/Pta+YdJsjbJ5iSbd+3aNc3qSZI0PPY5aCd5MbCzqq6fwfp4ByVJknZjOte0TwJekuQ04DF017TPp3twwKJ2Nj14l6SxOyhtS7IIOAi4axrblyRppOzzmXZVnV1VS6tqGXAm8KmqeiXwaeCMVmz8HZTG7qx0Riu/V/cxliRplM3G77TfCrwpyRa6a9YXtvQLgcNa+psY9zACSZK0ZzNyR7Sq+gzwmTZ9K3D8BGW+C7xsJrY33lQfizes/BmJhsWot2VpMt4RTZKknjBoS5LUEwZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0Jb0IB+1Ky1sBm1Jg8YetTtm7FG7xwD30D1iFwYetQuc18pJmmUGbUmAj9qV+sCgLWmMj9qVFjiDtiQftSv1xIzce1xS7/moXakHPNOW5KN2pZ4waEvaEx+1Ky0gdo9Lepj5ftSupN3zTFuSpJ4waEuS1BMGbUmSesKgLUlSTxi0JUnqCYO2JEk9YdCWJKknDNqSJPXEPgftJEcl+XSSm5LcmOSNLf3QJJuS3NLeD2npSfKeJFuS3JDkuJnaCUmSRsF0zrQfAN5cVccCJwJnJTmW7naG11TVCuAaHrq94anAivZaC1wwjW1LkjRy9jloV9WOqvpCm/5X4Ga6Z+yeDlzSil0CvLRNnw5cWp1r6Z4edOQ+11ySpBEzI9e0kywDng1cBxxRVTta1p3AEW16CbB1YLFtLW38utYm2Zxk865du2aiepIkDYVpPzAkyeOBvwJ+vaq+neTBvKqqJHv1uL6q2gBsAFi5cqWP+pshy9Z9fErlblv/olmuiSRpX03rTDvJo+kC9ger6sMt+Ztj3d7tfWdL3w4cNbD40pYmSZKmYDqjx0P3TN2bq+qPBrI2Aqvb9GrgowPpr2qjyE8E7hvoRpckSZOYzpn2ScB/A05O8qX2Og1YD/xckluA57d5gKuAW4EtwJ8CvzqNbUuaQf6EU+qHfb6mXVV/D2Q32asmKF/AWfu6PUmzauwnnF9I8gTg+iSbgFfT/YRzfZJ1dD/hfCsP/wnnCXQ/4TxhXmo+ZKYy/sSxJ6PLO6JJ8iecUk8YtCU9jD/hlBYug7akB43/CedgXrvEtdc/4ayqlVW1cvHixTNYU2k0Tft32pKGw55+wllVO/wJ58Lhde/R5Zm2JH/CKfWEZ9raax7lD6Wxn3B+OcmXWtrb6H6yeUWSNcDtwMtb3lXAaXQ/4bwfeM3cVlcaTQZtPcxUb3eq4eJPOKV+sHtckqSeMGhLktQTBm1JknrCoC1JUk8YtCVJ6gmDtiRJPWHQliSpJwzakiT1hEFbkqSe8I5okjSEvN3wcDJoS9KImuptiw3uC4dBW7PCo3xJmnle05YkqScM2pIk9YTd45o3Xk+TpL1j0NaC5/VxaX7ZBheOOe8eT3JKkq8l2ZJk3VxvX9LMsC1Lc29Oz7ST7Ae8D/g5YBvw+SQbq+qmuayHho9nAnPLtqzxpnq5azK20z2b6+7x44EtVXUrQJLLgNMBG7pm3UIM7AuxTlNkW9ascKzLns110F4CbB2Y3wacMFggyVpgbZv9tyR3Ad+am+rNq8MZ/v1c8PuYd87IamZ0P6dYp6fO1PamaNK2DBO256/NQd3myoL/f94HvdmnvWyrvdmvZrftecENRKuqDcCGsfkkm6tq5TxWaU6Mwn6Owj7C6OznVIxvz8NkGD/nYdwnGK79muuBaNuBowbml7Y0Sf1iW5bmwVwH7c8DK5IsT7I/cCawcY7rIGn6bMvSPJjT7vGqeiDJ64Grgf2Ai6rqxkkWG8qutQmMwn6Owj7CCOznPrblYTOMn/Mw7hMM0X6lqua7DpIkaQq897gkST1h0JYkqScWdNAextskJjkqyaeT3JTkxiRvbOmHJtmU5Jb2fsh813UmJNkvyReTfKzNL09yXftML2+DmHorycFJrkzy1SQ3J3nOsH6Wo2rY2+ywtdFhb5MLNmgP3CbxVOBY4BVJjp3fWs2IB4A3V9WxwInAWW2/1gHXVNUK4Jo2PwzeCNw8MP9O4LyqOga4B1gzL7WaOecDn6iqHwOeRbevw/pZjqphb7PD1kaHu01W1YJ8Ac8Brh6YPxs4e77rNQv7+VG6+zd/DTiypR0JfG2+6zYD+7aUroGcDHwMCN1diRZN9Bn37QUcBHyDNqBzIH3oPktfD/t8h6bNDlsbHYU2uWDPtJn4NolL5qkusyLJMuDZwHXAEVW1o2XdCRwxT9WaSe8G3gL8sM0fBtxbVQ+0+b5/psuBXcCfte7FDyR5HMP5WYqhbLPD1kaHvk0u5KA91JI8Hvgr4Ner6tuDedUdDvb6t3hJXgzsrKrr57sus2gRcBxwQVU9G/gO47rdhuGzVGfY2uyQttGhb5MLOWgP7W0SkzyarvF/sKo+3JK/meTIln8ksHO+6jdDTgJekuQ24DK67rfzgYOTjN3Up++f6TZgW1Vd1+avpPvCGLbPcuQNaZsdxjY69G1yIQftobxNYpIAFwI3V9UfDWRtBFa36dV01816q6rOrqqlVbWM7rP7VFW9Evg0cEYr1uv9rKo7ga1Jnt6SVtE9mnKoPstRN6xtdhjb6Ci0yQV9R7Qkp9Fdcxm7TeK581ylaUvyXOD/Al/moetIb6O7RnYFcDRwO/Dyqrp7Xio5w5I8D/jNqnpxkh+hO6o/FPgi8ItV9b35rN90JPlJ4APA/sCtwGvoDoaH8rMcRaPQZoepjQ57m1zQQVuSJD1kIXePS5KkAQZtSZJ6wqAtSVJPGLQlSeoJg7YkST1h0JYkqScM2pIk9cT/B/odrf8G1COhAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "src_length = map(len, [vars(x)['src'] for x in test_data.examples])\n", - "trg_length = map(len, [vars(x)['trg'] for x in test_data.examples])\n", - "\n", - "print('Length distribution in Test data')\n", - "plt.figure(figsize=[8, 4])\n", - "plt.subplot(1, 2, 1)\n", - "plt.title(\"source length\")\n", - "plt.hist(list(src_length), bins=20);\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.title(\"translation length\")\n", - "plt.hist(list(trg_length), bins=20);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model side\n", - "__Here comes simple pipeline of NMT model learning. It almost copies the week03 practice__" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "device(type='cuda', index=1)" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "device" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def _len_sort_key(x):\n", - " return len(x.src)\n", - "\n", - "BATCH_SIZE = 128\n", - "\n", - "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE, \n", - " device = device,\n", - " sort_key=_len_sort_key\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "[torchtext.data.batch.Batch of size 128]\n", - "\t[.trg]:[torch.cuda.LongTensor of size 55x128 (GPU 1)]\n", - "\t[.src]:[torch.cuda.LongTensor of size 59x128 (GPU 1)]\n", - "torch.Size([59, 128]) torch.Size([55, 128])\n" - ] - } - ], - "source": [ - "for x in train_iterator:\n", - " break\n", - "print(x)\n", - "print(x.src.shape, x.trg.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "import my_network\n", - "Encoder = my_network.Encoder\n", - "Decoder = my_network.Decoder\n", - "Seq2Seq = my_network.Seq2Seq" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(SRC.vocab)\n", - "OUTPUT_DIM = len(TRG.vocab)\n", - "ENC_EMB_DIM = 256\n", - "DEC_EMB_DIM = 256\n", - "HID_DIM = 512\n", - "N_LAYERS = 2\n", - "ENC_DROPOUT = 0.5\n", - "DEC_DROPOUT = 0.5\n", - "\n", - "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT)\n", - "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)\n", - "\n", - "# dont forget to put the model to the right device\n", - "model = Seq2Seq(enc, dec, device).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Seq2Seq(\n", - " (encoder): Encoder(\n", - " (embedding): Embedding(9267, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - " (decoder): Decoder(\n", - " (embedding): Embedding(6699, 256)\n", - " (rnn): LSTM(256, 512, num_layers=2, dropout=0.5)\n", - " (out): Linear(in_features=512, out_features=6699, bias=True)\n", - " (dropout): Dropout(p=0.5, inplace=False)\n", - " )\n", - ")" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def init_weights(m):\n", - " # \n", - " for name, param in m.named_parameters():\n", - " nn.init.uniform_(param, -0.08, 0.08)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The model has 14,880,299 trainable parameters\n" - ] - } - ], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [], - "source": [ - "PAD_IDX = TRG.vocab.stoi['']\n", - "optimizer = optim.Adam(model.parameters())\n", - "criterion = nn.CrossEntropyLoss(ignore_index = PAD_IDX)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, clip, train_history=None, valid_history=None):\n", - " model.train()\n", - " \n", - " epoch_loss = 0\n", - " history = []\n", - " for i, batch in enumerate(iterator):\n", - " \n", - " src = batch.src\n", - " trg = batch.trg\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " output = model(src, trg)\n", - " \n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - " \n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - " \n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - " \n", - " loss = criterion(output, trg)\n", - " \n", - " loss.backward()\n", - " \n", - " # Let's clip the gradient\n", - " torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " history.append(loss.cpu().data.numpy())\n", - " if (i+1)%10==0:\n", - " fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8))\n", - "\n", - " clear_output(True)\n", - " ax[0].plot(history, label='train loss')\n", - " ax[0].set_xlabel('Batch')\n", - " ax[0].set_title('Train loss')\n", - " if train_history is not None:\n", - " ax[1].plot(train_history, label='general train history')\n", - " ax[1].set_xlabel('Epoch')\n", - " if valid_history is not None:\n", - " ax[1].plot(valid_history, label='general valid history')\n", - " plt.legend()\n", - " \n", - " plt.show()\n", - "\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "def evaluate(model, iterator, criterion):\n", - " \n", - " model.eval()\n", - " \n", - " epoch_loss = 0\n", - " \n", - " history = []\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for i, batch in enumerate(iterator):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output[1:].view(-1, output.shape[-1])\n", - " trg = trg[1:].view(-1)\n", - "\n", - " #trg = [(trg sent len - 1) * batch size]\n", - " #output = [(trg sent len - 1) * batch size, output dim]\n", - "\n", - " loss = criterion(output, trg)\n", - " \n", - " epoch_loss += loss.item()\n", - " \n", - " return epoch_loss / len(iterator)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "def epoch_time(start_time, end_time):\n", - " elapsed_time = end_time - start_time\n", - " elapsed_mins = int(elapsed_time / 60)\n", - " elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n", - " return elapsed_mins, elapsed_secs" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "train_history = []\n", - "valid_history = []\n", - "\n", - "N_EPOCHS = 10\n", - "CLIP = 1\n", - "\n", - "best_valid_loss = float('inf')" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHwCAYAAABUqPIVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdeXxU1f3/8dfJAmFPAggikOCGInuCG7hiFTfcW6y20tri1tp+269V237VWv3Vqm1ta9WqdWndt7buxVYUFyqbgKIoLkEWhbAJAQJJ5vz+uDOTmclMZu5kMnfuzPvpAzOZucuZJclnPvM5n2OstYiIiIiISKsirwcgIiIiIpJrFCSLiIiIiMRQkCwiIiIiEkNBsoiIiIhIDAXJIiIiIiIxFCSLiIiIiMRQkCx5xRjzgjHmvDT3rTPGHJPpMYmIiIj/lHg9ABFjTEPEt92BnUBL8PsLrLUPpnosa+3xmRybiIiIFCYFyeI5a23P0GVjTB3wHWvtv2O3M8aUWGubszk2ERERKUwqt5CcZYw50hizyhhzuTHmC+BeY0yFMeZZY0y9MWZT8PLgiH1eMcZ8J3h5ujHmdWPMzcFtPzXGpJRpNsZ0NcbcYoxZE/x3izGma/C2fsHzbjbGbDTGvGaMKQredrkxZrUxZqsx5gNjzOROeGhERESkkylIllw3EKgEqoAZOK/Ze4PfDwV2ALe2s/9BwAdAP+BG4C/GGJPCeX8GHAyMBcYABwI/D972Y2AV0B8YAPwUsMaY4cD3gAnW2l7AcUBdivdTREREcoiCZMl1AeBqa+1Oa+0Oa+0Ga+2T1trt1tqtwPXAEe3sv8Jae5e1tgW4H9gdJ7BN5hzgWmvtOmttPfAL4BvB25qCx6my1jZZa1+z1lqcOuquwAhjTKm1ts5a+3Fa91pEREQ8pSBZcl29tbYx9I0xprsx5s/GmBXGmC3AbKDcGFOcYP8vQhestduDF3sm2DbSIGBFxPcrgtcB3AR8BMw0xnxijLkiePyPgB8C1wDrjDGPGGMGISIiIr6jIFlynY35/sfAcOAga21v4PDg9amUULixBqekI2Ro8DqstVuttT+21u4JTAV+FKo9ttY+ZK2dFNzXAr/O8LhEREQkCxQki9/0wqlD3myMqQSu7qTzPAz83BjT3xjTD7gKeADAGHOSMWbvYG3zlzhlFgFjzHBjzNHBCX6NwXEGOml8IiIi0okUJIvf3AJ0A9YD/wVe7KTzXAfMB5YA7wALg9cB7AP8G2gA5gC3WWtn4dQj3xAc2xfAbsCVnTQ+ERER6UTGmW8kIiIiIiIhyiSLiIiIiMRQkCwiIiIiEkNBsoiIiIhIDAXJIiIiIiIxFCSLiIiIiMQo8XoA8fTr189WV1d7PQwREdcWLFiw3lrb3+txZJN+Z4uIX7X3Ozsng+Tq6mrmz5/v9TBERFwzxqxIvlV+0e9sEfGr9n5nq9xCRERERCSGgmQRERERkRg5WW4hIiKdwxhTB2wFWoBma21tzO1HAv8EPg1e9ZS19tpsjlFEJBcoSBYRKTxHWWvXt3P7a9bak7I2GpEOaGpqYtWqVTQ2Nno9FMlhZWVlDB48mNLS0pT3UZAsIiIivrVq1Sp69epFdXU1xhivhyM5yFrLhg0bWLVqFcOGDUt5P9Uki4gUFgvMNMYsMMbMSLDNIcaYxcaYF4wxB2RzcCJuNTY20rdvXwXIkpAxhr59+7r+tEGZZBGRwjLJWrvaGLMb8JIxZpm1dnbE7QuBKmttgzHmBOAfwD6xBwkG2DMAhg4dmo1xiySkAFmSSec1okyyiEgBsdauDn5dB/wdODDm9i3W2obg5eeBUmNMvzjHudNaW2utre3fv6DWThHJaUceeWTcvuW33HIL27dvd328q666in//+98pb3/ffffxve99L+5tJ5xwAps3b064b7pj7CwKkkVECoQxpocxplfoMnAs8G7MNgNNMOVijDkQ5+/EhmyPVUTis9YSCARc79deANrS0pJwv2uvvZZjjjnG9fnief755ykvL094ezpBcntj7ygFySIihWMA8LoxZjEwF3jOWvuiMeZCY8yFwW3OBN4NbvMHYJq11no0XhFf+OUvf8nw4cOZNGkSZ599NjfffDMAH3/8MVOmTKGmpobDDjuMZcuWATB9+nQuvfRSDj30UPbcc0+eeOKJ8LFuuukmJkyYwOjRo7n66qsBqKurY/jw4Xzzm99k5MiRrFy5kosuuoja2loOOOCA8HaJ/OEPf2DNmjUcddRRHHXUUQD07NmTH//4x4wZM4Y5c+Zw7bXXMmHCBEaOHMmMGTMI/dhPnz49PL7q6mquvvpqxo8fz6hRo8L3J9aaNWuYMmUK++yzDz/5yU/C11dXV7N+/Xq2bdvGiSeeyJgxYxg5ciSPPvpo3DE+/PDDjBo1ipEjR3L55ZeHjxM59uuvv55TTz01fNtLL73EaaedlsKzlpxqkkVECoS19hNgTJzr74i4fCtwazbHJZIpv3hmKe+t2ZLRY44Y1JurT048f3XevHk8+eSTLF68mKamJsaPH09NTQ0AM2bM4I477mCfffbhrbfe4uKLL+bll18G4PPPP+f1119n2bJlTJ06lTPPPJOZM2eyfPly5s6di7WWqVOnMnv2bIYOHcry5cu5//77OfjggwG4/vrrqayspKWlhcmTJ7NkyRJGjx4dd4yXXnopv/3tb5k1axb9+jnVU9u2beOggw7iN7/5jXM/R4zgqquuAuAb3/gGzz77LCeffHKbY/Xr14+FCxdy2223cfPNN3P33Xe32WbRokW8/fbbdO3aleHDh/P973+fIUOGhG9/8cUXGTRoEM899xwAX375JX369Ika45o1a7j88stZsGABFRUVHHvssfzjH//g1FNPjRq7tZb999+f+vp6+vfvz7333su3v/3t9p/UFCmTLCIiIpKmN954g1NOOYWysjJ69eoVDiwbGhp48803Oeussxg7diwXXHABn3/+eXi/U089laKiIkaMGMHatWsBmDlzJjNnzmTcuHGMHz+eZcuWsXz5cgCqqqrCATLAY489xvjx4xk3bhxLly7lvffeczXu4uJizjjjjPD3s2bN4qCDDmLUqFG8/PLLLF26NO5+p59+OgA1NTXU1dXF3Wby5Mn06dOHsrIyRowYwYoVK6JuHzVqFC+99BKXX345r732Gn369GlzjHnz5nHkkUfSv39/SkpKOOecc5g9e3absRtj+MY3vsEDDzzA5s2bmTNnDscff7yrxyIRZZJFREQkL7SX8c22QCBAeXk5ixYtint7165dw5dDpQ3WWq688kouuOCCqG3r6uro0aNH+PtPP/2Um2++mXnz5lFRUcH06dNdtzcrKyujuLgYcNroXXzxxcyfP58hQ4ZwzTXXJDxeaNzFxcU0NzcnvW/xttt3331ZuHAhzz//PD//+c+ZPHlyOIvtduwA3/rWtzj55JMpKyvjrLPOoqQkM+GtMskiIiIiaZo4cSLPPPMMjY2NNDQ08OyzzwLQu3dvhg0bxuOPPw44AfDixYvbPdZxxx3HPffcQ0NDAwCrV69m3bp1bbbbsmULPXr0oE+fPqxdu5YXXngh6Th79erF1q1b494WCoj79etHQ0NDVI10Z1izZg3du3fn3HPP5bLLLmPhwoVtxnjggQfy6quvsn79elpaWnj44Yc54ogj4h5v0KBBDBo0iOuuu45vfetbGRunMskiIiIiaZowYQJTp05l9OjRDBgwgFGjRoXLBx588EEuuugirrvuOpqampg2bRpjxrSZFhB27LHH8v7773PIIYcAzgS1Bx54ICprCjBmzBjGjRvHfvvtx5AhQ5g4cWLScc6YMYMpU6YwaNAgZs2aFXVbeXk53/3udxk5ciQDBw5kwoQJbh8GV9555x0uu+wyioqKKC0t5fbbb487xhtuuIGjjjoKay0nnngip5xySsJjnnPOOdTX17P//vtnbJwmFyct19bW2ng9/kREcp0xZoG1ttbrcWSTfmeLl95///2MBkbpaGhooGfPnmzfvp3DDz+cO++8k/Hjx3s6pkLzve99j3HjxnH++ecn3Cbea6W939nKJIuISPZt+Bj67uX1KEQyYsaMGbz33ns0NjZy3nnnKUDOspqaGnr06BHu1JEpCpJFJKf86LFF9Opawi9OGen1UKSz1H8Id0yCkafD8TdCWW+vRyTSIQ899JDXQyhoCxYs6JTjauKeiOSUj+u38cn6bV4PQzpT5TCY9ENY8qgTLH/2ltcjEhFpQ0GyiIhkV3EpHPVT+NYLgIV7p8CsX0FL/HZSIiJeUJAsIrnFWnJwPrF0hqEHw4VvwOivwas3OMHyxk+8HpWICKAgWURyjAUsipILRllvOO0OOPOeYK3yYfD2g+idkoh4TUGyiOQUaxUfFaSRZ8BFb8DuY+GfF8Pj02H7Rq9HJeI7Rx55JJloyRh5nBNOOIHNmze32eaaa67h5ptvbnP99OnT4y5IsmbNGs4888yE59y8eTO33XZbB0adWQqSRSSnWFRuUbDKh8B5T8Pkq2HZs3D7RPh0ttej6nyBACx+FP50MLx5q9ejkRxnrSUQCGT1nM8//zzl5eUdPs6gQYPaXc0vnSA50dLYmaAgWURyigLkAldUDIf9CL7zb+jSHe6fCjP/D5p3eT2yzLMWPvwX/Pkw+PsM2FQHr/xKGXQf+uUvf8nw4cOZNGkSZ599dji7+vHHHzNlyhRqamo47LDDWLZsGeBkWi+99FIOPfRQ9txzz6jA8aabbmLChAmMHj2aq6++GoC6ujqGDx/ON7/5TUaOHMnKlSu56KKLqK2t5YADDghvl8iLL77IWWedFf7+lVde4aSTTgJI6TjV1dWsX78egOuvv559992XSZMm8cEHHyQ85+zZs9vcv7q6OkaOdNp7Ll26lAMPPJCxY8cyevRoli9fzhVXXMHHH3/M2LFjueyyy7DWctlllzFy5EhGjRrFo48+Gh7/YYcdxtSpUxkxYgRXXXUVt9xyS/jcP/vZz/j973/f7mOSCvVJFpGco5pkYdA4uGA2/Otn8OYf4JNX4Iy7of9wr0eWGZ/9F/59DXw2Byr3dGqy++8Ptx/q3N9jrvF4gD71whXwxTuZPebAUXD8DQlvnjdvHk8++SSLFy+mqamJ8ePHU1NTAziLjNxxxx3ss88+vPXWW1x88cW8/PLLAHz++ee8/vrrLFu2jKlTp3LmmWcyc+ZMli9fzty5c7HWMnXqVGbPns3QoUNZvnw5999/PwcffDDgBKuVlZW0tLQwefJklixZwujRo+OO8ZhjjmHGjBls27aNHj168OijjzJt2jTXx1mwYAGPPPIIixYtorm5Oeq+xop3/yLdcccd/OAHP+Ccc85h165dtLS0cMMNN/Duu++yaNEiAJ588kkWLVrE4sWLWb9+PRMmTODwww8HYOHChbz77rsMGzaMuro6Tj/9dH74wx8SCAR45JFHmDt3bsLnLFUKkkUkp6gmWcK69ICTb4F9vgJPfx/+fAQcdx3Ung/GeD269KxdCv/5JXz4AvQcACf+FsZ/02mLB05t9lt3wsGXQM/+3o5VUvLGG29wyimnUFZWRllZGSeffDLgLFX95ptvRmVwd+7cGb586qmnUlRUxIgRI1i7di0AM2fOZObMmYwbNy58jOXLlzN06FCqqqrCATLAY489xp133klzczOff/457733XsLgtqSkhClTpvDMM89w5pln8txzz3HjjTe6Ps5rr73GaaedRvfu3QGYOnVqwscl3v2LdMghh3D99dezatUqTj/9dPbZZ58227z++uucffbZFBcXM2DAAI444gjmzZtH7969OfDAAxk2bBjgZLr79u3L22+/zdq1axk3bhx9+/ZNOLZUKUgWkZxig/9EwvY7EfaogX9cDM/9GJa/BFNv9VcQuWmFU0qx+BHo2hsmXwUHXei8EYh0xOWw9Cl48/dw7HXejNXP2sn4ZlsgEKC8vDycFY3VtWvX8GUbzAxYa7nyyiu54IILoratq6ujR4/W18qnn37KzTffzLx586ioqGD69Ok0Nja2O55p06Zx6623UllZSW1tLb169UrrOKmKd/8iff3rX+eggw7iueee44QTTuDPf/4ze+65Z8rHj3w8AL7zne9w33338cUXX/Dtb387/YFHUE2yiOQUa62iZGmr10A45wmY8mv4eBbcfogTLOe6hnp44XL4Yw0s/TtMvBR+sAgO+3HbABmg/74w6qsw927Y2jb7Jrln4sSJPPPMMzQ2NtLQ0MCzzz4LQO/evRk2bBiPP/444PxuW7x4cbvHOu6447jnnntoaGgAYPXq1axbt67Ndlu2bKFHjx706dOHtWvX8sILLyQd5xFHHMHChQu56667wqUWbo9z+OGH849//IMdO3awdetWnnnmmaTnTeSTTz5hzz335NJLL+WUU05hyZIl9OrVi61bt4a3Oeyww3j00UdpaWmhvr6e2bNnc+CBB8Y93mmnncaLL77IvHnzOO6449IeVyRlkkVExB+KiuDgC2HYYfDkd+HBM+HAGfCVa6G0m9eji9a4Beb8CebcCk07YNy5Tpa4zx7J9z3iJ/DO4/DGLTDlV50/VumQCRMmMHXqVEaPHs2AAQMYNWoUffr0AeDBBx/koosu4rrrrqOpqYlp06YxZsyYhMc69thjef/99znkkEMA6NmzJw888ADFxcVR240ZM4Zx48ax3377MWTIECZOnJh0nMXFxZx00kncd9993H///WkdZ/z48Xzta19jzJgx7LbbbkyYMCHpeRN57LHH+Nvf/kZpaSkDBw7kpz/9KZWVlUycOJGRI0dy/PHHc+ONNzJnzhzGjBmDMYYbb7yRgQMHhidARurSpQtHHXUU5eXlbR6vdJl4KXCv1dbW2kz0+BMR/znud7Pp3a2Exy881OuhpMUYs8BaW+v1OLLJk9/ZTY3wn1/Af2+D/vs5k/oGjsruGOJp3gnz74HZN8H2DTDiFDj6/6Bf23rLdv3jEidQ/sFi6L1754w1T7z//vvsv//+no6hoaGBnj17sn37dg4//HDuvPNOxo8f7+mYCk0gEGD8+PE8/vjjceubIf5rpb3f2Sq3EJGck4Pv3SXXlJY5WdZzn4Idm+Cuo50ew1nuHxsWaIFFDzllFS9eAQNGwndfhq/+1X2ADHD4/4Jtgdd/m/mxSsbNmDGDsWPHMn78eM444wwFyFn23nvvsffeezN58uSEAXI6VG4hIjnFqgGcuLH3ZLhojtP9YubP4KOX4NTbofeg7JzfWvjgBfjPtVD/vrNi4NQ/wl5Hdey4lcNg7Dmw4D6Y+APoMzgjw5XO8dBDD3k9hII2YsQIPvnkk4wfV5lkEckpTgs4hcniQo++MO1BOPn3sHKu02v4vX92/nlXvAn3HAePnA0tu+Cs++C7szoeIIccfpnzA/HabzJzPBFxRUGyiOQUhceSFmOgZjpc8BqUV8Fj34R/XgI7GzJ/ri/egQfPgnuPh82fOcH5JW/BAac5kwszpXwI1JwHC//mtJCThPTGWpJJ5zWiIFlEcoq1KreQDui3N5z/Ekz6Ebz9oLPk86oFmTn2xk+drhp3HAYr34JjfgHfX+gE56HFQDJt0o/AFDkTASWusrIyNmzYoEBZErLWsmHDBsrKylztp5pkEckpFk3ckw4q6QLHXA17HwN/vwD+8hU48ko47EdQlEZrqIZ18OqNTn1wUQlM+qFTJ9ytIuNDb6PPHlD7LZh7lzP+ytQXWygUgwcPZtWqVdTX13s9FMlhZWVlDB7srrZfQbKI5BzFyPmtsamFOZ9sYPiAXgwq78T+xtUT4cLXnVX6Zl0HH/0bTr8TKqpSHOiX8OYfYc5t0NzolD4c/pPst2Sb9D9OgP7qTXDa7dk9tw+UlpaGlycWySSVW4hIblEqOe9t2LaLb907j5lLv+j8k3UrhzP/AqffBevegzsmwZLH2t+nqdEJjn8/xilz2Pc4+N48OOl33vQs7jUQJnwHljwC6z/K/vlFCpSCZBHJKQqP898e5d0Y1KeM+Ss2Ze+ko7/qZJV3GwFPfReeOB92bI7epqXZmST3x/Ew8+cwaBzMeAXOuhf67pW9scYz8QdQUgav/trbcYgUEAXJIpJTNHGvMNRUVzK/blN2J1tVVMH05+Con8PSvztZ5bo3nE8u3n/GaR339PeczO03n4Zv/N0JlHNBz93gwO86q/DVf+D1aEQKgoJkEckpqrYoDBOqK/hiSyOrN+/I7omLS+CIy+D8mc4kvPtOhNsOhkfPBRuAr/4NvvMf2POI7I4rFYf+ALr0gFdu8HokIgVBQbKI5BRr0Zp7BaCmyukMMb8uiyUXkQbXOuUX478BzTudVfIu/i+MmOr0XM5FPfrCQRc4WfC1S70ejUjeU5AsIjlHmeT8t9/A3vTsWsL8FRu9G0TXnk5w/INFMP6bTpY51x3yPejaS9lkkSxQkCwiOcViFSQXgOIiw7ih5d5lkv2qeyUcfDG8/zR8vsTr0YjkNQXJIpJTFCAXjtqqSj5Yu5UvdzR5PRR/OfgiKOujbLJIJ0s5SDbGFBtj3jbGPBvntq7GmEeNMR8ZY94yxlRH3HZl8PoPjDHHZWbYIpKvnJpkKQQTqiuwFt7+TNlkV7qVwyHfhw+egzVvez0akbzlJpP8A+D9BLedD2yy1u4N/A74NYAxZgQwDTgAmALcZoxJY01QESkkWW0LJp4ZO7Sc4iKjkot0HHSBsyz2rP/n9UhE8lZKQbIxZjBwInB3gk1OAe4PXn4CmGyMMcHrH7HW7rTWfgp8BBzYsSGLSD5TgFw4uncp4YBBvb2dvOdXZb3h0Eth+UxYOc/r0YjkpVQzybcAPwECCW7fA1gJYK1tBr4E+kZeH7QqeJ0ErdvSyPqGnV4PQySnKE4uHDVVFSxauZmmlkR/XiShA2dA977wirLJIp0haZBsjDkJWGetXdCZAzHGzDDGzDfGzK+vr+/MU+WUHz++mKv++a7XwxDJGYqPC0ttVSWNTQGWrtni9VD8p2tPmPhD+PhlWDHH69GI5J1UMskTganGmDrgEeBoY8wDMdusBoYAGGNKgD7AhsjrgwYHr2vDWnuntbbWWlvbv39/V3fCz7Y0NrO1sdnrYYjkDC0mUlhqq0OLiqjkIi0TvgM9dlM2WaQTJA2SrbVXWmsHW2urcSbhvWytPTdms6eB84KXzwxuY4PXTwt2vxgG7APMzdjo84FVT1iRSOqTXFgG9C5jSGU3Td5LV5fuMOl/4NPZ8OlrXo9GJK+k3SfZGHOtMWZq8Nu/AH2NMR8BPwKuALDWLgUeA94DXgQusda2dGzI+cUCAUUEImFqAVd4JlRVMn/FJk3aTFftt6DnQHjlVyroF8kgV0GytfYVa+1JwctXWWufDl5utNaeZa3d21p7oLX2k4h9rrfW7mWtHW6tfSGzw/c/a/U7TSSSRR0uCk1NdQXrG3ayYsN2r4fiT6Xd4LAfw4o34NNXvR6NSN7Qinses8H/REQK1YTqSgDmr1DJRdrGfxN67+H0TdabTJGMUJDsMWWSRaKp3KLw7N2/J73LSjR5ryNKy+Dw/4WVb8HH//F6NCJ5QUFyDlBAIBJJUXKhKSoy1FRVKJPcUWPPhT5DlU0WyRAFyR6zFgUEIhGUSS5MtdWVfLSugU3bdnk9FP8q6QJHXAarFzgr8YlIhyhI9pi6W4hE08S9wlRb5fRLXqBscseMORsqqmHW9comi3SQgmSPWatpeyKRFCAXpjFDyiktNiq56KjiUjjicvh8MSx7zuvRiPiaguQcoKBAJJp+IgpPWWkxI/foo8l7mTDqq1C5l9M3ORDwejQivqUg2WOqvxSJ5pRbeD0K8cKE6kqWrPqSxiatOdUhxSVw5BWw9l14/2mvRyPiWwqSPaYleEWiOW8c9UNRiGqqKtjVEuDd1V96PRT/G3kG9Ns3mE3Wmw6RdChI9pjTJ1kBgUiItXrjWKhqgpP3VJecAUXFTja5fhks/bvXoxHxJQXJHlMHOJFo+nkoXP16dmXPfj2YX6cgOSNGnAb994dXblA2WSQNCpI9pqyZSAytQlnQaqoqWLBioz5hy4SiIjjqStiwHN55wuvRiPiOgmSPOZlk/TEQEQFn8t6m7U18XL/N66Hkh/1OhgGj4NUboKXZ69G4Z606dIhnFCTnACVMRFppMZHCVlMdrEtWK7jMKCqCo34KGz+BJY96PRp3Vs6DPx0E950Au/SmSbJPQbLX9NGySBQtsFPY9uzXg8oeXTR5L5OGHw+7j4VXfw0tTV6PJrmmRnjpKrjnWNi5BVa+BY+d54+xS15RkOwxLUstEk0/DYXNGENNVYUyyZlkDBz1M9i8AhY95PVo2rdqAfz5cHjj9zDuG3DJXDjpd/DRS/D0pcoqSVYpSPaYPlYWiWb16UrBq62qoG7Dduq37vR6KPljn6/AHrUw+yZo3uX1aNpq3gn/vgb+coxTWnHuUzD1D1DWG2qmw5E/hcUPOduIZImCZI9pdTGRaDb4nxSu2upKABao5CJzjHFqk79cCW//1evRRFsdzB6//jsYew5c/CbsPTl6myN+ArXfhjdugf/e7s04peAoSPaYVhcTaUtvHAvbyD1606WkSCUXmbbX0TDkYJj9G6fu12vNO+E/18LdX4HGLXDOE3DKrVDWp+22xsAJN8P+J8OLV6ilnWSFgmSPaVlqyWefbdjuuqTIeeMohaxrSTFjB5dr8l6mhbLJW9fAwvu9Hcuat+HOI+G138CYs+HiOU5JSHuKiuH0u6FqEvz9Qvj45awMVQqXgmSPWauJe5KfPqlv4PCbZrHws82u9tNPg4DTCu7d1V+yY5dWisuoYYc7QeZrv4GmHdk/f/MuePk6uGsy7NgEX38cTv0TdCtPbf/SMpj2IPTbFx79hhNsi3QSBckeU9ZM8tXmHU67pi93uJwkpIl7gjN5rzlgWbzK3ZssScIYZxW+hrUw/57snnvNIid7PPsmGP1VJ3u877Huj9OtHM59ErpVwoNnOT2gRTqBguRcoIBA8lC6ga4NrkMpha2myllURJP3OkH1JBh2hDNRLhuLdDTvgln/D+6eDNs3wNmPwGl3QLeK9I/Ze3f4xlMQaIG/nQYN6zI3XtFd3JIAACAASURBVJEgBcke08IJkr+cV7bbYFkt4ASgvHsX9tmtJ/M0ea9zHPVT2FYP8+7u3PN88Q7cdbSzkMnIM5zs8fDjM3PsfvvAOY87AfIDZziT/0QySEGyx7QEr+S7dF7e+okQcFrBLVixiUBAr4iMG3ow7DUZXr8Fdm7N/PFbmuCVG5zyioa1MO0hOP1O6F6Z2fMMroWv/hXWvQePnut0zBDJEAXJOUC//yUfhYJjty9vvXGUkNqqCrY2NvPhuk4I4sRZhW/HRph7Z2aP+8W7Tvb4lV/BAafBJW/Bfidm9hyR9vkKTL0VPn3V6XoRCHTeuaSgKEj2mPokS74Kvardt4DTz4M4JgQXFZlfp7rkTjG4BvadAm/8ITOlCi1N8OpNTvZ46+fwtQfgjLsznz2OZ+zZcMwvYOlT8K+fqmZLMkJBssfUJ1nyVfoT91RuIY4hld3o36urFhXpTEdeCY2b4a07Onacte85E/NmXQcjpsLFbzkLf2TTxB/AwZfAW7c7K/OJdFCJ1wModJqkJPkqlBF2XW6hnwkJMsZQW1WhRUU606CxsN9J8OatcOCM1PsVh7Q0OwHpKzc4K+V99a8w4pTOGWsyxsCx18G2dfDva6DHbjDuHG/GInlBmWSPKRaQfNVabpHGvoqSJai2upJVm3bwxZc5sIxyvjryCtj5Jcz5k7v91i2DvxwDL//SqTm+5C3vAuSQoiI45TbY8yh4+vvw4b+8HY/4moJkjzlZMwUEhWLjtl1c+dQ7NDbl/ypirS9r969v/URISG2wX/L8FSq56DQDRznB7X9vh+0pPM4tzU6P5T8fBps/gzPvha/eDz36df5YU1HSBb72N+d+PXYerJzn9YjEpxQke86qu0UB+fULy3h47mc8vXiN10PJSXrD2PmMMXXGmHeMMYuMMfPj3G6MMX8wxnxkjFlijBnvxThDRgzqTbfSYk3e62xHXAG7GuDNP7a/Xf0HcM+xTjnDvlOc2uORp2dliK507QXnPAG9BsJDZ0H9h16PSHxIQbLH1N2isARCdboFEAzaNBYTCW+b/w+P146y1o611tbGue14YJ/gvxnA7VkdWYzS4iLGDilXJrmzDRjhBLtv/Rm2rW97e6AF3vg93HGYswz0GX9x6o979s/+WFPVs7+zKl9RKTxwOmxRckLcUZDsMacnrNejkGwxxvlaEM95Gn2SFSPnhFOAv1rHf4FyY8zuXg5oQnUF763ZQsPOZi+Hkf+OuAKadzjBcKT1y+Ge4+Clq5yexJfMhVFntv5Cy2WVe8K5T8COzc6qfDv0iYSkTkGyx7QsdWExOH9UCuE5T2fini2gTLuHLDDTGLPAGDMjzu17ACsjvl8VvM4zNdWVBCws+myzl8PIf/33hVFnwdy7nKWeAy1O+cUdk5xA+fS7nd7HPXfzeqTu7D4Gpj3o3IeHvw5NO7wekfiEgmSPKZNcWAopk9yR5agL4OHx0iRr7XicsopLjDGHp3MQY8wMY8x8Y8z8+vr6zI4wxrih5RijyXtZccTl0LILXrwS7j0eZv4c9jra6Vwx+ix/ZI/j2fMIOP3P8NkcePI7zhsAkSQUJHtM3S0KSzhILoAwMFyTXAD31U+stauDX9cBfwcOjNlkNTAk4vvBwetij3OntbbWWlvbv3/n1qX2Litlv4G9WaB+yZ2v714wZhq8+wTUL4PT7oRpDzkT4Pxu5Bkw5QZY9iw89+PCyFZIh2gxkRygH9NCEiy3KIAnPXQf05m4VwiPjxeMMT2AImvt1uDlY4FrYzZ7GvieMeYR4CDgS2vt51keahu1VRU8tXAVzS0BSoqV3+lUx1wDvfeA2m9Db0/L0TPv4AuhYS28/lsn8D/yCq9HJDlMv2k8Zq1VJrmAtGaSC4e7iXvKPneyAcDrxpjFwFzgOWvti8aYC40xFwa3eR74BPgIuAu42JuhRqutrmDbrhaWfbHV66Hkv567wdE/y78AOWTyVTD2XHjlVzDvL16PRnKYMskesxRWwFTowtV8BfDGqHXiXur3VZnkzmWt/QQYE+f6OyIuW+CSbI4rFbXVlQDMr9vIyD36eDwa8TVj4OTfw7Z6eP5/nTcF+5/s9agkBymT7DWrgKCQFFImuSOfkBTC4yPu7FHejd37lDFfdcmSCcUlcNZ9sEcNPHE+1L3h9YgkBylI9pjT3UIhQaEwhVSTnM4+BfC4SPpqqyuZX7dJvzMlM7p0h68/BhVV8PDZsHap1yOSHKMg2WNOTbLXo5BsaW0BVwBPekdKJwrg4RH3aqsq+GJLI6s3q8+tZEj3Sjj3KejSw1lsZPNnXo9IcoiCZI+pJrmwhGqSC+E5T2cSnibuSXtqqysA1ApOMqt8CJz7JDRth7+dDts2eD0iyREKkj2mPsmFxZgCKrdIZzERTdyTduw3sDc9u5Ywr06LikiGDRgBZz/iZJIf+irs2ub1iCQHKEj2mFXOrCAV0nPuqk9yzFeRSMVFhnFDy5lfp0yydIKqQ+HMe2DNQnh8OrQ0eT0i8ZiCZI9ZdbcoKIVUk5zeYiL5/7hIx9RWVfLB2q18uUMBjHSC/U+CE38Ly2fC05fqD3SBU5DsMQsE9ENYMExrp+S8l05WOJ3eylJYaqsrsBbe/kzZZOkktd+CI38Kix+Cf1/j9WjEQwqSvWb10XIhac0kezuObFCfZOkMY4eUU1xkNHlPOtcRP3GW5X7jFvjv7V6PRjyiFfdygSKCgtHa3SL/n3StuCedoUfXEkbs3luT96RzGQMn3OysyvfiFdCjP4w60+tRSZYpk+wxq6l7BaWwMsnBr6526oyRSL6pra5g0crNNLUEvB6K5LOiYjj9bqiaCH+/ED6e5fWIJMuSBsnGmDJjzFxjzGJjzFJjzC/ibPM7Y8yi4L8PjTGbI25ribjt6UzfAb/TxL3CEm4B5/E4ssN9lKw3jJKK2qpKGpsCLF2zxeuhSL4rLYNpD0G/feHRc2HJYxDQm7NCkUomeSdwtLV2DDAWmGKMOThyA2vt/1hrx1prxwJ/BJ6KuHlH6DZr7dSMjTxPaOJeYQmXWxTQU+5qMREbebmAHiRxJbSoyHyVXEg2dCt3Fhvpuzc89V246yj4dLbXo5IsSBokW0dD8NvS4L/2/nqdDTycgbEVBGuVOysooXKLAnjW01pMpIP7S2EY0LuMIZXd1C9Zsqf37vDdWXDanbBtPdx/Mjz0Naj/wOuRSSdKqSbZGFNsjFkErANesta+lWC7KmAY8HLE1WXGmPnGmP8aY07t8IjzjEXBQCEJtYArhOe8deJex/YXiae2qpL5KzbpEwfJnqIiGPM1+P58OOYaWPEm3HYIPPND2LrW69FJJ0gpSLbWtgRLKQYDBxpjRibYdBrwhLW2JeK6KmttLfB14BZjzF7xdjTGzAgG0/Pr6+td3AV/0+/3wmIKp01yWhP3IgMeBT/SntrqCtY37OSzjdu9HooUmtJuMOl/4NJFMOE78Pbf4A/j4NUbtZx1nnHV3cJauxmYBUxJsMk0YkotrLWrg18/AV4BxiU49p3W2lprbW3//v3dDCsvKCDwr+ornuOap5emtG1rTXL+P9+hkpJ0lqUWSaa2qhKAeSq5EK/06Asn3AiXzIW9j4ZZ18Mfa2DhXyHQknx/yXmpdLfob4wpD17uBnwFWBZnu/2ACmBOxHUVxpiuwcv9gInAe5kZuv9FZ808HIh02H1v1qW0XSG2gEt3nwJ4iKQD9tmtJ73LSliwQpP3xGN994KvPQDf/hf0GQxPfx/uOAw++rfXI5MOSiWTvDswyxizBJiHU5P8rDHmWmNMZLeKacAjNjpFtj8w3xizGCcDfYO1VkFyUOQjpQ4XhSFck+zxOLKhdVlqF90t0BtHSU1RkaGmqkKZZMkdQw+G81+Cs+6Dpm3wwBnw11Phi3e8HpmkKemKe9baJcQpkbDWXhXz/TVxtnkTGNWB8eU1m+Cy5K9CyiSHuLqvUZnkAnqQJC211ZXM+uADNm3bRUWPLl4PR8T5JX/AaTD8RJj/F3j1105WeezX4aifQZ89vB6huKAV9zykcovCU1DLUgdf1OneU/1MSDK1VU6/5AUrlE2WHFPSBQ6+CC59Gw79PrzzuFOv/J9fQqMWwfELBck5ohCCJiGcSlYAGJ8eFnFjzJBySosN8xUkS67qVgHH/hK+Nx/2Pwleu9nphDH3Lmhp8np0koSCZA9p4YTC05pJzn/h17SLF7d+DsSNstJiRu7RR5P3JPdVVMEZdzsLkvTfD57/X6fH8rLn9IsvhylI9lD0ErzejUM8UABPeLgFXBr7QEE8RJIBtVUVLF71JTub1XJLfGCP8TD9WTj7EeeTxUe+DvedCKsXeD0yiUNBsoeiAoKCyC3mH7f9jkPbBwrg6Q4vJuKmT7Im7olLtdWV7GoO8O7qL70eikhqjIHhx8NFc+DE38L6D+Guo+GJb8OmOq9HJxEUJHtImWT/cxvshjZvKYAnvDVIdtMCru3+Iu2pCU7eUys48Z3iEphwvjO57/DLYNnzcOsE+NfPYIdez7lAQXKOUDzgT277WwfCmWQ948noEZJU9OvZlWH9ejBfQbL4VddecPTP4dKFMOqrMOdP8PuxztfmnV6PrqApSPZQdCZZIYEfuQ12Q5sHCqDewsZ8TWkf/RxIGmqrKliwYqNeP+JvvQfBqX+CC1+HPWrgXz91MsvvPqWP1jyiINlD0TXJ4kduf2+FYuMCiJFb+ySnW5OsPwqSotrqCjZtb+Lj+m1eD0Wk4waOhG88Bec+5WSZn/gW3H0MrJjj9cgKjoJkD0UFBAHvxiHpcxvHhd4YtRRAlJxOJjne/iLJ1FZXAqgVnOSXvSfDBbPhlNtgy2q4dwo8cg6s/8jrkRUMBckeil6WWiGBH6VbblEQWdI07qIms0o69uzXg8oeXTR5T/JPUTGMOwe+v9CpW/7kFbjtIPj7RbDkcdiyxusR5rUSrwdQyLQstf+5D5KDmeQCeMLDfZJddbeI6gEnkhJjDDVVFVqeWvJXl+5OB4zx58ErNzjLXC9+yLmtYhhUTYTqiVB1KJRXhVd3lY5RkOwhm+Cy+Ifb562wapK9HoEUktqqCl56by31W3fSv1dXr4cj0jl67gYn/RZOuAm+eAdWvAEr3oQPnoNFDzjb9B7sBMtVh0L1JOi7t4LmNClIzhEF8fF7HnJbS15I3S1CtJiIZENttdMvecGKTUwZOdDj0Yh0sqJiGDTW+XfIJRAIQP2yYND8hlOW8c5jzrY9dgsGzcFsc//9oUjVtqlQkOyhyICggGKmvKI+yYmlcw+1mIika+QefehSUsSCFRsVJEvhKSqCASOcfwd+1/kFuuFjWPG6k2muewPe+4ezbbcKGBrKNE+EAaOchU2kDT0qXlLWzPfSDXZbCqCbSXiSoovXdlSdfqYHJHmta0kxYwb30eQ9EXDKK/rt7fyrme5ct2mFEzCHAucPnnOu79ILhh7cWp6x+1go6eLZ0HOJgmQPaZKS/7mvSXY/mc2vWifuudkn4nIBPEaSWbXVldz92ifs2NVCty7FXg9HJLdUVDn/xp7tfL9lTTBoDtY1/+cXzvWl3WHwhNbyjD1qobTMu3F7SEGyh6xiZN9Le8W9AggAWzPJ7vcRSUdtVQW3v2JZvGozB+/Z1+vhiOS23oNg1JnOP4Bt6yOC5jfglV8BFoq7OIFyqDxj8IHQtaenQ8+WvAmSv9zRxOKVmzl83/5eDyVlqr/0P/cr7oVawHXCYHJMeDGRNO9rATxEkmE1Va2T9xQki7jUox+MmOr8A9ixCT57qzVofv138NrNUFTilGSEyjMGjYee/om93MibIPn7D7/N7A/rmf/zY+jX0x/tfyI/Ti6EzGI+cp1JTnM/X0rrPqp3uKSvvHsX9tmtJ/PqtPKeSId1q4DhU5x/ADsbYOVbrdnmt+6AN//g3NZzAAwcBQNGtn7tu7fvJwT6e/QRPl3fAMD2nS3gk08B1CfZ/1wvSx3qblFA7UzcTdxLbz+RkNrqCp5b8jmBgKWoSL1hRTKma09nqey9JzvfN+2A1Qvg88Xwxbuw9h345FUINDm3l5TBbvtHB84DR0JZH+/ug0t5EySXBnv+7fJR24DoJXgVEPiRapITS1ZuYa3l8fmrOGnM7nTvUhK1T9tvRFJTW1XJw3NXsnxdA8MH9vJ6OCL5q7SbU25RPan1uuZdsP5DWPuus9jJ2nfhg+fh7b+1blM+1Gk7N3BkawBdXpWTvZvzJ0gudh7c5oCPgmR9tOx7adck++dlmrZkj03dhu385MkldO9azEmjB6W0j0gyoUVF5tVtVJAskm0lXZzgd+BIGDPNuc5a2PpFa+AcCp4/fKF1Ra4uvWDAARGB82gnC92lu3f3hXwKkkucj9Wamn30V9ZHQ5X40s0kF8InB8nuY3PwnUJzxCzGqDeOnTMsyXNDK7vTv1dXFqzYxLkHV3k9HBExBnrv7vzb5yut1+/aDvXvBwPnd53AefGjsOvu4H5FULlXdOA8cCT02j1ry2znT5Bc7MNyi8jLigh8yX0m2fnaUgBPeGu5Rfz7GkhSelIAD5F0AmMMtVUVmrwnkuu6dIc9apx/IYEAbF4RzDoHA+fVC2Dp31u36VYZzFaPbq1z7je8UxZAybsguclHQXKkQqhRzUfuu1uElqXujNF0nuaWAC3W0rUk9QUaWrPm8W9vLT2JX3akiXuSrpqqCl549wu++LKRgX0KcxEEEV8qKoLKYc6//U9uvb7xS1i71Amcv1jiBM/z7obmxuB+pdB/OHz9MeizR8aGk0dBcrDcwkdBci4tJrJy43YAhlR6W//jN26D3fDEPZ9FyTf+6wMWfbaZxy48JOV9bMzXWK2rD0bsk+CyiBsTqisBmL9iY7jeXUR8rKyP05e56tDW61qaYePHrXXO696DHpnt15xHQXJw4p6PVmmInrjn7bgPu3EWAHU3nOjpOPzG7fMWbgHnswjwiy8b+WJLo6t9kj028Tp9KHssmTBiUG+6lRYzv26TgmSRfFVc4mSP+w9vXTUww3Kv30aafFmTnEOZZEmP2+ctXJPss0yyJf0ANlm5RSBRJjmts4k4fw/GDilnwYpNXg9FRHwsb4LkLj6sSdbEPf9Ld8U9vz3fAWs7sLx0uhP3fPYgSU6pra7gvc+3sG1ns9dDERGfypsg2Z81yblTbiHpCbXlTrUbTXiymt+eb5vO6oLRX2O11iSru4VkXm11JS0By6KVm70eioj4VB4FycFyi2Y/BckRl70bhnRAKNBLuWOjT1fcczLJ6XXySLSXTVJuIdIR44aWYwxqBSciacubILnEh0FyJAUHhSFch+u3mmTr/o1cstd0vHILTdyTTOldVsrwAb1UlywiacubILlLsNxip4+CZPWE9b9wJjnFeovWjg6dNaLOYXFfkxzePNFiIoEkE/d89hhJ7plQXcnCFZvCqzuKiLiRN0FyiR+7W0S1gPNwIJK2UICXarlFvAU0/CBg3b+RC9ckt3NMiM6qR01m1RtH6aDa6gq27Wph2RdbvR6KiPhQHgXJTpjip3ILt1mzloDlhheWsW6ru3610nlaM8mpbR96mv1Wk2zTmLgXuW/869v2jI6ezJre+URCaoOLiqjkQkTSkTdBcpHxYZAccTmVoGnlxu3c8erHvPbh+s4blLjivuODPxcTAet+dcEkmeDWmuRE+4t0zB7l3di9T5km74lIWvImSA7FHH4Kkt0Kt8zyeBzSKhT0mhQLLvxak+yMN91yi0R9kuNkkqP299mDJDmptrqS+XWb9HoSEdfyJ0gO/nn1VU2yy4+Wky2+INkXDnZd9kn2X3eLDiwm4qJPsl7akmm1VRV8saWR1Zt3eD0UEfGZ/AmSfZhJdjtJycYJKsRbbvsk+7YmmXRawCXrk+x8jX6/ED+rLJKumqoKQHXJIuJeHgXJfswkx7+cSLIaTsk+t7Fu6Lnz24p7Aev+zVnyPsnxJu6lvr9IKvYb2IueXUuYX6cgWUTcyaMg2fnqp0xyZK4slcxieAUzBQ85I1yTnGp3i3C5RWeNqHNY674hW2j7xOUW0V8j94n3nUg6SoqLGDe0XJP3RMS1/AmSg1/9FCS7XZY6FFj57aP6fNbaJ9ntxD1/PYfWpl9HnXTiXoLj+uwhkhxWW1XJB2u3sqWxyeuhiIiP5E+QHMok+6ncIvJySuUWqknONe77JPuzBZxNY2mP8F101SfZ/dhEkqmtrsBaWKi6ZBFxIX+C5FB3C59mklPJJfu1fVg+c/tUhD4N8NF7OSD42kuzT3Ki3eKWWyRoByfSEWOHlFNcZDR5T0RcyZ8g2Yc1yW6Xpc50FnLHrhYWrdyckWMVKvfdLfz5aUAgnZrkFCfuJQqMffYQSQ7r0bWEEbv3Vl2yiLiSR0Gyv7tbpJIdznR3ix89tohT//QGG7ftyswBC1DrxL3UwmS/drew6XS3CO+bqCY59DVBdwvlkiWDaqoqWLRyM00++hshIt7KnyA5+NVXmeQEHzMnkuma5Lc/c7LIO5tbMnK8QhQqn0g1kxx6ofpuMRHSeHMWfr0mujn0yUjkeVSfLJ1jQnUljU0B3luzxeuhiIhP5E+Q7MNyi0ipxAM2SdDhViibWZTqrDNpI/xUuF1xz2cBoNMCLs1McsJjOl/9NolR/Km22llURCUXIpKq/AmSg3+Kd/ooSHabNcv0stShbKaC5PQVzIp7NvOZ3UC8N31Rn65k9nxS2Ab0LmNIZTdN3hORlOVPkOzHFnBJ6i/nfLyBv82pa7N9prKQbtuXSVvua5Kd7Vt8lkpOo7lF+PWabDGRyMciOl7212Mkua+2qpJ5dZt8N3FWRLyRNEg2xpQZY+YaYxYbY5YaY34RZ5vpxph6Y8yi4L/vRNx2njFmefDfeZm+AyGhv7N+LbeIFw+cfdd/+b9/Lg1/H28Z344IJAliJLnwYiIpr7gX/dUvAtZ2oAVcool77fdJ9ttjJLmvpqqC9Q07+Wzjdq+HIiI+UJLCNjuBo621DcaYUuB1Y8wL1tr/xmz3qLX2e5FXGGMqgauBWpw/sQuMMU9bazvh8y7nL6qfZi67726R2Yl7oXILZezS5/YNS+i582V3C7c1yUneECSbuCeSaROqKwGYV7eJqr49PB6NiOS6pJlk62gIflsa/JfqX7LjgJestRuDgfFLwJS0RppE6A9xs48+xo6qSXaxmEim4qtwgOefhyznhB7C/K9Jtq7LfJJt3vpJhjpaSHbss1tPepeVsGCFJu+JSHIp1SQbY4qNMYuAdThB71txNjvDGLPEGPOEMWZI8Lo9gJUR26wKXhfvHDOMMfONMfPr6+td3AVH6I+rn2o93X60nOnOCC0ZPl4haq3rdleTnE7fYS9Z0uiTnGTzZOVDoat3Nrfw8NzPfPV4SW4qKjLUVFUwv06T90QkuZSCZGtti7V2LDAYONAYMzJmk2eAamvtaJxs8f1uB2KtvdNaW2utre3fv7/b3cOZ2JaA9c0f0wST+hNvn/HuFqFz++PxykWuM8kuS2xyRRolyUlXF4y7LHWc/d/4aD1XPvUO732u/raZEEx6vG2MeTbObQnnl+SL2upKlq9rYPN2LaIkIu1z1d3CWrsZmEVMyYS1doO1dmfw27uBmuDl1cCQiE0HB6/LuMg/tH4puYj+mDn5mDNek6xMcoe57RAS+Vj76VOPgLVpl0Ik2s3GySTHK71oavFnR5Ac9gPg/XZuf9RaOzb47+5sDSpbaqucfslqBSciyaTS3aK/MaY8eLkb8BVgWcw2u0d8O5XWX8D/Ao41xlQYYyqAY4PXZZz1YfARlTVLYciZbgEXLrfwyePVWV75YB1bGpvS2tdt4BgZBPqpLrm1Ht7FmJPU0IcnjibMJEfvX+Av04wwxgwGTsRJZhSkMUPKKS02zFPJhYgkkUomeXdgljFmCTAPpyb5WWPMtcaYqcFtLg22h1sMXApMB7DWbgR+GdxvHnBt8LqMiywZ8E8mOeJyCh9mhzPJGSqP8GMdd6at29LI9Hvn8f2H3k5r/9ZAN3Eq+f3Pt7BuSyMQW27hn8c9NmB1s08icRfHibOTzfAnKAXuFuAnQHttgOLNL2mjo/NIvFJWWswBg/po8p6IJJW0BZy1dgkwLs71V0VcvhK4MsH+9wD3dGCMqYnMJLf45Y+pu1n9nZVR88ubis7Q2OTECp+sb0iyZXzhmuR2yi2O//1rdC0p4oPrjo96g+Onhz0cqKa1T6Ka5LZlFNGrUEaXA/no4cpJxpiTgHXW2gXGmCMTbPYM8LC1dqcx5gKc+SVHx9vQWnsncCdAbW2tr56eCdUV3D9nBTubW+haUuz1cEQkR+XPinsRl5sD/umVHOKuu0Vm/x4VciY5FNym+5Cmuix1aLl0v9Ykp1NukbxPcpLbw1/blmVIWiYCU40xdcAjwNHGmAciN2hnfkleqamqZFdzgHdXf+n1UEQkh+VPkBzxF9QvmdEknzK30Vkr5PkpWOssmZ6Ulvg87iZr5opAOpnkmK+Jjplsxb3Wl6d/Hq9cZK290lo72FpbDUwDXrbWnhu5TTvzS/JKbbUzeU+t4ESkPXkTJPuyu0Xk5RQCpvDHzxm+f36qjc20VLtSJOK2u4UfJ5hCmjXJSbaNV5McvU90LXIBv0w7VSrzS/JNv55dGdavhybviUi7UlmW2hci/376pSbZ/bLUqW/rhl/eVHSmdLO6oYfOpNgpOfIsTT55nULy+uL2941/fbwWhPE21cKQmWetfQV4JXg5pfkl+aa2qoJ/v78Wa23KiwGJSGHJm0xydLmFP2qSowMz285toS0y290ixE8ZzUwL/XFM9xGwLjPJkVnTxqaWNM+afeksiW6TFFzE61oRr09y6DiF3qpQMqu2uoJN25v4uH6b10MRkRyVP0FyxGW/BH3R5RbRt+1qaRvou6lJ3rGrhVc/TK0tk18er84Qim1TDf5e/bCew2+cxc7mFlf7WprMDwAAIABJREFUhVgLXUucH7vGZh8FyaGvaZRbJM4kR3+NPE/k5daVIUUyp6aqEkCt4EQkobwJkiP/gvrlY+z2Ju7tam4bJMdboSyR59/5nPPumUv91p1Jt/VL5r0zpZqdv/aZpXy2cTsrN24HUu9uEXmebl2cllOh9nN+0JFyi0SSdWuJXYZdNcmSSXv170FF91JN3hORhPImSI784+2XzGh0T9jo2+IFyW5awIWylPEy0m2O659YLePcvlJKi50fmV3N0fW0qdY0BgLQvdQJknfs8k8mOZ3OKskm3IWOGdUnOaq7RXRHjUyXGUlhM8ZQU1XJfC1PLSIJ5E2QHBno+SYzGjVxLzoAiFtuEQhtm/zQrUFN8o1bPE7RvbPqS6bfO5emFAL6TIu3NHJ7ugRLJUJjTfb4xrs9nEn2VblFdMCa2j7R+7a5PW4QHacWXzP3pJNMqK7g0/XbWN+Q/BM3ESk8eRMk+zOTHP8yJCi3CH1N4e65aZvV4vGbiv99fDGvfFDPx/XprXqXCam+ZEKZ5NYguf3tY1+LAWvp3sVpKrPThxP33LQLTF6T3PaTkXglSIqRpbOoX7KItCd/guSIv6B+aWkW76PlkJ0RQXJsLXJqPZWJ2qc9HiRwo3j5MbrbJHppsVNWsSu8gp5zgKIEP0mxWXproVupH2uSo7+62jfB9XH7JMc5Z7ztRDJh5B596FJSpMl7IhJX/gTJEZebszxx78O1W9m2s9n1fu0Fh7uiguTQ19Rrkt1MdorMJHu5ClyqvYYzqfWxTO1+h2uSW6KXmU409tgkfcC2Ttzb4atMsvt0brI3P3H7JEdlkmPfHKZ+7vbMq9vIA/9dkZmDia91LSlmzOA+WlREROLKnyA5KpOcvQxdS8By7O9mc/GDCzt0nNgAICqTHPzqZjERN9m35gQTp7LFywShmxIWgC7BIDmUBU624l6bTDLQPdzdwj9BcrjG3UWUnCz7HPvmr+0GUV8y9nnDkwtW8fv/LM/Q0cTvDhxWyburv2TdlkavhyIiOSZvgmQ8qknesqMJgIWfuc9ERK+4FzNxLyJIjq3dTCXwje0M0J6o7gIpbJ9poXN6seiVm8cJWifu7Uxx0l3s6o/W+rQFXHiJaDf7RO8bKzRpMrpPctvXYrxFRzoiYK3ayUnYWTVDaLGWv+nTBRGJkTdBcsBCUTDIymZN8pfBILl3WanrfePVX4ZEBmGxGblU/sC7qV+ODJK9rPsMxciPzP2Ml5etzco53XQBgdZyi1D7tlCglyi+j1eTXFbqx3KL4Nc09km0U9ya5DjbZnrinnNeRcniqO7Xg2P2H8AD/13hq093RKTz5U2QbK2lJBjAZDOTHAqSe5WVuN43agnemNviZZLdZNTclGYk6lObLbH35y+vf8pj81Zl6+wR/0+uNFxuEQySQzXJCdLg8bpbFBtD15IiX3W3SG/yXIo1yQk+yWgz+TRDr82AtSl3M5HCcP6kYWza3sRTC1d7PRQRySH5EyQDpcFUcjb77XYoSI68nEqfZFc1yW4m7uVIJtm0jiFbHS/cdm3oUuIMckdsTXKC7WMfT4vziUdZabHPslZplFskiW1bJ6K2vc7ZL/r2TL0mrPV2gqrknoOGVXLAoN785fVPot60iUhhy58g2UJpiZeZZPflFpF/81PJJLurSY7etz1et8yLPbu1qfct7ii35RYlRdGZ5GSTytpkkgMWYwxlpUUdrkm21mbtD7qNCVS372pm+r1z+WzD9hT2TVCTnOQ12qarS4be+yqTLLGMMXznsGF8XL+NV5fXez0cEckR+RMk0xrAeFGTnF4mOXGUvDNuC7jor+0eO412cakeO+PC53Tysc7EqiwFfy7LLUKPVThITlICExskW5yMeVlpcYdrkh+dt5LDbpzVoWOkKrbkYeXGHbzyQT2LV21OuE+yTHK8FnBR+6d4HLcCyiRLHCeOGsRuvbpyz+ufej0UEckR+RMkW0uX4EIP/qlJbr2c6e4WrRnS5OPIvXKL7AXrbs8TGySHy1qSbB95PoOhWwbKLVZv3sHqzTuyEvDFZsxTeXORvE9ycLtkE/fCpR6ZuZ/qbiHxdCkp4rxDq3lt+XqWfbHF6+GISA7IoyCZ8MS9bGaStzQ6QXKX4mLX+0YvnBBtV5w+yel1t0i+ba60gAtxPg7PzkjcTgoLLVSzI2biXqLhtskkW0uRga6lxTTGWXrcjUwvstGe2PKdUOlDe4FrstdrvE87olrAxWSaM3U3rfVyjUfJZeccNJSy0iJlk0UEyKcgGUtJMJPcnMWJe6E+yekEdZF7xO4eOXHPBi+mlUlOIRzImUxy8Gs2a5LdfpQfaukWqieOrdWNFft4Bmyw3KKkqMOZ5FSCx4adzSnVDSc/V3RAnkrf5Njsc6JjJl5xL/65OyoQ0BLXEl959y6cWTOYf7y9hvqtO70ejoh4LH+CZNu6GpoX5RbpiDeTPyR6xb3oYCKVuxevc0Ai3q+4FxtIZi+TnGgMiYQmyu1oU5Mcf/vY92sWS5ExdOvS8XKLVCZnfu3Pczj8plkdOo9zsqgvKZ07duJdrKR9kttkolVuIZ3vWxOHsasloKXLRSS/guRwJtmDIDnTmeRAVHY3tH3qtZnJss6Rx4/uk5z96CFeuUW2JKspjhV6bbWpSU6x3CJgAQNlJZkIkpNnWJeuyUxtZdte3UR9jTu+JHXL8R676Bg5+pyZ+rEOWGWSJbG9+vdk8n67aXEREcmjIBkbXujBi0xyR//oxsumxt6WSvaudf/QvvFvj1wJzuvFRMLnDn7NZhDjps4b4k3cS2371hNCUYZawLkpv+moRBnkjpw7tGuyn1e3z1Hy86omWdp3/qRhbNi2i38u0uIiIoUsb4LkgIXSUAu4Fi+CZPf7tjdxL/L78MfSgdQzask+6k5Uh+xFhi02CHL6/2bp3OGvqd3vlnAmOaYmOcHjFvupRsBaDJlZTCT2dbBk1WbWbW3s0DETiRPrR32Nv1P7G8ULtON1usj0YiIBa72ZoSq+cchefdl/99785fVP1S5QpIDlTZCMhaIiZ1JUc7YiLCKDpXR+kSbO4Mab8e9mln+yj6gjg7eomuQUjt15Wsec7e4WqZ4uVGO8I8UWcPH6JDuZ5I73SY4NMqfe+gbH/m52h46Z6rliyy/iSfYGJN6nHVHlFjHBccYm7qncQpIwxnD+pGF8uLaB15av93o4IuKRvAmSLRaDoaTIZLUmOZzdTSMujwoOEqXqoM1H3G5qkhNmkiOy7YEEWeVsiQ2CAln8ONx1d4vgE912MZH427ftbmHDi4ns7PCKe6Fjtl63eXv6E0nbPVfMOVOqSY55bN74aH3UZKi4JRtxLma6BVw2X1/iXyeP2Z1+PbvyF7WDEylY+RMkWyeLXFJUlNWa5JZ4f+hTZBNcjj1ebCDnpiY5cSY5EHG5nbqPLArfv0D2Vtxze4dD7y1CnyAkKwVo2yfZyVKVFJmouvB0JFvtL6NiXoOpTKaLvemcu9/i5/94t/X2eOUWcfokZ/p+WmWSJQVdS4o575AqXv2wnuVrt3o9HBHxQP4EyYSCZJPVmuRQEJRWsUWcbHG872OzwqlkrVvLAJIHb9H1ycmPnWnxspPZGkf4PCmeLxCuSQ5mkpOUAkRm6UPPn8F5rXY0UHPTErCj2nS3CF7fXk42WbY5EO7/3XafeMfJFLWAk1Sdc3AVXUuKuOcNZZNFClH+BMnW6T9bXGzCH4ln57zO1/QyyYnLHOIld910FEgWoDQnCJKTfRBdt34bm7fvSnp+N2JrT7PZJ9nt8xd6rHY2x6y4l2j7OJ8IFBmDMabDgZrb1QI7Iva9RCqTSJMF0slKgmJf94/MXcnhN85KccSJuSlbksJW2aMLp48fzJMLV7OhQYuLiBSavAmSQ3+sS4oMTdkstwhlktM4Zbv1nJEBdExAksq5kgXUUZnkOIFcItPvncsf/vNR8gGkIbLGNlvxi9tAKfS4NbU4JSHJFxNp+0bIGCgy6Z0/UmwQ2ZnarrgXGkTqb9jaXB8+dtvr4p3rs43b+WxjJlYPbH9cIpHOn1TNruYAD771mddDEZEsy5sg2Sm3MBQXmahJaZ0tYzXJ7ZRbxF6XyrSjZF0bojLJLW0DuUQadjazpbGTJodFZHWzleVzOykstr90sg8touttHUXGySZHnj8dsaUPqWyb/rmijxNvSWm35wzdnrhPd+y5MpQB7sCnP1J49t6tF0cO789f56wIf4IkIoUhb4JkrNN/tqSoyJvuFmllkhOXOUTe1qYmOYVzJSsjiCxJcZNJDlhojl1rOUMi63uz9RS6WcUQogO65oCNeN4SPc6tl1szyQYTc106Wmt608/mpsLGCfRbl4xOfOD2H5nImur4PwexJUNu2/Ul4naVRZHzJw1jfcNOnl60xuuhiEgW5U2Q7GSSnaWps1mTnKlVz2J3jzeZyc25kq64F4i87CZItp1WzhJ5P7OW5XOZSY4cV1NLIIXHue1jawwUBestOnI3I4PGQJLnpCOPpo37Woz+2t5JE07cixP0/n/23jzejqO6Fl7Vfc69V7ItyYM84HnAgDFgg/GADWFyAiExCYRA+GwMmCFfSML3SMiD5D14EJKQkADJBwmTAQdihzkhgAFjTIzxIAvbeMajLHmQLVuzdIdzuuv90b2rdlXt6j7n3itd6aiWf/6doburqvv0Va9atfbe8goKKc5Os7PGzqxUmDAaOOu4A/CUg/ZJxUUSEvYwjA5J1lXWgHwn5knWWjOSNAu7BTvEH7JUBW+YbAZ2KV7eOZYCrs3KUZZ63u0sUtntnfUcGtafyjOn9AvdqkpKqfwUFGq3xdyUZHbvtbUzp/LRznvtvTYd16wly0py2K+/31xJSvIkJwwLKi5y59otuPreJxZ6OAkJCTsJo0OSUWW32Jkp4NpSVw2DwG4hvJ9NMZGYqO4ElA2RAk7rHVfRkFcW3GnZLYbUJR0luSwDr64PKXMI9yTPyQbBiGpbzuV5s1t4E7VGu0ULGZXyJDcdP5d0i1K/iSQnDINzTnoSDth7LBUXSUjYgzAyJLksq2XsfCd6kqXMBcNA8l/azzp430Y6ONoC0qIp4AZQJHvzrSSbvnc+gRm2H8eTXLDsFpH9eUYLOlQpzIsneRjlfS415vifU5AvuaHZQaw7fvv8IJ4SUHp9x8U34Fs3Ptg6/rBft/2EhEEw0c1x7ulH4sd3PoZ7Htu60MNJSEjYCRgZklxTEXR3oifZtUQMf/ygBMNXKwfzJDfv6weg2ePa2t2RSrK8BN+GstS4ZpZLoMOSVJ8kty3d06pGppT5/TKlrJI85Hg5OMlsqzI5H4o1b8f3yTcdF51AkMc4Zrfw73sv3eJ3b34E/+Mrv2g/Ab/fpCQnzBLnnn4kxjoZvpCKiyQk7BEYHZKsda0k7zxPsus3nYWS7BBh93iJgA/nSW4eVz+a9q1dSZ5vOwsf62wyD1x73xP4vc9eizvXbh6+7yH3L+r7DKjsFm0WGLJBKLi/2872JM/NbiH1PfjvFLs2kpIsBu6Z/QfobADMZiKWkAAAB+w9jt8+6VB844YHsWHb/BZVSkhI2PUwMiQZQJ0CTrWqavOFuZZz5of4xzsE2lt2HoSQD1PkYjglWc/7JIQrjrPJPLB1ug8A2D4zixyms7BbjHeqP5vKbhFvZs367fjl2i0Aag9yvZOjJM9BlOe/8TD5mofvJ3w/kJLccG349pjdx5Lj6t1ccpK7/Q4/EUtIILz5rKMx1Stx8YpUXCQhYdQxMiRZa1gleWcF7gk5cIeBFBBl2wv7GUYBa1OduWWC21PafaQ7ME+yHs53bcfUPHnYPtPHn3/rFrEIyrC/W1lqjHdyAFUKOGNFEJp5/t9dYYN8lOtPng8lmV+rViV51r14dotgwjb7PqWVGNdu4fYxX351M/adlykyYYTwlIP3wfOffAAuunoVZvrpJkpIGGWMDkk22S2yHeaZ9TFnT7LzXnvbQmICQ1AGH9tAZamHCEDcIYF7pjk9KyWZlO3Ydbno6gdw8XWr8dkr72voe/C+JrqZeV8OmLs3s0LyDvAk6wGyW8zd1lG1U78O0G7bhGfQvx/axrNbzMf5pMC9hNnigrOOxmNbpvGdm1NxkYSEUcbIkORS82IiO/bhd8PqDfjSNau8SnVzY8n+4dIStxToFG26ZanbsViwOUVzMGFlL5jL9b30lkdwxS8fi7Q/uxy2NJ7YuKZ6lQ1DkXzLMLSSrF0leVALjIJylORsXjzJljS2FROZj/LXTp8DqbrNtgY5awbv1+2fe7Dncg8OUlI7IaEJv3L8cjz5wL3xuZ+m4iIJCaOMkSHJWmso1HmSd/DT71X/fDX+93/e5hCT2XFkrha7kIpQDBOV30be+Ni58t6krlFTvTko9Z+68j583sszynXy2SjJRdl8TK+2h3SzkCQP+7NxT7JTca/lOKXs9VNKGcI+H4F7VYXClp3nQpKF9/aebDiOkVrpPhTJd0M7dl+If+Nr1m/H534arhY0jSshYTZQSuHNZx2N2x/ZjGvvW7/Qw0lISNhBGB2SDAC1J7lNZdo63cemydCfOiwK4SE/DJxDvOP5x7DiXntfbYSaj527J5qapjbn4vkuyjL4ffiyvC7d7wYBEabYMbS92wlv92F/tqLUGO/awD2jlra0w1PAKVhP8mx42oZtM3jvN28xCrnWAxQTmQNL1sJKwyCWBb5FsuiINg4ncC8y+YmsZlx66yP40HfvwBbBe+72m5TkhLnjt08+FPvtlYqLJCSMMkaGJENTdovMKIcxnPKhy/CsD/xwzl1Ky8XDgB/RVJbaKq2DP9zbCDVvY9DAPTpmLoF7RRm3RfBUZsPmSW46hoJrurlEkof73YpSY6K2W/TLciCyCFT3Ju3heJJnQdR+/sAGXLJiNe54ZEvdht6xdgtBSx7mXgTk3NpS6XWn34hKryFnWCm8ANcYrGqdWHLC7DHRzXHuaUfg8jsfxf2Pb1vo4SQkJOwAjAxJ1qiWwLJMtT4kp3rzE9g3TDlnCY7/sqEsta8KD0IgfR9nbDvQlDPZhVGS5+IHLSU/qV1qn02hh7bAPSJoY/k82C20xkSXPMl64PEqL7vFXDzJJiVafV6DlPGei7VA9MdrYWOkT62BXr9ZSZa86LH7PVY8xZZib5kwzGK1IiFBwrlnHIluloqLJCSMKkaHJGuNTAG5mh3xmA2GKefchjBwL2x7GALZ5ruU7BxAM2mk3eZEknU8E4PWw1lKeJtNxxBBk5Xkgbup+ijh5Eke3JOsTF+Zwpw8yaX3O2i0e5LnoiRLWSjaJmHVuOhVY0ZYfZA8yeLxwt+GpExTe4Nm+kgkOWGuOHCfCZxz0pPwtZUPYuP2VFwkIWHUMDIkudTVknamdmIxEYE8DIOmwL0m9W4QQt5GqHnfg6aAo21tdpYmFDpUku2yuh6IfPkgJTymINJ4OwJJHrostWae5LK0v0VLMxlXkqFAmvZsiJohg6SgN1hYzDFz8SQLffPfLD5O+yrdMw75ZoQ/7Mu7XyCfr7VbtCjJs5iIJSTE8OYzj8Zkr8AlK9Ys9FASEhLmGa0kWSk1oZRaoZT6hVLqNqXUB4R93qWUul0pdbNS6nKl1JFsW6GUuqn+/9vzfQIEDW3sFvzZd//j23DUe76LH93+6Pz32aKEtR8vv/fb8x/q8+JJZpzFVcSb2qw2ziVwT7JbaPZmmImAP67YdZFUzKDvAUCpxyY61m4xCFm0x1evld1i9p5kOoQrya3Xaw580LUF0evgSjLg3jPSRKgp9Z/k15fuQWu3iI+J75cocsJ84IQnLcGZx+2Pi65eNScBISEhYdfDIEryNIAXa62fBeAkAC9TSp3u7XMjgFO01s8E8HUAf8e2TWqtT6r/P2deRi1AGyXZJYY3rt4AADsk6XsxYH7hGCSFTmrPVyuHym4xQN/9MiQwYpv1+c5FqZeUZD6m2RAY60luVpIlpXk4Ml692uwWPE9y+7GWJCtkGX0//LW0k5XS9N1mMZivPMlmVWMAXy9XnflERZrASb+79l75Buke8m1JMcxmIpaQ0IQLzjoaazdP4Xu3PLLQQ0lISJhHtJJkXWFr/bFb/6+9fa7QWm+vP14L4LB5HeUA0BqACu0W9DYTCknMFXP2JDd4gSWVbRglORb0ZLeHS93SONwxVVvnkidZsgZwMjWXPMmx34BUR4lIDvOzUT+mmEipHXK3YdsM3nHxDdHy16S+ZkxJnosnmauvbT/JvNktvAIhAxW2gXYUNskTLE42IvewhuyLp9+31XqSPMkJ84wXHn8gjlm+Fy68KhUXSUgYJQzkSVZK5UqpmwA8BuAyrfV1DbtfAOBS9nlCKbVSKXWtUuq35jDW9nEizG5hctPuAJI8r2Wpg39YOXF1ieMwnuR4dgv73q2+V71f9fg2nPKhH+HhjZNBmzqi5A2CslFJ1kypHJ4kt9ktRPVxCPJI528D90onQPJTV96L7978CL587QPhwdqOj9+Ks8uK4h5UFRNpI4bD98Pbtw253w3SrO9JlpRkybZi3nudxO6/Qb3Gs8mgkpDQhCxTePOZR+PmBzfh+lUbFno4CQkJ84SBSLLWutBan4RKIT5VKXWitJ9S6lwApwD4CPv6SK31KQBeD+DjSqljI8e+rSbTK9etWzfUSQDVgy+rU2tJy8NCsbU5oy3PaxuaPcnhtkEzKTjjiSnJkMdO7y65fjUe3zqN/7jpIXFMs/XeFWWY3cKqkrMjMAPbLeaoJFM/43UKuD7zJAMw4Xiyr9b6hnme5Nk4Y/32NdonLXMJUnMVX9Ypmicz2rlf4pO+2PhiqyGllrNbDLrSkgL3EnYEXv3sw7BscRcXXtVe9TEhIWH3wFDZLbTWGwFcAeBl/jal1EsB/AWAc7TW0+yYh+rX+wD8BMDJkbY/o7U+RWt9yvLly4cZVn28DYiSKuHtSLtFN7fq9fptM/jaysGinLVATgmiyjaEFcFfko9tB+TAPbIUTPfCZXL/mGEQC7oCyJNs3w/cZtm8zE4ErUl9HATWblGXpS6ZJxmuQuyDnxvA7RaD90+QSOOOVJK18969B5ua5dYM125RvXKe2xy4F06qZE9y9Tqw3aJxr4SE4bBoLMf/c9oR+OHtj+KBJ1JxkYSEUcAg2S2WK6WW1e8XATgbwJ3ePicD+DQqgvwY+35fpdR4/f4AAGcCuH3+hm+hoSu7hVJikY8dwJHNw7uTZebB+45/uwHv/vrNA/0j6ZAPYUnZ72coK4JHrIPN7GspcI+IoBRwBcw+w0VRhoSOTwJm40luK0vda7JbDNFP6ZHkvldMpOkWq/bjSjL/fjj4p6EHIMlzQSlMouibpn7NJi2T5Jgv3uznt2M+24p7fIWozZvu95+U5IT5xhvOOAqdTOELP1u10ENJSEiYBwyiJB8C4Aql1M0ArkflSf6OUuqDSinKVvERAHsD+JqX6u1pAFYqpX6BSoH+sNZ6x5BkpiRLBLPNk7xuyzTe+81bMN0vBu6TFMpOpkw/j26ecra1jdkfp/TZJySDPNtbPcmRPMn0joggV5L5mGYbvFeUYUlhruwNo5b744rbLeLbh6FJNO6xDs9uEe4nkbTKSlK9V8pO2mZzGaVMKO0Wg3CHlavW4yX/8BNMzgx+zw9zL3KfsVTVsdTV347bXkjIw0mBvWf5CpGpRJg8yQkLhIOWTOA3n/kkfHXlGmyaDAN4ExISdi902nbQWt8MwSKhtX4fe//SyLFXA3jGXAY4KGi5O1PuQ5LetXmS3/eft+LSW9fihU9Zjl97+sED9UkP6pzZLai/QZTrtme0Uq5PdxgvZRtxjNkt/OA0Pmngbc1WSS51gxKu+TkO3qYpJtKqJAtdzoKM55lCN1folW7YH/3mUpNac3+8MpO22WSdkPzrs7Fb/OV3bse967bhzrWbcfIR+w50bFBMpElJZufGVyQ40e7kCv2SVS7U0vHhpIB+c06SJRuHhJQCLmFH4s1nHY1v3vgQvnL9arztBWIITkJCwm6Ckam4VynJCjlTdavvw4ephK3TfQDARB2UNQhMJTdmtxjGA23HFj6wS62RG9Zlv2MfG9FKMiP+Yvqa1NKZfonP/fQ+vOZTV3sWjdkpyaUWlGTzymwDQ/CXdiV5fgL3zKRIKXSyrM5uMZg6zX3DVBly2P55W05/A9gtpO2ksEvlumPH+uS4qVe7r5/dgivJmfOdFo4PlGTYDClKsFsMnN2ica+EhNnhxEOX4vRj9sMXf7bK5DJPSEjYPTFCJFlDoSLKTo5hwbsoYXu95Lx4bHCS3DckQzmEAGj2p/rIlAoe2FpXaYWAZitGDG22BUdJ5t5QoyTXgXv9Eveu24Z7122bFyW5KBtSwGmrAg7nSY4XCwGAXr8pBdzgMCQ5U+jkCr1CJqeiksy+V8qWpZ5VVhTvc6nb1VOpF34+gx7r50ke6F5E3JNMfTcWsQnuf/ubi3aL1sC9wceekDAbXHDWMXh40xQuvXXtQg8lISFhDhgdkgxrt5AD95qJAJFkKjk8CEgR7eSKqVOhwtUG30cNuEqy/1AfxIrQZluQvJ+AJT+dvOp7pl8apdIJ3JuDkhyQGDbBmEsxkXie5Dh5mk0/ld0iQ78sRXIqWSgcJVmBVdwbuPvomB0FPgJpM/2GnTaSLPmEW4Ilq3HZ43tiWWrNPMnhOO3x4XgKYfKb8iQn7Cp4yVMPxFH7L8bnUnGRhITdGqNDknWV3cK3W3Bi0oTJmcpuMYxHlB7UnSxrTGEFVHYOX+m0yqK0hG4VPkMcTSng9jG2EYE28kzHTfeLiuB5CvAggYkSGpVk8AnBcG1Wx8gHka96znYL5knuZKrKkyzsR21y8sn7cTzJs3iAioFsrSQ53E6TvLYJpJSzWwvbwk7ti1xMJLzHnSsamTBpNnbXkzzYBGs2E7GEhGGQZQpvPuto/GLNRtywOhUXSUjYXTF4X9o5AAAgAElEQVQ6JBmUNaCyW/ikoM0jTEryUAFjTInzSSkngttn+jjx/T/A337fyZzHyhRH7BauJXkoJbltSTnWhH/cTFGiKBFcU35+26b7g6W8q9XoGKHz8/0OSiBtCrgISe4NZrdo669kxKybZ+gVWjyGvvFtDJIneVbppoUJlZhCjV9LoRmyzLRf57Ad2138WE54yfLC+3OUZDMBlHr12tVo8SRHh+RsTxw5YUfid55zGJYu6uLCq+5f6KEkJCTMEqNDkjWRD/sZ4IF0zcdPzsTVxhhMCrg8CwKa+IN682SlUvPqdXyMmQof2KXg1xzGS9m+/B4jqu7YYnYLrgye//kV+JWP/KR1THwCIRE4brfg+7ehbCBHRamby1I7qw7N/RC57xhPcilf5/q7MS8gju6XLLOe9flQkv3fRtpP6mZYUlm14xLrJteNZvdSrwivs9ZVZhg6ByAWuOdPClieZPaHTe02eZJjNqOEhPnG4rEOfu/UI/D9W9dizfrtCz2chISEWWCESLKuslsoz6JgiGgzS95GdouhPKoVQ+g6nmQ7HrMfLdN7Y6A9qrHpYFteG1ftErdLlpvge0dj28PvrcoHVIF7hSHJ9iCeoWLlA4MtJ7oBgmLvbtDlkEqytD9PYddUpS223enHkNzablFG8iTXr+TrJsz06X7J5r3injR2KbUfhwl4bJ1Q8b7c75rsSZq98rzavGqfn91CujF9Iq5h//ZEu0XDRXUIPzT+8ju34we3peCqhB2D8593JDKl8MWrVy30UBISEmaBESLJFBBVPTQLj+y1pZuYzRIskbM8U4HqJWbY8ORsM7SokuyOTVL0YhimmIh7nPs606+IoJ+LtzeL1EactPHAP66UD6Ps2v100D6BF0ORbB5uf4Nd0zxit/AP972+PGWgXfGYu5KsI+20qfJ0/7Zmg/DyW/Dvmg7VjPj2+uFYpMA96Xj/XtWa50mu/P7v/ebN2DzVax2Tf02+ccOD+O+71sUPSEiYAw5ZugiveOYh+Mr1a7BlKhUXSUjY3TA6JBkwZakBd6kXGCxvMdBOzDgZ6QsV97ilwOxX2v3cMVuSLQbueQUnyiEIpBgI5WyPHRkqyaWuyGCb6joocQdcdZArjr7SNwistzbcNsWLoYjZLVh/Ld0F2S2KUjwmZhOwSrIN3JuNkhxW3NPiBKBNSS4arhuH81t5k6hBOb6UAk4O3GN9we7njEfzPMkKn/vpfbhkxRr86I7HADQHMfpBvUWhzXVISNgRuOCso7F1uo+vXL9moYeSkJAwJEaHJGttUsABPLuDVZwGQZua6GZ4sMqgIQ1CO2ZpOKIkSyngtNYsTzKc10HG6ZOZ2PbY93TdpnsFylIHSrKUJ7nV0xtRknnfs/EkN2W3mOm3KMns/cBKcu1J7peR7BaQFVpzv+QZq843PEEL75WIH7ul7d6gdgshcM+/T+Tj7KuUi1trbSwpvu/eeS+cr81uAUz23LLaTefjtI+qHb+4TULCfOKZhy3DqUfthy+k4iIJCbsdRockw80a4AclDa4kNz8w+QPVpIDLVfCQ583M1EvNvieZoKTsFoAQuNesDErnEQ/Qa1aYaTMpyU2Be4S2ZXvOiyV10s/3O6gn2Vprwm08YKytLHXrb18wJTnLooF7MXvtjJlUsRUPbyxf/Nn9rcuyUnENMbuFcI056PdqTx8Xvh+kah2/Ds7kh407N55k2ibt5/bCK+5lSjmWGqDNk8wnYVU7s835nZAwKN581tF4aOMkfnj7ows9lISEhCEwOiRZu57k0nuYt+WC5e00oe+oodX7bp6xh29IPIhQ+inBuMotBWP5xURElS2C2aa58r2mM/0qBZzWsoXEH3PzmNqV5GHOkdCkJPN+5DRpbHwtXKkwv1etJBdaHGOM3E3XqvZYJwtWPADgmnufwP/5r9vx/v+8rXEcwYRKy8VEJPWWoy11nm3fvvcnX42qrXnVzqBL9nuRBakwYwnbkaxIPE/ydN/94Zoma769pl+WSUlO2OE4+4SDcMR+i1M6uISE3QyjQ5KhkSkVBETR86+JIruFDtrUxJK9t8qiT0qlIDdfzaZdeOAf30aEX/IktxFI6Ri3/WYl2XiSi1IkRLMhyZy08fe81PGslOQGssdtIbLdYvD+iERXdosMvbK5LLVP1sj60clkTzLl6t44ObyS3Jbd4qY1G3HzgxvFcbb68BvtFu6+L/v4lTj3c9cFG6U0blq3eJIjanU1YbN/hzMeSR40cK+obUTJk5ywo5FnCm868yj8/IENuDEVF0lI2G0wMiS51ABUaLegB22TkLxtuu+20wBXSQ5TwEk2B1pmzyJXO1Nh4F6prT2D+MAwnmRTnCG2PbLBt3bM1CngAHcyIXnr2u0WLkGxfdq+3XNsbM6OxWRpiG/z+/f7rvprmSCVdkWgm6l44F4kVR9PAacEJZneta15hH3KijZv+6+/dwf+7vu/FNtrsidU7fC+3cmnf83uXLsFV93zeD0qO17/OtNnyiU9qPpbtcvzJLtp/qQxOceye4TaSEpyws7Aa045HPtMdJKanJCwG2FkSDJ0nd0iYrcwqpnwQNzqkOQ2JZkrxKQkZ4YR0FZO2Mx+kTzJXIk227RmSrL9btBxti2H09c0JBtIRu3bfek9P3cxcG9AuwIQJ0WzqbjHl++DPtmgJCV5GFIeBO7FFMgIgbSBe9aT7FZhbp/QAeE9XGr53Pg1nilKM1kL9hvwXnK+Y+p//Dj76tsc6NpQ4J5kt7AOJl+FtuqvZLcotcbF163G11aG2QT4b2KLzCRPcsKOx97jVXGRS29di4c2Ti70cBISEgbAyJBkDTe7BT3YiReYzwIJ2zZd8IYawT2uppiIkALOsVv027JbyL5Lmyc5JIHthM7tI9zuEvdOg7WjNEotOyeBWAyTGcRRkulVu4RsQI7c6K11A/fiZA8YZPzVa67IbhEJ3KP9tcbZJxyE804/EgC3W8ieZPuumSX7Pbop0dzvzTFavj60bdD+/EnUIBMZjfA60ye/mIhr7XBVa348/eYKED3JX/v5GnzzBrfCJe8HsBO9pCQn7Cyc/7yjAAAXpeIiCQm7BUaHJGsvu4UhyZTmCvXnkGhStb3qu8GVZF5MxFeuJU9yqCTXalimAubDA/eMSi2ojjG0ZbegbzNDkmuyIlg7aDlbyuzBMUyWhBhhdTJgDKoka5lM+f00+YcH6c/Jk5w1BO5pm1f6hEOW4NjlewGw90GXKcm+wgoMoCQLEyqjQvP9vHnMIOq9BLeEuHszNh1qbSfuddKsz65RktnGlvFpMOUZVZpCd7zVxK5t5YB+j+iKQELCPOPQZYvw8hMPxiXXrXZWMBMSEnZNjA5JRkUuco98+CWLJVLEH5JDeZKpmAjPbkFkmREU60mOBO4JnmQNnqkjJN5t/NGSdnk75ZUmVtVpsHaQ0s6XpXtinuTZKcmcd/EW+qXGRy+7C09snW5sl34HqX/uo24rgNL223OSnCkFXf/nnYbTVqasBWi6sJ5ku5+j0wJo9yRLyiovrmHG612P2Pm1VtzjfzPeJKopMDQ2qePbOp4nWVKt/S60dv+uJSW50HKpbj6OnrFbJJKcsPNwwVlHY8t0X7QDJSQk7FoYHZKsq+wWJiCKHqLmYVp9L6XFalMbObjdggfueRzZU5JrctXoSfYJDUsBZ87R3d6EQTzJjCMjz+OEnIo1uBMEwW4xjCdZUCe5VxUAfnr34/iny+/G+1pSovk5sTn6rXmS7fu2ADb6vTu5AlR1vtIh/DwyZYkr5fPlnmSXSFavrdkKGxRSfqhP/tr86dHuhPex4ETCZK/w7lf3vVGSvYmg1K+UGYMmbFoLdgutUZSx1QqLnrFbJE9yws7DyUfsi+ccuS8+/7P70wQtIWEXx8iQ5FK7dgujEnt+1VhWBftdC1ESgteqintuP6Ldwi/7R0vkKgzcg25WkgcNMoudjk2Z53qSbdCZ3XdyhpRkdu6zsFuUZXjt3DG51397bYOJBZz5/TblApYmItUx/H3z+IlUdbMMCiHJJWjWVpYpQ1yN3SLLTKYT18pQQbVoyZKSbPOBx88ndn7tmVLCvw9ppYKfy5apvus154q7dgvxAG42Gj+I1IcG/7sGZvq+3UJXdgvhHnU9yXumkqyUypVSNyqlviNsG1dKfUUpdY9S6jql1FE7f4SjjwvOOhpr1k/islRcJCFhl8bIkGQAgFIs72r1lV9ogj/wjZI8BPmMVdzzfbH8Ad+L2S0AE2zok3OxmAg/tk1J9vZ7fOs0Nm23+XdLXfVNhMTPVysV/mjPbtE8plhxCz5ZcbzQtfI6ljffpuY3FvqnsY93sgEC9xq7MeplpQTTOYTnVGrrreYTkZl+iawueDMbT/KL/v4nuGTFatGTLE8A3O9ic41Wu4Xw3jbNJ4OcJPdsOkH4qf0sgR3reIF72k50Y7m+OcmW7RaI2i0cT3L9QbIOjTjeCeCOyLYLAGzQWh8H4GMA/nanjWoPwq+ecBAO23cRPp/SwSUk7NIYCZLMg5b8/LPWu1h9XwjkbCi7BWMa9JDt8GIiVHGPp9/qU+CeP26rfvvPc42wLPVQSrLn8TzlQz/Csz90mdu3spolBe6FJMiSCFdJnlt2C0mJ5goswMo4+xcu0m5T4N5YJxPHN4yFpVfY31spIqfseOE+yjN7T/aK0nhwlbcfYO8diST3ihL3P74N9z++TVCSLQHmKnRgt4gG7olf23EJLNlXlAE3X/HmKTcoybeVmN8lzwHYSZeGzVAj9l/33eRJLpuUZPYdZZ3Zk5RkpdRhAF4B4HORXV4J4KL6/dcBvEQNWq40YWB08gxvfN5RWLFqfVDkJyEhYdfBiJDk6lWxYiJ+2rImJXkY8in5cjuMzFJToifZqyZSpa1TYgq4Uod5kofJ/CCNw7WaaMeeYgm53W7HX3Xcaw3caxxS1P/LRUmXdNmUaU1oKktN44wqyQNYbW55cBO2Tvft751nNnCP2QNsukHuSbZVIGeK0nhwlXJ/WzoOCO0WazdN4YEntgOoMjlouON0ylJzu4U3j5HS+jWdt7Tdz4/Mt3GyuoWRZK39AEmbeYKUZPpMkzd674+7+mzzJJc6zG5B7cdKnxPMCsme5Un+OIA/AxA76UMBrAEArXUfwCYA+++coe1ZeO1zD8fe46m4SELCrozRIMn1q0LcbmFUY4EQ82XooTzJxu+amT7sg90eYz3J3rhrJRmSkqyt8mxUYcG/GkNTWjQ6XspuIR3Xl5TkWVTca1eStUjOuwMqydJvRxaJsajdgu8btl2WGr/5iatw3oXXmTF38kpJLjUpn2HwGfGuSq23dotuTQr9fN7OWLzTPf1vLsdLP/rfACoiGiqr9h5xAvd8u4Wg0kv7+eBbjRdZUO9nHJLcc2xCfhd0fcZyd1KroQNHdnC+0E6sge9ZL2oVWbodpcnrnqIkK6V+A8BjWuufz1N7b1NKrVRKrVy3bt18NLlHYZ+JLl773MPx3ZsfwSObUnGRhIRdEaNBko1qZ8kHPfh8ldFXU8Pvmvtyi4lop8+qSEKoavZaAs8kT7LWTN0Ftc+8wy0Pdskb7bZfZwOpP4d9hYRKmiC4fbaQ5BYl0/fWmuIbLSTZlKVuVJLzWdktqO0bV28077tZBkAZckrecU7WbbEWa5+Y7pdGFZeItZTr2MdMvxQq7llltilwj3/kxHLQIFDehlGS2X5cSd461XfyJLurNXa8lA6P/3bKm0AEw9Nu/vPQblH9Lm2WoD2wmMiZAM5RSq0C8O8AXqyU+rK3z0MADgcApVQHwFIAT0iNaa0/o7U+RWt9yvLly3fcqEcYb3zeUSi1xkVXP7DQQ0lISBAwEiSZnnGKpduynuTS+eySItTfuQ/wJrhlqcsqZy5Tr6UlYlv+1m2L1NxMqVBpq0msM06EwXwxtBUTKWsVm8ZOXlnJa0qQ0t9JfUbHFFGSOeHizZLHtdsauOcWjHHGXNjgvzZi3xb81mceaZrY0G/I99XsPQ/Sm+mXRhW3qQqrV+2QxGrj11auwRu/sMIZz3S/DEijBr+X2di985V88n7fEvhmfxLo2i2s7WHLVN87zm2PxkbKul0tYYF7wt9S9dnNkxyq1BUJl1wU0grPnlJMRGv9Xq31YVrrowC8DsCPtdbnert9G8D59fvfqffZMy7QAuDw/RbjZScejIuvewDbUnGRhIRdDiNBkm3AkwpIpB/UJQXpzTa7Rb/wcjNr7ai+hF4/VJdpjAqyJ1lrq6By4upnoYghRjD4dq4kdxo8yeZ8vXP34S9b94sSX71+TaDqA7LqG1OS20lyfEJAYx7vZqL67pO3WNuAELhXN+CrwlqDKbvKC9yrPtgMDhVe/o8/xed+en91TP3du79+M37yS3cZe7oflsKmQDX/fJryJLcVWeGQrpH/Cgh2C3a8Pxmh32XMKyYChNdGslvQ/jN9ebJWlPJEzrHzGE/yns0BlVIfVEqdU3+8EMD+Sql7ALwLwHsWbmR7Bi446xhsnurjGzc8uNBDSUhI8NBZ6AHMB/hDlGK8iKQEeZIF5XC2xUSKUpvqazQOKUAwVtlLo5JzlVBxjyvJnLj63uEY2oo9lNr2DTSngDPn66jogurqcZKvrFyDv/jWrdgy3ccFZx3tTBx4W3YJX0Nr6xewJHnA7BaCcki//0Qnx5bpXrCdn2ZrXt2yWjlQdVo3yk9M185WjbPqZs5TwBVlbdUI1efV67eb37Ypl8B0vwhJo+YEPT4R4fckJ5dD2S1Mn+F9wm0Pm6f6diwaDtPWsNeqawL3bLv++UsTSPpdt8+4QXtVW7W9owwvpDR5LfaswD0AgNb6JwB+Ur9/H/t+CsBrFmZUeyaec+S+OOnwZfj8Vffj3NOODFKFJiQkLBxGQkkmONktPAJsyukK/mPJFxqDnwaNq7F86VdS7QLiqSkFXEhUNLhP2CrJfsaLGEyAVcOOPPOCryTLdgtGwAawW9C1uuexrcH2gZRkkz2kRUluIvakWHaywO5y/+Pb8DgreS3aLdgxmyZ7lsjW49VAkN2iKJndgnmSe30dKslsMkfXt+kRWQXu+aRRs2sgj93f5pLklruJE1xmKak+y21OzhRRJZl7lMPAPXv+klpN+zSp35T+re0e7SclOWEXwQVnHY1VT2zH5Xc+ttBDSUhIYBgpJZkXbvCD8mRrBe0D9l1zXz0veC1T4fKw385MVEm2xN5/oGtt/cd//b07ce196x3Vso3MW3+svF9pFDvyJA+gJJP6lyuTI9rZ7h2z11h1exERdewWEesD/5qKibRlaW3Kk0wTlLGOa7d4bPMUXvT3P3H2lY7n5/Tghklj/VBKgVL++RMznqO3suNYJXnxWO6ck2MDqt83paWd6ZfihEqyyQSTFmEC4h8jwa+Wx9t2lWSr6hYeKeY9aM3vpTAFXOZNDqVCO03ElgL3tHAZpcnrnpLdImHXxctPPBiHLluEC6+6D2efcNBCDychIaHGSCjJ9ODjeX+JAwTFRCQleQi7BSezvULXy+/hsa7dwiXstn9de5LlFHB82e3Hdz7mZLxoeq7zfM2x06ndFmbsvlorHcd9pFIKOJ/MbK/z1xqS7Kh4Qgdae6SrVuBbSExTnmTKQNLNldP/Ry+7q3X8vG2gIskdFnhHyrdvVSnY9edlqad7hQmQ5FYarStSTcS1TUkOrDmMlJcamOoV+JtL7wgCgfj5cdW3JfmKo0jzTCSAOzGkSU3VpnY2lt57v+Iev86+khz8/LrZIlHUgXtyXmz7nv4u95TAvYRdF508w/nPOxLX3rcetz60aaGHk5CQUGMkSDI94pSynmTfbmE8lEJBDinFVQxcSS5qj6ohPJFiH1TZSwzcq5fjJU9y7imKpZY9yd+68UF8+xcPi+cQDdyDduwWgSdZUopZzmE5cM/9vL0maUSSncmI6P91idxkTbKb8vjyUtaxYiKdrCr+wfu8d91WYfzSmDhJ3m4mEwrKWG39tIOl5koymCdZs2Iitn2/2yYlebpfiEoyH+e/Xbcan/7v+/CJK+6Jnt8wdgstvKemHOLNboB+qR27hV+0hfrseoF7VGCH9xUUT4FuJLY06WjLZmKyW+yBnuSEXQ+vfe4R2Hu8g7/41i3YmjJdJCTsEhgNkmyUZBUsffNUUUCkLLX3AG+CW1DDVswDgA9//w6zTUo1JdotQIF7CLb5ARyOksye6//jK7/AH19yI+u7nfSXRNDJbjGAJ5kmCGMdOZ2af37b6qCqdVumDXEhiMVEPMI4WR/fpHS6QZfh9n6dUSLPXCVZJsTh8fxaTvVKc51MRhIdph0sS0sCnYp7/UL0JPskrcleUpFbf0IlT0A2TfaC/Qg9J3CyhSSL95P2PlsleVE3r+wOTHV2J26IZregFY7YuKvza7ZIUOCenNLPvu+zqn1tqxUJCTsaSxd18bHXnoRbH96Mt160ElO9MCg1ISFh52I0SHL96palRv1qH4SAnMmibCFaHD1PLcuVzZN8yYo1Zpvk//TFL10TLKmYSFnqoEKftLQvwV3ajijJ2k1P5ts4JC8zXyJvS68FAJMzlRoy1Svx+NYZL6hMIMneeI2S3KD09YXf09+eZ1VqwDZC3Wa3ACDbLbxiIoW27WcZTwGnraeZjcPvo9Vu4efb9iYXlA3Ef8jya75281T0HH3wzUZJpr8vwZO8eCxHvywdJdjxNcOmrCO7Es8xTasrMCTbHd8Nqzdg5QMbGsfrK8mrn9iOX/vYlVi3xQZqOmnw2paQEhJ2As4+4SD8/WueiWvuewJ/dMmNoq0tISFh52E0SDJ7vuWe3cJXkt3Apup1mBRwfmlmSgkWjslVnIGQGFIJXkon5m5DYLfQ2v3u5gc34gs/u998Puo938Xfff/OQEmWyF/lh7aTCpOTuSF1nKk4l2diCjifaGxj6bk2T/W8zALSmFxCNqySLOdJLtHNM2SZavWei0qy13eXeYopL7a1W9i23ewWNnDPKtFcSfZIclMKuF4h2nb4tSff86SXHo0f97N7HmfWnXh/dQ+sLyKz2ttiPeSLxnIUpaceO75m+7v5E5jqGM9u4Y3vn39yb+NoKyW/vp/qdu96dAt++egWrHp8m9lvmFzRCQk7C7998mH4wDlPx2W3P4o/+/rNaZUjIWEBMRIkmZ6mPJOA9SRXD0JjrRCIEud7bf8eOQU1So0sk5W/gfIkawCKylIj2ObbLXwl+ZxP/Awf+K/bnX18AsELL/jtczKWZ5n5ntr3YTzJkcA9v5vtzFc33Svbi4lA9iQ3TVzcUtfC9rLycedKiZlNYm3FviNiCWXHa7IxCHaLnKUILErtkOxqHG7O6KrphuwWRVhxjxcTAayFYdJTkm0GCY2r7nkcZz35AHN8ExwlWbvf8W1Ekvca66AoS2fCxZXkUtuUdVm9EmOvs65tQM33YhOcuAEzWa7G1ivdlSD+fqpX4Gsr17TaTxISdjTOf95ReNfZx+ObNz6ED37n9nRPJiQsEEaCJJvsFkKe5GZPcqjwtv1jxAOeqhRw1gftjsm+j+ZJriF5kqXAPceT3DDM0iOOsQAmXi2w4xM9SZWtycd4J2P5p3lf7jG80MNM4WZlKIoS///ld2PtJrvszwO6AG63aCDJAiHiqAL3aiLGSLSkTlPf37n5Ybz3m7eIfXdyG7iHWvk2GVXMxMyWRFbe/WHsFmzFYygluV8KKxLu79XtyPcIfV71xHas2zKNs44bjCTzzcFKg2O3qE56Yix3Jp7+CkGl8FbvSUnm50Q+fUmtHgSSQkzXmAf8OX/LRYnL73gM7/76zSavd0LCQuKPXnwcLjjraHzx6lX42I/uXujhJCTskRgJkkyPPQVrR6CHsJ8ezM1uUe/TojByOJ7koqwrqoX7uUUx4sSzsjxUKu0H/us23PXoluqcIkqyn8/YR565hFtDDmDSpm9ljqN+q/bDtiW7RVMwnEOS+66SfOvDm/EPl93lBBz6/RKJeWD9drz9Syvx3ZsfCcbUFnRZBe5lyDM/oFFS16vv/vDiG3HJitXifl0TeGfbC7NbuNt4dj3jaTZ9Cp7kBpKsNTDtMfwqm4b9LE3agDCg75Cli8x4myDlSeal0gkz/RJjnQydrMr77ZBr9p4ryXmm6v3d/fgZSMvNpx61H/70V48Xx+tXxQQsOe4XspL8g9vWmiwsvgKfkLAQUErhf73iaXjNcw7DP11+Ny686v6FHlJCwh6HESkmQkpymLPYLzQhZbdwlNcWwuAH7mVZTEnWwTFinuRaaXxsyzS+8LNV+MLPVuGev3o5SuMZ5kvbVl2OKd7dXAXnE/P/8uvVzV31sSmIbayTYfNULzgnn4xvm+ljopthqlcGqcsoqIwTEq1l8n/dfU9gul/iB7c9ihc/9WVYVBfkCPsPDkWf2y243UNS1z11mfLtctBkQjkk2ctuwRTxPFOOfYJSyNkCNGEfRBGlSoxA5Uvm0Fr2wAfnV+9D154KmwxltwheuZJcYDzPqkwipXYItZMhA0BZX+zcqPzWFkWBe/b4cExZFk+VN9MPVxfob3CGXRv+t/w/v3ELnnX4suD7hISFhFIKf/OqZ2DLVB9/+Z3bsc9EB797yuELPayEhD0Go6UkC3aLME9ySOqkUtUx+HaLnBFNDtluEY7bVNxjD++f3v14vc0l4BqsGllknN08c/I1a60Dzyt9T95PQCiBLfCEvpAnuSm7xORMgX0XjwGoK8UJPm2ulmto8bx4/l1+/YH2oMt+UanvgwTu+WR161Q/INPdjAfuVd/ZstT2njPZLbz7o+ulgBM9yQrOPj6meu414GWp+Th8lDWZJpJMk422wCCpip+0MjPdLzHezYIJCeBaJjRTvnPlpuerglm9wD1hTDGbE+ApyV4hH64k+8Gn6+qMH9P9RJITdh108gz/+Hsn4flPPgDv+cbN+P6t4YpaQkLCjsFokGS2ROt7dpsq7lyqBxQAACAASURBVMllqZsJA1ei+kXpEHMO3g8VE+HffeQHd+Jfr3mgogPKDSha9cQ2Q2L982zLSDCWZ0F2C1FJhks0cuW221SWeiy3eZIdT7DHLbbN9LGMkWQnLV59TXJ2jpVXVVa9Cd+/7RF8+doHgjHxsXNUSnImBO6F+/rq+eapXjBZ8O0SgEt4q1cbLKmUq3jyFHK0b5AnmdqVfDwI7QC+57epZLPWlgRaJTm6u3g8P4YfOtMvMd7J0clJSdZmH/+eNMVWMirLbrdR/m5ppYfA80/78Fd7AJgy6vza+Irx+u0z5jwSEnYljHdyfOrc5+BZhy/DH19yE666+/GFHlJCwh6B0SDJIEKiAn+o70meazER/mAtNS0XC2MSPMmc0H3yinvZmF3l7eGNk5UnWYUEoS1P8ljHJcnxogo1CSfVkvL5Cl5TglNMRKhW5vezfbrAvou79TVwPcnUVu4oye2E7X9+4xb8r/+41XzmpCeWAs4UE2lTkutTmehWP+jmqV6Y3YJyDDLia/3c9l6j91VFRnu8CdxTNOa48usHblLWiiD/sTe5aCzZzJTkwe0W8koEfwUq8j3Wycz9bJRgHQadcjtKntnxa8BMHJvSEcYmp4BrtzABvML96mdoIYU+keSEXRF7jXfwxTeeimOW74W3fWklblgdzxWekJAwPxgNksyWvf0UcH3P6tBmt2gjaf4DNI96ku37tuwWmXKXfh/aOFl7ksOsF7bintxWJ3ePafQkg9kt6lRl3JPczd3z4p7knjcJkc5v20zf2C2me252C7JQcJU15kluQpvPmFLAZZnv1Rb2rb/be7yy6m+Z6gf7UVlpTnzpHKJ2C6Y7+yngpDzJ9JFPIP7k7OPx/539ZAASSXYnQlIOa74vVcab6A6mJPNLwKvo+dtm+gXG8jpwTzNPch08yv30Jk+yCtPzKeUmwYspybEAx76Q5s0P4APi12kmeZITdlEsXdzFv15wKpbvM443fn4F7ly7eaGHlJAw0hgtkgwVzdIgRePzYL5Oi0JL8JdoY2oWJ2xTDanMekUZtPHQhkmnSAWHyWccGV83z7zqZqHnlb7ndgsKQDSeZK1NkBmByMe4oyTLxLMoNaZ6JZbVSvK0pyTTZIOfY+VJnj1Jln67XlFauwXfV8ypTDaEiiRvnuwFkxFrt7ADN6sXZiIGx27Bz9EvJvLwpimsXr/d6YOuAT/uecftj+MP3AeA4ElG+2SBUJbAVF0Zb6KbV6sVLSzZCdxjxLfaZjdy/7cUPJiZtHduMRHuF+cTXpi+QmQNSrKz2kN2i8J99feLHZ+QsKvhwH0m8OULTsPisQ7Ou3CFUyAnISFhfjEaJBkhIbH5UV0VV0r35hbpkPuY6hX45BX3YNtM4SissYc1EZ1+URoPqETipvulo4gt6uZ4qLZbSFJZG5kfy7NA+ZNKSJe1lMyDxBTLpqBZXwTRk+yQM7sv+WadwD02Lpo4ZIGSLJ5WFK2eZB64pwfLA71XrSRvFgL3bHYK+52v7jt2C6U8T7Jblvri61bjT7/6C6cPbkUgjOU5xjqy3cLPL91rsFuUzG4x0c3rlIHNF93PTFGdq/sZqP62KKVbVY3QEl8377d2SLIfuOf3JU2cfE8y/1PhRNjep6HdIkaGk90iYVfH4fstxpffcir6RYlzL7zOyTefkJAwfxgNkmyU5Hh2C3o2SoVDytIW1ogpmdfdvx4f+cEvseL+JzDesSnI2uwW2xmhkRS+mb6rJB++3yI8vrUKIJKUZPKlxtTCQT3J0K4VIK9TanEFvuPZLXieZDG7BXtP1faMktwvnO1E1PxLNxe7hexJ1qZgBd+fH0fXmfreq/bqbmGeZFN0xQu8q95795xmFeUyL7tFFh6/hVUmrNqpXjlJ7nYUxjtyJT1/ciGtHJhtWhsleqKTiYVsfPDNpUdm/XSDVEGvX3K7BZy836W2k9VMeX5xzQP3tNnfh6r7IZCNBZCLifRMnmRX+ZaQSHLC7oDjDtwHF735VGzc3sN5F16HDdtmFnpICQkjh9EgyfWrUmGKtKDinkOqUH9nyW6MpE3WhTGmeqVR9IB4lD2RgO3T1XG8Sp0Pfvzh+y522vaxZFHHGbuPbp4Fy+OxinsKnpIMV2nt5O7tQePvdjKjyBWRwD0qJLKvkN2ikylD9FwlWbcSNh9tGSv6ZYlunhlyWwjEi86TrtNiUpIn+2Zytaj275KSzNXhoJiI40l2z9EoyQ2eWltBkivJGcbrMUieZH5vNWa3KKvj80yhk2dicGhwjA7/ZkwX7NBqsom6OIgbuMdXa7S2EyqawJhUjZDyJEtKsnt9xltIsilLzdMJRpTklAIuYXfBMw9bhs++4RQ8sH473viFFdjqTbgTEhLmhtEgyYxUWH+oRsnVLG2/J5hgPq1rMhMnn9N9S0zGGUnOM3c53bZdNbRtpvpHa5+JbpQAchK1315j5r3EofaZ6Drt++jmYT5gSTHT2u3XKOlmuzaqJ6FXVNepmyn0Ch0EnTkp3gobHNbJFK68ax3+6fK76zFmZtLhj2l4T3I8uwZQ2y2Y2m9WFNi+dJ4mLVl92punemY/CnLrCkpy7inJPHtDLE8ybZMgXYJunpn7zr+PtHc+3FKwmBVeAarfaLpfYqJuyy8J3TYe7X3n+9A56XVtP+514hX3uJKsdRWwqgDcsHoDNmybEf9u/Mlpl/1N9gt3TAArSz3AZCIF7iXsTjjj2P3xz69/Nm59eDPeetHKYBKdkJAwe4wISa5eFeQHMX0G5OwW9HBvWnrm//AQYQLiSjJ1TUrykokOilKjV5T4+QPrnX1V5GEv5cldUpPkmCrdDTzJkepy2s3DTEq6taBAUJIrawh9X3ptSxk9OrnCWCfDDas3GnV5rJOZJX8nuwVmY7eg85bJXr/UdQq4en/Bk9zxLCz0yu0WRkkWA/eE7BaGcMsp4Ko2ZEirHmOdzFnB8Pd3SbJ9f/CSiWDfqV5h7uGs4Z4nOJvJbmFsFxaF1p59wsrN3G6h4SrJ/goO3RI3rN6I8z5/nTimLHN/gzF2XWckJdkE7oXbfPT6w92DCQkLjZeecBD+4TXPwrX3P4E/vPjGFHyakDBPGC2SzIKkQgJXEw+BOFPAUdbgSeYZBcY9Iit6kktfSe6g0BqX3f4oXv0v1zj7+svqTSC7RYzYSMVEwrLHoSKYKTgp4Cq7RehJJhIEVITDzRgRKnjdXDnXq/ousyng+Jj04IF7l9/xKG5/eLNRTX2bCR9HJ8tEImvH426j61fZLUhJrs5BCtyzhUFgjrd2C+WcZWcAJZmO5Xl8x5iSHEC72Tq4J/kgnySXlSeZSLIawG7h3E/eq3uvaUcZ5qs4GjaDDF+ByJWnJNdtTdWWh1sfklNcqUBJth8c3zEF7BWh3SKGmSIpcQm7H37r5EPxwXOejh/d8Sj+7Os3t64QJSQktKOz0AOYD5jsFmBBWKX20pPZ781x7DtKhxYjDFxJduwWCqK31HiSa5K890QHWmtsmuwF+/KHve939kF2ixiZV0oFgVaSYkZ2i7625aEzL3Cvm4WeZKUsqex711guFhIqoPz68VOkfLqD4IKLVgIAvnTBqQBgMir4qFLAqSADBd+ViK+fsWPLdM+8pxLORPR8FR5wFWBjt8iAjPEyR0lu8SQ7RL6TYayMK8l8IsR/k+X7jHv7VingxmvSP1h2i3BsJoCPbStKjfEOI8l0PGxQnxkvKe11CripXokHN2w3+bt59gvp/vXLUo9FPMkmX7qnKDchBe4l7K4474yjsGmyh7//4V1YMtHB/znn6aIdMCEhYTC0KslKqQml1Aql1C+UUrcppT4g7DOulPqKUuoepdR1Sqmj2Lb31t//Uin1a/M7/ApWSXYJC1fU5DzJ9oFvSbLch6sk8+wWmUhmaUzbarvFPuNdFKUWH8D8eO5Zlf5tWzLRrCT7S+8aMjGgstcmk0KVUoCRH0FJLijFV00qfSWZE7WaqHQzFZBkUmWBUEke0m1hyE+V1cPd9up/uRoPbpg0FfcAO3nh4+74SnL9E22atJ5ka7dwi4Hw91yJdu0WLMCsZRIEsPSB3G6RZ0HeakKp3UkT9yRTdhFCoTWmewUmOtxu0UKSnbHJr1XbFemlYiL8fEhlpmOcinsKuOa+J3DW316B7TOFmzJP8jKBAvfsZze7Bb8PXXLci/3hMCSSnLA74x0vOg5vff7RuOiaB/Cxy+5a6OEkJOzWGERJngbwYq31VqVUF8BVSqlLtdbXsn0uALBBa32cUup1AP4WwGuVUicAeB2ApwN4EoAfKaWO11rP63omPfYUi3gvtW+tqF4lDy1lt2haep7igXuM5FWp08L9qZ3tzG5RavkB7JJkTqLCdpcsqj3JkXHyHL30WSwrXPfrlk9WznF0Tejwfu1JdpTkwu2LwJVAPqkAbLEOYO6e5Ec2VvlBKy+2e+zPH9hQj8FOZHguY0LX9yTX2yqSXO3jB+5xGLuFadtN4xYjc1Elub5FfEuIX6aa4PvO+fuli1ySTHYLuoeVUmhzIIh5kj1vMrWdq4ooF4WGYj97qd2y1NQn2S0I22f6zsSpm2ditommyYfjO9ZEjl3bRRNS4F7C7gylFP7815+GzZN9/NOP78GSRV285fnHLPSwEhJ2S7QqybrC1vpjt/7fZzKvBHBR/f7rAF6iKvbzSgD/rrWe1lrfD+AeAKfOy8jdMQJwg6TKUovEQSpNXGpdBwLFlcyY3aITUZKpP6MkT9h8wT6iJEoI7aLAvV5E7eIlkYGKkHBFkqd449ksKCUZz9CglEvMyG6RGyXZv8a2X1LsOnkW+KwXscBHhxRHCH0TblpTEeFuHk+x1xWUZN4PbfPtFhu3W7vFREMKOEmlttktmiZBMU+ymzmkm1de+1wg6LS/GzTZQJK1xnTfKsl51p5RxPuJvPbctnOmJHO7hYa9z3nxkyxzr8NUr3CWF/KIklyle7SfOy0p4KSy1DHMpMC9hN0cSin89auegV9/xsH40HfvwFevX7PQQ0pI2C0xUOCeUipXSt0E4DEAl2mt/ZDzQwGsAQCtdR/AJgD78+9rPFh/N6/gD2put5AIsVR8oiiramBZpqKEIW63iBcTue6+J/DD29cCqJRkICwEAQxntyBv7JRAtqt+dUA83TRp5mtnUpAreJ5k7RRnAWzgHtkTekXpLO2LdotcsFuwtGTO7wH3twTigYxU8OOWOrCrk1e2gS9f+wD+8OIbnH2pLDXvj/eT1TYdX2XeMtU3yr+f3YJzN2u3sNdBynUMuJ7zuCfZHR9dg5iSzMfMzxEAzjzugKDtKnDP2kaGC9xzrxGvkFdNooRiIroO6lPWbkFj7GSZY+vZPlM4U8O43UJF/27EFHDGbuFOLqX2k5KcMArIM4WPvfYkPP/JB+A937wZ37vlkYUeUkLCboeBSLLWutBanwTgMACnKqVOnO+BKKXeppRaqZRauW7duiGPtoTEBgf5kffVaxH5jgLXYpbF6VjgXhZLAafx2s9ci2vvW18Xgqirpc1IS8f2fZvSSESDk3aOsnQzHZRaJg2VkuzaLRTcgKzMU+sKypPM7Bax7BYme4HgSV7E7CrcfkJkioNbWziozS1TVSAkleO++cGNuO5+N8VevyxttTfByqBQKc1+4RkA2LB9ph6zlyeZtU+/H6/gaDy3DWSuyZPMJx+UFjBiSQ6CM+k87v3rX8eTli5y9i3KMAVcu92Cf3C/C5RkVZelLq3tp/Ik2wmshlXJfaXdn0T6vniCX0xkjE1cxRRwxm7h3V9CxpCZyAQ0IWF3w3gnx6fPew5OPmJfvPPfb8SVdw37bE1I2LMxVAo4rfVGAFcAeJm36SEAhwOAUqoDYCmAJ/j3NQ6rv5Pa/ozW+hSt9SnLly8fZlhOnmSbjssjcIzA2O9gvstrq8bwnmS5mAjve9FYbshtu5LcrDTSd7GE8UUQuCcr6tSWa7dQbIm8TvcmKckZeXhLMYMIYMlIN88CBZR7kh0rCDQ8oc+o9j6ZocPIr0pKsh/EBgCPbp5y8iT725WiKnFURdBue6Iu9UoKPllNeA5royTTyoS251GVaebn064kF96926Yka095pvPIFKAyf1+NqX7hpIBrtVuw923ZLaiYSL/UbsBffT9Vx1JGmdrS43iS/cA9+Z+oIP90RHH2lWTfkzzezYNjUuBewihh8VgHnz//uTh2+d54+5d+HuTpT0hIiGOQ7BbLlVLL6veLAJwN4E5vt28DOL9+/zsAfqyrJ++3Abyuzn5xNIAnA1gxX4MnWKLHiomU2lNUyVoRfmeD1GaR3ULJ5YVdj2hpiIBIbtnxbSngqJ1Y6VxeZbD6DI/IcqXYLrXbID1tjsvqpXMCLafTEnXP9yRzu0V98SuF1h0rL8bC/aNah5MUsgWEJLnajxR+ypMspbx7dPO0k4HC307ea6Mkl9r0u35rRZJpYiQryaGVg65FlTRkWE+y+5vRMTF/Ls+mAVS/CwWx+kcUWjt2i8FSwIWrL/bV3cY96ybdHqr7iWe3oNzkgEv+J327RURJ5qtGgFx4B7C/Sc97JYhKcrJbJIwYli7u4ksXnIaDlozjTV+4Hnc8IucfT0hIcDGIknwIgCuUUjcDuB6VJ/k7SqkPKqXOqfe5EMD+Sql7ALwLwHsAQGt9G4CvArgdwPcBvGO+M1tU/VSv3Kc4mN1C49r7nsCGbT1TnCDuSY4oybnsSebtbJ8pzD4SSY7le5U4FO073aAkk4ZHuYOljB7kObaeZOV8psA+Tsx6ZYlM2SCpfuHlouZL/qQkZxlmvCVuHrjn2C2AYJJCJGaRV16Z+iJCQ0VU/IBNoFKSrfqtgz4Uqt+R2iy0xv57VfmFjZLsBe7BIWjumLjVwLfjtGUvAap7h6cvpImTX0DD7I/qtxljWTqIePqrHGVZ3YPjTgo4eRx2PG5f/Dt+KBFfOkVzb+g6mwqPFyitspw5SnLf+XtoSgHHN8VUdhuMKWe3mBCU5FRxL2EUsXyfcXz5Ladh8VgH5124Avc/vm2hh5SQsMtjkOwWN2utT9ZaP1NrfaLW+oP19+/TWn+7fj+ltX6N1vo4rfWpWuv72PF/pbU+Vmv9FK31pTviJEyQFCx3KXR7WeqZQuN1n7kWK1atR541BzE5JJkRnU7Ek1xojcVjboAfELNb2Peu3SJsmPaN2S140BeRD+7DfOclN+K6+54wSrKxqij3/Gk7Jx/0nQncK0svKLA69p7HtmB9TS47uQoycSwas+foq9yUv5lAJMYnM77doptntRfYzZsNAO/+taewDBZl8BtnWfU7ciV5/73HAADrt01XYx4kcI/ZLQpmt+Dn4wbuxZVhGssBe4/jmYctNdskNVnrKhUf94rTb+/vXmqNaScFXDydoGm/psLVJNL9zg8YrFK6uSn1NChwD+YzWTMAl+CWGlg8Hv7d+MiUa3NqUtkBm/GjN4AneTopyQkjisP2XYwvv+VUlFrj3M9dh0c2TS70kBISdmmMWFlqOIowEeJurhwFlcDJGwVYDWS3YITNf1gTSg0cuswGTRFpmZwZIruFMA5qp8luQedolGR2zpff+Riuu399tQ9Tzv2cyJQWz1/GzpStxFeUbp5k8vu+9KNX4q++d4cZg18KmCvJjrJXq/9cPSQSM9HxSbIl84D9jfn5ZkrhHS86Fq969mFO6Wk61lonVB3AxpXkiiT7nmR+jL0mdvXC7yMIMHNS/MkoS0sw/+RXj8c/vu7koC9pfyLg/aI05Ng3XPRLjZmitCngVDyji2mfVhsyxchx9Z1r7anIeaAk1/tZu4V27RbePUbpEqtzkcfmB/zF7Ba+F9m/FzlJPuGQJTjhkCXJk5ww0jjuwH1w0ZtOxabJHs67cIURNBISEkKMBkm2rmQAMISnMGQxs55kwTcLuEvBHFO9Ake957u4nXm43DzJoZJMS/+Lx22Ams1KIeVJZiS5xZMcs2284Ywj8bRDllTlgBnxLTUC+wGVDFawy+WUyo4+UxVCfxm7IkHkSXZVWa0RlN3u5FlIklngHlf2yG7BSRMpyH6WC5/Xdeprrtn5am2X9KnN+9Ztw6W3VGn5bM5jCtyjpXmNZYvHkCnrSfbzJLtKcvXKS16b3N2ZS1MHSQHXL0vcsXazM25znpKSDI1eUZpViL5jt3D3pUkaz27hB0sG7ZtJqF154Pm2CaWmdHr+b6XN/QQAazdNYf22GXNuPsGldIlAfDLoW08i1uWgwmLf+1sg28nLnn4wvvfO5+PoA/ZK2S0SRh7POGwpLjz/FKxZvx3nf36FyRKUkJDgYjRIsre8ToowPfw7Gcv/K6TKAmolOUNQJmW7oPxyJZnnSR7LM6z68Ctw5P6L68DBqrG/fOXTjbrWarfI4iTq5ScezOwWLnl4xqFLcdi+i5zCEnmd99knBqWuTKKu3cKdJJQ6zDxQjZVV3PM8yUWp8cimKWf/Tq6CJe5F0cA9XSvJ9hrElGQ/GHCMBe5ZNdyeF7X5xatX4U++9gszNsB6km1Z6krlXLqoiy3TVcVES5JD8pl5144HB4b5fO25xQTc61dtwJu+cL3Tn+lLIMkU6McrB2Yxktyrzme8M7zdgqvO5mdnhxpPsvKPr/6ne+mff3Iv/vOmhw2R989xyQAk2b+urYF7hUuWCePdDFe++0X4+OtOAlBNYlLgXsKegNOO2R//cu6zcccjm/GWi1ZGLXwJCXsyRoIkE+gxmWVuVS9KDwa42S34Uq7NkywQSg/ck8xJMnE7aqfUGi992oE474yjmALcVpaaL+Pbfa5894vwydc/25A+v3Ifpd7imQ5oclB6xKBf2wGUsufnTxIqFTbMzesE7pVlUNVwrU+SMxWQDu5J9rNbaO2eN5HTsU7mED7fFtPNlfnNeZo3aosIMZUJB6z1Qdfnb/MkV8ctWzxmx1GTSjp3rg/zUugApeGD6T/2+7YFzAGhkix7kis7wTi3WxCZ9+wW0/X9RysWNJFqgmbn4nNjR0muyXmeh6o/ZZDhsL7puN0ipupWNhb+uZkk+5MqQidTOGL/xfY+y7MUuJewx+DFTz0I//C7z8KKVevxjn+7IVj1S0jY0zESJNk+xD27BZHFPGMKKfMks38QiMz4xMUnmIC79N/JbGBWh+XQpQA6pVwiIGe3sO9jgV0T3cyxOvhkO6+38ewNZJ8IlOTabsHtFXlWESquJIt2C2VTwPULbRQ6VfubAyU5E+wWjifZt1top8QwkZc8U47K7qNb/8ZFieAaVOMIrxsRZyJwPPAuzxT2ZnaZJXV550UsvzDBLwroll2OB+4NUoLbzxMsZXGg1QLHbhEJ3KMJS4cR1PbsFlYV9+0W/FCrJLud6jq9RUD4jRXG7W8fdt1jqi5Nav22fMRsFuY47/omJTlhT8MrTzoUf/nKE3H5nY/hT7/2C/GZl5Cwp2IkSLItAVx9NnYLCtDKLBGQKpMBNnDNJy7SUjTPk8xJECcmulYzM2+bZLdwPMmRwC6f9PhlqSmnMffDUg7cwlPRKLCMB+qRtYI4a2mUZJd8KGXJZb/UJiBqvJOh0BprvWjpTqbw9hcc63zneJJLV0n2FccJlp84lsEAqCZC3I9t7Q52O+BOUuhal2VFRintGimiXPU96fBl+OTrn40zjt3fXC+Cr2K6gXsuSeZE31/6lzCQkoxqstHtKNOuCdzzxkZBafx+ahsHz5ZC1gse4Gn2K7UziXLb0AGRjQfuMZIc9SQPFrjnFxPx0fW8Id08S4F7CXsczj39SLz7156C/7zpYbz/27e1ri4lJOwpGAmSTH/OliTD2B2Aym9qyBMP3HOUZOUoZQSJQPhKsh8cRko2D1aih7gcuGffx4qJ2ICxuJKcKcqTbL/j6cTMeZe6LvzAylLXJNnk+tUAEFOSq7H0CltxbyzPUJRwlGQi2f/vC4/Fqg+/AicfsQx7jeUNSnKlALvZLaySHCssUfWvnGIk9LtZT3J4/bvMbpFl1m5BimjHs9W84pmHiAptkItYW1tP7mU/4WRuMCW5nSSXWjt5kntFaT3J3r6k/OfsvmwvJmL79smxk92CVh+8MdYW+IDI8r8XDm63iPF3v+Ie/VQ+6eXBmFLfnVxQkhNJTtgD8QcvPBZvf8Ex+NK1D+AffnjXQg8nIWGXQKd9l10fRPTIf0kPfiIq3SzDtrqGCS/SwAPKiGQGnmThecmzW/Bl38whHm4AFZES3ufv/8qxeNsLjsFXV64x38XKUpsgs/o735NM1giuYhr7hRC4p7UOsltUvtzqhEkFbw3cq89nrJNDa421my1J9gnIt/7gTADALQ9uMt/5acJ4RgrAksROljnXxgdZaqg5Uqh9lX+KESBqW+vKqmDLUldj4Ona/MlCk90CcFc3YgL4IGKNdP19FPWkhwfu0XH+7mR9sfmew4lhME6WJ9lP/caPrfoNx0w2mtkoyTFUnuTQblHZe+zfBt1fvuUnVwoFdFDOmuwWVb7u+KQsIWHUoJTCe17+VGye6uETV9yDpYu6eOsLjlnoYSUkLChGSkkm2cwEsAmBe9yvyYN5bJ7k4ewWPAWc9XnaLAuWpIXj3ndxF/vtNeaQKK4cOp5LIhQUuCcpybXSR3kvSfnzlWSTAk6xCYaypBqwhUN8UlbZLWzgXr8soVSl4BWlxqOcJEfYIQ/c81FqtxQxJ1Kx9siPzS02NBmi8ROx5yohnYeu/bLGalKSkmz78/mSlCeZg0hZplQQPEeYL7sFnZPJk+xkt3D3p3HNxm5RtUUrDeRJtscWWju+eQPt5kkm0EdJSf7r334GTjx0SXRMvpJMKrW/2sD/7jnyyP40AfYzsiQk7AlQSuFDv/UMvOKZh+CvvncHPvXf9y70kBISFhQjoiRXr/S4U+RJLklRzcz7fr0sPVkWQXYLIFzeFe0WHb4MnxkiwpePiaTHlpT9/QkxYtbx9vVtG0TWH9o4iXf++00AKgV9IaRjsAAAIABJREFUUhdynmTtBm0RKTRp0HRVTERSMomoF2VFPshyUmo3ZVeM1EqlgAEb8Mb7NOSnhSSTn5zuBVKSrSc8JOZEnDV0nSe5VpLrMZAy66uWQHtmBU5GY0rybOwW0jWgvkj55v5kGit1Rb9Px7tXG2FWakL1m99atAoQy8ARs1tISvLrTzsCD6zfhlsf2gwJqvbgm7bMZMj9nf1iIn7fgd2i/jzZKxzrU0LCnoI8U/jY756ETCl8+NI7sWHbDN7z8qemlZWEPRIj8hRwVcNMVQSZVOBubpeU+4V2FDcCKcmcBKzbMh0UxwBc33DOltOJh+WqKtBAHk1AVgBtzl1ZlZTK7lq7hfvQ9xW85x61L048dGlUSS7JbuF5kvuMJPtEhM6RzrPUVeBeJ8uQZfTZ9hWzRywek+dm1bK8SwTNMnquAkLD96Hfzg/U8rNbcJjAvZrA9U3gnmsrkX67psA9t/94+elZ2S1Ekkz3eagk++Oz5N163NvtFjblWuhJZkpyqc195B6vnbLU/rjCPMmVJ3mswV6Tsb+76nzktujfAP9vgHbz7RbHH7wPAODnD6yP9p2QMOoY62T4+GtPwnmnH4lPX3kf3vONW4KJZkLCnoCRIMl2Obh6tVkdartFxu0WrDIZD9yryR9/6D/3r36EV//L1QCA5xy5L/7i158GwCV/eZ6ZBWdewY2UZBNMKJFkplSacXCSzPYlokWVxvzgIiKKhD940XEY67h5kwmF5oF7tn1edY6UZp/Y8H4oKLCTWz+0Y2GJ2S2iSnLoSabrlmdZNHCPVPSCWWyCwD3hWCJUVAqbpwnMFJiSHB7r/mbhmGxqPBVYNQgDKcneuKVUZ5SyjCZvRVk6+/EjZnwlOWsfh5kwqbAstfUoazPZCFLAaTsRkc7Nn0SQJ9lPf8fhl4Ontv2JGa0O8MnbWJ5F9z/9mP2w93gHP7zt0WjfCQl7AvJM4YOvfDr++CVPxldWrsE7Lr4hFRxJ2OMwEiTZ2i34ErL9vipLXb3vFxpj9cO5x8gj+W/Xb5/BbQ/bwDLC215wDJ5//AF43rH745ClE+b7SjW2gXK2f2tpoP18+BaK6j3Ye5ldZUo5qdOAmuQHAW/xPMkmBRydRxYqyZkKia5y7BaVctyp/dB8YgLEleTxyDK2RtVeVwiY62TxPMm8EIxZMSCSzM7Ph+mn9sv62S1ou6gkIyRoHP2ybLTaUD9t8G0iUl9EfB0lOeJtDz3Jqr3inq6VZPCAPc22syqPSiH3iL1G9dsGgXtsssWxd02SuWXER6UkyysOHDQP5pO3sU5mVWzvHh3v5PiVpyzHj+98LNp3QsKeAqUU3nX28Xj/b56AH9z2KN70hetTCeuEPQojQpJt8Bm9lk4xEasS9kpmtyhc1VMphRtXb8TvfebaoI9cKTz14CW4+K2nO57aDiNXtjgCz25h2/chKcl8v5gCKS2R+8vcnSxDpqpr43uSOYG0OZXrXMEsBZwUuJcpS9RIOe7kmSGpfu5pcfyZwiVvPR2vOvlQ53uyS3SdwD3bVqw98kRrzZbXC9eTLBHsjvEk26BFzSY3XZYBIjiHlslMr2C5isVRD2a3CFPAhfsYT3JNKvusb38AYZ7k6p7/5BX3RMdQ6mpSQF5/f+yVUmztJdKEsBQC98AmW4RF3dyQ/abiMbSiQqD3gd2iLOtJof2um9spjp8yDgCO3G+xCX5NSEgA3nTm0fj4a0/C9avW4/WfvQ5PbJ1e6CElJOwUjAZJrl+5akjliYGKjNJDvShLRpJ9T3L1fuu0LV1stjcEjRWeksztFk1qonmgR3zIsYAviTzz8thARQCrcYQleWlsPHCPAq5cT7IcuEdfaV0HidVL7GXpXtOmvMZnHLs/li8Zd77TqEguV/d44J5EaGifzCNcdB7GriEcaz3J2lhN+G/ZFSYxBCUQNI5+UQbVFmeDMLtF+CdLJNkqyaXTp2O3KNx7lZr/yA9+6bSptcYPb1uLux/dgk/9973G0mGKifB9wYq3ZGGAJU08fFsFTWTo2Fc9+1D87984wWyP/d7VuN37XbJPVBlXwpWUrmCR4hjv5OiX4eQyIWFPxm+dfCg++4ZTcPdjW/CaT12DhzZOth+UkLCbYzRIsseSrd2iDmhiftMeW8537BYZ99qGWS1iFb0kkpzVFoySEQOJZEuFKVwlOd6njyxzlTWrrtp80QTKblHZLWyAG8/wYDzJAkmm74paOc5rQl59tp01KYE0Rg4KMpTyE+cNgXs8DZ8fuOcXE+Ew2S20/R1pYlWlnIvbLdCSAq5fMnvB7DlyqCR7bWVKDtxzMoQIdgvJ6rONTQ7/46aH8LYv/RwXXLQSQFVxUCkYduxU2tNubm7/b0XD5t3mMJkn6nvm5MOX4fWnHWHPvSVwj192utY8x/JYnuG7tzyMB57Y5hzbzTNnlckHFQtKRUUSEly86KkH4ksXnIZ1W6fx6n++Gnc/umWhh5SQsEMxGiTZy25BhM1UPWMkuV+Ust3CW76Vig9IyDOFo5fvBQB4x4uOZf25xEB63kvL+XwMMW4lppMT7RZk+wjLUmu4tg0iN3Ta5EmWUp9x1bZXlOhmmanWxycXTWWkacwuqhLart2iVgjrPiRQSW7A/m5+CjhJMeRKMqnodKkypYwnVpyUcCVZ2F5VvQv3HRZteZLzTAV5krkXHnDJZOBJZu09zpZQH9pQqURr6wqKX37LaY6H3bdbmImikpRkmPuNo1e6SrJ/Hflkyb9VYkryElatb9tMgUc3T+PtX/65227HkmRJrSbPvF+wJyEhAXjuUfvhq28/A4XWeM2nr8FNazYu9JASEnYYRoMkm8C9CjkpqNoqbMTdCqZU9pw8yW4wlk+SY6Joniksmehi1YdfgZedeEi1r3ItDdV3kpLcnN0itkwftVtwkpyTKh6WpTZV+ZS9dr6SXKWIC7NbcCWZ/M6dXNXFOLRzTZuWy6sxuxeV7BuO3YIp8bH2yFoC2KV1Q7walGQizpWSnGH9thn86zWr6v6sEi79Dm22mMoXPLjdIpYD2if3oUdc2ewWgk0FiGW3CO89TpJ9W4ZNs2ftOARe6VApedLAM70QfNU/yAmdx/8W+GSNj3PJoi7a0M1tZcmY3QII0ywmJCRUeNohS/D13z8DSya6eP1nr8VVdz++0ENKSNghGC2SzEgJBWEBLECrJnE2TzJLAafcVF3+A7JJSfZBAU68LHWTJzmeJ1ns0uwz5pFJPpRuXittGkIKOAC12mhU+MzN8FCl7JLy9HIluZpoUEEVnhsYaF4ur7aHiqMfuEf7dJj9gUDnnys34wbA7BZm3GEqti67LzqZwvptM/ibS++sz0GhqQx2W+BevywNWRzEkhzL+OF7qYOUcJligXv8fpDHJwXuEdZtmQ7267BzUKjui3vXbcWWKde3zzO8cLJLzfvqNmB/I5rM+tu7uXw+tK9jU1KkJIc5uI8/cJ+gXfrTb1SSe4kkJyTEcOT+e+Hrv38GjthvMd70xRX43i2PLPSQEhLmHaNBkiFkt+BLwBmpqm6eZN8awB/S26fdpdZo4J7AgDJVkQYnBZykZLbYLWIKpLEgeLYEPpa8zm4hK8klKyZiz8PPk0y5cd1zY/7f2oPczSvFmYgVFQtptVsE3lXtTCz4NcjzUEnmxT5oP5r49I3dwh7je6T97Bb+2Lga78Pxw7JjaUzbpovWFHAc45Hc0b666reVZwo9kwKO/f58P/aWVGdz77H21221GR0MmWbXWKkqpeDvfuoap8gOD5L1/44U28cH/UZEsP0JQLdBSfatQMZuISjJfuBqN8+skixMhMaS3SIhYSAcuGQCX3nbGXjWYcvwjotvwMXXrV7oISUkzCtGgyT7dotMgac+I3JEhNHaLVwlmXOo7T1XKYsG7glKFHmgyddL7Uv7VdvC75wT8mB9zpwUu2Ps1MopzzxAIE+yUix1V6aQZ5lR98hPLeVJpvRbdI2JGE3X15NS5LXZLSTvalGruv716GRhECGRGSpLDQCFCQZzJ068LYLjSRYmA2Nmezh2bs3hhy5dNAagKg++j6BqxhBVkgfwJJPFZRi7RZOSfNejWwwJpt8iV8qkDFy/3U2PptlqRaYU3NLq9UTDm/wAPHCPCLZ7DbiSvGjMnUT4KwPUJfckX/rO5wMIV4XGmP1KWi2wnuSkJCcktGHp4i6+dMFpeOHxy/Hn37oFn7ziHiePekLC7ozRIMn1Kz00yW5RenYLKpsspYCj4wjbfCU5ogZKXtLKbmErkAEuuaG3UkW3tnLHfB+eO9gPZOrktsBGWEzELn/TFutJtp5THhBnr4PdnwfuZSyAbHFNaKR0ZU5bgZJc/Sb8OFtMJAtUP17swyrJrt3CvybS8VqHk52MKclSKjD+0/A+li6qiPHDGycNYRtESR6LkOQwu0X4mdThbsxuwT6YwD1jA7L7rdsyjV5R4lc/diW+ccOD1X7cMlKvFvjPP+5JpvvIH0epQ9tJz0sB558b/72X7+2mC/RXOawn2U5MnnbIEhy8ZCKoEsaLlIh2i27yJCckDINFYzk+84ZT8FsnPQkf+cEv8aHv3hHY/BISdkcMLnXtwrDFROyDv9QI1KIqMMzaLXgKOCq9S5icmYvdQtX922Alzu/2Gutgy3TfJSBCHzFqxYuW5EqhgA6KbXSqSEQxnR2Vb66UQdtmnnueZBWWGDYe6zpQr/IQV3moZ+rlaSLJ3WHtFvVYHZJFSnKugvaIWPIUcJYk13YLdt19wtk1kyeZjCpSkkWSLFsbli0eA7AN/VIbwtZyGQDEleRgkhJ4xDnhy8T9+BE9ISCP8PjW6YAYUlEaVSvJk0JZWg0wu4U06XNXVQjmt/JsUfZ87OcDl4zj7se22vNWXiaYus+lnt0izxSmeqHdgp+fj5TdIiFheHTzDB/93ZOwbPEYLrzqfmzc3sPfvvoZrbEpCQm7MkaEJFev9Mz0SyRbT3JttzCVyezDk1fHA4BtM57dYojAvby2MXBP8iFLF+HZRyzDYfsuxsbJHq68a10QuCcFJ0ngFo48U0AhKMmZDdybnCkw1smM0tsvw+IOWVYdQ+p7rJgI5ZClcty9QmOiWx1HiiYtjTcVE5GunYbNu2z3sfv6/9iSvSDLwhRwppiI59PmsAQptALkGZDXd5RUttklaPb9MkbSiLDF8l075xIhyb6P2r9mnNyPO0qyvDrR8zzJyiPJfm5g7WVokYLZdOnZLbJQ0RaLiXiZSMJJjG1nv71cJdm/3+m67DPhkuROrgKyy20pYp7kZLdISJgVskzh/b95AvbbawwfvewubJrs4ROvP9mpUpuQsDthNEiyCdyzD3Ne4KDDFMN+qY06xu0WfsDYwEqy8D31XzD1bK/xDr75B2cCAP74khurcXlpuKTgpMv/5FcCWwgvkGEUwcwrJpIrE7i3dbqPJRNdk+KrrDN/+JaBTCkzcSCC749pok6PRWo9BUL2ijKwW0gqHUdgVdFVUKG7XG/PNUaizIQAlnBJFgl/aZ3a04KSzFPdiYF7jifZvl+62JI0a7cIDg9AaccIR+2/GP/65tNELy616Zd6drNBhJYHQPIk222TM0WQ+rBXaidLh6wkuxNS/rMrNtHw5wp+xT3f8uIqvuHfhutJrj4smYgryWN5hpmidGwpcnaL2m6RslskJAwNpRT++CVPxr6Lu3jft2/DGz6/Ap87/5TgbzMhYXfASKyD+EpyJ1Po9bVRt3jgXq8ojYe1x6Le+6V2lqV9JTm2YhRNAVfKAWEAS2vmZbcIlGMFHLt8bzzlYDeFFZEQXmVPslsoKEOSlzKvpi0m4p5H6EkOz5uU5Kzet19oQ9aJgC3qDqoke3mSUf0OkkKYZ1nQHqmvnMzThILIntQWgchSqXVA0HhZ6lLgSlLQGFCdOymVvpJ89gkHhQ3Vx/h2i4lujiP2Xxzsyz3a/jnF7BbccGGyW2Rhhhc+0SH0i9L0qaBEkkzWIiCuJFPebf84APjfv3ECnv/kA3D60fs7251Ucl6f3JOslJ08THT91QKrJNM1HmuzW3ST3SIhYa4474yj8I+vOxk3PLABr/v0tU6KyYSE3QWjRZLrJ+nSRV1smuyBBFhT/KKsSBTlEOaqWVk2e5Jj1gfpIZtnvPRzeNxY7hIcTnQH6dPxJDtKMiPJtZKsAWyZ6jupsSiokZOWLPAka0dNJZDKRhk8+nUxkUwxkjxgCrgwu0U1sXGyW9Tn1M2VUbEJXTbZsJ5kr4qbZ0GR+i+1EBCXKeZZbgnc85Rv8iLza37te1+CT7z+5KAdAHjX2cfjTWce7Y4tVjiF/d6AO+54nmT73leSuaWgX+rAYtAr7GqIUgiC4ID6d2NZUvifBF3/amVCPCUcd+De+NIFoWruWFC8Y/nESMFO5vy/tzzLjJJM5FfKw+30WzeWylInJMwN5zzrSfjc+afg/se34TWfuhpr1m9f6CElJAyF0SDJ9Ss9NPddPIYN22eskszITlUhrgpqK7jdwgssCrJbRJ7wkqMgUypaIAGwD2ZSuBVTxDhiFNPkDmYk1i+r3anXo7UGtkz1nICmQhOB523CU5JdtY6uoeNJ1hr9ojQlsEmlXEwp4IbMbkF2mDxXOOnwZXj/b57gZAc5/3lH4fwzjjT7O0oyI7yADcp0LSiep9mUcdai3cLk026xW/jWBvLFcpJ88NKJwFJB+O1nH4qXPPVAvPzEg3HyEcvq820O5KPxuoF78pj472w8yQJJ7vVDJblXlE4fnCTz/OOmBLynJPObeABrtgO3kElot+A2pacdsgQnHroEBy+dCNqgMRP55Yq7mALOKMmJJCckzBUvfMqB+PJbTsOG7T38zqeuxi/XblnoISUkDIyRIMm+0rdscRcbJ3uG3BA5KrVGryzRrW0KPLuF70kO8iQPoSQr5u2VuLVZKo/YLairWG5mvj1nZMnJaKCUsT1M90vHD1aaPMmuFSHPsjqoz3qW/Xy6pOYqpVCU5PGur2c96Rg0cC9QkmGzW/zHO87Em848mtkLFA7fbzFe+JQDzf7ck+wrwXT9+TmGSjJLASeUf6b7Rkr56Sq17nWnqm9S9TcJnawi+f9y7nNw6lH7AXAtHBx0nkcesBhLJjrO5IdbNtzsFsxuESjJFYGc6GbolTb4ktCv82DTefIVFhqL60l21W1+GoOkwuOg33fZoq4Y1Monl089eAm+80fPx97j7jXPM2XI7rjJ3x33OgOpLHVCwnzjOUfu+3/bO/MwKarr/b+nl9mHYRvGYXPYF9kFBGQREEVRcddEoyQajcbdmK9mVZNfNO5J3GLcTeJuohITRYMYkaiogLKoGIiKC6gRFQRmub8/qm7Vrapb1dVr9fScz/PM0z3VtZyuru5669R7z8EDp06GEMDRv1+GV/77v6hDYphQlIRIdtstOleVYVdLG7bvNISuPBHue80SSxCpg9QA016g7A233cJP8Om8yvGYUqdXcxJOWplkvd1CvY2s36aSSVasF+5tdasus557M8nCYxlQ7QdtQmbrnJYDK5McM8R2S6tht1BFmSWSU9gt3PEK077hqJOseJIBZzbStq14/cqt2kyySyQrHfe87Z6Dm6E461kr74nIyiDrur/pcF/cAP6DHuU+mzG4Hqsu3h/ViigMM3DPXQJOiubaiqTWk6xePLoH7skQheJJJtO2444X8D+e/ZCCvUt1mfcui5JJDqoeoqv+UZZIkUnmEnAMk3OG7FaLh0+bgi5VSRx/64tY8taWqENimJSUhEiWhguZMetiVhj4ZJvRGUyeKP+33ewiFpeVHOwUYUur05O83V3dws8frBEzMSJtCTJJwuNJdorlVCd/tVZxTBHM7m11UUSy2mShpVV4PKKq/7ilrc0q/WVPM96PzCRbdgvTvqKKoSpr4F561S10pcA89gKNB9ddm1eN15ktdzcjMUWy8JaAU+0WOvw9yXaFBXfNXj90A9387D3u/aDO5hDJGuEN2MJTHs/y/9qKBFpahdaHq4pktWiIlUlWRLJ64aYu644jDE3dqnH8pL74wwnjNXYLSnkxCTj3oxS/6sVPoEjm6hYMk1P6dK3Cg9+bgqbu1Tj5rpfx2MoPog6JYQIpCZHszSQb4uRTs+SZt1Ob0dJWrW7R5hJK2911klOIFse8RNYAMt1itk1AjjaylzPehy1KdOhKwMViXiHftdpbsxeQHdKcA/cML6mZNW6zPcnWoEdzH8tMMpm+6xazZJu67zKtk+zOcsq41GnqdtT96F6Xzu7iV0JO20wkFiyS/e0W9gVJ2JJHzm6M5Jmmm5c0AlHNkAbt+hjZx7OaSd7V2oZdrd7sqTUozi1UrTsPzhJwuu6SRszO9V54wFD/IM11/fLQkRhQX+NZVhXJ7gucHx84DMft1ReA8xhMajzJumOUiFCWiLHdgmHyQH1tOe4/dRLG9u2Cs+97Dfcs2xh1SAzjS4nUSTZQ7RYA8Jkrkyyx7RZuT7I9T9hMsk48E+mbWUikzcKvBFzc5+RvbVOKasWLGyev3UJtwKAKNl0JuFjMFmDNisD3q5Mcj5Ftt4jFHKKsyqxukcpu4bYUtLY5/bIAUFVubK+63K7PLLHaese8WXTd/vf1JEN4RKlqP9GjF4Ixogwyyd51+Q4UdYljR2bdz27h0vrqfpciuVNFAi0au4Vjm66QrAsowFHdQmcfccf0t7OmYo+eddr3qMMrkgGK6V/77vT+SozefeIYuOdjaylXmu8wDJNbOlUkcfd3JuKMP7+Knz66Gp9ta8ZZswemfbeJYfJNSWWS1eoWAPDptl2OGqqSpFkyrEUduCecWbntruoW6WSS40Se7LZKv/pq9KyrsG7rej3J5rLaLaqeZDsudzMRAOha5eNJbhOekneqtaK5pU2Z5lyn7UkmtFrNRJzd8CrLXJnyFO9D0qKxW4zt0xl3LJiAcX27AHCKLrUttTeTLO0W9jRvnWWZCfXGEjeziX6os7tFYe+uVah1DaoLQmeN8BPJ8v3ohKuv3cJ1JKnr3tli2y3aBDwtnB3bck2XF2htbcKyYRgXVkq80D9PdWx4cW5dPQ6CTqsJTTCpOu4BhkhmTzLD5I+KZBw3Hb8nDh/XC9c+/RYueXyNVZGKYYqFksgkW4OGzP9tu8UurVdXlixzrCNFdYtUmT3HNM3tc5WDRvXEQaN6euZxCyC/TLIqpBIxwzqiWiMktRUJQ8y2CWedZCHg/ilS7RbS9hAjbwbd2XHPyCTHY+Ro4lCZDJlJVgRKr86VVnkyt+icOdSuaKGuslwpAecW83Y2XM0kuxtNyJFnGpGcIpPsEGmuQXLHTuiDA0bsFiiy/dB5jZ2vOx/VbTtLwKmxOtehvi/LblFuHB9f7XQe94CdiXZnedTj3LJbkFF1IhknNLd6B4dKUtXQ9sSgySS7L4516Kwf6ufiL5LjbLdgmDyTjMdw1ZGj0aWqDLc9vwGfb9+FK48aHWh1Y5hCUhIi2Wu3ME74W79uRlki5jnBSmGp0urKrLozyX52C52QcgiDEFpAzm+d0Mk53Y0tlIwLAD97RixG6FKVxCdf7fLYLSDct+TJujXdrJRPc2fQrY57ZNotzIF7ag1gqy11ih86dfs15Qkr+xskTnV1gd2eaMCugR1ot7CqW6Q/cC/m8xnL5brXlHsXCoFO/KpYrddddx3c+0Dnc7ZeU4ShOnAP8HrxAfvYd4ckRWqbsLtbyrhqK5L4bNsuXwGb6gLKjc6TLNfRv77adzln1z7jeVi7BYtkhsk/sRjhJ/OGoWt1Ga588k1s/boZNx63p6e5EMNEQUlcrg3qUYPT9xlgeZHLE3FLqOl8tYk4eUSIp5mIZ+CeftvudsaAU1CHyZi5M8d+HlD3/HKQlC2YvAt0NStcdHK1pW5zlYADbEGxS2npXF2WcGTeKpK2J1lmIZMxctToratMoktVEr26VAa+b7n8waN7mgMp5eCvcOJU9ST7WTfU9+hpS60M3Gtx9Z6Okb7ygcQ96NGanp728yA/S7/VuNcv43B3XHQMJnStQ413p1nSTTZA+Wqn12LgF5P0kAthN1yR+1jWK/aLKd1MsnfQIFBdnsAdCybgjm9P9F3OcbFgfpypOu4BRrZ5p6a7IMMwuYeI8P2ZA/H/DhuBZ9/agm/d9iK2ft0cdVgMUxoieVhjJ/xw7lBH9q6zaS8wbv8650/EndnleSMbccURo1zVLUKWgNNM9xu45IfXbuGc7rdNKZDt/73zSn92bYqBe2rczUpliGMn9sGDp0625lEtDlY5sTh5Mskv/mhfHDyqMfB9T+zXFT+ZNwyXHz4SgF2RIqzNIbCZSJvMhsMzv/v/NuEtfaa2pdbHoY8p24EnbsuN53WNWASM90LK2/PruAc4j095F6ZGZpJ1dgvr+HTt41a7hbnqSQbgaerhjiNV5ZOgZdVYZg7tYV0I6nAOijSep6qTDBiNR9yNVRiGyS/H7bU7rv/GOKx8/3Mc8/tl2PzFjqhDYjo4JSGSdciMp67yQdI17dpjxqCpe7XjROwRyT7CTTc96Fa3DmvgXshMsmrPiJO+PJqka3UZasoTjpisEnCaDDugNEIhQm1FEqP7dLbmsferPegrGYs5PMnxmDHoLZVgjMcIJ0/rj+ryhNnBz78Bi0RfJ9lrC2nVVLdwf1ZSQAnhHMQpYwiyi/hZatJMkHqwa2D7ve6KwxTNibhTPgddqKnC8YFTJ+OMmQNRY1YPcd9BARS7hWu63GcCsO0W5rxSdPt0qM4gk+wkbPc+3V2JVG2pAdNuwXWSGabgzBvViNsXTMC7n23HkTcvw7ufbo86JKYDU7Ii2boVrxm45741LU/YanvrVkU0yYFxOnQne11JryDIJY6D7BPq9Lg5WM+deVYXG9bYCQN71DjibG0zmokQgEdOn4Lz5wx2vJddrd4srERmkuNkt/uNu+wWmQy6INgZ7EBPsvJSuSKS/WouB3msZulPAAAgAElEQVSS1ThbXFnDOJGjCoI3Xr34Trf1shvblh782be57CRuT7Ij0+1ah2oRGtGrDj/Yf4i1L7YF2S1cK2qx7BbOOskAUJvCbpFudQvPoMGQu9k5mNG+AyPxE+tc3YJhomPaoHr86eS98MWOZhxx8wtY/cHWqENiOiilK5IT9gnRfXJPxmPaTKDf3VU/q4Xfa6mqW7ixbrG7BmP5LWk3ECGHQNRllM+aPQh//f7eDlHRJozsKRFhXN8uOHP2IMd7kSXgdBcGaobe8iTHCeVJ226RbpbQ2JadmQxaXt9MRFMnWddMxKctNWCLakkqu0XMx9qQzluv0gxMCXsXQV7DqW2sHRd+GnFqD/Lzfu1l1lwO3NtnSL2yvHNbErnPVE+y3JYcCOgn3DM5RlTCZ5K9n03Q91nCA/cYJlrG9u2CB0+djDgRDr1hKa74xzrtwGKGySclK5KlEIgReTJCsi218bp98lczySqBt/9TZZLTGrjn/N+/woHxKDPJqmgG9OLa22yjzderajfi8K7H4Uk296tR3UIpq5WpSA5R3ULnMQ4euKdmDV2eZOX/5lb3wD3vOh1xKHvZz5+ciud+OBPPnD/DuV7XXQXvdg1kET9L+MbJV4Ra2WZXO3SVMvOCYNvOVtSWJ3DntyeiUhmkqW7LjXHR5RbJSfM19Y6Mf1Y/Fe79EXY3J5TYZSRhkthcAo5homdQQy0WnjUV88f0wo3PvoM51zyHp1Z/ZP3eMEy+KVmRXGYJAu8tZNWioAoGty/Vmj/NW+jOLl+p53fHkk51C2d7XufrumUkbcIbm8ysqtUt3FjVLdSBezGyphvrycRuQdo6yW6cmWSZRfXWNG7VVLdwZ4bl3QbA+9kbdyDCiXWdIA1D95pyDKivcUyzs7Z+25UXdOb/5vREzFmyThe73Ec6gSovKrftanF4vY2YjMdenau0MQkI6y6MVd3CzCRvUwYCqltN25Psmj1dT3IiFtP61P0Y2KMGQ3arTStGhmFyT/eaclx11Gg8+L3JqClP4JR7XsFJdy1nrzJTEEpWJEsBFCPy3KIx7BbejJ1fJjndE7qqxcIN3HNmji37ha8n2Xx02y0CxLXuPXg7sZl1klu8VgVJmZW9hTWwKRl3epKztVsEVT7QDdzT+c51nmTvwD075kNG93S8lkrnx3w+4+w9ycGZZDndnblNxGOIx8iycMQ1dhApjnWfTzIhPcktyn6F43FYo140trWpJeCMabK6xZc77O+ew/pSoIF78lhKxMnjmw7i3DmDccM3x6UVY3uAiCqI6CUiWklEq4noEs08C4hoCxGtMP9OjiJWhlGZ0NQVC8+aip/MG4YX//Mp5ly7BL995m0eO8DkldIVycrAPXftV7WZiKPqg08mOV3NF3dk9FLPTy4xIv/3W1RtSy0tF0DwgD/de/Bkks0Jza1eq4K1jCK0dlkD7ZzNRDKyW0AtPRckku3n5QFtqVvbNJ5k1zzqwLwRveqw8fJ5djwpPzh99jhLq22ITLLxKDyeZDODa4pTXe3mZIDdImlO27az1RbJLlE9rLGTNiYB4alu0cnMJKuWBQq4YEmFN5Mcbjm5nUQsPZFcwuwEMEsIMRrAGABziWiSZr77hRBjzL9bCxsiw+hJxmM4eVp/PHP+Pth3eAOuWfQW5l73Lzz31paoQ2NKlJIXyaq1QqIOclLFRKuf3SLtTHJ6mUV3LFa20Gd+uy11zOHHDfKOakWfa5rtSQ4jVu2Be4kYWZ34gPRr4MpY5P4Pqnygq5Ns7APnfC2aW+tuG0hQnOpnuEdPrzjMVybZupvgc4kkp0pPsu03dopk0ohkaYEIzCTvarGtSq47HL4iWdh3Ydwl4FTUzaZbT9o9f9jlLYtJPGbFmK59qpQQBl+Z/ybNPzZ4Mu2K3eoqcMM3x+Gek4xGQifc/hK+/6dX8eHWryOOjCk1Slgk24Lx1OkDrDJngBy4ZzxXB9blym6ha2AQhPsWuzyJ+w4ktLLgzkYaQZ5kHe65rI57AXYLNWaZJUzEY6hwZJIzKwEnB+4Fe5Lt50HVLZo1Zez82lLrkDG88pN98dD3pnjj9RHG2TYTcfvLPa/H5LFh/C8/q927Gq2Zq8qdg+3UdcqKE3pPsswkt3gzyebyfk07hIAnS1tTnvTOmMW+cS/JmeTMIaI4Ea0AsBnAIiHEi5rZjiCiVUT0EBH1KXCIDBOKaYPq8Y9zpuH8OYPx9NqPMfvqJbjluXc8A7EZJlNKWCTbJ/rKsrhV5sx4zRZVziYb+nWlmx101mFNPb8t2I1H+5Z6sGiPxwjDGjthj151jjjDhqurHw3o/by6GOzMrzOTnKknublNiu7gDLZEvVvgtVuk9iQng9pfm/N2qylHpaZUm7omR4bUd43hSOVHl5PlBdRJU/vhogOG4oojRwEAqpKy9Jo3plpTuGozyUqLbtuqBM/8T54zHXcsmOBYVjanAez9puu4l4029dZJTjOTHCM0dKoAAFSVeWPrSAghWoUQYwD0BjCRiEa4ZnkcQJMQYhSARQDu0q2HiE4houVEtHzLFr7dzURDecI4vz993gxM7t8Nv3piHeb99l94acNnUYfGlAAle7ZIum4ZqyRi+oF7ObNbOKodpF7WLYyshhE+ol2N/SxF/Lu9yWG3K5EZYF0W1o3D6+sauJcJBNtnG7a6haPjnivYFm11C2eMQfsp1S15h8XCkbUNXCwlttdY/7plwzDnG92ns6MjohT0jvfmySQHd6ErS7jtFvZ8Q3ar9Xw+AnaNcbnM+KYuOGrP3qitSOL2pRucsWdA1tUt4jFcfsQozBnegOEa+0xHRAjxOREtBjAXwBvK9E+V2W4FcIXP8rcAuAUAxo8fz5YNJlL6dK3CbQsmYNGaj3HxY6tx9O+X4fBxvfCjA4ehe0151OEx7ZTSzyRrTqZxxcOq6iZ/e0PmIjlcMxFnVtsWycHxuMVKqsoI3vW4/jf3hW4A3e7dnOW/3N3T1IF7mRC2hq761uoqE6gqi6NHbYW3TnKI6hZBpHKMOAfr6QVzJtheY/165Or9FEl1uV2iz71Mp8qgTLI9rdxn4J5EbUEOAKfesxw7mlsd20rGY7jyqNHoV1/tiSMT3IuG/UranmRCXWUSh4/rnXkQJQAR1RNRZ/N5JYA5ANa55mlU/j0EwNrCRcgw2TFneAOePm8Gvj9zAB5f+QFmXfUs7lm20TcJxjBBlGwmWTZH0ImWZNwWdXkZuJdmZtGvBFybj63Kz3tsrSdknG4hJjOMVp1kRQs9fd4Mh2hX32MiTh7hlC5O+0JAJlnZbk15Ei9cOAudKpLY8tVOx3y6hii7dapAbXkCX+50lgQcqqmHmyqTrL7srCSRnUiWh6CvJ1leQPkcq5VJTac78zHIk+zIJLsuMN37olfnSvzowKGoTMbx00dX4+MvdmLT58aAGff3Tf0vm12TeSbZtluovPrTOdYA1Q5GI4C7iCgOI0nygBBiIRFdCmC5EOIxAGcR0SEAWgB8BmBBZNEyTAZUlsVxwf5DcdjY3vj5Y2/gp4+uxgPL38cvDx3huPPGMKlIKZLNQRt3A2iAkcC6RQjxG9c8FwA4TlnnMAD1QojPiGgjgC8BtAJoEUKMz134/tjdxXSv2aJOPam7RfLQ3Wqx7qMvsxTJqZe1vJ/m4wEjGrHuoy/RvVY/UEqKFndc8v+wQs09m+VJbjGzmYrE8VgVXJnf7DPJ9vNgT7LzeeeqMk88gN2WWt0Xh4zuiVnDemDUxU9Z0/5xzjQ01lV6tpPqM9dVj3DHlwnSh+5rt5AXUD5JEVknWZfprtVUvpBo7RY+xxMR4ZTpA7Dyvc+taTKT7BbU6r/ZXEC4rRphx4ZadZJdC/gNQix1hBCrAIzVTP+Z8vwiABcVMi6GyQcDe9TgjyfthcdXfYhfLlyDQ29cim9O7IsL9h9inTsYJogwp5oWAOcLIYYDmATg+0Q0XJ1BCHGlrKkJ48d1iRBCdc3PNF8viEAGUnmSyarG4By451Qeo3rLAXHpbTvzttTG45mzBmLlz/ZDj9oK7fzu2rj2epyPqXALD7k+mWEL9CQrG0nGY5awyhQ1liC7hV8lCb+21G4rRKcKZ9WFobt1Ql2ltxJDKkHnl/nOxncL2J7kVBdXwsdwIatb7GpV6xMbx0S1KZJ3aUZ+qxcm7mYifoNPR/Wuw2WHjwQA7GiWdx8CrDK+r6Qm20yyu9siwzAdAyLCIaN74pnzZ+DbU/rhvpffw6yrl+DB5e/53pFjGElKZSOE+FAI8ar5/EsY/rReAYt8A8C9uQkvc6TdQtueNx5ztFeWuDPJI3sbt2XSzSSnO5BLhmANyIsR6qo0JbRc63QLkqBmIkHblaRV3UK1G2iqS6SLw74QWCfZuV07HrdITl3GLohU78fP65xtJlleqPmtxu64p39dVrfYrjTQIRgDG2Xt4h27vB2qAu0WPm+KiNDQyRgQI7teuWdVLxqyqSHtrW4RbrmgLoMMw3QcaiuS+NnBw/H4GVPRr3s1LnhoFY7+/TKs/fCLqENjipi00n9E1ATjVp2uriaIqArGSOmHlckCwFNE9AoRnRKw7pyWE5InfZ2WUEuWOeskO+drMger6U7u35jYF3sP7KbddqZ1ksOUi3PM7y7hZmWk9cut/38H4Mbj7Fa77vlkNtGukxyQ0XVlkrPFYbcImUl2WC9cIbS2ei0j6ZCOJ9lpAclOjMljMOXAPR+VLAfubVeEsJFJJqss2w5NG1fHwL1ksN3CuZwxr6yZ7RajTruF72pS4l40rHVDrW7BMAwzvGcnPHjqZFxxxCj855NtOOh3z+MXC9fgK9dYFYYB0hi4R0Q1MMTvOUIIv0uvgwEsdVktpgohNhFRDwCLiGidEOI594K5LickO4ipt1MaOpXj4y92GnYLM5OccIhkY95L5++B+ppyy8eoy0LJ28w6dC2Bg0i7KoVPqTf3AEA3iXjM8X7dAlK+z+27jB+LoLJujlJsuRDJSixhS8A5K2I4Y2jWlIBLh3SqW/j5kzNBiOAsvtyW311CWQJOfoYyJiKg2qwP/HWKTHJX06tnVV0JIZLt6hYukaw8zy6T7Pw/3TrJbLdgGEYSixGOntAH++3RgCuefBO3L92Ahas+wE8PGo55IxuzHoDNlA6h1A0RJWEI5D8JIR4JmPVYuKwWQohN5uNmAH8BMDGzUNPD9tfaauKh703BFUeOcnSI09VJHtunCw4Y2eg7Mj4VmQ7cC1s+zB7o5yek/JdVvafu+aTQlFfU8kJCh7rtZCL7H5SwmWRdaTNAk0mWnuQ0P7tu1fqBgG78RH22meRUdZLtEnA+nuQyTSYZZIjkgEyyus/loDapm4N2oZVJbs53Jjkzu4XdeIczyQzDOOlcVYZfHTYSj5w2Bd1rynHGn1/Dt257Ce9s+Sr1wkyHIOWZg4xLqtsArBVCXBMwXx2AGQAeVaZVE1GtfA5gPyhF6/NJmSaT3KdrFY4eb3RYrUh6s8S2sILjtXSFlipEQ3khpUgOm0n2sWe0pchCul9zXy1LASpFcmVZ6o50QO7tFoGDv5RNBXmS7VrP6cXx8GlTcMkhe6R8T36iPutMMryl63Qb9q9uYXqSFZEcixmfuywBJwfZOVeriGSz8L5l3wnYiWWW3UKfSfbbRrpknUlmTzLDMD6M7dsFj50xFZfO3wMr3/8cc697Dlc9+ab2rhvTsQhjt9gbwLcAvE5EK8xpPwLQFwCEEDeb0w4D8JQQYpuybAOAv5gnxwSAPwsh/pGLwFMhRU6rj3dTZknVjJwUmWrbZyC1P9WNn2821fxh7wj7iXdbJKdeFvD6PONmANt2SruFfyZZ3UZQe+ewZFLdIqhRiK6ZSBiaulejqXt1yvnU9daUJ1AWj2FXa1vOPMl+67E9yfrlBzcYNZ+nDLD98nLgnpVJTvHDL7PpoewW5l0Eq7qFa1b1c81mz7iXDbub3d9lhmEYHfEY4YTJTThgRCMue2Itrl+8Hn9dsQkXH7wH9h3eEHV4TESkFMlCiOcR4vwmhLgTwJ2uaf8BMDrD2LJCimS/Ei8ykyzFFGBnkqUoSGSaSVZEY5jsWbqeZDmbW0ymElhAcCUGub5tZmWEQLuFmknOsd0i2JOsX4bIsBRI8diapSc5FepqiYC9B3bD4je3WIMeM8UqQ+gTtxSdfgP3+nWvxis/2ddRB9jyJJuD+nR2CxXbbpH6uJTH+o6WVsRIc7yrdwiIcOLk3fF1c/rZmYwzyeaFXy7udjAMU/rU15bjmmPG4OgJffDTv76Bk+9ejn2H9cDPD94DfbpWpV4BU1KU7JlDDtRJlUlWa8ZaItMlDtLNJOffk6wXL7Ic14K9m/xjC7JbSJG8S3qSww3cy7UAcQ/C89uu+/2r761Z1nrOsm5xmDgIhFlDewAAPvpiR1brTVUnOVUmGQC61ZS7BhOSo7pFc2vAwrAzyenYLXY0t2ovbtwXE5fMH4Erjkz/utlbAi7c52pXt+BMMsMw4ZnUvxueOHsafnTgULzwzqeYc+0SXP/Pty1rGdMxKNm21Jbdwi+TbFoJ1Exy95oybPhkm1XVQZ5Y892W2qqPHHIzdpMH5wK1FUlsvHxexrFZnuQdpic5ZCZZZqDv+s7EjIuzBzUGcc5nP/e05Y6RdaVji82MwkmNmsWOGSUBv25uxTET+ma1WpHCMrPv8AY0PvM2TprWL/Q6yVyf/Dy7BNTgBoAult3C+D/oGkjeRdjZ0pbyrkk2n4U3kxxuOa6TzDBMpiTjMZwyfQAOGtUTv1i4Blc99RYeXfEBLj9iJPbcvWvU4TEFoORFsp9ok7Vgm5VM8o3H7YnFb25G7y7O+sjp2i0yziSnWQIukxO/471oRGaMwlW3kKtJxskSRzMG16cdjxWK8jyTOsmAPuOfrUfYPw77OcEor3fK9AFZr9eqk+yTAe9eU45lF81Oa50xMvYDEeH6b47FqF6dA+e3ulWGsFuoJeB0+98pnDP/LNz7I+wgQHvgXsneNGMYJs/07FyJm47fE4vXbcaP//I6jrx5Gb41aXdcsP8Q1FYEJx2Y9k3JnjlS2S3koDRVJNfXllvVLwAlC5XmuT19kZyeGE/Xw6ySCMgkG6/HrMYQgSLZqj+bm0MobHULuxa0v11EN79KqkxqGNx2hlyRjww4kX0hc9ConujbLZyvLj2R3JbSbhFJJtn88rLdgmGYbJk5tAeeOm8GTpzchHv+/V/MueY5LFrzcdRhMXmkZEVymWW30L8u/bZBA60yHRnvEKIh9rAtesOt389uEW5Zp5fWjT3QKbjVtHtwY7bItaRaX1DWXbcoafb/kh/OxEs/Ti8b61lviu1mitWWOofCW9ot0l6OUh//aiMZ7f533LjIJpPsJG1PMtstGIbJATXlCVx8yB545LQpqKtM4rt3L8f3//QqNn+Z3XgUpjgpWZFsddxLMXAvaBBTmExa0HJhl7W8n2FP/BkOKHTHpltcDu6qCCj/BtjvqyygK186hBFk6ny62cJmkjtVJNGjtiKDKPXrzeXgwANHNjoec4EcuJeKymQcA3vUWP/L5GtwMxH1glBnt7Cf5zaTnJ7dgttSMwyTS8b27YLHz5yKH+w3GIvWfox9r16C+19+17fyENM+KVlPstVxzyeVbIvk3GeS0x+4Jx/DbccSihmojlR2i9qKBDZ/uRMVZSlEshQfOfJ6hs0kA7bH1k2q2/25xJkhzd16h+xWm3LwZboQhTsO37hkf8f/YZrpxGN26b1Ux0I2+8nT7jrkYWd1zWS7BcMwOaYsEcMZswbhgJGNuOiR1/F/D7+Ov7y2CZcdPgr9QtTbZ4qfkk2vWAP3fC7qrDrJAdUYMu2458zWhvckhxXjcv5MbiGr70WXAZWDEILKvwGK3SJH4sOupBBuf+ntFuGm5YJ8ieR8QAh3HMZjpD12U3XRk9+1Ks2FlaOZSA53VNqZZLZbMAyTJwbU1+C+707CZYePxOoPvsD+1z2HGxavD0zCMe2DkhXJ0gaQqgRcEJnaGtT5wyybqSc5k0yys06y9/VOlaZITmm3MNeXM/ER/ra4IZK903VCKH/NRPJjt8gHMaJQ3ng3YY9/2fJZVzLQcTGRfgjKepxLhz3s4jm+48EwDKMjFiN8Y2JfPHPeDMwe2gNXPvkmDv7d81jx3udRh8ZkQcmeOcK2pQ4inmmdZNWnmY9mIrHMxDuQOstdW2F6klPsH3fDlWyRqwnr4dZmjUN6knNBu8ok++yvVIS9kyL9/yktOhEM3EvwwD2GYQpIj04VuOn4PXHLt/bE/7bvwuE3LsWlj6/BNrO0KtO+KGGRbJwUU7WlDsLqOJbmyT3hEKKp5w9zW1vFtmekFRYA5y1xnW7oZIrkoEYixrbTy36nIj1PMmn3q37gXpaBBcSge16MZCqS7QuX4PnkBWlliu9UNrvJvWzYdcWtEnAl+1PHMEwRst8eu2HReTPwzb364valG7Dftc9h8Zubow6LSZOSPXPkJJMcy0yMOgRUSJUWj1HozHC6zUdUqsvtsZpau4XpSS4P6UnOld0iPU+yfr5CNhMhn+fFSCIWy+hzioe8Y1FmiWSd3SI3FxOZZ5KN+ZI8cI9hmALTqSKJXx46Eg9+bzIqkjF8+46XcfZ9r+HTr3ZGHRoTkpKtbiFFsl81lvIQpctyUic55KK/OmwEJjSFa3OZ7kA/FfV96wfuJUKtOx17RBhkLGFKyvkO3CukJ7kd2S1OndEfn361K+3lwja5kQK0UjtwT/887VhcMfDAPYZh2gsTmrriibOn4cbF7+DGZ9fjube24CfzhuPwcb1yOqCZyT0lnElOJfJSH5iZ2i3SrZMMAMdM6Iv+9TWpZ4TiB87gxO/sFOd9XVa3aAmoHw1kfgHhH5fxGCbjR6T//FK3Rc4d+eq4lw/26FmH6Rm0DA/b2dG2W3ivuZ0XE7nMJIdbrmt1GX4ybxjmjshd3WmGYZh0KU/Ece6cwfjbWdPQr3s1zn9wJU64/SW8++n2qENjAihhkZz9W4uZNWDTFYK5usXshwwn2+xY0MC9XSlK18RznKGzRXKITHJMX91CXjSE9dJmiyHW87uNKJEfRaqPxBLJZd4ZnSXgMo+l3FVtJZ2a4idP64/d6rJrHsMwDJMLBjfU4qHvTcEv5u+B1979HPtdtwS3PPeOb08HJlo6tEg+bGwvXH3U6MB5ErFw3cr8yIdQs+wWGcYVJCKlJzlVfcdsGppo15d2CThNJtlcNGlWNMj3gDq/OEqF0JnkhL8nWbe+TJg3qhF3LJiQ8fIMwzDFQixG+NbkJiw6bzqmDuyOXz2xDofeuBRvbNoadWiMi5IVyWGyv9ceMwZH7Nk7cJ4YUVaWgrxkkrOwWwBAdZmRLQ7yJKcSyZlaUXwxV1MWwm6RauCebHCSbwFLKP5Be9kQtsxf6DrJWeys6vIEZg7tkfkKGIZhiozGukr84YTxuOGb4/DR1p2Yf8NSXPb3tfh6V2vUoTEmJSuSAWBEr0644ohRWa3juL12xz5D0vdzSvJpt8hUvMsycLrQaqTdoiWV3cJ8zNnAPYMwdwDIpwRczOWTzneS168UXakQtoJJmzk6VlcnWV2Sx84xDMM4ISLMG9WIZ86bgSPH9cbvl/wH+1/3HJau/yTq0BiUuEheeOY0HD2hT1br+NnBwzFtUOYimfKwh8PeBvdDimSdaJHez+YUA/dsu0VGIfiuL5zdQv/epaiTQjvvVgifAYSlQtgmN7KrZapMcmnn3RmGYTKnriqJXx85Cvd+dxLiMcJxt76I8x9Yif9tS78yEZM7SlokFwO5yrSqZJ9JllUIvMv36FQOAJg/pmfgOnJeJ9l8DGe3CB64lyhQJrnj2C2C52sJEMmO9ZXyzmIYhskBkwd0w9/PnobT9xmAR1dswr7XLMGjKzZB+NWzZfIKi+Q8kx+7RXYD94IyyZ0qklhz6f44a9agwHXEQ/pVwyJXkwiRmo4RabObhc4kdxi7RYo3aWWSNXYL9TMo5aw7wzBMrqhIxvHDuUPx2BlT0btLJc6+bwW+c+fL2PT511GH1uFgkZxn8qELsmkmAgBVZtc9P9FSVZZIeYvdnbXNFvmekiGaifi1WZYD9mTDlLxnkn3iKBXCDtyTmWRdF8vKFG3QGYZhGD3De3bCI6fvjZ8eNBwvbvgMc65Zgtuf32AlJpj8wyI5z+SnuoXxmKlIrjaFS6rBeYExZGn5cGMP3MvCbmHu6/Jk3PF/voiRrj5I6RC2416QJ1mdVtp7i2EYJvfEY4STpvbDU+dOx8R+XXHpwjU4/KYXsO6jL6IOrUPAIjnP5LVOcpae5G27WjKOQd6Cz9ktdHM1yVB2C5+BezFnJjnvzURQ2haCsM1EWtqMiy2d3ULNLpfwrmIYhskrvbtU4Y4FE/CbY8fgvc+2Y95vn8cvFq7BFzuaow6tpGGRnGfykc3cZ3AP/GC/wejfvTqj5avLDeGyfWfmIlne7MlZxz1Iu0W4TLJOnMp9XZEsjCe51Dvuha2iIluYazPJZSySGYZhcgERYf6YXnjmvBk4enwf3L50A2ZdtQSPvPo+D+zLEyyS80yuOtKp1FUlccasQRmvWwqXbVkULJc+1Jx13JOZ5FB1kvXZTTlNlrHLvye5tA0EYUWyLBeo8yRXlbHdgmEYJpd0qS7DZYePxF9P3xu9ulTivAdW4ujfL8OaD9iCkWtYJHdAZMe9bLr6yKvWnFW3MB/DiGT/ttTOTHK+rRBU8nWSw9l6WgPsFmp2OVc1tRmGYRhgdJ/O+MtpU3D54SOxfvNXOOh3/8LFj63G1q/ZgpEr+LTVATlsbC/UlCdw2LheGa9DDtYKMc4uFHYmORd2C/8Sd7nEbwBhqSCvV7Kpk1zBA/cYhmHyRixGOHZiXyz+wT44bq/dcfeyjZh11bN4YPl7aOMqGFnDIrkD0qdrFd64ZH8MqK/JeB25tltIwtotdJv1DrX8cRYAABZJSURBVNzLcyYZJZ5JDlkCTloqyjTl+8qVabk4VIY01Ga/EoZhmBKjc1UZfnHoCDx2xlQ0da/GDx9ahSNufgFvbNoadWjtmkTqWRjGi7Rb5KqjoBTdWdktZAk405Oc73EMRJT/jURIWLvFn787CU+v+Rg15d6fE/UiIheHyqNn7I2dzZmXLmQYhillRvSqw4OnTsYjr23C5X9fi4Ovfx7fnNgXF+w/BJ2ryqIOr93BIpnJiFZTp+SqTnKrJZJD2C1ienEec3mSW/J8q8kIoXQzyfGQA/cG1NdgwIzUdyVykXWvSMa1AwQZhmEYg1iMcOSevTFneAOue/ot3L3sv3ji9Q/xw7lDccz4PnkpKFCqsN2CyYhWmUnO0ZdNlhEL25Zap7fiLk+yrN+bLwy7RV43ESmW3SLHDWMYhmGY/FNXmcTPD94DC8+cikE9anHRI6/jsBuXYsV7n0cdWruBRTKTEXJAQK5EcrOZmg7TltrPbmFnkk2R3JrfTHLpd9xzPma/vlLeWwzDMMXJsMZOuP/USbjumDH4YOsOHHbjUlz48Cp8tm1X1KEVPSySmYzoZzYyGdGzLifrk3aLslDVLfTlxOw6ydJukedMsk/nv1JBXgDlyndewruKYRimqCEiHDq2F/55/gycPLUfHnrlfcy86lnc8+//WudfxguLZCYjpg+ux1PnTsdR43vnZH3NbeHtFidOacJxe+3umS6XlSI53198P9tHqWA1E8lRKrmULygYhmHaA7UVSfx43nD8/expGN7YCT/96xs45Prn8cp//xd1aEUJi2QmYwY31OasBJpsSBHGbjF/TC8cOLLRM12KsHLTbtGcZ7sFUNo+27Ad9xiGYZj2xaCGWvz5u3vh+m+Oxadf7cIRN72AHzy4Ep98tTPq0IoKFslMUSAFbRi7hR/SblGo6gel3nFP7s8QVflCwSOqGYZhigciwkGjeuKZ82fgezMG4NEVmzDzqmdx59INaGnlUpsAi2SmSJBfyDB2Cz+sgXshstG5oMPYLXLcepxhGIYpHqrLE7jwgKH4xznTMaZPZ1z8+Boc9Lvn8dKGz6IOLXJYJDNFgVUnOQuBG3fZLfJNqQ/cy5VItqtklO6+YhiGae8MqK/B3d+ZiJuPH4cvd7Tg6N8vw7n3r8DmL3ZEHVpksEhmigJpt0hmcUve3ZY635R6JtmqbpGlTULaX0p5XzEMw5QCRIS5Ixrx9HkzcOasgfjbqg8x6+oluPVf/7FKtXYkWCTniZuOG4ezZw+KOox2Q0saA/f8GN27M6YPri+YSCaUtoXAaiaSpbq1RHLWETEMwzCFoLIsjvP3G4Knzp2OCU1d8Mu/rcW83/4Ly975NOrQCgqL5DxxwMhGnDtncNRhtBtarLbUmR+S+w5vwN3fmZjVOtKixO0W0r6SbSZZXrQ0cy1OhmGYdkVT92rcvmACbj1hPL5ubsU3/vBvnHnva/hoa8ewYLBIZooCuy119qKzrIB2i1JOj04b3B3nzRmMgT1qslpP365VAOwujQzDMEz7gYiw7/AGLDp3Bs7ZdxCeWv0RZl39LG5e8g52tZS2BYNFMlMUWB33ciBwywqUSS51u0WniiTOmj0o60zyTcfviauOGo0+plhmGIZh2h8VyTjO2Xcwnj5vBqYM6I7L/74Oc3/zHBau+qBk/coskpmiQH7BcmGVyMbXnA4xopK2W+SKrtVlOHLP3HRmZBiGYaKlT9cq3HrieNyxYAIggDP+/BqmX7EYNyxej8+27Yo6vJySiDoAhgFsT3JO7BaFyiSXttuCYRiGYXyZObQHpg+ux7NvbsadL2zElU++id888zYOHdMTJ05pwh4966IOMWtYJDNFgWwm0q7sFkQgttkyDMMwHZR4jDB7WANmD2vA2x9/ibuWbcTDr2zCA8vfx8R+XfHtKU2YM7wBiUINqM8xLJKZoiCXmeRkojD5XQLX/mUYhmEYABjUUItfHjoSF+w/FA8ufw93LduI0/70KnrWVeBbk5tw7IQ+6FJdFnWYadE+pT1TcrTkoOOepKB2C1bJDMMwDGNRV5nEydP649kfzMQfThiPfvXV+PU/1mHSZc/gwodXYe2HX0QdYmg4k8wUBVZ1ixwI3GyrMYQlRgQh2G/BMAzDMG7iMcKc4Q2YM7wBb370Je58YSP+8tr7uO/l9zCpf1csmNIPc4Y3FOycnQkpFQkR9SGixUS0hohWE9HZmnn2IaKtRLTC/PuZ8tpcInqTiNYT0YW5fgNMadCaQ7tFobK7RECM78UwDMMwTCBDdqvFZYePxL8vmo2LDhiK9z77Gt/74yuYfsVi3PLcO9i6vTnqELWEySS3ADhfCPEqEdUCeIWIFgkh1rjm+5cQ4iB1AhHFAdwAYA6A9wG8TESPaZZlOjiVyTi+bm4t6itKNzxwj2EYhmHC07mqDKfOGICTpvbD02s3484XNuBXT6zDtYvexmHjemHBlCYMbqiNOkyLlCJZCPEhgA/N518S0VoAvQCEEboTAawXQvwHAIjoPgDzQy7LdCAeO2NvLF3/Sbvy+PLAPYZhGIZJn0Q8hrkjdsPcEbth7Ydf4K4XNuLhV97Hn198F3sP7IYFU/ph1tAekSfO0vIkE1ETgLEAXtS8PJmIVgL4AMAPhBCrYYjp95R53gewV0aRMiXNoIZaDCqiq8cwGHWSWSUzDMMwTKYMa+yEy48Yhf+bOxT3vvwu7ln2X3z37uXo07USJ05uwlHj+6CuMhlJbKEdlURUA+BhAOcIIdxDE18FsLsQYjSA3wH4a7qBENEpRLSciJZv2bIl3cUZpuDEiCUywzAMw+SCLtVlOH2fgfjXD2fixuPGobFTJX75t7WY9Ktn8JO/vo71m78seEyhMslElIQhkP8khHjE/boqmoUQTxDRjUTUHcAmAH2UWXub0zwIIW4BcAsAjB8/np2eTNHDdguGYRiGyS2JeAwHjmzEgSMb8camrbjrhY14YPn7+OO/38W0Qd3x7b2bsM/gHogVwIoRproFAbgNwFohxDU+8+xmzgcimmiu91MALwMYRET9iKgMwLEAHstV8AwTJTEixFglMwzDMExeGNGrDlceNRrLLpyFC/Yfgrc//grfuXM5Zl79LG5/fgO+2JHfqhhhMsl7A/gWgNeJaIU57UcA+gKAEOJmAEcCOI2IWgB8DeBYYRSQbSGiMwA8CSAO4HbTq8wweeWFC2ehPAeNSQIhcHULhmEYhskz3WrK8f2ZA3HK9P54cvVHuGPpRly6cA2ufupNHLlnb5wwpQkD6mtyvt0w1S2eB4Ktl0KI6wFc7/PaEwCeyCg6hsmQnp0r874NtlswDMMwTOFIxmM4aFRPHDSqJ1a9/znufGEj7n3pPdy17L+YMbgelx0+Mqfnf+64xzAZcvi4XmhtizoKhmEYhul4jOrdGdccPQYXHTAM9770Lp54/UN0rS7L6TZYJDNMhhwzoW/UITAMwzBMh6a+thxnzR6EM2cNzHmvBW6qyzAMwzAMw7Rr8tGMjEUywzAMwzAMw7hgkcwwDMMwDMMwLlgkMwzDMAzDMIwLFskMwzAMwzAM44JFMsMwDMMwDMO4YJHMMAzTQSCiCiJ6iYhWEtFqIrpEM085Ed1PROuJ6EUiaip8pAzDMNHDIplhGKbjsBPALCHEaABjAMwlokmueU4C8D8hxEAA1wL4dYFjZBiGKQpYJDMMw3QQhMFX5r9J80+4ZpsP4C7z+UMAZlM+CpAyDMMUOSySGYZhOhBEFCeiFQA2A1gkhHjRNUsvAO8BgBCiBcBWAN006zmFiJYT0fItW7bkO2yGYZiCwyKZYRimAyGEaBVCjAHQG8BEIhqR4XpuEUKMF0KMr6+vz22QDMMwRQCLZIZhmA6IEOJzAIsBzHW9tAlAHwAgogSAOgCfFjY6hmGY6GGRzDAM00Egonoi6mw+rwQwB8A612yPATjRfH4kgH8KIdy+ZYZhmJInEXUADMMwTMFoBHAXEcVhJEkeEEIsJKJLASwXQjwG4DYA9xDRegCfATg2unAZhmGig0UywzBMB0EIsQrAWM30nynPdwA4qpBxMQzDFCNst2AYhmEYhmEYFyySGYZhGIZhGMYFi2SGYRiGYRiGcUHFOGiZiLYA+G+ai3UH8EkewsknHHNhaG8xt7d4AY5ZZXchRIcqHJzhbzZQPMdNscQBFE8sxRIHwLHoKJY4gOKJJdM4fH+zi1IkZwIRLRdCjI86jnTgmAtDe4u5vcULcMxMZhTLZ1AscQDFE0uxxAFwLMUcB1A8seQjDrZbMAzDMAzDMIwLFskMwzAMwzAM46KURPItUQeQARxzYWhvMbe3eAGOmcmMYvkMiiUOoHhiKZY4AI5FR7HEARRPLDmPo2Q8yQzDMAzDMAyTK0opk8wwDMMwDMMwOaEkRDIRzSWiN4loPRFdGHU8fhDRRiJ6nYhWENFyc1pXIlpERG+bj10ijvF2ItpMRG8o07QxksFvzf2+iojGFUm8FxPRJnM/ryCiA5XXLjLjfZOI9i90vGYMfYhoMRGtIaLVRHS2Ob2Y97NfzEW5r4mogoheIqKVZryXmNP7EdGLZlz3E1GZOb3c/H+9+XpTIePtaBTLb7bu9yOiOLTfr4hi0X53IownTkSvEdHCiOPwnL8jjKUzET1EROuIaC0RTY4ghiHK7/4KIvqCiM4pdBxKPOeax+sbRHQvEVXkZMVCiHb9ByAO4B0A/QGUAVgJYHjUcfnEuhFAd9e0KwBcaD6/EMCvI45xOoBxAN5IFSOAAwH8HQABmATgxSKJ92IAP9DMO9w8PsoB9DOPm3gEMTcCGGc+rwXwlhlbMe9nv5iLcl+b+6rGfJ4E8KK57x4AcKw5/WYAp5nPTwdws/n8WAD3F3ofd5S/YvrN1v1+RBSH9vsVUSza706E++Y8AH8GsDDiz8hz/o4wlrsAnGw+LwPQOeJ44gA+glFvOIrt9wKwAUCl+f8DABbkYt2lkEmeCGC9EOI/QohdAO4DMD/imNJhPowDHubjoRHGAiHEcwA+c032i3E+gLuFwb8BdCaixsJEauATrx/zAdwnhNgphNgAYD2M46egCCE+FEK8aj7/EsBaGF/yYt7PfjH7Eem+NvfVV+a/SfNPAJgF4CFzunsfy33/EIDZREQFCrejUTS/2Wn+fuQzjnS/X/mMxe+7U3CIqDeAeQBujWL7xQgR1cG4uLsNAIQQu4QQn0cbFWYDeEcIkUlDoVyRAFBJRAkAVQA+yMVKS0Ek9wLwnvL/+4joxyUEAsBTRPQKEZ1iTmsQQnxoPv8IQEM0oQXiF2Mx7/szTGvC7YqFpejiNW/rj4WRrWkX+9kVM1Ck+9q8TbsCwGYAi2BkLz8XQrRoYrLiNV/fCqBbIePtQER+bBQzmu9XFDE4vjtCiKhiuQ7ADwG0RbR9Fd35Owr6AdgC4A7ThnIrEVVHGA9g3H27N6qNCyE2AbgKwLsAPgSwVQjxVC7WXQoiuT0xVQgxDsABAL5PRNPVF4Vxn6Coy420hxgB3ARgAIAxML4wV0cbjh4iqgHwMIBzhBBfqK8V637WxFy0+1oI0SqEGAOgN4zs5dCIQ2KYQIJ+EwqJ+7tDRCMKHQMRHQRgsxDilUJv24fA83cBScCwCN0khBgLYBsMe14kmOM6DgHwYIQxdIFxN6ofgJ4Aqono+FysuxRE8iYAfZT/e5vTig7zagdCiM0A/gLjxP2xvHVuPm6OLkJf/GIsyn0vhPjY/JFvA/AH2Lf5iyZeIkrCOBn+SQjxiDm5qPezLub2sK/NW5GLAUyGYVVJaGKy4jVfrwPwaYFD7SgUzbFRTPj8JkSK8t2ZG8Hm9wZwCBFthGHJmUVEf4wgDgC+5+8oeB/A+0p2/yEYojkqDgDwqhDi4whj2BfABiHEFiFEM4BHAEzJxYpLQSS/DGCQOWq9DEba/7GIY/JARNVEVCufA9gPwBswYj3RnO1EAI9GE2EgfjE+BuAEMpgE4xbHh7oVFBKXX/cwGPsZMOI91qxk0A/AIAAvRRAfwfCTrRVCXKO8VLT72S/mYt3XRFRPRJ3N55UA5sDweS4GcKQ5m3sfy31/JIB/mtl8Jve0i9/sQhLwmxBFLLrvzrpCxyGEuEgI0VsI0QTjGPmnECIn2cF0CTh/FxwhxEcA3iOiIeak2QDWRBGLyTcQodXC5F0Ak4ioyvwuzYbxe589uRj9F/UfjNH/b8HwHP446nh8YuwPYxT3SgCrZZwwfI/PAHgbwNMAukYc570wbps3w7hiPckvRhijoG8w9/vrAMYXSbz3mPGsgnHybVTm/7EZ75sADohoH0+FYaVYBWCF+Xdgke9nv5iLcl8DGAXgNTOuNwD8zJzeH4ZYXw/j9mC5Ob3C/H+9+Xr/KI6NjvJXLL/Zut+PiOLQfr8iikX73Yn4eNkHEVa38Dt/RxjPGADLzc/orwC6RBRHNYw7bnVFcIxcAuNi7g3zvFSei/Vyxz2GYRiGYRiGcVEKdguGYRiGYRiGySkskhmGYRiGYRjGBYtkhmEYhmEYhnHBIplhGIZhGIZhXLBIZhiGYRiGYRgXLJKZdgsRtRLRCiJaSUSvElFg8XAi6kxEp4dY77NEND53kTIMwzDKb7b8y1mnOCJqIqJIahczpUsi9SwMU7R8LYzWqSCi/QFcBmBGwPydAZwO4MYCxMYwDMM4sX6zGaY9wJlkplToBOB/AEBENUT0jJldfp2I5pvzXA5ggJnBuNKc9//MeVYS0eXK+o4iopeI6C0imlbYt8IwDNNxIKKNRHSF+Vv8EhENNKc3EdE/iWiV+Zve15zeQER/MX+3Vyp3EeNE9AciWk1ET5kdAxkmYziTzLRnKoloBYxuaY0AZpnTdwA4TAjxBRF1B/BvInoMwIUARijZ5wMAzAewlxBiOxF1VdadEEJMJKIDAfwcRm94hmEYJnPkb7bkMiHE/ebzrUKIkUR0AoDrABwE4HcA7hJC3EVE3wHwWwCHmo9LhBCHEVEcQA2ALgAGAfiGEOK7RPQAgCMA/LEwb40pRVgkM+0Z1W4xGcDdRDQCRhvnXxHRdABtAHoBaNAsvy+AO4QQ2wFACPGZ8toj5uMrAJryEz7DMEyHIshuca/yeK35fDKAw83n9wC4wnw+C8AJACCEaAWwlYi6ANgghJAinH+7maxhkcyUBEKIZWbWuB7AgebjnkKIZiLaCCPbnA47zcdW8PeEYRgm3wif5+mwU3neCoDtFkxWsCeZKQmIaCiAOIBPAdQB2GwK5JkAdjdn+xJArbLYIgDfJqIqcx2q3YJhGIYpHMcoj8vM5y8AONZ8fhyAf5nPnwFwGgAQUZyI6goVJNOx4AwZ055R/W0E4EQhRCsR/QnA40T0OoDlANYBgBDiUyJaapYJ+rsQ4gIiGgNgORHtAvAEgB9F8D4YhmE6Am5P8j+EELIMXBciWgUjG/wNc9qZAO4gogsAbAHwbXP62QBuIaKTYGSMTwPwYd6jZzocJESmdzUYhmEYhmGyw7TEjRdCfBJ1LAyjwnYLhmEYhmEYhnHBmWSGYRiGYRiGccGZZIZhGIZhGIZxwSKZYRiGYRiGYVywSGYYhmEYhmEYFyySGYZhGIZhGMYFi2SGYRiGYRiGccEimWEYhmEYhmFc/H8jUJGRag2ougAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 10 | Time: 1m 10s\n", - "\tTrain Loss: 2.998 | Train PPL: 20.040\n", - "\t Val. Loss: 4.710 | Val. PPL: 111.007\n" - ] - } - ], - "source": [ - "for epoch in range(N_EPOCHS):\n", - " \n", - " start_time = time.time()\n", - " \n", - " train_loss = train(model, train_iterator, optimizer, criterion, CLIP, train_history, valid_history)\n", - " valid_loss = evaluate(model, valid_iterator, criterion)\n", - " \n", - " end_time = time.time()\n", - " \n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut1-model.pt')\n", - " \n", - " train_history.append(train_loss)\n", - " valid_history.append(valid_loss)\n", - " print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__Let's take a look at our network quality__:" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [], - "source": [ - "del utils" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "import utils\n", - "import imp\n", - "imp.reload(utils)\n", - "generate_translation = utils.generate_translation\n", - "remove_tech_tokens = utils.remove_tech_tokens\n", - "get_text = utils.get_text\n", - "flatten = utils.flatten" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "batch = next(iter(test_iterator))" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Original: there is a 24 - hour front desk at the property .\n", - "Generated: the property offers a 24 - hour front desk . .\n", - "\n", - "Original: this property also features free wifi .\n", - "Generated: free wifi access . . . .\n", - "\n" - ] - } - ], - "source": [ - "for idx in [1,2]:\n", - " src = batch.src[:, idx:idx+1]\n", - " trg = batch.trg[:, idx:idx+1]\n", - " generate_translation(src, trg, model, TRG.vocab)" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [], - "source": [ - "from nltk.translate.bleu_score import corpus_bleu\n", - "\n", - "# \"\"\" Estimates corpora-level BLEU score of model's translations given inp and reference out \"\"\"\n", - "# translations, _ = model.translate_lines(inp_lines, **flags)\n", - "# # Note: if you experience out-of-memory error, split input lines into batches and translate separately\n", - "# return corpus_bleu([[ref] for ref in out_lines], translations) * 100" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "59it [00:03, 18.87it/s]\n" - ] - } - ], - "source": [ - "original_text = []\n", - "generated_text = []\n", - "model.eval()\n", - "with torch.no_grad():\n", - "\n", - " for i, batch in tqdm.tqdm(enumerate(test_iterator)):\n", - "\n", - " src = batch.src\n", - " trg = batch.trg\n", - "\n", - " output = model(src, trg, 0) #turn off teacher forcing\n", - "\n", - " #trg = [trg sent len, batch size]\n", - " #output = [trg sent len, batch size, output dim]\n", - "\n", - " output = output.argmax(dim=-1)\n", - " \n", - " original_text.extend([get_text(x, TRG.vocab) for x in trg.cpu().numpy().T])\n", - " generated_text.extend([get_text(x, TRG.vocab) for x in output[1:].detach().cpu().numpy().T])\n", - "\n", - "# original_text = flatten(original_text)\n", - "# generated_text = flatten(generated_text)" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "14.139920232081806" - ] - }, - "execution_count": 111, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "corpus_bleu([[text] for text in original_text], generated_text) * 100" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Baseline solution BLEU score is quite low. Try to achieve at least __24__ BLEU on the test set. \n", - "The checkpoints are:\n", - "\n", - "* __22__ - minimal score to submit the homework, 30% of points\n", - "\n", - "* __27__ - good score, 70% of points\n", - "\n", - "* __29__ - excellent score, 100% of points" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "homework.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Py3 Research", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/homeworks/lab01_nlp/my_network.py b/homeworks/lab01_nlp/my_network.py deleted file mode 100644 index 966416d..0000000 --- a/homeworks/lab01_nlp/my_network.py +++ /dev/null @@ -1,182 +0,0 @@ -import torch -import torch.nn as nn -import torch.optim as optim - -import torchtext -from torchtext.datasets import TranslationDataset, Multi30k -from torchtext.data import Field, BucketIterator - -import random -import math -import time - - -class Encoder(nn.Module): - def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout): - super().__init__() - - self.input_dim = input_dim - self.emb_dim = emb_dim - self.hid_dim = hid_dim - self.n_layers = n_layers -# self.dropout = dropout - - self.embedding = nn.Embedding( - num_embeddings=input_dim, - embedding_dim=emb_dim - ) - # - - self.rnn = nn.LSTM( - input_size=emb_dim, - hidden_size=hid_dim, - num_layers=n_layers, - dropout=dropout - ) - # - - self.dropout = nn.Dropout(p=dropout)# - - def forward(self, src): - - #src = [src sent len, batch size] - - # Compute an embedding from the src data and apply dropout to it - embedded = self.embedding(src)# - - embedded = self.dropout(embedded) - - output, (hidden, cell) = self.rnn(embedded) - #embedded = [src sent len, batch size, emb dim] - - # Compute the RNN output values of the encoder RNN. - # outputs, hidden and cell should be initialized here. Refer to nn.LSTM docs ;) - - # - - #outputs = [src sent len, batch size, hid dim * n directions] - #hidden = [n layers * n directions, batch size, hid dim] - #cell = [n layers * n directions, batch size, hid dim] - - #outputs are always from the top hidden layer - - return hidden, cell - - -class Decoder(nn.Module): - def __init__(self, output_dim, emb_dim, hid_dim, n_layers, dropout): - super().__init__() - - self.emb_dim = emb_dim - self.hid_dim = hid_dim - self.output_dim = output_dim - self.n_layers = n_layers - self.dropout = dropout - - self.embedding = nn.Embedding( - num_embeddings=output_dim, - embedding_dim=emb_dim - ) - # - - self.rnn = nn.LSTM( - input_size=emb_dim, - hidden_size=hid_dim, - num_layers=n_layers, - dropout=dropout - ) - # - - self.out = nn.Linear( - in_features=hid_dim, - out_features=output_dim - ) - # - - self.dropout = nn.Dropout(p=dropout)# - - def forward(self, input, hidden, cell): - - #input = [batch size] - #hidden = [n layers * n directions, batch size, hid dim] - #cell = [n layers * n directions, batch size, hid dim] - - #n directions in the decoder will both always be 1, therefore: - #hidden = [n layers, batch size, hid dim] - #context = [n layers, batch size, hid dim] - - input = input.unsqueeze(0) - - #input = [1, batch size] - - # Compute an embedding from the input data and apply dropout to it - embedded = self.dropout(self.embedding(input))# - - #embedded = [1, batch size, emb dim] - - # Compute the RNN output values of the encoder RNN. - # outputs, hidden and cell should be initialized here. Refer to nn.LSTM docs ;) - # - - - #output = [sent len, batch size, hid dim * n directions] - #hidden = [n layers * n directions, batch size, hid dim] - #cell = [n layers * n directions, batch size, hid dim] - - #sent len and n directions will always be 1 in the decoder, therefore: - #output = [1, batch size, hid dim] - #hidden = [n layers, batch size, hid dim] - #cell = [n layers, batch size, hid dim] - - - output, (hidden, cell) = self.rnn(embedded, (hidden, cell)) - prediction = self.out(output.squeeze(0)) - - #prediction = [batch size, output dim] - - return prediction, hidden, cell - - -class Seq2Seq(nn.Module): - def __init__(self, encoder, decoder, device): - super().__init__() - - self.encoder = encoder - self.decoder = decoder - self.device = device - - assert encoder.hid_dim == decoder.hid_dim, \ - "Hidden dimensions of encoder and decoder must be equal!" - assert encoder.n_layers == decoder.n_layers, \ - "Encoder and decoder must have equal number of layers!" - - def forward(self, src, trg, teacher_forcing_ratio = 0.5): - - #src = [src sent len, batch size] - #trg = [trg sent len, batch size] - #teacher_forcing_ratio is probability to use teacher forcing - #e.g. if teacher_forcing_ratio is 0.75 we use ground-truth inputs 75% of the time - - # Again, now batch is the first dimention instead of zero - batch_size = trg.shape[1] - max_len = trg.shape[0] - trg_vocab_size = self.decoder.output_dim - - #tensor to store decoder outputs - outputs = torch.zeros(max_len, batch_size, trg_vocab_size).to(self.device) - - #last hidden state of the encoder is used as the initial hidden state of the decoder - hidden, cell = self.encoder(src) - - #first input to the decoder is the tokens - input = trg[0,:] - - for t in range(1, max_len): - - output, hidden, cell = self.decoder(input, hidden, cell) - outputs[t] = output - teacher_force = random.random() < teacher_forcing_ratio - top1 = output.max(1)[1] - input = (trg[t] if teacher_force else top1) - - return outputs diff --git a/homeworks/lab01_nlp/utils.py b/homeworks/lab01_nlp/utils.py deleted file mode 100644 index f3691d2..0000000 --- a/homeworks/lab01_nlp/utils.py +++ /dev/null @@ -1,33 +0,0 @@ - -def flatten(l): - return [item for sublist in l for item in sublist] - -def remove_tech_tokens(mystr, tokens_to_remove=['', '', '', '']): - return [x for x in mystr if x not in tokens_to_remove] - - -def get_text(x, TRG_vocab): - text = [TRG_vocab.itos[token] for token in x] - try: - end_idx = text.index('') - text = text[:end_idx] - except ValueError: - pass - text = remove_tech_tokens(text) - if len(text) < 1: - text = [] - return text - - -def generate_translation(src, trg, model, TRG_vocab): - model.eval() - - output = model(src, trg, 0) #turn off teacher forcing - output = output.argmax(dim=-1).cpu().numpy() - - original = get_text(list(trg[:,0].cpu().numpy()), TRG_vocab) - generated = get_text(list(output[1:, 0]), TRG_vocab) - - print('Original: {}'.format(' '.join(original))) - print('Generated: {}'.format(' '.join(generated))) - print() diff --git a/homeworks/lab02_qa/LICENSE b/homeworks/lab02_qa/LICENSE deleted file mode 100644 index e1b9ab0..0000000 --- a/homeworks/lab02_qa/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) 2019 Christopher Chute http://chrischute.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/homeworks/lab02_qa/README.md b/homeworks/lab02_qa/README.md deleted file mode 100644 index de29ad3..0000000 --- a/homeworks/lab02_qa/README.md +++ /dev/null @@ -1,40 +0,0 @@ -#### Lab02: QA system - -In this homework your goal is to build the QA system for specific language. The default code is available for English and Russian languages. Russian example using the [SberQuAD dataset](https://arxiv.org/pdf/1912.09723.pdf). The preprocessing code and baseline solution (BiDAF) are the slightly adapted version of the [Stanford CS224n Starter code](https://github.com/chrischute/squad) for the SQuAD dataset. - -**To use any other language, please, refer to [this post](https://medium.com/deepset-ai/going-beyond-squad-part-1-question-answering-in-different-languages-8eac6cf56f21) or to the Table 2 in the paper [Deep learning based question answering systemin Bengali](https://www.researchgate.net/publication/346129818_Deep_learning_based_question_answering_system_in_Bengali), where the authors provide an overview of available datasets.** - -The available languages are (but not limited to): Korean, Arabic, French, Spanish, Italian, Russian, English, Hindi and Chinese. - -The starting point of this assighnment is the `SberQuAD_preprocessing_and_problem_statement.ipynb` notebook. -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github//natural-language-processing/tree/master/homeworks/lab02_qa/SberQuAD_preprocessing_and_problem_statement.ipynb) - - -You may choose either this assignment or the `homework05` on the Image Captioning. Or do both ;) - -Next comes the original instructions from the https://github.com/chrischute/squad repository. - -P.s. Downgrading PyTorch is not required, starter code works fine on PyTorch 1.4 -P.p.s. If you are running in Colab, mount your Google Drive and store the checkpoints/word vectors there. [Official instruction (en)](https://colab.research.google.com/notebooks/io.ipynb), [Habr post (ru)](https://habr.com/ru/post/348058/). Restarting the kernel after you finished the preprocessing (and saved the data to your disk) might be a good idea to release the memory. - -#### Setup - -1. Make sure you have [Miniconda](https://docs.conda.io/en/latest/miniconda.html) installed - 1. Conda is a package manager that sandboxes your project’s dependencies in a virtual environment - 2. Miniconda contains Conda and its dependencies with no extra packages by default (as opposed to Anaconda, which installs some extra packages) - -2. cd into src, run `conda env create -f environment.yml` - 1. This creates a Conda environment called `squad` - -3. Run `source activate squad` - 1. This activates the `squad` environment - 2. Do this each time you want to write/test your code - -4. Run `python setup.py` - 1. This downloads SQuAD 2.0 training and dev sets, as well as the GloVe 300-dimensional word vectors (840B) - 2. This also pre-processes the dataset for efficient data loading - 3. For a MacBook Pro on the Stanford network, `setup.py` takes around 30 minutes total - -5. Browse the code in `train.py` - 1. The `train.py` script is the entry point for training a model. It reads command-line arguments, loads the SQuAD dataset, and trains a model. - 2. You may find it helpful to browse the arguments provided by the starter code. Either look directly at the `parser.add_argument` lines in the source code, or run `python train.py -h`. diff --git a/homeworks/lab02_qa/SberQuAD_preprocessing_and_problem_statement.ipynb b/homeworks/lab02_qa/SberQuAD_preprocessing_and_problem_statement.ipynb deleted file mode 100644 index 7dafe1f..0000000 --- a/homeworks/lab02_qa/SberQuAD_preprocessing_and_problem_statement.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Credits: the provided initial code is an adaptation of the [Starter code for Stanford CS224n default final project on SQuAD 2.0](https://github.com/chrischute/squad) which is shared under MIT License. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook does initial preprocessing for the SberQuAD dataset and will give you the starting point in this assignment. If it looks too complex and/or time/resourse-expensive, you may stick to homework05 as well." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1. Preprocessing\n", - "This code is a bit changed version of the code from `setup.py`. If you want to work with the SQuAD dataset, stick to the original instructions from the https://github.com/chrischute/squad repository." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# If running on Colab, uncomment the following lines \n", - "\n", - "# !wget https://raw.githubusercontent.com/girafe-ai/natural-language-processing/master/homeworks/lab02_qa/args.py -nc\n", - "# !wget https://raw.githubusercontent.com/girafe-ai/natural-language-processing/master/homeworks/lab02_qa/layers.py -nc\n", - "# !wget https://raw.githubusercontent.com/girafe-ai/natural-language-processing/master/homeworks/lab02_qa/models.py -nc\n", - "# !wget https://raw.githubusercontent.com/girafe-ai/natural-language-processing/master/homeworks/lab02_qa/setup.py -nc\n", - "# !wget https://raw.githubusercontent.com/girafe-ai/natural-language-processing/master/homeworks/lab02_qa/test.py -nc\n", - "# !wget https://raw.githubusercontent.com/girafe-ai/natural-language-processing/master/homeworks/lab02_qa/train.py -nc\n", - "# !wget https://raw.githubusercontent.com/girafe-ai/natural-language-processing/master/homeworks/lab02_qa/util.py -nc" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# If running on Colab, uncomment the following lines \n", - "\n", - "# !pip install ujson\n", - "# !pip install tensorboardX\n", - "# !pip install pymorphy2==0.8" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\"\"\"Train a model on SQuAD.\n", - "\n", - "Author:\n", - " Chris Chute (chute@stanford.edu)\n", - "\"\"\"\n", - "\n", - "import numpy as np\n", - "import random\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "import torch.optim.lr_scheduler as sched\n", - "import torch.utils.data as data\n", - "import util\n", - "\n", - "from args import get_train_args\n", - "from collections import OrderedDict\n", - "from json import dumps\n", - "from models import BiDAF\n", - "from tensorboardX import SummaryWriter\n", - "from tqdm import tqdm\n", - "from ujson import load as json_load\n", - "from util import collate_fn, SQuAD" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "Path(\"./data\").mkdir(parents=True, exist_ok=True)\n", - "Path(\"./save\").mkdir(parents=True, exist_ok=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Downloading the SberQuAD data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!wget http://files.deeppavlov.ai/datasets/sber_squad_clean-v1.1.tar.gz -nc -O ./data/sber_squad_clean-v1.1.tar.gz" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! tar -xzvf ./data/sber_squad_clean-v1.1.tar.gz\n", - "! mv train-v1.1.json data\n", - "! mv dev-v1.1.json data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Downloading the word vectors (this may take a while)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! wget http://files.deeppavlov.ai/embeddings/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize.vec -nc -O ./data/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize.vec" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And finally the preprocessing for the SberQuAD dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_file = './data/train-v1.1.json'\n", - "dev_file = './data/dev-v1.1.json'\n", - "glove_file = './data/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize.vec'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from setup import *" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Uncomment this cell if needed\n", - "# !pip install pymorphy2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nlp = spacy.blank(\"ru\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following cell may take a while (usually 10 minutes or less)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Process training set and use it to decide on the word/character vocabularies\n", - "word_counter, char_counter = Counter(), Counter()\n", - "train_examples, train_eval = process_file(train_file, \"train\", word_counter, char_counter, nlp)\n", - "word_emb_mat, word2idx_dict = get_embedding(\n", - " word_counter, 'word', emb_file=glove_file, vec_size=300, num_vectors=1560132)\n", - "char_emb_mat, char2idx_dict = get_embedding(\n", - " char_counter, 'char', emb_file=None, vec_size=64)\n", - "\n", - "\n", - "dev_examples, dev_eval = process_file(dev_file, \"dev\", word_counter, char_counter, nlp)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we have the preprocessed data:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_record_file = './data/train.npz'\n", - "dev_record_file = './data/dev.npz'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from args import add_common_args, get_setup_args" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Retreiving the default arguments for the preprocessing script\n", - "_args = get_setup_args(bypass=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "_args" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "build_features(_args, train_examples, \"train\", train_record_file, word2idx_dict, char2idx_dict)\n", - "dev_meta = build_features(_args, dev_examples, \"dev\", dev_record_file, word2idx_dict, char2idx_dict)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "save(_args.word_emb_file, word_emb_mat, message=\"word embedding\")\n", - "save(_args.char_emb_file, char_emb_mat, message=\"char embedding\")\n", - "save(_args.train_eval_file, train_eval, message=\"train eval\")\n", - "save(_args.dev_eval_file, dev_eval, message=\"dev eval\")\n", - "save(_args.word2idx_file, word2idx_dict, message=\"word dictionary\")\n", - "save(_args.char2idx_file, char2idx_dict, message=\"char dictionary\")\n", - "save(_args.dev_meta_file, dev_meta, message=\"dev meta\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2. The experiment\n", - "\n", - "Now you are almost ready to go. You may follow these steps to begin (or just start your experiments here).\n", - "\n", - "1. Try running the `train.py` script from the console (or via `!`) (default command-line arguments are ok for the start). If will run the BiDAF model on the preprocessed data. Set `--use_squad_v2` flag to False (SberQuAD is similar to SQuAD v1.1).\n", - "\n", - "Example code (be careful with the path and the names of the variables):\n", - "```\n", - "python train.py --name first_run_on_sberquad --use_squad_v2 False\n", - "```\n", - "\n", - "2. After if finishes (might take an 1-2-3 hours depending on the hardware), evaluate your model on the `dev` set and measure the quality.\n", - "Example code (be careful with the path and the names of the variables):\n", - "```\n", - " python test.py --split dev --load_path ./save/train/first_run_on_sberquad-02/best.pth.tar --name best_evaluation_experiment\n", - "```\n", - "The result should be similar to the following:\n", - "```\n", - ">>> Dev NLL: 02.47, F1: 75.62, EM: 55.73, AvNA: 99.42\n", - "```\n", - "\n", - "The [DeepPavlov's RuBERT](http://docs.deeppavlov.ai/en/master/features/models/squad.html) achieves $F1 = 84.60\\pm0.11$ and $EM = 66.30\\pm0.24$" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Here comes your quest: try to improve the quality of this QA system. \n", - "\n", - "This is a very creative assignment. It is all about experimenting, trying different approaches (and a lot of computations). But if you wish to stick to some numbers, try to increase F1 at least by $5$ points.\n", - "\n", - "Here are some ideas that might help you on your way:\n", - "* Try adapting the optimization hyperparameters/network structure to Russian language (the baseline is designed for English SQuAD dataset).\n", - "* Incorporating the additional information about the data (like PoS tags) might be a good idea.\n", - "* __Distilling the knowledge from a pre-trained RuBERT__ (e.g. try to use the predictions of the model we've discussed on `week10` as soft targets).\n", - "* Or anything else.\n", - "\n", - "\n", - "And, first of all, read the initial code carefully.\n", - "\n", - "\n", - "Good luck! Feel free to share your results :)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Py3 Research", - "language": "python", - "name": "py3_research_kernel" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/homeworks/lab02_qa/args.py b/homeworks/lab02_qa/args.py deleted file mode 100644 index 47a59d1..0000000 --- a/homeworks/lab02_qa/args.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Command-line arguments for setup.py, train.py, test.py. - -Author: - Chris Chute (chute@stanford.edu) -""" - -import argparse - - -def get_setup_args(bypass=False): - """Get arguments needed in setup.py.""" - parser = argparse.ArgumentParser('Download and pre-process SQuAD') - - add_common_args(parser) - - parser.add_argument('--train_url', - type=str, - default='https://github.com/chrischute/squad/data/train-v2.0.json') - parser.add_argument('--dev_url', - type=str, - default='https://github.com/chrischute/squad/data/dev-v2.0.json') - parser.add_argument('--test_url', - type=str, - default='https://github.com/chrischute/squad/data/test-v2.0.json') - parser.add_argument('--glove_url', - type=str, - default='http://nlp.stanford.edu/data/glove.840B.300d.zip') - parser.add_argument('--dev_meta_file', - type=str, - default='./data/dev_meta.json') - parser.add_argument('--test_meta_file', - type=str, - default='./data/test_meta.json') - parser.add_argument('--word2idx_file', - type=str, - default='./data/word2idx.json') - parser.add_argument('--char2idx_file', - type=str, - default='./data/char2idx.json') - parser.add_argument('--answer_file', - type=str, - default='./data/answer.json') - parser.add_argument('--para_limit', - type=int, - default=400, - help='Max number of words in a paragraph') - parser.add_argument('--ques_limit', - type=int, - default=50, - help='Max number of words to keep from a question') - parser.add_argument('--test_para_limit', - type=int, - default=1000, - help='Max number of words in a paragraph at test time') - parser.add_argument('--test_ques_limit', - type=int, - default=100, - help='Max number of words in a question at test time') - parser.add_argument('--char_dim', - type=int, - default=64, - help='Size of char vectors (char-level embeddings)') - parser.add_argument('--glove_dim', - type=int, - default=300, - help='Size of GloVe word vectors to use') - parser.add_argument('--glove_num_vecs', - type=int, - default=2196017, - help='Number of GloVe vectors') - parser.add_argument('--ans_limit', - type=int, - default=30, - help='Max number of words in a training example answer') - parser.add_argument('--char_limit', - type=int, - default=16, - help='Max number of chars to keep from a word') - parser.add_argument('--include_test_examples', - type=lambda s: s.lower().startswith('t'), - default=True, - help='Process examples from the test set') - - if bypass: - args = parser.parse_args('') - else: - args = parser.parse_args() - - return args - - -def get_train_args(): - """Get arguments needed in train.py.""" - parser = argparse.ArgumentParser('Train a model on SQuAD') - - add_common_args(parser) - add_train_test_args(parser) - - parser.add_argument('--eval_steps', - type=int, - default=50000, - help='Number of steps between successive evaluations.') - parser.add_argument('--lr', - type=float, - default=0.5, - help='Learning rate.') - parser.add_argument('--l2_wd', - type=float, - default=0, - help='L2 weight decay.') - parser.add_argument('--num_epochs', - type=int, - default=30, - help='Number of epochs for which to train. Negative means forever.') - parser.add_argument('--drop_prob', - type=float, - default=0.2, - help='Probability of zeroing an activation in dropout layers.') - parser.add_argument('--metric_name', - type=str, - default='F1', - choices=('NLL', 'EM', 'F1'), - help='Name of dev metric to determine best checkpoint.') - parser.add_argument('--max_checkpoints', - type=int, - default=5, - help='Maximum number of checkpoints to keep on disk.') - parser.add_argument('--max_grad_norm', - type=float, - default=5.0, - help='Maximum gradient norm for gradient clipping.') - parser.add_argument('--seed', - type=int, - default=224, - help='Random seed for reproducibility.') - parser.add_argument('--ema_decay', - type=float, - default=0.999, - help='Decay rate for exponential moving average of parameters.') - - args = parser.parse_args() - - if args.metric_name == 'NLL': - # Best checkpoint is the one that minimizes negative log-likelihood - args.maximize_metric = False - elif args.metric_name in ('EM', 'F1'): - # Best checkpoint is the one that maximizes EM or F1 - args.maximize_metric = True - else: - raise ValueError(f'Unrecognized metric name: "{args.metric_name}"') - - return args - - -def get_test_args(): - """Get arguments needed in test.py.""" - parser = argparse.ArgumentParser('Test a trained model on SQuAD') - - add_common_args(parser) - add_train_test_args(parser) - - parser.add_argument('--split', - type=str, - default='dev', - choices=('train', 'dev', 'test'), - help='Split to use for testing.') - parser.add_argument('--sub_file', - type=str, - default='submission.csv', - help='Name for submission file.') - - # Require load_path for test.py - args = parser.parse_args() - if not args.load_path: - raise argparse.ArgumentError('Missing required argument --load_path') - - return args - - -def add_common_args(parser): - """Add arguments common to all 3 scripts: setup.py, train.py, test.py""" - parser.add_argument('--train_record_file', - type=str, - default='./data/train.npz') - parser.add_argument('--dev_record_file', - type=str, - default='./data/dev.npz') - parser.add_argument('--test_record_file', - type=str, - default='./data/test.npz') - parser.add_argument('--word_emb_file', - type=str, - default='./data/word_emb.json') - parser.add_argument('--char_emb_file', - type=str, - default='./data/char_emb.json') - parser.add_argument('--train_eval_file', - type=str, - default='./data/train_eval.json') - parser.add_argument('--dev_eval_file', - type=str, - default='./data/dev_eval.json') - parser.add_argument('--test_eval_file', - type=str, - default='./data/test_eval.json') - - -def add_train_test_args(parser): - """Add arguments common to train.py and test.py""" - parser.add_argument('--name', - '-n', - type=str, - required=True, - help='Name to identify training or test run.') - parser.add_argument('--max_ans_len', - type=int, - default=15, - help='Maximum length of a predicted answer.') - parser.add_argument('--num_workers', - type=int, - default=4, - help='Number of sub-processes to use per data loader.') - parser.add_argument('--save_dir', - type=str, - default='./save/', - help='Base directory for saving information.') - parser.add_argument('--batch_size', - type=int, - default=64, - help='Batch size per GPU. Scales automatically when \ - multiple GPUs are available.') - parser.add_argument('--use_squad_v2', - type=lambda s: s.lower().startswith('t'), - default=True, - help='Whether to use SQuAD 2.0 (unanswerable) questions.') - parser.add_argument('--hidden_size', - type=int, - default=100, - help='Number of features in encoder hidden layers.') - parser.add_argument('--num_visuals', - type=int, - default=10, - help='Number of examples to visualize in TensorBoard.') - parser.add_argument('--load_path', - type=str, - default=None, - help='Path to load as a model checkpoint.') diff --git a/homeworks/lab02_qa/layers.py b/homeworks/lab02_qa/layers.py deleted file mode 100644 index 6859e4d..0000000 --- a/homeworks/lab02_qa/layers.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Assortment of layers for use in models.py. - -Author: - Chris Chute (chute@stanford.edu) -""" - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence -from util import masked_softmax - - -class Embedding(nn.Module): - """Embedding layer used by BiDAF, without the character-level component. - - Word-level embeddings are further refined using a 2-layer Highway Encoder - (see `HighwayEncoder` class for details). - - Args: - word_vectors (torch.Tensor): Pre-trained word vectors. - hidden_size (int): Size of hidden activations. - drop_prob (float): Probability of zero-ing out activations - """ - def __init__(self, word_vectors, hidden_size, drop_prob): - super(Embedding, self).__init__() - self.drop_prob = drop_prob - self.embed = nn.Embedding.from_pretrained(word_vectors) - self.proj = nn.Linear(word_vectors.size(1), hidden_size, bias=False) - self.hwy = HighwayEncoder(2, hidden_size) - - def forward(self, x): - emb = self.embed(x) # (batch_size, seq_len, embed_size) - emb = F.dropout(emb, self.drop_prob, self.training) - emb = self.proj(emb) # (batch_size, seq_len, hidden_size) - emb = self.hwy(emb) # (batch_size, seq_len, hidden_size) - - return emb - - -class HighwayEncoder(nn.Module): - """Encode an input sequence using a highway network. - - Based on the paper: - "Highway Networks" - by Rupesh Kumar Srivastava, Klaus Greff, Jürgen Schmidhuber - (https://arxiv.org/abs/1505.00387). - - Args: - num_layers (int): Number of layers in the highway encoder. - hidden_size (int): Size of hidden activations. - """ - def __init__(self, num_layers, hidden_size): - super(HighwayEncoder, self).__init__() - self.transforms = nn.ModuleList([nn.Linear(hidden_size, hidden_size) - for _ in range(num_layers)]) - self.gates = nn.ModuleList([nn.Linear(hidden_size, hidden_size) - for _ in range(num_layers)]) - - def forward(self, x): - for gate, transform in zip(self.gates, self.transforms): - # Shapes of g, t, and x are all (batch_size, seq_len, hidden_size) - g = torch.sigmoid(gate(x)) - t = F.relu(transform(x)) - x = g * t + (1 - g) * x - - return x - - -class RNNEncoder(nn.Module): - """General-purpose layer for encoding a sequence using a bidirectional RNN. - - Encoded output is the RNN's hidden state at each position, which - has shape `(batch_size, seq_len, hidden_size * 2)`. - - Args: - input_size (int): Size of a single timestep in the input. - hidden_size (int): Size of the RNN hidden state. - num_layers (int): Number of layers of RNN cells to use. - drop_prob (float): Probability of zero-ing out activations. - """ - def __init__(self, - input_size, - hidden_size, - num_layers, - drop_prob=0.): - super(RNNEncoder, self).__init__() - self.drop_prob = drop_prob - self.rnn = nn.LSTM(input_size, hidden_size, num_layers, - batch_first=True, - bidirectional=True, - dropout=drop_prob if num_layers > 1 else 0.) - - def forward(self, x, lengths): - # Save original padded length for use by pad_packed_sequence - orig_len = x.size(1) - - # Sort by length and pack sequence for RNN - lengths, sort_idx = lengths.sort(0, descending=True) - x = x[sort_idx] # (batch_size, seq_len, input_size) - x = pack_padded_sequence(x, lengths, batch_first=True) - - # Apply RNN - x, _ = self.rnn(x) # (batch_size, seq_len, 2 * hidden_size) - - # Unpack and reverse sort - x, _ = pad_packed_sequence(x, batch_first=True, total_length=orig_len) - _, unsort_idx = sort_idx.sort(0) - x = x[unsort_idx] # (batch_size, seq_len, 2 * hidden_size) - - # Apply dropout (RNN applies dropout after all but the last layer) - x = F.dropout(x, self.drop_prob, self.training) - - return x - - -class BiDAFAttention(nn.Module): - """Bidirectional attention originally used by BiDAF. - - Bidirectional attention computes attention in two directions: - The context attends to the query and the query attends to the context. - The output of this layer is the concatenation of [context, c2q_attention, - context * c2q_attention, context * q2c_attention]. This concatenation allows - the attention vector at each timestep, along with the embeddings from - previous layers, to flow through the attention layer to the modeling layer. - The output has shape (batch_size, context_len, 8 * hidden_size). - - Args: - hidden_size (int): Size of hidden activations. - drop_prob (float): Probability of zero-ing out activations. - """ - def __init__(self, hidden_size, drop_prob=0.1): - super(BiDAFAttention, self).__init__() - self.drop_prob = drop_prob - self.c_weight = nn.Parameter(torch.zeros(hidden_size, 1)) - self.q_weight = nn.Parameter(torch.zeros(hidden_size, 1)) - self.cq_weight = nn.Parameter(torch.zeros(1, 1, hidden_size)) - for weight in (self.c_weight, self.q_weight, self.cq_weight): - nn.init.xavier_uniform_(weight) - self.bias = nn.Parameter(torch.zeros(1)) - - def forward(self, c, q, c_mask, q_mask): - batch_size, c_len, _ = c.size() - q_len = q.size(1) - s = self.get_similarity_matrix(c, q) # (batch_size, c_len, q_len) - c_mask = c_mask.view(batch_size, c_len, 1) # (batch_size, c_len, 1) - q_mask = q_mask.view(batch_size, 1, q_len) # (batch_size, 1, q_len) - s1 = masked_softmax(s, q_mask, dim=2) # (batch_size, c_len, q_len) - s2 = masked_softmax(s, c_mask, dim=1) # (batch_size, c_len, q_len) - - # (bs, c_len, q_len) x (bs, q_len, hid_size) => (bs, c_len, hid_size) - a = torch.bmm(s1, q) - # (bs, c_len, c_len) x (bs, c_len, hid_size) => (bs, c_len, hid_size) - b = torch.bmm(torch.bmm(s1, s2.transpose(1, 2)), c) - - x = torch.cat([c, a, c * a, c * b], dim=2) # (bs, c_len, 4 * hid_size) - - return x - - def get_similarity_matrix(self, c, q): - """Get the "similarity matrix" between context and query (using the - terminology of the BiDAF paper). - - A naive implementation as described in BiDAF would concatenate the - three vectors then project the result with a single weight matrix. This - method is a more memory-efficient implementation of the same operation. - - See Also: - Equation 1 in https://arxiv.org/abs/1611.01603 - """ - c_len, q_len = c.size(1), q.size(1) - c = F.dropout(c, self.drop_prob, self.training) # (bs, c_len, hid_size) - q = F.dropout(q, self.drop_prob, self.training) # (bs, q_len, hid_size) - - # Shapes: (batch_size, c_len, q_len) - s0 = torch.matmul(c, self.c_weight).expand([-1, -1, q_len]) - s1 = torch.matmul(q, self.q_weight).transpose(1, 2)\ - .expand([-1, c_len, -1]) - s2 = torch.matmul(c * self.cq_weight, q.transpose(1, 2)) - s = s0 + s1 + s2 + self.bias - - return s - - -class BiDAFOutput(nn.Module): - """Output layer used by BiDAF for question answering. - - Computes a linear transformation of the attention and modeling - outputs, then takes the softmax of the result to get the start pointer. - A bidirectional LSTM is then applied the modeling output to produce `mod_2`. - A second linear+softmax of the attention output and `mod_2` is used - to get the end pointer. - - Args: - hidden_size (int): Hidden size used in the BiDAF model. - drop_prob (float): Probability of zero-ing out activations. - """ - def __init__(self, hidden_size, drop_prob): - super(BiDAFOutput, self).__init__() - self.att_linear_1 = nn.Linear(8 * hidden_size, 1) - self.mod_linear_1 = nn.Linear(2 * hidden_size, 1) - - self.rnn = RNNEncoder(input_size=2 * hidden_size, - hidden_size=hidden_size, - num_layers=1, - drop_prob=drop_prob) - - self.att_linear_2 = nn.Linear(8 * hidden_size, 1) - self.mod_linear_2 = nn.Linear(2 * hidden_size, 1) - - def forward(self, att, mod, mask): - # Shapes: (batch_size, seq_len, 1) - logits_1 = self.att_linear_1(att) + self.mod_linear_1(mod) - mod_2 = self.rnn(mod, mask.sum(-1)) - logits_2 = self.att_linear_2(att) + self.mod_linear_2(mod_2) - - # Shapes: (batch_size, seq_len) - log_p1 = masked_softmax(logits_1.squeeze(), mask, log_softmax=True) - log_p2 = masked_softmax(logits_2.squeeze(), mask, log_softmax=True) - - return log_p1, log_p2 diff --git a/homeworks/lab02_qa/models.py b/homeworks/lab02_qa/models.py deleted file mode 100644 index 3487ea2..0000000 --- a/homeworks/lab02_qa/models.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Top-level model classes. - -Author: - Chris Chute (chute@stanford.edu) -""" - -import layers -import torch -import torch.nn as nn - - -class BiDAF(nn.Module): - """Baseline BiDAF model for SQuAD. - - Based on the paper: - "Bidirectional Attention Flow for Machine Comprehension" - by Minjoon Seo, Aniruddha Kembhavi, Ali Farhadi, Hannaneh Hajishirzi - (https://arxiv.org/abs/1611.01603). - - Follows a high-level structure commonly found in SQuAD models: - - Embedding layer: Embed word indices to get word vectors. - - Encoder layer: Encode the embedded sequence. - - Attention layer: Apply an attention mechanism to the encoded sequence. - - Model encoder layer: Encode the sequence again. - - Output layer: Simple layer (e.g., fc + softmax) to get final outputs. - - Args: - word_vectors (torch.Tensor): Pre-trained word vectors. - hidden_size (int): Number of features in the hidden state at each layer. - drop_prob (float): Dropout probability. - """ - def __init__(self, word_vectors, hidden_size, drop_prob=0.): - super(BiDAF, self).__init__() - self.emb = layers.Embedding(word_vectors=word_vectors, - hidden_size=hidden_size, - drop_prob=drop_prob) - - self.enc = layers.RNNEncoder(input_size=hidden_size, - hidden_size=hidden_size, - num_layers=1, - drop_prob=drop_prob) - - self.att = layers.BiDAFAttention(hidden_size=2 * hidden_size, - drop_prob=drop_prob) - - self.mod = layers.RNNEncoder(input_size=8 * hidden_size, - hidden_size=hidden_size, - num_layers=2, - drop_prob=drop_prob) - - self.out = layers.BiDAFOutput(hidden_size=hidden_size, - drop_prob=drop_prob) - - def forward(self, cw_idxs, qw_idxs): - c_mask = torch.zeros_like(cw_idxs) != cw_idxs - q_mask = torch.zeros_like(qw_idxs) != qw_idxs - c_len, q_len = c_mask.sum(-1), q_mask.sum(-1) - - c_emb = self.emb(cw_idxs) # (batch_size, c_len, hidden_size) - q_emb = self.emb(qw_idxs) # (batch_size, q_len, hidden_size) - - c_enc = self.enc(c_emb, c_len) # (batch_size, c_len, 2 * hidden_size) - q_enc = self.enc(q_emb, q_len) # (batch_size, q_len, 2 * hidden_size) - - att = self.att(c_enc, q_enc, - c_mask, q_mask) # (batch_size, c_len, 8 * hidden_size) - - mod = self.mod(att, c_len) # (batch_size, c_len, 2 * hidden_size) - - out = self.out(att, mod, c_mask) # 2 tensors, each (batch_size, c_len) - - return out diff --git a/homeworks/lab02_qa/setup.py b/homeworks/lab02_qa/setup.py deleted file mode 100644 index c270cdf..0000000 --- a/homeworks/lab02_qa/setup.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Download and pre-process SQuAD and GloVe. - -Usage: - > source activate squad - > python setup.py - -Pre-processing code adapted from: - > https://github.com/HKUST-KnowComp/R-Net/blob/master/prepro.py - -Author: - Chris Chute (chute@stanford.edu) -""" - -import numpy as np -import os -import spacy -import ujson as json -import urllib.request - -from args import get_setup_args -from codecs import open -from collections import Counter -from subprocess import run -from tqdm import tqdm -from zipfile import ZipFile - - -def download_url(url, output_path, show_progress=True): - class DownloadProgressBar(tqdm): - def update_to(self, b=1, bsize=1, tsize=None): - if tsize is not None: - self.total = tsize - self.update(b * bsize - self.n) - - if show_progress: - # Download with a progress bar - with DownloadProgressBar(unit='B', unit_scale=True, - miniters=1, desc=url.split('/')[-1]) as t: - urllib.request.urlretrieve(url, - filename=output_path, - reporthook=t.update_to) - else: - # Simple download with no progress bar - urllib.request.urlretrieve(url, output_path) - - -def url_to_data_path(url): - return os.path.join('./data/', url.split('/')[-1]) - - -def download(args): - downloads = [ - # Can add other downloads here (e.g., other word vectors) - ('GloVe word vectors', args.glove_url), - ] - - for name, url in downloads: - output_path = url_to_data_path(url) - if not os.path.exists(output_path): - print(f'Downloading {name}...') - download_url(url, output_path) - - if os.path.exists(output_path) and output_path.endswith('.zip'): - extracted_path = output_path.replace('.zip', '') - if not os.path.exists(extracted_path): - print(f'Unzipping {name}...') - with ZipFile(output_path, 'r') as zip_fh: - zip_fh.extractall(extracted_path) - - print('Downloading spacy language model...') - run(['python', '-m', 'spacy', 'download', 'en']) - -def word_tokenize(sent, nlp): - doc = nlp(sent) - return [token.text for token in doc] - - -def convert_idx(text, tokens): - current = 0 - spans = [] - for token in tokens: - current = text.find(token, current) - if current < 0: - print(f"Token {token} cannot be found") - raise Exception() - spans.append((current, current + len(token))) - current += len(token) - return spans - - -def process_file(filename, data_type, word_counter, char_counter, nlp): - print(f"Pre-processing {data_type} examples...") - examples = [] - eval_examples = {} - total = 0 - with open(filename, "r") as fh: - source = json.load(fh) - for article in tqdm(source["data"]): - for para in article["paragraphs"]: - context = para["context"].replace( - "''", '" ').replace("``", '" ') - context_tokens = word_tokenize(context, nlp) - context_chars = [list(token) for token in context_tokens] - spans = convert_idx(context, context_tokens) - for token in context_tokens: - word_counter[token] += len(para["qas"]) - for char in token: - char_counter[char] += len(para["qas"]) - for qa in para["qas"]: - total += 1 - ques = qa["question"].replace( - "''", '" ').replace("``", '" ') - ques_tokens = word_tokenize(ques, nlp) - ques_chars = [list(token) for token in ques_tokens] - for token in ques_tokens: - word_counter[token] += 1 - for char in token: - char_counter[char] += 1 - y1s, y2s = [], [] - answer_texts = [] - for answer in qa["answers"]: - answer_text = answer["text"] - answer_start = answer['answer_start'] - answer_end = answer_start + len(answer_text) - answer_texts.append(answer_text) - answer_span = [] - for idx, span in enumerate(spans): - if not (answer_end <= span[0] or answer_start >= span[1]): - answer_span.append(idx) - y1, y2 = answer_span[0], answer_span[-1] - y1s.append(y1) - y2s.append(y2) - example = {"context_tokens": context_tokens, - "context_chars": context_chars, - "ques_tokens": ques_tokens, - "ques_chars": ques_chars, - "y1s": y1s, - "y2s": y2s, - "id": total} - examples.append(example) - eval_examples[str(total)] = {"context": context, - "question": ques, - "spans": spans, - "answers": answer_texts, - "uuid": qa["id"]} - print(f"{len(examples)} questions in total") - return examples, eval_examples - - -def get_embedding(counter, data_type, limit=-1, emb_file=None, vec_size=None, num_vectors=None): - print(f"Pre-processing {data_type} vectors...") - embedding_dict = {} - filtered_elements = [k for k, v in counter.items() if v > limit] - if emb_file is not None: - assert vec_size is not None - with open(emb_file, "r", encoding="utf-8") as fh: - for line in tqdm(fh, total=num_vectors): - array = line.split() - word = "".join(array[0:-vec_size]) - vector = list(map(float, array[-vec_size:])) - if word in counter and counter[word] > limit: - embedding_dict[word] = vector - print(f"{len(embedding_dict)} / {len(filtered_elements)} tokens have corresponding {data_type} embedding vector") - else: - assert vec_size is not None - for token in filtered_elements: - embedding_dict[token] = [np.random.normal( - scale=0.1) for _ in range(vec_size)] - print(f"{len(filtered_elements)} tokens have corresponding {data_type} embedding vector") - - NULL = "--NULL--" - OOV = "--OOV--" - token2idx_dict = {token: idx for idx, token in enumerate(embedding_dict.keys(), 2)} - token2idx_dict[NULL] = 0 - token2idx_dict[OOV] = 1 - embedding_dict[NULL] = [0. for _ in range(vec_size)] - embedding_dict[OOV] = [0. for _ in range(vec_size)] - idx2emb_dict = {idx: embedding_dict[token] - for token, idx in token2idx_dict.items()} - emb_mat = [idx2emb_dict[idx] for idx in range(len(idx2emb_dict))] - return emb_mat, token2idx_dict - - -def convert_to_features(args, data, word2idx_dict, char2idx_dict, is_test): - example = {} - context, question = data - context = context.replace("''", '" ').replace("``", '" ') - question = question.replace("''", '" ').replace("``", '" ') - example['context_tokens'] = word_tokenize(context) - example['ques_tokens'] = word_tokenize(question) - example['context_chars'] = [list(token) for token in example['context_tokens']] - example['ques_chars'] = [list(token) for token in example['ques_tokens']] - - para_limit = args.test_para_limit if is_test else args.para_limit - ques_limit = args.test_ques_limit if is_test else args.ques_limit - char_limit = args.char_limit - - def filter_func(example): - return len(example["context_tokens"]) > para_limit or \ - len(example["ques_tokens"]) > ques_limit - - if filter_func(example): - raise ValueError("Context/Questions lengths are over the limit") - - context_idxs = np.zeros([para_limit], dtype=np.int32) - context_char_idxs = np.zeros([para_limit, char_limit], dtype=np.int32) - ques_idxs = np.zeros([ques_limit], dtype=np.int32) - ques_char_idxs = np.zeros([ques_limit, char_limit], dtype=np.int32) - - def _get_word(word): - for each in (word, word.lower(), word.capitalize(), word.upper()): - if each in word2idx_dict: - return word2idx_dict[each] - return 1 - - def _get_char(char): - if char in char2idx_dict: - return char2idx_dict[char] - return 1 - - for i, token in enumerate(example["context_tokens"]): - context_idxs[i] = _get_word(token) - - for i, token in enumerate(example["ques_tokens"]): - ques_idxs[i] = _get_word(token) - - for i, token in enumerate(example["context_chars"]): - for j, char in enumerate(token): - if j == char_limit: - break - context_char_idxs[i, j] = _get_char(char) - - for i, token in enumerate(example["ques_chars"]): - for j, char in enumerate(token): - if j == char_limit: - break - ques_char_idxs[i, j] = _get_char(char) - - return context_idxs, context_char_idxs, ques_idxs, ques_char_idxs - - -def is_answerable(example): - return len(example['y2s']) > 0 and len(example['y1s']) > 0 - - -def build_features(args, examples, data_type, out_file, word2idx_dict, char2idx_dict, is_test=False): - para_limit = args.test_para_limit if is_test else args.para_limit - ques_limit = args.test_ques_limit if is_test else args.ques_limit - ans_limit = args.ans_limit - char_limit = args.char_limit - - def drop_example(ex, is_test_=False): - if is_test_: - drop = False - else: - drop = len(ex["context_tokens"]) > para_limit or \ - len(ex["ques_tokens"]) > ques_limit or \ - (is_answerable(ex) and - ex["y2s"][0] - ex["y1s"][0] > ans_limit) - - return drop - - print(f"Converting {data_type} examples to indices...") - total = 0 - total_ = 0 - meta = {} - context_idxs = [] - context_char_idxs = [] - ques_idxs = [] - ques_char_idxs = [] - y1s = [] - y2s = [] - ids = [] - for n, example in tqdm(enumerate(examples)): - total_ += 1 - - if drop_example(example, is_test): - continue - - total += 1 - - def _get_word(word): - for each in (word, word.lower(), word.capitalize(), word.upper()): - if each in word2idx_dict: - return word2idx_dict[each] - return 1 - - def _get_char(char): - if char in char2idx_dict: - return char2idx_dict[char] - return 1 - - context_idx = np.zeros([para_limit], dtype=np.int32) - context_char_idx = np.zeros([para_limit, char_limit], dtype=np.int32) - ques_idx = np.zeros([ques_limit], dtype=np.int32) - ques_char_idx = np.zeros([ques_limit, char_limit], dtype=np.int32) - - for i, token in enumerate(example["context_tokens"]): - context_idx[i] = _get_word(token) - context_idxs.append(context_idx) - - for i, token in enumerate(example["ques_tokens"]): - ques_idx[i] = _get_word(token) - ques_idxs.append(ques_idx) - - for i, token in enumerate(example["context_chars"]): - for j, char in enumerate(token): - if j == char_limit: - break - context_char_idx[i, j] = _get_char(char) - context_char_idxs.append(context_char_idx) - - for i, token in enumerate(example["ques_chars"]): - for j, char in enumerate(token): - if j == char_limit: - break - ques_char_idx[i, j] = _get_char(char) - ques_char_idxs.append(ques_char_idx) - - if is_answerable(example): - start, end = example["y1s"][-1], example["y2s"][-1] - else: - start, end = -1, -1 - - y1s.append(start) - y2s.append(end) - ids.append(example["id"]) - - np.savez(out_file, - context_idxs=np.array(context_idxs), - context_char_idxs=np.array(context_char_idxs), - ques_idxs=np.array(ques_idxs), - ques_char_idxs=np.array(ques_char_idxs), - y1s=np.array(y1s), - y2s=np.array(y2s), - ids=np.array(ids)) - print(f"Built {total} / {total_} instances of features in total") - meta["total"] = total - return meta - - -def save(filename, obj, message=None): - if message is not None: - print(f"Saving {message}...") - with open(filename, "w") as fh: - json.dump(obj, fh) - - -def pre_process(args): - # Process training set and use it to decide on the word/character vocabularies - word_counter, char_counter = Counter(), Counter() - train_examples, train_eval = process_file(args.train_file, "train", word_counter, char_counter) - word_emb_mat, word2idx_dict = get_embedding( - word_counter, 'word', emb_file=args.glove_file, vec_size=args.glove_dim, num_vectors=args.glove_num_vecs) - char_emb_mat, char2idx_dict = get_embedding( - char_counter, 'char', emb_file=None, vec_size=args.char_dim) - - # Process dev and test sets - dev_examples, dev_eval = process_file(args.dev_file, "dev", word_counter, char_counter) - build_features(args, train_examples, "train", args.train_record_file, word2idx_dict, char2idx_dict) - dev_meta = build_features(args, dev_examples, "dev", args.dev_record_file, word2idx_dict, char2idx_dict) - if args.include_test_examples: - test_examples, test_eval = process_file(args.test_file, "test", word_counter, char_counter) - save(args.test_eval_file, test_eval, message="test eval") - test_meta = build_features(args, test_examples, "test", - args.test_record_file, word2idx_dict, char2idx_dict, is_test=True) - save(args.test_meta_file, test_meta, message="test meta") - - save(args.word_emb_file, word_emb_mat, message="word embedding") - save(args.char_emb_file, char_emb_mat, message="char embedding") - save(args.train_eval_file, train_eval, message="train eval") - save(args.dev_eval_file, dev_eval, message="dev eval") - save(args.word2idx_file, word2idx_dict, message="word dictionary") - save(args.char2idx_file, char2idx_dict, message="char dictionary") - save(args.dev_meta_file, dev_meta, message="dev meta") - - -if __name__ == '__main__': - # Get command-line args - args_ = get_setup_args() - - # Download resources - download(args_) - - # Import spacy language model - nlp = spacy.blank("en") - - # Preprocess dataset - args_.train_file = url_to_data_path(args_.train_url) - args_.dev_file = url_to_data_path(args_.dev_url) - if args_.include_test_examples: - args_.test_file = url_to_data_path(args_.test_url) - glove_dir = url_to_data_path(args_.glove_url.replace('.zip', '')) - glove_ext = f'.txt' if glove_dir.endswith('d') else f'.{args_.glove_dim}d.txt' - args_.glove_file = os.path.join(glove_dir, os.path.basename(glove_dir) + glove_ext) - pre_process(args_) diff --git a/homeworks/lab02_qa/test.py b/homeworks/lab02_qa/test.py deleted file mode 100644 index 745fa36..0000000 --- a/homeworks/lab02_qa/test.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Test a model and generate submission CSV. - -Usage: - > python test.py --split SPLIT --load_path PATH --name NAME - where - > SPLIT is either "dev" or "test" - > PATH is a path to a checkpoint (e.g., save/train/model-01/best.pth.tar) - > NAME is a name to identify the test run - -Author: - Chris Chute (chute@stanford.edu) -""" - -import csv -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.utils.data as data -import util - -from args import get_test_args -from collections import OrderedDict -from json import dumps -from models import BiDAF -from os.path import join -from tensorboardX import SummaryWriter -from tqdm import tqdm -from ujson import load as json_load -from util import collate_fn, SQuAD - - -def main(args): - # Set up logging - args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False) - log = util.get_logger(args.save_dir, args.name) - log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}') - device, gpu_ids = util.get_available_devices() - args.batch_size *= max(1, len(gpu_ids)) - - # Get embeddings - log.info('Loading embeddings...') - word_vectors = util.torch_from_json(args.word_emb_file) - - # Get model - log.info('Building model...') - model = BiDAF(word_vectors=word_vectors, - hidden_size=args.hidden_size) - model = nn.DataParallel(model, gpu_ids) - log.info(f'Loading checkpoint from {args.load_path}...') - model = util.load_model(model, args.load_path, gpu_ids, return_step=False) - model = model.to(device) - model.eval() - - # Get data loader - log.info('Building dataset...') - record_file = vars(args)[f'{args.split}_record_file'] - dataset = SQuAD(record_file, args.use_squad_v2) - data_loader = data.DataLoader(dataset, - batch_size=args.batch_size, - shuffle=False, - num_workers=args.num_workers, - collate_fn=collate_fn) - - # Evaluate - log.info(f'Evaluating on {args.split} split...') - nll_meter = util.AverageMeter() - pred_dict = {} # Predictions for TensorBoard - sub_dict = {} # Predictions for submission - eval_file = vars(args)[f'{args.split}_eval_file'] - with open(eval_file, 'r') as fh: - gold_dict = json_load(fh) - with torch.no_grad(), \ - tqdm(total=len(dataset)) as progress_bar: - for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in data_loader: - # Setup for forward - cw_idxs = cw_idxs.to(device) - qw_idxs = qw_idxs.to(device) - batch_size = cw_idxs.size(0) - - # Forward - log_p1, log_p2 = model(cw_idxs, qw_idxs) - y1, y2 = y1.to(device), y2.to(device) - loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2) - nll_meter.update(loss.item(), batch_size) - - # Get F1 and EM scores - p1, p2 = log_p1.exp(), log_p2.exp() - starts, ends = util.discretize(p1, p2, args.max_ans_len, args.use_squad_v2) - - # Log info - progress_bar.update(batch_size) - if args.split != 'test': - # No labels for the test set, so NLL would be invalid - progress_bar.set_postfix(NLL=nll_meter.avg) - - idx2pred, uuid2pred = util.convert_tokens(gold_dict, - ids.tolist(), - starts.tolist(), - ends.tolist(), - args.use_squad_v2) - pred_dict.update(idx2pred) - sub_dict.update(uuid2pred) - - # Log results (except for test set, since it does not come with labels) - if args.split != 'test': - results = util.eval_dicts(gold_dict, pred_dict, args.use_squad_v2) - results_list = [('NLL', nll_meter.avg), - ('F1', results['F1']), - ('EM', results['EM'])] - if args.use_squad_v2: - results_list.append(('AvNA', results['AvNA'])) - results = OrderedDict(results_list) - - # Log to console - results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items()) - log.info(f'{args.split.title()} {results_str}') - - # Log to TensorBoard - tbx = SummaryWriter(args.save_dir) - util.visualize(tbx, - pred_dict=pred_dict, - eval_path=eval_file, - step=0, - split=args.split, - num_visuals=args.num_visuals) - - # Write submission file - sub_path = join(args.save_dir, args.split + '_' + args.sub_file) - log.info(f'Writing submission file to {sub_path}...') - with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh: - csv_writer = csv.writer(csv_fh, delimiter=',') - csv_writer.writerow(['Id', 'Predicted']) - for uuid in sorted(sub_dict): - csv_writer.writerow([uuid, sub_dict[uuid]]) - - -if __name__ == '__main__': - main(get_test_args()) diff --git a/homeworks/lab02_qa/train.py b/homeworks/lab02_qa/train.py deleted file mode 100644 index 42e4265..0000000 --- a/homeworks/lab02_qa/train.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Train a model on SQuAD. - -Author: - Chris Chute (chute@stanford.edu) -""" - -import numpy as np -import random -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -import torch.optim.lr_scheduler as sched -import torch.utils.data as data -import util - -from args import get_train_args -from collections import OrderedDict -from json import dumps -from models import BiDAF -from tensorboardX import SummaryWriter -from tqdm import tqdm -from ujson import load as json_load -from util import collate_fn, SQuAD - - -def main(args): - # Set up logging and devices - args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True) - log = util.get_logger(args.save_dir, args.name) - tbx = SummaryWriter(args.save_dir) - - import warnings - warnings.filterwarnings('ignore') - - device, args.gpu_ids = util.get_available_devices() - log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}') - args.batch_size *= max(1, len(args.gpu_ids)) - - # Set random seed - log.info(f'Using random seed {args.seed}...') - random.seed(args.seed) - np.random.seed(args.seed) - torch.manual_seed(args.seed) - torch.cuda.manual_seed_all(args.seed) - - # Get embeddings - log.info('Loading embeddings...') - word_vectors = util.torch_from_json(args.word_emb_file) - - # Get model - log.info('Building model...') - model = BiDAF(word_vectors=word_vectors, - hidden_size=args.hidden_size, - drop_prob=args.drop_prob) - model = nn.DataParallel(model, args.gpu_ids) - if args.load_path: - log.info(f'Loading checkpoint from {args.load_path}...') - model, step = util.load_model(model, args.load_path, args.gpu_ids) - else: - step = 0 - model = model.to(device) - model.train() - ema = util.EMA(model, args.ema_decay) - - # Get saver - saver = util.CheckpointSaver(args.save_dir, - max_checkpoints=args.max_checkpoints, - metric_name=args.metric_name, - maximize_metric=args.maximize_metric, - log=log) - - # Get optimizer and scheduler - optimizer = optim.Adadelta(model.parameters(), args.lr, - weight_decay=args.l2_wd) - scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR - - # Get data loader - log.info('Building dataset...') - train_dataset = SQuAD(args.train_record_file, args.use_squad_v2) - train_loader = data.DataLoader(train_dataset, - batch_size=args.batch_size, - shuffle=True, - num_workers=args.num_workers, - collate_fn=collate_fn) - dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2) - dev_loader = data.DataLoader(dev_dataset, - batch_size=args.batch_size, - shuffle=False, - num_workers=args.num_workers, - collate_fn=collate_fn) - - # Train - log.info('Training...') - steps_till_eval = args.eval_steps - epoch = step // len(train_dataset) - while epoch != args.num_epochs: - epoch += 1 - log.info(f'Starting epoch {epoch}...') - with torch.enable_grad(), \ - tqdm(total=len(train_loader.dataset)) as progress_bar: - for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader: - # Setup for forward - cw_idxs = cw_idxs.to(device) - qw_idxs = qw_idxs.to(device) - batch_size = cw_idxs.size(0) - optimizer.zero_grad() - - # Forward - log_p1, log_p2 = model(cw_idxs, qw_idxs) - y1, y2 = y1.to(device), y2.to(device) - loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2) - loss_val = loss.item() - - # Backward - loss.backward() - nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) - optimizer.step() - scheduler.step(step // batch_size) - ema(model, step // batch_size) - - # Log info - step += batch_size - progress_bar.update(batch_size) - progress_bar.set_postfix(epoch=epoch, - NLL=loss_val) - tbx.add_scalar('train/NLL', loss_val, step) - tbx.add_scalar('train/LR', - optimizer.param_groups[0]['lr'], - step) - - steps_till_eval -= batch_size - if steps_till_eval <= 0: - steps_till_eval = args.eval_steps - - # Evaluate and save checkpoint - log.info(f'Evaluating at step {step}...') - ema.assign(model) - results, pred_dict = evaluate(model, dev_loader, device, - args.dev_eval_file, - args.max_ans_len, - args.use_squad_v2) - saver.save(step, model, results[args.metric_name], device) - ema.resume(model) - - # Log to console - results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items()) - log.info(f'Dev {results_str}') - - # Log to TensorBoard - log.info('Visualizing in TensorBoard...') - for k, v in results.items(): - tbx.add_scalar(f'dev/{k}', v, step) - util.visualize(tbx, - pred_dict=pred_dict, - eval_path=args.dev_eval_file, - step=step, - split='dev', - num_visuals=args.num_visuals) - - -def evaluate(model, data_loader, device, eval_file, max_len, use_squad_v2): - nll_meter = util.AverageMeter() - - model.eval() - pred_dict = {} - with open(eval_file, 'r') as fh: - gold_dict = json_load(fh) - with torch.no_grad(), \ - tqdm(total=len(data_loader.dataset)) as progress_bar: - for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in data_loader: - # Setup for forward - cw_idxs = cw_idxs.to(device) - qw_idxs = qw_idxs.to(device) - batch_size = cw_idxs.size(0) - - # Forward - log_p1, log_p2 = model(cw_idxs, qw_idxs) - y1, y2 = y1.to(device), y2.to(device) - loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2) - nll_meter.update(loss.item(), batch_size) - - # Get F1 and EM scores - p1, p2 = log_p1.exp(), log_p2.exp() - starts, ends = util.discretize(p1, p2, max_len, use_squad_v2) - - # Log info - progress_bar.update(batch_size) - progress_bar.set_postfix(NLL=nll_meter.avg) - - preds, _ = util.convert_tokens(gold_dict, - ids.tolist(), - starts.tolist(), - ends.tolist(), - use_squad_v2) - pred_dict.update(preds) - - model.train() - - results = util.eval_dicts(gold_dict, pred_dict, use_squad_v2) - results_list = [('NLL', nll_meter.avg), - ('F1', results['F1']), - ('EM', results['EM'])] - if use_squad_v2: - results_list.append(('AvNA', results['AvNA'])) - results = OrderedDict(results_list) - - return results, pred_dict - - -if __name__ == '__main__': - main(get_train_args()) diff --git a/homeworks/lab02_qa/util.py b/homeworks/lab02_qa/util.py deleted file mode 100644 index 1cad0bb..0000000 --- a/homeworks/lab02_qa/util.py +++ /dev/null @@ -1,725 +0,0 @@ -"""Utility classes and methods. - -Author: - Chris Chute (chute@stanford.edu) -""" -import logging -import os -import queue -import re -import shutil -import string -import torch -import torch.nn.functional as F -import torch.utils.data as data -import tqdm -import numpy as np -import ujson as json - -from collections import Counter - - -class SQuAD(data.Dataset): - """Stanford Question Answering Dataset (SQuAD). - - Each item in the dataset is a tuple with the following entries (in order): - - context_idxs: Indices of the words in the context. - Shape (context_len,). - - context_char_idxs: Indices of the characters in the context. - Shape (context_len, max_word_len). - - question_idxs: Indices of the words in the question. - Shape (question_len,). - - question_char_idxs: Indices of the characters in the question. - Shape (question_len, max_word_len). - - y1: Index of word in the context where the answer begins. - -1 if no answer. - - y2: Index of word in the context where the answer ends. - -1 if no answer. - - id: ID of the example. - - Args: - data_path (str): Path to .npz file containing pre-processed dataset. - use_v2 (bool): Whether to use SQuAD 2.0 questions. Otherwise only use SQuAD 1.1. - """ - def __init__(self, data_path, use_v2=True): - super(SQuAD, self).__init__() - - dataset = np.load(data_path) - self.context_idxs = torch.from_numpy(dataset['context_idxs']).long() - self.context_char_idxs = torch.from_numpy(dataset['context_char_idxs']).long() - self.question_idxs = torch.from_numpy(dataset['ques_idxs']).long() - self.question_char_idxs = torch.from_numpy(dataset['ques_char_idxs']).long() - self.y1s = torch.from_numpy(dataset['y1s']).long() - self.y2s = torch.from_numpy(dataset['y2s']).long() - - if use_v2: - # SQuAD 2.0: Use index 0 for no-answer token (token 1 = OOV) - batch_size, c_len, w_len = self.context_char_idxs.size() - ones = torch.ones((batch_size, 1), dtype=torch.int64) - self.context_idxs = torch.cat((ones, self.context_idxs), dim=1) - self.question_idxs = torch.cat((ones, self.question_idxs), dim=1) - - ones = torch.ones((batch_size, 1, w_len), dtype=torch.int64) - self.context_char_idxs = torch.cat((ones, self.context_char_idxs), dim=1) - self.question_char_idxs = torch.cat((ones, self.question_char_idxs), dim=1) - - self.y1s += 1 - self.y2s += 1 - - # SQuAD 1.1: Ignore no-answer examples - self.ids = torch.from_numpy(dataset['ids']).long() - self.valid_idxs = [idx for idx in range(len(self.ids)) - if use_v2 or self.y1s[idx].item() >= 0] - - def __getitem__(self, idx): - idx = self.valid_idxs[idx] - example = (self.context_idxs[idx], - self.context_char_idxs[idx], - self.question_idxs[idx], - self.question_char_idxs[idx], - self.y1s[idx], - self.y2s[idx], - self.ids[idx]) - - return example - - def __len__(self): - return len(self.valid_idxs) - - -def collate_fn(examples): - """Create batch tensors from a list of individual examples returned - by `SQuAD.__getitem__`. Merge examples of different length by padding - all examples to the maximum length in the batch. - - Args: - examples (list): List of tuples of the form (context_idxs, context_char_idxs, - question_idxs, question_char_idxs, y1s, y2s, ids). - - Returns: - examples (tuple): Tuple of tensors (context_idxs, context_char_idxs, question_idxs, - question_char_idxs, y1s, y2s, ids). All of shape (batch_size, ...), where - the remaining dimensions are the maximum length of examples in the input. - - Adapted from: - https://github.com/yunjey/seq2seq-dataloader - """ - def merge_0d(scalars, dtype=torch.int64): - return torch.tensor(scalars, dtype=dtype) - - def merge_1d(arrays, dtype=torch.int64, pad_value=0): - lengths = [(a != pad_value).sum() for a in arrays] - padded = torch.zeros(len(arrays), max(lengths), dtype=dtype) - for i, seq in enumerate(arrays): - end = lengths[i] - padded[i, :end] = seq[:end] - return padded - - def merge_2d(matrices, dtype=torch.int64, pad_value=0): - heights = [(m.sum(1) != pad_value).sum() for m in matrices] - widths = [(m.sum(0) != pad_value).sum() for m in matrices] - padded = torch.zeros(len(matrices), max(heights), max(widths), dtype=dtype) - for i, seq in enumerate(matrices): - height, width = heights[i], widths[i] - padded[i, :height, :width] = seq[:height, :width] - return padded - - # Group by tensor type - context_idxs, context_char_idxs, \ - question_idxs, question_char_idxs, \ - y1s, y2s, ids = zip(*examples) - - # Merge into batch tensors - context_idxs = merge_1d(context_idxs) - context_char_idxs = merge_2d(context_char_idxs) - question_idxs = merge_1d(question_idxs) - question_char_idxs = merge_2d(question_char_idxs) - y1s = merge_0d(y1s) - y2s = merge_0d(y2s) - ids = merge_0d(ids) - - return (context_idxs, context_char_idxs, - question_idxs, question_char_idxs, - y1s, y2s, ids) - - -class AverageMeter: - """Keep track of average values over time. - - Adapted from: - > https://github.com/pytorch/examples/blob/master/imagenet/main.py - """ - def __init__(self): - self.avg = 0 - self.sum = 0 - self.count = 0 - - def reset(self): - """Reset meter.""" - self.__init__() - - def update(self, val, num_samples=1): - """Update meter with new value `val`, the average of `num` samples. - - Args: - val (float): Average value to update the meter with. - num_samples (int): Number of samples that were averaged to - produce `val`. - """ - self.count += num_samples - self.sum += val * num_samples - self.avg = self.sum / self.count - - -class EMA: - """Exponential moving average of model parameters. - Args: - model (torch.nn.Module): Model with parameters whose EMA will be kept. - decay (float): Decay rate for exponential moving average. - """ - def __init__(self, model, decay): - self.decay = decay - self.shadow = {} - self.original = {} - - # Register model parameters - for name, param in model.named_parameters(): - if param.requires_grad: - self.shadow[name] = param.data.clone() - - def __call__(self, model, num_updates): - decay = min(self.decay, (1.0 + num_updates) / (10.0 + num_updates)) - for name, param in model.named_parameters(): - if param.requires_grad: - assert name in self.shadow - new_average = \ - (1.0 - decay) * param.data + decay * self.shadow[name] - self.shadow[name] = new_average.clone() - - def assign(self, model): - """Assign exponential moving average of parameter values to the - respective parameters. - Args: - model (torch.nn.Module): Model to assign parameter values. - """ - for name, param in model.named_parameters(): - if param.requires_grad: - assert name in self.shadow - self.original[name] = param.data.clone() - param.data = self.shadow[name] - - def resume(self, model): - """Restore original parameters to a model. That is, put back - the values that were in each parameter at the last call to `assign`. - Args: - model (torch.nn.Module): Model to assign parameter values. - """ - for name, param in model.named_parameters(): - if param.requires_grad: - assert name in self.shadow - param.data = self.original[name] - - -class CheckpointSaver: - """Class to save and load model checkpoints. - - Save the best checkpoints as measured by a metric value passed into the - `save` method. Overwrite checkpoints with better checkpoints once - `max_checkpoints` have been saved. - - Args: - save_dir (str): Directory to save checkpoints. - max_checkpoints (int): Maximum number of checkpoints to keep before - overwriting old ones. - metric_name (str): Name of metric used to determine best model. - maximize_metric (bool): If true, best checkpoint is that which maximizes - the metric value passed in via `save`. Otherwise, best checkpoint - minimizes the metric. - log (logging.Logger): Optional logger for printing information. - """ - def __init__(self, save_dir, max_checkpoints, metric_name, - maximize_metric=False, log=None): - super(CheckpointSaver, self).__init__() - - self.save_dir = save_dir - self.max_checkpoints = max_checkpoints - self.metric_name = metric_name - self.maximize_metric = maximize_metric - self.best_val = None - self.ckpt_paths = queue.PriorityQueue() - self.log = log - self._print(f"Saver will {'max' if maximize_metric else 'min'}imize {metric_name}...") - - def is_best(self, metric_val): - """Check whether `metric_val` is the best seen so far. - - Args: - metric_val (float): Metric value to compare to prior checkpoints. - """ - if metric_val is None: - # No metric reported - return False - - if self.best_val is None: - # No checkpoint saved yet - return True - - return ((self.maximize_metric and self.best_val < metric_val) - or (not self.maximize_metric and self.best_val > metric_val)) - - def _print(self, message): - """Print a message if logging is enabled.""" - if self.log is not None: - self.log.info(message) - - def save(self, step, model, metric_val, device): - """Save model parameters to disk. - - Args: - step (int): Total number of examples seen during training so far. - model (torch.nn.DataParallel): Model to save. - metric_val (float): Determines whether checkpoint is best so far. - device (torch.device): Device where model resides. - """ - ckpt_dict = { - 'model_name': model.__class__.__name__, - 'model_state': model.cpu().state_dict(), - 'step': step - } - model.to(device) - - checkpoint_path = os.path.join(self.save_dir, - f'step_{step}.pth.tar') - torch.save(ckpt_dict, checkpoint_path) - self._print(f'Saved checkpoint: {checkpoint_path}') - - if self.is_best(metric_val): - # Save the best model - self.best_val = metric_val - best_path = os.path.join(self.save_dir, 'best.pth.tar') - shutil.copy(checkpoint_path, best_path) - self._print(f'New best checkpoint at step {step}...') - - # Add checkpoint path to priority queue (lowest priority removed first) - if self.maximize_metric: - priority_order = metric_val - else: - priority_order = -metric_val - - self.ckpt_paths.put((priority_order, checkpoint_path)) - - # Remove a checkpoint if more than max_checkpoints have been saved - if self.ckpt_paths.qsize() > self.max_checkpoints: - _, worst_ckpt = self.ckpt_paths.get() - try: - os.remove(worst_ckpt) - self._print(f'Removed checkpoint: {worst_ckpt}') - except OSError: - # Avoid crashing if checkpoint has been removed or protected - pass - - -def load_model(model, checkpoint_path, gpu_ids, return_step=True): - """Load model parameters from disk. - - Args: - model (torch.nn.DataParallel): Load parameters into this model. - checkpoint_path (str): Path to checkpoint to load. - gpu_ids (list): GPU IDs for DataParallel. - return_step (bool): Also return the step at which checkpoint was saved. - - Returns: - model (torch.nn.DataParallel): Model loaded from checkpoint. - step (int): Step at which checkpoint was saved. Only if `return_step`. - """ - device = f"cuda:{gpu_ids[0] if gpu_ids else 'cpu'}" - ckpt_dict = torch.load(checkpoint_path, map_location=device) - - # Build model, load parameters - model.load_state_dict(ckpt_dict['model_state']) - - if return_step: - step = ckpt_dict['step'] - return model, step - - return model - - -def get_available_devices(): - """Get IDs of all available GPUs. - - Returns: - device (torch.device): Main device (GPU 0 or CPU). - gpu_ids (list): List of IDs of all GPUs that are available. - """ - gpu_ids = [] - if torch.cuda.is_available(): - gpu_ids += [gpu_id for gpu_id in range(torch.cuda.device_count())] - device = torch.device(f'cuda:{gpu_ids[0]}') - torch.cuda.set_device(device) - else: - device = torch.device('cpu') - - return device, gpu_ids - - -def masked_softmax(logits, mask, dim=-1, log_softmax=False): - """Take the softmax of `logits` over given dimension, and set - entries to 0 wherever `mask` is 0. - - Args: - logits (torch.Tensor): Inputs to the softmax function. - mask (torch.Tensor): Same shape as `logits`, with 0 indicating - positions that should be assigned 0 probability in the output. - dim (int): Dimension over which to take softmax. - log_softmax (bool): Take log-softmax rather than regular softmax. - E.g., some PyTorch functions such as `F.nll_loss` expect log-softmax. - - Returns: - probs (torch.Tensor): Result of taking masked softmax over the logits. - """ - mask = mask.type(torch.float32) - masked_logits = mask * logits + (1 - mask) * -1e30 - softmax_fn = F.log_softmax if log_softmax else F.softmax - probs = softmax_fn(masked_logits, dim) - - return probs - - -def visualize(tbx, pred_dict, eval_path, step, split, num_visuals): - """Visualize text examples to TensorBoard. - - Args: - tbx (tensorboardX.SummaryWriter): Summary writer. - pred_dict (dict): dict of predictions of the form id -> pred. - eval_path (str): Path to eval JSON file. - step (int): Number of examples seen so far during training. - split (str): Name of data split being visualized. - num_visuals (int): Number of visuals to select at random from preds. - """ - if num_visuals <= 0: - return - if num_visuals > len(pred_dict): - num_visuals = len(pred_dict) - - visual_ids = np.random.choice(list(pred_dict), size=num_visuals, replace=False) - - with open(eval_path, 'r') as eval_file: - eval_dict = json.load(eval_file) - for i, id_ in enumerate(visual_ids): - pred = pred_dict[id_] or 'N/A' - example = eval_dict[str(id_)] - question = example['question'] - context = example['context'] - answers = example['answers'] - - gold = answers[0] if answers else 'N/A' - tbl_fmt = (f'- **Question:** {question}\n' - + f'- **Context:** {context}\n' - + f'- **Answer:** {gold}\n' - + f'- **Prediction:** {pred}') - tbx.add_text(tag=f'{split}/{i+1}_of_{num_visuals}', - text_string=tbl_fmt, - global_step=step) - - -def save_preds(preds, save_dir, file_name='predictions.csv'): - """Save predictions `preds` to a CSV file named `file_name` in `save_dir`. - - Args: - preds (list): List of predictions each of the form (id, start, end), - where id is an example ID, and start/end are indices in the context. - save_dir (str): Directory in which to save the predictions file. - file_name (str): File name for the CSV file. - - Returns: - save_path (str): Path where CSV file was saved. - """ - # Validate format - if (not isinstance(preds, list) - or any(not isinstance(p, tuple) or len(p) != 3 for p in preds)): - raise ValueError('preds must be a list of tuples (id, start, end)') - - # Make sure predictions are sorted by ID - preds = sorted(preds, key=lambda p: p[0]) - - # Save to a CSV file - save_path = os.path.join(save_dir, file_name) - np.savetxt(save_path, np.array(preds), delimiter=',', fmt='%d') - - return save_path - - -def get_save_dir(base_dir, name, training, id_max=100): - """Get a unique save directory by appending the smallest positive integer - `id < id_max` that is not already taken (i.e., no dir exists with that id). - - Args: - base_dir (str): Base directory in which to make save directories. - name (str): Name to identify this training run. Need not be unique. - training (bool): Save dir. is for training (determines subdirectory). - id_max (int): Maximum ID number before raising an exception. - - Returns: - save_dir (str): Path to a new directory with a unique name. - """ - for uid in range(1, id_max): - subdir = 'train' if training else 'test' - save_dir = os.path.join(base_dir, subdir, f'{name}-{uid:02d}') - if not os.path.exists(save_dir): - os.makedirs(save_dir) - return save_dir - - raise RuntimeError('Too many save directories created with the same name. \ - Delete old save directories or use another name.') - - -def get_logger(log_dir, name): - """Get a `logging.Logger` instance that prints to the console - and an auxiliary file. - - Args: - log_dir (str): Directory in which to create the log file. - name (str): Name to identify the logs. - - Returns: - logger (logging.Logger): Logger instance for logging events. - """ - class StreamHandlerWithTQDM(logging.Handler): - """Let `logging` print without breaking `tqdm` progress bars. - - See Also: - > https://stackoverflow.com/questions/38543506 - """ - def emit(self, record): - try: - msg = self.format(record) - tqdm.tqdm.write(msg) - self.flush() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.handleError(record) - - # Create logger - logger = logging.getLogger(name) - logger.setLevel(logging.DEBUG) - - # Log everything (i.e., DEBUG level and above) to a file - log_path = os.path.join(log_dir, 'log.txt') - file_handler = logging.FileHandler(log_path) - file_handler.setLevel(logging.DEBUG) - - # Log everything except DEBUG level (i.e., INFO level and above) to console - console_handler = StreamHandlerWithTQDM() - console_handler.setLevel(logging.INFO) - - # Create format for the logs - file_formatter = logging.Formatter('[%(asctime)s] %(message)s', - datefmt='%m.%d.%y %H:%M:%S') - file_handler.setFormatter(file_formatter) - console_formatter = logging.Formatter('[%(asctime)s] %(message)s', - datefmt='%m.%d.%y %H:%M:%S') - console_handler.setFormatter(console_formatter) - - # add the handlers to the logger - logger.addHandler(file_handler) - logger.addHandler(console_handler) - - return logger - - -def torch_from_json(path, dtype=torch.float32): - """Load a PyTorch Tensor from a JSON file. - - Args: - path (str): Path to the JSON file to load. - dtype (torch.dtype): Data type of loaded array. - - Returns: - tensor (torch.Tensor): Tensor loaded from JSON file. - """ - with open(path, 'r') as fh: - array = np.array(json.load(fh)) - - tensor = torch.from_numpy(array).type(dtype) - - return tensor - - -def discretize(p_start, p_end, max_len=15, no_answer=False): - """Discretize soft predictions to get start and end indices. - - Choose the pair `(i, j)` of indices that maximizes `p1[i] * p2[j]` - subject to `i <= j` and `j - i + 1 <= max_len`. - - Args: - p_start (torch.Tensor): Soft predictions for start index. - Shape (batch_size, context_len). - p_end (torch.Tensor): Soft predictions for end index. - Shape (batch_size, context_len). - max_len (int): Maximum length of the discretized prediction. - I.e., enforce that `preds[i, 1] - preds[i, 0] + 1 <= max_len`. - no_answer (bool): Treat 0-index as the no-answer prediction. Consider - a prediction no-answer if `preds[0, 0] * preds[0, 1]` is greater - than the probability assigned to the max-probability span. - - Returns: - start_idxs (torch.Tensor): Hard predictions for start index. - Shape (batch_size,) - end_idxs (torch.Tensor): Hard predictions for end index. - Shape (batch_size,) - """ - if p_start.min() < 0 or p_start.max() > 1 \ - or p_end.min() < 0 or p_end.max() > 1: - raise ValueError('Expected p_start and p_end to have values in [0, 1]') - - # Compute pairwise probabilities - p_start = p_start.unsqueeze(dim=2) - p_end = p_end.unsqueeze(dim=1) - p_joint = torch.matmul(p_start, p_end) # (batch_size, c_len, c_len) - - # Restrict to pairs (i, j) such that i <= j <= i + max_len - 1 - c_len, device = p_start.size(1), p_start.device - is_legal_pair = torch.triu(torch.ones((c_len, c_len), device=device)) - is_legal_pair -= torch.triu(torch.ones((c_len, c_len), device=device), - diagonal=max_len) - if no_answer: - # Index 0 is no-answer - p_no_answer = p_joint[:, 0, 0].clone() - is_legal_pair[0, :] = 0 - is_legal_pair[:, 0] = 0 - else: - p_no_answer = None - p_joint *= is_legal_pair - - # Take pair (i, j) that maximizes p_joint - max_in_row, _ = torch.max(p_joint, dim=2) - max_in_col, _ = torch.max(p_joint, dim=1) - start_idxs = torch.argmax(max_in_row, dim=-1) - end_idxs = torch.argmax(max_in_col, dim=-1) - - if no_answer: - # Predict no-answer whenever p_no_answer > max_prob - max_prob, _ = torch.max(max_in_col, dim=-1) - start_idxs[p_no_answer > max_prob] = 0 - end_idxs[p_no_answer > max_prob] = 0 - - return start_idxs, end_idxs - - -def convert_tokens(eval_dict, qa_id, y_start_list, y_end_list, no_answer): - """Convert predictions to tokens from the context. - - Args: - eval_dict (dict): Dictionary with eval info for the dataset. This is - used to perform the mapping from IDs and indices to actual text. - qa_id (int): List of QA example IDs. - y_start_list (list): List of start predictions. - y_end_list (list): List of end predictions. - no_answer (bool): Questions can have no answer. E.g., SQuAD 2.0. - - Returns: - pred_dict (dict): Dictionary index IDs -> predicted answer text. - sub_dict (dict): Dictionary UUIDs -> predicted answer text (submission). - """ - pred_dict = {} - sub_dict = {} - for qid, y_start, y_end in zip(qa_id, y_start_list, y_end_list): - context = eval_dict[str(qid)]["context"] - spans = eval_dict[str(qid)]["spans"] - uuid = eval_dict[str(qid)]["uuid"] - if no_answer and (y_start == 0 or y_end == 0): - pred_dict[str(qid)] = '' - sub_dict[uuid] = '' - else: - if no_answer: - y_start, y_end = y_start - 1, y_end - 1 - start_idx = spans[y_start][0] - end_idx = spans[y_end][1] - pred_dict[str(qid)] = context[start_idx: end_idx] - sub_dict[uuid] = context[start_idx: end_idx] - return pred_dict, sub_dict - - -def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): - if not ground_truths: - return metric_fn(prediction, '') - scores_for_ground_truths = [] - for ground_truth in ground_truths: - score = metric_fn(prediction, ground_truth) - scores_for_ground_truths.append(score) - return max(scores_for_ground_truths) - - -def eval_dicts(gold_dict, pred_dict, no_answer): - avna = f1 = em = total = 0 - for key, value in pred_dict.items(): - total += 1 - ground_truths = gold_dict[key]['answers'] - prediction = value - em += metric_max_over_ground_truths(compute_em, prediction, ground_truths) - f1 += metric_max_over_ground_truths(compute_f1, prediction, ground_truths) - if no_answer: - avna += compute_avna(prediction, ground_truths) - - eval_dict = {'EM': 100. * em / total, - 'F1': 100. * f1 / total} - - if no_answer: - eval_dict['AvNA'] = 100. * avna / total - - return eval_dict - - -def compute_avna(prediction, ground_truths): - """Compute answer vs. no-answer accuracy.""" - return float(bool(prediction) == bool(ground_truths)) - - -# All methods below this line are from the official SQuAD 2.0 eval script -# https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ -def normalize_answer(s): - """Convert to lowercase and remove punctuation, articles and extra whitespace.""" - - def remove_articles(text): - regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) - return re.sub(regex, ' ', text) - - def white_space_fix(text): - return ' '.join(text.split()) - - def remove_punc(text): - exclude = set(string.punctuation) - return ''.join(ch for ch in text if ch not in exclude) - - def lower(text): - return text.lower() - - return white_space_fix(remove_articles(remove_punc(lower(s)))) - - -def get_tokens(s): - if not s: - return [] - return normalize_answer(s).split() - - -def compute_em(a_gold, a_pred): - return int(normalize_answer(a_gold) == normalize_answer(a_pred)) - - -def compute_f1(a_gold, a_pred): - gold_toks = get_tokens(a_gold) - pred_toks = get_tokens(a_pred) - common = Counter(gold_toks) & Counter(pred_toks) - num_same = sum(common.values()) - if len(gold_toks) == 0 or len(pred_toks) == 0: - # If either is no-answer, then F1 is 1 if they agree, 0 otherwise - return int(gold_toks == pred_toks) - if num_same == 0: - return 0 - precision = 1.0 * num_same / len(pred_toks) - recall = 1.0 * num_same / len(gold_toks) - f1 = (2 * precision * recall) / (precision + recall) - return f1 diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index e86885d..0000000 --- a/poetry.lock +++ /dev/null @@ -1,3181 +0,0 @@ -[[package]] -name = "appnope" -version = "0.1.2" -description = "Disable App Nap on macOS >= 10.9" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "argon2-cffi" -version = "20.1.0" -description = "The secure Argon2 password hashing algorithm." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -cffi = ">=1.0.0" -six = "*" - -[package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest", "sphinx", "wheel", "pre-commit"] -docs = ["sphinx"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest"] - -[[package]] -name = "async-generator" -version = "1.10" -description = "Async generators and context managers for Python 3.5+" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "attrs" -version = "21.2.0" -description = "Classes Without Boilerplate" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] - -[[package]] -name = "backcall" -version = "0.2.0" -description = "Specifications for callback functions passed in to an API" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "backports.entry-points-selectable" -version = "1.1.0" -description = "Compatibility shim providing selectable entry points for older implementations" -category = "dev" -optional = false -python-versions = ">=2.7" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3.7)", "pytest-mypy", "pytest-checkdocs (>=2.4)", "pytest-enabler (>=1.0.1)"] - -[[package]] -name = "bleach" -version = "4.0.0" -description = "An easy safelist-based HTML-sanitizing tool." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -packaging = "*" -six = ">=1.9.0" -webencodings = "*" - -[[package]] -name = "blis" -version = "0.7.4" -description = "The Blis BLAS-like linear algebra library, as a self-contained C-extension." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -numpy = ">=1.15.0" - -[[package]] -name = "bokeh" -version = "2.3.3" -description = "Interactive plots and applications in the browser from Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -Jinja2 = ">=2.9" -numpy = ">=1.11.3" -packaging = ">=16.8" -pillow = ">=7.1.0" -python-dateutil = ">=2.1" -PyYAML = ">=3.10" -tornado = ">=5.1" -typing_extensions = ">=3.7.4" - -[[package]] -name = "boto3" -version = "1.18.19" -description = "The AWS SDK for Python" -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -botocore = ">=1.21.19,<1.22.0" -jmespath = ">=0.7.1,<1.0.0" -s3transfer = ">=0.5.0,<0.6.0" - -[package.extras] -crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] - -[[package]] -name = "botocore" -version = "1.21.19" -description = "Low-level, data-driven core of boto 3." -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -jmespath = ">=0.7.1,<1.0.0" -python-dateutil = ">=2.1,<3.0.0" -urllib3 = ">=1.25.4,<1.27" - -[package.extras] -crt = ["awscrt (==0.11.24)"] - -[[package]] -name = "catalogue" -version = "2.0.4" -description = "Super lightweight function registries for your library" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "certifi" -version = "2021.5.30" -description = "Python package for providing Mozilla's CA Bundle." -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "cffi" -version = "1.14.6" -description = "Foreign Function Interface for Python calling C code." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -pycparser = "*" - -[[package]] -name = "cfgv" -version = "3.3.0" -description = "Validate configuration and produce human readable error messages." -category = "dev" -optional = false -python-versions = ">=3.6.1" - -[[package]] -name = "charset-normalizer" -version = "2.0.4" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" -optional = false -python-versions = ">=3.5.0" - -[package.extras] -unicode_backport = ["unicodedata2"] - -[[package]] -name = "click" -version = "7.1.2" -description = "Composable command line interface toolkit" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "cloudpickle" -version = "1.6.0" -description = "Extended pickling support for Python objects" -category = "main" -optional = true -python-versions = ">=3.5" - -[[package]] -name = "colorama" -version = "0.4.4" -description = "Cross-platform colored terminal text." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "cycler" -version = "0.10.0" -description = "Composable style cycles" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -six = "*" - -[[package]] -name = "cymem" -version = "2.0.5" -description = "Manage calls to calloc/free through Cython" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "cython" -version = "0.29.14" -description = "The Cython compiler for writing C extensions for the Python language." -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "decorator" -version = "5.0.9" -description = "Decorators for Humans" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "distlib" -version = "0.3.2" -description = "Distribution utilities" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "eli5" -version = "0.11.0" -description = "Debug machine learning classifiers and explain their predictions" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -attrs = ">16.0.0" -graphviz = "*" -jinja2 = "*" -numpy = ">=1.9.0" -scikit-learn = ">=0.20" -scipy = "*" -six = "*" -tabulate = ">=0.7.7" - -[[package]] -name = "entrypoints" -version = "0.3" -description = "Discover and load entry points from installed packages." -category = "main" -optional = false -python-versions = ">=2.7" - -[[package]] -name = "filelock" -version = "3.0.12" -description = "A platform independent file lock." -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "gensim" -version = "3.8.3" -description = "Python framework for fast Vector Space Modelling" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -Cython = "0.29.14" -numpy = ">=1.11.3" -scipy = ">=0.18.1" -six = ">=1.5.0" -smart-open = ">=1.8.1" - -[package.extras] -distributed = ["Pyro4 (>=4.27)"] -docs = ["pytest", "pytest-rerunfailures", "mock", "cython", "nmslib", "pyemd", "testfixtures", "Morfessor (==2.0.2a4)", "python-Levenshtein (>=0.10.2)", "visdom (>0.1.8.7)", "scikit-learn", "Pyro4 (>=4.27)", "sphinx (<=2.4.4)", "sphinx-gallery", "sphinxcontrib.programoutput", "sphinxcontrib-napoleon", "matplotlib", "plotly", "memory-profiler", "annoy", "pyro4", "nltk", "statsmodels", "pandas"] -test = ["pytest", "pytest-rerunfailures", "mock", "cython", "nmslib", "pyemd", "testfixtures", "Morfessor (==2.0.2a4)", "python-Levenshtein (>=0.10.2)", "visdom (>0.1.8.7)", "scikit-learn"] -test-win = ["pytest", "pytest-rerunfailures", "mock", "cython", "nmslib", "pyemd", "testfixtures", "Morfessor (==2.0.2a4)", "python-Levenshtein (>=0.10.2)", "visdom (>0.1.8.7)", "scikit-learn"] - -[[package]] -name = "graphviz" -version = "0.16" -description = "Simple Python interface for Graphviz" -category = "main" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" - -[package.extras] -dev = ["tox (>=3)", "flake8", "pep8-naming", "wheel", "twine"] -docs = ["sphinx (>=1.8)", "sphinx-rtd-theme"] -test = ["mock (>=3)", "pytest (>=4)", "pytest-mock (>=2)", "pytest-cov"] - -[[package]] -name = "gym" -version = "0.18.3" -description = "The OpenAI Gym: A toolkit for developing and comparing your reinforcement learning agents." -category = "main" -optional = true -python-versions = ">=3.6" - -[package.dependencies] -cloudpickle = ">=1.2.0,<1.7.0" -numpy = ">=1.10.4" -Pillow = "<=8.2.0" -pyglet = ">=1.4.0,<=1.5.15" -scipy = "*" - -[package.extras] -all = ["box2d-py (>=2.3.5,<2.4.0)", "opencv-python (>=3)", "imageio", "atari_py (>=0.2.0,<0.3.0)", "mujoco_py (>=1.50,<2.0)"] -atari = ["atari_py (>=0.2.0,<0.3.0)", "opencv-python (>=3)"] -box2d = ["box2d-py (>=2.3.5,<2.4.0)"] -mujoco = ["mujoco_py (>=1.50,<2.0)", "imageio"] -nomujoco = ["box2d-py (>=2.3.5,<2.4.0)", "opencv-python (>=3)", "atari_py (>=0.2.0,<0.3.0)"] -robotics = ["mujoco_py (>=1.50,<2.0)", "imageio"] - -[[package]] -name = "h5py" -version = "3.3.0" -description = "Read and write HDF5 files from Python" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -numpy = [ - {version = ">=1.17.5", markers = "python_version == \"3.8\""}, - {version = ">=1.19.3", markers = "python_version >= \"3.9\""}, -] - -[[package]] -name = "identify" -version = "2.2.13" -description = "File identification library for Python" -category = "dev" -optional = false -python-versions = ">=3.6.1" - -[package.extras] -license = ["editdistance-s"] - -[[package]] -name = "idna" -version = "3.2" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "imageio" -version = "2.9.0" -description = "Library for reading and writing a wide range of image, video, scientific, and volumetric data formats." -category = "main" -optional = true -python-versions = ">=3.5" - -[package.dependencies] -numpy = "*" -pillow = "*" - -[package.extras] -ffmpeg = ["imageio-ffmpeg"] -fits = ["astropy"] -full = ["astropy", "gdal", "imageio-ffmpeg", "itk"] -gdal = ["gdal"] -itk = ["itk"] - -[[package]] -name = "ipykernel" -version = "5.5.5" -description = "IPython Kernel for Jupyter" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -appnope = {version = "*", markers = "platform_system == \"Darwin\""} -ipython = ">=5.0.0" -jupyter-client = "*" -tornado = ">=4.2" -traitlets = ">=4.1.0" - -[package.extras] -test = ["pytest (!=5.3.4)", "pytest-cov", "flaky", "nose", "jedi (<=0.17.2)"] - -[[package]] -name = "ipython" -version = "7.26.0" -description = "IPython: Productive Interactive Computing" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -appnope = {version = "*", markers = "sys_platform == \"darwin\""} -backcall = "*" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -decorator = "*" -jedi = ">=0.16" -matplotlib-inline = "*" -pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} -pickleshare = "*" -prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0" -pygments = "*" -traitlets = ">=4.2" - -[package.extras] -all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.17)", "pygments", "qtconsole", "requests", "testpath"] -doc = ["Sphinx (>=1.3)"] -kernel = ["ipykernel"] -nbconvert = ["nbconvert"] -nbformat = ["nbformat"] -notebook = ["notebook", "ipywidgets"] -parallel = ["ipyparallel"] -qtconsole = ["qtconsole"] -test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.17)"] - -[[package]] -name = "ipython-genutils" -version = "0.2.0" -description = "Vestigial utilities from IPython" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "ipywidgets" -version = "7.6.3" -description = "IPython HTML widgets for Jupyter" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -ipykernel = ">=4.5.1" -ipython = {version = ">=4.0.0", markers = "python_version >= \"3.3\""} -jupyterlab-widgets = {version = ">=1.0.0", markers = "python_version >= \"3.6\""} -nbformat = ">=4.2.0" -traitlets = ">=4.3.1" -widgetsnbextension = ">=3.5.0,<3.6.0" - -[package.extras] -test = ["pytest (>=3.6.0)", "pytest-cov", "mock"] - -[[package]] -name = "jedi" -version = "0.18.0" -description = "An autocompletion tool for Python that can be used for text editors." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -parso = ">=0.8.0,<0.9.0" - -[package.extras] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"] - -[[package]] -name = "jinja2" -version = "3.0.1" -description = "A very fast and expressive template engine." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jmespath" -version = "0.10.0" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "joblib" -version = "1.0.1" -description = "Lightweight pipelining with Python functions" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "jsonschema" -version = "3.2.0" -description = "An implementation of JSON Schema validation for Python" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -attrs = ">=17.4.0" -pyrsistent = ">=0.14.0" -six = ">=1.11.0" - -[package.extras] -format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] -format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"] - -[[package]] -name = "jupyter-client" -version = "6.2.0" -description = "Jupyter protocol implementation and client libraries" -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -jupyter-core = ">=4.6.0" -nest-asyncio = ">=1.5" -python-dateutil = ">=2.1" -pyzmq = ">=13" -tornado = ">=4.1" -traitlets = "*" - -[package.extras] -doc = ["sphinx (>=1.3.6)", "sphinx-rtd-theme", "sphinxcontrib-github-alt"] -test = ["async-generator", "ipykernel", "ipython", "mock", "pytest-asyncio", "pytest-timeout", "pytest", "mypy", "pre-commit", "jedi (<0.18)"] - -[[package]] -name = "jupyter-core" -version = "4.7.1" -description = "Jupyter core package. A base package on which Jupyter projects rely." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pywin32 = {version = ">=1.0", markers = "sys_platform == \"win32\""} -traitlets = "*" - -[[package]] -name = "jupyterlab-pygments" -version = "0.1.2" -description = "Pygments theme using JupyterLab CSS variables" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -pygments = ">=2.4.1,<3" - -[[package]] -name = "jupyterlab-widgets" -version = "1.0.0" -description = "A JupyterLab extension." -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "kiwisolver" -version = "1.3.1" -description = "A fast implementation of the Cassowary constraint solver" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "llvmlite" -version = "0.34.0" -description = "lightweight wrapper around basic LLVM functionality" -category = "main" -optional = true -python-versions = ">=3.6" - -[[package]] -name = "markupsafe" -version = "2.0.1" -description = "Safely add untrusted strings to HTML/XML markup." -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "matplotlib" -version = "3.4.2" -description = "Python plotting package" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -cycler = ">=0.10" -kiwisolver = ">=1.0.1" -numpy = ">=1.16" -pillow = ">=6.2.0" -pyparsing = ">=2.2.1" -python-dateutil = ">=2.7" - -[[package]] -name = "matplotlib-inline" -version = "0.1.2" -description = "Inline Matplotlib backend for Jupyter" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -traitlets = "*" - -[[package]] -name = "mistune" -version = "0.8.4" -description = "The fastest markdown parser in pure Python" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "murmurhash" -version = "1.0.5" -description = "Cython bindings for MurmurHash" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "nbclient" -version = "0.5.3" -description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -async-generator = "*" -jupyter-client = ">=6.1.5" -nbformat = ">=5.0" -nest-asyncio = "*" -traitlets = ">=4.2" - -[package.extras] -dev = ["codecov", "coverage", "ipython", "ipykernel", "ipywidgets", "pytest (>=4.1)", "pytest-cov (>=2.6.1)", "check-manifest", "flake8", "mypy", "tox", "bumpversion", "xmltodict", "pip (>=18.1)", "wheel (>=0.31.0)", "setuptools (>=38.6.0)", "twine (>=1.11.0)", "black"] -sphinx = ["Sphinx (>=1.7)", "sphinx-book-theme", "mock", "moto", "myst-parser"] -test = ["codecov", "coverage", "ipython", "ipykernel", "ipywidgets", "pytest (>=4.1)", "pytest-cov (>=2.6.1)", "check-manifest", "flake8", "mypy", "tox", "bumpversion", "xmltodict", "pip (>=18.1)", "wheel (>=0.31.0)", "setuptools (>=38.6.0)", "twine (>=1.11.0)", "black"] - -[[package]] -name = "nbconvert" -version = "6.1.0" -description = "Converting Jupyter Notebooks" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -bleach = "*" -defusedxml = "*" -entrypoints = ">=0.2.2" -jinja2 = ">=2.4" -jupyter-core = "*" -jupyterlab-pygments = "*" -mistune = ">=0.8.1,<2" -nbclient = ">=0.5.0,<0.6.0" -nbformat = ">=4.4" -pandocfilters = ">=1.4.1" -pygments = ">=2.4.1" -testpath = "*" -traitlets = ">=5.0" - -[package.extras] -all = ["pytest", "pytest-cov", "pytest-dependency", "ipykernel", "ipywidgets (>=7)", "pyppeteer (==0.2.2)", "tornado (>=4.0)", "sphinx (>=1.5.1)", "sphinx-rtd-theme", "nbsphinx (>=0.2.12)", "ipython"] -docs = ["sphinx (>=1.5.1)", "sphinx-rtd-theme", "nbsphinx (>=0.2.12)", "ipython"] -serve = ["tornado (>=4.0)"] -test = ["pytest", "pytest-cov", "pytest-dependency", "ipykernel", "ipywidgets (>=7)", "pyppeteer (==0.2.2)"] -webpdf = ["pyppeteer (==0.2.2)"] - -[[package]] -name = "nbformat" -version = "5.1.3" -description = "The Jupyter Notebook format" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -ipython-genutils = "*" -jsonschema = ">=2.4,<2.5.0 || >2.5.0" -jupyter-core = "*" -traitlets = ">=4.1" - -[package.extras] -fast = ["fastjsonschema"] -test = ["check-manifest", "fastjsonschema", "testpath", "pytest", "pytest-cov"] - -[[package]] -name = "nest-asyncio" -version = "1.5.1" -description = "Patch asyncio to allow nested event loops" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "networkx" -version = "2.6.2" -description = "Python package for creating and manipulating graphs and networks" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.extras] -default = ["numpy (>=1.19)", "scipy (>=1.5,!=1.6.1)", "matplotlib (>=3.3)", "pandas (>=1.1)"] -developer = ["black (==21.5b1)", "pre-commit (>=2.12)"] -doc = ["sphinx (>=4.0,<5.0)", "pydata-sphinx-theme (>=0.6,<1.0)", "sphinx-gallery (>=0.9,<1.0)", "numpydoc (>=1.1)", "pillow (>=8.2)", "nb2plots (>=0.6)", "texext (>=0.6.6)"] -extra = ["lxml (>=4.5)", "pygraphviz (>=1.7)", "pydot (>=1.4.1)"] -test = ["pytest (>=6.2)", "pytest-cov (>=2.12)", "codecov (>=2.1)"] - -[[package]] -name = "nltk" -version = "3.6.2" -description = "Natural Language Toolkit" -category = "main" -optional = false -python-versions = ">=3.5.*" - -[package.dependencies] -click = "*" -joblib = "*" -regex = "*" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "twython", "scipy", "numpy", "gensim (<4.0.0)", "python-crfsuite", "pyparsing", "scikit-learn", "requests"] -corenlp = ["requests"] -machine_learning = ["gensim (<4.0.0)", "numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - -[[package]] -name = "nodeenv" -version = "1.6.0" -description = "Node.js virtual environment builder" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "notebook" -version = "6.4.3" -description = "A web-based notebook environment for interactive computing" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -argon2-cffi = "*" -ipykernel = "*" -ipython-genutils = "*" -jinja2 = "*" -jupyter-client = ">=5.3.4" -jupyter-core = ">=4.6.1" -nbconvert = "*" -nbformat = "*" -prometheus-client = "*" -pyzmq = ">=17" -Send2Trash = ">=1.5.0" -terminado = ">=0.8.3" -tornado = ">=6.1" -traitlets = ">=4.2.1" - -[package.extras] -docs = ["sphinx", "nbsphinx", "sphinxcontrib-github-alt", "sphinx-rtd-theme", "myst-parser"] -json-logging = ["json-logging"] -test = ["pytest", "coverage", "requests", "nbval", "selenium", "pytest-cov", "requests-unixsocket"] - -[[package]] -name = "numba" -version = "0.51.2" -description = "compiling Python code using LLVM" -category = "main" -optional = true -python-versions = ">=3.6" - -[package.dependencies] -llvmlite = ">=0.34.0.dev0,<0.35" -numpy = ">=1.15" - -[[package]] -name = "numpy" -version = "1.21.1" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "opencv-python" -version = "4.5.3.56" -description = "Wrapper package for OpenCV python bindings." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -numpy = ">=1.21.0" - -[[package]] -name = "packaging" -version = "21.0" -description = "Core utilities for Python packages" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2" - -[[package]] -name = "pandas" -version = "1.3.1" -description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" -optional = false -python-versions = ">=3.7.1" - -[package.dependencies] -numpy = ">=1.17.3" -python-dateutil = ">=2.7.3" -pytz = ">=2017.3" - -[package.extras] -test = ["hypothesis (>=3.58)", "pytest (>=6.0)", "pytest-xdist"] - -[[package]] -name = "pandocfilters" -version = "1.4.3" -description = "Utilities for writing pandoc filters in python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "parso" -version = "0.8.2" -description = "A Python Parser" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["docopt", "pytest (<6.0.0)"] - -[[package]] -name = "pathy" -version = "0.6.0" -description = "pathlib.Path subclasses for local and cloud bucket storage" -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -smart-open = ">=5.0.0,<6.0.0" -typer = ">=0.3.0,<1.0.0" - -[package.extras] -all = ["google-cloud-storage (>=1.26.0,<2.0.0)", "boto3", "pytest", "pytest-coverage", "mock", "typer-cli"] -gcs = ["google-cloud-storage (>=1.26.0,<2.0.0)"] -s3 = ["boto3"] -test = ["pytest", "pytest-coverage", "mock", "typer-cli"] - -[[package]] -name = "patsy" -version = "0.5.1" -description = "A Python package for describing statistical models and for building design matrices." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -numpy = ">=1.4" -six = "*" - -[[package]] -name = "pdpbox" -version = "0.2.0" -description = "python partial dependence plot toolbox" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -joblib = "*" -matplotlib = ">=2.1.2" -numpy = "*" -pandas = "*" -psutil = "*" -scikit-learn = "*" -scipy = "*" - -[[package]] -name = "pexpect" -version = "4.8.0" -description = "Pexpect allows easy control of interactive console applications." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -ptyprocess = ">=0.5" - -[[package]] -name = "pickleshare" -version = "0.7.5" -description = "Tiny 'shelve'-like database with concurrency support" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "pillow" -version = "7.2.0" -description = "Python Imaging Library (Fork)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "platformdirs" -version = "2.2.0" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] - -[[package]] -name = "pre-commit" -version = "2.14.0" -description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -cfgv = ">=2.0.0" -identify = ">=1.0.0" -nodeenv = ">=0.11.1" -pyyaml = ">=5.1" -toml = "*" -virtualenv = ">=20.0.8" - -[[package]] -name = "preshed" -version = "3.0.5" -description = "Cython hash table that trusts the keys are pre-hashed" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -cymem = ">=2.0.2,<2.1.0" -murmurhash = ">=0.28.0,<1.1.0" - -[[package]] -name = "prometheus-client" -version = "0.11.0" -description = "Python client for the Prometheus monitoring system." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.extras] -twisted = ["twisted"] - -[[package]] -name = "prompt-toolkit" -version = "3.0.19" -description = "Library for building powerful interactive command lines in Python" -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -wcwidth = "*" - -[[package]] -name = "psutil" -version = "5.8.0" -description = "Cross-platform lib for process and system monitoring in Python." -category = "main" -optional = true -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.extras] -test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"] - -[[package]] -name = "ptyprocess" -version = "0.7.0" -description = "Run a subprocess in a pseudo terminal" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "py" -version = "1.10.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pycparser" -version = "2.20" -description = "C parser in Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pydantic" -version = "1.8.2" -description = "Data validation and settings management using python 3.6 type hinting" -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -typing-extensions = ">=3.7.4.3" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pydotplus" -version = "2.0.2" -description = "Python interface to Graphviz's Dot language" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -pyparsing = ">=2.0.1" - -[[package]] -name = "pyglet" -version = "1.5.15" -description = "Cross-platform windowing and multimedia library" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pygments" -version = "2.9.0" -description = "Pygments is a syntax highlighting package written in Python." -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "pyparsing" -version = "2.4.7" -description = "Python parsing module" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "pyrsistent" -version = "0.18.0" -description = "Persistent/Functional/Immutable data structures" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytorch-transformers" -version = "1.2.0" -description = "Repository of pre-trained NLP Transformer models: BERT & RoBERTa, GPT & GPT-2, Transformer-XL, XLNet and XLM" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -boto3 = "*" -numpy = "*" -regex = "*" -requests = "*" -sacremoses = "*" -sentencepiece = "*" -torch = ">=1.0.0" -tqdm = "*" - -[[package]] -name = "pytz" -version = "2021.1" -description = "World timezone definitions, modern and historical" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "pywavelets" -version = "1.1.1" -description = "PyWavelets, wavelet transform module" -category = "main" -optional = true -python-versions = ">=3.5" - -[package.dependencies] -numpy = ">=1.13.3" - -[[package]] -name = "pywin32" -version = "301" -description = "Python for Window Extensions" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "pywinpty" -version = "1.1.3" -description = "Pseudo terminal support for Windows from Python." -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "pyyaml" -version = "5.4.1" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[[package]] -name = "pyzmq" -version = "22.2.1" -description = "Python bindings for 0MQ" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = {version = "*", markers = "implementation_name == \"pypy\""} -py = {version = "*", markers = "implementation_name == \"pypy\""} - -[[package]] -name = "regex" -version = "2021.8.3" -description = "Alternative regular expression module, to replace re." -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "requests" -version = "2.26.0" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} -idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] - -[[package]] -name = "s3transfer" -version = "0.5.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "sacremoses" -version = "0.0.45" -description = "SacreMoses" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -click = "*" -joblib = "*" -regex = "*" -six = "*" -tqdm = "*" - -[[package]] -name = "scikit-image" -version = "0.18.2" -description = "Image processing in Python" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -imageio = ">=2.3.0" -matplotlib = ">=2.0.0,<3.0.0 || >3.0.0" -networkx = ">=2.0" -numpy = ">=1.16.5" -pillow = ">=4.3.0,<7.1.0 || >7.1.0,<7.1.1 || >7.1.1" -PyWavelets = ">=1.1.1" -scipy = ">=1.0.1" -tifffile = ">=2019.7.26" - -[package.extras] -data = ["pooch (>=1.3.0)"] -docs = ["sphinx (>=1.8,<=2.4.4)", "sphinx-gallery (>=0.7.0,!=0.8.0)", "numpydoc (>=1.0)", "sphinx-copybutton", "pytest-runner", "scikit-learn", "matplotlib (>=3.0.1)", "dask[array] (>=0.15.0,!=2.17.0)", "cloudpickle (>=0.2.1)", "pandas (>=0.23.0)", "seaborn (>=0.7.1)", "pooch (>=1.3.0)", "tifffile (>=2020.5.30)", "myst-parser", "ipywidgets", "plotly (>=4.10.0)"] -optional = ["simpleitk", "astropy (>=3.1.2)", "qtpy", "pyamg", "dask[array] (>=1.0.0,!=2.17.0)", "cloudpickle (>=0.2.1)", "pooch (>=1.3.0)"] -test = ["pytest (>=5.2.0)", "pytest-cov (>=2.7.0)", "pytest-localserver", "pytest-faulthandler", "flake8", "codecov", "pooch (>=1.3.0)"] - -[[package]] -name = "scikit-learn" -version = "0.24.2" -description = "A set of python modules for machine learning and data mining" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -joblib = ">=0.11" -numpy = ">=1.13.3" -scipy = ">=0.19.1" -threadpoolctl = ">=2.0.0" - -[package.extras] -benchmark = ["matplotlib (>=2.1.1)", "pandas (>=0.25.0)", "memory-profiler (>=0.57.0)"] -docs = ["matplotlib (>=2.1.1)", "scikit-image (>=0.13)", "pandas (>=0.25.0)", "seaborn (>=0.9.0)", "memory-profiler (>=0.57.0)", "sphinx (>=3.2.0)", "sphinx-gallery (>=0.7.0)", "numpydoc (>=1.0.0)", "Pillow (>=7.1.2)", "sphinx-prompt (>=1.3.0)"] -examples = ["matplotlib (>=2.1.1)", "scikit-image (>=0.13)", "pandas (>=0.25.0)", "seaborn (>=0.9.0)"] -tests = ["matplotlib (>=2.1.1)", "scikit-image (>=0.13)", "pandas (>=0.25.0)", "pytest (>=5.0.1)", "pytest-cov (>=2.9.0)", "flake8 (>=3.8.2)", "mypy (>=0.770)", "pyamg (>=4.0.0)"] - -[[package]] -name = "scipy" -version = "1.6.1" -description = "SciPy: Scientific Library for Python" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -numpy = ">=1.16.5" - -[[package]] -name = "seaborn" -version = "0.11.1" -description = "seaborn: statistical data visualization" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -matplotlib = ">=2.2" -numpy = ">=1.15" -pandas = ">=0.23" -scipy = ">=1.0" - -[[package]] -name = "send2trash" -version = "1.8.0" -description = "Send file to trash natively under Mac OS X, Windows and Linux." -category = "main" -optional = false -python-versions = "*" - -[package.extras] -nativelib = ["pyobjc-framework-cocoa", "pywin32"] -objc = ["pyobjc-framework-cocoa"] -win32 = ["pywin32"] - -[[package]] -name = "sentencepiece" -version = "0.1.96" -description = "SentencePiece python wrapper" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "shap" -version = "0.38.1" -description = "A unified approach to explain the output of any machine learning model." -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -cloudpickle = "*" -numba = "*" -numpy = "*" -pandas = "*" -scikit-learn = "*" -scipy = "*" -slicer = "0.0.7" -tqdm = ">4.25.0" - -[package.extras] -all = ["xgboost", "lightgbm", "transformers", "lime", "pyspark", "torch", "pytest", "pytest-mpl", "pytest-cov", "nbsphinx", "matplotlib", "sphinx-rtd-theme", "sphinx", "catboost", "pyod", "sentencepiece", "opencv-python", "ipython", "numpydoc"] -docs = ["matplotlib", "ipython", "numpydoc", "sphinx-rtd-theme", "sphinx", "nbsphinx"] -others = ["lime"] -plots = ["matplotlib", "ipython"] -test = ["pytest", "pytest-mpl", "pytest-cov", "xgboost", "lightgbm", "catboost", "pyspark", "pyod", "transformers", "torch", "sentencepiece", "opencv-python"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "slicer" -version = "0.0.7" -description = "A small package for big slicing." -category = "main" -optional = true -python-versions = ">=3.6" - -[[package]] -name = "smart-open" -version = "5.1.0" -description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" -category = "main" -optional = false -python-versions = ">=3.6.*" - -[package.extras] -all = ["boto3", "google-cloud-storage", "azure-storage-blob", "azure-common", "azure-core", "requests"] -azure = ["azure-storage-blob", "azure-common", "azure-core"] -gcs = ["google-cloud-storage"] -http = ["requests"] -s3 = ["boto3"] -test = ["boto3", "google-cloud-storage", "azure-storage-blob", "azure-common", "azure-core", "requests", "moto[server] (==1.3.14)", "pathlib2", "responses", "paramiko", "parameterizedtestcase", "pytest", "pytest-rerunfailures"] -webhdfs = ["requests"] - -[[package]] -name = "spacy" -version = "3.1.1" -description = "Industrial-strength Natural Language Processing (NLP) in Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -blis = ">=0.4.0,<0.8.0" -catalogue = ">=2.0.4,<2.1.0" -cymem = ">=2.0.2,<2.1.0" -jinja2 = "*" -murmurhash = ">=0.28.0,<1.1.0" -numpy = ">=1.15.0" -packaging = ">=20.0" -pathy = ">=0.3.5" -preshed = ">=3.0.2,<3.1.0" -pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<1.9.0" -requests = ">=2.13.0,<3.0.0" -spacy-legacy = ">=3.0.7,<3.1.0" -srsly = ">=2.4.1,<3.0.0" -thinc = ">=8.0.8,<8.1.0" -tqdm = ">=4.38.0,<5.0.0" -typer = ">=0.3.0,<0.4.0" -wasabi = ">=0.8.1,<1.1.0" - -[package.extras] -cuda = ["cupy (>=5.0.0b4,<10.0.0)"] -cuda100 = ["cupy-cuda100 (>=5.0.0b4,<10.0.0)"] -cuda101 = ["cupy-cuda101 (>=5.0.0b4,<10.0.0)"] -cuda102 = ["cupy-cuda102 (>=5.0.0b4,<10.0.0)"] -cuda110 = ["cupy-cuda110 (>=5.0.0b4,<10.0.0)"] -cuda111 = ["cupy-cuda111 (>=5.0.0b4,<10.0.0)"] -cuda112 = ["cupy-cuda112 (>=5.0.0b4,<10.0.0)"] -cuda80 = ["cupy-cuda80 (>=5.0.0b4,<10.0.0)"] -cuda90 = ["cupy-cuda90 (>=5.0.0b4,<10.0.0)"] -cuda91 = ["cupy-cuda91 (>=5.0.0b4,<10.0.0)"] -cuda92 = ["cupy-cuda92 (>=5.0.0b4,<10.0.0)"] -ja = ["sudachipy (>=0.4.9)", "sudachidict-core (>=20200330)"] -ko = ["natto-py (==0.9.0)"] -lookups = ["spacy-lookups-data (>=1.0.2,<1.1.0)"] -ray = ["spacy-ray (>=0.1.0,<1.0.0)"] -th = ["pythainlp (>=2.0)"] -transformers = ["spacy-transformers (>=1.0.1,<1.1.0)"] - -[[package]] -name = "spacy-legacy" -version = "3.0.8" -description = "Legacy registered functions for spaCy backwards compatibility" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "srsly" -version = "2.4.1" -description = "Modern high-performance serialization utilities for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -catalogue = ">=2.0.1,<2.1.0" - -[[package]] -name = "statsmodels" -version = "0.12.2" -description = "Statistical computations and models for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -numpy = ">=1.15" -pandas = ">=0.21" -patsy = ">=0.5" -scipy = ">=1.1" - -[package.extras] -build = ["cython (>=0.29)"] -develop = ["cython (>=0.29)"] -docs = ["sphinx", "nbconvert", "jupyter-client", "ipykernel", "matplotlib", "nbformat", "numpydoc", "pandas-datareader"] - -[[package]] -name = "subword-nmt" -version = "0.3.7" -description = "Unsupervised Word Segmentation for Neural Machine Translation and Text Generation" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "tabulate" -version = "0.8.9" -description = "Pretty-print tabular data" -category = "main" -optional = true -python-versions = "*" - -[package.extras] -widechars = ["wcwidth"] - -[[package]] -name = "terminado" -version = "0.11.0" -description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -ptyprocess = {version = "*", markers = "os_name != \"nt\""} -pywinpty = {version = ">=1.1.0", markers = "os_name == \"nt\""} -tornado = ">=4" - -[package.extras] -test = ["pytest"] - -[[package]] -name = "testpath" -version = "0.5.0" -description = "Test utilities for code working with files and commands" -category = "main" -optional = false -python-versions = ">= 3.5" - -[package.extras] -test = ["pytest", "pathlib2"] - -[[package]] -name = "thinc" -version = "8.0.8" -description = "A refreshing functional take on deep learning, compatible with your favorite libraries" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -blis = ">=0.4.0,<0.8.0" -catalogue = ">=2.0.4,<2.1.0" -cymem = ">=2.0.2,<2.1.0" -murmurhash = ">=0.28.0,<1.1.0" -numpy = ">=1.15.0" -preshed = ">=3.0.2,<3.1.0" -pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<1.9.0" -srsly = ">=2.4.0,<3.0.0" -wasabi = ">=0.8.1,<1.1.0" - -[package.extras] -cuda = ["cupy (>=5.0.0b4)"] -cuda100 = ["cupy-cuda100 (>=5.0.0b4)"] -cuda101 = ["cupy-cuda101 (>=5.0.0b4)"] -cuda102 = ["cupy-cuda102 (>=5.0.0b4)"] -cuda110 = ["cupy-cuda110 (>=5.0.0b4)"] -cuda111 = ["cupy-cuda111 (>=5.0.0b4)"] -cuda80 = ["cupy-cuda80 (>=5.0.0b4)"] -cuda90 = ["cupy-cuda90 (>=5.0.0b4)"] -cuda91 = ["cupy-cuda91 (>=5.0.0b4)"] -cuda92 = ["cupy-cuda92 (>=5.0.0b4)"] -datasets = ["ml-datasets (>=0.2.0,<0.3.0)"] -mxnet = ["mxnet (>=1.5.1,<1.6.0)"] -tensorflow = ["tensorflow (>=2.0.0,<2.3.0)"] -torch = ["torch (>=1.5.0)"] - -[[package]] -name = "threadpoolctl" -version = "2.2.0" -description = "threadpoolctl" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "tifffile" -version = "2021.8.8" -description = "Read and write TIFF files" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -numpy = ">=1.15.1" - -[package.extras] -all = ["imagecodecs (>=2021.7.30)", "matplotlib (>=3.2)", "lxml"] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "torch" -version = "1.7.1" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -category = "main" -optional = false -python-versions = ">=3.6.2" - -[package.dependencies] -numpy = "*" -typing-extensions = "*" - -[[package]] -name = "torchsummary" -version = "1.5.1" -description = "Model summary in PyTorch similar to `model.summary()` in Keras" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "torchtext" -version = "0.8.1" -description = "Text utilities and datasets for PyTorch" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -numpy = "*" -requests = "*" -torch = "1.7.1" -tqdm = "*" - -[[package]] -name = "torchvision" -version = "0.8.2" -description = "image and video datasets and models for torch deep learning" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -numpy = "*" -pillow = ">=4.1.1" -torch = "1.7.1" - -[package.extras] -scipy = ["scipy"] - -[[package]] -name = "tornado" -version = "6.1" -description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "main" -optional = false -python-versions = ">= 3.5" - -[[package]] -name = "tqdm" -version = "4.62.0" -description = "Fast, Extensible Progress Meter" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] -telegram = ["requests"] - -[[package]] -name = "traitlets" -version = "5.0.5" -description = "Traitlets Python configuration system" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -ipython-genutils = "*" - -[package.extras] -test = ["pytest"] - -[[package]] -name = "typer" -version = "0.3.2" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -click = ">=7.1.1,<7.2.0" - -[package.extras] -test = ["pytest-xdist (>=1.32.0,<2.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "mypy (==0.782)", "black (>=19.10b0,<20.0b0)", "isort (>=5.0.6,<6.0.0)", "shellingham (>=1.3.0,<2.0.0)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "coverage (>=5.2,<6.0)"] -all = ["colorama (>=0.4.3,<0.5.0)", "shellingham (>=1.3.0,<2.0.0)"] -dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)"] -doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=5.4.0,<6.0.0)", "markdown-include (>=0.5.1,<0.6.0)"] - -[[package]] -name = "typing-extensions" -version = "3.10.0.0" -description = "Backported and Experimental Type Hints for Python 3.5+" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "urllib3" -version = "1.26.6" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" - -[package.extras] -brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "virtualenv" -version = "20.7.2" -description = "Virtual Python Environment builder" -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[package.dependencies] -"backports.entry-points-selectable" = ">=1.0.4" -distlib = ">=0.3.1,<1" -filelock = ">=3.0.0,<4" -platformdirs = ">=2,<3" -six = ">=1.9.0,<2" - -[package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] - -[[package]] -name = "wasabi" -version = "0.8.2" -description = "A lightweight console printing and formatting toolkit" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "wcwidth" -version = "0.2.5" -description = "Measures the displayed width of unicode strings in a terminal" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "webencodings" -version = "0.5.1" -description = "Character encoding aliases for legacy web content" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "widgetsnbextension" -version = "3.5.1" -description = "IPython HTML widgets for Jupyter" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -notebook = ">=4.4.1" - -[[package]] -name = "xgboost" -version = "1.4.2" -description = "XGBoost Python Package" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -numpy = "*" -scipy = "*" - -[package.extras] -dask = ["dask", "pandas", "distributed"] -datatable = ["datatable"] -pandas = ["pandas"] -plotting = ["graphviz", "matplotlib"] -scikit-learn = ["scikit-learn"] - -[extras] -basic = ["Pillow", "tqdm", "scikit-image", "h5py", "pydotplus", "eli5", "PDPbox", "shap"] -nlp = ["nltk", "gensim", "spacy", "torchtext", "bokeh"] -rl = ["gym", "graphviz"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.8" -content-hash = "ff0500ffaf9219e3d7ea9dedcfc79ac15c1347545c910a921a43ad75c60c651f" - -[metadata.files] -appnope = [ - {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, - {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, -] -argon2-cffi = [ - {file = "argon2-cffi-20.1.0.tar.gz", hash = "sha256:d8029b2d3e4b4cea770e9e5a0104dd8fa185c1724a0f01528ae4826a6d25f97d"}, - {file = "argon2_cffi-20.1.0-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:6ea92c980586931a816d61e4faf6c192b4abce89aa767ff6581e6ddc985ed003"}, - {file = "argon2_cffi-20.1.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:05a8ac07c7026542377e38389638a8a1e9b78f1cd8439cd7493b39f08dd75fbf"}, - {file = "argon2_cffi-20.1.0-cp27-cp27m-win32.whl", hash = "sha256:0bf066bc049332489bb2d75f69216416329d9dc65deee127152caeb16e5ce7d5"}, - {file = "argon2_cffi-20.1.0-cp27-cp27m-win_amd64.whl", hash = "sha256:57358570592c46c420300ec94f2ff3b32cbccd10d38bdc12dc6979c4a8484fbc"}, - {file = "argon2_cffi-20.1.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7d455c802727710e9dfa69b74ccaab04568386ca17b0ad36350b622cd34606fe"}, - {file = "argon2_cffi-20.1.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:b160416adc0f012fb1f12588a5e6954889510f82f698e23ed4f4fa57f12a0647"}, - {file = "argon2_cffi-20.1.0-cp35-cp35m-win32.whl", hash = "sha256:9bee3212ba4f560af397b6d7146848c32a800652301843df06b9e8f68f0f7361"}, - {file = "argon2_cffi-20.1.0-cp35-cp35m-win_amd64.whl", hash = "sha256:392c3c2ef91d12da510cfb6f9bae52512a4552573a9e27600bdb800e05905d2b"}, - {file = "argon2_cffi-20.1.0-cp36-cp36m-win32.whl", hash = "sha256:ba7209b608945b889457f949cc04c8e762bed4fe3fec88ae9a6b7765ae82e496"}, - {file = "argon2_cffi-20.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:da7f0445b71db6d3a72462e04f36544b0de871289b0bc8a7cc87c0f5ec7079fa"}, - {file = "argon2_cffi-20.1.0-cp37-abi3-macosx_10_6_intel.whl", hash = "sha256:cc0e028b209a5483b6846053d5fd7165f460a1f14774d79e632e75e7ae64b82b"}, - {file = "argon2_cffi-20.1.0-cp37-cp37m-win32.whl", hash = "sha256:18dee20e25e4be86680b178b35ccfc5d495ebd5792cd00781548d50880fee5c5"}, - {file = "argon2_cffi-20.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6678bb047373f52bcff02db8afab0d2a77d83bde61cfecea7c5c62e2335cb203"}, - {file = "argon2_cffi-20.1.0-cp38-cp38-win32.whl", hash = "sha256:77e909cc756ef81d6abb60524d259d959bab384832f0c651ed7dcb6e5ccdbb78"}, - {file = "argon2_cffi-20.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:9dfd5197852530294ecb5795c97a823839258dfd5eb9420233c7cfedec2058f2"}, - {file = "argon2_cffi-20.1.0-cp39-cp39-win32.whl", hash = "sha256:e2db6e85c057c16d0bd3b4d2b04f270a7467c147381e8fd73cbbe5bc719832be"}, - {file = "argon2_cffi-20.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a84934bd818e14a17943de8099d41160da4a336bcc699bb4c394bbb9b94bd32"}, - {file = "argon2_cffi-20.1.0-pp36-pypy36_pp73-macosx_10_7_x86_64.whl", hash = "sha256:b94042e5dcaa5d08cf104a54bfae614be502c6f44c9c89ad1535b2ebdaacbd4c"}, - {file = "argon2_cffi-20.1.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:8282b84ceb46b5b75c3a882b28856b8cd7e647ac71995e71b6705ec06fc232c3"}, - {file = "argon2_cffi-20.1.0-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3aa804c0e52f208973845e8b10c70d8957c9e5a666f702793256242e9167c4e0"}, - {file = "argon2_cffi-20.1.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:36320372133a003374ef4275fbfce78b7ab581440dfca9f9471be3dd9a522428"}, -] -async-generator = [ - {file = "async_generator-1.10-py3-none-any.whl", hash = "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b"}, - {file = "async_generator-1.10.tar.gz", hash = "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144"}, -] -attrs = [ - {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, - {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, -] -backcall = [ - {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, - {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, -] -"backports.entry-points-selectable" = [ - {file = "backports.entry_points_selectable-1.1.0-py2.py3-none-any.whl", hash = "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc"}, - {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, -] -bleach = [ - {file = "bleach-4.0.0-py2.py3-none-any.whl", hash = "sha256:c1685a132e6a9a38bf93752e5faab33a9517a6c0bb2f37b785e47bf253bdb51d"}, - {file = "bleach-4.0.0.tar.gz", hash = "sha256:ffa9221c6ac29399cc50fcc33473366edd0cf8d5e2cbbbb63296dc327fb67cc8"}, -] -blis = [ - {file = "blis-0.7.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:5b403deb2ad5515e1edb3c0867bccb5b974b461f24283d9219a3a761fd6dacc6"}, - {file = "blis-0.7.4-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9f9b829480c12fc834549306821e5c51cb28b216ca5f88c5b2cfedbeb9daf67d"}, - {file = "blis-0.7.4-cp36-cp36m-win_amd64.whl", hash = "sha256:c2d8064217c326dd9a0dcbae294ffe8557263e2a00d76101ffa222b9c9d9c62d"}, - {file = "blis-0.7.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d717b5dea407aac89a646908e7d9849105abab9c88a539c120518c200f899f4e"}, - {file = "blis-0.7.4-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5ecddc4c6daf80558154b091db0a9839bb15dbe65d2906a543a73b93fbce4f73"}, - {file = "blis-0.7.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6814991b3e3193db4f9b2417174c6f24b9c0197409d864fa7628583bd2df1f0f"}, - {file = "blis-0.7.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4222bbc7b9c47bc3cf6f36f2241862c1512ca7ebac3828267a2e05ef6c47fc54"}, - {file = "blis-0.7.4-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:445e4838b809e99677f5c0982fb9af320f0d91328fb28c8097e5f1173c4df9d6"}, - {file = "blis-0.7.4-cp38-cp38-win_amd64.whl", hash = "sha256:94890b2296f1449baa56aede46627ea7fc8de11c788f9c261ee38c2eb4a2cc7d"}, - {file = "blis-0.7.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:168fd7bd763ebe529aa25a066d3a6b89f4c3f492f6297f881df6942741b95787"}, - {file = "blis-0.7.4-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:5c1a2023f7d8431daa8d87d32f539bb23e1a009500c37f9eba0ac7b3f20f73eb"}, - {file = "blis-0.7.4-cp39-cp39-win_amd64.whl", hash = "sha256:78a8e0ee72a42c3b2f5b9114500a781119995f76fa6c21d4b02c6fb9c21df2cc"}, - {file = "blis-0.7.4.tar.gz", hash = "sha256:7daa615a97d4f28db0f332b710bfe1900b15d0c25841c6d727965e4fd91e09cf"}, -] -bokeh = [ - {file = "bokeh-2.3.3.tar.gz", hash = "sha256:a5fdcc181835561447fcc5a371300973fce4114692d5853addec284d1cdeb677"}, -] -boto3 = [ - {file = "boto3-1.18.19-py3-none-any.whl", hash = "sha256:72b1f70a5a42dff0a9c26a71486d3dcb3e098fac5b36126bc8fdcdec8c4d3cf4"}, - {file = "boto3-1.18.19.tar.gz", hash = "sha256:096f771c259484dc7140af2b7a9078e9c3efba28e2a298d1e8e40fed404fa38e"}, -] -botocore = [ - {file = "botocore-1.21.19-py3-none-any.whl", hash = "sha256:2fa40a39b338888c9492dc1e36734d8807f9e1c6f5dd3514247338e97f4da0f6"}, - {file = "botocore-1.21.19.tar.gz", hash = "sha256:7dce88db827e9b5c88701c978df00742c854d2b751fbda8db7656fb9a571afc5"}, -] -catalogue = [ - {file = "catalogue-2.0.4-py3-none-any.whl", hash = "sha256:62572ad1a641face0eb1436921ee4e03169162879bdc25ab8d535219b5f65b48"}, - {file = "catalogue-2.0.4.tar.gz", hash = "sha256:9ed345d12855af315f1715583612b26b8621a2b0a2e3bef974dc5d712f7983aa"}, -] -certifi = [ - {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, - {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, -] -cffi = [ - {file = "cffi-1.14.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c"}, - {file = "cffi-1.14.6-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99"}, - {file = "cffi-1.14.6-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819"}, - {file = "cffi-1.14.6-cp27-cp27m-win32.whl", hash = "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20"}, - {file = "cffi-1.14.6-cp27-cp27m-win_amd64.whl", hash = "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224"}, - {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7"}, - {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33"}, - {file = "cffi-1.14.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5"}, - {file = "cffi-1.14.6-cp35-cp35m-win32.whl", hash = "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca"}, - {file = "cffi-1.14.6-cp35-cp35m-win_amd64.whl", hash = "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218"}, - {file = "cffi-1.14.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f"}, - {file = "cffi-1.14.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872"}, - {file = "cffi-1.14.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195"}, - {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d"}, - {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b"}, - {file = "cffi-1.14.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb"}, - {file = "cffi-1.14.6-cp36-cp36m-win32.whl", hash = "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a"}, - {file = "cffi-1.14.6-cp36-cp36m-win_amd64.whl", hash = "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e"}, - {file = "cffi-1.14.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5"}, - {file = "cffi-1.14.6-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf"}, - {file = "cffi-1.14.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69"}, - {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56"}, - {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c"}, - {file = "cffi-1.14.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762"}, - {file = "cffi-1.14.6-cp37-cp37m-win32.whl", hash = "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771"}, - {file = "cffi-1.14.6-cp37-cp37m-win_amd64.whl", hash = "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a"}, - {file = "cffi-1.14.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0"}, - {file = "cffi-1.14.6-cp38-cp38-manylinux1_i686.whl", hash = "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e"}, - {file = "cffi-1.14.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346"}, - {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc"}, - {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd"}, - {file = "cffi-1.14.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc"}, - {file = "cffi-1.14.6-cp38-cp38-win32.whl", hash = "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548"}, - {file = "cffi-1.14.6-cp38-cp38-win_amd64.whl", hash = "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156"}, - {file = "cffi-1.14.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d"}, - {file = "cffi-1.14.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e"}, - {file = "cffi-1.14.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c"}, - {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202"}, - {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f"}, - {file = "cffi-1.14.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87"}, - {file = "cffi-1.14.6-cp39-cp39-win32.whl", hash = "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728"}, - {file = "cffi-1.14.6-cp39-cp39-win_amd64.whl", hash = "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2"}, - {file = "cffi-1.14.6.tar.gz", hash = "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd"}, -] -cfgv = [ - {file = "cfgv-3.3.0-py2.py3-none-any.whl", hash = "sha256:b449c9c6118fe8cca7fa5e00b9ec60ba08145d281d52164230a69211c5d597a1"}, - {file = "cfgv-3.3.0.tar.gz", hash = "sha256:9e600479b3b99e8af981ecdfc80a0296104ee610cab48a5ae4ffd0b668650eb1"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, - {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, -] -click = [ - {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, - {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, -] -cloudpickle = [ - {file = "cloudpickle-1.6.0-py3-none-any.whl", hash = "sha256:3a32d0eb0bc6f4d0c57fbc4f3e3780f7a81e6fee0fa935072884d58ae8e1cc7c"}, - {file = "cloudpickle-1.6.0.tar.gz", hash = "sha256:9bc994f9e9447593bd0a45371f0e7ac7333710fcf64a4eb9834bf149f4ef2f32"}, -] -colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] -cycler = [ - {file = "cycler-0.10.0-py2.py3-none-any.whl", hash = "sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d"}, - {file = "cycler-0.10.0.tar.gz", hash = "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8"}, -] -cymem = [ - {file = "cymem-2.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9d72d69f7a62a280199c3aa7bc550685c47d6d0689b2d299e6492253b86d2437"}, - {file = "cymem-2.0.5-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:8ea57e6923f40eb51012352161bb5707c14a5a5ce901ff72021e59df06221655"}, - {file = "cymem-2.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:4bd023c2477198b39b660c2a6b0242880649765ecee8461688a57fd4afd2bfc0"}, - {file = "cymem-2.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1f0eb9b3d03623dcfc746cf8bff0663b0e347f4aea759965c8932087a0307ee9"}, - {file = "cymem-2.0.5-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:a440d63577fcdc9c528c9cc026b7b4f8648193bac462bc0596c9eac10f9fba62"}, - {file = "cymem-2.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:3d48902d7441645835fefc7832df49feb5362c7300d182475b63a01d25ae44ef"}, - {file = "cymem-2.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2167c9959fcd639b95d51fa5efaa7c61eef8d686cb75a25412a914f428ce980"}, - {file = "cymem-2.0.5-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:734d82d0d03c2ceb929bc1744c04dbe0a105e68a4947c8406056a36f86c41830"}, - {file = "cymem-2.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:01d3ea159f7a3f3192b1e800ed8207dac7586794d903a153198b9ea317f144bc"}, - {file = "cymem-2.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d307f7f6230d861a938837cae4b855226b6845a21c010242a15e9ce6853856cd"}, - {file = "cymem-2.0.5-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:ce1e81c1d031f56b67bac2136e73b4512cbc794706cd570178972d54ba6115d8"}, - {file = "cymem-2.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d19f68b90411e02ab33b1654118337f96f41c13a3cd00c4f44f7abed2bc712e7"}, - {file = "cymem-2.0.5.tar.gz", hash = "sha256:190e15d9cf2c3bde60ae37bddbae6568a36044dc4a326d84081a5fa08818eee0"}, -] -cython = [ - {file = "Cython-0.29.14-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:47e5e1502d52ef03387cf9d3b3241007961a84a466e58a3b74028e1dd4957f8c"}, - {file = "Cython-0.29.14-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1dcdaa319558eb924294a554dcf6c12383ec947acc7e779e8d3622409a7f7d28"}, - {file = "Cython-0.29.14-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:7bc18fc5a170f2c1cef5387a3d997c28942918bbee0f700e73fd2178ee8d474d"}, - {file = "Cython-0.29.14-cp27-cp27m-win32.whl", hash = "sha256:89458b49976b1dee5d89ab4ac943da3717b4292bf624367e862e4ee172fcce99"}, - {file = "Cython-0.29.14-cp27-cp27m-win_amd64.whl", hash = "sha256:c0b24bfe3431b3cb7ced323bca813dbd13aca973a1475b512d3331fd0de8ec60"}, - {file = "Cython-0.29.14-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7f89eff20e4a7a64b55210dac17aea711ed8a3f2e78f2ff784c0e984302583dd"}, - {file = "Cython-0.29.14-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:6c53338c1811f8c6d7f8cb7abd874810b15045e719e8207f957035c9177b4213"}, - {file = "Cython-0.29.14-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:521340844cf388d109ceb61397f3fd5250ccb622a1a8e93559e8de76c80940a9"}, - {file = "Cython-0.29.14-cp34-cp34m-manylinux1_i686.whl", hash = "sha256:75c2dda47dcc3c77449712b1417bb6b89ec3b7b02e18c64262494dceffdf455e"}, - {file = "Cython-0.29.14-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:05eb79efc8029d487251c8a2702a909a8ba33c332e06d2f3980866541bd81253"}, - {file = "Cython-0.29.14-cp34-cp34m-win32.whl", hash = "sha256:1fc5bdda28f25fec44e4721677458aa509d743cd350862270309d61aa148d6ff"}, - {file = "Cython-0.29.14-cp34-cp34m-win_amd64.whl", hash = "sha256:0c70e842e52e2f50cc43bad43b5e5bc515f30821a374e544abb0e0746f2350ff"}, - {file = "Cython-0.29.14-cp35-cp35m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:094d28a34c3fa992ae02aea1edbe6ff89b3cc5870b6ee38b5baeb805dc57b013"}, - {file = "Cython-0.29.14-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:280573a01d9348d44a42d6a9c651d9f7eb1fe9217df72555b2a118f902996a10"}, - {file = "Cython-0.29.14-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:773c5a98e463b52f7e8197254b39b703a5ea1972aef3a94b3b921515d77dd041"}, - {file = "Cython-0.29.14-cp35-cp35m-win32.whl", hash = "sha256:986f871c0fa649b293061236b93782d25c293a8dd8117c7ba05f8a61bdc261ae"}, - {file = "Cython-0.29.14-cp35-cp35m-win_amd64.whl", hash = "sha256:78c3068dcba300d473fef57cdf523e34b37de522f5a494ef9ee1ac9b4b8bbe3f"}, - {file = "Cython-0.29.14-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:f3818e578e687cdb21dc4aa4a3bc6278c656c9c393e9eda14dd04943f478863d"}, - {file = "Cython-0.29.14-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bb487881608ebd293592553c618f0c83316f4f13a64cb18605b1d2fb9fd3da3e"}, - {file = "Cython-0.29.14-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:03f6bbb380ad0acb744fb06e42996ea217e9d00016ca0ff6f2e7d60f580d0360"}, - {file = "Cython-0.29.14-cp36-cp36m-win32.whl", hash = "sha256:b8ab3ab38afc47d8f4fe629b836243544351cef681b6bdb1dc869028d6fdcbfb"}, - {file = "Cython-0.29.14-cp36-cp36m-win_amd64.whl", hash = "sha256:298ceca7b0f0da4205fcb0b7c9ac9e120e2dafffd5019ba1618e84ef89434b5a"}, - {file = "Cython-0.29.14-cp37-cp37m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:fe666645493d72712c46e4fbe8bec094b06aec3c337400479e9704439c9d9586"}, - {file = "Cython-0.29.14-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4074a8bff0040035673cc6dd365a762476d6bff4d03d8ce6904e3e53f9a25dc8"}, - {file = "Cython-0.29.14-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a14aa436586c41633339415de82a41164691d02d3e661038da533be5d40794a5"}, - {file = "Cython-0.29.14-cp37-cp37m-win32.whl", hash = "sha256:41e7068e95fbf9ec94b41437f989caf9674135e770a39cdb9c00de459bafd1bc"}, - {file = "Cython-0.29.14-cp37-cp37m-win_amd64.whl", hash = "sha256:05e8cfd3a3a6087aec49a1ae08a89171db991956209406d1e5576f9db70ece52"}, - {file = "Cython-0.29.14-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e8fab9911fd2fa8e5af407057cb8bdf87762f983cba483fa3234be20a9a0af77"}, - {file = "Cython-0.29.14-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:d4039bb7f234ad32267c55e72fd49fb56078ea102f9d9d8559f6ec34d4887630"}, - {file = "Cython-0.29.14-cp38-cp38-win32.whl", hash = "sha256:c7894c06205166d360ab2915ae306d1f7403e9ce3d3aaeff4095eaf98e42ce66"}, - {file = "Cython-0.29.14-cp38-cp38-win_amd64.whl", hash = "sha256:a0f495a4fe5278aab278feee35e6102efecde5176a8a74dd28c28e3fc5c8d7c7"}, - {file = "Cython-0.29.14.tar.gz", hash = "sha256:e4d6bb8703d0319eb04b7319b12ea41580df44fd84d83ccda13ea463c6801414"}, -] -decorator = [ - {file = "decorator-5.0.9-py3-none-any.whl", hash = "sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323"}, - {file = "decorator-5.0.9.tar.gz", hash = "sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5"}, -] -defusedxml = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] -distlib = [ - {file = "distlib-0.3.2-py2.py3-none-any.whl", hash = "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c"}, - {file = "distlib-0.3.2.zip", hash = "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736"}, -] -eli5 = [ - {file = "eli5-0.11.0-py2.py3-none-any.whl", hash = "sha256:1ea45cd0722d20c8c9e9bb89c7c5909feeface4e5942e24b7a89809f0fe593a2"}, - {file = "eli5-0.11.0.tar.gz", hash = "sha256:aea7b51be9157ce615b319711467f358de03da12328e5639818b3cb3755aa056"}, -] -entrypoints = [ - {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, - {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, -] -filelock = [ - {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, - {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, -] -gensim = [ - {file = "gensim-3.8.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:61eed1d6b5fbe6dda0586ea447ebc2dc7890a7f70c2ed953d5abc3fe3cfb94bb"}, - {file = "gensim-3.8.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3af62709369331c85552fd26caa21504baa64accc426dc094172f5c688750013"}, - {file = "gensim-3.8.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:8ff471921b3b10ffb3ae6cbb598dd9c07d9dc030dee5aa167e7682b549c42f87"}, - {file = "gensim-3.8.3-cp27-cp27m-win32.whl", hash = "sha256:440700e29b494bc2e1d52e14b69a821f46ab09ecf85cf36c8988f18e1d6c7a8b"}, - {file = "gensim-3.8.3-cp27-cp27m-win_amd64.whl", hash = "sha256:f8ea67bf8c47ee55cb1b32c97fa1474b7d6d22959dd8097c019a5d9c9df34f5f"}, - {file = "gensim-3.8.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7a90549dfc8ee3822fcad6da957de07d927e4e90ef42b3699543dee35ab2da13"}, - {file = "gensim-3.8.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7629b33cf35f672efdd5269381f7e301958ee2638f27dfc63b80c5bfeaa827d3"}, - {file = "gensim-3.8.3-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:6711b6d3a0007530ee7de7adc30a4c48a1d26ec6312ac50e1d1e0a1d54f9de5b"}, - {file = "gensim-3.8.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:ef2ddeceff482aee17c1e185f63bf027c8de8f595fdd9fd2d2503de96008f3b7"}, - {file = "gensim-3.8.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:41dcf6ecdc9acc657157967c791b8cbaba90ee6391f64efd28339b72f5e0c327"}, - {file = "gensim-3.8.3-cp35-cp35m-win32.whl", hash = "sha256:685a7657278161628821c8f873c5d7d2ffc0c28866648e39f76b450e4c7d5390"}, - {file = "gensim-3.8.3-cp35-cp35m-win_amd64.whl", hash = "sha256:b61a7c841a752c84b685674aa0d610289faad38795b325176481abe19b487e98"}, - {file = "gensim-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a61179df454a0d4b06a111c4ede0536f61c8121b4c0d0d02d23560a2fd4b3aff"}, - {file = "gensim-3.8.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:cc387d0d8bddbf3609ab95b3453296e4c9ff92c35e9799a17d86b1571d77a5fc"}, - {file = "gensim-3.8.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:b36e6330471061cfd78aad751e24c6b4f56d575697af0fbab42655128927d296"}, - {file = "gensim-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:1e3d66c2eec494376fc599701d9c2868549aed6e93e47177e39217f0188e2d88"}, - {file = "gensim-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:91fa62d61b21f1878f140b10520f9de4a26a52672fbe407edfc7e09ca2eff235"}, - {file = "gensim-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:637fc5969f3cef4b7c8fd3e78e31ef09565c5566d5ceabf076b4170eb6444a80"}, - {file = "gensim-3.8.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:22f45fd239cacd0e3715ac447a2c8a5eea02e730ec1f701c55b359e9298e63a8"}, - {file = "gensim-3.8.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d79370f78e9013b9d1e867c85ecc678d46a7ae0f01a8ca29e8f4291e5373b170"}, - {file = "gensim-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:9c214b341f5304b906c79844e2787c13b46505df9dc70afca79a9a7dc0894478"}, - {file = "gensim-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fe98277a7b3b4987b40c928056bbaae1d0715022cf27bba89d05cd0d4fe51a84"}, - {file = "gensim-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a47903d104469a7a8b6f22ad5ef74681b19c4f4b71ff2c2893271b53161a43e4"}, - {file = "gensim-3.8.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:05bfc02e102a34c9c795095b688b1b4aaa2529c624821368c9c3ea6a16536f77"}, - {file = "gensim-3.8.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a8807ebf324dd11e1298a91a92d6e57c7bdabb91d0d5240bf1efa0c0eacd86f0"}, - {file = "gensim-3.8.3-cp38-cp38-win32.whl", hash = "sha256:90115d12ee545c21cc75521ef1bb3dd66aae8a378e9c2eb029c9f22df173c125"}, - {file = "gensim-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:4e34cf2e50f3eab3e303da46089ea4972567bf216e28f7535ada155770784ac8"}, - {file = "gensim-3.8.3.tar.gz", hash = "sha256:786adb0571f75114e9c5f7a31dd2e6eb39a9791f22c8757621545e2ded3ea367"}, -] -graphviz = [ - {file = "graphviz-0.16-py2.py3-none-any.whl", hash = "sha256:3cad5517c961090dfc679df6402a57de62d97703e2880a1a46147bb0dc1639eb"}, - {file = "graphviz-0.16.zip", hash = "sha256:d2d25af1c199cad567ce4806f0449cb74eb30cf451fd7597251e1da099ac6e57"}, -] -gym = [ - {file = "gym-0.18.3.tar.gz", hash = "sha256:81a3e3fbf7fcf57c8cf98f7e22d1bdd5815f3824d9c148a7eb42420d3d642967"}, -] -h5py = [ - {file = "h5py-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f3bba8ffddd1fd2bf06127c5ff7b73f022cc1c8b7164355ddc760dc3f8570136"}, - {file = "h5py-3.3.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baef1a2cdef287a83e7f95ce9e0f4d762a9852fe7117b471063442c78b973695"}, - {file = "h5py-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8e09b682e4059c8cd259ddcc34bee35d639b9170105efeeae6ad195e7c1cea7a"}, - {file = "h5py-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:89d7e10409b62fed81c571e35798763cb8375442b98f8ebfc52ba41ac019e081"}, - {file = "h5py-3.3.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7ca7d23ebbdd59a4be9b4820de52fe67adc74e6a44d5084881305461765aac47"}, - {file = "h5py-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e0ea3330bf136f8213e43db67448994046ce501585dddc7ea4e8ceef0ef1600c"}, - {file = "h5py-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:13355234c004ff8bd819f7d3420188aa1936b17d7f8470d622974a373421b7a5"}, - {file = "h5py-3.3.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:09e78cefdef0b7566ab66366c5c7d9984c7b23142245bd51b82b744ad1eebf65"}, - {file = "h5py-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:5e2f22e66a3fb1815405cfe5711670450c973b8552507c535a546a23a468af3d"}, - {file = "h5py-3.3.0.tar.gz", hash = "sha256:e0dac887d779929778b3cfd13309a939359cc9e74756fc09af7c527a82797186"}, -] -identify = [ - {file = "identify-2.2.13-py2.py3-none-any.whl", hash = "sha256:7199679b5be13a6b40e6e19ea473e789b11b4e3b60986499b1f589ffb03c217c"}, - {file = "identify-2.2.13.tar.gz", hash = "sha256:7bc6e829392bd017236531963d2d937d66fc27cadc643ac0aba2ce9f26157c79"}, -] -idna = [ - {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, - {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, -] -imageio = [ - {file = "imageio-2.9.0-py3-none-any.whl", hash = "sha256:3604d751f03002e8e0e7650aa71d8d9148144a87daf17cb1f3228e80747f2e6b"}, - {file = "imageio-2.9.0.tar.gz", hash = "sha256:52ddbaeca2dccf53ba2d6dec5676ca7bc3b2403ef8b37f7da78b7654bb3e10f0"}, -] -ipykernel = [ - {file = "ipykernel-5.5.5-py3-none-any.whl", hash = "sha256:29eee66548ee7c2edb7941de60c0ccf0a7a8dd957341db0a49c5e8e6a0fcb712"}, - {file = "ipykernel-5.5.5.tar.gz", hash = "sha256:e976751336b51082a89fc2099fb7f96ef20f535837c398df6eab1283c2070884"}, -] -ipython = [ - {file = "ipython-7.26.0-py3-none-any.whl", hash = "sha256:892743b65c21ed72b806a3a602cca408520b3200b89d1924f4b3d2cdb3692362"}, - {file = "ipython-7.26.0.tar.gz", hash = "sha256:0cff04bb042800129348701f7bd68a430a844e8fb193979c08f6c99f28bb735e"}, -] -ipython-genutils = [ - {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"}, - {file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"}, -] -ipywidgets = [ - {file = "ipywidgets-7.6.3-py2.py3-none-any.whl", hash = "sha256:e6513cfdaf5878de30f32d57f6dc2474da395a2a2991b94d487406c0ab7f55ca"}, - {file = "ipywidgets-7.6.3.tar.gz", hash = "sha256:9f1a43e620530f9e570e4a493677d25f08310118d315b00e25a18f12913c41f0"}, -] -jedi = [ - {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"}, - {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"}, -] -jinja2 = [ - {file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"}, - {file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"}, -] -jmespath = [ - {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, - {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, -] -joblib = [ - {file = "joblib-1.0.1-py3-none-any.whl", hash = "sha256:feeb1ec69c4d45129954f1b7034954241eedfd6ba39b5e9e4b6883be3332d5e5"}, - {file = "joblib-1.0.1.tar.gz", hash = "sha256:9c17567692206d2f3fb9ecf5e991084254fe631665c450b443761c4186a613f7"}, -] -jsonschema = [ - {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, - {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, -] -jupyter-client = [ - {file = "jupyter_client-6.2.0-py3-none-any.whl", hash = "sha256:9715152067e3f7ea3b56f341c9a0f9715c8c7cc316ee0eb13c3c84f5ca0065f5"}, - {file = "jupyter_client-6.2.0.tar.gz", hash = "sha256:e2ab61d79fbf8b56734a4c2499f19830fbd7f6fefb3e87868ef0545cb3c17eb9"}, -] -jupyter-core = [ - {file = "jupyter_core-4.7.1-py3-none-any.whl", hash = "sha256:8c6c0cac5c1b563622ad49321d5ec47017bd18b94facb381c6973a0486395f8e"}, - {file = "jupyter_core-4.7.1.tar.gz", hash = "sha256:79025cb3225efcd36847d0840f3fc672c0abd7afd0de83ba8a1d3837619122b4"}, -] -jupyterlab-pygments = [ - {file = "jupyterlab_pygments-0.1.2-py2.py3-none-any.whl", hash = "sha256:abfb880fd1561987efaefcb2d2ac75145d2a5d0139b1876d5be806e32f630008"}, - {file = "jupyterlab_pygments-0.1.2.tar.gz", hash = "sha256:cfcda0873626150932f438eccf0f8bf22bfa92345b814890ab360d666b254146"}, -] -jupyterlab-widgets = [ - {file = "jupyterlab_widgets-1.0.0-py3-none-any.whl", hash = "sha256:caeaf3e6103180e654e7d8d2b81b7d645e59e432487c1d35a41d6d3ee56b3fef"}, - {file = "jupyterlab_widgets-1.0.0.tar.gz", hash = "sha256:5c1a29a84d3069208cb506b10609175b249b6486d6b1cbae8fcde2a11584fb78"}, -] -kiwisolver = [ - {file = "kiwisolver-1.3.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fd34fbbfbc40628200730bc1febe30631347103fc8d3d4fa012c21ab9c11eca9"}, - {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:d3155d828dec1d43283bd24d3d3e0d9c7c350cdfcc0bd06c0ad1209c1bbc36d0"}, - {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5a7a7dbff17e66fac9142ae2ecafb719393aaee6a3768c9de2fd425c63b53e21"}, - {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:f8d6f8db88049a699817fd9178782867bf22283e3813064302ac59f61d95be05"}, - {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:5f6ccd3dd0b9739edcf407514016108e2280769c73a85b9e59aa390046dbf08b"}, - {file = "kiwisolver-1.3.1-cp36-cp36m-win32.whl", hash = "sha256:225e2e18f271e0ed8157d7f4518ffbf99b9450fca398d561eb5c4a87d0986dd9"}, - {file = "kiwisolver-1.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:cf8b574c7b9aa060c62116d4181f3a1a4e821b2ec5cbfe3775809474113748d4"}, - {file = "kiwisolver-1.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:232c9e11fd7ac3a470d65cd67e4359eee155ec57e822e5220322d7b2ac84fbf0"}, - {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:b38694dcdac990a743aa654037ff1188c7a9801ac3ccc548d3341014bc5ca278"}, - {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ca3820eb7f7faf7f0aa88de0e54681bddcb46e485beb844fcecbcd1c8bd01689"}, - {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:c8fd0f1ae9d92b42854b2979024d7597685ce4ada367172ed7c09edf2cef9cb8"}, - {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:1e1bc12fb773a7b2ffdeb8380609f4f8064777877b2225dec3da711b421fda31"}, - {file = "kiwisolver-1.3.1-cp37-cp37m-win32.whl", hash = "sha256:72c99e39d005b793fb7d3d4e660aed6b6281b502e8c1eaf8ee8346023c8e03bc"}, - {file = "kiwisolver-1.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:8be8d84b7d4f2ba4ffff3665bcd0211318aa632395a1a41553250484a871d454"}, - {file = "kiwisolver-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:31dfd2ac56edc0ff9ac295193eeaea1c0c923c0355bf948fbd99ed6018010b72"}, - {file = "kiwisolver-1.3.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:563c649cfdef27d081c84e72a03b48ea9408c16657500c312575ae9d9f7bc1c3"}, - {file = "kiwisolver-1.3.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:78751b33595f7f9511952e7e60ce858c6d64db2e062afb325985ddbd34b5c131"}, - {file = "kiwisolver-1.3.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a357fd4f15ee49b4a98b44ec23a34a95f1e00292a139d6015c11f55774ef10de"}, - {file = "kiwisolver-1.3.1-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:5989db3b3b34b76c09253deeaf7fbc2707616f130e166996606c284395da3f18"}, - {file = "kiwisolver-1.3.1-cp38-cp38-win32.whl", hash = "sha256:c08e95114951dc2090c4a630c2385bef681cacf12636fb0241accdc6b303fd81"}, - {file = "kiwisolver-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:44a62e24d9b01ba94ae7a4a6c3fb215dc4af1dde817e7498d901e229aaf50e4e"}, - {file = "kiwisolver-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:50af681a36b2a1dee1d3c169ade9fdc59207d3c31e522519181e12f1b3ba7000"}, - {file = "kiwisolver-1.3.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:a53d27d0c2a0ebd07e395e56a1fbdf75ffedc4a05943daf472af163413ce9598"}, - {file = "kiwisolver-1.3.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:834ee27348c4aefc20b479335fd422a2c69db55f7d9ab61721ac8cd83eb78882"}, - {file = "kiwisolver-1.3.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5c3e6455341008a054cccee8c5d24481bcfe1acdbc9add30aa95798e95c65621"}, - {file = "kiwisolver-1.3.1-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:acef3d59d47dd85ecf909c359d0fd2c81ed33bdff70216d3956b463e12c38a54"}, - {file = "kiwisolver-1.3.1-cp39-cp39-win32.whl", hash = "sha256:c5518d51a0735b1e6cee1fdce66359f8d2b59c3ca85dc2b0813a8aa86818a030"}, - {file = "kiwisolver-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:b9edd0110a77fc321ab090aaa1cfcaba1d8499850a12848b81be2222eab648f6"}, - {file = "kiwisolver-1.3.1-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0cd53f403202159b44528498de18f9285b04482bab2a6fc3f5dd8dbb9352e30d"}, - {file = "kiwisolver-1.3.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:33449715e0101e4d34f64990352bce4095c8bf13bed1b390773fc0a7295967b3"}, - {file = "kiwisolver-1.3.1-pp36-pypy36_pp73-win32.whl", hash = "sha256:401a2e9afa8588589775fe34fc22d918ae839aaaf0c0e96441c0fdbce6d8ebe6"}, - {file = "kiwisolver-1.3.1.tar.gz", hash = "sha256:950a199911a8d94683a6b10321f9345d5a3a8433ec58b217ace979e18f16e248"}, -] -llvmlite = [ - {file = "llvmlite-0.34.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:11342e5ac320c953590bdd9d0dec8c52f4b5252c4c6335ba25f1e7b9f91f9325"}, - {file = "llvmlite-0.34.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:5bdf0ce430adfaf938ced5844d12f80616eb8321b5b9edfc45ef84ada5c5242c"}, - {file = "llvmlite-0.34.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:e08d9d2dc5a31636bfc6b516d2d7daba95632afa3419eb8730dc76a7951e9558"}, - {file = "llvmlite-0.34.0-cp36-cp36m-win32.whl", hash = "sha256:9ff1dcdad03be0cf953aca5fc8cffdca25ccee2ec9e8ec7e95571722cdc02d55"}, - {file = "llvmlite-0.34.0-cp36-cp36m-win_amd64.whl", hash = "sha256:5acdc3c3c7ea0ef7a1a6b442272e05d695bc8492e5b07666135ed1cfbf4ab9d2"}, - {file = "llvmlite-0.34.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bb96989bc57a1ccb131e7a0e061d07b68139b6f81a98912345d53d9239e231e1"}, - {file = "llvmlite-0.34.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:6d3f81992f52a94077e7b9b16497029daf5b5eebb2cce56f3c8345bbc9c6308e"}, - {file = "llvmlite-0.34.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d841248d1c630426c93e3eb3f8c45bca0dab77c09faeb7553b1a500220e362ce"}, - {file = "llvmlite-0.34.0-cp37-cp37m-win32.whl", hash = "sha256:408b15ffec30696406e821c89da010f1bb1eb0aa572be4561c98eb2536d610ab"}, - {file = "llvmlite-0.34.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5d1f370bf150db7239204f09cf6a0603292ea28bac984e69b167e16fe160d803"}, - {file = "llvmlite-0.34.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:132322bc084abf336c80dd106f9357978c8c085911fb656898d3be0d9ff057ea"}, - {file = "llvmlite-0.34.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:8f344102745fceba6eb5bf03c228bb290e9bc79157e9506a4a72878d636f9b3c"}, - {file = "llvmlite-0.34.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:05253f3f44fab0148276335b2c1b2c4a78143dfa78e6bafd7f937d6248f297cc"}, - {file = "llvmlite-0.34.0-cp38-cp38-win32.whl", hash = "sha256:28264f9e2b3df4135cbcfca5a91c5b0b31dd3fc02fa623b4bb13327f0cd4fc80"}, - {file = "llvmlite-0.34.0-cp38-cp38-win_amd64.whl", hash = "sha256:964f8f7a2184963cb3617d057c2382575953e488b7bb061b632ee014cfef110a"}, - {file = "llvmlite-0.34.0.tar.gz", hash = "sha256:f03ee0d19bca8f2fe922bb424a909d05c28411983b0c2bc58b020032a0d11f63"}, -] -markupsafe = [ - {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, - {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, -] -matplotlib = [ - {file = "matplotlib-3.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c541ee5a3287efe066bbe358320853cf4916bc14c00c38f8f3d8d75275a405a9"}, - {file = "matplotlib-3.4.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:3a5c18dbd2c7c366da26a4ad1462fe3e03a577b39e3b503bbcf482b9cdac093c"}, - {file = "matplotlib-3.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a9d8cb5329df13e0cdaa14b3b43f47b5e593ec637f13f14db75bb16e46178b05"}, - {file = "matplotlib-3.4.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:7ad19f3fb6145b9eb41c08e7cbb9f8e10b91291396bee21e9ce761bb78df63ec"}, - {file = "matplotlib-3.4.2-cp37-cp37m-win32.whl", hash = "sha256:7a58f3d8fe8fac3be522c79d921c9b86e090a59637cb88e3bc51298d7a2c862a"}, - {file = "matplotlib-3.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6382bc6e2d7e481bcd977eb131c31dee96e0fb4f9177d15ec6fb976d3b9ace1a"}, - {file = "matplotlib-3.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a6a44f27aabe720ec4fd485061e8a35784c2b9ffa6363ad546316dfc9cea04e"}, - {file = "matplotlib-3.4.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1c1779f7ab7d8bdb7d4c605e6ffaa0614b3e80f1e3c8ccf7b9269a22dbc5986b"}, - {file = "matplotlib-3.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5826f56055b9b1c80fef82e326097e34dc4af8c7249226b7dd63095a686177d1"}, - {file = "matplotlib-3.4.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0bea5ec5c28d49020e5d7923c2725b837e60bc8be99d3164af410eb4b4c827da"}, - {file = "matplotlib-3.4.2-cp38-cp38-win32.whl", hash = "sha256:6475d0209024a77f869163ec3657c47fed35d9b6ed8bccba8aa0f0099fbbdaa8"}, - {file = "matplotlib-3.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:21b31057bbc5e75b08e70a43cefc4c0b2c2f1b1a850f4a0f7af044eb4163086c"}, - {file = "matplotlib-3.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b26535b9de85326e6958cdef720ecd10bcf74a3f4371bf9a7e5b2e659c17e153"}, - {file = "matplotlib-3.4.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:32fa638cc10886885d1ca3d409d4473d6a22f7ceecd11322150961a70fab66dd"}, - {file = "matplotlib-3.4.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:956c8849b134b4a343598305a3ca1bdd3094f01f5efc8afccdebeffe6b315247"}, - {file = "matplotlib-3.4.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:85f191bb03cb1a7b04b5c2cca4792bef94df06ef473bc49e2818105671766fee"}, - {file = "matplotlib-3.4.2-cp39-cp39-win32.whl", hash = "sha256:b1d5a2cedf5de05567c441b3a8c2651fbde56df08b82640e7f06c8cd91e201f6"}, - {file = "matplotlib-3.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:df815378a754a7edd4559f8c51fc7064f779a74013644a7f5ac7a0c31f875866"}, - {file = "matplotlib-3.4.2.tar.gz", hash = "sha256:d8d994cefdff9aaba45166eb3de4f5211adb4accac85cbf97137e98f26ea0219"}, -] -matplotlib-inline = [ - {file = "matplotlib-inline-0.1.2.tar.gz", hash = "sha256:f41d5ff73c9f5385775d5c0bc13b424535c8402fe70ea8210f93e11f3683993e"}, - {file = "matplotlib_inline-0.1.2-py3-none-any.whl", hash = "sha256:5cf1176f554abb4fa98cb362aa2b55c500147e4bdbb07e3fda359143e1da0811"}, -] -mistune = [ - {file = "mistune-0.8.4-py2.py3-none-any.whl", hash = "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4"}, - {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"}, -] -murmurhash = [ - {file = "murmurhash-1.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ef8819d15973e0d6f69688bafc097a1fae081675c1de39807028869a1320b1a9"}, - {file = "murmurhash-1.0.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:76251513a2acad6c2e4b7aeffc5fcb807ee97a66cad5c2990557556555a6b7e9"}, - {file = "murmurhash-1.0.5-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:d58315961dc5a5e740f41f2ac5c3a0ebc61ef472f8afeb4db7eeb3b863243105"}, - {file = "murmurhash-1.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:23c56182822a1ed88e2a098ac56958dfec380696a9a943df203b9b41e4bcf5e4"}, - {file = "murmurhash-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:023391cfefe584ac544c1ea0936976c0119b17dd27bb8280652cef1704f76428"}, - {file = "murmurhash-1.0.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:f00321998f0a6bad3fd068babf448a296d4b0b1f4dd424cab863ebe5ed54182f"}, - {file = "murmurhash-1.0.5-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:8381172e03c5f6f947005fb146a53c5e5a9e0d630be4a40cbf8838e9324bfe1c"}, - {file = "murmurhash-1.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fed7578fbaa6c301f27ed80834c1f7494ea7d335e269e98b9aee477cf0b3b487"}, - {file = "murmurhash-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d4c3a0242014cf4c84e9ea0ba3f13b48f02a3992de3da7b1116d11b816451195"}, - {file = "murmurhash-1.0.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:99e55488476a5f70e8d305fd31258f140e52f724f788bcc50c31ec846a2b3766"}, - {file = "murmurhash-1.0.5-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:b9292c532538cf47846ca81056cfeab08b877c35fe7521d6524aa92ddcd833e2"}, - {file = "murmurhash-1.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:fd17973fd4554715efd8d86b3e9200358e49e437fdb92a897ca127aced48b61c"}, - {file = "murmurhash-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:81474a45c4074637a6dfc8fea4cdebf091ab5aa781c2cfcb94c43b16030badd7"}, - {file = "murmurhash-1.0.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:a9bd2312996e6e47605af305a1e5f091eba1bdd637cdd9986aec4885cb4c5530"}, - {file = "murmurhash-1.0.5-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:892749023da26420d194f37bfa30df1368aaac0149cfa3b2105db36b66549e37"}, - {file = "murmurhash-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:add366944eb8ec73013a4f36e166c5a4f0f7628ffe1746bc5fe031347489e5e8"}, - {file = "murmurhash-1.0.5.tar.gz", hash = "sha256:98ec9d727bd998a35385abd56b062cf0cca216725ea7ec5068604ab566f7e97f"}, -] -nbclient = [ - {file = "nbclient-0.5.3-py3-none-any.whl", hash = "sha256:e79437364a2376892b3f46bedbf9b444e5396cfb1bc366a472c37b48e9551500"}, - {file = "nbclient-0.5.3.tar.gz", hash = "sha256:db17271330c68c8c88d46d72349e24c147bb6f34ec82d8481a8f025c4d26589c"}, -] -nbconvert = [ - {file = "nbconvert-6.1.0-py3-none-any.whl", hash = "sha256:37cd92ff2ae6a268e62075ff8b16129e0be4939c4dfcee53dc77cc8a7e06c684"}, - {file = "nbconvert-6.1.0.tar.gz", hash = "sha256:d22a8ff202644d31db254d24d52c3a96c82156623fcd7c7f987bba2612303ec9"}, -] -nbformat = [ - {file = "nbformat-5.1.3-py3-none-any.whl", hash = "sha256:eb8447edd7127d043361bc17f2f5a807626bc8e878c7709a1c647abda28a9171"}, - {file = "nbformat-5.1.3.tar.gz", hash = "sha256:b516788ad70771c6250977c1374fcca6edebe6126fd2adb5a69aa5c2356fd1c8"}, -] -nest-asyncio = [ - {file = "nest_asyncio-1.5.1-py3-none-any.whl", hash = "sha256:76d6e972265063fe92a90b9cc4fb82616e07d586b346ed9d2c89a4187acea39c"}, - {file = "nest_asyncio-1.5.1.tar.gz", hash = "sha256:afc5a1c515210a23c461932765691ad39e8eba6551c055ac8d5546e69250d0aa"}, -] -networkx = [ - {file = "networkx-2.6.2-py3-none-any.whl", hash = "sha256:5fcb7004be69e8fbdf07dcb502efa5c77cadcaad6982164134eeb9721f826c2e"}, - {file = "networkx-2.6.2.tar.gz", hash = "sha256:2306f1950ce772c5a59a57f5486d59bb9cab98497c45fc49cbc45ac0dec119bb"}, -] -nltk = [ - {file = "nltk-3.6.2-py3-none-any.whl", hash = "sha256:240e23ab1ab159ef9940777d30c7c72d7e76d91877099218a7585370c11f6b9e"}, - {file = "nltk-3.6.2.zip", hash = "sha256:57d556abed621ab9be225cc6d2df1edce17572efb67a3d754630c9f8381503eb"}, -] -nodeenv = [ - {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, - {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, -] -notebook = [ - {file = "notebook-6.4.3-py3-none-any.whl", hash = "sha256:b50eafa8208d5db966efd1caa4076b4dfc51815e02a805b32ecd717e9e6cc071"}, - {file = "notebook-6.4.3.tar.gz", hash = "sha256:e6b6dfed36b00cf950f63c0d42e947c101d4258aec21624de62b9e0c11ed5c0d"}, -] -numba = [ - {file = "numba-0.51.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:af798310eeb318c56cdb83254abbe9a938cc0182d08671d7f9f032dc817e064d"}, - {file = "numba-0.51.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:93e18350f2094e7432321c1275730a3143b94af012fb609cc180fa376c44867f"}, - {file = "numba-0.51.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9e2bb1f129bfadd757ad7a9c18ab79c3ab25ce6d6a68e58565d6c52ad07b3566"}, - {file = "numba-0.51.2-cp36-cp36m-win32.whl", hash = "sha256:31cdf6b6d1301d5fb6c4fcb8b4c711ba5c9f60ba2fca008b550da9b56185367c"}, - {file = "numba-0.51.2-cp36-cp36m-win_amd64.whl", hash = "sha256:df6edca13c04a31fdb5addf5205199478a7da372712829157ef491e8a6e7031f"}, - {file = "numba-0.51.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:a628122dacfcba9a3ea68a9e95578c6b6391016e34962c46550ea8e189e0412e"}, - {file = "numba-0.51.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:106736d5a8dab6bebce989d4ab1b3f169c264582598f172e6e5b736210d2e834"}, - {file = "numba-0.51.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:a12f16fdb4ca5edc94e2ef412e4e768c29217ef9b6fdfc237d064ebe30acfe14"}, - {file = "numba-0.51.2-cp37-cp37m-win32.whl", hash = "sha256:025b033fd31c44bba17802293c81270084b5454b5b055b8c10c394385c232f00"}, - {file = "numba-0.51.2-cp37-cp37m-win_amd64.whl", hash = "sha256:081788f584fa500339e9b74bf02e3c5029d408c114e555ada19cae0b92721416"}, - {file = "numba-0.51.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:5416b584183fd599afda11b947b64f89450fcf26a9c15b408167f412b98a3a94"}, - {file = "numba-0.51.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:05da65dca2ac28a192c9d8f20e9e477eb1237205cfc4d131c414f5f8092c6639"}, - {file = "numba-0.51.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:aee435e3b7e465dd49971f8ea76aa414532a87736916cb399534e017334d1138"}, - {file = "numba-0.51.2-cp38-cp38-win32.whl", hash = "sha256:bbbe2432433b11d3fadab0226a84c1a81918cb905ba1aeb022249e8d2ba8856c"}, - {file = "numba-0.51.2-cp38-cp38-win_amd64.whl", hash = "sha256:259e7c15b24feec4a99fb41eb8c47b5ad49b544d1a5ad40ad0252ef531ba06fd"}, - {file = "numba-0.51.2.tar.gz", hash = "sha256:16bd59572114adbf5f600ea383880d7b2071ae45477e84a24994e089ea390768"}, -] -numpy = [ - {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, - {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, - {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, - {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, - {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, - {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, - {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, - {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, - {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, -] -opencv-python = [ - {file = "opencv-python-4.5.3.56.tar.gz", hash = "sha256:3c001d3feec7f3140f1fb78dfc52ca28122db8240826882d175a208a89d2731b"}, - {file = "opencv_python-4.5.3.56-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:9a78558b5ae848386edbb843c761e5fed5a8480be9af16274a5a78838529edeb"}, - {file = "opencv_python-4.5.3.56-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:8d3282138f3a8646941089aae142684910ebe40776266448eab5f4bb609fc63f"}, - {file = "opencv_python-4.5.3.56-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:881f3d85269500e0c7d72b140a6ebb5c14a089f8140fb9da7ce01f12a245858e"}, - {file = "opencv_python-4.5.3.56-cp36-cp36m-win32.whl", hash = "sha256:f1bda4d144f5204e077ca4571453ebb2015e5748d5e0043386c92c2bbf7f52eb"}, - {file = "opencv_python-4.5.3.56-cp36-cp36m-win_amd64.whl", hash = "sha256:6763729fcfee2a08e069aa1982c9a8c1abf55b9cdf2fb9640eda1d85bdece19a"}, - {file = "opencv_python-4.5.3.56-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:68813b720b88e4951e84399b9a8a7b532d45a07a96ea8f539636242f862e32e0"}, - {file = "opencv_python-4.5.3.56-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:c360cb76ad1ddbd5d2d3e730b42f2ff6e4be08ea6f4a6eefacca175d27467e8f"}, - {file = "opencv_python-4.5.3.56-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:437f30e300725e1d1b3744dbfbc66a523a4744792b58f3dbe1e9140c8f4dfba5"}, - {file = "opencv_python-4.5.3.56-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:e42c644a70d5c54f53a4b114dbd88b4eb83f42a9ca998f07bd5682f3f404efcc"}, - {file = "opencv_python-4.5.3.56-cp37-cp37m-win32.whl", hash = "sha256:f3ac2355217114a683f3f72a9c40a5890914a59c4a2df62e4083c66ff65c9cf9"}, - {file = "opencv_python-4.5.3.56-cp37-cp37m-win_amd64.whl", hash = "sha256:7f41b97d84ac66bdf13cb4d9f4dad3e159525ba1e3f421e670c787ce536eb70a"}, - {file = "opencv_python-4.5.3.56-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:cdc3363c2911d7cfc6c9f55308c51c2841a7aecbf0bf5e791499d220ce89d880"}, - {file = "opencv_python-4.5.3.56-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:18a4a14015eee30d9cd514db8cdefbf594b1d5c234762d27abe512d62a333bc3"}, - {file = "opencv_python-4.5.3.56-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:05c5139d620e8d02f7ce0921796d55736fa19fa15e2ec00a388db2eb1ae1e9a1"}, - {file = "opencv_python-4.5.3.56-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:831b92fe63ce18dd628f71104da7e60596658b75e2fa16b83aefa3eb10c115e2"}, - {file = "opencv_python-4.5.3.56-cp38-cp38-win32.whl", hash = "sha256:e1f54736272830a1e895cedf7a4ee67737e31e966d380c82a81ef22515d043a3"}, - {file = "opencv_python-4.5.3.56-cp38-cp38-win_amd64.whl", hash = "sha256:b42bbba9f5421865377c7960bd4f3dd881003b322a6bf46ed2302b89224d102b"}, - {file = "opencv_python-4.5.3.56-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5366fcd6eae4243add3c8c92142045850f1db8e464bcf0b75313e1596b2e3671"}, - {file = "opencv_python-4.5.3.56-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54c64e86a087841869901fd34462bb6bec01cd4652800fdf5d92fe7b0596c82f"}, - {file = "opencv_python-4.5.3.56-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:8852be06c0749fef0d9c58f532bbcb0570968c59e41cf56b90f5c92593c6e108"}, - {file = "opencv_python-4.5.3.56-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8b5bc61be7fc8565140b746288b370a4bfdb4edb9d680b66bb914e7690485db1"}, - {file = "opencv_python-4.5.3.56-cp39-cp39-win32.whl", hash = "sha256:085232718f28bddd265da480874c37db5c7354cb08f23f4a68a8639b16276a89"}, - {file = "opencv_python-4.5.3.56-cp39-cp39-win_amd64.whl", hash = "sha256:205a73adb29c37e42475645519e612e843a985475da993d10b4d5daa6afec36a"}, -] -packaging = [ - {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, - {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, -] -pandas = [ - {file = "pandas-1.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1ee8418d0f936ff2216513aa03e199657eceb67690995d427a4a7ecd2e68f442"}, - {file = "pandas-1.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d9acfca191140a518779d1095036d842d5e5bc8e8ad8b5eaad1aff90fe1870d"}, - {file = "pandas-1.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e323028ab192fcfe1e8999c012a0fa96d066453bb354c7e7a4a267b25e73d3c8"}, - {file = "pandas-1.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d06661c6eb741ae633ee1c57e8c432bb4203024e263fe1a077fa3fda7817fdb"}, - {file = "pandas-1.3.1-cp37-cp37m-win32.whl", hash = "sha256:23c7452771501254d2ae23e9e9dac88417de7e6eff3ce64ee494bb94dc88c300"}, - {file = "pandas-1.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7150039e78a81eddd9f5a05363a11cadf90a4968aac6f086fd83e66cf1c8d1d6"}, - {file = "pandas-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5c09a2538f0fddf3895070579082089ff4ae52b6cb176d8ec7a4dacf7e3676c1"}, - {file = "pandas-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:905fc3e0fcd86b0a9f1f97abee7d36894698d2592b22b859f08ea5a8fe3d3aab"}, - {file = "pandas-1.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ee927c70794e875a59796fab8047098aa59787b1be680717c141cd7873818ae"}, - {file = "pandas-1.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c976e023ed580e60a82ccebdca8e1cc24d8b1fbb28175eb6521025c127dab66"}, - {file = "pandas-1.3.1-cp38-cp38-win32.whl", hash = "sha256:22f3fcc129fb482ef44e7df2a594f0bd514ac45aabe50da1a10709de1b0f9d84"}, - {file = "pandas-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:45656cd59ae9745a1a21271a62001df58342b59c66d50754390066db500a8362"}, - {file = "pandas-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:114c6789d15862508900a25cb4cb51820bfdd8595ea306bab3b53cd19f990b65"}, - {file = "pandas-1.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:527c43311894aff131dea99cf418cd723bfd4f0bcf3c3da460f3b57e52a64da5"}, - {file = "pandas-1.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb3b33dde260b1766ea4d3c6b8fbf6799cee18d50a2a8bc534cf3550b7c819a"}, - {file = "pandas-1.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c28760932283d2c9f6fa5e53d2f77a514163b9e67fd0ee0879081be612567195"}, - {file = "pandas-1.3.1-cp39-cp39-win32.whl", hash = "sha256:be12d77f7e03c40a2466ed00ccd1a5f20a574d3c622fe1516037faa31aa448aa"}, - {file = "pandas-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:9e1fe6722cbe27eb5891c1977bca62d456c19935352eea64d33956db46139364"}, - {file = "pandas-1.3.1.tar.gz", hash = "sha256:341935a594db24f3ff07d1b34d1d231786aa9adfa84b76eab10bf42907c8aed3"}, -] -pandocfilters = [ - {file = "pandocfilters-1.4.3.tar.gz", hash = "sha256:bc63fbb50534b4b1f8ebe1860889289e8af94a23bff7445259592df25a3906eb"}, -] -parso = [ - {file = "parso-0.8.2-py2.py3-none-any.whl", hash = "sha256:a8c4922db71e4fdb90e0d0bc6e50f9b273d3397925e5e60a717e719201778d22"}, - {file = "parso-0.8.2.tar.gz", hash = "sha256:12b83492c6239ce32ff5eed6d3639d6a536170723c6f3f1506869f1ace413398"}, -] -pathy = [ - {file = "pathy-0.6.0-py3-none-any.whl", hash = "sha256:bffa0bd74c66575cf51c96d3ab312f34d08d6bff54aabb8c7a2b9f8b701fe6ef"}, - {file = "pathy-0.6.0.tar.gz", hash = "sha256:f83f1eddf77dd86e824143eef8d9adbe0785c3cdd5ec0ed6c0edea3227385048"}, -] -patsy = [ - {file = "patsy-0.5.1-py2.py3-none-any.whl", hash = "sha256:5465be1c0e670c3a965355ec09e9a502bf2c4cbe4875e8528b0221190a8a5d40"}, - {file = "patsy-0.5.1.tar.gz", hash = "sha256:f115cec4201e1465cd58b9866b0b0e7b941caafec129869057405bfe5b5e3991"}, -] -pdpbox = [ - {file = "PDPbox-0.2.0-py2-none-any.whl", hash = "sha256:def6840f5a3ada5d4269aced1e0b1244c417a471cf3ed87bf4c4f60ee4f64d2b"}, - {file = "PDPbox-0.2.0.tar.gz", hash = "sha256:2eae5a20004657f48ddd5b00f2fb74dd54f9de891c25ec7935a8fd471f9186f9"}, -] -pexpect = [ - {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, - {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, -] -pickleshare = [ - {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, - {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, -] -pillow = [ - {file = "Pillow-7.2.0-cp35-cp35m-macosx_10_10_intel.whl", hash = "sha256:1ca594126d3c4def54babee699c055a913efb01e106c309fa6b04405d474d5ae"}, - {file = "Pillow-7.2.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:c92302a33138409e8f1ad16731568c55c9053eee71bb05b6b744067e1b62380f"}, - {file = "Pillow-7.2.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8dad18b69f710bf3a001d2bf3afab7c432785d94fcf819c16b5207b1cfd17d38"}, - {file = "Pillow-7.2.0-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:431b15cffbf949e89df2f7b48528be18b78bfa5177cb3036284a5508159492b5"}, - {file = "Pillow-7.2.0-cp35-cp35m-win32.whl", hash = "sha256:09d7f9e64289cb40c2c8d7ad674b2ed6105f55dc3b09aa8e4918e20a0311e7ad"}, - {file = "Pillow-7.2.0-cp35-cp35m-win_amd64.whl", hash = "sha256:0295442429645fa16d05bd567ef5cff178482439c9aad0411d3f0ce9b88b3a6f"}, - {file = "Pillow-7.2.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:ec29604081f10f16a7aea809ad42e27764188fc258b02259a03a8ff7ded3808d"}, - {file = "Pillow-7.2.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:612cfda94e9c8346f239bf1a4b082fdd5c8143cf82d685ba2dba76e7adeeb233"}, - {file = "Pillow-7.2.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0a80dd307a5d8440b0a08bd7b81617e04d870e40a3e46a32d9c246e54705e86f"}, - {file = "Pillow-7.2.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:06aba4169e78c439d528fdeb34762c3b61a70813527a2c57f0540541e9f433a8"}, - {file = "Pillow-7.2.0-cp36-cp36m-win32.whl", hash = "sha256:f7e30c27477dffc3e85c2463b3e649f751789e0f6c8456099eea7ddd53be4a8a"}, - {file = "Pillow-7.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:ffe538682dc19cc542ae7c3e504fdf54ca7f86fb8a135e59dd6bc8627eae6cce"}, - {file = "Pillow-7.2.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:94cf49723928eb6070a892cb39d6c156f7b5a2db4e8971cb958f7b6b104fb4c4"}, - {file = "Pillow-7.2.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6edb5446f44d901e8683ffb25ebdfc26988ee813da3bf91e12252b57ac163727"}, - {file = "Pillow-7.2.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:52125833b070791fcb5710fabc640fc1df07d087fc0c0f02d3661f76c23c5b8b"}, - {file = "Pillow-7.2.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:9ad7f865eebde135d526bb3163d0b23ffff365cf87e767c649550964ad72785d"}, - {file = "Pillow-7.2.0-cp37-cp37m-win32.whl", hash = "sha256:c79f9c5fb846285f943aafeafda3358992d64f0ef58566e23484132ecd8d7d63"}, - {file = "Pillow-7.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d350f0f2c2421e65fbc62690f26b59b0bcda1b614beb318c81e38647e0f673a1"}, - {file = "Pillow-7.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:6d7741e65835716ceea0fd13a7d0192961212fd59e741a46bbed7a473c634ed6"}, - {file = "Pillow-7.2.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:edf31f1150778abd4322444c393ab9c7bd2af271dd4dafb4208fb613b1f3cdc9"}, - {file = "Pillow-7.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:d08b23fdb388c0715990cbc06866db554e1822c4bdcf6d4166cf30ac82df8c41"}, - {file = "Pillow-7.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:5e51ee2b8114def244384eda1c82b10e307ad9778dac5c83fb0943775a653cd8"}, - {file = "Pillow-7.2.0-cp38-cp38-win32.whl", hash = "sha256:725aa6cfc66ce2857d585f06e9519a1cc0ef6d13f186ff3447ab6dff0a09bc7f"}, - {file = "Pillow-7.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:a060cf8aa332052df2158e5a119303965be92c3da6f2d93b6878f0ebca80b2f6"}, - {file = "Pillow-7.2.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:9c87ef410a58dd54b92424ffd7e28fd2ec65d2f7fc02b76f5e9b2067e355ebf6"}, - {file = "Pillow-7.2.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:e901964262a56d9ea3c2693df68bc9860b8bdda2b04768821e4c44ae797de117"}, - {file = "Pillow-7.2.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:25930fadde8019f374400f7986e8404c8b781ce519da27792cbe46eabec00c4d"}, - {file = "Pillow-7.2.0.tar.gz", hash = "sha256:97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626"}, -] -platformdirs = [ - {file = "platformdirs-2.2.0-py3-none-any.whl", hash = "sha256:4666d822218db6a262bdfdc9c39d21f23b4cfdb08af331a81e92751daf6c866c"}, - {file = "platformdirs-2.2.0.tar.gz", hash = "sha256:632daad3ab546bd8e6af0537d09805cec458dce201bccfe23012df73332e181e"}, -] -pre-commit = [ - {file = "pre_commit-2.14.0-py2.py3-none-any.whl", hash = "sha256:ec3045ae62e1aa2eecfb8e86fa3025c2e3698f77394ef8d2011ce0aedd85b2d4"}, - {file = "pre_commit-2.14.0.tar.gz", hash = "sha256:2386eeb4cf6633712c7cc9ede83684d53c8cafca6b59f79c738098b51c6d206c"}, -] -preshed = [ - {file = "preshed-3.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:572899224578d30f6a67fadecb3d62b824866b4d2b6bad73f71abf7585db1389"}, - {file = "preshed-3.0.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:67c11e384ce4c008bc487ba3a29bafdfe038b9a2546ccfe0fe2160480b356fed"}, - {file = "preshed-3.0.5-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:6e833f1632a1d0232bdc6df6c3542fb130ef044d8656b24576d9fd19e5f1e0d1"}, - {file = "preshed-3.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:1ce0846cb7ebb2ea913d44ec2e296098c285443ecdea80ddf02656bbef4deacb"}, - {file = "preshed-3.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8a560850b8c53c1487ba51c2b0f5769535512b36d3b129ad5796b64653abe2f9"}, - {file = "preshed-3.0.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:6f126bcc414a0304b54956f9dac2628a0f9bef1657d1b3a3837fc82b791aa2a1"}, - {file = "preshed-3.0.5-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:1bdededa7fd81f26a42bc9d11d542657c74746b7ea7fc2b2ca6d0ddbf1f93792"}, - {file = "preshed-3.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9ebf444f8487782c84d7b5acb1d7195e603155882fafc4697344199eeeafbe5f"}, - {file = "preshed-3.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8a3adffde3126c2a0ab7d57cab1d605cb5f63da1ba88088ad3cf8debfd9aa4dc"}, - {file = "preshed-3.0.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:56b9603517bb2a364418163236d6a147a1d722ff7546cbe085e76e25ae118e89"}, - {file = "preshed-3.0.5-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:5e06a49477bd257eea02bf823b5d3e201d00a19d6976523a58da8606b2358481"}, - {file = "preshed-3.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:ca4a7681b643b8356e7dfdab9cf668b2b34bd07ef4b09ebed44c8aeb3b1626ee"}, - {file = "preshed-3.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:85074eebf90a858a6b68242f1ae265ca99e1af45bf9dafcb9a83d49b0815a2e1"}, - {file = "preshed-3.0.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:12cbe1e378b4f1c6b06f5e4130408befe916e55ea1616e6aa63c5cd0ccd9c927"}, - {file = "preshed-3.0.5-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:30f0c8ea85113d0565a1e3eb6222d00513ec39b56f3f9a2615e304575e65422e"}, - {file = "preshed-3.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:fb4d2e82add82d63b2c97802b759a58ff200d06b632e2edc48a9ced1e6472faf"}, - {file = "preshed-3.0.5.tar.gz", hash = "sha256:c6d3dba39ed5059aaf99767017b9568c75b2d0780c3481e204b1daecde00360e"}, -] -prometheus-client = [ - {file = "prometheus_client-0.11.0-py2.py3-none-any.whl", hash = "sha256:b014bc76815eb1399da8ce5fc84b7717a3e63652b0c0f8804092c9363acab1b2"}, - {file = "prometheus_client-0.11.0.tar.gz", hash = "sha256:3a8baade6cb80bcfe43297e33e7623f3118d660d41387593758e2fb1ea173a86"}, -] -prompt-toolkit = [ - {file = "prompt_toolkit-3.0.19-py3-none-any.whl", hash = "sha256:7089d8d2938043508aa9420ec18ce0922885304cddae87fb96eebca942299f88"}, - {file = "prompt_toolkit-3.0.19.tar.gz", hash = "sha256:08360ee3a3148bdb5163621709ee322ec34fc4375099afa4bbf751e9b7b7fa4f"}, -] -psutil = [ - {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, - {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"}, - {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df"}, - {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131"}, - {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60"}, - {file = "psutil-5.8.0-cp27-none-win32.whl", hash = "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876"}, - {file = "psutil-5.8.0-cp27-none-win_amd64.whl", hash = "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65"}, - {file = "psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8"}, - {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6"}, - {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac"}, - {file = "psutil-5.8.0-cp36-cp36m-win32.whl", hash = "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2"}, - {file = "psutil-5.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d"}, - {file = "psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935"}, - {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d"}, - {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023"}, - {file = "psutil-5.8.0-cp37-cp37m-win32.whl", hash = "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394"}, - {file = "psutil-5.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"}, - {file = "psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef"}, - {file = "psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28"}, - {file = "psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b"}, - {file = "psutil-5.8.0-cp38-cp38-win32.whl", hash = "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d"}, - {file = "psutil-5.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d"}, - {file = "psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7"}, - {file = "psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4"}, - {file = "psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b"}, - {file = "psutil-5.8.0-cp39-cp39-win32.whl", hash = "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0"}, - {file = "psutil-5.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3"}, - {file = "psutil-5.8.0.tar.gz", hash = "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"}, -] -ptyprocess = [ - {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, - {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, -] -py = [ - {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, - {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, -] -pycparser = [ - {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, - {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, -] -pydantic = [ - {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, - {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, - {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, - {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, - {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, - {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, - {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, - {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, - {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, - {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, -] -pydotplus = [ - {file = "pydotplus-2.0.2.tar.gz", hash = "sha256:91e85e9ee9b85d2391ead7d635e3d9c7f5f44fd60a60e59b13e2403fa66505c4"}, -] -pyglet = [ - {file = "pyglet-1.5.15-py3-none-any.whl", hash = "sha256:4401cc176580e4e17e2df8bbf7536f27e691327dc3f38f209a12f1859c70aed2"}, - {file = "pyglet-1.5.15.zip", hash = "sha256:da9d8337388cedabf1f1c5dc21a45bb2b0e5327fba47f996c8573818c3dfa478"}, -] -pygments = [ - {file = "Pygments-2.9.0-py3-none-any.whl", hash = "sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e"}, - {file = "Pygments-2.9.0.tar.gz", hash = "sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f"}, -] -pyparsing = [ - {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, - {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, -] -pyrsistent = [ - {file = "pyrsistent-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f4c8cabb46ff8e5d61f56a037974228e978f26bfefce4f61a4b1ac0ba7a2ab72"}, - {file = "pyrsistent-0.18.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:da6e5e818d18459fa46fac0a4a4e543507fe1110e808101277c5a2b5bab0cd2d"}, - {file = "pyrsistent-0.18.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5e4395bbf841693eaebaa5bb5c8f5cdbb1d139e07c975c682ec4e4f8126e03d2"}, - {file = "pyrsistent-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:527be2bfa8dc80f6f8ddd65242ba476a6c4fb4e3aedbf281dfbac1b1ed4165b1"}, - {file = "pyrsistent-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2aaf19dc8ce517a8653746d98e962ef480ff34b6bc563fc067be6401ffb457c7"}, - {file = "pyrsistent-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58a70d93fb79dc585b21f9d72487b929a6fe58da0754fa4cb9f279bb92369396"}, - {file = "pyrsistent-0.18.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4916c10896721e472ee12c95cdc2891ce5890898d2f9907b1b4ae0f53588b710"}, - {file = "pyrsistent-0.18.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:73ff61b1411e3fb0ba144b8f08d6749749775fe89688093e1efef9839d2dcc35"}, - {file = "pyrsistent-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:b29b869cf58412ca5738d23691e96d8aff535e17390128a1a52717c9a109da4f"}, - {file = "pyrsistent-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:097b96f129dd36a8c9e33594e7ebb151b1515eb52cceb08474c10a5479e799f2"}, - {file = "pyrsistent-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:772e94c2c6864f2cd2ffbe58bb3bdefbe2a32afa0acb1a77e472aac831f83427"}, - {file = "pyrsistent-0.18.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:c1a9ff320fa699337e05edcaae79ef8c2880b52720bc031b219e5b5008ebbdef"}, - {file = "pyrsistent-0.18.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cd3caef37a415fd0dae6148a1b6957a8c5f275a62cca02e18474608cb263640c"}, - {file = "pyrsistent-0.18.0-cp38-cp38-win32.whl", hash = "sha256:e79d94ca58fcafef6395f6352383fa1a76922268fa02caa2272fff501c2fdc78"}, - {file = "pyrsistent-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:a0c772d791c38bbc77be659af29bb14c38ced151433592e326361610250c605b"}, - {file = "pyrsistent-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d5ec194c9c573aafaceebf05fc400656722793dac57f254cd4741f3c27ae57b4"}, - {file = "pyrsistent-0.18.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:6b5eed00e597b5b5773b4ca30bd48a5774ef1e96f2a45d105db5b4ebb4bca680"}, - {file = "pyrsistent-0.18.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:48578680353f41dca1ca3dc48629fb77dfc745128b56fc01096b2530c13fd426"}, - {file = "pyrsistent-0.18.0-cp39-cp39-win32.whl", hash = "sha256:f3ef98d7b76da5eb19c37fda834d50262ff9167c65658d1d8f974d2e4d90676b"}, - {file = "pyrsistent-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:404e1f1d254d314d55adb8d87f4f465c8693d6f902f67eb6ef5b4526dc58e6ea"}, - {file = "pyrsistent-0.18.0.tar.gz", hash = "sha256:773c781216f8c2900b42a7b638d5b517bb134ae1acbebe4d1e8f1f41ea60eb4b"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -pytorch-transformers = [ - {file = "pytorch_transformers-1.2.0-py2-none-any.whl", hash = "sha256:15f12a04424c0f6d3a7c7b57d6c79628dc9c117a204fb7db8c1ea330c77a6898"}, - {file = "pytorch_transformers-1.2.0-py3-none-any.whl", hash = "sha256:bdb606fe1f2d27586710ed03cfa49dbbd80215c38bf965862daada0c137fd7ce"}, - {file = "pytorch_transformers-1.2.0.tar.gz", hash = "sha256:293e4a864ae9d9401f9fba13f16b8696e4a1cb38bcd0b56562d03af5489daeb9"}, -] -pytz = [ - {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"}, - {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"}, -] -pywavelets = [ - {file = "PyWavelets-1.1.1-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:35959c041ec014648575085a97b498eafbbaa824f86f6e4a59bfdef8a3fe6308"}, - {file = "PyWavelets-1.1.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:55e39ec848ceec13c9fa1598253ae9dd5c31d09dfd48059462860d2b908fb224"}, - {file = "PyWavelets-1.1.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c06d2e340c7bf8b9ec71da2284beab8519a3908eab031f4ea126e8ccfc3fd567"}, - {file = "PyWavelets-1.1.1-cp35-cp35m-win32.whl", hash = "sha256:be105382961745f88d8196bba5a69ee2c4455d87ad2a2e5d1eed6bd7fda4d3fd"}, - {file = "PyWavelets-1.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:076ca8907001fdfe4205484f719d12b4a0262dfe6652fa1cfc3c5c362d14dc84"}, - {file = "PyWavelets-1.1.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7947e51ca05489b85928af52a34fe67022ab5b81d4ae32a4109a99e883a0635e"}, - {file = "PyWavelets-1.1.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:9e2528823ccf5a0a1d23262dfefe5034dce89cd84e4e124dc553dfcdf63ebb92"}, - {file = "PyWavelets-1.1.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:80b924edbc012ded8aa8b91cb2fd6207fb1a9a3a377beb4049b8a07445cec6f0"}, - {file = "PyWavelets-1.1.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c2a799e79cee81a862216c47e5623c97b95f1abee8dd1f9eed736df23fb653fb"}, - {file = "PyWavelets-1.1.1-cp36-cp36m-win32.whl", hash = "sha256:d510aef84d9852653d079c84f2f81a82d5d09815e625f35c95714e7364570ad4"}, - {file = "PyWavelets-1.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:889d4c5c5205a9c90118c1980df526857929841df33e4cd1ff1eff77c6817a65"}, - {file = "PyWavelets-1.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:68b5c33741d26c827074b3d8f0251de1c3019bb9567b8d303eb093c822ce28f1"}, - {file = "PyWavelets-1.1.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:18a51b3f9416a2ae6e9a35c4af32cf520dd7895f2b69714f4aa2f4342fca47f9"}, - {file = "PyWavelets-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cfe79844526dd92e3ecc9490b5031fca5f8ab607e1e858feba232b1b788ff0ea"}, - {file = "PyWavelets-1.1.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:2f7429eeb5bf9c7068002d0d7f094ed654c77a70ce5e6198737fd68ab85f8311"}, - {file = "PyWavelets-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:720dbcdd3d91c6dfead79c80bf8b00a1d8aa4e5d551dc528c6d5151e4efc3403"}, - {file = "PyWavelets-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:bc5e87b72371da87c9bebc68e54882aada9c3114e640de180f62d5da95749cd3"}, - {file = "PyWavelets-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:98b2669c5af842a70cfab33a7043fcb5e7535a690a00cd251b44c9be0be418e5"}, - {file = "PyWavelets-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e02a0558e0c2ac8b8bbe6a6ac18c136767ec56b96a321e0dfde2173adfa5a504"}, - {file = "PyWavelets-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6162dc0ae04669ea04b4b51420777b9ea2d30b0a9d02901b2a3b4d61d159c2e9"}, - {file = "PyWavelets-1.1.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:39c74740718e420d38c78ca4498568fa57976d78d5096277358e0fa9629a7aea"}, - {file = "PyWavelets-1.1.1-cp38-cp38-win32.whl", hash = "sha256:79f5b54f9dc353e5ee47f0c3f02bebd2c899d49780633aa771fed43fa20b3149"}, - {file = "PyWavelets-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:935ff247b8b78bdf77647fee962b1cc208c51a7b229db30b9ba5f6da3e675178"}, - {file = "PyWavelets-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6ebfefebb5c6494a3af41ad8c60248a95da267a24b79ed143723d4502b1fe4d7"}, - {file = "PyWavelets-1.1.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:6bc78fb9c42a716309b4ace56f51965d8b5662c3ba19d4591749f31773db1125"}, - {file = "PyWavelets-1.1.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:411e17ca6ed8cf5e18a7ca5ee06a91c25800cc6c58c77986202abf98d749273a"}, - {file = "PyWavelets-1.1.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:83c5e3eb78ce111c2f0b45f46106cc697c3cb6c4e5f51308e1f81b512c70c8fb"}, - {file = "PyWavelets-1.1.1-cp39-cp39-win32.whl", hash = "sha256:2b634a54241c190ee989a4af87669d377b37c91bcc9cf0efe33c10ff847f7841"}, - {file = "PyWavelets-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:732bab78435c48be5d6bc75486ef629d7c8f112e07b313bf1f1a2220ab437277"}, - {file = "PyWavelets-1.1.1.tar.gz", hash = "sha256:1a64b40f6acb4ffbaccce0545d7fc641744f95351f62e4c6aaa40549326008c9"}, -] -pywin32 = [ - {file = "pywin32-301-cp35-cp35m-win32.whl", hash = "sha256:93367c96e3a76dfe5003d8291ae16454ca7d84bb24d721e0b74a07610b7be4a7"}, - {file = "pywin32-301-cp35-cp35m-win_amd64.whl", hash = "sha256:9635df6998a70282bd36e7ac2a5cef9ead1627b0a63b17c731312c7a0daebb72"}, - {file = "pywin32-301-cp36-cp36m-win32.whl", hash = "sha256:c866f04a182a8cb9b7855de065113bbd2e40524f570db73ef1ee99ff0a5cc2f0"}, - {file = "pywin32-301-cp36-cp36m-win_amd64.whl", hash = "sha256:dafa18e95bf2a92f298fe9c582b0e205aca45c55f989937c52c454ce65b93c78"}, - {file = "pywin32-301-cp37-cp37m-win32.whl", hash = "sha256:98f62a3f60aa64894a290fb7494bfa0bfa0a199e9e052e1ac293b2ad3cd2818b"}, - {file = "pywin32-301-cp37-cp37m-win_amd64.whl", hash = "sha256:fb3b4933e0382ba49305cc6cd3fb18525df7fd96aa434de19ce0878133bf8e4a"}, - {file = "pywin32-301-cp38-cp38-win32.whl", hash = "sha256:88981dd3cfb07432625b180f49bf4e179fb8cbb5704cd512e38dd63636af7a17"}, - {file = "pywin32-301-cp38-cp38-win_amd64.whl", hash = "sha256:8c9d33968aa7fcddf44e47750e18f3d034c3e443a707688a008a2e52bbef7e96"}, - {file = "pywin32-301-cp39-cp39-win32.whl", hash = "sha256:595d397df65f1b2e0beaca63a883ae6d8b6df1cdea85c16ae85f6d2e648133fe"}, - {file = "pywin32-301-cp39-cp39-win_amd64.whl", hash = "sha256:87604a4087434cd814ad8973bd47d6524bd1fa9e971ce428e76b62a5e0860fdf"}, -] -pywinpty = [ - {file = "pywinpty-1.1.3-cp36-none-win_amd64.whl", hash = "sha256:81dc6f16d917b756e06fc58943e9750d59dbefc0ffd2086871d3fa5f33824446"}, - {file = "pywinpty-1.1.3-cp37-none-win_amd64.whl", hash = "sha256:54557887e712ea3215ab0d9f089ed55a6cc8d826cd5d1e340d75300654c9663f"}, - {file = "pywinpty-1.1.3-cp38-none-win_amd64.whl", hash = "sha256:f5e25197397f1fef0362caf3eb89f25441827a1e48bf15827c27021592fd2160"}, - {file = "pywinpty-1.1.3-cp39-none-win_amd64.whl", hash = "sha256:b767276224f86b7560eb9173ba7956758cafcdfab97bb33837d42d2a0f1dbf67"}, - {file = "pywinpty-1.1.3.tar.gz", hash = "sha256:3a1d57b338390333812a5eed31c93c7d8ba82b131078063703e731946d90c9f2"}, -] -pyyaml = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, -] -pyzmq = [ - {file = "pyzmq-22.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:d60a407663b7c2af781ab7f49d94a3d379dd148bb69ea8d9dd5bc69adf18097c"}, - {file = "pyzmq-22.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:631f932fb1fa4b76f31adf976f8056519bc6208a3c24c184581c3dd5be15066e"}, - {file = "pyzmq-22.2.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0471d634c7fe48ff7d3849798da6c16afc71676dd890b5ae08eb1efe735c6fec"}, - {file = "pyzmq-22.2.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f520e9fee5d7a2e09b051d924f85b977c6b4e224e56c0551c3c241bbeeb0ad8d"}, - {file = "pyzmq-22.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1b6619ceb33a8907f1cb82ff8afc8a133e7a5f16df29528e919734718600426"}, - {file = "pyzmq-22.2.1-cp310-cp310-win32.whl", hash = "sha256:31c5dfb6df5148789835128768c01bf6402eb753d06f524f12f6786caf96fb44"}, - {file = "pyzmq-22.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:4842a8263cbaba6fce401bbe4e2b125321c401a01714e42624dabc554bfc2629"}, - {file = "pyzmq-22.2.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b921758f8b5098faa85f341bbdd5e36d5339de5e9032ca2b07d8c8e7bec5069b"}, - {file = "pyzmq-22.2.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:240b83b3a8175b2f616f80092cbb019fcd5c18598f78ffc6aa0ae9034b300f14"}, - {file = "pyzmq-22.2.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:da7f7f3bb08bcf59a6b60b4e53dd8f08bb00c9e61045319d825a906dbb3c8fb7"}, - {file = "pyzmq-22.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e66025b64c4724ba683d6d4a4e5ee23de12fe9ae683908f0c7f0f91b4a2fd94e"}, - {file = "pyzmq-22.2.1-cp36-cp36m-win32.whl", hash = "sha256:50d007d5702171bc810c1e74498fa2c7bc5b50f9750697f7fd2a3e71a25aad91"}, - {file = "pyzmq-22.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b4a51c7d906dc263a0cc5590761e53e0a68f2c2fefe549cbef21c9ee5d2d98a4"}, - {file = "pyzmq-22.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:93705cb90baa9d6f75e8448861a1efd3329006f79095ab18846bd1eaa342f7c3"}, - {file = "pyzmq-22.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:620b0abb813958cb3ecb5144c177e26cde92fee6f43c4b9de6b329515532bf27"}, - {file = "pyzmq-22.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2dd3896b3c952cf6c8013deda53c1df16bf962f355b5503d23521e0f6403ae3d"}, - {file = "pyzmq-22.2.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6e9c030222893afa86881d7485d3e841969760a16004bd23e9a83cca28b42778"}, - {file = "pyzmq-22.2.1-cp37-cp37m-win32.whl", hash = "sha256:262f470e7acde18b7217aac78d19d2e29ced91a5afbeb7d98521ebf26461aa7e"}, - {file = "pyzmq-22.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:246f27b88722cfa729bb04881e94484e40b085720d728c1b05133b3f331b0b7b"}, - {file = "pyzmq-22.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0d17bac19e934e9f547a8811b7c2a32651a7840f38086b924e2e3dcb2fae5c3a"}, - {file = "pyzmq-22.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5933d1f4087de6e52906f72d92e1e4dcc630d371860b92c55d7f7a4b815a664c"}, - {file = "pyzmq-22.2.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ac4497e4b7d134ee53ce5532d9cc3b640d6e71806a55062984e0c99a2f88f465"}, - {file = "pyzmq-22.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:66375a6094af72a6098ed4403b15b4db6bf00013c6febc1baa832e7abda827f4"}, - {file = "pyzmq-22.2.1-cp38-cp38-win32.whl", hash = "sha256:b2c16d20bd0aef8e57bc9505fdd80ea0d6008020c3740accd96acf1b3d1b5347"}, - {file = "pyzmq-22.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:ff345d48940c834168f81fa1d4724675099f148f1ab6369748c4d712ed71bf7c"}, - {file = "pyzmq-22.2.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:f5c84c5de9a773bbf8b22c51e28380999ea72e5e85b4db8edf5e69a7a0d4d9f9"}, - {file = "pyzmq-22.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2534a036b777f957bd6b89b55fb2136775ca2659fb0f1c85036ba78d17d86fd5"}, - {file = "pyzmq-22.2.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a649065413ba4eab92a783a7caa4de8ce14cf46ba8a2a09951426143f1298adb"}, - {file = "pyzmq-22.2.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c9cb0bd3a3cb7ccad3caa1d7b0d18ba71ed3a4a3610028e506a4084371d4d223"}, - {file = "pyzmq-22.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4428302c389fffc0c9c07a78cad5376636b9d096f332acfe66b321ae9ff2c63"}, - {file = "pyzmq-22.2.1-cp39-cp39-win32.whl", hash = "sha256:6a5b4566f66d953601d0d47d4071897f550a265bafd52ebcad5ac7aad3838cbb"}, - {file = "pyzmq-22.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:89200ab6ef9081c72a04ed84c52a50b60dcb0655375aeedb40689bc7c934715e"}, - {file = "pyzmq-22.2.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed67df4eaa99a20d162d76655bda23160abdf8abf82a17f41dfd3962e608dbcc"}, - {file = "pyzmq-22.2.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:021e22a8c58ab294bd4b96448a2ca4e716e1d76600192ff84c33d71edb1fbd37"}, - {file = "pyzmq-22.2.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:200ac096cee5499964c90687306a7244b79ef891f773ed4cf15019fd1f3df330"}, - {file = "pyzmq-22.2.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:b3f57bee62e36be5c97712de32237c5589caee0d1154c2ad01a888accfae20bc"}, - {file = "pyzmq-22.2.1.tar.gz", hash = "sha256:6d18c76676771fd891ca8e0e68da0bbfb88e30129835c0ade748016adb3b6242"}, -] -regex = [ - {file = "regex-2021.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8764a78c5464ac6bde91a8c87dd718c27c1cabb7ed2b4beaf36d3e8e390567f9"}, - {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4551728b767f35f86b8e5ec19a363df87450c7376d7419c3cac5b9ceb4bce576"}, - {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:577737ec3d4c195c4aef01b757905779a9e9aee608fa1cf0aec16b5576c893d3"}, - {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c856ec9b42e5af4fe2d8e75970fcc3a2c15925cbcc6e7a9bcb44583b10b95e80"}, - {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3835de96524a7b6869a6c710b26c90e94558c31006e96ca3cf6af6751b27dca1"}, - {file = "regex-2021.8.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cea56288eeda8b7511d507bbe7790d89ae7049daa5f51ae31a35ae3c05408531"}, - {file = "regex-2021.8.3-cp36-cp36m-win32.whl", hash = "sha256:a4eddbe2a715b2dd3849afbdeacf1cc283160b24e09baf64fa5675f51940419d"}, - {file = "regex-2021.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:57fece29f7cc55d882fe282d9de52f2f522bb85290555b49394102f3621751ee"}, - {file = "regex-2021.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a5c6dbe09aff091adfa8c7cfc1a0e83fdb8021ddb2c183512775a14f1435fe16"}, - {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff4a8ad9638b7ca52313d8732f37ecd5fd3c8e3aff10a8ccb93176fd5b3812f6"}, - {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b63e3571b24a7959017573b6455e05b675050bbbea69408f35f3cb984ec54363"}, - {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fbc20975eee093efa2071de80df7f972b7b35e560b213aafabcec7c0bd00bd8c"}, - {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14caacd1853e40103f59571f169704367e79fb78fac3d6d09ac84d9197cadd16"}, - {file = "regex-2021.8.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bb350eb1060591d8e89d6bac4713d41006cd4d479f5e11db334a48ff8999512f"}, - {file = "regex-2021.8.3-cp37-cp37m-win32.whl", hash = "sha256:18fdc51458abc0a974822333bd3a932d4e06ba2a3243e9a1da305668bd62ec6d"}, - {file = "regex-2021.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:026beb631097a4a3def7299aa5825e05e057de3c6d72b139c37813bfa351274b"}, - {file = "regex-2021.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:16d9eaa8c7e91537516c20da37db975f09ac2e7772a0694b245076c6d68f85da"}, - {file = "regex-2021.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3905c86cc4ab6d71635d6419a6f8d972cab7c634539bba6053c47354fd04452c"}, - {file = "regex-2021.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937b20955806381e08e54bd9d71f83276d1f883264808521b70b33d98e4dec5d"}, - {file = "regex-2021.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:28e8af338240b6f39713a34e337c3813047896ace09d51593d6907c66c0708ba"}, - {file = "regex-2021.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c09d88a07483231119f5017904db8f60ad67906efac3f1baa31b9b7f7cca281"}, - {file = "regex-2021.8.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:85f568892422a0e96235eb8ea6c5a41c8ccbf55576a2260c0160800dbd7c4f20"}, - {file = "regex-2021.8.3-cp38-cp38-win32.whl", hash = "sha256:bf6d987edd4a44dd2fa2723fca2790f9442ae4de2c8438e53fcb1befdf5d823a"}, - {file = "regex-2021.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:8fe58d9f6e3d1abf690174fd75800fda9bdc23d2a287e77758dc0e8567e38ce6"}, - {file = "regex-2021.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7976d410e42be9ae7458c1816a416218364e06e162b82e42f7060737e711d9ce"}, - {file = "regex-2021.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9569da9e78f0947b249370cb8fadf1015a193c359e7e442ac9ecc585d937f08d"}, - {file = "regex-2021.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459bbe342c5b2dec5c5223e7c363f291558bc27982ef39ffd6569e8c082bdc83"}, - {file = "regex-2021.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f421e3cdd3a273bace013751c345f4ebeef08f05e8c10757533ada360b51a39"}, - {file = "regex-2021.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea212df6e5d3f60341aef46401d32fcfded85593af1d82b8b4a7a68cd67fdd6b"}, - {file = "regex-2021.8.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a3b73390511edd2db2d34ff09aa0b2c08be974c71b4c0505b4a048d5dc128c2b"}, - {file = "regex-2021.8.3-cp39-cp39-win32.whl", hash = "sha256:f35567470ee6dbfb946f069ed5f5615b40edcbb5f1e6e1d3d2b114468d505fc6"}, - {file = "regex-2021.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:bfa6a679410b394600eafd16336b2ce8de43e9b13f7fb9247d84ef5ad2b45e91"}, - {file = "regex-2021.8.3.tar.gz", hash = "sha256:8935937dad2c9b369c3d932b0edbc52a62647c2afb2fafc0c280f14a8bf56a6a"}, -] -requests = [ - {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, - {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, -] -s3transfer = [ - {file = "s3transfer-0.5.0-py3-none-any.whl", hash = "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803"}, - {file = "s3transfer-0.5.0.tar.gz", hash = "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c"}, -] -sacremoses = [ - {file = "sacremoses-0.0.45-py3-none-any.whl", hash = "sha256:fa93db44bc04542553ba6090818b892f603d02aa0d681e6c5c3023baf17e8564"}, - {file = "sacremoses-0.0.45.tar.gz", hash = "sha256:58176cc28391830789b763641d0f458819bebe88681dac72b41a19c0aedc07e9"}, -] -scikit-image = [ - {file = "scikit-image-0.18.2.tar.gz", hash = "sha256:32ff472355fbf8ab40a8e9ed685906c6c51a863f1ea8737882d26be9221631f3"}, - {file = "scikit_image-0.18.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6f6d0e79a91c62360708111951abb3a774cefac865902ea797c3b72d8ece6382"}, - {file = "scikit_image-0.18.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:49a9b48bc428d2d56aaefbc042fd79c67ebc908a1cbf542e9c863c49339ca496"}, - {file = "scikit_image-0.18.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:32620792e989beb2c3eb67eae38b59291be412be59ad3485ee0f67cb7b37c16f"}, - {file = "scikit_image-0.18.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:38efc3fd9023c7849175fc18b2cc96a08629da840b6100ec5038f487fba7d34e"}, - {file = "scikit_image-0.18.2-cp37-cp37m-win32.whl", hash = "sha256:5510b133999a45b2c8ed4c1b659fa0a1cf4ca0db949353d0f54fc6290dac4d5a"}, - {file = "scikit_image-0.18.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b3aa7230d84b12d8a4a9f0b65ee895603d27fe85366bf2b57929ba1cce2e8987"}, - {file = "scikit_image-0.18.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b21d65dcee453539fe70b5903edd8429ad9fe46233b049dd622368bad435f39e"}, - {file = "scikit_image-0.18.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f61d65de826abe2f5f6c171c75d2bb93df56aa4a690d1bab5911412f49b9e768"}, - {file = "scikit_image-0.18.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a4fdba1bdd883a8028ddff0b8fe8d43c8dd43360bdab6e1f40599fa210613f1d"}, - {file = "scikit_image-0.18.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8e62228a91b770fbe89d310e833f8797f14136b9635bb67d8b780f1b8cf237e6"}, - {file = "scikit_image-0.18.2-cp38-cp38-win32.whl", hash = "sha256:f80d16ce57e05af8e282620a23e90bb8886e5efa6eedcacb4da1c15293ba5e9a"}, - {file = "scikit_image-0.18.2-cp38-cp38-win_amd64.whl", hash = "sha256:6d576a8249114e6169ea1c2b05a22168745eedba90b06d5765368dbd59b27c7f"}, - {file = "scikit_image-0.18.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:278b0034c509d8b31a9b117837b3d45957dd3408e062ad0f2b24edeb1a460e91"}, - {file = "scikit_image-0.18.2-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:f9648093a0865150fc4ac9eaf02256afbf471a43216b0b6ee6585a4d57674563"}, - {file = "scikit_image-0.18.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:2c7e91fb3df5cc58cb13c39094a32bb2e990ced30b08ee34bf0976ff8a1ba579"}, - {file = "scikit_image-0.18.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:4f42a36e34a3e659dd5e0fe9c0b07f797f9b66680224a7f2545a564484574d78"}, - {file = "scikit_image-0.18.2-cp39-cp39-win32.whl", hash = "sha256:66ea3bc8f53efbaf751fdae472fe1cbc55ad5e4fadbf6d3a0a268dc7e34d83b6"}, - {file = "scikit_image-0.18.2-cp39-cp39-win_amd64.whl", hash = "sha256:74f7c5920c0b893608ef0d159a61a15e87aa9f31d2707d1ed6621a65233646cd"}, -] -scikit-learn = [ - {file = "scikit-learn-0.24.2.tar.gz", hash = "sha256:d14701a12417930392cd3898e9646cf5670c190b933625ebe7511b1f7d7b8736"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:d5bf9c863ba4717b3917b5227463ee06860fc43931dc9026747de416c0a10fee"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5beaeb091071625e83f5905192d8aecde65ba2f26f8b6719845bbf586f7a04a1"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:06ffdcaaf81e2a3b1b50c3ac6842cfb13df2d8b737d61f64643ed61da7389cde"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:fec42690a2eb646b384eafb021c425fab48991587edb412d4db77acc358b27ce"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:5ff3e4e4cf7592d36541edec434e09fb8ab9ba6b47608c4ffe30c9038d301897"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:3cbd734e1aefc7c5080e6b6973fe062f97c26a1cdf1a991037ca196ce1c8f427"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-win32.whl", hash = "sha256:f74429a07fedb36a03c159332b914e6de757176064f9fed94b5f79ebac07d913"}, - {file = "scikit_learn-0.24.2-cp36-cp36m-win_amd64.whl", hash = "sha256:dd968a174aa82f3341a615a033fa6a8169e9320cbb46130686562db132d7f1f0"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:49ec0b1361da328da9bb7f1a162836028e72556356adeb53342f8fae6b450d47"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f18c3ed484eeeaa43a0d45dc2efb4d00fc6542ccdcfa2c45d7b635096a2ae534"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cdf24c1b9bbeb4936456b42ac5bd32c60bb194a344951acb6bfb0cddee5439a4"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d177fe1ff47cc235942d628d41ee5b1c6930d8f009f1a451c39b5411e8d0d4cf"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f3ec00f023d84526381ad0c0f2cff982852d035c921bbf8ceb994f4886c00c64"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ae19ac105cf7ce8c205a46166992fdec88081d6e783ab6e38ecfbe45729f3c39"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-win32.whl", hash = "sha256:f0ed4483c258fb23150e31b91ea7d25ff8495dba108aea0b0d4206a777705350"}, - {file = "scikit_learn-0.24.2-cp37-cp37m-win_amd64.whl", hash = "sha256:39b7e3b71bcb1fe46397185d6c1a5db1c441e71c23c91a31e7ad8cc3f7305f9a"}, - {file = "scikit_learn-0.24.2-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:90a297330f608adeb4d2e9786c6fda395d3150739deb3d42a86d9a4c2d15bc1d"}, - {file = "scikit_learn-0.24.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f1d2108e770907540b5248977e4cff9ffaf0f73d0d13445ee938df06ca7579c6"}, - {file = "scikit_learn-0.24.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:1eec963fe9ffc827442c2e9333227c4d49749a44e592f305398c1db5c1563393"}, - {file = "scikit_learn-0.24.2-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:2db429090b98045d71218a9ba913cc9b3fe78e0ba0b6b647d8748bc6d5a44080"}, - {file = "scikit_learn-0.24.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:62214d2954377fcf3f31ec867dd4e436df80121e7a32947a0b3244f58f45e455"}, - {file = "scikit_learn-0.24.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8fac72b9688176922f9f54fda1ba5f7ffd28cbeb9aad282760186e8ceba9139a"}, - {file = "scikit_learn-0.24.2-cp38-cp38-win32.whl", hash = "sha256:ae426e3a52842c6b6d77d00f906b6031c8c2cfdfabd6af7511bb4bc9a68d720e"}, - {file = "scikit_learn-0.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:038f4e9d6ef10e1f3fe82addc3a14735c299866eb10f2c77c090410904828312"}, - {file = "scikit_learn-0.24.2-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:48f273836e19901ba2beecd919f7b352f09310ce67c762f6e53bc6b81cacf1f0"}, - {file = "scikit_learn-0.24.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:a2a47449093dcf70babc930beba2ca0423cb7df2fa5fd76be5260703d67fa574"}, - {file = "scikit_learn-0.24.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:0e71ce9c7cbc20f6f8b860107ce15114da26e8675238b4b82b7e7cd37ca0c087"}, - {file = "scikit_learn-0.24.2-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:2754c85b2287333f9719db7f23fb7e357f436deed512db3417a02bf6f2830aa5"}, - {file = "scikit_learn-0.24.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:7be1b88c23cfac46e06404582215a917017cd2edaa2e4d40abe6aaff5458f24b"}, - {file = "scikit_learn-0.24.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:4e6198675a6f9d333774671bd536668680eea78e2e81c0b19e57224f58d17f37"}, - {file = "scikit_learn-0.24.2-cp39-cp39-win32.whl", hash = "sha256:cbdb0b3db99dd1d5f69d31b4234367d55475add31df4d84a3bd690ef017b55e2"}, - {file = "scikit_learn-0.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:40556bea1ef26ef54bc678d00cf138a63069144a0b5f3a436eecd8f3468b903e"}, -] -scipy = [ - {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"}, - {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"}, - {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"}, - {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"}, - {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"}, - {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"}, - {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"}, - {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"}, - {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"}, - {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, -] -seaborn = [ - {file = "seaborn-0.11.1-py3-none-any.whl", hash = "sha256:4e1cce9489449a1c6ff3c567f2113cdb41122f727e27a984950d004a88ef3c5c"}, - {file = "seaborn-0.11.1.tar.gz", hash = "sha256:44e78eaed937c5a87fc7a892c329a7cc091060b67ebd1d0d306b446a74ba01ad"}, -] -send2trash = [ - {file = "Send2Trash-1.8.0-py3-none-any.whl", hash = "sha256:f20eaadfdb517eaca5ce077640cb261c7d2698385a6a0f072a4a5447fd49fa08"}, - {file = "Send2Trash-1.8.0.tar.gz", hash = "sha256:d2c24762fd3759860a0aff155e45871447ea58d2be6bdd39b5c8f966a0c99c2d"}, -] -sentencepiece = [ - {file = "sentencepiece-0.1.96-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc969e6694fb27fba7cee2953f350804faf03913f25ae1ee713a7b8a1bc08018"}, - {file = "sentencepiece-0.1.96-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36e9ff61e7b67c5b7ee96733613622620b4802fc8cf188a4dbc1f355b03dde02"}, - {file = "sentencepiece-0.1.96-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e9e9fe8094ca57549d801e9a2017ac5c24108bbf485ea4f8994a72e8e96ee135"}, - {file = "sentencepiece-0.1.96-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b77d27f59d515c43b61745b8173fbe7c7b3014b14b3702a75bf1793471e7def6"}, - {file = "sentencepiece-0.1.96-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1dac8c2ad02b5ebc1179c0a14cbc7d7c6f4fd73d4dd51820626402d0aefc974e"}, - {file = "sentencepiece-0.1.96-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:e8ec5bb6777e2060e1499750c50e1b69dca5a0f80f90f2c66656c5f3e5244593"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:99ea2d9db19e63a2d17d5dc64f9ace83fb9308a735be05a1aaf98eb4b496fba7"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeb090ad462833df03af1debce4ae607a2766ef861f992003ad0c56d074ab805"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f8c90df663cd9759b2cf8dd29998b63140ac39e51ada2e739dc13bdac0b4f001"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26d20d713b3ba1b7a19205336afb1e93a4327c372b2f795e907b8dc2315ac92e"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5388882bb24d083f6cc8cffc5c435f3694a7772b018e06ea6fd84d1044009efb"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a92e1932ee8fd500680ccbe1bf53eb33228f4c9d6524ed6f300bcc80ac359f27"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-win32.whl", hash = "sha256:bedf0355117fb4e9b1fc9fc92b4d5ee743a7d468be9f6196e3b94447710ea589"}, - {file = "sentencepiece-0.1.96-cp36-cp36m-win_amd64.whl", hash = "sha256:4997c7ccf2ae462320250314aa5709a88d8a09fa271d073458a07bebf33f8e7c"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:a697257a2cd7581732d7741a8d32a06927f0311c3d277dbc47fa1043350c9d17"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff7d752a7f82d87711ec1a95c2262cb74f98be5b457f0300d81a1aefe5be2a95"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3e61e0757e49c306fff78ea75d6b75773418fe22214b4a460959203be934e834"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef59ba19340dc1d002ce5713b911c0ef23c577b08f8ed57998ee3c8e62c5bf6e"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89c038da7f827a6e2ca4c73aeb4e4b25b99d981ce47dd61b04d446c8200cba1e"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d954d25a8705f972e8bfc1dea5464d7e697dd6f4ade092f1a487387e6d6c829a"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-win32.whl", hash = "sha256:fd907a8f744e5337de7fc532dd800c4416b571ea47f8c3c66be10cd1bc67c925"}, - {file = "sentencepiece-0.1.96-cp37-cp37m-win_amd64.whl", hash = "sha256:335bf84d72112cc91f3c3b691d61802fc963503b7772fd8280d20368048b8f3e"}, - {file = "sentencepiece-0.1.96-cp38-cp38-macosx_10_6_x86_64.whl", hash = "sha256:e811984b0908c14c56de7d8226fdd494d87a7ccb75af8ac3a07423037aaafc35"}, - {file = "sentencepiece-0.1.96-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8179785883b556cd517416cdbda6244745414b00ec83132cfe1d26000971f3ae"}, - {file = "sentencepiece-0.1.96-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:466e381f0a812da8fda97a9707498cef3210ea8385a3421bcbadcb5384063969"}, - {file = "sentencepiece-0.1.96-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8cb24d8d0b2f8b7463815a59183eb81ec1d7a06e3217bed456063f3303eddfb"}, - {file = "sentencepiece-0.1.96-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e88354b61f59dfdeb41023f7be8ae31dc627c2dc2dacbc2de8b2d82a0997135c"}, - {file = "sentencepiece-0.1.96-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a336575463d75d3aac1f7e32470b8998643ccd9a73786bd726f6b0470520b6b4"}, - {file = "sentencepiece-0.1.96-cp38-cp38-win32.whl", hash = "sha256:81bb77ba3651114943b2f8f77829cf764137dff06e38f4bf7fa43efea12c7f84"}, - {file = "sentencepiece-0.1.96-cp38-cp38-win_amd64.whl", hash = "sha256:eba0471ab0bb2e07ed06d91ecf5185d402c83d194155a41d8e2aa547d187712e"}, - {file = "sentencepiece-0.1.96-cp39-cp39-macosx_10_6_x86_64.whl", hash = "sha256:78e18d9106c36dcca929e18fd2c412378deac661d47fa3ee25defc55eef8a215"}, - {file = "sentencepiece-0.1.96-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1c24c1d9405b2148184ff27c062493d5e3be5c144575f95b5a0d7c660a515af"}, - {file = "sentencepiece-0.1.96-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940a6999c7d3f55e9d7b194fd5e1f41a7dbed26d3519fb95333216292a39599e"}, - {file = "sentencepiece-0.1.96-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:384148cead5cdab34a4d74fe1fb6a5a8abaafed25eaa4a7698b49dd9482e4c4e"}, - {file = "sentencepiece-0.1.96-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c703e68ea192e45b65c5d5836f6980849d828a18da4189899d7150fad82dc9e"}, - {file = "sentencepiece-0.1.96-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d501713a8396193883aa526f48dc609f5f031a5df1afbafa561cf9ab492ffc76"}, - {file = "sentencepiece-0.1.96-cp39-cp39-win32.whl", hash = "sha256:b8b1dd2712f8a7de5b4c8ec912e6c041d25750bf03e1ce325cdba43bae0944ae"}, - {file = "sentencepiece-0.1.96-cp39-cp39-win_amd64.whl", hash = "sha256:d45e3f78e746aa161bc9f5a31c6a2839c512101113a4065f4d2e7a3ab8198d8c"}, - {file = "sentencepiece-0.1.96-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5513298d62fe63dd0862d08a6eb52a9aa3537006f597f2386184e3f95bb88889"}, - {file = "sentencepiece-0.1.96-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dadccb2e49244b6e64b4527d13ec14d5e094a90b41cf9b963e457e64182f1941"}, - {file = "sentencepiece-0.1.96-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48c6d13b3bfff08060c138248e85df60f6fad11135ad7a8fc2ef6005aacca839"}, - {file = "sentencepiece-0.1.96.tar.gz", hash = "sha256:9bdf097d5bd1d8ce42dfee51f6ff05f5578b96e48c6f6006aa4eff69edfa3639"}, -] -shap = [ - {file = "shap-0.38.1-cp36-cp36m-win_amd64.whl", hash = "sha256:34913391184180f9359e2627131960a473d67143e94b7f649c75a2d0c7d4cd40"}, - {file = "shap-0.38.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4cc1e1ac2e1e30aa9857fcf3fcfa0a6b2bf5e6aa0670c16a36bc28cd9b11aae5"}, - {file = "shap-0.38.1-cp38-cp38-win_amd64.whl", hash = "sha256:0457e7fb80d2398454a16d16c7cd7934003e8c8bd9c1e002d965fce6a3815e54"}, - {file = "shap-0.38.1.tar.gz", hash = "sha256:8f23e2ee3c80774d8c0942ecbd71b4dc0c2beba6d3de41dfc3a86e55adb9d28a"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -slicer = [ - {file = "slicer-0.0.7-py3-none-any.whl", hash = "sha256:0b94faa5251c0f23782c03f7b7eedda91d80144059645f452c4bc80fab875976"}, - {file = "slicer-0.0.7.tar.gz", hash = "sha256:f5d5f7b45f98d155b9c0ba6554fa9770c6b26d5793a3e77a1030fb56910ebeec"}, -] -smart-open = [ - {file = "smart_open-5.1.0-py3-none-any.whl", hash = "sha256:2059b07f530c8c9e2158e4e1575309aacb74bd813da2325c1f348015d04f3bd6"}, - {file = "smart_open-5.1.0.tar.gz", hash = "sha256:e4dc1350b240ef0759e343e4e2f361bfd4e5477bb2619866e97f80240652e92e"}, -] -spacy = [ - {file = "spacy-3.1.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:667a7fd991b49a99403f47003d6b28fd9d1ad0e79d022823b0f608e55660ce06"}, - {file = "spacy-3.1.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bc38a83d48d6e57f0db88ee8d0683540b97aabc1102797366a0f345e7dc4288"}, - {file = "spacy-3.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:05730587cd620148a9fc824aea097a3848e955f2d01b8181a6dfb795c351a061"}, - {file = "spacy-3.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fee107573987df50e1c692bd62dfc54a3e36dcb1498ae370b3a381e9ddb0b719"}, - {file = "spacy-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e15e12b3a9ce9f4c631e5047147eb92309f9b9cbcbf7f4a81e72c822f886af27"}, - {file = "spacy-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0472162a8a46adcbde2390517657928b2c09cc506ca7835def4ff49ba3dc1bd4"}, - {file = "spacy-3.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cb6e01bead3c99deb2d5f9cfc9cc7ac033bbfa16106bfe232681581e1772a4b6"}, - {file = "spacy-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e7114d3c35820ef4d0c972cb717788291fc7131da379d5c76773e79a92d9de8"}, - {file = "spacy-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:ee4051e81022168999de9faca85ec2c71a54a28c0d5a6a77edb4da950497b688"}, - {file = "spacy-3.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:62a98490cf7fd2ed161da185579c037df1f35cde46049d39bd222fe241595176"}, - {file = "spacy-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a50d9eba682c4f33daef34af84021a1b072aef7776793a712b602056ece724be"}, - {file = "spacy-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:04d6b4f29eb737e89c2f31d99cef976f7fd9288412e32a74b48a81349bc7a156"}, - {file = "spacy-3.1.1.tar.gz", hash = "sha256:77a0f78d7e65335e5fae13af7e55684770c76a0457ee1baac414ddb89b5df6e4"}, -] -spacy-legacy = [ - {file = "spacy-legacy-3.0.8.tar.gz", hash = "sha256:b4725c5c161f0685ab4fce3fc912bc68aefdb7e102ba9848e852bb5842256c2f"}, - {file = "spacy_legacy-3.0.8-py2.py3-none-any.whl", hash = "sha256:eb37a3540bb461b5fe9348d4976784f18a0e345982e41e2c5c7cd8229889e825"}, -] -srsly = [ - {file = "srsly-2.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f7c3374184bfb1aa852bcb8e45747b02f2dde0ebe62b4ddf4b0141affeab32e1"}, - {file = "srsly-2.4.1-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9625a584b26e522b6afb7c24be8783228ff44d7ac624e500020b0b888e09c6b6"}, - {file = "srsly-2.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:129c85db752b5945c6398a1952294e03b7d20fa111eb7fd1083c4a6b1d02f7c7"}, - {file = "srsly-2.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:20f11d5d6ae29e3cc97e93c862d7bf8b75023668daf1ac5892598c512302e5d3"}, - {file = "srsly-2.4.1-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cefe06912f3944b5729d555ee110f434a0787843c6676b90f4987ff7a0a69500"}, - {file = "srsly-2.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b1bd4a55bafbb8cf86be15bf18aa2ba2c953161ad71ce7d2dae0c141201a7d89"}, - {file = "srsly-2.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e896d516ca2e2e89cc01df8c9c8b1528701d6f49e9c814332582cc701af64a91"}, - {file = "srsly-2.4.1-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:ff36dc01df8890a239e5d15cffa3ae3b272c19e5ae279840f2d30085d361c20a"}, - {file = "srsly-2.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:867d1154ff7b60043584fe048de9b6d9a7d5a7fc61437850922ae4bd46d3be16"}, - {file = "srsly-2.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:76b11e0ec0056bda4ad009b6e0db37f3ad0005a0501d587080023d4312ad2ada"}, - {file = "srsly-2.4.1-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:178aa6d350c9cfedb8adadb5e1f96b7aadde203d088917063415fcd689eb6e42"}, - {file = "srsly-2.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:869fdcf664edf20cd374cf1add869d67960061276478025a5887e080d8f99e1c"}, - {file = "srsly-2.4.1.tar.gz", hash = "sha256:b0f2aec0a329e6e7e742a0a60e99a74968ca29be71f35c5c4de221e328176926"}, -] -statsmodels = [ - {file = "statsmodels-0.12.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c1d98ce2072f5e772cbf91d05475490368da5d3ee4a3150062330c7b83221ceb"}, - {file = "statsmodels-0.12.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4184487e9c281acad3d0bda19445c69db292f0dbb18f25ebf56a7966a0a28eef"}, - {file = "statsmodels-0.12.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:37e107fa11299090ed90f93c7172162b850c28fd09999937b971926813e887c5"}, - {file = "statsmodels-0.12.2-cp36-none-win32.whl", hash = "sha256:5d3e7333e1c5b234797ed57c3d1533371374c1e1e7e7ed54d27805611f96e2d5"}, - {file = "statsmodels-0.12.2-cp36-none-win_amd64.whl", hash = "sha256:aaf3c75fd22cb9dcf9c1b28f8ae87521310870f4dd8a6a4f1010f1e46d992377"}, - {file = "statsmodels-0.12.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:c48b7cbb37a651bb1cd23614abc10f447845ad3c3a713bf74e2aad20cfc94ae7"}, - {file = "statsmodels-0.12.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a3bd3922463dda8ad33e5e5075d2080e9e012aeb2032b5cdaeea9b79c2472000"}, - {file = "statsmodels-0.12.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:43de84bc08c8b9f778502aed7a476d6e68674e6878718e533b07d569cf0927a9"}, - {file = "statsmodels-0.12.2-cp37-none-win32.whl", hash = "sha256:0197855aa1d40c42532d6a75b4ca72e30826a50d90ec3047a404f9702d8b814f"}, - {file = "statsmodels-0.12.2-cp37-none-win_amd64.whl", hash = "sha256:93273aa1c31caf59bcce9790ca4c3f54fdc45a37c61084d06f1ba4fbe56e7752"}, - {file = "statsmodels-0.12.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:3e94306d4c07e332532ea4911d1f1d1f661c79aa73f22c5bb22e6dd47b40d562"}, - {file = "statsmodels-0.12.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f3a7622f3d0ce2fc204f43b74de4e03e42775609705bf94d656b730482ca935a"}, - {file = "statsmodels-0.12.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:587deb788e7f8f3f866d28e812cf5c082b4d4a2d3f5beea94d0e9699ea71ef22"}, - {file = "statsmodels-0.12.2-cp38-none-win32.whl", hash = "sha256:cbbdf6f708c9a1f1fad5cdea5e4342d6fdb37e42e92288c2cf906b99976ffe15"}, - {file = "statsmodels-0.12.2-cp38-none-win_amd64.whl", hash = "sha256:1fa720e895112a1b04b27002218b0ea7f10dd1d9cffd1c018c88bbfb82520f57"}, - {file = "statsmodels-0.12.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:c3782ce846a52862ac72f89d22b6b1ca13d877bc593872309228a6f05d934321"}, - {file = "statsmodels-0.12.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:8f93cb3f7d87c1fc7e51b3b239371c25a17a0a8e782467fdf4788cfef600724a"}, - {file = "statsmodels-0.12.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f61f33f64760a22100b6b146217823f73cfedd251c9bdbd58453ca94e63326c7"}, - {file = "statsmodels-0.12.2-cp39-none-win32.whl", hash = "sha256:3aab85174444f1bcad1e9218a3d3db08f0f86eeb97985236ca8605a0a39ce305"}, - {file = "statsmodels-0.12.2-cp39-none-win_amd64.whl", hash = "sha256:94d3632d56c13eebebaefb52bd4b43144ad5a131337b57842f46db826fa7d2d3"}, - {file = "statsmodels-0.12.2.tar.gz", hash = "sha256:8ad7a7ae7cdd929095684118e3b05836c0ccb08b6a01fe984159475d174a1b10"}, -] -subword-nmt = [ - {file = "subword_nmt-0.3.7-py2.py3-none-any.whl", hash = "sha256:a2d92eed5dea55f2b1c9b21225a57b3ae7009ce8a1fa4d2e3f01ab11435c28c9"}, -] -tabulate = [ - {file = "tabulate-0.8.9-py3-none-any.whl", hash = "sha256:d7c013fe7abbc5e491394e10fa845f8f32fe54f8dc60c6622c6cf482d25d47e4"}, - {file = "tabulate-0.8.9.tar.gz", hash = "sha256:eb1d13f25760052e8931f2ef80aaf6045a6cceb47514db8beab24cded16f13a7"}, -] -terminado = [ - {file = "terminado-0.11.0-py3-none-any.whl", hash = "sha256:221eef83e6a504894842f7dccfa971ca2e98ec22a8a9118577e5257527674b42"}, - {file = "terminado-0.11.0.tar.gz", hash = "sha256:1e01183885f64c1bba3cf89a5a995ad4acfed4e5f00aebcce1bf7f089b0825a1"}, -] -testpath = [ - {file = "testpath-0.5.0-py3-none-any.whl", hash = "sha256:8044f9a0bab6567fc644a3593164e872543bb44225b0e24846e2c89237937589"}, - {file = "testpath-0.5.0.tar.gz", hash = "sha256:1acf7a0bcd3004ae8357409fc33751e16d37ccc650921da1094a86581ad1e417"}, -] -thinc = [ - {file = "thinc-8.0.8-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:68401890470062eaa3bcd0cd0bc5ad52a6fa77da87336a927df18c21dbf0ba30"}, - {file = "thinc-8.0.8-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d0303de94e12cd288fdffebef97a460fc95700a527d4e898548477be5406a25"}, - {file = "thinc-8.0.8-cp36-cp36m-win_amd64.whl", hash = "sha256:101047df534a4861ba6fab25a1849c673c83536e067bd917ae735aeb9090fb52"}, - {file = "thinc-8.0.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e35ff1cea8b1ec73fed5c04923ee88ec4799e7948496fe7eca1f754019da87e7"}, - {file = "thinc-8.0.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f7c5381e804d641fff39061d3b9e01feb25790282aa8ed6684c62b8e2731e7f"}, - {file = "thinc-8.0.8-cp37-cp37m-win_amd64.whl", hash = "sha256:d0f46905fdd737a8090609ddc54a48f70fa997e5b304d8c362db93b95365646d"}, - {file = "thinc-8.0.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6901e6d7dbb5cf08d1877920e893fe6d721627c946004a495f77c151bf07eb72"}, - {file = "thinc-8.0.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03a0e2679363fafe0c7312dc9eb46697b6fa3e65ffa7a1702ea369e93389fbfd"}, - {file = "thinc-8.0.8-cp38-cp38-win_amd64.whl", hash = "sha256:793cb9113b36df6607089806d9d08b371748b201dda05150f7f531cd63df84b8"}, - {file = "thinc-8.0.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:37c31b83f46372283d5f394db9272d35ec6c26b8a0481f1b1995f9ed0cb72a47"}, - {file = "thinc-8.0.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c37c96335d74b34e8128569070c0e17e13a213c9564a3553e44c3769a948a35e"}, - {file = "thinc-8.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:998b87d6cd334b5bf080ef5594bc0d1afda36d088deffc1caf7e8fe0bae553c6"}, - {file = "thinc-8.0.8.tar.gz", hash = "sha256:cf2abbd99c56f21b8804f31f995460515d95a5c5988be39e0964469e0070987b"}, -] -threadpoolctl = [ - {file = "threadpoolctl-2.2.0-py3-none-any.whl", hash = "sha256:e5a995e3ffae202758fa8a90082e35783b9370699627ae2733cd1c3a73553616"}, - {file = "threadpoolctl-2.2.0.tar.gz", hash = "sha256:86d4b6801456d780e94681d155779058759eaef3c3564758b17b6c99db5f81cb"}, -] -tifffile = [ - {file = "tifffile-2021.8.8-py3-none-any.whl", hash = "sha256:1309d1f5cc2ee2e8274916dc609922cb2364f947a9d09b388069c63180710dfd"}, - {file = "tifffile-2021.8.8.tar.gz", hash = "sha256:8260f31c4700143e8374ff6cde5cef7fe54fc9b7313afe88329f407881901dc5"}, -] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] -torch = [ - {file = "torch-1.7.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:422e64e98d0e100c360993819d0307e5d56e9517b26135808ad68984d577d75a"}, - {file = "torch-1.7.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f0aaf657145533824b15f2fd8fde8f8c67fe6c6281088ef588091f03fad90243"}, - {file = "torch-1.7.1-cp36-none-macosx_10_9_x86_64.whl", hash = "sha256:af464a6f4314a875035e0c4c2b07517599704b214634f4ed3ad2e748c5ef291f"}, - {file = "torch-1.7.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5d76c255a41484c1d41a9ff570b9c9f36cb85df9428aa15a58ae16ac7cfc2ea6"}, - {file = "torch-1.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:d241c3f1c4d563e4ba86f84769c23e12606db167ee6f674eedff6d02901462e3"}, - {file = "torch-1.7.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:de84b4166e3f7335eb868b51d3bbd909ec33828af27290b4171bce832a55be3c"}, - {file = "torch-1.7.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:dd2fc6880c95e836960d86efbbc7f63d3287f2e1893c51d31f96dbfe02f0d73e"}, - {file = "torch-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:e000b94be3aa58ad7f61e7d07cf379ea9366cf6c6874e68bd58ad0bdc537b3a7"}, - {file = "torch-1.7.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:2e49cac969976be63117004ee00d0a3e3dd4ea662ad77383f671b8992825de1a"}, - {file = "torch-1.7.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:a3793dcceb12b1e2281290cca1277c5ce86ddfd5bf044f654285a4d69057aea7"}, - {file = "torch-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:6652a767a0572ae0feb74ad128758e507afd3b8396b6e7f147e438ba8d4c6f63"}, - {file = "torch-1.7.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:38d67f4fb189a92a977b2c0a38e4f6dd413e0bf55aa6d40004696df7e40a71ff"}, -] -torchsummary = [ - {file = "torchsummary-1.5.1-py3-none-any.whl", hash = "sha256:10f41d1743fb918f83293f13183f532ab1bb8f6639a1b89e5f8592ec1919a976"}, - {file = "torchsummary-1.5.1.tar.gz", hash = "sha256:981bf689e22e0cf7f95c746002f20a24ad26aa6b9d861134a14bc6ce92230590"}, -] -torchtext = [ - {file = "torchtext-0.8.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a0c0b7221fdfdd124f98de854d922c111084a4defe11ea32ecc22b56d1f46fd9"}, - {file = "torchtext-0.8.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:19c9976400e09ab1008c3fb0d1162dc80214b6ac45012d2e1692c25337119157"}, - {file = "torchtext-0.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:991d9d38fd1d47a8517e624223e3537123a48175b00b74c6508daa2906431176"}, - {file = "torchtext-0.8.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:46cae2155fa28ab9920e23e6fb8d445911183e88e7f9eeb74024ee0a20671961"}, - {file = "torchtext-0.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5b479e2c98525a77ab112e6dd624a1ccc783e927b25b618218793254fc09e2d2"}, - {file = "torchtext-0.8.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:90c4699d3f923cf937c89579e08f560094874ecdcd0a62603bef2bda961553ed"}, - {file = "torchtext-0.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e2e82629a682064e21c20c2d6b34a3a4212e0ec816de0e69db6ee43da48f3eb0"}, - {file = "torchtext-0.8.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:28036a61bf97d965775b32065ff31661637662124f6aabf4eccd2ef12d9f3d43"}, -] -torchvision = [ - {file = "torchvision-0.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:86fae370d222f76ad57c57c3bee03f78b8db727743bfb4c1559a3d395159cea8"}, - {file = "torchvision-0.8.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:951239b5fcb911dbf78c1385d677f5f48c7a1b12859e3d3ec287562821b17cf2"}, - {file = "torchvision-0.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:24db8f4c3d812a032273f68563ad5dbd724f5bfbed523d0c6dce8cede26bb153"}, - {file = "torchvision-0.8.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:b068f6bcbe91bdd34dda0a39e8a26392add45a3be82543f6dd523b76484fb56f"}, - {file = "torchvision-0.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:afb76a66b9b0693f758a881a2bf333ed97e3c0c3f15a413c4f49d8dd8bd21307"}, - {file = "torchvision-0.8.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cd8817e9197fc60ebae37162a445db90bbf35591314a5767ad3d1490b5d65b0f"}, - {file = "torchvision-0.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1bd58acc3366ec02266aae56a7a752d43ef07de4a6ba420c4f907d0c9168bb8c"}, - {file = "torchvision-0.8.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:976750a49db2e23dc5a1ed0b5c31f7af51ed2702eee410ee09ef985c3a3e48cf"}, -] -tornado = [ - {file = "tornado-6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675"}, - {file = "tornado-6.1-cp35-cp35m-win32.whl", hash = "sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5"}, - {file = "tornado-6.1-cp35-cp35m-win_amd64.whl", hash = "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68"}, - {file = "tornado-6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085"}, - {file = "tornado-6.1-cp36-cp36m-win32.whl", hash = "sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575"}, - {file = "tornado-6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795"}, - {file = "tornado-6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d"}, - {file = "tornado-6.1-cp37-cp37m-win32.whl", hash = "sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df"}, - {file = "tornado-6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37"}, - {file = "tornado-6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f"}, - {file = "tornado-6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6"}, - {file = "tornado-6.1-cp38-cp38-win32.whl", hash = "sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326"}, - {file = "tornado-6.1-cp38-cp38-win_amd64.whl", hash = "sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c"}, - {file = "tornado-6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0"}, - {file = "tornado-6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd"}, - {file = "tornado-6.1-cp39-cp39-win32.whl", hash = "sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c"}, - {file = "tornado-6.1-cp39-cp39-win_amd64.whl", hash = "sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4"}, - {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, -] -tqdm = [ - {file = "tqdm-4.62.0-py2.py3-none-any.whl", hash = "sha256:706dea48ee05ba16e936ee91cb3791cd2ea6da348a0e50b46863ff4363ff4340"}, - {file = "tqdm-4.62.0.tar.gz", hash = "sha256:3642d483b558eec80d3c831e23953582c34d7e4540db86d9e5ed9dad238dabc6"}, -] -traitlets = [ - {file = "traitlets-5.0.5-py3-none-any.whl", hash = "sha256:69ff3f9d5351f31a7ad80443c2674b7099df13cc41fc5fa6e2f6d3b0330b0426"}, - {file = "traitlets-5.0.5.tar.gz", hash = "sha256:178f4ce988f69189f7e523337a3e11d91c786ded9360174a3d9ca83e79bc5396"}, -] -typer = [ - {file = "typer-0.3.2-py3-none-any.whl", hash = "sha256:ba58b920ce851b12a2d790143009fa00ac1d05b3ff3257061ff69dbdfc3d161b"}, - {file = "typer-0.3.2.tar.gz", hash = "sha256:5455d750122cff96745b0dec87368f56d023725a7ebc9d2e54dd23dc86816303"}, -] -typing-extensions = [ - {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, - {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, - {file = "typing_extensions-3.10.0.0.tar.gz", hash = "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342"}, -] -urllib3 = [ - {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, - {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, -] -virtualenv = [ - {file = "virtualenv-20.7.2-py2.py3-none-any.whl", hash = "sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06"}, - {file = "virtualenv-20.7.2.tar.gz", hash = "sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0"}, -] -wasabi = [ - {file = "wasabi-0.8.2-py3-none-any.whl", hash = "sha256:a493e09d86109ec6d9e70d040472f9facc44634d4ae6327182f94091ca73a490"}, - {file = "wasabi-0.8.2.tar.gz", hash = "sha256:b4a36aaa9ca3a151f0c558f269d442afbb3526f0160fd541acd8a0d5e5712054"}, -] -wcwidth = [ - {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, - {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, -] -webencodings = [ - {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, - {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, -] -widgetsnbextension = [ - {file = "widgetsnbextension-3.5.1-py2.py3-none-any.whl", hash = "sha256:bd314f8ceb488571a5ffea6cc5b9fc6cba0adaf88a9d2386b93a489751938bcd"}, - {file = "widgetsnbextension-3.5.1.tar.gz", hash = "sha256:079f87d87270bce047512400efd70238820751a11d2d8cb137a5a5bdbaf255c7"}, -] -xgboost = [ - {file = "xgboost-1.4.2-py3-none-macosx_10_14_x86_64.macosx_10_15_x86_64.macosx_11_0_x86_64.whl", hash = "sha256:e8f1a366a403784afd30a56eb99a429cefc45d906943cd362025ccf942208e13"}, - {file = "xgboost-1.4.2-py3-none-manylinux2010_x86_64.whl", hash = "sha256:ec3f60d53dcd23273a5c7a495ba0f8205656ce750eb2ce7798726a4b2ef4955a"}, - {file = "xgboost-1.4.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:15dd5987827030b3f68e741490a8b3a4ead7c6064bd911e36235b84e0a9d0765"}, - {file = "xgboost-1.4.2-py3-none-win_amd64.whl", hash = "sha256:7c8973204b2c2362012850605e81de5a180513fc08db36d0da9befb77c3d57c8"}, - {file = "xgboost-1.4.2.tar.gz", hash = "sha256:5a364c152095824445ac56a83fb7f7e75913b4bb128c2fcd99b85877c9f4f8fe"}, -] diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index b26eb4d..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,87 +0,0 @@ -[tool.poetry] -name = "ml-mipt" -version = "1.0.0" -description = "Machine learning course at MIPT" -authors = ["Vladislav Goncharenko , Radoslav Neychev "] -license = "MIT License" - -[tool.poetry.dependencies] -python = "^3.8" -scikit-learn = "^0.24.1" -matplotlib = "^3.3.4" -pandas = "^1.2.2" -numpy = "^1.20.1" -scipy = "^1.6.0" -statsmodels = "^0.12.2" -seaborn = "^0.11.1" -xgboost = "^1.3.3" -opencv-python = "^4.5.1" -torch = "^1.7.1" -torchvision = "^0.8.2" -torchsummary = "^1.5.1" - -# basic -Pillow = {version = "^7.2.0", optional = true} # TODO: remove -tqdm = {version = "^4.56.2", optional = true} # TODO: remove -scikit-image = {version = "^0.18.1", optional = true} # TODO: remove week0_12 imread and resize -h5py = {version = "^3.1.0", optional = true} # parse cats and dogs dataset, maybe remove? -pydotplus = {version = "^2.0.2", optional = true} # graph visualization -eli5 = {version = "^0.11.0", optional = true} # week0_07 feature importance -PDPbox = {version = "^0.2.0", optional = true} # week0_07 feature importance -shap = {version = "^0.38.1", optional = true} # week0_07 feature importance - -# advanced -ipywidgets = "^7.6.3" # week1_15 downloading mnist via torchvision - -# nlp -nltk = "^3.5" -gensim = "^3.8.3" -spacy = "^3.1.1" -subword-nmt = "^0.3.7" - -pytorch-transformers = "^1.2.0" -torchtext = "^0.8" - -bokeh = "^2.3.0" - -# rl -gym = {version = "^0.18.0", optional = true} -graphviz = "^0.16" - -[tool.poetry.extras] -basic = ["Pillow", "tqdm", "scikit-image", "h5py", "pydotplus", "eli5", "PDPbox", "shap"] -nlp = ["nltk", "gensim", "spacy", "subword-nmt", "pytorch-transformers", "torchtext", "bokeh"] -rl = ["gym", "graphviz"] - -[tool.poetry.dev-dependencies] -pre-commit = "^2.10.1" -ipykernel = "^5.4.3" - -[tool.black] -line-length = 100 -target-version = ["py38"] - -[tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -line_length = 100 -lines_after_imports = 2 - -[tool.nbqa.config] -black = "pyproject.toml" -isort = "pyproject.toml" -flake8 = "setup.cfg" - -[tool.nbqa.addopts] -flake8 = ["--extend-ignore=E402"] - -[tool.nbqa.mutate] -black = 1 -isort = 1 - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8392765..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -max-line-length = 100 -ignore = E203, E501, W503, B950 -max-complexity = 12 -select = B, C, E, F, W, B9 diff --git a/week05_transformer_pos_tagging/README.md b/week05_transformer_pos_tagging/README.md deleted file mode 100644 index 1443d4f..0000000 --- a/week05_transformer_pos_tagging/README.md +++ /dev/null @@ -1,24 +0,0 @@ -PoS Tagging with BiLSTM: -* Self-practice version: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging.ipynb) -* Completed version: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging__completed.ipynb) - - -Understanding the positional encoding: -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/week05_transformer_pos_tagging/week05_positional_encoding_carriers.ipynb) - -Full Transformer architecture and training pipeline by Harvard NLP: -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/harvardnlp/annotated-transformer/blob/master/The%20Annotated%20Transformer.ipynb) - - - -__Further readings__: -* [en] The Illustrated Transformer [blog post](https://jalammar.github.io/illustrated-transformer/) - -* [en] Harvard NLP [full implementation in PyTorch](http://nlp.seas.harvard.edu/2018/04/03/attention.html) - -* [en] OpenAI blog post [Better Language Models -and Their Implications (GPT-2)](https://openai.com/blog/better-language-models/) - -* [en] Paper describing positional encoding ["Convolutional Sequence to Sequence Learning"](https://arxiv.org/pdf/1705.03122) - -* [en] Paper presenting [Layer Normalization](https://arxiv.org/abs/1607.06450) diff --git a/week05_transformer_pos_tagging/assets/pos-bert.png b/week05_transformer_pos_tagging/assets/pos-bert.png deleted file mode 100644 index 139b8df..0000000 Binary files a/week05_transformer_pos_tagging/assets/pos-bert.png and /dev/null differ diff --git a/week05_transformer_pos_tagging/assets/pos-bert.xml b/week05_transformer_pos_tagging/assets/pos-bert.xml deleted file mode 100644 index 4317308..0000000 --- a/week05_transformer_pos_tagging/assets/pos-bert.xml +++ /dev/null @@ -1 +0,0 @@ -7Vxbc5s4GP01PCaDkAHzWDtpd3Z72dadSfLUUUAGNoBcLAe7v34FCANG2Dgxl0nwg40+ELqcc75PEsISnPvbTyFaOV+IhT1Jka2tBG8kRdH1CfuODbvUMAHcYIeulZpAbli4fzA3yty6cS28Ll1ICfGouyobTRIE2KQlGwpDEpUvWxKvXOoK2bhiWJjIq1rvXIs6qXWqaLn9L+zaTlYy0Iz0zCMyn+yQbAJenqTAZfJJT/souxcvYe0gi0SpKWk7vJXgPCSEpkf+do69uGuzbks76GPN2X29QxzQRhl4jmfkbXBW5aRidJd1BrZY3/BkQAL2M0taiON7yCxlEt81+bFDfY8dAna4b1ts/w9TuuNAow0lzERC6hCbBMj7TMiKZ1qSgH5EvuvFzJmTTejikFXnK474SX4PoLF0tbVZt7KMJq++mpriVhSu4d3xCRMf03DHLgixh6j7XGYA4kSy99fts/5LXFaqInPOA924hjDNlbFelct3oSi0MeUZc1TYQaEmuSnBSoybKoCNSWyy/cUPKihSvKUH+NCQPOE58UiYI7t0Pe/AhDzXDljSZL3MwICzZxxSl4nlAz/hu5YVFzOLHJfixQolXR8xz1ChSokTtofWawGHLsOBuJZ4exTy7KxagU6VOXRRLv8JNzkF5StyPU9K8B7BMnN6b1iDWjcaVKpAtqhBrVaDcNTguRpUYK8azCT3hjWowG5ECKtItijCrFUCFU5GFZ6rQgj6VaFxMdHhrUvvY/O1ylMP/CLWO+GucCpOPgxMqtNupGp0qtRprVLBqNRzlVqdbXQp1PvF4+LvyFC/L9Qoir7//EIfwyuwp7JYvrl4bnPrQV8LBSYQYjOls+ObbTGxa+ADkkSeLUntCiifP/jlk7VUSsc6EFw8RDcFVBNNKA+wLCMl0k3doKdONGU1S2wMqOkQaQeSZnYMLBXr58qs4gTaiZAQNhPepD3hyaPwBMLT5cbCU/oSnt5g8j8KTxJOEPsXHjg+YB2Q8F4moGzt/bSAQG8CarCCPQpIEq5ydikgPFve/fB//3rY/v6GjA3Q/0RPV9M3P2JspDNx38iXllT9JJB97T8HFNFbmxCa35TnuxlY/4O/zu4/rKynKTCvdNgnI0CRD9e62pASRUJk2U5y4gDX15Gk6KHFFyod+eOj1ezZSVvao6YKnPRyuVRMcwhOWvAQEfTso41eFTlwH33xecORh8sDctLVpbrRSZ9myWknDTty0mJUp70KvYSq0hBVUEZVOYWqhdZOUlvQF8RyRxAfreYYh8+fLPUfiKtYjYG45DnbD8TJDpNiIO43DqtjHH4BSU476UmvcVgf43D7EPc7HxbtWhnjcKNV//7j8LhoecJzth+Hk01mw4nD2uixX8uY4UyOj1Zz9Njnb5Do0mPXPJ7sV6BtP6cVNtpovu3o4k77VWgZ416klz4sGMCOCNHj+GRzpzp3EJX02U7Sb97dW0prVpwb2DNCKfFj49Fh1qvWqfZbinrbB5rtpjrBgff2lkxbHBDMkQbAgbr3MsoceG/vaLTFAcGoawAcEI2Zqxx4b7v/2+JAdTjQJQXEE2OjwoDZ7Y+fA4W6psMFsNTrUD+Y+ig9IyDYefPZDTAK3w8GcNIaBiyZ/1dDupqU/x8GvP0f \ No newline at end of file diff --git a/week05_transformer_pos_tagging/assets/pos-bidirectional-lstm.png b/week05_transformer_pos_tagging/assets/pos-bidirectional-lstm.png deleted file mode 100644 index 3f0e7ae..0000000 Binary files a/week05_transformer_pos_tagging/assets/pos-bidirectional-lstm.png and /dev/null differ diff --git a/week05_transformer_pos_tagging/assets/pos-bidirectional-lstm.xml b/week05_transformer_pos_tagging/assets/pos-bidirectional-lstm.xml deleted file mode 100644 index e873dc6..0000000 --- a/week05_transformer_pos_tagging/assets/pos-bidirectional-lstm.xml +++ /dev/null @@ -1 +0,0 @@ -7V3blto2FP0aHifL98tjIJN0dWXapqRN8pTlwQLcMWhiPAHy9ZWxDBgdewQIyWDzMrYs37TP3kc6Otb0zMFs9SEJnqcPOERxz9DCVc981zMM3XU08icrWecltmnlBZMkCmmlXcEw+oVoIT1v8hKFaFGqmGIcp9FzuXCE53M0SktlQZLgZbnaGMfluz4HE8QUDEdBzJZ+icJ0mpd6hrMr/w1Fk2lxZ93x8yOPwehpkuCXOb1fzzDHm19+eBYU16J3WEyDEC/zos27m/c9c5BgnOZbs9UAxVnbFs2WN9D7iqPb507QPOU54evwcfj70rc/De3lcvnp80P6mNx5FKyfQfyCivfYPG26Llpo844ou4rWM/vLaZSi4XMwyo4uiU2Qsmk6i8meTja3b5nVncTBYkG3R3gWjej2Ik3wExrgGCebW5ie8Wg6DjkyjuJ4r7yv3Zv3m3I8T98HsyjOrGyAX5IIJeQp/0BLepAalp5VDuJoMic7I9IypJrZZ5uKtt5PlKRotVdEm+4DwjOUJmtSpTiq+W9MMz+LWrqpURNe7uzGokXTPZMpygJqqZPtxXdokQ0K2DHgmR14fOB5DQTP6MDjAs9wGgie1oHHBZ5pKQXP1BlYUEjcP92d4zn50y8jtd/iewihVZR+3dv+llV5Y2d7c/JUX+kZm53dsRKu/6E0XdMWD15STIpwkk7xBM+D+CPGz/TyR6FWCdKCnDiiL02tMw2SCaK1KChZc9QCmaA4SKOf5d7SOZg0W/dCG3mh1QTq6DZLHVMidVhYWkkdk6VOUU0+d5rd4WsOdwxTKXd0/XWcTiVTQ2hhi6YAPfUvHJG77kTQZYC07AOEcmrSEw9A2j4JF242ABsZIFur73SDQZHYY3qAT4ktFNl9otAixuwz645GQfyWHphFYZjdBuRw2VS4aCzGBs7yXzZn188QwUGOLvqVc9CRw0GDBfKCHHQqOWh2HBTgB2Vy0ODwg11/Jeuc6Er7KzxjsivXSvHjXVgsTZZxFxRLcECQq6XVqaUAFspUy4LwbR9xF12A0pBbNHt5QbF84aBQIApYmtfwl9ZIn6GZ4V9OIqEOZdcHYdWPHWlL7YLI4NlO/HZa+K1h/RRPFQcv2U3xKrspetdNOZ+oMnspdoun1Eo9EknOkownHMfc/iyvLNFWdnj3c63y9XNpERIbheJpGXWnPfu+Zw+SzMw2CVnftY7Tx3L6TnvjuAfIyuM0mOOga3Yt03c8u9+VHrQ1yEWAs3zem2y/W+3vrDn8+mZnd9pmrziPyyPbBkt+sMF84XMi3JoMxW9gbrbN3y7I7aL55HNufbWgn+OBDUeiB67n5dUPSRXMKfKOP3mutR01iffBFTrtdDqd0QIIG7nKMjXsqnksVpKNTpLPl2RgtlmqJrudJgueYz5Zk4Frydfk+nBWazTZAzRZU6bJVWEoVpPblmtwEU0Gsg+kavKNT90omMo+WZOBa0nXZL1+frUtmlwwcF+THWWhC4c/rNi2jIaLaDKQ4yBTk2Fmbs30Zpl5Wn6nzTK1QtqUfc3jQDnTB1hKmF73HdcMgOl1pIc2co9lnqQUP13i/HoF8W7eJZ5EPBdwkRXEM1QRz4UcZ0c8rlGIeuLpVxMfOI1AwIR5RUMoi9AWz9gR6PjQqkwCof74y9+zH9+/rX78Gfgvuvtr+XTn3XyPkYtncNsA3kt8tsp58DXDeYXOo2MD3BuPx8Zo1ATuAZ8/6oqp55sd9Srt2gCo1yzmGR3zTvV66qnHYtVRrzBsYKEGZVGR2ifsqHf8iE099boOZ7VhW4DXUzauq33EjnvHhyllcg9ebe1qEtzCYDHd3FPnJVTt8nIXn+Fll9bbTgaJ/woITppWG4HeR3aL8yu6uhXSg+9VKnRVsE28GlIrevjylRcGWMlwsdTopyJXCdbrGEgisAWsG3Eow7wpGpbBXutwyvfCKRpefRblDQq9cK5WpESyC+BKF3q1HegrEnoghlS7/HNDhN7z2kZeSUvKGOzS49LJqzbwdEXkBaJQtQvvN4W8VzMfLQoo4e0Pk1dne1XSydu2XpUvCVsoQVrjw/ZtkgTrvWrPWYVFza2AkbqmHVhLflGxtgPFzfbSc2M07j4sFpidyy6Xb3CmWlwuO1e/mvDbSblKtd/ycyQwNcuR+10+7qmZFeqzAjW13WwFnlrSalfQ56UX8tRQqEWKp37l48adpzY6T32+fAD/lUi9p9auJjVLlH74cvQDWjLkUvoBRHuk6IfPqx/dt9EC9AP4l3QN0A8W1NvWD12TNFEHLHl7If2AAk4y9GPblK8LSPchrwgBAfyEegGBvqkBbaBbI/JoxKEJes7UrAsGh6oQJ2hPg7Tn9tc9911bxxx9nKZ4JpL17EDWVW4CBpcJtLTbKNwEgOxo9SZQ9X9IyibQUscv3ASAJF31JlA1U1Q2gZZOEwk3ATYgIdECPkToH3/RXw7/nYex+WT7g/HwDvie+n72iMKQvP6ioXhXtDqATXWvrBj3FNnyhmocWHf8MZqjIGkPBrrEtdNhDFh/2I/uwihBozTKgiPk2Mfh54cbhsQrQwKNVDwxkJDdBON0P5xBXmr6gEOU1fgf \ No newline at end of file diff --git a/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging.ipynb b/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging.ipynb deleted file mode 100644 index e79553d..0000000 --- a/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging.ipynb +++ /dev/null @@ -1,1533 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Practice: BiLSTM for PoS Tagging\n", - "*This notebook is based on [open-source implementation](https://github.com/bentrevett/pytorch-pos-tagging) of PoS Tagging in PyTorch.*\n", - "\n", - "### Introduction\n", - "\n", - "In this series we'll be building a machine learning model that produces an output for every element in an input sequence, using PyTorch and TorchText. Specifically, we will be inputting a sequence of text and the model will output a part-of-speech (PoS) tag for each token in the input text. This can also be used for named entity recognition (NER), where the output for each token will be what type of entity, if any, the token is.\n", - "\n", - "In this notebook, we'll be implementing a multi-layer bi-directional LSTM (BiLSTM) to predict PoS tags using the Universal Dependencies English Web Treebank (UDPOS) dataset.\n", - "\n", - "### Preparing Data\n", - "\n", - "First, let's import the necessary Python modules." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "\n", - "from torchtext.legacy import data\n", - "from torchtext.legacy import datasets\n", - "\n", - "import spacy\n", - "import numpy as np\n", - "\n", - "import time\n", - "import random" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we'll set the random seeds for reproducability." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "SEED = 1234\n", - "\n", - "random.seed(SEED)\n", - "np.random.seed(SEED)\n", - "torch.manual_seed(SEED)\n", - "torch.backends.cudnn.deterministic = True" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "One of the key parts of TorchText is the `Field`. The `Field` handles how your dataset is processed.\n", - "\n", - "Our `TEXT` field handles how the text that we need to tag is dealt with. All we do here is set `lower = True` which lowercases all of the text.\n", - "\n", - "Next we'll define the `Fields` for the tags. This dataset actually has two different sets of tags, [universal dependency (UD) tags](https://universaldependencies.org/u/pos/) and [Penn Treebank (PTB) tags](https://www.sketchengine.eu/penn-treebank-tagset/). We'll only train our model on the UD tags, but will load the PTB tags to show how they could be used instead.\n", - "\n", - "`UD_TAGS` handles how the UD tags should be handled. Our `TEXT` vocabulary - which we'll build later - will have *unknown* tokens in it, i.e. tokens that are not within our vocabulary. However, we won't have unknown tags as we are dealing with a finite set of possible tags. TorchText `Fields` initialize a default unknown token, ``, which we remove by setting `unk_token = None`.\n", - "\n", - "`PTB_TAGS` does the same as `UD_TAGS`, but handles the PTB tags instead." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "TEXT = data.Field(lower = True)\n", - "UD_TAGS = data.Field(unk_token = None)\n", - "PTB_TAGS = data.Field(unk_token = None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then define `fields`, which handles passing our fields to the dataset.\n", - "\n", - "Note that order matters, if you only wanted to load the PTB tags your field would be:\n", - "\n", - "```\n", - "fields = ((\"text\", TEXT), (None, None), (\"ptbtags\", PTB_TAGS))\n", - "```\n", - "\n", - "Where `None` tells TorchText to not load those tags." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fields = ((\"text\", TEXT), (\"udtags\", UD_TAGS), (\"ptbtags\", PTB_TAGS))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we load the UDPOS dataset using our defined fields." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_data, valid_data, test_data = datasets.UDPOS.splits(fields)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can check how many examples are in each section of the dataset by checking their length." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"Number of training examples: {len(train_data)}\")\n", - "print(f\"Number of validation examples: {len(valid_data)}\")\n", - "print(f\"Number of testing examples: {len(test_data)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's print out an example:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(vars(train_data.examples[0]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also view the text and tags separately:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(vars(train_data.examples[0])['text'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(vars(train_data.examples[0])['udtags'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(vars(train_data.examples[0])['ptbtags'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we'll build the vocabulary - a mapping of tokens to integers. \n", - "\n", - "We want some unknown tokens within our dataset in order to replicate how this model would be used in real life, so we set the `min_freq` to 2 which means only tokens that appear twice in the training set will be added to the vocabulary and the rest will be replaced by `` tokens.\n", - "\n", - "We also load the [GloVe](https://nlp.stanford.edu/projects/glove/) pre-trained token embeddings. Specifically, the 100-dimensional embeddings that have been trained on 6 billion tokens. Using pre-trained embeddings usually leads to improved performance - although admittedly the dataset used in this tutorial is too small to take advantage of the pre-trained embeddings. \n", - "\n", - "`unk_init` is used to initialize the token embeddings which are not in the pre-trained embedding vocabulary. By default this sets those embeddings to zeros, however it is better to not have them all initialized to the same value, so we initialize them from a Normal/Gaussian distribution.\n", - "\n", - "These pre-trained vectors are now loaded into our vocabulary and we will initialize our model with these values later." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MIN_FREQ = 2\n", - "\n", - "TEXT.build_vocab(train_data, \n", - " min_freq = MIN_FREQ,\n", - " vectors = \"glove.6B.100d\",\n", - " unk_init = torch.Tensor.normal_)\n", - "\n", - "\n", - "UD_TAGS.build_vocab(train_data)\n", - "PTB_TAGS.build_vocab(train_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can check how many tokens and tags are in our vocabulary by getting their length:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\n", - "print(f\"Unique tokens in UD_TAG vocabulary: {len(UD_TAGS.vocab)}\")\n", - "print(f\"Unique tokens in PTB_TAG vocabulary: {len(PTB_TAGS.vocab)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Exploring the vocabulary, we can check the most common tokens within our texts:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(TEXT.vocab.freqs.most_common(20))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the vocabularies for both of our tags:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(UD_TAGS.vocab.itos)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(PTB_TAGS.vocab.itos)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also see how many of each tag are in our vocabulary:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(UD_TAGS.vocab.freqs.most_common())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(PTB_TAGS.vocab.freqs.most_common())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also view how common each of the tags are within the training set:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def tag_percentage(tag_counts):\n", - " \n", - " total_count = sum([count for tag, count in tag_counts])\n", - " \n", - " tag_counts_percentages = [(tag, count, count/total_count) for tag, count in tag_counts]\n", - " \n", - " return tag_counts_percentages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Tag\\t\\tCount\\t\\tPercentage\\n\")\n", - "\n", - "for tag, count, percent in tag_percentage(UD_TAGS.vocab.freqs.most_common()):\n", - " print(f\"{tag}\\t\\t{count}\\t\\t{percent*100:4.1f}%\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Tag\\t\\tCount\\t\\tPercentage\\n\")\n", - "\n", - "for tag, count, percent in tag_percentage(PTB_TAGS.vocab.freqs.most_common()):\n", - " print(f\"{tag}\\t\\t{count}\\t\\t{percent*100:4.1f}%\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The final part of data preparation is handling the iterator. \n", - "\n", - "This will be iterated over to return batches of data to process. Here, we set the batch size and the `device` - which is used to place the batches of tensors on our GPU, if we have one. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "BATCH_SIZE = 128\n", - "\n", - "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n", - "\n", - "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE,\n", - " device = device)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Building the Model\n", - "\n", - "Next up, we define our model - a multi-layer bi-directional LSTM. The image below shows a simplified version of the model with only one LSTM layer and omitting the LSTM's cell state for clarity.\n", - "\n", - "![](assets/pos-bidirectional-lstm.png)\n", - "\n", - "The model takes in a sequence of tokens, $X = \\{x_1, x_2,...,x_T\\}$, passes them through an embedding layer, $e$, to get the token embeddings, $e(X) = \\{e(x_1), e(x_2), ..., e(x_T)\\}$.\n", - "\n", - "These embeddings are processed - one per time-step - by the forward and backward LSTMs. The forward LSTM processes the sequence from left-to-right, whilst the backward LSTM processes the sequence right-to-left, i.e. the first input to the forward LSTM is $x_1$ and the first input to the backward LSTM is $x_T$. \n", - "\n", - "The LSTMs also take in the the hidden, $h$, and cell, $c$, states from the previous time-step\n", - "\n", - "$$h^{\\rightarrow}_t = \\text{LSTM}^{\\rightarrow}(e(x^{\\rightarrow}_t), h^{\\rightarrow}_{t-1}, c^{\\rightarrow}_{t-1})$$\n", - "$$h^{\\leftarrow}_t=\\text{LSTM}^{\\leftarrow}(e(x^{\\leftarrow}_t), h^{\\leftarrow}_{t-1}, c^{\\leftarrow}_{t-1})$$\n", - "\n", - "After the whole sequence has been processed, the hidden and cell states are then passed to the next layer of the LSTM.\n", - "\n", - "The initial hidden and cell states, $h_0$ and $c_0$, for each direction and layer are initialized to a tensor full of zeros.\n", - "\n", - "We then concatenate both the forward and backward hidden states from the final layer of the LSTM, $H = \\{h_1, h_2, ... h_T\\}$, where $h_1 = [h^{\\rightarrow}_1;h^{\\leftarrow}_T]$, $h_2 = [h^{\\rightarrow}_2;h^{\\leftarrow}_{T-1}]$, etc. and pass them through a linear layer, $f$, which is used to make the prediction of which tag applies to this token, $\\hat{y}_t = f(h_t)$.\n", - "\n", - "When training the model, we will compare our predicted tags, $\\hat{Y}$ against the actual tags, $Y$, to calculate a loss, the gradients w.r.t. that loss, and then update our parameters.\n", - "\n", - "We implement the model detailed above in the `BiLSTMPOSTagger` class.\n", - "\n", - "`nn.Embedding` is an embedding layer and the input dimension should be the size of the input (text) vocabulary. We tell it what the index of the padding token is so it does not update the padding token's embedding entry.\n", - "\n", - "`nn.LSTM` is the LSTM. We apply dropout as regularization between the layers, if we are using more than one.\n", - "\n", - "`nn.Linear` defines the linear layer to make predictions using the LSTM outputs. We double the size of the input if we are using a bi-directional LSTM. The output dimensions should be the size of the tag vocabulary.\n", - "\n", - "We also define a dropout layer with `nn.Dropout`, which we use in the `forward` method to apply dropout to the embeddings and the outputs of the final layer of the LSTM." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class BiLSTMPOSTagger(nn.Module):\n", - " def __init__(self, \n", - " input_dim, \n", - " embedding_dim, \n", - " hidden_dim, \n", - " output_dim, \n", - " n_layers, \n", - " bidirectional, \n", - " dropout, \n", - " pad_idx):\n", - " \n", - " super().__init__()\n", - " \n", - " self.embedding = nn.Embedding(input_dim, embedding_dim, padding_idx = pad_idx)\n", - " \n", - " self.lstm = nn.LSTM(embedding_dim, \n", - " hidden_dim, \n", - " num_layers = n_layers, \n", - " bidirectional = bidirectional,\n", - " dropout = dropout if n_layers > 1 else 0)\n", - " \n", - " self.fc = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)\n", - " \n", - " self.dropout = nn.Dropout(dropout)\n", - " \n", - " def forward(self, text):\n", - "\n", - " #text = [sent len, batch size]\n", - " \n", - " #pass text through embedding layer\n", - " embedded = self.dropout(self.embedding(text))\n", - " \n", - " #embedded = [sent len, batch size, emb dim]\n", - " \n", - " #pass embeddings into LSTM\n", - " outputs, (hidden, cell) = self.lstm(embedded)\n", - " \n", - " #outputs holds the backward and forward hidden states in the final layer\n", - " #hidden and cell are the backward and forward hidden and cell states at the final time-step\n", - " \n", - " #output = [sent len, batch size, hid dim * n directions]\n", - " #hidden/cell = [n layers * n directions, batch size, hid dim]\n", - " \n", - " #we use our outputs to make a prediction of what the tag should be\n", - " predictions = self.fc(self.dropout(outputs))\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " \n", - " return predictions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training the Model\n", - "\n", - "Next, we instantiate the model. We need to ensure the embedding dimensions matches that of the GloVe embeddings we loaded earlier.\n", - "\n", - "The rest of the hyperparmeters have been chosen as sensible defaults, though there may be a combination that performs better on this model and dataset.\n", - "\n", - "The input and output dimensions are taken directly from the lengths of the respective vocabularies. The padding index is obtained using the vocabulary and the `Field` of the text." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(TEXT.vocab)\n", - "EMBEDDING_DIM = 100\n", - "HIDDEN_DIM = 128\n", - "OUTPUT_DIM = len(UD_TAGS.vocab)\n", - "N_LAYERS = 2\n", - "BIDIRECTIONAL = True\n", - "DROPOUT = 0.25\n", - "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n", - "\n", - "model = BiLSTMPOSTagger(INPUT_DIM, \n", - " EMBEDDING_DIM, \n", - " HIDDEN_DIM, \n", - " OUTPUT_DIM, \n", - " N_LAYERS, \n", - " BIDIRECTIONAL, \n", - " DROPOUT, \n", - " PAD_IDX)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We initialize the weights from a simple Normal distribution. Again, there may be a better initialization scheme for this model and dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def init_weights(m):\n", - " for name, param in m.named_parameters():\n", - " nn.init.normal_(param.data, mean = 0, std = 0.1)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, a small function to tell us how many parameters are in our model. Useful for comparing different models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll now initialize our model's embedding layer with the pre-trained embedding values we loaded earlier.\n", - "\n", - "This is done by getting them from the vocab's `.vectors` attribute and then performing a `.copy` to overwrite the embedding layer's current weights." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pretrained_embeddings = TEXT.vocab.vectors\n", - "\n", - "print(pretrained_embeddings.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.embedding.weight.data.copy_(pretrained_embeddings)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's common to initialize the embedding of the pad token to all zeros. This, along with setting the `padding_idx` in the model's embedding layer, means that the embedding should always output a tensor full of zeros when a pad token is input." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n", - "\n", - "print(model.embedding.weight.data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then define our optimizer, used to update our parameters w.r.t. their gradients. We use Adam with the default learning rate." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = optim.Adam(model.parameters())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we define our loss function, cross-entropy loss.\n", - "\n", - "Even though we have no `` tokens within our tag vocab, we still have `` tokens. This is because all sentences within a batch need to be the same size. However, we don't want to calculate the loss when the target is a `` token as we aren't training our model to recognize padding tokens.\n", - "\n", - "We handle this by setting the `ignore_index` in our loss function to the index of the padding token in our tag vocabulary." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "TAG_PAD_IDX = UD_TAGS.vocab.stoi[UD_TAGS.pad_token]\n", - "\n", - "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then place our model and loss function on our GPU, if we have one." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = model.to(device)\n", - "criterion = criterion.to(device)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will be using the loss value between our predicted and actual tags to train the network, but ideally we'd like a more interpretable way to see how well our model is doing - accuracy.\n", - "\n", - "The issue is that we don't want to calculate accuracy over the `` tokens as we aren't interested in predicting them.\n", - "\n", - "The function below only calculates accuracy over non-padded tokens. `non_pad_elements` is a tensor containing the indices of the non-pad tokens within an input batch. We then compare the predictions of those elements with the labels to get a count of how many predictions were correct. We then divide this by the number of non-pad elements to get our accuracy value over the batch." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def categorical_accuracy(preds, y, tag_pad_idx):\n", - " \"\"\"\n", - " Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n", - " \"\"\"\n", - " max_preds = preds.argmax(dim = 1, keepdim = True) # get the index of the max probability\n", - " non_pad_elements = (y != tag_pad_idx).nonzero()\n", - " correct = max_preds[non_pad_elements].squeeze(1).eq(y[non_pad_elements])\n", - " return correct.sum() / torch.FloatTensor([y[non_pad_elements].shape[0]]).to(device)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next is the function that handles training our model.\n", - "\n", - "We first set the model to `train` mode to turn on dropout/batch-norm/etc. (if used). Then we iterate over our iterator, which returns a batch of examples. \n", - "\n", - "For each batch: \n", - "- we zero the gradients over the parameters from the last gradient calculation\n", - "- insert the batch of text into the model to get predictions\n", - "- as PyTorch loss functions cannot handle 3-dimensional predictions we reshape our predictions\n", - "- calculate the loss and accuracy between the predicted tags and actual tags\n", - "- call `backward` to calculate the gradients of the parameters w.r.t. the loss\n", - "- take an optimizer `step` to update the parameters\n", - "- add to the running total of loss and accuracy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.train()\n", - " \n", - " for batch in iterator:\n", - " \n", - " text = batch.text\n", - " tags = batch.udtags\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " #text = [sent len, batch size]\n", - " \n", - " predictions = model(text)\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " #tags = [sent len, batch size]\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " #predictions = [sent len * batch size, output dim]\n", - " #tags = [sent len * batch size]\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - " \n", - " loss.backward()\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `evaluate` function is similar to the `train` function, except with changes made so we don't update the model's parameters.\n", - "\n", - "`model.eval()` is used to put the model in evaluation mode, so dropout/batch-norm/etc. are turned off. \n", - "\n", - "The iteration loop is also wrapped in `torch.no_grad` to ensure we don't calculate any gradients. We also don't need to call `optimizer.zero_grad()` and `optimizer.step()`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def evaluate(model, iterator, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.eval()\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for batch in iterator:\n", - "\n", - " text = batch.text\n", - " tags = batch.udtags\n", - " \n", - " predictions = model(text)\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - "\n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we have a small function that tells us how long an epoch takes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def epoch_time(start_time, end_time):\n", - " elapsed_time = end_time - start_time\n", - " elapsed_mins = int(elapsed_time / 60)\n", - " elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n", - " return elapsed_mins, elapsed_secs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, we train our model!\n", - "\n", - "After each epoch we check if our model has achieved the best validation loss so far. If it has then we save the parameters of this model and we will use these \"best\" parameters to calculate performance over our test set." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "N_EPOCHS = 15\n", - "\n", - "best_valid_loss = float('inf')\n", - "\n", - "for epoch in range(N_EPOCHS):\n", - "\n", - " start_time = time.time()\n", - " \n", - " train_loss, train_acc = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n", - " valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)\n", - " \n", - " end_time = time.time()\n", - "\n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut1-model.pt')\n", - " \n", - " print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}%')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then load our \"best\" parameters and evaluate performance on the test set." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.load_state_dict(torch.load('tut1-model.pt'))\n", - "\n", - "test_loss, test_acc = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n", - "\n", - "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inference\n", - "\n", - "88% accuracy looks pretty good, but let's see our model tag some actual sentences.\n", - "\n", - "We define a `tag_sentence` function that will:\n", - "- put the model into evaluation mode\n", - "- tokenize the sentence with spaCy if it is not a list\n", - "- lowercase the tokens if the `Field` did\n", - "- numericalize the tokens using the vocabulary\n", - "- find out which tokens are not in the vocabulary, i.e. are `` tokens\n", - "- convert the numericalized tokens into a tensor and add a batch dimension\n", - "- feed the tensor into the model\n", - "- get the predictions over the sentence\n", - "- convert the predictions into readable tags\n", - "\n", - "As well as returning the tokens and tags, it also returns which tokens were `` tokens." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def tag_sentence(model, device, sentence, text_field, tag_field):\n", - " \n", - " model.eval()\n", - " \n", - " if isinstance(sentence, str):\n", - " nlp = spacy.load('en')\n", - " tokens = [token.text for token in nlp(sentence)]\n", - " else:\n", - " tokens = [token for token in sentence]\n", - "\n", - " if text_field.lower:\n", - " tokens = [t.lower() for t in tokens]\n", - " \n", - " numericalized_tokens = [text_field.vocab.stoi[t] for t in tokens]\n", - "\n", - " unk_idx = text_field.vocab.stoi[text_field.unk_token]\n", - " \n", - " unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n", - " \n", - " token_tensor = torch.LongTensor(numericalized_tokens)\n", - " \n", - " token_tensor = token_tensor.unsqueeze(-1).to(device)\n", - " \n", - " predictions = model(token_tensor)\n", - " \n", - " top_predictions = predictions.argmax(-1)\n", - " \n", - " predicted_tags = [tag_field.vocab.itos[t.item()] for t in top_predictions]\n", - " \n", - " return tokens, predicted_tags, unks" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll get an already tokenized example from the training set and test our model's performance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "example_index = 1\n", - "\n", - "sentence = vars(train_data.examples[example_index])['text']\n", - "actual_tags = vars(train_data.examples[example_index])['udtags']\n", - "\n", - "print(sentence)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then use our `tag_sentence` function to get the tags. Notice how the tokens referring to subject of the sentence, the \"respected cleric\", are both `` tokens!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tokens, pred_tags, unks = tag_sentence(model, \n", - " device, \n", - " sentence, \n", - " TEXT, \n", - " UD_TAGS)\n", - "\n", - "print(unks)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then check how well it did. Surprisingly, it got every token correct, including the two that were unknown tokens!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Pred. Tag\\tActual Tag\\tCorrect?\\tToken\\n\")\n", - "\n", - "for token, pred_tag, actual_tag in zip(tokens, pred_tags, actual_tags):\n", - " correct = '✔' if pred_tag == actual_tag else '✘'\n", - " print(f\"{pred_tag}\\t\\t{actual_tag}\\t\\t{correct}\\t\\t{token}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's now make up our own sentence and see how well the model does.\n", - "\n", - "Our example sentence below has every token within the model's vocabulary." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sentence = 'The Queen will deliver a speech about the conflict in North Korea at 1pm tomorrow.'\n", - "\n", - "tokens, tags, unks = tag_sentence(model, \n", - " device, \n", - " sentence, \n", - " TEXT, \n", - " UD_TAGS)\n", - "\n", - "print(unks)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Looking at the sentence it seems like it gave sensible tags to every token!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Pred. Tag\\tToken\\n\")\n", - "\n", - "for token, tag in zip(tokens, tags):\n", - " print(f\"{tag}\\t\\t{token}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We've now seen how to implement PoS tagging with PyTorch and TorchText! \n", - "\n", - "The BiLSTM isn't a state-of-the-art model, in terms of performance, but is a strong baseline for PoS tasks and is a good tool to have in your arsenal." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Going deeper\n", - "What if we could combine word-level and char-level approaches? \n", - "![title](https://i.postimg.cc/tT9hsBfj/ive-put-an-rnn-in-your-rnn-so-you-can-train-an-rnn-on-every-step-of-your-rnn-training-loop.jpg)\n", - "\n", - "\n", - "Actually, we can. Let's use LSTM or GRU to generate embedding for every word on char-level.\n", - "![title](https://guillaumegenthial.github.io/assets/char_representation.png)\n", - "*Image source: https://guillaumegenthial.github.io/sequence-tagging-with-tensorflow.html*\n", - "\n", - "![title](https://guillaumegenthial.github.io/assets/bi-lstm.png)\n", - "*Image source: https://guillaumegenthial.github.io/sequence-tagging-with-tensorflow.html*" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To do that we need to make few adjustments to the code above" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Now lets try both word and character embeddings\n", - "WORD = data.Field(lower = True)\n", - "UD_TAG = data.Field(unk_token = None)\n", - "PTB_TAG = data.Field(unk_token = None)\n", - "\n", - "# We'll use NestedField to tokenize each word into list of chars\n", - "CHAR_NESTING = data.Field(tokenize=list, init_token=\"\", eos_token=\"\")\n", - "CHAR = data.NestedField(CHAR_NESTING)#, init_token=\"\", eos_token=\"\")\n", - "\n", - "fields = [(('word', 'char'), (WORD, CHAR)), ('udtag', UD_TAG), ('ptbtag', PTB_TAG)]\n", - "train_data, valid_data, test_data = datasets.UDPOS.splits(fields)\n", - "# train, val, test = datasets.UDPOS.splits(fields=fields)\n", - "\n", - "print(train_data.fields)\n", - "print(len(train_data))\n", - "print(vars(train_data[0]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "WORD.build_vocab(\n", - " train_data,\n", - " min_freq = MIN_FREQ,\n", - " vectors=\"glove.6B.100d\",\n", - " unk_init = torch.Tensor.normal_\n", - ")\n", - "\n", - "\n", - "CHAR.build_vocab(train_data)\n", - "UD_TAG.build_vocab(train_data)\n", - "PTB_TAG.build_vocab(train_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"Unique tokens in WORD vocabulary: {len(WORD.vocab)}\")\n", - "print(f\"Unique tokens in CHAR vocabulary: {len(CHAR.vocab)}\")\n", - "print(f\"Unique tokens in UD_TAG vocabulary: {len(UD_TAG.vocab)}\")\n", - "print(f\"Unique tokens in PTB_TAG vocabulary: {len(PTB_TAG.vocab)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "BATCH_SIZE = 64\n", - "\n", - "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n", - "\n", - "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE,\n", - " device = device)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch = next(iter(train_iterator))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "text = batch.word\n", - "chars = batch.char\n", - "tags = batch.udtag\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class BiLSTMPOSTaggerWithChars(nn.Module):\n", - " def __init__(self, \n", - " word_input_dim, \n", - " word_embedding_dim,\n", - " char_input_dim,\n", - " char_embedding_dim,\n", - " char_hidden_dim,\n", - " hidden_dim,\n", - " output_dim, \n", - " n_layers, \n", - " bidirectional, \n", - " dropout, \n", - " pad_idx):\n", - " \n", - " super().__init__()\n", - " \n", - " self.char_embedding = # YOUR CODE HERE\n", - " self.char_gru = # YOUR CODE HERE\n", - " \n", - " self.word_embedding = nn.Embedding(word_input_dim, word_embedding_dim, padding_idx = pad_idx)\n", - " self.lstm = nn.LSTM(word_embedding_dim + # YOUR CODE HERE, \n", - " hidden_dim, \n", - " num_layers = n_layers, \n", - " bidirectional = bidirectional,\n", - " dropout = dropout if n_layers > 1 else 0)\n", - " \n", - " self.fc = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)\n", - " \n", - " self.dropout = nn.Dropout(dropout)\n", - " \n", - " def forward(self, text, chars):\n", - "\n", - " #text = [sent len, batch size]\n", - " \n", - " #pass text through embedding layer\n", - " embedded = self.dropout(self.word_embedding(text))\n", - " #embedded = [sent len, batch size, emb dim]\n", - " \n", - " chars_embedded = # YOUR CODE HERE\n", - " hid_from_chars = # YOUR CODE HERE\n", - " \n", - " embedded_with_chars = torch.cat([embedded, hid_from_chars], dim=2)\n", - " \n", - " \n", - " #pass embeddings into LSTM\n", - " outputs, (hidden, cell) = self.lstm(embedded_with_chars)\n", - "# outputs, (hidden, cell) = self.lstm(hid)\n", - "\n", - " \n", - " #outputs holds the backward and forward hidden states in the final layer\n", - " #hidden and cell are the backward and forward hidden and cell states at the final time-step\n", - " \n", - " #output = [sent len, batch size, hid dim * n directions]\n", - " #hidden/cell = [n layers * n directions, batch size, hid dim]\n", - " \n", - " #we use our outputs to make a prediction of what the tag should be\n", - " predictions = self.fc(self.dropout(outputs))\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " \n", - " return predictions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(WORD.vocab)\n", - "EMBEDDING_DIM = 100\n", - "HIDDEN_DIM = 160\n", - "CHAR_INPUT_DIM = 112\n", - "CHAR_EMBEDDING_DIM = 30\n", - "CHAR_HIDDEN_DIM = 30\n", - "OUTPUT_DIM = len(UD_TAGS.vocab)\n", - "N_LAYERS = 2\n", - "BIDIRECTIONAL = True\n", - "DROPOUT = 0.25\n", - "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n", - "\n", - "model = BiLSTMPOSTaggerWithChars(\n", - " INPUT_DIM, \n", - " EMBEDDING_DIM,\n", - " CHAR_INPUT_DIM,\n", - " CHAR_EMBEDDING_DIM,\n", - " CHAR_HIDDEN_DIM,\n", - " HIDDEN_DIM, \n", - " OUTPUT_DIM, \n", - " N_LAYERS, \n", - " BIDIRECTIONAL, \n", - " DROPOUT, \n", - " PAD_IDX\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Congratulations, you've got LSTM which relies on GRU output on each step.**\n", - "\n", - "Now we need only to train it. Same actions, very small adjustments." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def init_weights(m):\n", - " for name, param in m.named_parameters():\n", - " nn.init.normal_(param.data, mean = 0, std = 0.1)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pretrained_embeddings = TEXT.vocab.vectors\n", - "\n", - "print(pretrained_embeddings.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.word_embedding.weight.data.copy_(pretrained_embeddings)\n", - "model.word_embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n", - "\n", - "print(model.word_embedding.weight.data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = optim.Adam(model.parameters())\n", - "\n", - "TAG_PAD_IDX = UD_TAGS.vocab.stoi[UD_TAGS.pad_token]\n", - "\n", - "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n", - "\n", - "model = model.to(device)\n", - "criterion = criterion.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.train()\n", - " \n", - " for batch in iterator:\n", - " \n", - " text = batch.word\n", - " chars = batch.char\n", - " tags = batch.udtag\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " #text = [sent len, batch size]\n", - " \n", - " predictions = model(text, chars)\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " #tags = [sent len, batch size]\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " #predictions = [sent len * batch size, output dim]\n", - " #tags = [sent len * batch size]\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - " \n", - " loss.backward()\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)\n", - "\n", - "\n", - "def evaluate(model, iterator, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.eval()\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for batch in iterator:\n", - "\n", - " text = batch.word\n", - " chars = batch.char\n", - " tags = batch.udtag\n", - " \n", - " predictions = model(text, chars)\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - "\n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "N_EPOCHS = 15\n", - "\n", - "best_valid_loss = float('inf')\n", - "\n", - "for epoch in range(N_EPOCHS):\n", - "\n", - " start_time = time.time()\n", - " \n", - " train_loss, train_acc = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n", - " valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)\n", - " \n", - " end_time = time.time()\n", - "\n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut2-model.pt')\n", - " \n", - " print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}%')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Let's take a look at the model from the last epoch\n", - "test_loss, test_acc = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n", - "\n", - "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# And at the best checkpoint (based on validation score)\n", - "model.load_state_dict(torch.load('tut2-model.pt'))\n", - "\n", - "test_loss, test_acc = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n", - "\n", - "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Py3 research env", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging__completed.ipynb b/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging__completed.ipynb deleted file mode 100644 index cd5e9a7..0000000 --- a/week05_transformer_pos_tagging/week05_bilstm_for_pos_tagging__completed.ipynb +++ /dev/null @@ -1,2358 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Practice: BiLSTM for PoS Tagging\n", - "*This notebook is based on [open-source implementation](https://github.com/bentrevett/pytorch-pos-tagging) of PoS Tagging in PyTorch.*\n", - "\n", - "### Introduction\n", - "\n", - "In this series we'll be building a machine learning model that produces an output for every element in an input sequence, using PyTorch and TorchText. Specifically, we will be inputting a sequence of text and the model will output a part-of-speech (PoS) tag for each token in the input text. This can also be used for named entity recognition (NER), where the output for each token will be what type of entity, if any, the token is.\n", - "\n", - "In this notebook, we'll be implementing a multi-layer bi-directional LSTM (BiLSTM) to predict PoS tags using the Universal Dependencies English Web Treebank (UDPOS) dataset.\n", - "\n", - "### Preparing Data\n", - "\n", - "First, let's import the necessary Python modules." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "\n", - "from torchtext.legacy import data\n", - "from torchtext.legacy import datasets\n", - "\n", - "import spacy\n", - "import numpy as np\n", - "\n", - "import time\n", - "import random" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we'll set the random seeds for reproducability." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "SEED = 1234\n", - "\n", - "random.seed(SEED)\n", - "np.random.seed(SEED)\n", - "torch.manual_seed(SEED)\n", - "torch.backends.cudnn.deterministic = True" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "One of the key parts of TorchText is the `Field`. The `Field` handles how your dataset is processed.\n", - "\n", - "Our `TEXT` field handles how the text that we need to tag is dealt with. All we do here is set `lower = True` which lowercases all of the text.\n", - "\n", - "Next we'll define the `Fields` for the tags. This dataset actually has two different sets of tags, [universal dependency (UD) tags](https://universaldependencies.org/u/pos/) and [Penn Treebank (PTB) tags](https://www.sketchengine.eu/penn-treebank-tagset/). We'll only train our model on the UD tags, but will load the PTB tags to show how they could be used instead.\n", - "\n", - "`UD_TAGS` handles how the UD tags should be handled. Our `TEXT` vocabulary - which we'll build later - will have *unknown* tokens in it, i.e. tokens that are not within our vocabulary. However, we won't have unknown tags as we are dealing with a finite set of possible tags. TorchText `Fields` initialize a default unknown token, ``, which we remove by setting `unk_token = None`.\n", - "\n", - "`PTB_TAGS` does the same as `UD_TAGS`, but handles the PTB tags instead." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "TEXT = data.Field(lower = True)\n", - "UD_TAGS = data.Field(unk_token = None)\n", - "PTB_TAGS = data.Field(unk_token = None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then define `fields`, which handles passing our fields to the dataset.\n", - "\n", - "Note that order matters, if you only wanted to load the PTB tags your field would be:\n", - "\n", - "```\n", - "fields = ((\"text\", TEXT), (None, None), (\"ptbtags\", PTB_TAGS))\n", - "```\n", - "\n", - "Where `None` tells TorchText to not load those tags." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "fields = ((\"text\", TEXT), (\"udtags\", UD_TAGS), (\"ptbtags\", PTB_TAGS))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we load the UDPOS dataset using our defined fields." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "downloading en-ud-v2.zip\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "en-ud-v2.zip: 100%|██████████| 688k/688k [00:00<00:00, 1.02MB/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "extracting\n" - ] - } - ], - "source": [ - "train_data, valid_data, test_data = datasets.UDPOS.splits(fields)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can check how many examples are in each section of the dataset by checking their length." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training examples: 12543\n", - "Number of validation examples: 2002\n", - "Number of testing examples: 2077\n" - ] - } - ], - "source": [ - "print(f\"Number of training examples: {len(train_data)}\")\n", - "print(f\"Number of validation examples: {len(valid_data)}\")\n", - "print(f\"Number of testing examples: {len(test_data)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's print out an example:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'text': ['i', 'will', 'never', 'return', 'there', 'again', '(', 'and', 'now', 'have', 'some', 'serious', 'doubts', 'about', 'the', 'quality', 'of', 'work', 'they', 'actually', 'performed', 'on', 'my', 'car', ')', '.'], 'udtags': ['PRON', 'AUX', 'ADV', 'VERB', 'ADV', 'ADV', 'PUNCT', 'CCONJ', 'ADV', 'VERB', 'DET', 'ADJ', 'NOUN', 'ADP', 'DET', 'NOUN', 'ADP', 'NOUN', 'PRON', 'ADV', 'VERB', 'ADP', 'PRON', 'NOUN', 'PUNCT', 'PUNCT'], 'ptbtags': ['PRP', 'MD', 'RB', 'VB', 'RB', 'RB', '-LRB-', 'CC', 'RB', 'VBP', 'DT', 'JJ', 'NNS', 'IN', 'DT', 'NN', 'IN', 'NN', 'PRP', 'RB', 'VBD', 'IN', 'PRP$', 'NN', '-RRB-', '.']}\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[-1]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also view the text and tags separately:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['i', 'will', 'never', 'return', 'there', 'again', '(', 'and', 'now', 'have', 'some', 'serious', 'doubts', 'about', 'the', 'quality', 'of', 'work', 'they', 'actually', 'performed', 'on', 'my', 'car', ')', '.']\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[-1])['text'])" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['PRON', 'AUX', 'ADV', 'VERB', 'ADV', 'ADV', 'PUNCT', 'CCONJ', 'ADV', 'VERB', 'DET', 'ADJ', 'NOUN', 'ADP', 'DET', 'NOUN', 'ADP', 'NOUN', 'PRON', 'ADV', 'VERB', 'ADP', 'PRON', 'NOUN', 'PUNCT', 'PUNCT']\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[-1])['udtags'])" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['PRP', 'MD', 'RB', 'VB', 'RB', 'RB', '-LRB-', 'CC', 'RB', 'VBP', 'DT', 'JJ', 'NNS', 'IN', 'DT', 'NN', 'IN', 'NN', 'PRP', 'RB', 'VBD', 'IN', 'PRP$', 'NN', '-RRB-', '.']\n" - ] - } - ], - "source": [ - "print(vars(train_data.examples[-1])['ptbtags'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we'll build the vocabulary - a mapping of tokens to integers. \n", - "\n", - "We want some unknown tokens within our dataset in order to replicate how this model would be used in real life, so we set the `min_freq` to 2 which means only tokens that appear twice in the training set will be added to the vocabulary and the rest will be replaced by `` tokens.\n", - "\n", - "We also load the [GloVe](https://nlp.stanford.edu/projects/glove/) pre-trained token embeddings. Specifically, the 100-dimensional embeddings that have been trained on 6 billion tokens. Using pre-trained embeddings usually leads to improved performance - although admittedly the dataset used in this tutorial is too small to take advantage of the pre-trained embeddings. \n", - "\n", - "`unk_init` is used to initialize the token embeddings which are not in the pre-trained embedding vocabulary. By default this sets those embeddings to zeros, however it is better to not have them all initialized to the same value, so we initialize them from a Normal/Gaussian distribution.\n", - "\n", - "These pre-trained vectors are now loaded into our vocabulary and we will initialize our model with these values later." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|█████████▉| 398413/400000 [00:16<00:00, 24234.96it/s]" - ] - } - ], - "source": [ - "MIN_FREQ = 2\n", - "\n", - "TEXT.build_vocab(train_data, \n", - " min_freq = MIN_FREQ,\n", - " vectors = \"glove.6B.100d\",\n", - " unk_init = torch.Tensor.normal_)\n", - "\n", - "\n", - "UD_TAGS.build_vocab(train_data)\n", - "PTB_TAGS.build_vocab(train_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can check how many tokens and tags are in our vocabulary by getting their length:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unique tokens in TEXT vocabulary: 8866\n", - "Unique tokens in UD_TAG vocabulary: 18\n", - "Unique tokens in PTB_TAG vocabulary: 51\n" - ] - } - ], - "source": [ - "print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\n", - "print(f\"Unique tokens in UD_TAG vocabulary: {len(UD_TAGS.vocab)}\")\n", - "print(f\"Unique tokens in PTB_TAG vocabulary: {len(PTB_TAGS.vocab)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Exploring the vocabulary, we can check the most common tokens within our texts:" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[('the', 9076), ('.', 8640), (',', 7021), ('to', 5137), ('and', 5002), ('a', 3782), ('of', 3622), ('i', 3379), ('in', 3112), ('is', 2239), ('you', 2156), ('that', 2036), ('it', 1850), ('for', 1842), ('-', 1426), ('have', 1359), ('\"', 1296), ('on', 1273), ('was', 1244), ('with', 1216)]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|█████████▉| 398413/400000 [00:30<00:00, 24234.96it/s]" - ] - } - ], - "source": [ - "print(TEXT.vocab.freqs.most_common(20))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the vocabularies for both of our tags:" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['', 'NOUN', 'PUNCT', 'VERB', 'PRON', 'ADP', 'DET', 'PROPN', 'ADJ', 'AUX', 'ADV', 'CCONJ', 'PART', 'NUM', 'SCONJ', 'X', 'INTJ', 'SYM']\n" - ] - } - ], - "source": [ - "print(UD_TAGS.vocab.itos)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['', 'NN', 'IN', 'DT', 'NNP', 'PRP', 'JJ', 'RB', '.', 'VB', 'NNS', ',', 'CC', 'VBD', 'VBP', 'VBZ', 'CD', 'VBN', 'VBG', 'MD', 'TO', 'PRP$', '-RRB-', '-LRB-', 'WDT', 'WRB', ':', '``', \"''\", 'WP', 'RP', 'UH', 'POS', 'HYPH', 'JJR', 'NNPS', 'JJS', 'EX', 'NFP', 'GW', 'ADD', 'RBR', '$', 'PDT', 'RBS', 'SYM', 'LS', 'FW', 'AFX', 'WP$', 'XX']\n" - ] - } - ], - "source": [ - "print(PTB_TAGS.vocab.itos)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also see how many of each tag are in our vocabulary:" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[('NOUN', 34781), ('PUNCT', 23679), ('VERB', 23081), ('PRON', 18577), ('ADP', 17638), ('DET', 16285), ('PROPN', 12946), ('ADJ', 12477), ('AUX', 12343), ('ADV', 10548), ('CCONJ', 6707), ('PART', 5567), ('NUM', 3999), ('SCONJ', 3843), ('X', 847), ('INTJ', 688), ('SYM', 599)]\n" - ] - } - ], - "source": [ - "print(UD_TAGS.vocab.freqs.most_common())" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[('NN', 26915), ('IN', 20724), ('DT', 16817), ('NNP', 12449), ('PRP', 12193), ('JJ', 11591), ('RB', 10831), ('.', 10317), ('VB', 9476), ('NNS', 8438), (',', 8062), ('CC', 6706), ('VBD', 5402), ('VBP', 5374), ('VBZ', 4578), ('CD', 3998), ('VBN', 3967), ('VBG', 3330), ('MD', 3294), ('TO', 3286), ('PRP$', 3068), ('-RRB-', 1008), ('-LRB-', 973), ('WDT', 948), ('WRB', 869), (':', 866), ('``', 813), (\"''\", 785), ('WP', 760), ('RP', 755), ('UH', 689), ('POS', 684), ('HYPH', 664), ('JJR', 503), ('NNPS', 498), ('JJS', 383), ('EX', 359), ('NFP', 338), ('GW', 294), ('ADD', 292), ('RBR', 276), ('$', 258), ('PDT', 175), ('RBS', 169), ('SYM', 156), ('LS', 117), ('FW', 93), ('AFX', 48), ('WP$', 15), ('XX', 1)]\n" - ] - } - ], - "source": [ - "print(PTB_TAGS.vocab.freqs.most_common())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also view how common each of the tags are within the training set:" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "def tag_percentage(tag_counts):\n", - " \n", - " total_count = sum([count for tag, count in tag_counts])\n", - " \n", - " tag_counts_percentages = [(tag, count, count/total_count) for tag, count in tag_counts]\n", - " \n", - " return tag_counts_percentages" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tag\t\tCount\t\tPercentage\n", - "\n", - "NOUN\t\t34781\t\t17.0%\n", - "PUNCT\t\t23679\t\t11.6%\n", - "VERB\t\t23081\t\t11.3%\n", - "PRON\t\t18577\t\t 9.1%\n", - "ADP\t\t17638\t\t 8.6%\n", - "DET\t\t16285\t\t 8.0%\n", - "PROPN\t\t12946\t\t 6.3%\n", - "ADJ\t\t12477\t\t 6.1%\n", - "AUX\t\t12343\t\t 6.0%\n", - "ADV\t\t10548\t\t 5.2%\n", - "CCONJ\t\t6707\t\t 3.3%\n", - "PART\t\t5567\t\t 2.7%\n", - "NUM\t\t3999\t\t 2.0%\n", - "SCONJ\t\t3843\t\t 1.9%\n", - "X\t\t847\t\t 0.4%\n", - "INTJ\t\t688\t\t 0.3%\n", - "SYM\t\t599\t\t 0.3%\n" - ] - } - ], - "source": [ - "print(\"Tag\\t\\tCount\\t\\tPercentage\\n\")\n", - "\n", - "for tag, count, percent in tag_percentage(UD_TAGS.vocab.freqs.most_common()):\n", - " print(f\"{tag}\\t\\t{count}\\t\\t{percent*100:4.1f}%\")" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tag\t\tCount\t\tPercentage\n", - "\n", - "NN\t\t26915\t\t13.2%\n", - "IN\t\t20724\t\t10.1%\n", - "DT\t\t16817\t\t 8.2%\n", - "NNP\t\t12449\t\t 6.1%\n", - "PRP\t\t12193\t\t 6.0%\n", - "JJ\t\t11591\t\t 5.7%\n", - "RB\t\t10831\t\t 5.3%\n", - ".\t\t10317\t\t 5.0%\n", - "VB\t\t9476\t\t 4.6%\n", - "NNS\t\t8438\t\t 4.1%\n", - ",\t\t8062\t\t 3.9%\n", - "CC\t\t6706\t\t 3.3%\n", - "VBD\t\t5402\t\t 2.6%\n", - "VBP\t\t5374\t\t 2.6%\n", - "VBZ\t\t4578\t\t 2.2%\n", - "CD\t\t3998\t\t 2.0%\n", - "VBN\t\t3967\t\t 1.9%\n", - "VBG\t\t3330\t\t 1.6%\n", - "MD\t\t3294\t\t 1.6%\n", - "TO\t\t3286\t\t 1.6%\n", - "PRP$\t\t3068\t\t 1.5%\n", - "-RRB-\t\t1008\t\t 0.5%\n", - "-LRB-\t\t973\t\t 0.5%\n", - "WDT\t\t948\t\t 0.5%\n", - "WRB\t\t869\t\t 0.4%\n", - ":\t\t866\t\t 0.4%\n", - "``\t\t813\t\t 0.4%\n", - "''\t\t785\t\t 0.4%\n", - "WP\t\t760\t\t 0.4%\n", - "RP\t\t755\t\t 0.4%\n", - "UH\t\t689\t\t 0.3%\n", - "POS\t\t684\t\t 0.3%\n", - "HYPH\t\t664\t\t 0.3%\n", - "JJR\t\t503\t\t 0.2%\n", - "NNPS\t\t498\t\t 0.2%\n", - "JJS\t\t383\t\t 0.2%\n", - "EX\t\t359\t\t 0.2%\n", - "NFP\t\t338\t\t 0.2%\n", - "GW\t\t294\t\t 0.1%\n", - "ADD\t\t292\t\t 0.1%\n", - "RBR\t\t276\t\t 0.1%\n", - "$\t\t258\t\t 0.1%\n", - "PDT\t\t175\t\t 0.1%\n", - "RBS\t\t169\t\t 0.1%\n", - "SYM\t\t156\t\t 0.1%\n", - "LS\t\t117\t\t 0.1%\n", - "FW\t\t93\t\t 0.0%\n", - "AFX\t\t48\t\t 0.0%\n", - "WP$\t\t15\t\t 0.0%\n", - "XX\t\t1\t\t 0.0%\n" - ] - } - ], - "source": [ - "print(\"Tag\\t\\tCount\\t\\tPercentage\\n\")\n", - "\n", - "for tag, count, percent in tag_percentage(PTB_TAGS.vocab.freqs.most_common()):\n", - " print(f\"{tag}\\t\\t{count}\\t\\t{percent*100:4.1f}%\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The final part of data preparation is handling the iterator. \n", - "\n", - "This will be iterated over to return batches of data to process. Here, we set the batch size and the `device` - which is used to place the batches of tensors on our GPU, if we have one. " - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "BATCH_SIZE = 128\n", - "\n", - "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n", - "\n", - "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE,\n", - " device = device)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Building the Model\n", - "\n", - "Next up, we define our model - a multi-layer bi-directional LSTM. The image below shows a simplified version of the model with only one LSTM layer and omitting the LSTM's cell state for clarity.\n", - "\n", - "![](assets/pos-bidirectional-lstm.png)\n", - "\n", - "The model takes in a sequence of tokens, $X = \\{x_1, x_2,...,x_T\\}$, passes them through an embedding layer, $e$, to get the token embeddings, $e(X) = \\{e(x_1), e(x_2), ..., e(x_T)\\}$.\n", - "\n", - "These embeddings are processed - one per time-step - by the forward and backward LSTMs. The forward LSTM processes the sequence from left-to-right, whilst the backward LSTM processes the sequence right-to-left, i.e. the first input to the forward LSTM is $x_1$ and the first input to the backward LSTM is $x_T$. \n", - "\n", - "The LSTMs also take in the the hidden, $h$, and cell, $c$, states from the previous time-step\n", - "\n", - "$$h^{\\rightarrow}_t = \\text{LSTM}^{\\rightarrow}(e(x^{\\rightarrow}_t), h^{\\rightarrow}_{t-1}, c^{\\rightarrow}_{t-1})$$\n", - "$$h^{\\leftarrow}_t=\\text{LSTM}^{\\leftarrow}(e(x^{\\leftarrow}_t), h^{\\leftarrow}_{t-1}, c^{\\leftarrow}_{t-1})$$\n", - "\n", - "After the whole sequence has been processed, the hidden and cell states are then passed to the next layer of the LSTM.\n", - "\n", - "The initial hidden and cell states, $h_0$ and $c_0$, for each direction and layer are initialized to a tensor full of zeros.\n", - "\n", - "We then concatenate both the forward and backward hidden states from the final layer of the LSTM, $H = \\{h_1, h_2, ... h_T\\}$, where $h_1 = [h^{\\rightarrow}_1;h^{\\leftarrow}_T]$, $h_2 = [h^{\\rightarrow}_2;h^{\\leftarrow}_{T-1}]$, etc. and pass them through a linear layer, $f$, which is used to make the prediction of which tag applies to this token, $\\hat{y}_t = f(h_t)$.\n", - "\n", - "When training the model, we will compare our predicted tags, $\\hat{Y}$ against the actual tags, $Y$, to calculate a loss, the gradients w.r.t. that loss, and then update our parameters.\n", - "\n", - "We implement the model detailed above in the `BiLSTMPOSTagger` class.\n", - "\n", - "`nn.Embedding` is an embedding layer and the input dimension should be the size of the input (text) vocabulary. We tell it what the index of the padding token is so it does not update the padding token's embedding entry.\n", - "\n", - "`nn.LSTM` is the LSTM. We apply dropout as regularization between the layers, if we are using more than one.\n", - "\n", - "`nn.Linear` defines the linear layer to make predictions using the LSTM outputs. We double the size of the input if we are using a bi-directional LSTM. The output dimensions should be the size of the tag vocabulary.\n", - "\n", - "We also define a dropout layer with `nn.Dropout`, which we use in the `forward` method to apply dropout to the embeddings and the outputs of the final layer of the LSTM." - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [], - "source": [ - "example_batch = next(iter(train_iterator))" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[1355, 50, 9, ..., 11, 368, 2],\n", - " [ 4, 23, 660, ..., 14, 107, 781],\n", - " [ 69, 79, 9, ..., 844, 48, 20],\n", - " ...,\n", - " [ 1, 1, 1, ..., 1, 1, 1],\n", - " [ 1, 1, 1, ..., 1, 1, 1],\n", - " [ 1, 1, 1, ..., 1, 1, 1]], device='cuda:0')" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "example_batch.text" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [], - "source": [ - "class BiLSTMPOSTagger(nn.Module):\n", - " def __init__(self, \n", - " input_dim, \n", - " embedding_dim, \n", - " hidden_dim, \n", - " output_dim, \n", - " n_layers, \n", - " bidirectional, \n", - " dropout, \n", - " pad_idx):\n", - " \n", - " super().__init__()\n", - " \n", - " self.embedding = nn.Embedding(input_dim, embedding_dim, padding_idx = pad_idx)\n", - " \n", - " self.lstm = nn.LSTM(embedding_dim, \n", - " hidden_dim, \n", - " num_layers = n_layers, \n", - " bidirectional = bidirectional,\n", - " dropout = dropout if n_layers > 1 else 0)\n", - " \n", - " self.fc = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)\n", - " \n", - " self.dropout = nn.Dropout(dropout)\n", - " \n", - " def forward(self, text):\n", - "\n", - " #text = [sent len, batch size]\n", - " \n", - " #pass text through embedding layer\n", - " embedded = self.dropout(self.embedding(text))\n", - " \n", - " #embedded = [sent len, batch size, emb dim]\n", - " \n", - " #pass embeddings into LSTM\n", - " outputs, (hidden, cell) = self.lstm(embedded)\n", - " \n", - " #outputs holds the backward and forward hidden states in the final layer\n", - " #hidden and cell are the backward and forward hidden and cell states at the final time-step\n", - " \n", - " #output = [sent len, batch size, hid dim * n directions]\n", - " #hidden/cell = [n layers * n directions, batch size, hid dim]\n", - " \n", - " #we use our outputs to make a prediction of what the tag should be\n", - " predictions = self.fc(self.dropout(outputs))\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " \n", - " return predictions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training the Model\n", - "\n", - "Next, we instantiate the model. We need to ensure the embedding dimensions matches that of the GloVe embeddings we loaded earlier.\n", - "\n", - "The rest of the hyperparmeters have been chosen as sensible defaults, though there may be a combination that performs better on this model and dataset.\n", - "\n", - "The input and output dimensions are taken directly from the lengths of the respective vocabularies. The padding index is obtained using the vocabulary and the `Field` of the text." - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(TEXT.vocab)\n", - "EMBEDDING_DIM = 100\n", - "HIDDEN_DIM = 128\n", - "OUTPUT_DIM = len(UD_TAGS.vocab)\n", - "N_LAYERS = 2\n", - "BIDIRECTIONAL = True\n", - "DROPOUT = 0.25\n", - "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n", - "\n", - "model = BiLSTMPOSTagger(INPUT_DIM, \n", - " EMBEDDING_DIM, \n", - " HIDDEN_DIM, \n", - " OUTPUT_DIM, \n", - " N_LAYERS, \n", - " BIDIRECTIONAL, \n", - " DROPOUT, \n", - " PAD_IDX)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We initialize the weights from a simple Normal distribution. Again, there may be a better initialization scheme for this model and dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "BiLSTMPOSTagger(\n", - " (embedding): Embedding(8866, 100, padding_idx=1)\n", - " (lstm): LSTM(100, 128, num_layers=2, dropout=0.25, bidirectional=True)\n", - " (fc): Linear(in_features=256, out_features=18, bias=True)\n", - " (dropout): Dropout(p=0.25, inplace=False)\n", - ")" - ] - }, - "execution_count": 72, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def init_weights(m):\n", - " for name, param in m.named_parameters():\n", - " nn.init.normal_(param.data, mean = 0, std = 0.1)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, a small function to tell us how many parameters are in our model. Useful for comparing different models." - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The model has 1,522,010 trainable parameters\n" - ] - } - ], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll now initialize our model's embedding layer with the pre-trained embedding values we loaded earlier.\n", - "\n", - "This is done by getting them from the vocab's `.vectors` attribute and then performing a `.copy` to overwrite the embedding layer's current weights." - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([8866, 100])\n" - ] - } - ], - "source": [ - "pretrained_embeddings = TEXT.vocab.vectors\n", - "\n", - "print(pretrained_embeddings.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [], - "source": [ - "a = next(model.embedding.parameters())" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([8866, 100])" - ] - }, - "execution_count": 76, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[-0.1117, -0.4966, 0.1631, ..., 1.2647, -0.2753, -0.1325],\n", - " [-0.8555, -0.7208, 1.3755, ..., 0.0825, -1.1314, 0.3997],\n", - " [-0.0382, -0.2449, 0.7281, ..., -0.1459, 0.8278, 0.2706],\n", - " ...,\n", - " [ 0.9261, 2.3049, 0.5502, ..., -0.3492, -0.5298, -0.1577],\n", - " [-0.5972, 0.0471, -0.2406, ..., -0.9446, -0.1126, -0.2260],\n", - " [-0.4809, 2.5629, 0.9530, ..., 0.5278, -0.4588, 0.7294]])" - ] - }, - "execution_count": 77, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.embedding.weight.data.copy_(pretrained_embeddings)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's common to initialize the embedding of the pad token to all zeros. This, along with setting the `padding_idx` in the model's embedding layer, means that the embedding should always output a tensor full of zeros when a pad token is input." - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([[-0.1117, -0.4966, 0.1631, ..., 1.2647, -0.2753, -0.1325],\n", - " [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0382, -0.2449, 0.7281, ..., -0.1459, 0.8278, 0.2706],\n", - " ...,\n", - " [ 0.9261, 2.3049, 0.5502, ..., -0.3492, -0.5298, -0.1577],\n", - " [-0.5972, 0.0471, -0.2406, ..., -0.9446, -0.1126, -0.2260],\n", - " [-0.4809, 2.5629, 0.9530, ..., 0.5278, -0.4588, 0.7294]])\n" - ] - } - ], - "source": [ - "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n", - "\n", - "print(model.embedding.weight.data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then define our optimizer, used to update our parameters w.r.t. their gradients. We use Adam with the default learning rate." - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = optim.Adam(model.parameters())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we define our loss function, cross-entropy loss.\n", - "\n", - "Even though we have no `` tokens within our tag vocab, we still have `` tokens. This is because all sentences within a batch need to be the same size. However, we don't want to calculate the loss when the target is a `` token as we aren't training our model to recognize padding tokens.\n", - "\n", - "We handle this by setting the `ignore_index` in our loss function to the index of the padding token in our tag vocabulary." - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [], - "source": [ - "TAG_PAD_IDX = UD_TAGS.vocab.stoi[UD_TAGS.pad_token]\n", - "\n", - "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then place our model and loss function on our GPU, if we have one." - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [], - "source": [ - "model = model.to(device)\n", - "criterion = criterion.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [], - "source": [ - "model.embedding.weight.requires_grad = True" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 85, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.embedding.weight.requires_grad" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will be using the loss value between our predicted and actual tags to train the network, but ideally we'd like a more interpretable way to see how well our model is doing - accuracy.\n", - "\n", - "The issue is that we don't want to calculate accuracy over the `` tokens as we aren't interested in predicting them.\n", - "\n", - "The function below only calculates accuracy over non-padded tokens. `non_pad_elements` is a tensor containing the indices of the non-pad tokens within an input batch. We then compare the predictions of those elements with the labels to get a count of how many predictions were correct. We then divide this by the number of non-pad elements to get our accuracy value over the batch." - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [], - "source": [ - "def categorical_accuracy(preds, y, tag_pad_idx):\n", - " \"\"\"\n", - " Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n", - " \"\"\"\n", - " max_preds = preds.argmax(dim = 1, keepdim = True) # get the index of the max probability\n", - " non_pad_elements = (y != tag_pad_idx).nonzero()\n", - " correct = max_preds[non_pad_elements].squeeze(1).eq(y[non_pad_elements])\n", - " return correct.sum() / torch.FloatTensor([y[non_pad_elements].shape[0]])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next is the function that handles training our model.\n", - "\n", - "We first set the model to `train` mode to turn on dropout/batch-norm/etc. (if used). Then we iterate over our iterator, which returns a batch of examples. \n", - "\n", - "For each batch: \n", - "- we zero the gradients over the parameters from the last gradient calculation\n", - "- insert the batch of text into the model to get predictions\n", - "- as PyTorch loss functions cannot handle 3-dimensional predictions we reshape our predictions\n", - "- calculate the loss and accuracy between the predicted tags and actual tags\n", - "- call `backward` to calculate the gradients of the parameters w.r.t. the loss\n", - "- take an optimizer `step` to update the parameters\n", - "- add to the running total of loss and accuracy" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.train()\n", - " \n", - " for batch in iterator:\n", - " \n", - " text = batch.text\n", - " tags = batch.udtags\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " #text = [sent len, batch size]\n", - " \n", - " predictions = model(text)\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " #tags = [sent len, batch size]\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " #predictions = [sent len * batch size, output dim]\n", - " #tags = [sent len * batch size]\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - " \n", - " loss.backward()\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `evaluate` function is similar to the `train` function, except with changes made so we don't update the model's parameters.\n", - "\n", - "`model.eval()` is used to put the model in evaluation mode, so dropout/batch-norm/etc. are turned off. \n", - "\n", - "The iteration loop is also wrapped in `torch.no_grad` to ensure we don't calculate any gradients. We also don't need to call `optimizer.zero_grad()` and `optimizer.step()`." - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": {}, - "outputs": [], - "source": [ - "def evaluate(model, iterator, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.eval()\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for batch in iterator:\n", - "\n", - " text = batch.text\n", - " tags = batch.udtags\n", - " \n", - " predictions = model(text)\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - "\n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we have a small function that tells us how long an epoch takes." - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [], - "source": [ - "def epoch_time(start_time, end_time):\n", - " elapsed_time = end_time - start_time\n", - " elapsed_mins = int(elapsed_time / 60)\n", - " elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n", - " return elapsed_mins, elapsed_secs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, we train our model!\n", - "\n", - "After each epoch we check if our model has achieved the best validation loss so far. If it has then we save the parameters of this model and we will use these \"best\" parameters to calculate performance over our test set." - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 01 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 1.343 | Train Acc: 58.15%\n", - "\t Val. Loss: 0.684 | Val. Acc: 78.59%\n", - "Epoch: 02 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.477 | Train Acc: 85.02%\n", - "\t Val. Loss: 0.499 | Val. Acc: 83.89%\n", - "Epoch: 03 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.347 | Train Acc: 89.12%\n", - "\t Val. Loss: 0.446 | Val. Acc: 85.16%\n", - "Epoch: 04 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.287 | Train Acc: 90.97%\n", - "\t Val. Loss: 0.406 | Val. Acc: 86.60%\n", - "Epoch: 05 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.250 | Train Acc: 92.03%\n", - "\t Val. Loss: 0.397 | Val. Acc: 86.90%\n", - "Epoch: 06 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.223 | Train Acc: 92.95%\n", - "\t Val. Loss: 0.384 | Val. Acc: 87.23%\n", - "Epoch: 07 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.203 | Train Acc: 93.52%\n", - "\t Val. Loss: 0.366 | Val. Acc: 87.35%\n", - "Epoch: 08 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.189 | Train Acc: 93.96%\n", - "\t Val. Loss: 0.360 | Val. Acc: 87.65%\n", - "Epoch: 09 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.175 | Train Acc: 94.35%\n", - "\t Val. Loss: 0.356 | Val. Acc: 87.71%\n", - "Epoch: 10 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.165 | Train Acc: 94.64%\n", - "\t Val. Loss: 0.366 | Val. Acc: 87.55%\n", - "Epoch: 11 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.154 | Train Acc: 95.05%\n", - "\t Val. Loss: 0.367 | Val. Acc: 87.82%\n", - "Epoch: 12 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.146 | Train Acc: 95.27%\n", - "\t Val. Loss: 0.349 | Val. Acc: 88.14%\n", - "Epoch: 13 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.139 | Train Acc: 95.43%\n", - "\t Val. Loss: 0.340 | Val. Acc: 88.35%\n", - "Epoch: 14 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.130 | Train Acc: 95.76%\n", - "\t Val. Loss: 0.338 | Val. Acc: 88.56%\n", - "Epoch: 15 | Epoch Time: 0m 2s\n", - "\tTrain Loss: 0.124 | Train Acc: 95.91%\n", - "\t Val. Loss: 0.336 | Val. Acc: 88.58%\n" - ] - } - ], - "source": [ - "N_EPOCHS = 15\n", - "\n", - "best_valid_loss = float('inf')\n", - "\n", - "for epoch in range(N_EPOCHS):\n", - "\n", - " start_time = time.time()\n", - " \n", - " train_loss, train_acc = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n", - " valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)\n", - " \n", - " end_time = time.time()\n", - "\n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut1-model.pt')\n", - " \n", - " print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}%')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then load our \"best\" parameters and evaluate performance on the test set." - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Loss: 0.349 | Test Acc: 88.66%\n" - ] - } - ], - "source": [ - "model.load_state_dict(torch.load('tut1-model.pt'))\n", - "\n", - "test_loss, test_acc = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n", - "\n", - "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inference\n", - "\n", - "88% accuracy looks pretty good, but let's see our model tag some actual sentences.\n", - "\n", - "We define a `tag_sentence` function that will:\n", - "- put the model into evaluation mode\n", - "- tokenize the sentence with spaCy if it is not a list\n", - "- lowercase the tokens if the `Field` did\n", - "- numericalize the tokens using the vocabulary\n", - "- find out which tokens are not in the vocabulary, i.e. are `` tokens\n", - "- convert the numericalized tokens into a tensor and add a batch dimension\n", - "- feed the tensor into the model\n", - "- get the predictions over the sentence\n", - "- convert the predictions into readable tags\n", - "\n", - "As well as returning the tokens and tags, it also returns which tokens were `` tokens." - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def tag_sentence(model, device, sentence, text_field, tag_field):\n", - " \n", - " model.eval()\n", - " \n", - " if isinstance(sentence, str):\n", - " nlp = spacy.load('en')\n", - " tokens = [token.text for token in nlp(sentence)]\n", - " else:\n", - " tokens = [token for token in sentence]\n", - "\n", - " if text_field.lower:\n", - " tokens = [t.lower() for t in tokens]\n", - " \n", - " numericalized_tokens = [text_field.vocab.stoi[t] for t in tokens]\n", - "\n", - " unk_idx = text_field.vocab.stoi[text_field.unk_token]\n", - " \n", - " unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n", - " \n", - " token_tensor = torch.LongTensor(numericalized_tokens)\n", - " \n", - " token_tensor = token_tensor.unsqueeze(-1).to(device)\n", - " \n", - " predictions = model(token_tensor)\n", - " \n", - " top_predictions = predictions.argmax(-1)\n", - " \n", - " predicted_tags = [tag_field.vocab.itos[t.item()] for t in top_predictions]\n", - " \n", - " return tokens, predicted_tags, unks" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll get an already tokenized example from the training set and test our model's performance." - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['[', 'this', 'killing', 'of', 'a', 'respected', 'cleric', 'will', 'be', 'causing', 'us', 'trouble', 'for', 'years', 'to', 'come', '.', ']']\n" - ] - } - ], - "source": [ - "example_index = 1\n", - "\n", - "sentence = vars(train_data.examples[example_index])['text']\n", - "actual_tags = vars(train_data.examples[example_index])['udtags']\n", - "\n", - "print(sentence)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then use our `tag_sentence` function to get the tags. Notice how the tokens referring to subject of the sentence, the \"respected cleric\", are both `` tokens!" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['respected', 'cleric']\n" - ] - } - ], - "source": [ - "tokens, pred_tags, unks = tag_sentence(model, \n", - " device, \n", - " sentence, \n", - " TEXT, \n", - " UD_TAGS)\n", - "\n", - "print(unks)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then check how well it did. Surprisingly, it got every token correct, including the two that were unknown tokens!" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pred. Tag\tActual Tag\tCorrect?\tToken\n", - "\n", - "PUNCT\t\tPUNCT\t\t✔\t\tthe\n", - "DET\t\tDET\t\t✔\t\tqueen\n", - "NOUN\t\tNOUN\t\t✔\t\twill\n", - "ADP\t\tADP\t\t✔\t\tdeliver\n", - "DET\t\tDET\t\t✔\t\ta\n", - "ADJ\t\tADJ\t\t✔\t\tspeech\n", - "NOUN\t\tNOUN\t\t✔\t\tabout\n", - "AUX\t\tAUX\t\t✔\t\tthe\n", - "AUX\t\tAUX\t\t✔\t\tconflict\n", - "VERB\t\tVERB\t\t✔\t\tin\n", - "PRON\t\tPRON\t\t✔\t\tnorth\n", - "NOUN\t\tNOUN\t\t✔\t\tkorea\n", - "ADP\t\tADP\t\t✔\t\tat\n", - "NOUN\t\tNOUN\t\t✔\t\t1\n", - "PART\t\tPART\t\t✔\t\tpm\n", - "VERB\t\tVERB\t\t✔\t\ttomorrow\n", - "PUNCT\t\tPUNCT\t\t✔\t\t.\n" - ] - } - ], - "source": [ - "print(\"Pred. Tag\\tActual Tag\\tCorrect?\\tToken\\n\")\n", - "\n", - "for token, pred_tag, actual_tag in zip(tokens, pred_tags, actual_tags):\n", - " correct = '✔' if pred_tag == actual_tag else '✘'\n", - " print(f\"{pred_tag}\\t\\t{actual_tag}\\t\\t{correct}\\t\\t{token}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's now make up our own sentence and see how well the model does.\n", - "\n", - "Our example sentence below has every token within the model's vocabulary." - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[]\n" - ] - } - ], - "source": [ - "sentence = 'The Queen will deliver a speech about the conflict in North Korea at 1pm tomorrow.'\n", - "\n", - "tokens, tags, unks = tag_sentence(model, \n", - " device, \n", - " sentence, \n", - " TEXT, \n", - " UD_TAGS)\n", - "\n", - "print(unks)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Looking at the sentence it seems like it gave sensible tags to every token!" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pred. Tag\tToken\n", - "\n", - "DET\t\tthe\n", - "NOUN\t\tqueen\n", - "AUX\t\twill\n", - "VERB\t\tdeliver\n", - "DET\t\ta\n", - "NOUN\t\tspeech\n", - "ADP\t\tabout\n", - "DET\t\tthe\n", - "NOUN\t\tconflict\n", - "ADP\t\tin\n", - "PROPN\t\tnorth\n", - "PROPN\t\tkorea\n", - "ADP\t\tat\n", - "NUM\t\t1\n", - "NOUN\t\tpm\n", - "NOUN\t\ttomorrow\n", - "PUNCT\t\t.\n" - ] - } - ], - "source": [ - "print(\"Pred. Tag\\tToken\\n\")\n", - "\n", - "for token, tag in zip(tokens, tags):\n", - " print(f\"{tag}\\t\\t{token}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We've now seen how to implement PoS tagging with PyTorch and TorchText! \n", - "\n", - "The BiLSTM isn't a state-of-the-art model, in terms of performance, but is a strong baseline for PoS tasks and is a good tool to have in your arsenal." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Going deeper\n", - "What if we could combine word-level and char-level approaches? \n", - "![title](https://i.postimg.cc/tT9hsBfj/ive-put-an-rnn-in-your-rnn-so-you-can-train-an-rnn-on-every-step-of-your-rnn-training-loop.jpg)\n", - "\n", - "\n", - "Actually, we can. Let's use LSTM or GRU to generate embedding for every word on char-level.\n", - "![title](https://guillaumegenthial.github.io/assets/char_representation.png)\n", - "*Image source: https://guillaumegenthial.github.io/sequence-tagging-with-tensorflow.html*\n", - "\n", - "![title](https://guillaumegenthial.github.io/assets/bi-lstm.png)\n", - "*Image source: https://guillaumegenthial.github.io/sequence-tagging-with-tensorflow.html*" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To do that we need to make few adjustments to the code above" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'udtag': , 'ptbtag': , 'word': , 'char': }\n", - "12543\n", - "{'word': ['i', 'will', 'never', 'return', 'there', 'again', '(', 'and', 'now', 'have', 'some', 'serious', 'doubts', 'about', 'the', 'quality', 'of', 'work', 'they', 'actually', 'performed', 'on', 'my', 'car', ')', '.'], 'char': [['I'], ['w', 'i', 'l', 'l'], ['n', 'e', 'v', 'e', 'r'], ['r', 'e', 't', 'u', 'r', 'n'], ['t', 'h', 'e', 'r', 'e'], ['a', 'g', 'a', 'i', 'n'], ['('], ['a', 'n', 'd'], ['n', 'o', 'w'], ['h', 'a', 'v', 'e'], ['s', 'o', 'm', 'e'], ['s', 'e', 'r', 'i', 'o', 'u', 's'], ['d', 'o', 'u', 'b', 't', 's'], ['a', 'b', 'o', 'u', 't'], ['t', 'h', 'e'], ['q', 'u', 'a', 'l', 'i', 't', 'y'], ['o', 'f'], ['w', 'o', 'r', 'k'], ['t', 'h', 'e', 'y'], ['a', 'c', 't', 'u', 'a', 'l', 'l', 'y'], ['p', 'e', 'r', 'f', 'o', 'r', 'm', 'e', 'd'], ['o', 'n'], ['m', 'y'], ['c', 'a', 'r'], [')'], ['.']], 'udtag': ['PRON', 'AUX', 'ADV', 'VERB', 'ADV', 'ADV', 'PUNCT', 'CCONJ', 'ADV', 'VERB', 'DET', 'ADJ', 'NOUN', 'ADP', 'DET', 'NOUN', 'ADP', 'NOUN', 'PRON', 'ADV', 'VERB', 'ADP', 'PRON', 'NOUN', 'PUNCT', 'PUNCT'], 'ptbtag': ['PRP', 'MD', 'RB', 'VB', 'RB', 'RB', '-LRB-', 'CC', 'RB', 'VBP', 'DT', 'JJ', 'NNS', 'IN', 'DT', 'NN', 'IN', 'NN', 'PRP', 'RB', 'VBD', 'IN', 'PRP$', 'NN', '-RRB-', '.']}\n" - ] - } - ], - "source": [ - "# Now lets try both word and character embeddings\n", - "WORD = data.Field(lower = True)\n", - "UD_TAG = data.Field(unk_token = None)\n", - "PTB_TAG = data.Field(unk_token = None)\n", - "\n", - "# We'll use NestedField to tokenize each word into list of chars\n", - "CHAR_NESTING = data.Field(tokenize=list, init_token=\"\", eos_token=\"\")\n", - "CHAR = data.NestedField(CHAR_NESTING)#, init_token=\"\", eos_token=\"\")\n", - "\n", - "fields = [(('word', 'char'), (WORD, CHAR)), ('udtag', UD_TAG), ('ptbtag', PTB_TAG)]\n", - "train_data, valid_data, test_data = datasets.UDPOS.splits(fields)\n", - "# train, val, test = datasets.UDPOS.splits(fields=fields)\n", - "\n", - "print(train_data.fields)\n", - "print(len(train_data))\n", - "print(vars(train_data[-1]))" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "WORD.build_vocab(\n", - " train_data,\n", - " min_freq = MIN_FREQ,\n", - " vectors=\"glove.6B.100d\",\n", - " unk_init = torch.Tensor.normal_\n", - ")\n", - "\n", - "\n", - "CHAR.build_vocab(train_data)\n", - "UD_TAG.build_vocab(train_data)\n", - "PTB_TAG.build_vocab(train_data)" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unique tokens in WORD vocabulary: 8866\n", - "Unique tokens in CHAR vocabulary: 112\n", - "Unique tokens in UD_TAG vocabulary: 18\n", - "Unique tokens in PTB_TAG vocabulary: 51\n" - ] - } - ], - "source": [ - "print(f\"Unique tokens in WORD vocabulary: {len(WORD.vocab)}\")\n", - "print(f\"Unique tokens in CHAR vocabulary: {len(CHAR.vocab)}\")\n", - "print(f\"Unique tokens in UD_TAG vocabulary: {len(UD_TAG.vocab)}\")\n", - "print(f\"Unique tokens in PTB_TAG vocabulary: {len(PTB_TAG.vocab)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [], - "source": [ - "BATCH_SIZE = 64\n", - "\n", - "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n", - "\n", - "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n", - " (train_data, valid_data, test_data), \n", - " batch_size = BATCH_SIZE,\n", - " device = device)" - ] - }, - { - "cell_type": "code", - "execution_count": 116, - "metadata": {}, - "outputs": [], - "source": [ - "batch = next(iter(train_iterator))" - ] - }, - { - "cell_type": "code", - "execution_count": 127, - "metadata": {}, - "outputs": [], - "source": [ - "text = batch.word\n", - "chars = batch.char\n", - "tags = batch.udtag\n" - ] - }, - { - "cell_type": "code", - "execution_count": 128, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([46, 64])" - ] - }, - "execution_count": 128, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# seq len, batch_size\n", - "text.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 129, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([64, 46, 19])" - ] - }, - "execution_count": 129, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# not another_seq_len, batch_size\n", - "chars.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 130, - "metadata": {}, - "outputs": [], - "source": [ - "# new seq_len, batch_size_1, batch_size_2\n", - "chars = chars.permute(2, 0, 1)" - ] - }, - { - "cell_type": "code", - "execution_count": 135, - "metadata": {}, - "outputs": [], - "source": [ - "# new seq_len, new batch_size\n", - "\n", - "chars_new = chars.view(chars.shape[0], -1)" - ] - }, - { - "cell_type": "code", - "execution_count": 139, - "metadata": {}, - "outputs": [], - "source": [ - "emb_test = nn.Embedding(112, 64).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 143, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([19, 2944, 64])" - ] - }, - "execution_count": 143, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = emb_test(chars_new)\n", - "a.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 144, - "metadata": {}, - "outputs": [], - "source": [ - "lstm_test = nn.LSTM(64, 32).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 146, - "metadata": {}, - "outputs": [], - "source": [ - "b = lstm_test(a)" - ] - }, - { - "cell_type": "code", - "execution_count": 156, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([46, 64, 32])" - ] - }, - "execution_count": 156, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "b[1][0].permute(1, 2, 0).reshape(32, -1).reshape(*text.shape, -1).shape" - ] - }, - { - "cell_type": "code", - "execution_count": 158, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([46, 64])" - ] - }, - "execution_count": 158, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "text.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 120, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([46, 64])" - ] - }, - "execution_count": 120, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tags.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 195, - "metadata": {}, - "outputs": [], - "source": [ - "class BiLSTMPOSTaggerWithChars(nn.Module):\n", - " def __init__(self, \n", - " word_input_dim, \n", - " word_embedding_dim,\n", - " char_input_dim,\n", - " char_embedding_dim,\n", - " char_hidden_dim,\n", - " hidden_dim,\n", - " output_dim, \n", - " n_layers, \n", - " bidirectional, \n", - " dropout, \n", - " pad_idx):\n", - " \n", - " super().__init__()\n", - " \n", - " self.word_embedding = nn.Embedding(word_input_dim, word_embedding_dim, padding_idx = pad_idx)\n", - " self.char_embedding = nn.Embedding(char_input_dim, char_embedding_dim, padding_idx = pad_idx)\n", - " self.char_lstm = nn.LSTM(char_embedding_dim, char_hidden_dim, bidirectional=True)\n", - " \n", - " self.lstm = nn.LSTM(word_embedding_dim + char_hidden_dim*2, \n", - " hidden_dim, \n", - " num_layers = n_layers, \n", - " bidirectional = bidirectional,\n", - " dropout = dropout if n_layers > 1 else 0)\n", - " \n", - " self.fc = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)\n", - " \n", - " self.dropout = nn.Dropout(dropout)\n", - " \n", - " def forward(self, text, chars):\n", - "\n", - " #text = [sent len, batch size]\n", - " \n", - " #pass text through embedding layer\n", - " embedded = self.dropout(self.word_embedding(text))\n", - " #embedded = [sent len, batch size, emb dim]\n", - " \n", - " chars = chars.permute(2, 0, 1)\n", - " chars = chars.view(chars.shape[0], -1)\n", - "\n", - " chars_embedded = self.char_embedding(chars)\n", - " _, (hid, _) = self.char_lstm(chars_embedded)\n", - " hid = hid.permute(1, 2, 0)\n", - " hid = hid.reshape(hid.shape[0], -1)\n", - " hid = hid.reshape(*text.shape, -1)\n", - " \n", - " embedded_with_chars = torch.cat([embedded, hid], dim=2)\n", - " \n", - " \n", - " #pass embeddings into LSTM\n", - " outputs, (hidden, cell) = self.lstm(self.dropout(embedded_with_chars))\n", - " \n", - " #outputs holds the backward and forward hidden states in the final layer\n", - " #hidden and cell are the backward and forward hidden and cell states at the final time-step\n", - " \n", - " #output = [sent len, batch size, hid dim * n directions]\n", - " #hidden/cell = [n layers * n directions, batch size, hid dim]\n", - " \n", - " #we use our outputs to make a prediction of what the tag should be\n", - " predictions = self.fc(self.dropout(outputs))\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " \n", - " return predictions" - ] - }, - { - "cell_type": "code", - "execution_count": 196, - "metadata": {}, - "outputs": [], - "source": [ - "INPUT_DIM = len(WORD.vocab)\n", - "EMBEDDING_DIM = 100\n", - "HIDDEN_DIM = 160\n", - "CHAR_INPUT_DIM = 112\n", - "CHAR_EMBEDDING_DIM = 30\n", - "CHAR_HIDDEN_DIM = 30\n", - "OUTPUT_DIM = len(UD_TAGS.vocab)\n", - "N_LAYERS = 2\n", - "BIDIRECTIONAL = True\n", - "DROPOUT = 0.25\n", - "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n", - "\n", - "model = BiLSTMPOSTaggerWithChars(\n", - " INPUT_DIM, \n", - " EMBEDDING_DIM,\n", - " CHAR_INPUT_DIM,\n", - " CHAR_EMBEDDING_DIM,\n", - " CHAR_HIDDEN_DIM,\n", - " HIDDEN_DIM, \n", - " OUTPUT_DIM, \n", - " N_LAYERS, \n", - " BIDIRECTIONAL, \n", - " DROPOUT, \n", - " PAD_IDX\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Congratulations, you've got LSTM which relies on GRU output on each step.**\n", - "\n", - "Now we need only to train it. Same actions, very small adjustments." - ] - }, - { - "cell_type": "code", - "execution_count": 197, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "BiLSTMPOSTaggerWithChars(\n", - " (word_embedding): Embedding(8866, 100, padding_idx=1)\n", - " (char_embedding): Embedding(112, 30, padding_idx=1)\n", - " (char_lstm): LSTM(30, 30, bidirectional=True)\n", - " (lstm): LSTM(160, 160, num_layers=2, dropout=0.25, bidirectional=True)\n", - " (fc): Linear(in_features=320, out_features=18, bias=True)\n", - " (dropout): Dropout(p=0.25, inplace=False)\n", - ")" - ] - }, - "execution_count": 197, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def init_weights(m):\n", - " for name, param in m.named_parameters():\n", - " nn.init.normal_(param.data, mean = 0, std = 0.1)\n", - " \n", - "model.apply(init_weights)" - ] - }, - { - "cell_type": "code", - "execution_count": 198, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The model has 1,939,738 trainable parameters\n" - ] - } - ], - "source": [ - "def count_parameters(model):\n", - " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", - "\n", - "print(f'The model has {count_parameters(model):,} trainable parameters')" - ] - }, - { - "cell_type": "code", - "execution_count": 199, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([8866, 100])\n" - ] - } - ], - "source": [ - "pretrained_embeddings = TEXT.vocab.vectors\n", - "\n", - "print(pretrained_embeddings.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 200, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([[-0.1117, -0.4966, 0.1631, ..., 1.2647, -0.2753, -0.1325],\n", - " [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n", - " [-0.0382, -0.2449, 0.7281, ..., -0.1459, 0.8278, 0.2706],\n", - " ...,\n", - " [ 0.9261, 2.3049, 0.5502, ..., -0.3492, -0.5298, -0.1577],\n", - " [-0.5972, 0.0471, -0.2406, ..., -0.9446, -0.1126, -0.2260],\n", - " [-0.4809, 2.5629, 0.9530, ..., 0.5278, -0.4588, 0.7294]])\n" - ] - } - ], - "source": [ - "model.word_embedding.weight.data.copy_(pretrained_embeddings)\n", - "model.word_embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n", - "\n", - "print(model.word_embedding.weight.data)" - ] - }, - { - "cell_type": "code", - "execution_count": 201, - "metadata": {}, - "outputs": [], - "source": [ - "optimizer = optim.Adam(model.parameters())\n", - "\n", - "TAG_PAD_IDX = UD_TAGS.vocab.stoi[UD_TAGS.pad_token]\n", - "\n", - "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n", - "\n", - "model = model.to(device)\n", - "criterion = criterion.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 202, - "metadata": {}, - "outputs": [], - "source": [ - "def train(model, iterator, optimizer, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.train()\n", - " \n", - " for batch in iterator:\n", - " \n", - " text = batch.word\n", - " chars = batch.char\n", - " tags = batch.udtag\n", - " \n", - " optimizer.zero_grad()\n", - " \n", - " #text = [sent len, batch size]\n", - " \n", - " predictions = model(text, chars)\n", - " \n", - " #predictions = [sent len, batch size, output dim]\n", - " #tags = [sent len, batch size]\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " #predictions = [sent len * batch size, output dim]\n", - " #tags = [sent len * batch size]\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - " \n", - " loss.backward()\n", - " \n", - " optimizer.step()\n", - " \n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)\n", - "\n", - "\n", - "def evaluate(model, iterator, criterion, tag_pad_idx):\n", - " \n", - " epoch_loss = 0\n", - " epoch_acc = 0\n", - " \n", - " model.eval()\n", - " \n", - " with torch.no_grad():\n", - " \n", - " for batch in iterator:\n", - "\n", - " text = batch.word\n", - " chars = batch.char\n", - " tags = batch.udtag\n", - " \n", - " predictions = model(text, chars)\n", - " \n", - " predictions = predictions.view(-1, predictions.shape[-1])\n", - " tags = tags.view(-1)\n", - " \n", - " loss = criterion(predictions, tags)\n", - " \n", - " acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n", - "\n", - " epoch_loss += loss.item()\n", - " epoch_acc += acc.item()\n", - " \n", - " return epoch_loss / len(iterator), epoch_acc / len(iterator)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 203, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 01 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 1.029 | Train Acc: 67.43%\n", - "\t Val. Loss: 0.560 | Val. Acc: 81.96%\n", - "Epoch: 02 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.453 | Train Acc: 85.30%\n", - "\t Val. Loss: 0.465 | Val. Acc: 84.58%\n", - "Epoch: 03 | Epoch Time: 0m 13s\n", - "\tTrain Loss: 0.345 | Train Acc: 88.82%\n", - "\t Val. Loss: 0.431 | Val. Acc: 85.22%\n", - "Epoch: 04 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.289 | Train Acc: 90.62%\n", - "\t Val. Loss: 0.402 | Val. Acc: 86.37%\n", - "Epoch: 05 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.253 | Train Acc: 91.79%\n", - "\t Val. Loss: 0.379 | Val. Acc: 87.08%\n", - "Epoch: 06 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.228 | Train Acc: 92.57%\n", - "\t Val. Loss: 0.368 | Val. Acc: 86.99%\n", - "Epoch: 07 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.207 | Train Acc: 93.28%\n", - "\t Val. Loss: 0.357 | Val. Acc: 87.62%\n", - "Epoch: 08 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.192 | Train Acc: 93.71%\n", - "\t Val. Loss: 0.353 | Val. Acc: 89.27%\n", - "Epoch: 09 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.179 | Train Acc: 94.13%\n", - "\t Val. Loss: 0.343 | Val. Acc: 89.94%\n", - "Epoch: 10 | Epoch Time: 0m 13s\n", - "\tTrain Loss: 0.168 | Train Acc: 94.47%\n", - "\t Val. Loss: 0.343 | Val. Acc: 89.78%\n", - "Epoch: 11 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.158 | Train Acc: 94.82%\n", - "\t Val. Loss: 0.336 | Val. Acc: 90.07%\n", - "Epoch: 12 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.150 | Train Acc: 95.05%\n", - "\t Val. Loss: 0.337 | Val. Acc: 90.19%\n", - "Epoch: 13 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.143 | Train Acc: 95.29%\n", - "\t Val. Loss: 0.329 | Val. Acc: 90.39%\n", - "Epoch: 14 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.137 | Train Acc: 95.46%\n", - "\t Val. Loss: 0.344 | Val. Acc: 89.93%\n", - "Epoch: 15 | Epoch Time: 0m 12s\n", - "\tTrain Loss: 0.129 | Train Acc: 95.74%\n", - "\t Val. Loss: 0.330 | Val. Acc: 90.45%\n" - ] - } - ], - "source": [ - "N_EPOCHS = 15\n", - "\n", - "best_valid_loss = float('inf')\n", - "\n", - "for epoch in range(N_EPOCHS):\n", - "\n", - " start_time = time.time()\n", - " \n", - " train_loss, train_acc = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n", - " valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)\n", - " \n", - " end_time = time.time()\n", - "\n", - " epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n", - " \n", - " if valid_loss < best_valid_loss:\n", - " best_valid_loss = valid_loss\n", - " torch.save(model.state_dict(), 'tut2-model.pt')\n", - " \n", - " print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n", - " print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n", - " print(f'\\t Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}%')" - ] - }, - { - "cell_type": "code", - "execution_count": 206, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Loss: 0.342 | Test Acc: 89.85%\n" - ] - } - ], - "source": [ - "# Let's take a look at the model from the last epoch\n", - "test_loss, test_acc = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n", - "\n", - "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')" - ] - }, - { - "cell_type": "code", - "execution_count": 207, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Loss: 0.342 | Test Acc: 89.85%\n" - ] - } - ], - "source": [ - "# And at the best checkpoint (based on validation score)\n", - "model.load_state_dict(torch.load('tut2-model.pt'))\n", - "\n", - "test_loss, test_acc = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n", - "\n", - "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "py3_research env", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/week05_transformer_pos_tagging/week05_positional_encoding_carriers.ipynb b/week05_transformer_pos_tagging/week05_positional_encoding_carriers.ipynb deleted file mode 100644 index 160ef2c..0000000 --- a/week05_transformer_pos_tagging/week05_positional_encoding_carriers.ipynb +++ /dev/null @@ -1,2438 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## week04: understanding the positional encoding\n", - "\n", - "_This notebook is brought to you by [Vladislav Goncharenko](https://www.linkedin.com/in/vladislav-goncharenko/)_" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# If using Colab, uncomment this cell\n", - "#! pip install plotly --upgrade" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:51:09.014457Z", - "start_time": "2019-09-26T22:51:08.160758Z" - } - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "\n", - "import plotly.express as px" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:51:09.019768Z", - "start_time": "2019-09-26T22:51:09.016810Z" - } - }, - "outputs": [], - "source": [ - "plt.rcParams.update({'font.size': 14})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Positional Encoding matrix components proposed in the article\n", - "\n", - "$$\n", - "PE_{(pos,2i)} = \n", - "\\sin \\left( \\frac{pos}{10000^{2i/d_{\\text{model}}}} \\right) \\sim\n", - "$$\n", - "\n", - "$$\n", - "\\sim \\sin \\left( \\exp \\left( -\\frac{2i}{d_{\\text{model}}} \\right) \\cdot \\text{pos} \\right) =\n", - "\\sin(\\omega \\cdot t)\n", - "$$\n", - "\n", - "$$ \\\\ $$\n", - "\n", - "$$\n", - "PE_{(pos,2i+1)} =\n", - "\\cos(\\dots) \\sim \\cos (\\omega \\cdot t)\n", - "$$\n", - "\n", - "Let's treat $\\text{pos}$ as time and number of embedding component as carrier frequency of our signal.\n", - "\n", - "Note that carrier frequencies decrease exponentionally." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:51:10.035945Z", - "start_time": "2019-09-26T22:51:10.032742Z" - } - }, - "outputs": [], - "source": [ - "def make_carriers(d_mod, denom):\n", - " return 1 / np.power(denom, np.arange(d_mod) / d_mod)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:51:10.770645Z", - "start_time": "2019-09-26T22:51:10.473266Z" - } - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4IAAAGxCAYAAAA3eAhLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3yNV77H8c/auchNoq1cSBCCJK5BXIJqwmlVUVMzndI0FdrDqdFpp1WXVhCXlimdMdMadDpMWwytnp5OL2nVtYg2hGpJXIIaIRUUlZCQPOePkIpL7ETuvu/Xa7/Yz7PWb/12tj/ys9azlrEsCxEREREREbl92Co7AREREREREalYKgRFRERERERuMyoERUREREREbjMqBEVERERERG4zKgRFRERERERuM46VnUB5qVu3rhUYGFjZaYiIiIiIiFSKrVu3Hrcsy/t692psIRgYGMiWLVsqOw0REREREZFKYYz54Ub3tDRURERERETkNqNCUERERERE5DajQlBEREREROQ2o0JQRERERETkNqNCUERERERE5DZTY3cNFREREZHb25kzZzh27BgXLlyo7FREyoW7uzsBAQHYbCWf31MhKCIiIiI1zpkzZ/jxxx/x9/fH1dUVY0xlpyRSpvLz80lPT+f48eP4+PiUuL+WhoqIiIhIjXPs2DH8/f1xc3NTESg1ks1mw9fXl9OnT5eufxnnIyIiIiJS6S5cuICrq2tlpyFSrpycnLh48WKp+qoQFBEREZEaSTOBUtPdyr9xFYIiIiIiIiK3GRWCIiIiIiIit5kKLQSNMT2MMR8ZY9KNMZYxJtaOPq2NMeuMMecu9ZtoNM8vIiIiIreJfv36ERsbW9lp2OXgwYMYY9iyZUtlp1LlnDp1irZt22Kz2Vi0aFFlp1PhM4IewPfAM8C5mzU2xngCK4EfgY6X+r0APFeOOYqIiIiISCk0aNCAo0ePEhYWVqZxIyMjGTVqVJnGvJkFCxYQFRVFnTp1MMZw8ODBa9r89NNPxMTE4OXlhZeXFzExMZw6deqadtnZ2fTt2xdnZ2dmzpzJ8OHD+fDDD6877pw5cwgJCcHV1ZWAgAB+97vfcfbs2bL+eBVbCFqW9allWS9alvU+kG9Hl2jADRhiWdb3l/rNBJ6rjrOCObk5/N+6BZWdhoiIiIhIuXBwcMDPzw9Hx6p5XHlubq7dbbOzs7nvvvuYPHnyDds8+uijJCcnk5CQQEJCAsnJycTExFwz5sCBA3FwcGDVqlW88MILLFy4kMcee4zVq1cXabtkyRLGjBnDSy+9REpKCm+//TaffvopzzzzTIk+pz2q+jOCEcBXlmVdOXv4OVAfCKyUjG7Bq8ufJO7AX1iZ/F5lpyIiIiIiVVB2djaxsbF4eHjg6+vLyy+/fE2b3Nxcxo4dS0BAAG5ubnTs2JHPP/+88P7atWsxxrBq1So6d+6Mm5sb4eHhJCcnF4nzwQcf0Lp1a2rVqkWDBg2YPn06lmUV3g8MDGTKlCnExsZSu3ZtGjRowLJlyzh16hSDBg3Cw8ODZs2a8cUXXxT2uXppqD25nDhxgsGDBxMQEICrqystW7Zk4cKFhfdjY2NZt24db7zxBsaYIrNz69evp3Pnzri4uODr68sf/vCHIsVeZGQkTz31FKNHj8bb25tu3brZ/V08++yzjB8/nu7du1/3fkpKCgkJCSxYsICIiAgiIiKYP38+H3/8Mbt37wYKDn2PiYnBsiwSEhLw9PQEIDo6msWLF/PII4+QlJRUGHPTpk106dKFmJgYAgMD6dmzJ48//jhff/213Xnbq2qW6r/wAw5fde3HK+4duPKGMWY4MBygYcOG5Z5cSf2m64t8tXogr+54lbvb9MfF0aWyUxIRERG5bcT/eye7jpyp0DFb1PdkUv+WdrcfPXo0K1euZMWKFfj7+xMfH8/69esZOHBgYZuhQ4eSlpbGkiVLCAgI4NNPP6V///4kJSXRtm3bwnbjx49n5syZ1KtXj2eeeYbo6Gh27dqFMYatW7fy8MMPM2HCBKKjo0lKSmLEiBF4enry9NNPF8b485//zLRp03jppZeYN28eQ4YMoWfPngwaNIhp06bxyiuv8Nhjj3Ho0CFcXG78u21xuZw/f5727dszduxYPD09+fLLLxkxYgQNGzakV69ezJkzhz179hASElJYGHt7e5Oenk6fPn2IiYlh0aJFpKWl8eSTT2Kz2Zg9e3bh2O+++y7Dhw/nq6++Kix0AwMDiYyMvKVn9RITE/Hw8KBr166F17p164a7uzubNm0iODgYm83GsmXLrtt/wIABDBgwoMi17t27884777B582a6dOnCoUOH+Oijj3jggQdKneeNVPUZwRKxLGuBZVnhlmWFe3t7V3Y61whpEspDJ+tx1JzjjeQ5lZ2OiIiIiFQhZ8+e5a233uKPf/wjvXv3plWrVixcuBCb7Zdf2dPS0li6dCnLly+nR48eNGnShFGjRvHAAw8wf/78IvGmTp1KVFQUISEhTJw4kdTUVNLT0wF47bXXuOeee4iPj6d58+ZER0czevRoZs6cWSRG7969GTlyJM2aNSM+Pp6cnByaNm3K448/TtOmTYmLiyMzM5Pvv/++2M9WXC7+/v688MILhIWF0aRJE4YPH87AgQNZunQpAF5eXjg7O+Pm5oafnx9+fn44ODgwd+5c6tevz9y5cwkNDaVfv37MmDGD119/nezs7MKxGzduzOzZswkJCSE0NBSAoKAg6tWrV8pvqkBGRgbe3t5FzvIzxuDj40NGRkapYg4aNIiXX36ZHj164OTkRKNGjWjduvU130tZqOozghmA71XXfK+4V+24BsTy8ImJvJ2ymP9qfD9tvdvevJOIiIiI3LKSzMxVhrS0NHJzc4mIiCi85uHhQevWrQvfJycnY1kWLVq0KNI3JyeHnj17FrnWpk2bwr/Xr18fgGPHjhEQEEBKSgp9+/Yt0r579+7Ex8dz5syZwiWMV8bw8PDAzc2tSD6+vr6FcYtTXC55eXnMmDGDZcuWkZ6eTk5ODrm5uURGRhYbMyUlhS5duhQplLt3705ubi779u0rHLNDhw7X9F21alWxsSvLunXrmDp1KnPnzqVz587s27ePZ555hkmTJjFlypQyHauqF4KJwExjjItlWecvXbsXOAIcrLSsbkHjDvfTfsnLrPcwxG2M473+71HLoVZlpyUiIiIi1UB+fj7GGJKSknBycipyz9XVtcj7K+9fnrXKz7/5fo1XznBdPYYxplRxi+sza9YsZs+ezZw5c2jdujUeHh68+OKLNy0u7f0M7u7upY5THD8/PzIzM7Esq3A8y7I4duwYfn5+pYo5YcIEBg8ezJNPPglA69atycrK4sknn2TixIlluglPRZ8j6GGMCTPGhF0au+Gl9w0v3X/FGHNleb4EyAYWGWNaGWMGAuOA16wrn2StRjo1uYtPrHuZciyDA6cPMHf73MpOSURERESqgKCgIJycnNi8eXPhtaysrCLLLtu1a4dlWWRkZNC0adMiL39/f7vHCg0NZePGjUWubdiwgYCAAGrXrn3rH6YENmzYQP/+/YmJiSEsLIygoCD27NlTpI2zszN5eXlFroWGhrJ58+YiReiGDRtwdnYmKCio3POOiIjg7NmzJCYmFl5LTEwkKyuryHODJZGdnY2Dg0ORaw4ODpRH6VPRzwiGA9suvVyB+Et/vzzPWQ8o/NYsyzpNwQxgfWAL8AYwG3it4lIuWy5ODhwNfIgO5/IYWMufRTsX8V3md5WdloiIiIhUMg8PD5544gnGjh3LypUr2blzJ8OGDStSAF1+ni82Npb333+f/fv3s2XLFmbNmsUHH3xg91jPP/8869atY/LkyezZs4fFixcze/ZsxowZUx4frVjNmzdn1apVbNiwgdTUVEaNGsWBA0X2hCQwMJBvvvmGgwcPcvz4cfLz8xk5ciRHjhxh5MiRpKSk8MknnzBu3DhGjRqFm5tbsWP26tWL8ePHF9smIyOD7du3Fxalu3btYvv27Zw8eRIoKETvv/9+RowYQWJiIomJiYwYMYJ+/foRHBxcqp9F//79WbBgAf/61784cOAAK1euJC4ujn79+pX5kRwVujTUsqy1wA3P/7MsK/Y6174DepRfVhWvU8sgPj7Qhef2b2dj02DiNsaxvP9ynB2cKzs1EREREalEs2bNIisri4ceegg3NzeefvppsrKyirRZuHAh06dPZ8yYMRw+fJg777yTTp06ERUVZfc47du357333mPSpEm8/PLL+Pr6FhZRFW3ChAkcOHCAPn364OrqSmxsbOGuopeNHj2aIUOG0KJFC86dO8eBAwcIDAzks88+K9xopk6dOjz66KPXPXLjamlpaTRo0KDYNvPmzSM+Pr7w/eVnKhcuXEhsbCxQcO7f008/Te/evQF48MEHef3110v6Iyg0YcIEjDHExcVx+PBh6tatS//+/Zk+fXqpY96IqaYrLG8qPDzcunx+SVWTfuocT8+cxwe1JvNV5LOM/OED/rv1f/P79r+v7NREREREaoSUlJTCHSJFarLi/q0bY7ZalhV+vXs16viI6sK/jivZPu35wbExd6euZkDQAP7x/T/YeWJnZacmIiIiIiK3ARWClSQyxJe3zkdBxg5eCLiPu1zuIm5jHBfyLlR2aiIiIiIiUsOpEKwkUcHefHCxKxcd3PD6dhkTIyay96e9LPhuQWWnJiIiIiIiNZwKwUrSodEdGBdPtnj2gu9WcE/dtvRv0p+/7/g7qSdTKzs9ERERERGpwVQIVhJHBxs9mnvz1597wMVzsGMZYzuNpY5LHSZsmMCFfC0RFRERERGR8qFCsBJFBfuw8aw/2d5hsOUfeDl7Etcljt0/7ebv3/29stMTEREREZEaSoVgJYoM9gZgU50HITMVDiXSs2FP+jTuw4IdC9h9cnclZygiIiIiIjWRCsFKVNejFm0DvHjzVBjU8oIt/wBgfKfxeDp7FuwiqiWiIiIiIiJSxlQIVrKoEB++OXye8y1/C7v+D7JOcIfLHUzoMoGUkyks+n5RZacoIiIiIiI1jArBShYV7INlwUav/pCXC9sXA3Bvo3vpHdibud/OZe9Peys5SxERERGpLP369SM2Nray07DLwYMHMcawZcuWyk6lyjl16hRt27bFZrOxaNGiyk5HhWBla+3vRV0PZ/7viCc07ApbF0J+PgAvdn6R2k61idsYx8X8i5WcqYiIiIhI8Ro0aMDRo0cJCwsr07iRkZGMGjWqTGPezIIFC4iKiqJOnToYYzh48OA1bX766SdiYmLw8vLCy8uLmJgYTp06dU277Oxs+vbti7OzMzNnzmT48OF8+OGHxY6/dOlSjDH069evrD5SESoEK5nNZrinuQ/r9mSS32EonNwPB9YBcKfLnbzY5UV2ntjJP3f+s5IzFREREREpnoODA35+fjg6OlZ2KteVm5trd9vs7Gzuu+8+Jk+efMM2jz76KMnJySQkJJCQkEBycjIxMTHXjDlw4EAcHBxYtWoVL7zwAgsXLuSxxx5j9erV1427f/9+XnjhBe6++2678y0pFYJVQFSIN6fPXWB77bvB7a7CTWMAejfqzb2N7uWN7W+w/9T+SsxSRERERMpbdnY2sbGxeHh44Ovry8svv3xNm9zcXMaOHUtAQABubm507NiRzz//vPD+2rVrMcawatUqOnfujJubG+Hh4SQnJxeJ88EHH9C6dWtq1apFgwYNmD59OpZlFd4PDAxkypQpxMbGUrt2bRo0aMCyZcs4deoUgwYNwsPDg2bNmvHFF18U9rl6aag9uZw4cYLBgwcTEBCAq6srLVu2ZOHChYX3Y2NjWbduHW+88QbGmCKzc+vXr6dz5864uLjg6+vLH/7whyLFXmRkJE899RSjR4/G29ubbt262f1dPPvss4wfP57u3btf935KSgoJCQksWLCAiIgIIiIimD9/Ph9//DG7dxfs/p+fn09MTAyWZZGQkICnpycA0dHRLF68mEceeYSkpKQicS9cuMDgwYOZPn06TZo0sTvfkqqapfpt5u5m3jjYDKv2nqZ9WDQkvgFnjoJnPYwxvNj5RZIykojbGMfbfd7GweZQ2SmLiIiIVD+fjYOM7yp2TL/W0GeG3c1Hjx7NypUrWbFiBf7+/sTHx7N+/XoGDhxY2Gbo0KGkpaWxZMkSAgIC+PTTT+nfvz9JSUm0bdu2sN348eOZOXMm9erV45lnniE6Oppdu3ZhjGHr1q08/PDDTJgwgejoaJKSkhgxYgSenp48/fTThTH+/Oc/M23aNF566SXmzZvHkCFD6NmzJ4MGDWLatGm88sorPPbYYxw6dAgXF5cbfq7icjl//jzt27dn7NixeHp68uWXXzJixAgaNmxIr169mDNnDnv27CEkJKSwMPb29iY9PZ0+ffoQExPDokWLSEtL48knn8RmszF79uzCsd99912GDx/OV199VVjoBgYGEhkZeUvP6iUmJuLh4UHXrl0Lr3Xr1g13d3c2bdpEcHAwNpuNZcuWXbf/gAEDGDBgwDXXX3rpJQIDAxkyZAhr1qwpdX43oxnBKsDL1YkOje5gTWomdIgFKw+2vVt4v65rXcZ3Gs+O4zt4Z9c7lZeoiIiIiJSbs2fP8tZbb/HHP/6R3r1706pVKxYuXIjN9suv7GlpaSxdupTly5fTo0cPmjRpwqhRo3jggQeYP39+kXhTp04lKiqKkJAQJk6cSGpqKunp6QC89tpr3HPPPcTHx9O8eXOio6MZPXo0M2fOLBKjd+/ejBw5kmbNmhEfH09OTg5Nmzbl8ccfp2nTpsTFxZGZmcn3339f7GcrLhd/f39eeOEFwsLCaNKkCcOHD2fgwIEsXboUAC8vL5ydnXFzc8PPzw8/Pz8cHByYO3cu9evXZ+7cuYSGhtKvXz9mzJjB66+/TnZ2duHYjRs3Zvbs2YSEhBAaGgpAUFAQ9erVK+U3VSAjIwNvb2+MMYXXjDH4+PiQkZFRqphffPEFy5cvv+a7LA+aEawiooJ9mJmQSoZjR/yaRMHWRXD3c3Bp9q9P4z4kHEzg9e2vc0+De2js1bhyExYRERGpbkowM1cZ0tLSyM3NJSIiovCah4cHrVu3LnyfnJyMZVm0aNGiSN+cnBx69uxZ5FqbNm0K/16/fn0Ajh07RkBAACkpKfTt27dI++7duxMfH8+ZM2cKlzBeGcPDwwM3N7ci+fj6+hbGLU5xueTl5TFjxgyWLVtGeno6OTk55ObmEhkZWWzMlJQUunTpUqRQ7t69O7m5uezbt69wzA4dOlzTd9WqVcXGrgyZmZnExsaydOlS6tSpU+7jqRCsIqJCvJmZkMra3ccYFD4MlsfA3pUQfD9Q8L8LcV3i+NX//YqJGyey6P5FWiIqIiIicpvJz8/HGENSUhJOTk5F7rm6uhZ5f+X9y7NW+Zd2py/OlTNcV49hjClV3OL6zJo1i9mzZzNnzhxat26Nh4cHL7744k2LS3s/g7u7e6njFMfPz4/MzEwsyyocz7Isjh07hp+fX4nj7dy5k6NHj9KrV6/Ca5d/Ro6OjuzcuZPg4OCySR4tDa0ygn1rU9/LhTW7j0FwH/DwK7JpDIC3mzfjOo1je+Z2lqQuqaRMRURERKQ8BAUF4eTkxObNmwuvZWVlFVl22a5dOyzLIiMjg6ZNmxZ5+fv72z1WaGgoGzduLHJtw4YNBAQEULt27Vv/MCWwYcMG+vfvT0xMDGFhYQQFBbFnz54ibZydncnLyytyLTQ0lM2bNxcpQjds2ICzszNBQUHlnndERARnz54lMTGx8FpiYiJZWVlFnhu0V8eOHfnuu+/Yvn174evBBx/k7rvvZvv27TRuXLYrAlUIVhHGGCJDfNiw9zi5lgO0fxz2fgGnDhVp169JP+4JuIe/JP+FQ2cO3SCaiIiIiFQ3Hh4ePPHEE4wdO5aVK1eyc+dOhg0bVqQAuvw8X2xsLO+//z779+9ny5YtzJo1iw8++MDusZ5//nnWrVvH5MmT2bNnD4sXL2b27NmMGTOmPD5asZo3b86qVavYsGEDqampjBo1igMHDhRpExgYyDfffMPBgwc5fvw4+fn5jBw5kiNHjjBy5EhSUlL45JNPGDduHKNGjcLNza3YMXv16sX48eOLbZORkcH27dsLi9Jdu3axfft2Tp48CRQUovfffz8jRowgMTGRxMRERowYQb9+/Uo1c+fu7k6rVq2KvOrUqUPt2rVp1aoVzs7OJY5ZHBWCVUhUsA9ZuXkkHTxZUAgaA1uLnh94eYmok82JuI1x5Fs3n94XERERkeph1qxZREVF8dBDDxEVFUWrVq3o0aNHkTYLFy5k6NChjBkzhpCQEPr168f69etp1KiR3eO0b9+e9957jxUrVtCqVSvGjRtXWERVtAkTJtCpUyf69OlDjx49cHd3Jzo6ukib0aNH4+zsTIsWLfD29ubQoUP4+/vz2WefsW3bNsLCwhg2bBiDBw++7pEbV0tLS+Po0aPFtpk3bx7t2rUrzKVv3760a9eOjz76qLDNkiVLaNu2Lb1796Z37960bduWd96pHps7mivPCqlJwsPDrcvnl1QX2bkXCYtfyeMRjZjQrwUsGQTpW+G5XeBQdH32h/s+JG5jHOM6jSM6NPoGEUVERERuTykpKYU7RIrUZMX9WzfGbLUsK/x69zQjWIW4OTvSucmdrN596cHY8GGQdQxSP7mm7YCgAXT3786c5Dn85+f/VHCmIiIiIiJSnakQrGKign3Yn5nFDyeyoGkv8Gp4zaYxULBEdFLEJByMA5M2TdISURERERERsZsKwSqmZ4gPAGt3ZxacIdhhCBxYB8f3XdPWz92P0eGjScpIYvnu5RWdqoiIiIiIVFMqBKuYwLruNK7rzurUS8tD28WAzRG2Lrxu+4HNBhJRL4LXtr5G+tn0CsxURERERESqKxWCVVBksDeJ+09wLjcPavtCSD/YvhgunL+mrTGG+K7x2IyNSZsmUVM3/xERERERkbKjQrAK6hniQ+7FfBL3Hy+4ED4Mzv0Eu/7vuu3redTjuQ7P8fXRr3l/7/sVmKmIiIiIiFRHKgSroE6N78TN2eGX5aGNe8BdTa+7acxlDzd/mM5+nZm9ZTZHzxZ/JoqIiIiIiNzeVAhWQbUcHejWtC5rUjMLlnoaAx2Gwn82w4+7rtvHGEN8t3jyrXwmJ07WElEREREREbkhFYJVVFSwD+mnzrHv2NmCC2GPgkOtG24aA+Dv4c9zHZ5j05FN/O++/62gTEVEREREpLpRIVhFRQZ7A/yyPNTtTmj5K/j2X5CbdcN+vw3+LR39OvJq0qtkZGVURKoiIiIiUo769etHbGxsZadhl4MHD2KMYcuWLZWdSpVz6tQp2rZti81mY9GiRZWdjgrBqqp+HVdC/GqzZvexXy6GD4OcM/D9ihv2sxkb8RHx5Fl5xCfGa4moiIiIiFSYBg0acPToUcLCwso0bmRkJKNGjSrTmDezYMECoqKiqFOnDsYYDh48eE2bn376iZiYGLy8vPDy8iImJoZTp05d0y47O5u+ffvi7OzMzJkzGT58OB9++OE17d58803uvvtu7rjjDurUqUNUVBQbNmwoj4+nQrAqiwrxYcvBnzhz/kLBhQadwadFsZvGADTwbMAz7Z9hQ/oGPkr7qAIyFREREREBBwcH/Pz8cHR0rOxUris3N9futtnZ2dx3331Mnjz5hm0effRRkpOTSUhIICEhgeTkZGJiYq4Zc+DAgTg4OLBq1SpeeOEFFi5cyGOPPcbq1auLtF27di2PPPIIq1ev5uuvvyY4OJjevXuzd+/eEn1Oe6gQrMKign24mG+xYe+lYySMKZgVPLIN0pOL7Ts4ZDDtfdozM2kmx7KPFdtWRERERKqG7OxsYmNj8fDwwNfXl5dffvmaNrm5uYwdO5aAgADc3Nzo2LEjn3/+eeH9tWvXYoxh1apVdO7cGTc3N8LDw0lOLvr74wcffEDr1q2pVasWDRo0YPr06UVWkwUGBjJlyhRiY2OpXbs2DRo0YNmyZZw6dYpBgwbh4eFBs2bN+OKLLwr7XL001J5cTpw4weDBgwkICMDV1ZWWLVuycOEv+2LExsaybt063njjDYwxRWbn1q9fT+fOnXFxccHX15c//OEPRYq9yMhInnrqKUaPHo23tzfdunWz+7t49tlnGT9+PN27d7/u/ZSUFBISEliwYAERERFEREQwf/58Pv74Y3bv3g1Afn4+MTExWJZFQkICnp6eAERHR7N48WIeeeQRkpKSCmMuXryYUaNG0a5dO4KDg/nb3/5G7dq1SUhIsDtve1XNUl0AaN+wDp4ujqxJPcYDresVXGzzW1g5sWDTGP/2N+xrMzamdJvCrz/6NVMSp/DXnn/FGFNBmYuIiIhUPTO/mUnqydQKHTPkzhDGdhprd/vRo0ezcuVKVqxYgb+/P/Hx8axfv56BAwcWthk6dChpaWksWbKEgIAAPv30U/r3709SUhJt27YtbDd+/HhmzpxJvXr1eOaZZ4iOjmbXrl0YY9i6dSsPP/wwEyZMIDo6mqSkJEaMGIGnpydPP/10YYw///nPTJs2jZdeeol58+YxZMgQevbsyaBBg5g2bRqvvPIKjz32GIcOHcLFxeWGn6u4XM6fP0/79u0ZO3Ysnp6efPnll4wYMYKGDRvSq1cv5syZw549ewgJCSksjL29vUlPT6dPnz7ExMSwaNEi0tLSePLJJ7HZbMyePbtw7HfffZfhw4fz1VdfFRa6gYGBREZG3tKzeomJiXh4eNC1a9fCa926dcPd3Z1NmzYRHByMzWZj2bJl1+0/YMAABgwYUOwYubm5nD9/njvuuKPUed6IZgSrMEcHGz2ae7N2Tyb5+Zf+d8bFC1r/Br57H86fLrZ/I89G/L7d71l3eB0f7/+4AjIWERERkdI6e/Ysb731Fn/84x/p3bs3rVq1YuHChdhsv/zKnpaWxtKlS1m+fDk9evSgSZMmjBo1igceeID58+cXiTd16lSioqIICQlh4sSJpKamkp6eDsBrr73GPffcQ3x8PM2bNyc6OprRo0czc+bMIjF69+7NyJEjadasGfHx8eTk5NC0aVMef/xxmjZtSlxcHJmZmXz//ffFfrbicqP6WhIAACAASURBVPH39+eFF14gLCyMJk2aMHz4cAYOHMjSpUsB8PLywtnZGTc3N/z8/PDz88PBwYG5c+dSv3595s6dS2hoKP369WPGjBm8/vrrZGdnF47duHFjZs+eTUhICKGhoQAEBQVRr169Un5TBTIyMvD29i4y2WKMwcfHh4yMstm0ccKECXh4ePDggw+WSbwraUawiosK9uHjHUfZeeQMrQO8Ci6GD4Pkt2HHcuj038X2jw6N5osfvmDGNzPoUq8L3m7eFZC1iIiISNVTkpm5ypCWlkZubi4RERGF1zw8PGjdunXh++TkZCzLokWLFkX65uTk0LNnzyLX2rRpU/j3+vXrA3Ds2DECAgJISUmhb9++Rdp3796d+Ph4zpw5U7iE8coYHh4euLm5FcnH19e3MG5xisslLy+PGTNmsGzZMtLT08nJySE3N5fIyMhiY6akpNClS5cihXL37t3Jzc1l3759hWN26NDhmr6rVq0qNnZVMGfOHObPn8+XX35Z+H2UJRWCVdw9wd4YA2t2H/ulEKzfruC15R/Q8cmCZwdvwMHmwNRuU/nNR79h6uapzImaoyWiIiIiItVUfn4+xhiSkpJwcnIqcs/V1bXI+yvvX/79Lz8//6ZjXPm74tVjGGNKFbe4PrNmzWL27NnMmTOH1q1b4+HhwYsvvnjT4tLez+Du7l7qOMXx8/MjMzMTy7IKx7Msi2PHjuHn53dLsf/85z8TFxfHZ599RqdOncoi3WtoaWgVV9ejFm0C6vxynuBl4cPg2C74z9c3jdHYqzGj2o1izX/W8NmBz8opUxERERG5FUFBQTg5ObF58+bCa1lZWUWWXbZr1w7LssjIyKBp06ZFXv7+/naPFRoaysaNG4tc27BhAwEBAdSuXfvWP0wJbNiwgf79+xMTE0NYWBhBQUHs2bOnSBtnZ2fy8vKKXAsNDWXz5s1FitANGzbg7OxMUFBQuecdERHB2bNnSUxMLLyWmJhIVlZWkecGS+q1114jLi6OTz755IYb1ZQFFYLVQFSwN98ePsWJszm/XGz1a6jledOjJC57vMXjtKnbhle+eYXj546XU6YiIiIiUloeHh488cQTjB07lpUrV7Jz506GDRtWpAC6/DxfbGws77//Pvv372fLli3MmjWLDz74wO6xnn/+edatW8fkyZPZs2cPixcvZvbs2YwZM6Y8PlqxmjdvzqpVq9iwYQOpqamMGjWKAwcOFGkTGBjIN998w8GDBzl+/Dj5+fmMHDmSI0eOMHLkSFJSUvjkk08YN24co0aNws3Nrdgxe/Xqxfjx44ttk5GRwfbt2wuL0l27drF9+3ZOnjwJFBSi999/PyNGjCAxMZHExERGjBhBv379CA4OLtXP4tVXX2XcuHG89dZbNG/enIyMDDIyMjh9uvi9QUpDhWA10DPEB8uC9Xszf7no7A5tB8HODyHrxE1jONgcmNJtClkXsnj562u3IRYRERGRyjdr1iyioqJ46KGHiIqKolWrVvTo0aNIm4ULFzJ06FDGjBlDSEgI/fr1Y/369TRq1Mjucdq3b897773HihUraNWqFePGjSssoirahAkT6NSpE3369KFHjx64u7sTHR1dpM3o0aNxdnamRYsWeHt7c+jQIfz9/fnss8/Ytm0bYWFhDBs2jMGDB1/3yI2rpaWlcfTo0WLbzJs3j3bt2hXm0rdvX9q1a8dHH/1yTveSJUto27YtvXv3pnfv3rRt25Z33nmnFD+FAm+88QYXLlzgkUceoV69eoWvZ555ptQxb8RceVZITRIeHm5dPr+kusvPt+j08pdEBNXlr4Pb/XLjx13wtwi4bxp0ffrGAa7w9+/+zpzkOcy6Zxa9A3uXU8YiIiIilSslJaVwh0iRmqy4f+vGmK2WZYVf755mBKsBm81wT3Mf1u/J5GLeFQ/i+raAhhGwZSHY8eAvQGzLWFre1ZLpm6dz8vzJcspYRERERESqMhWC1UTPEB9On7vA9v+cKnojfBicTIOD6+2K42hzZGq3qfx84Wde+fqVcshURERERESqOhWC1UT3ZnVxsJlrdw8NfRBc77R70xiAZnc046m2T5FwMIEvf/iyjDMVEREREZGqToVgNeHl6kSHRnewZndm0RtOLtAuGlI/gZ8z7I43tNVQQu8MZermqZw6f+rmHUREREREpMZQIViN9AzxIeXoGTJOny96o8NQyL8I2+zfocjJ5sTUblM5k3OGV77RElERERGpeew5PF2kOruVjT9VCFYjUcE+AKzZfdXy0LuCoEkkbP0n5Odd0+9Ggu8MZnib4Xx64FNWH1pddomKiIiIVDJ3d3fS09PJzc29pV+WRaoqy7I4ceIELi4upervWMb5SDlq7uuBfx1X1qQeY3CnhkVvhg+D5Y/Dvi+huf3HQjzZ+klWHVrF1M1T6eDbAa9aXmWctYiIiEjFCwgI4Pjx4/zwww9cvHixstMRKRcuLi4EBASUqq8KwWrEGENksDcfbksn52IetRwdfrkZ/AB4+BYcJVGCQtDJoWCJ6KOfPMrMb2by8t06bF5ERESqP5vNho+PDz4+PpWdikiVpKWh1UxUsA9ZuXkkHfip6A0HJ2j/OOz9HE79p0QxQ+8K5YnWT/Dv/f9m3X/WlWG2IiIiIiJSFakQrGa6Nr0LZ0fbtc8JQkEhaFmQ/HaJ445oM4KmdZoyJXEKZ3LPlEGmIiIiIiJSVakQrGbcnB3p0uSu6xeCdRpCs/sKCsG8CyWK6+TgxLRu0zhx/gSvJr1aRtmKiIiIiEhVpEKwGooK9mZ/ZhY/nMi69mb4MDibAbs/K3HclnVbMrTVUD7c9yFfHf6qDDIVEREREZGqSIVgNVR4jETqdWYFm90LngGw5R+liv1U26cI8goiPjGen3N/vpU0RURERESkilIhWA0F1nWnSV13Vu/OvPamzQE6xML+NXAircSxnR2cmdptKpnnMpm9ZfatJysiIiIiIlWOCsFqKjLYh837T5Cde51zcdrHgHGArYtKFbu1d2uGtBzCir0r2JS+6dYSFRERERGRKqfCC0FjzEhjzAFjzHljzFZjzN03af+oMWa7MSbbGJNhjHnXGONXUflWVT1DfMi9mE9i2olrb9b2g5C+sO1duJhTqvi/C/sdjb0aMzlxMlkXrvMsooiIiIiIVFsVWggaYx4B5gAvA+2ATcBnxpiGN2jfDXgH+CfQEvgV0AJYXCEJV2EdG9+Bm7MDq6/3nCAUbBpz7iTs+qhU8Ws51GJK1ylkZGXw2pbXbiFTERERERGpaip6RvA5YJFlWW9alpViWdbTwFHgqRu0jwAOW5b1J8uyDliWtRn4K9C5gvKtsmo5OtCtaV3W7s7EsqxrGzS+B+5sUupNYwDCfMJ4vMXjLN+znK+Pfn0L2YqIiIiISFVSYYWgMcYZ6AB8cdWtL4CuN+i2EahnjOlvCtQFBgGfll+m1UfPEB/ST51j77Gz19602aDDUDi0CY6llHqMUe1G0cizEZM2TSL7QvYtZCsiIiIiIlVFRc4I1gUcgB+vuv4jcN1n/izLSqSg8FsM5AKZgAGGXK+9MWa4MWaLMWZLZuZ1dtSsYSKDvQFuvDw0LBocnGHLwlKP4eLowpSuUzhy9gh/2vqnUscREREREZGqo0rvGmqMaUHBUtCpFMwm3k9B0Tj/eu0ty1pgWVa4ZVnh3t7eFZdoJann5UqIX+3rnycI4H4XtPgVfPsvyC39hi/tfdsTHRrNv3b/i6SMpFLHERERERGRqqEiC8HjQB7ge9V1XyDjBn3GA99YlvWqZVk7LMv6HBgJxBhjAsov1eqjZ4gPW374iTPnL1y/QfgwyDkN339wS+M83e5pGtRuwMSNE7VEVERERESkmquwQtCyrFxgK3DvVbfupWD30Otxo6B4vNLl91V6NrOiRIX4kJdv8dWe49dv0LALeIfe0qYxAG5ObsR3jefw2cP8ZdtfbimWiIiIiIhUrooupl4DYo0xTxpjQo0xc4D6wDwAY8zbxpi3r2j/b2CAMeYpY0yTS8dJ/AVItizrUAXnXiW1a1AHL1cn1uy+wfJQYwpmBY8kw5FttzRWR7+ODA4ZzJKUJWz9cestxRIRERERkcpToYWgZVnLgGeBCcB2oDvwgGVZP1xq0vDS63L7RRQcOTEK+B54H9gDDKi4rKs2RwcbPZp7s3Z3Jvn51zlGAqDtI+Dkdkubxlz2bPtnqe9Rn4kbJ3Lu4rlbjiciIiIiIhWvwpdXWpY117KsQMuyalmW1cGyrPVX3Iu0LCvyqvZ/tSyrpWVZbpZl1bMsK9qyrMMVnXdVFhXszfGzOXx/5PT1G7h4Qatfw3fvw/kbtLHT5SWih34+xOvbXr+lWCIiIiIiUjn0nF0NcE9zb4yBNanFHJkRPgwuZMGO5bc8Xud6nflt89/yzq532H5s+y3HExERERGRiqVCsAa4y6MWbQPq3Pg5QQD/9lAvrGB5qHWDJaQl8Fz4c/i5+xG3MY7zF8/fcjwREREREak4KgRriKhgH749fIoTZ3Nu3Ch8GBzbCYdv/SxAdyd3JnedzMEzB5m7fe4txxMRERERkYqjQrCG6Bnig2XBuj3FLA9t9Wtwrn3LR0lc1rV+V37d7Nf8c9c/2ZG5o0xiioiIiIhI+VMhWEO0rO9JXY9arNldTCFYy6NgB9HvP4Dsk2Uy7vPhz+Pt6k3cxjhy8oqZjRQRERERkSpDhWANYbMZIoO9Wbf7GBfz8m/csMNQyMuBb5eWybi1nWszuetk9p/ez7xv55VJTBERERERKV8qBGuQniE+nDl/kW3/OXXjRn6toEHnguWhZbBpDEB3/+481PQh/vH9P/j++PdlElNERERERMqPCsEapHuzujjYDKtTi9k9FAo2jTmxDw5+VWZjj+44mroudYnbGEduXm6ZxRURERERkbKnQrAG8XRxIrzRHay5WSHYYgC43lFmm8YAeDp7MqnrJPad2sf8HfPLLK6IiIiIiJQ9FYI1TM8QH1Izfubo6XM3buTkCmHRkPJvOHuTorEEegT04MGgB3nru7fYdWJXmcUVEREREZGypUKwhokK8QFgTWoxu4cCdIiF/Iuw7Z0yHX9MxzHc6XIncRvjuJB3oUxji4iIiIhI2VAhWMM08/HAv44ra3bfZKavbjNo3AO2LoL8vDIb36uWF3Fd4tjz0x7e/O7NMosrIiIiIiJlR4VgDWOMISrEm437jpNz8SYFXvgwOHUI0laXaQ5RDaPo26Qvb+54k90nd5dpbBERERERuXUqBGugqGAfsnPz+ObATQ6ND+4L7j5lumnMZeM6jsOrlhcTNk7gQr6WiIqIiIiIVCUqBGugiKC7cHa03fw5QUdnaB8DexLg9OEyzaGOSx3iusSRejKVf3xX9oWmiIiIiIiUngrBGsjN2ZGIJnex9mbPCQK0H1JwsHzy22WeR69Gvbg/8H7m7ZjHnp/2lHl8EREREREpHRWCNVRUsDf7j2dx8HhW8Q3vaATN7oWt/4Ry2OVzfOfxeDp7Ercxjov5F8s8voiIiIiIlJwKwRqqZ4gvwM13D4WCTWPOZhQsES1jd7rcyYudX2TXiV0s2rmozOOLiIiIiEjJqRCsoRre5UYTb3fW7L7Jc4IAze4Dz4By2TQGoHdgb+5tdC9zt88l7VRauYwhIiIiIiL2UyFYg0UF+7B5/wmyc2+yJNPmAB2GFBwjcXJ/ueTyUueXcHdy1xJREREREZEqQIVgDdYzxIfci/ls2nfi5o3bxYBxKDhgvhzc5XoXL3Z+ke+Of8fbu8p+YxoREREREbGfCsEaLDzwDtydHex7TtCzHoQ8ANvehYs55ZLP/YH306thL97Y9gb7T5fPzKOIiIiIiNycCsEarJajA92a1mVN6jEsy7p5h/BhkH0CUv5dLvkYY5jQZQKuTq7EbYwjLz+vXMYREREREZHiqRCs4XqG+HDk9Hn2/Hj25o0bR8IdgbBlYbnlU9e1LuM6jWNH5g7eTXm33MYREREREZEbUyFYw0UG+wCwOtWO5aE2G3QYCj9sgMzd5ZZT38Z9iQyI5K/b/srB0wfLbRwREREREbk+FYI1nJ+XC6H1PO17ThCg3WNgcyrXWUFjDHERcTg7ODNx00QtERURERERqWAqBG8DPUO82frDT5w+d+Hmjd3rQosB8O0SyM0ut5x83HwY23Es245tY2nq0nIbR0RERERErqVC8DYQFexDXr7FV3vtOFweCjaNOX8adv5vueb1YNCD3O1/N3OS53DozKFyHUtERERERH6hQvA2ENagDl6uTqxJtbMQbNQV6gbDln+Ua17GGCZGTMTR5sjETRPJt/LLdTwRERERESmgQvA24Ohg457m3qzbc4z8fDuOkTCmYFYwfQsc/bZcc/Nz92NMxzFs/XEry3YvK9exRERERESkgArB20RUiDfHz+byXfpp+zq0fQQcXct105jLftX0V3Sr340/bf0Th38+XO7jiYiIiIjc7lQI3iZ6NPPGGOzfPdT1Dmj1a/juPcj5uVxzM8YwKWISNmNj0qZJWiIqIiIiIlLOVAjeJu7yqEVYgzqs2W3nc4JQsDw09yzsWF5+iV1Sz6Mez4c/zzcZ3/D+nvfLfTwRERERkduZCsHbSFSwDzsOn+L42Rz7Ovi3B782BctDLTueLbxFv2n2G7rU68LsLbM5cvZIuY8nIiIiInK7UiF4G4kK9sGyYJ29s4KXN4358Ts4vKV8k6NgiejkrpOxsJi0aRJWBRSfIiIiIiK3IxWCt5GW9T3xrl3L/ucEAVr/Bpxrl/tREpf5e/jzfIfn2Xx0Myv2rqiQMUVEREREbjcqBG8jNpshsrk36/dkcjHPzg1ZatWGNr+FnR9A9snyTfCSh4MfpqNfR2ZtmcXRs0crZEwRERERkduJXYWgMeZXxhiH8k5Gyl/PEB/OnL9I8qFT9ncKHwoXz8O3/yq/xK5gMzbiu8aTb+UTnxivJaIiIiIiImXM3hnBxUC6MWamMaZ5eSYk5atbs7o42kzJlof6tYaATgXLQyuoKGtQuwHPtn+WjUc28uG+DytkTBERERGR24W9haAfMAm4B0gxxmwwxgw1xriXX2pSHjxdnAgPvIM1qSUoBKFg05gTe+HghvJJ7DoGhQyig28HXk16lR+zfqywcUVEREREajq7CkHLsn62LGu+ZVldgDbA18ArwFFjzJvGmC7lmaSUrZ4hPqRm/MyRU+fs79TyV+BSp8I2jYGCJaJTuk7hQv4FpmyeoiWiIiIiIiJlpMSbxViWtRP4E7AAcAYeAb4yxnxtjGlTxvlJOYgK9gEo2fJQJ1cIi4aUf8PZEhxKf4saejbk9+1/z/rD6/n3/n9X2LgiIiIiIjWZ3YWgMcbJGPNbY0wCcADoCfwP4As0AlKAZeWSpZSppj4e+NdxZU1qCQu68KGQfwG2v1s+id3AoyGP0s6nHTO+mUFmdsUVoSIiIiIiNZW9u4b+FTgKvAHsAtpaltXdsqxFlmWdsyzrCDAOCC6/VKWsGGPoGeLDxn3HybmYZ3/Hus0g8G7YshDy7Tx+ogw42ByY0nUKuXm5WiIqIiIiIlIG7J0RbAGMAvwty3rOsqxd12lzHIgqs8ykXEWFeHPuQh5f7y/h2YDhQ+HUD7B/dfkkdgOBXoE83e5p1v5nLZ8e+LRCxxYRERERqWns3Syml2VZ/7IsK7eYNhcty1pXdqlJeYpoUpdajraSPScIENIf3OoWzApWsMdCH6ONdxte+eYVjp87XuHji4iIiIjUFPYuDZ1ujPmf61z/H2PM1LJPS8qbq7MDEUF3sXZ3CZ+5c3SG9jGw+zM4nV4+yd2Ag82BqV2ncu7COaZtnqYloiIiIiIipWTv0tAYYNt1rm8FHi+7dKQiRQX7cOB4FgeOZ5WsY/shYOXDtnfKJ7FiNKnThJFhI1l1aBWfH/y8wscXEREREakJ7C0EfYDrTR2doGDXUKmGCo+RKOnh8nc2hqa9YOs/Ie9iOWRWvCEth9DqrlZM/3o6J86dqPDxRURERESqO3sLwUPA3de53gM4XHbpSEVqeJcbQd7uJX9OECB8GPx8BPZW/Kyco82Rqd2mknUhi5e/frnCxxcRERERqe7sLQTnA38yxvy3MSbo0ms4MJuCg+WlmooK9uHr/SfJyinhzF6z3lC7Pmz5R/kkdhNN72jKU22f4osfviDhQEKl5CAiIiIiUl3Zu2vobAqKwb8Aey695gBvWpb1x/JLT8pbVIgPuXn5bEor4RJLB0foMAT2rYKTB8onuZsY2moobeq24cUNL7L6UMUeZyEiIiIiUp3ZOyOIZVnjgbpAl0svb8uyxpVXYlIxOgbeibuzQ+mWh7Z/HIwNkv9Z9onZwdHmyNz/mkvInSE8t/Y5Pt2v8wVFREREROxhdyEIYFlWlmVZSZdeZ8srKak4zo42ujery5rUYyU/jsGzPgT3geR34OINj5gsV161vHjzvjdp59OOcV+NY8WeFZWSh4iIiIhIdWLvOYIuxpixxpgvjDHbjTE7rnyVd5JSvqKCfTh6+jy7f/y55J3Dh0L2cUj9d9knZid3J3fm/tdcuvp3ZXLiZN7d9W6l5SIiIiIiUh3YOyM4FxgHHAQ+BFZc9ZJqLCrk8jESJTxcHqBJT6jTCLYsLOOsSsbV0ZW/RP2FXg17MTNpJm/ueLNS8xERERERqcoc7Wz3K+Bhy7K+LM9kpHL4errQop4na1KP8VRkUMk622wFs4JfTobMPeDdvFxytIezgzOz7plF3MY4/rLtL2RdyOKZ9s9gjKm0nEREREREqiJ7ZwSzgf+UZyJSuXqG+LD10E+czr5Q8s5hj4HNCbZW7qwgFGwgM737dB5u/jBvff8WM76ZQb6VX9lpiYiIiIhUKfYWgn8EnjNlMLVijBlpjDlgjDlvjNlqjLneQfVXtnc2xky51CfHGHPIGPP7W81DiooK8SYv3+KrfaVYHurhDS0ehO2L4cK5sk+uhGzGRlyXOB5v8ThLUpcwadMk8vLzKjstEREREZEqw96lofcCdwP3G2N2AUWmjSzLetCeIMaYRyg4f3AksOHSn58ZY1pYlnXoBt3+BQQAw4G9gC/gamfeYqewBndQx82J1anH6NemfskDhA+D71fAzv+FsEfLPsESMsYwOnw07k7u/O3bv3H+4nlevvtlnGxOlZ2aiIiIiEils7cQPA78bxmM9xywyLKsyzt5PG2MuR94Chh/dWNjzH1ALyDIsqzjly4fLIM85CoONsM9zb1ZtzuT/HwLm62Ek7+NukHd5rDlH1WiEISCYnBk2EjcHN2YvXU25y+eZ1bkLGo51Krs1EREREREKpVdS0Mtyxpa3MueGMYYZ6AD8MVVt74Aut6g26+AJAqWpR42xuw1xvzFGONhz5hSMlHBPpzIymVH+umSdzamYFbwcBIcrVonisS2imVC5wmsPbyW3636HdkXsis7JRERERGRSlWiA+WNMeHGmEeMMe6X3rsbY+ydVawLOAA/XnX9R8DvBn2aAN2BtsCvgVHA/cCiG+Q33BizxRizJTOzFM+63eZ6NPfGGFiTeqx0AdoOAkeXKrFpzNUeCXmEad2mkZSRxP98+T/8nFuKMxNFRERERGoIew+U9zXGbAa+AZZQ8JwewGvA7HLKDQrys4BHLcv62rKszykoBn9tjPG9urFlWQssywq3LCvc29u7HNOqme50d6Zdgzqs3V3KQtD1Dmj1a9ixHHKqXqE1oOkAXu3xKt8d/44nPn+Cn87/VNkpiYiIiIhUCntnBP9EwczdXRQcJXHZe8B9dsY4DuTxSxF5mS+QcYM+R4F0y7KuXKuYcunPhnaOKyUQFezDt4dPk/lzTukCdBgKuWfhu/fLNrEycl/gfcyJmkPaqTSGfT6MzGzNHIuIiIjI7cfeQrAX8JJlWVdPoaRhZ0FmWVYusJWCHUivdC+w6QbdNgL1r3om8PKJ5T/YM66UTFSIDwDr9pSyQAoIB9/WBZvGWFYZZlZ2egT04G//9TfSz6YTmxDLkbNHKjslEREREZEKZW8h6ArkXue6N3C+BOO9BsQaY540xoQaY+YA9YF5AMaYt40xb1/RfglwAlhojGlpjOlGwfET71uWVcr1i1KclvU98aldizWlXR5qDIQPhYwdkJ5ctsmVoU71OrHg3gX8dP4nhiQM4Ycz+n8FEREREbl92FsIrgdir3hvGWMcgLHAKnsHsyxrGfAsMAHYTsFGMA9YlnX5t/CGXDHDaFnWWeC/AC8Kdg9dDqwDhtk7ppSMMYbIYG/W78nkQl5+6YK0+S04exTMClZhYT5hvNX7LXIu5jDksyHs/WlvZackIiIiIlIh7C0ExwD/bYxZCdSiYIOYXUA3rnP+X3Esy5prWVagZVm1LMvqYFnW+ivuRVqWFXlV+92WZd1nWZabZVn+lvX/7N13fNXl3f/x15W9QyYzECCQICBTsbgIiCCKRG83ynDd1daun7PDDltXW6u13rWOMqxVRCGAICAKbkGIgigr7CFkMLLnuX5/nBBiDBDhJN9zTt7Px+P74CQ5fM/bPry9eXNd389lf2St9b5JJH4kMz2Z4ooacnae4jCV0Gjof7X7gPly7x7I0iehD9PGTiPABDB1yVS+KvzK6UgiIiIiIi2uuecIfg30x/0s31IgDPegmEHW2q0tF0+ccF6vRIICDMs3ncYglaFToaYc1s7yXLAW0rNdT2aMnUFkUCS3LrmVnAPeu6VVRERERMQTmn2OoLV2v7X2t9bay6y146y1v7bWftOS4cQZ0WHBnJUaf+rnCQJ0HACdh3r10JiGUmJSmHHJDBLDE/nhsh/yyb5PnI4kIiIiItJimnuO4OATXS0dUlpfZkYSmw4Us/dw+anfZOjNULAJdh5vKKx36RDZgWljp9Elugs/eudHrNi9wulIIiIiIiItorkrgqtxD2tZ3eD6rMElfmZk3TESp3y4PEDfKyAs1uuHxjSUGxAICAAAIABJREFUGJ7ItDHTSI9L5+fLf87i7YudjiQiIiIi4nHNLYLdgR51v3bHfZbfdcCXwGUtE02c1DMpii5x4ae3PTQkAgbcAF/PgxLfObg9NjSW5y9+ngHJA7j3/XuZu2Wu05FERERERDyqucNidja6cq21s3FPE/11y0YUJxhjyExP5qPcQiqqa0/9RkOngqsavnjZc+FaQVRIFP+86J/8oNMPePDjB3l5g2/lFxERERE5kWYPizmO7cBATwQR7zMyI5ny6lpWbT946jdJSodu58GaaeA6xXMJHRIeFM7TI59mZMpIHl31KC98+YLTkUREREREPKK5w2LiG10Jxph+wCPAppaNKE45p0cCoUEBvHs620PBvSp4aAdsW+6RXK0pJDCEv4z4C+O6j+OpnKf4e87fsT4wBVVERERE5ESCmvm+AqDxn34NsBu41qOJxGuEhwQyvGdC3cCYvqd+oz7jISLRPTQmbZTH8rWW4IBgHj7vYcKDwnn+y+cpqynjvrPuwxjjdDQRERERkVPS3CKY2ehrF5AP5FprazwbSbxJZkYyy+d9xbb8EnokRZ3aTYJCYdCN8PHTULQPYjp5NmQrCAwI5Lc/+C3hQeH8Z8N/KK8p58FzHiQwINDpaCIiIiIi31tzh8W81+j6wFq7USXQ/2Wmu4+RWL7pNKd+DpkCthZyXjr9UA4xxnDvWffyv2f+L3O2zOGBDx6g2lXtdCwRERERke+tWSuCxpgLmntDa+37px5HvE1KfARpyVGs2JTHLed1P/UbxXeHnqMgZwac//8gsLmL0d7FGMOPB/2YiOAI/rbmb5TXlvOXC/9CaGCo09FERERERJqtuX8aX8GxZwSPPhjV+Ouj39NeOT+TmZ7EjI93UlpZQ2ToaRS4oTfDrImwZSlkjPNcQAfc3O9mwoPCeXjlw9z1zl08mfkkEcERTscSEREREWmW5h4fcRnu6aCTgLS6axKwERgPJNVdyS2QURyWmZ5MVa2Lj3ILTu9GvcdCdEf30Bg/cH3G9Tx07kOs3L+SO5bdQXFVsdORRERERESapblF8CHgp9bal6212+qul4GfAX+01hYevVouqjhlaGo8UaFBp/+cYGAQDJ4Mucvg0E7PhHNYVloWj1/wOOvy13Hr0ls5XHHY6UgiIiIiIifV3CJ4BrCnie/vBTI8F0e8UUhQAOelJbJiU97pn6E3eBIY435W0E+MSR3Dk5lPknsol6lLplJQfporpyIiIiIiLay5RfAr4LfGmPCj36h7/WDdz8TPZWYk8c2RCjbuP83tj7Gd3VtEc16CmirPhPMCF6ZcyDMXPcPekr1MWTyFb0q+cTqSiIiIiMhxNbcI3oH7LMG9xpgVxpgVuFcIR9b9TPzciPpjJPJO/2ZDb4bSPNi08PTv5UXO6XgOz41+joPlB5m8eDK7inY5HUlEREREpEnNPUfwM6AHcD+QU3fdD3Sv+5n4ufYxYfTtFMPyjR4ogj1HQruufjM0pqGByQN5YcwLlNeUM3nxZHIP5TodSURERETkO5q7Ioi1ttRa+5y19hd11/PW2tKWDCfeJTM9mTU7D3Gk7DQPUQ8IdB8wv/19KNjikWze5IyEM5g2ZhoGw9QlU/m68GunI4mIiIiIfEuzi6Ax5hJjzJvGmK+NMSl137vVGDOq5eKJN8nMSMZl4f0tpzk9FGDQTRAQBGumn/69vFBaXBrTx04nPCicW5bcwhd5XzgdSURERESkXrOKoDFmIvAasAXoDgTX/SgQuLdloom3GZjSjnYRwZ7ZHhqVDH3GwxcvQ3X56d/PC3WN6crMS2aSEJ7A7W/fzqfffOp0JBERERERoPkrgvcCt1lrfw7UNPj+p8BAj6cSrxQYYLiwdxIrNufjcp3mMRLgHhpTfgi+nnf69/JSHSI7MH3sdDpHdeZHy37Ee7vfczqSiIiIiEizi2Av4JMmvl8CxHgujni7kRnJHCytYt3eI6d/s9TzISHNL4fGNJQYnsi0MdPoFdeLny3/GYt3LHY6koiIiIi0cc0tgvuA3k18/wJgq+fiiLe7oFcSAQbe9cT2UGPcq4K7V8L+9ad/Py/WLqwdL1z8Amcmncl9799Hdm6205FEREREpA1rbhF8Dvi7Mebcuq9TjDGTgceBf7ZIMvFKcZEhDOoaxwpPnCcIMOB6CAyFNdM8cz8vFhUSxT8v+ifDOgzjNx/9hlc2vuJ0JBERERFpo5p7juDjwBzgbSASWA48CzxrrX2m5eKJN8pMT2LdniPkF1ee/s0i4qHflbB2FlSWnP79vFxEcARPj3qaESkjeHjlw7z45YtORxIRERGRNuikRdAYE2SMGQc8ASQCZwPnAEnW2t+0cD7xQiPSkwE8tyo49GaoKob1r3vmfl4uNDCUJ0Y8wSXdL+HJnCd5+vOnsdYDw3dERERERJrppEXQWluDezUw2lpbZq1dba1dZa31/+UbaVLfTjEkR4eyYpMHzhME6HIWtO8Hn70IbaQQBQcE88h5j3Blryt5bt1zPP7Z4yqDIiIiItJqmvuM4FogrSWDiO8wxpCZnsz7m/OprnV54oYwdCrsXwf7ck7/fj4iMCCQ3/7gt0zsM5H/bPgPv//k99S6ap2OJSIiIiJtQHOL4O+AvxpjsowxKcaY+IZXC+YTL5WZkURxZQ1rdh7yzA37XwPBkX5/lERjASaA+866j9v638YbW97glx/+kmpXtdOxRERERMTPNbcILgT6494iugPIr7sK6n6VNubctESCAw3LPfWcYFgMnHk1fPkGlB/2zD19hDGGnwz+CT8d/FMWbV/E3Svupqq2yulYIiIiIuLHmlsEMxtcIxtcR7+WNiY6LJizUuNZ7onzBI8aejPUlMO6WZ67pw+5tf+t3H/2/by7+13uevcuymvKnY4kIiIiIn7quEXQGPOuMaZd3ZfdgE+tte81dbVOVPE2menJbD5Qwp5DZZ65YccB0HmIe3toGx2cMrHPRP4w/A98+s2n/PDtH1JSpZlMIiIiIuJ5J1oRPBeIqHs9DYht+TjiSzIzjh4j4cHdwUNvhvyNsOsTz93Tx1zR6woeO/8x1uWv47alt3Gk8ojTkURERETEz5yoCG4EHjbGTAYMcI0xZlJTV+tEFW/TMymSlPhwz24P7XslhMbC6mmeu6cPGtt9LH/L/BubDm1i6pKpFJQXOB1JRERERPzIiYrgHUBf4EnAAo8CzzRx/aOFM4qXOnqMxEdbC6io9tCxByERMOA6+DobSgs9c08fNSJlBM+MeoY9xXuYungq+0v3Ox1JRERERPzEcYugtfZja+1Z1to43CuCPay10U1cMa0XV7xNZkYyFdUuVm4/6LmbDp0KtVXwxcueu6eP+kGnH/Cv0f+ioLyAyW9NZnfRbqcjiYiIiIgfaO7U0O7omAhpwg96JBAaFODZ7aHJfaDrcFgzDVweOLDexw1KHsQLY16gtKaUKYunsO3wNqcjiYiIiIiPa1YRtNbutLaNjnGUEwoLDmR4zwTe3ZiHR/8VGXozHNwG2zWUFqBvQl+mjZmGCxdTFk9hQ+EGpyOJiIiIiA9r7oqgyHGNzEhm18EytheUeu6mZ1wOEQnuoyQEgF5xvZg+djqhQaHcsuQWvsj7wulIIiIiIuKjVATltI1Idx8j8a4nt4cGhcLAibBxIRR947n7+rhuMd2YOXYmcWFx3P727az6ZpXTkURERETEBzWrCBpjIowxKo3SpJT4CNKSozx7niDAkClga+Hz/3j2vj6uY1RHpo+dTueoztz5zp28v+d9pyOJiIiIiI85abkzxgQCR4CMlo8jvmpkRjIrtxdSWlnjuZsm9IQembBmOrg8dDyFn0iKSOLfY/5Nj9ge/HT5T1m6Y6nTkURERETEh5y0CFpra4GdQEjLxxFfNSI9iepay4e5Hj74fOjNULQHtrzt2fv6gbiwOF4c8yL9E/tzz/v3MH/rfKcjiYiIiIiPaO52z4eAR40xiS0ZRnzXWanxRIUGsWKTB58TBEi/BKI6aGjMcUSHRPPsRc9ydoez+dWHv2LWxllORxIRERERH9DcIng3cB6w1xiz1RizruHVgvnERwQHBnB+r0SWb8z37DESgcEweBJsWQqHd3nuvn4kIjiCf4z6ByO6jOCPK//ItPXTnI4kIiIiIl6uuUXwdeDPwMPATOCNRpcImenJ7C+qYMM3xZ698eBJYAysmeHZ+/qR0MBQnsh8grGpY3lizRM888Uzni3kIiIiIuJXgprzJmvt71s6iPi+EelJACzflMcZnWI8d+N2KdBrDOTMhBH3u1cJ5TuCA4J59PxHCQsK49m1z1JWXcbdQ+/GGON0NBERERHxMs0+EsIYE2aMucoYc58xpl3d93oaY+JbLp74kuSYMPp1jmG5J88TPGrozVCa5z5XUI4rMCCQ3w//PTdk3MDMr2fy0KcP4bIup2OJiIiIiJdp7jmCacBG4FngT8DR8ncH8HjLRBNflJmeTM6uQxwuq/LsjdNGQWxXDY1phgATwP1n38+t/W9l9ubZ/OrDX1Hj8uCxHiIiIiLi85q7IvgksBRoD5Q3+P58INPTocR3ZWYk47Lw/hYPHyMREAhDJsP296Ag17P39kPGGH46+Kf8ZNBPeHPbm9zz3j1U1Xq4nIuIiIiIz2puERwO/KXuTMGGdgGdPBtJfNmALu2Iiwhume2hg26CgCBYo6mYzXXbmbdx31n3sWzXMn6y/CeU15Sf/DeJiIiIiN9r9jOCQFMTOroCRzyURfxAYIDhwt5JvLc5n1qXh6dWRreHjMvgi5ehusKz9/ZjN55xI78f/ns+3vsxdy67k9LqUqcjiYiIiIjDmlsElwK/aPC1NcbEAL8HNL1DviUzI5mDpVWs23PY8zcfejOUH4Kv53n+3n7syl5X8uj5j/J53ufctvQ2jlTq729ERERE2rLmFsFfAOcZYzYBYcAsYAfQAbi/ZaKJr7qgVxIBhpbZHtr9AkhI0/bQUzCuxzieGPEEGw9u5JYlt1BYXuh0JBERERFxSLOKoLV2HzAQeAz4F7AauBcYbK3Nb7l44oviIkMY1DWO5Zta4F8NY2DIFNj1CRz42vP393Mju47kH6P+wc6inUxZPIX9pfudjiQiIiIiDmj2M4LW2nJr7b+ttT+21t5prX3BWqvJE9KkkRnJfLn3CHnFLfAs34AbIDBUq4KnaHin4Tw7+lnyy/OZsngKu4t3Ox1JRERERFrZcYugMeZKY0xwg9fHvVovrviKEelJAKxoiVXByATomwVrX4UqDT45FUPaD+HFi1+kpLqEKYunsO3INqcjiYiIiEgrOtGK4OtAXIPXx7tmf58PNMbcaYzZboypMMasMcac38zfd54xpsYYs/77fJ4444yOMbSPCWXFphZ4ThDcQ2Mqi2D9Gy1z/zagb2Jf/j3m39S6apm6eCobD250OpKIiIiItJLjFkFrbYC1Nq/B6+Ndgc39MGPMtcBTwMPAIOBj4C1jTNeT/L44YCbwTnM/S5xljCEzPZkPNhdQXevy/AekDIPkM2D1vz1/7zakd1xvpo+dTkhgCDcvuZl1+eucjiQiIiIireCkzwgaY4KNMbOMMT098Hm/AKZba5+31m6w1t4FfAPccZLf9yIwA/jEAxmklYxIT6a4sobVOw55/ubGuFcF930Oe3M8f/82JDU2lRljZ9AutB23Lb2Nz/Z/5nQkEREREWlhJy2C1tpq4GLgtE4HN8aEAENwn0nY0FJg+Al+351Ae+CPzfiM240xq40xq/PzNczUaef1SiQ40LTc9tAzr4HgCFj+J6ipapnPaCM6RXVixtgZdIzsyB3L7uDFL1+kurba6VgiIiIi0kKaOzV0DnC6Q2ESgUDgQKPvH8B9HuF3GGP6A78FbrTW1p7sA6y1z1lrh1prhyYlJZ1mXDldUaFBnN09nndb4jxBgLBYGP0HyF0GsydDTWXLfE4bkRSRxLSx0xjeaThP5jzJlfOv5NNvPnU6loiIiIi0gOYWwV3Ar40x84wxvzHG/KLh1RLBjDGhuA+uv9tau70lPkNaXmZ6MlvySth9sKxlPuDs22DcX2DTIph1I1S3wHEVbUhcWBx/H/l3nhn1DLW2ltuW3sbd792t8wZFRERE/Exzi+AU4BBwJnAzcFeD68fNvEcBUIt7m2dD7YGm/pTZEegDTKubFloDPAj0rfv64mZ+rjgoMyMZgBWbW3Cr7tm3wWVPwpal8Or1UK3jLU/XBV0uYO6Eufxo4I9YsXsFl2dfzrT107RdVERERMRPNGdYTABwGdDfWtu9iatHcz7IWlsFrAFGN/rRaNzTQxvbC/QHBja4ngVy61439XvEy/RIjKRrfATLW2p76FFDp8Ll/4Cty+G/10JVC61AtiGhgaH8cMAPyZ6QzbCOw3hizRNcteAqVn6z0uloIiIiInKamrMiaIHPOc5zfN/TE8AUY8ytxpg+xpingE64Cx7GmJnGmJngHlJjrV3f8ALygMq6r0s8kEdamPsYiSQ+3lpARfVJH/M8PYNvgqx/wo4P4OWroVL/inhCl+guPD3yaf4x8h9U1VZx69Jbuee9ezhQ2vhxXxERERHxFc2ZGmqBTcBpT1+x1s4Cfgb8GvgCOA8YZ63dWfeWrnWX+JHMjGQqql18uq2w5T9s4PVwxXOw6xN4+SqoLG75z2wjLky5kOysbO4ceCfLdy/n8uzLmb5+OtUubRcVERER8TXNfUbwXuAvxpiBxhhzOh9orf0/a22qtTbUWjvEWvt+g5+NsNaOOMHv/Z21tt/pfL60vnN6JBAWHNDy20OPOvNquOpF2PMZvHQFVBxpnc9tA0IDQ7ljwB3MnTCXszuczV/X/JWr51/Nqm9WOR1NRERERL6H5hbB14CzcT/jV2GMKWp4tVw88QdhwYEM75nI8k35uBeYW0HfK+Dq6bDvC5iZBeUtcKh9G5YSncLTo57m6ZFPU1FbwS1Lb+He9+8lr6yVyr6IiIiInJagZr6vuZNBRZqUmZHMuxvz2FZQSs+kqNb50D7j4dqX4LVJMONymDQPIuJb57PbiBEpIzin4zn8e/2/efHLF3lv93vcOfBObuhzA8EBwU7HExEREZHjMK22QtPKhg4dalevXu10DKmz+2AZ5z++nF9f2odbz2/WoFnP2fI2vDoREnu5y2BkYut+fhuxu2g3j6x6hA/2fkBauzR+OeyXnNXhLKdjiYiIiLRZxpg11tqhTf2suVtDG96sgzGma8Pr9COKv0uJj6BXchTLNzmwdbDXaLj+FSjMhemXQYm2L7aElJgUnhn1DH/P/DvlNeXcvORm7nv/PvLLWvAMSRERERE5Jc0qgsaYWGPMDGNMOe7z/bY3ukROamRGMqu2H6Sksqb1PzxtFNzwGhzeCdMvheL9rZ+hDTDGkNk1k+wJ2fxwwA9ZtnMZ47PHM/OrmZouKiIiIuJFmrsi+BdgAJAFVAA3APcAe4BrWyaa+JsR6clU11o+3FLgTIAeF8LE1+HIXncZLNrnTI42ICwojB8N/BFzJ8xlUPIg/rz6z1yz4BpW79d2bRERERFv0NwieAlwl7V2CVALrLHWPgHcD/xvS4UT/zI0NY7o0CBWOLE99KjUc+GmOVB8AKaNg8O7ncvSBnSN6cr/jfo/nsp8irLqMqYumcoDHzyg7aIiIiIiDmtuEWwHHD30/QiQUPf6E2C4p0OJfwoODOD83oks35TXesdINKXrOXDTXCg7CNPHwaGdJ/89csqMMYzsOpLsrGxuP/N2luxYwuXZl/Ofr/9DjcuBbcIiIiIi0uwiuBU4OupxA3Bd3cHyVwIHWyKY+KcR6ckcKKrk628cPn4y5SyYlA0VRe5toge3OZunDQgPCueuQXcxd8JcBiQN4LHPHuOaN69hzYE1TkcTERERaXOaWwSnA2fWvX4U93bQKuDPwGOejyX+akR6EgArNnnB1sDOg2HyfKgqhWmXQuFWpxO1Cd1iuvHPi/7JkyOepKSqhCmLp/DLD35JQblDz46KiIiItEGndI5g3ZERQ4Et1tovPZ7KA3SOoPca//SHhAYF8PodXrKreP96mHk5BATD5AWQ1NvpRG1GeU05z697nulfTSc0MJQfD/ox16ZfS1BAkNPRRERERHyeR88RBLDW7rLWzvHWEijeLTM9iZxdhzhUWuV0FLcO/WDKQrAu9zODeRucTtRmhAeF85PBP2HO5XM4M+lMHl31KNe+eS05B3KcjiYiIiLi105YBI0xlxhjdhhjYpr4WWzdz0a3XDzxRyMyknFZeH+LF2wPPSq5j7sMmkD3M4P71zudqE1JjU3l2Yue5W8j/kZRVRGTF0/mVx/+SttFRURERFrIyVYEfwz82Vr7ncke1tojuJ8P/FlLBBP/NaBLO+IjQ1i+0cFjJJqS1BumLoLAUJhxGXyz1ulEbYoxhou6XcS8CfO4tf+tLNq+iMvnXs7LG17WdFERERERDztZETwTWHaCn7+L+6B5kWYLDDBc2DuJ9zbnU+ty8BiJpiT0hKkLISQKZoyHvdqi2NoigiP46eCfMufyOfRL7Mejqx7lujev4/O8z52OJiIiIuI3TlYEkwDXCX5uOXamoEizZWYkc6ismrV7Djsd5bvie7i3iYbFwsws2KOhQ07oHtudf43+F3+98K8crjzMpLcm8esPf01heaHT0URERER83smK4B6OHRvRlDOBvZ6LI23FBb0SCTB43/bQo+K6wZRFEBHvLoO7PnU6UZtkjOHi1IuZnzWfW/rdwsLtCxmfPZ5XNr5CravW6XgiIiIiPutkRXAh8JAxJrzxD4wxEcAf6t4j8r20iwhhcNc4lm/y0iII0C7F/cxgdHt46UrY8ZHTidqsiOAIfjbkZ7xx+Rv0TejLwysf5vqF1/NF3hdORxMRERHxSScrgn8CYoHNxpj7jDET6q77gc11P3u4pUOKf8rMSGb93iLyiiqcjnJ8MZ3c20Rju8DLV8G295xO1Kb1iO3Bc6Of4y8X/oXCikJueusmHvzoQQ5WHHQ6moiIiIhPOWERtNbmAcOBL3EXvrl115+AdcB51toDLR1S/FNmejIAKzZ50TESTYnuAFPehLhU+O81kPuO04naNGMMY1LHsCBrAVP7TWXB1gVcNvcyXt34qraLioiIiDTTSQ+Ut9butNaOAxKBYcA5QKK1dpy1dntLBxT/1adjNB1iwrx7e+hRUckweQEkpMEr18PmpU4navMigiP4xZBf8Mblb3BG/Bn8aeWfuH7h9azN17EfIiIiIidz0iJ4lLX2kLX2M2vtKmvtoZYMJW2DMYbMjCQ+2FJAde2JhtN6ichEdxlMSodZE2HTW04nEqBHux48f/Hz/PnCP1NYXsiNi27ktx//VttFRURERE6g2UVQpCWMSE+mpLKGz3b4yB/aI+Jh8nxo3w9m3QgbFjidSHD/pcLY1LHMv2I+U/tOZX7ufMbPHc9rm17TdlERERGRJqgIiqPOTUskONB4/3OCDYXHwaRs6DQIXpsMX811OpHUiQyO5BdDf8Hrl79ORnwGD336EDcsuoEv8790OpqIiIiIV1ERFEdFhQYxrHuC954neDxhsXDjHEg5G16/Bb583elE0kDPdj154eIXePyCx8kvy2fioon87uPfcahCu9pFREREQEVQvMCI9CS25JWw+2CZ01G+n7AYmPg6dBsOc26Dta86nUgaMMZwSfdLWHDFAiadMYl5ufMYn63toiIiIiKgIiheIDPj6DESPrYqCBAaBTe8Bqnnw9wfQs5LTieSRiKDI7n7rLuZPX42veN689CnDzFx0UTWF6x3OpqIiIiIY1QExXE9EiPplhDBcl96TrChkAi4YRb0HAnzfwyrpzmdSJqQFpfGixe/yGPnP0ZeWR43LLyB33/yew5XHHY6moiIiEirUxEUxxljyExP5uOtBVRU++iWveBwuO6/0GsMvPkzWPW804mkCcYYxvUYx/ys+dx0xk3M3TKXy7IvY/bm2bisDxxhIiIiIuIhKoLiFUakJ1FR7eKTbYVORzl1wWFw7UuQfiksuhs++T+nE8lxRIVEcc9Z9zB7/GzS2qXxh0/+wMSF2i4qIiIibYeKoHiFc3okEBYc4HvTQxsLCoWrp0Of8bDkAfjoKacTyQn0iuvFtDHTeOT8R9hftp8bFt7AHz75g7aLioiIiN9TERSvEBYcyLk9E3l3Yx7WWqfjnJ6gELhqGvS9At5+EN7/i9OJ5ASMMVzW4zLmZ81nYp+JzNkyh/HZ43lj8xvaLioiIiJ+S0VQvMaIjGT2HCpna36p01FOX2AwXPkC9L8a3n0IVjzmdCI5ieiQaO47+z5eG/8aPWJ78LtPfsdNi27iq8KvnI4mIiIi4nEqguI1MtOTAHx/e+hRgUFwxb9gwA2w4mF494/g66udbUDvuN5MHzudh897mL0le7n+zev546d/5EjlEaejiYiIiHiMiqB4jS5xEfRuH8VyXzxP8HgCAmHCMzB4Erz/Z1j2O5VBH2CMYXzP8Sy4YgET+0xk9ubZjJ87njlb5mi7qIiIiPgFFUHxKpkZyXy24yDFFdVOR/GcgAC47CkYegt89CQs/bXKoI+o3y562Wukxqby249/y02LbuLrwq+djiYiIiJyWlQExatkpidTXWv5KLfA6SieFRAAl/4Vzv5f+OQfsPh+lUEfkh6fzoyxM/jTeX9iT8kernvzOm0XFREREZ+mIiheZUi3OKLDgli+Md/pKJ5nDFzyGPzgx7DyWVj4/8ClbYa+whjD5T0vZ8EVC7g+4/r67aJzt8zVdlERERHxOSqC4lWCAwO4oFcSyzf5wTESTTEGLv4jnPszWP0ivPlTlUEfExMSwwPDHmDWZbPoFtONBz9+kElvTWJD4Qano4mIiIg0m4qgeJ0R6UnkFVfy1b4ip6O0DGPgot/BBfdAzkyY9yNw1TqdSr6njPgMZlwygz+e+0d2F+/muoXX8fDKhymq8tOBbItWAAAgAElEQVR/b0VERMSvqAiK17mw7hiJFf40PbQxY2Dkr2HEL2Htf2HuD6G2xulU8j0FmAAmpE1gwRULuDb9WmZtmsX4uePJzs3WdlERERHxaiqC4nWSo8M4s0ssyzf54XOCjY24D0b+Br58DebcBrV+NC21DYkJieGXw37JrMtmkRKdwm8++g2T35rMxoMbnY4mIiIi0iQVQfFKI9KT+XzXIQ6VVjkdpeVdcDeM/gN8NQdevxlq2sA/s5/KiM9g5iUzeejch9hVvItr37yWR1Y+ou2iIiIi4nVUBMUrZaYn4bLw/pY2sCoIcO5PYcwjsGE+zJ4CNZVOJ5JTFGACyErLYn7WfK7pfQ2vbnqVi2ZfxK8+/BWf7f9MW0ZFRETEK6gIilca0KUdCZEhLN/ox88JNvaDO2HcX2DTQph1E1RXOJ1ITkNsaCy/OudXvHbZa4zrPo53dr3DzUtu5tI5l/Ls2mf5puQbpyOKiIhIG2b8ckQ/MHToULt69WqnY8hp+MWsL1i+KY/Vvx5NYIBxOk7rWT0N3vwZ9BwF170MweFOJxIPKK8pZ9nOZczLncfK/SsxGIZ1HEZWWhajuo4iLCjM6YgiIiLiZ4wxa6y1Q5v8mYqgeKv5a/fxk1c+5407hjOkW5zTcVpXzksw/y7ocSFc9wqERDidSDxob8le5ufOZ97Weewt2Ut0cDRju48lKy2L/on9MaYN/cWHiIiItBgVQfFJR8qqGfTQUu4ckcbdY9KdjtP6vngF5t0J3c6F61+F0CinE4mHuayLz/Z/RnZuNst2LqOitoKesT2ZkDaB8T3Hkxie6HREERER8WEqguKzrn72Y8qqaln4k/OdjuKMdbNh7u2QMgwmzobQaKcTSQspripmyY4lZOdmszZ/LYEmkPM7n09WWhYXdLmA4MBgpyOKiIiIjzlREQxq7TAi38eI9GT+vGQTB4oqaB/TBp+hOvNqCAyC12+Bl66AG9+AsFinU0kLiA6J5qreV3FV76vYdmQb83LnsWDrAlbsWUFcaByX9riUrLQs0uPb4Oq4iIiIeJxWBMWrbfimiEue+oDH/qc/157V1ek4ztmwAGZPhQ794aY5EN7Gnplso2pcNXy872Oyc7NZvns5Na4a+sT3ISsti0t7XEpsqP5SQERERI5PW0PFZ1lrGf7ouwzo0o5nbxridBxnbXoLXpsEyX3gpmyIiHc6kbSiQxWHWLR9Edm52Ww8uJHggGAyUzLJSstieKfhBAYEOh1RREREvIyKoPi0B+Z8yYK1+8j5zWhCgtr40Zdb3oZXJ0Jib5iUDZEaJtIWbTy4kezcbBZuW8jhysMkhydzedrlTOg5gdTYVKfjiYiIiJdQERSftvSr/dz+0hr+e+swhqep+JD7Drx6A8T3gEnzISrJ6UTikKraKt7b8x7Zudl8uPdDXNbFoORBZKVlMSZ1DJHBkU5HFBEREQepCIpPK62sYdAf3mby8G786tIznI7jHba9B69cB7EpMHk+RHdwOpE4LL8snwXbFpCdm832I9sJDwpndLfRZKVlMaT9EAJMG19NFxERaYNUBMXn3fTiSr45UsGyX1zodBTvseMjePlqiOkIkxdATCenE4kXsNayrmAdc7fMZfGOxZRWl9IlqgsT0iYwoecEOkZ1dDqiiIiItBIVQfF5L364nYfe/JoP7s0kJT7C6TjeY9en8J+r3M8KTnkTYrs4nUi8SHlNOct2LmNe7jxW7l+JwTCs4zCy0rIY1XUUYUFt8EgWERGRNkRFUHzetvwSRv71Pf4woS+TfpDqdBzvsvsz+M+V7iMlJi+AuG5OJxIvtLdkL/Nz5zNv6zz2luwlOjiasd3HkpWWRf/E/hhjnI4oIiIiHnaiItjqD40YY+40xmw3xlQYY9YYY84/wXuvNMYsNcbkG2OKjTErjTGXt2Ze8Q49kqJITYhg+cY8p6N4n5SzYNI8qDgM0y+Fg9ucTiReqHNUZ+4YeAeLrlzEixe/yIiUESzYuoCJiyZyxbwrmLZ+GgXlBU7HFBERkVbSqkXQGHMt8BTwMDAI+Bh4yxhzvJPCLwTeBS6te/8iYO6JyqP4rxHpyXy8tZDyqlqno3ifzoPdq4FVJTDtUijc6nQi8VIBJoCzO57Nw+c/zPJrlvO7H/yOqJAonljzBBfNvoi73rmLd3a+Q3VttdNRRUREpAW16tZQY8xKYJ219rYG39sCvG6tfaCZ91gFfGCt/X8nep+2hvqf9zbnM/nfq5g25SwyM5KdjuOd9q+HmZdDQLC7GCb1djqR+IhtR7YxL3ceC7YuIL88n7jQOC7tcSlZaVmkx6c7HU9EREROgVdsDTXGhABDgKWNfrQUGP49bhUNHDrOZ9xujFltjFmdn59/akHFaw3rHk94cCDLN2l76HF16AdTFoJ1ubeJ5m1wOpH4iB6xPfj5kJ+z9KqlPDPqGYZ2GMqrm17lqgVXcc2Ca/jvhv9ypPKI0zFFRETEQ1ptRdAY0wnYC1xorX2/wfcfBCZaa0/6V87GmB8BjwL9rLU7T/RerQj6p1tnfMbG/cV8cG+mhlucSP5mmDEeXDXu5wc79HM6kfigwxWHWbh9Idm52Ww8uJHggGAyUzLJSstieKfhBAYEOh1RRERETsArVgRPlzHmf4A/AzecrASK/xqRnsyeQ+VszS9xOop3S+oNUxdBYIi7EH6zzulE4oPahbVjYp+JzB4/m9njZ3NN+jWs2r+KO9+5k4tfv5incp5ix5EdTscUERGRU9CaRbAAqAXaN/p+e2D/iX6jMeYq4CVgkrV2QcvEE19w9NnA5Ru19fekEnrC1IUQEukug/s+dzqR+LCM+AzuP/t+3rn6HZ4Y8QQZCRn8e/2/GZ89nklvTWLOljmUVpc6HVNERESayYlhMWuttbc3+N5m4I3jDYsxxlwDzAAmW2tfa+5naWuo/xrzt/eJjwzhldvPcTqKbzi0E2ZcBuVH4KY50KXJ3QEi31t+WT4Lti0gOzeb7Ue2Ex4Uzuhuo8lKy2JI+yEEGJ/ZdCIiIuKXvOZA+brjI14C7gQ+An4I3AL0tdbuNMbMBLDWTqp7/3V1778bmNXgVlXW2oMn+iwVQf/1yFsbePGD7Xz+4Giiw4KdjuMbDu92rwqWFsCNb0DXYU4nEj9irWVdwTrmbpnL4h2LKa0upXNUZ7LSspjQcwIdozo6HVFERKRN8poiWBfmTuBeoCOwHvj50eExxpgVANbaEQ2+vrCJ27x39D3HoyLov1ZuK+Ta5z7lnxMHc0l//QGz2Yr2wfTLoOQA3PAapJ7rdCLxQ+U15SzbuYx5ufNYuX8lBsOwjsPISstiVNdRhAWFOR1RRESkzfCqIthaVAT9V3Wti8EPvc0l/Trw+FUDnI7jW4r3u1cGj+yB61+FHk39PYuIZ+wt2cv83PnM2zqPvSV7iQ6OZmz3sWSlZdE/sb8m/4qIiLQwFUHxOz96OYdVOw6y6pej9IfJ76skD2ZOgIPb4PpXoOdIpxOJn3NZF6v3ryY7N5u3d75NRW0FPWN7MiFtAuN7jicxPNHpiCIiIn5JRVD8zutr9nD37LW8edd59Osc63Qc31Na4C6DBVvgupeh12inE0kbUVJVwpIdS5ibO5e1+WsJNIGc3/l8stKyuKDLBQQH6rlfERERT1ERFL+TX1zJWX9axv8b3Zu7RvVyOo5vKjvoLoP5G+GamZB+idOJpI3ZdmQb83LnsWDrAvLL84kLjePSHpeSlZZFeny60/FERER8noqg+KUJ//iQwADDnDs19OSUlR+Cl66E/V/C1dOgz3inE0kbVOOq4eN9H5Odm83y3cupcdXQJ74PWWlZXNrjUmJDteovIiJyKlQExS/97e3N/P3dLaz59WjiI0OcjuO7Ko7Af/7HfeD8/7wIfbOcTiRt2OGKwyzcvpB5ufPYcHADwQHBZKZkkpWWxfBOwwkMCHQ6ooiIiM9QERS/9MXuw2Q98xFPXjuQrEGdnY7j2yqK4OWrYc9ncOVz0P8qpxOJsOngJrJzs3lz25scrjxMcngy43uOJysti9TYVKfjiYiIeD0VQfFLLpfl7IeXER4SyK/GncGYvu01QfR0VJbAf6+FnR9Bz0wYPAnSx0FQqNPJpI2rrq1mxZ4VZOdm8+HeD3FZF52jOjOk/RAGJw9mSPshdIvppv/7FxERaURFUPzWJ1sL+c289eTmlTCkWxy/HJfBkG7xTsfyXVVl8PHT8PlLcGQ3RCTAgOth0E2QnOF0OhHyy/JZunMpq/evJicvh4MVBwGID4v/VjHsHddb20hFRKTNUxEUv1ZT62L2mj088fZm8osrGdO3PfeOzaBnUpTT0XyXqxa2LYecmbBxEbiqIWWYe5XwjCwI1f+24jxrLTuKdpBzIIc1B9aQk5fD3pK9AEQFRzEgeQBDkocwpP0Q+iX2IyRQzxKLiEjboiIobUJZVQ0vfLCdf723lYoaF9efncJPR/UmKVpbG09LST6se9VdCgs2Q0g09LsSBk+GzoNB2/HEi+wv3e8uhQdyyMnLIfdwLgAhASH0S+zHkPbuYjgweSCRwZEOpxUREWlZKoLSphSUVPL3d7bw35W7CAkK4PYLenDb+T2IDA1yOppvsxZ2r3IXwq/mQHUZJPd1rxKeeQ1EaEuueJ/DFYfJycupL4ZfF35Nra0lwASQHpdeXwwHJQ8iITzB6bgiIiIepSIobdK2/BL+vGQTb63fT2JUKD8f3Ytrh6YQFBjgdDTfV1EE699wl8J9ORAY6j6DcPAkSD0fAvS/sXinsuoy1uavrS+Ha/PXUllbCUBqTGp9MRzcfjCdIjtpAI2IiPg0FUFp09bsPMSjb23gsx2H6JEUyX1jM7j4DE0Y9Zj9X0LOS+7toxVHIC7VPVxm4ESI6eh0OpETqq6t5qvCr761alhcVQxA+4j2x4ph8mB6tOtBgNFfcoiIiO9QEZQ2z1rLsg15PPrWBrbmlzK0WxwPjOvDkG5xTkfzH9XlsOFNyJkBOz4AEwC9xsDgm6DXxRAY7HRCkZNyWRdbDm2pL4ZrDqwhvzwfgHah7RiUPKi+GGYkZBAcoH+vRUTEe6kIitSpqXXx2uo9/G2Ze8Lo2L4duHdsOj00YdSzDm6Dz/8Dn78MJfshqj0MvMG9UpjQ0+l0Is1mrWVP8R7W5K2pH0Kzq3gXAOFB4QxIGsDg9oMZkjyE/kn9CQ8KdzixiIjIMSqCIo1owmgrqa2B3LfdzxJuXgK21v0M4eBJ7mcKg/WHZvE9+WX55OTl1BfDzYc2Y7EEBQTRN6FvfTEcmDyQ2NBYp+OKiEgbpiIochz5xe4Jo6+s2kVoUAC3X9CTW8/vrgmjLaHoG1j7X3cpPLQDwmLhzGvdpbBDf6fTiZyyoqoivsj7or4Yri9cT42rBoOhV1yv+kPuB7cfTHJEstNxRUSkDVERFDmJhhNGk6JD+dlFmjDaYlwu2PmhuxB+PR9qK6HTIHch7HcVhMU4nVDktFTUVPBlwZf1xfCL/C8orykHICU6pb4YDmk/hJToFA2uEhGRFqMiKNJMa3Ye4pFFG1i98xA96yaMjtaE0ZZTdhC+nA1rZkDeVxAcAX2vcJfClGE6rF78Qo2rho0HN37roPvDlYcBSAxPZHDyYAa3H8zQ9kNJa5dGYECgw4lFRMRfqAiKfA/WWt7++gCPLt7ItvxSzkp1Txgd3FUTRluMte7zCHNmwpdvQFUxJPRyF8IB10NUktMJRTzGZV1sP7LdXQzrnjXcX7ofgOjgaAYmD6xfMeyb0JdgTdwVEZFTpCIocgpqal3MWr2bv729hYKSSi7p14F7xmjCaIurKoWvst2lcPenEBAE6eNg8GTomQlaLRE/tK9k37eK4fYj2wEIDQylf2L/+mcMByYNJCI4wuG0IiLiK1QERU5DaWXdhNH3t1JV4+L6s7vyk1G9NGG0NeRvchfCta9AWSHEdIFBN8KgidCuq9PpRFrMwYqDfH7g8/pjKzYe3IjLugg0gWTEZ9QXw8HJg4kL024FERFpmoqgiAfkF1fy1DubeWXVbsKCAvjfC90TRiNCNGG0xdVUwaZF7lK49V3393qOdG8dTR8HQSHO5hNpYaXVpazNW1tfDL/M/5IqVxUAPWJ71BfDIclD6BjV0eG0IiLiLVQERTxoa34Jf168icVfuSeM/vyi3lwztIsmjLaWw7vcB9V//h8o2gMRCe7nCAdPgqR0p9OJtIqq2iq+KvyKNQfcxfCLvC8oqS4BoFNkJ/dqYV0x7B7bXQOvRETaKBVBkRawZudBHlm0sX7C6P2X9OGiPsn6A1drcdXC1uWQM8O9WuiqgZRzYPBN7smjIZFOJxRpNbWuWrYc3lJfDHMO5FBYUQhAfFg8g5IH1R9bkR6fTlCAdjKIiLQFKoIiLcRay9KvD/CYJow6qyQf1r3q3jpasBlCoqH//7hXCTsN1jEU0uZYa9lVvOtbxXBPyR4AIoIiGJg8sL4Y9k/qT2ignnkWEfFHKoIiLazxhNFx/Ttwz5gMuidqVapVWQu7V7oL4fo5UFMO7fu5C2H/qyEi3umEIo45UHqgfippTl4OWw5tASA4IJj0uHRSY1PpFtON1NhUusd0p2tMV8KDwh1OLSIip0NFUKSVlFbW8PwH23ju/W1U1bi4YZh7wmhilP62vdVVHIH1b7hL4b7PITAUzrjcXQq7nQcBeqZT2rYjlUf4PO9zcg7ksOHgBnYU7ag/z/CoDpEdSI1xF8Tusd3rX3eM7KiD70VEfICKoEgryyuu4O/vbNGEUW/xzTr4/CVYN8tdEOO6u4+hGDgRYjRhUeSo8ppydhXtYnvRdnYe2cmOoh3sOLKDHUU76ofRAIQEhNA1piupMamkxqZ+qyzGhsY6+E8gIiINqQiKOGRrfgmPL97Ikq8OkBwdys9H9+bqIZow6pjqctjwpnvAzI4PwARCr4vdq4S9LoZAFXWRplhrKawoZMeRHewsqiuIdSVxT/EeamxN/XvjQuPqt5imxqTWl8WU6BRCAnXUi4hIa1IRFHHY6h0HeeStjazZeYi05CjuG5uhCaNOK9zqPoLii5eh5ABEdYCBN7hXChN6Op1OxGdUu6rZV7KvfuWw4SpiQXlB/fsCTACdIjt9pyB2i+lG+4j2+u+hiEgLUBEU8QLWWpZ8dYDHF29kW0EpZ6fG88C4DAZpwqizamtgy1L3s4RbloB1Qer5MHgy9BkPwWFOJxTxWSVVJews2unealq0s74g7izaSXlNef37woPC67eXNi6KkcEauiUicqpUBEW8SHWti1mf7ebJZe4Jo5f278g9Y9JJ1YRR5xXtgy/+636e8NAOCGsHZ17r3jraoZ/T6UT8hrWWA2UH3KWw7lnEo88l7ivdh8u66t+bFJ7UZEHsHNVZ5yGKiJyEiqCIF2o8YXTisK7cpQmj3sHlcj9DmDMTNsyH2ir3eYSDJ0G//4GwGKcTivitqtoqdhXtql9JbPhc4uHKw/XvCzJBdInuUn/cRcOyGB8Wr62mIiKoCIp4tbziCp5atoVXP9tNeHAgP7ywBzefpwmjXqPsIKx7zV0K876C4Ajoe4W7FKYM02H1Iq3ocMXh+ucQG2413VW0iypXVf37ooOjvzXN9OhrnY0oIm2NiqCID2g8YfQXo3tzlSaMeg9rYV+OuxB++TpUlUBib3chHHA9RCY6nVCkzap11fJN6Tf1BXH7ke31rxufjdgxsuN3n0eMTaVjZEcCjP57KyL+RUVQxIes3nGQhxdtIGfXYdKSo7h/bAajNGHUu1SWwNfZ7lK4eyUEBEPGOHcp7JEJOmhbxGuUVZexq3jXt6aZHn0useHZiKGBoaREp9A9tvt3iqLORhQRX6UiKOJj3BNG9/P44k3uCaPd43ngEk0Y9Up5G93DZda+AmWFEJty7LD6dilOpxOR42h4NuLRknj0WcSmzkZsvNW0e0x3ukR30dmIIuLVVARFfFR1rYtXP9vNU8s2U1BSpQmj3qymEjYtcq8Sbl3u/l7Pke5VwvRLIEhDgER8RbWrmr3Fe5vcatr4bMTOUZ3rC2L32LqhNTGpJIYnEqjdASLiMBVBER9XUlnD8++7J4xW17q48Zxu3DUyjQRNGPVOh3fB5y+7D6wv2gMmEOJS3c8UJvaqu3q7r4h4p9OKyPdQXFVcv3LY8FzExmcjBpgA4sPiSQpPIiE8gaTwJBLDE7/1+ujPIoIjHPwnEhF/piIo4ifyiip48p0tzGowYfSW83oQHqK/dfZKrlr36uDuT6FgMxTkQmEu1FYee09EAiQ0KoeJvaBdNwjU5FgRX+GyLvLK8uqfQcwrz6OwvJD88nwKygsoKC+gsLyQWlv7nd8bERRBUkQSCWEJ7oIY4S6Kja+40DitMorI96IiKOJncvPcE0aXfn2A9jHuCaP/M1gTRn2Cq9a9YliwxV0OC7cce12af+x9AcGQ0PNYQUw4WhTTIEyDK0R8kcu6OFx5mPyyfArLCymoKCC/7FhRbHg1HGRzVKAJJD4s/jsri1plFJHjUREU8VMNJ4z2So7i/ksyGJmhCaM+q/yQe9WwYHPdVVcQD20H17HBFUR1aLTFtO7XmC4QoL8MEPEH5TXl9auIBeUF31pZrL/KCiisOPkqY1MrjEcLo1YZRfybiqCIHzs6YfSxxZvYXlDKsO7xPDCuDwNT2jkdTTylthoO7WhQEI+WxU1QceTY+4LCISGtUUHs5f5eiAYMifgjl3VxqOJQfWlssjCewipjU6VRq4wivkdFUKQNqK518eqqXTz1zhb3hNEzO3LvmHS6JagA+C1robTgu1tMCza7t59a17H3xqbUlcJGzyNGdwCtIIu0CQ1XGZsqjEe3rB5vlTEyONJdGLXKKOIzVARF2pCSyhqee38bz7+/jRqXi4nDNGG0TaqugIPbvr3F9GhZrGqwKhAS7X7u8OgK4tFnEeN7QHCYc/lFxDGeXGVs6kqKSCIxLFGrjCKtQEVQpA1qPGH0jhE9ufnc7pow2tZZC8XfNCiIW469Ltpz7H0mwD25tMkjLxK0iigiQNOrjPll+RRWFGqVUcQLqAiKtGG5ecU8tngTbzeYMHrVkBQCA/QHeWmkqtR9vEXDLaYFue6VxJqKY+8La/ftoy6O/hqXCoHBjsUXEe9V66rlcOXh464s5pfn1w/GaWqVESAqOIqokCiigqOICYmpfx0dEl1/Nfy68fvCg8I1TE3aHBVBEeGzugmjn+86TO/27gmjmemaMCrN4HLBkd1NH3lRcuDY+wKC3FtKE3vXDa3pfezIi/A45/KLiE8pqy6rX1E8eh2sOEhJVQnFVcUUVxVTUv3d102tODYUZIKIColqsjA2+bruvdHBx74ODtBfdolvUREUEcA9YXTx+v08tngjOwrLGNY9nl+O68MATRiVU1VxpNGRF3XbTA9uA1f1sfdFJjVaQawri+26grZ7ichpstZSXlP+rYJ4vMJ4vNel1aUn/ZywwLDvlMSmCmPD1ciG74sMjiTA6JgfaT0qgiLyLdW1Ll5ZtYunlm2hsLSKy87syD2aMCqeVFsDh3c2OhNxi/vIi/JDx94XGNrgyIte3x5aExrlXH4RaXNqXbWUVJe4r6oSiqqKKKlyf3309ckKZWVt5Qk/w2DqVx1Pus01JOpb5TImJIao4ChCA0O1m0eaTUVQRJpUUlnDc+9t5fkPttdPGP3JqF7ER4Y4HU38WWlhgy2mDaaaHtrx7SMvYjo32mJaVxZjOmtYjYh4paraqu8UxO+7Mulq+N/BJgQFBNWXwqYKY1MrlY1fBwUEtdL/IuI0FUEROaG8ogr+tmwLsz7bRXBgAJ3bhZMQFUJCZKj716hQEht8nVj3dUxYMAEaOiOeUlMJB7cfW0UszD1WFCuLjr0vKNw9uTQsttEV08T36q7QGPeQm7AYDbQREa91dItrU6uRjVcmi6uLv7VKefRnZTVlJ/2c8KDw+issMIywoDBCA0PdXzd6HRYYRmhQKOGB3/1Z4/c1/FUrl95BRVBEmiU3r5hXVu3mQFEFhSVVFJZWUlhSxcGyKpr6T0VQgCE+smFRdJfEhKijhfHb5TEsWM+CySmw1j2UpuHzh+WH3M8nVhyBisMNXhcBJ/n/a8GR36M8xtYVyAbvD9KZnCLivWpcNZRWl550ZbKipoKK2grKa8qpqKmgsraSipoKymvLqaw59rqipoLqhs98fw/15bBRUWxYLI9bLhsVy8avj74vNDBUK5wnoCIoIqelptbFobLq+mJYUFJZXxQLiut+bVAcy6qantwWGRJIYnQoCZHfXWU8+nVilPvn7SJCdMSFfH8uF1SVNCiGdVdlUaPvHT5WHBu/9ySTBwkKa0Z5jG1UIht8PyhMW1tFxKfUumqprK2kvKb8W4Wxoqbi/7d350GSlGUex7+/qu6eoYfhHORQuRGRYwHxQDldWVHEAwlFF1fWAEREXV1F8QhR15sFZ3EJREVgVwNCDUFYFGRhVQRREIThEFAOAUFmhmNmYLqnu579483qzsrO6uqeo7Nm6veJyKjK933zzaeycmrq6TfzLYZGhsaeN5PL5uPQSNomX9Zs1+yvWdZcXxn9tf7SJLHdSOWEuk7tsvL+Wv9aN8o5WSLo9NnMOuqr19hs7iw2mzu1kZBnhkeyRHGYhUuGxhPFXLL4l8XPcPODT7J42RCNkr9H1UQabWyTKI6NOmb1c2b548yAWi1LujYAnj/97SPS7ylOSBwnWZ5ZnO5vXP4UPPtk62ypZeoDHZLHDkv/oBNJM5tR9Vqdwdogg/2Da3Q/EcHQ6NB4klhILIvJZMf60eUsXr64tL7Tz42Uqak26Ujl/IPnM7tv9ho4MmuGvzmZ2Wo3ONDH4CZ9PH+Tzv9hNBrBk8+uYNHS1lHFRUuHeDx7XLRsmNseepJFS4dZMjRS2s96/fXxhHHOwNjz5uWq89ZvJpQDbDI4QF/d03dbCSnNVjprfdhgq7RnSdcAABM+SURBVOlvHwEjy1svVZ1w+WrJCOXTD4+37/QX8Vrf9JPHfMI5a64TSTPrSpLGkqsNZ224Rve1orFiSgllyyho7nnzstrm+tLhpWvdJaprV7Rmts6pZfcZbjJngJ0279x++YpRFi8bvzx1YZYoLmquLxvm0aeXs+CRp1i0dJiRsuFGYOPB/kKiOD7KuOmc1vW5s/rWuktBrCIS9K+XlrlbrFwfI0OFS1afnGSEMitfeM94WaffQlNtYiI5MCeNVNYH0j2Q9f7x9bGlP6vLntfzzwegr9i+w3Y1/zHGzKrTX+unf6CfuQNzqw6lMjOeCEo6EfgosCVwO/AvEfGrSdofCJwO7Ao8Anw1Is6eiVjNrPvM7q+z1UbrsdVG63VsGxE8/ewIC3OjjAtzl6suyi5XvfPRp1m0dJinni2/pG+gr5aNMhYTxez53PGEcpM5Awz0+QuurYK+WbD+ZmlZGaMrYGjJxFHIdvdELn8Knn4kbTc6nD0OjT8fGep8uevKUH2SpDOfWDbrZ61k0tlf0ucUt/MfgMxsHTajiaCktwHzgROBa7PHn0p6UUQ8WNJ+O+By4FzgaGA/4CxJj0fEj2YucjNbG0liw8F+NhzsZ4cpfKceHmmMjza2jDLmEsmlw9z96BIWLhtmeKT8t542mN03finqnFmsP7uPukS9rvRYS0tfTdSaj0qPHdvUm21r1GtQzz/mtmvZvmW7iW3axeZR0LVUvR8GN0nL6hKRSxRzy8jwxLJ8AtmSWGbbjwyV99XSXyEZHVraeZuVuN+no1r/NEZIC/W1ehp9VS33vF5YL6urtWlbT4npZH0V209WN7auNvuaaixt6vz5Ydb1ZnTWUEk3ALdGxHG5snuAH0bEKSXtvwIcERE75cq+DewaEftOti/PGmpma1JEsHRoZHz21PxsqtnIYzNxfGZohNEIRhtpGWkEjeZjpMdunMC5JlqTxbHkMSWffbUateajmuutyWtZclpMPluS3WJiXJYQ14RIXzKb3zXV8ry1rklq1rRu16xrlovWRhO3Ucn2rftGJX0X22ryWDu9rtb+2tWV9FMSa36bTqbzB4Kp9znlLsfjLmqMUmusQI0VqDFMPVag0eGxstroMGo014epjTbbrqDeGEajw6ldI3scXdHSvqWvxopC+/H+ao1hNLoCxSgQKEZRowE0Ulk0UDQKj6Oo08+erGVCNYKUSEaWIEaWdMZYWY1QHVB6LJYrlYdqpH9UGnsM1FIW0FKfyvIJqdL+i9uN/fsp7xcxsWysL1q3K/QfufrWeJofEBO3i0J9edwU+irGXXi9LftnQl9j/TXjbdLEshj/gMy924UPl1xZlPbHhHalsZT01xrL1Pobi6VDf+PHdmr9lfW94wHvoNbXXXfedcWsoZIGgBcDpxWqrgRe0WazfbP6vCuAd0nqj4g1cK2KmVlnkpg7u5+5s/vZdt6cVe6v0YiWZHE0gtHRQlk+kYxgZHQ8kSy2Sds1GG3AaKMxSZuSvgtJanN9WrGNbddgtBEMjYwyGrTENFl/xXjb3etpNn01YHa2dJMgpU4N6jQQQZ3GWFkqD5TV1whqSo/1rL71+fh6S19qjO1jvO9iu9z2mhhL2T5a+lKjNc5O+6BBTcV26fU2n9cIxtOddBwYe54vBykYT8OK9ePPaVPeub5D31qVfTfToKnUTyyvyZ+VVVq+75HM7rJEcDIzGek8oA48Vih/DHh1m222AK4qad+X9ffXfIWk44HjAbbeeutVDNfMbObUaqKG6K9XHUn3yifLwNgoahC559ljVjC+Pr4SFLcf36alfa5t/qFYV9pP4btY2TaTxtpsPaHvkn7axJPf92Sx5tc7md7I9dQaT6fPNRHndK6MWjPHyaYipno+MfX3aZ3W/NCLZtI5/hxALSdp5LYplOWOpkpP7InbqmTb8frp9zcWc4f+yvud7msrian0tZX3vcfA1H5mq1usPSnrFETEOcA5kC4NrTgcMzNbjZwsm5mZrT4zObXdQmAUKE4QvznwaJttHm3TfiTrz8zMzMzMzKZpxhLBiBgGbgIOKVQdAlzXZrPr27S/0fcHmpmZmZmZrZyZ/rGr04FjJB0raRdJ84GtgLMBJF0g6YJc+7OB50r6etb+WOAYJk44Y2ZmZmZmZlM0o/cIRsRFkjYFPkX6QfkFwOsi4oGsydaF9vdJeh1wBvBe0g/Kf8C/IWhmZmZmZrbyZnyymIg4CzirTd1BJWW/APZew2GZmZmZmZn1jJm+NNTMzMzMzMwq5kTQzMzMzMysxzgRNDMzMzMz6zFOBM3MzMzMzHqME0EzMzMzM7Me40TQzMzMzMysxzgRNDMzMzMz6zFOBM3MzMzMzHqME0EzMzMzM7Me40TQzMzMzMysxygiqo5hjZD0OPBA1XGUmAcsrDoIs0n4HLW1gc9T63Y+R63b+RztDdtExGZlFetsItitJN0YEftUHYdZOz5HbW3g89S6nc9R63Y+R82XhpqZmZmZmfUYJ4JmZmZmZmY9xongzDun6gDMOvA5amsDn6fW7XyOWrfzOdrjfI+gmZmZmZlZj/GIoJmZmZmZWY9xImhmZmZmZtZjnAiamZmZmZn1GCeCM0jSiZLuk7Rc0k2S9q86JjMASadI+p2kpyU9LulSSbtVHZdZO9k5G5K+UXUsZk2StpR0fvY5ulzSHZIOrDouMwBJdUmfz30XvU/Sv0nqqzo2q4YTwRki6W3AfOCLwF7AdcBPJW1daWBmyUHAWcArgFcBI8BVkjapMiizMpJeDhwP3Fp1LGZNkjYCfg0IOAzYBXg/8Lcq4zLL+RjwPuADwAuBD2brp1QZlFXHs4bOEEk3ALdGxHG5snuAH0aE/wFaV5G0PvAU8KaIuLTqeMyaJG0I/B44FvgMsCAiTqo2KjOQ9EXgwIh4ZdWxmJWRdBmwKCLelSs7H9g0Il5fXWRWFY8IzgBJA8CLgSsLVVeSRmDMus1c0ufDE1UHYlZwDukPaNdUHYhZwZuAGyRdJOlvkm6RdJIkVR2YWeZa4GBJLwSQ9CLSVUCXVxqVVcbXBM+MeUAdeKxQ/hjw6pkPx6yj+cAtwPVVB2LWJOk4YEfg6KpjMSuxPXAicAbwZWBP4MyszveyWjf4CukPvXdIGiXlAV+IiLOqDcuq4kTQzFpIOh3YD9gvIkarjscMQNLOpHus94uIFVXHY1aiBtyYu93jZkk7ke7BciJo3eBtwD8B7wBuJ/2xYr6k+yLiO5VGZpVwIjgzFgKjwOaF8s2BR2c+HLNyks4AjgIOjog/Vx2PWc6+pKsrbs9daVcHDpB0AjAnIoaqCs4M+CtwR6HsTtKEHGbd4GvAaRFxYbZ+m6RtSJPFOBHsQb5HcAZExDBwE3BIoeoQ0uyhZpWTNB94O/CqiLir6njMCi4Gdif9Bbu53AhcmD0fri40MyDNGLpzoewFwAMVxGJWZpA0MJE3ivOBnuURwZlzOvBfkn5L+s/iBGAr4OxKozIDJP0n8E7SZAdPSNoiq1oaEUuri8wsiYgngSfzZZKWAYsjYkE1UZm1OAO4TtIngYtIPxX1AeATlUZlNu5S4OOS7iNdGroX8GHggkqjssr45yNmkKQTgZOBLYEFwIci4pfVRmUGktp9EHw2Ik6dyVjMpkrS/+Gfj7AuIukw0r2sOwMPku4NPDP8Zcu6gKS5wOeBNwPPIV3OfCHwuYhYXmVsVg0ngmZmZmZmZj3G1wSbmZmZmZn1GCeCZmZmZmZmPcaJoJmZmZmZWY9xImhmZmZmZtZjnAiamZmZmZn1GCeCZmZmZmZmPcaJoJlZj5N0nqTLqo4jT9IbJd0jaUTSeTO87zVyPCTNkxSSDpqkzT5Zm22z9YOy9XmrOx4zM+ttTgTNzCqUJR0h6dOF8l5PAL4D/AjYBvhgxbFU6TpgS2BR1YFYd/7RxMxsZTkRNDOr3nLgo5I2qzqQ1UlS/0putxGwKXBFRDwcEU+t3sjWHhExHBGPRkRUHYuZma1bnAiamVXvGuB+4NPtGpSNEEraNivbp9DmtZJukvSspF9Jep6kAyX9QdJSSZdJ2rRkH5+S9FjW5ruS1svVSdLJkv6U9XubpKNLYnm7pKslPQu8p81r2VjS+ZKeyPq6StKuzdcAPJE1vXqySyklDUj6iqSHJD0j6XeSXlNyzGb8eGRtXpLtd7mkm4GXlezjUEl3ZW1+BbygUN/yvks6Jovn7yUtkLRM0jWStitsd0ou9gskfUbS/WXHMbfNVpK+J2lRdjxvkXRwrv49ku6VNJw9HlfYPiS9V9Il2fZ3Szo4O95XZLHeImnv3DbN13N41n559nq2L/Q9lX0fL+kH2X7+XPJ+PFfShdl594Sk/5G0U67+1OyYHpW9r0skXZw79qcC7wIOy/Y36WW+ZmZdLyK8ePHixUtFC3AecBnwOmAY2CErPwgIYF7Zela2bVa2T6HNb4H9gT2ABcCvgf8lJSL7APcBZxZiWAL8ANgNeA3wMPAfuTZfAP4IHApsB7wDWAYcVojlfuDIrM3z2rzmS4C7gAOA3YGfAH8B1gMGgBdlfR0BbAEMtOnne8Bvsn62B07KjuHfdcHxWB/4W6GPO7N4DsraPJ80Gnwm8ELgrcBDWZtt25wHxwArgKuAl2av6WbS6GkztqOyfo8lJZanAE8B909yHs4B7smOzf7ADtnxPzirf3O235OyPt+frR+e6yOy4/R2YCfg+8BjwBXAG7PtLgduzW3TfD03Aq8E9gJ+CdwCaJr7fgg4GtgR+FJ2Lmyd1Q8Cd2fv7R7Z8f428AAwmLU5FVgK/Dhrs29W/83ce3oR8HPSedn23PTixYuXtWGpPAAvXrx46eUl+2J6Wfb8GuDC7HkxAWhZz8q2pTwRfE2uzUlZ2d65slOBBYUYngTWz5UdDQxlCcIc4Flg/0LsXwcuL8Tyrx1e705ZuwNyZRuSEpVjs/V55BKmNv3sADSaX/Rz5RcDZ3XB8Ti+TR/5RPCLpOREuTafonMiGMDOuW3+MYutmThdD5xdiO1KJk8EjyMlv/Pa1P8aOLfk3L02tx7Al3Lru2VlH86VtXs9r8y12QYYBV69CvvuA54Bjs7W301KdPPHuk669/KtufNgObBhrs0ngXvL/r168eLFy9q+9GFmZt3iY8D1kr62iv3cmnv+WPZ4W6HsOcVtImJpbv160ujcDsAsYDbwM0n5e9X6SSOAeTd2iG0XUgJ3fbMgIp6SdBtpJHCq9gYE3CEpXz4LuLrQtorjsUubPvJ2AX4TETFJmzJDEfHH3PojWWwbA4tJo13fKmxzA4XLTgv2yuJd2KZ+F+DcQtm1wBsKZVM51pCOd3NfDdKoLQAR8YCkR0jnw1Urs++IGJH0OOPv64tJI7dLCufLIOk9bXogWu9JfYSJ54aZ2TrBiaCZWZeIiN9K+hHwVeDzhepG9pj/FttuMpYV+W6zvotl07lHvNn2cODBSfYF6fLIlTWdCVFqWfuXlMTwbGG9yuOxJowU1pvHrYr7/ovv2YRj3aasGOvKTIYz2b6b9c391EiXmx5V0s/iKfZhZrZO8YebmVl3+QTpHq1DC+WPZ49b5sr2XI373V3SnNz6y0n3WP0JuIN06eE2EXFvYXlgmvu5k/R/z77NAkkbkO4VvGMa/dxMSoq3KInp4WnGVGZVj8edbfrIuxN4mVqHqIptVsZdpAQ576UdtrkZ2EPtf67kTtI9fHn7Mb33rJ0aufgkbQ1sle1zde3796R7BxeWvGeLO22cM0y6pNTMbK3nRNDMrItExL3AOUz87bx7SROqnCrpBZL+gXQ/2erSB5wraVdJhwBfBr4VEcsiYglwGnCapHdL2lHSnpJOkHT8dHYSEfeQJov5pqT9Je0O/DfwNGlykan2czdpspjzJB0paXulH2P/iKQjphNTG6t6PL5PGrnL9/HJwj7OJt1b+XVJO0s6EjhhNcQ+Hzgmi20nSSeTJsaZbNTt+6TJbS7J3pftJb0hN2vo14B3Snpf1uf7SfcmfnU1xDtCOgb7StoTOB+4nXRZ6Ora9/dIl6VeojRj7HaSDpD07/mZQ6fgfmC37P2ap5X8iRQzs27gRNDMrPt8jsLlf9mljEeRZsf8A/BZ0ujh6vIL0pfva0izJl4NnJyr/zRpMo2PZO1+DryFNOPmdP0z6Z6wn2SPg8ChEVG8pHMq/XyXlBDcRZp99QDSTI+rapWOR3Zv4OtJk+P8npQ4fiy/g4h4kDQz56Gk9/RDwMdXNfCIuJB0afGXSSN9u5GSzuWTbLMMOJA08+alpNlVP8v4pbQXk2br/BBpJO6DwIkRcemqxksaXf0CcAHpXsYacETz3snVse+IeIZ0bvyZNJPrXaSEc2PGf65kKr5FGqG8kTRKXxypNDNbazRnGDMzM7N1lKQfA30RcXjVseRJOgb4RkSsX3UsZma9xpPFmJmZrUMkDQLvBX5GGll+C+l3/N5SZVxmZtZdnAiamZmtWwJ4LenS4fVIv593dET8uNKozMysq/jSUDMzMzMzsx7jyWLMzMzMzMx6jBNBMzMzMzOzHuNE0MzMzMzMrMc4ETQzMzMzM+sxTgTNzMzMzMx6zP8D+PO/QTunw5IAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "carriers = {i: make_carriers(10, 10**i) for i in (8, 4, 2)}\n", - "\n", - "plt.figure(figsize=(15, 7))\n", - "for i, carrier in carriers.items():\n", - " plt.plot(carrier, label=f'denominator: 10^{i}')\n", - "plt.legend()\n", - "plt.xlabel('Number of embedding component')\n", - "plt.ylabel('Carrier frequency')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In fact different frequencies correspond to different \"clocks\" that we have to measure $\\text{pos}$ as time.\n", - "\n", - "Model doesn't know $\\text{pos}$ value directly but it sees all the \"times\" (in fact phases) of differend \"clocks\" ($\\sin$s and $\\cos$s of different frequencies)\n", - "\n", - "Having representative sutie of \"clocks\" we can definetly say _what time is it now ($\\text{pos}$ value)_ for every given \"moment\"" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:51:11.741823Z", - "start_time": "2019-09-26T22:51:11.736491Z" - } - }, - "outputs": [], - "source": [ - "def make_pa_matrix(n_pos, d_mod, denom):\n", - " res= np.empty((n_pos, d_mod))\n", - " carriers = make_carriers(d_mod, denom)\n", - "\n", - " for pos in range(n_pos):\n", - " if pos % 2:\n", - " funct = np.sin\n", - " else:\n", - " funct = np.cos\n", - "\n", - " for i in range(d_mod):\n", - " res[pos, i] = funct((pos // 2) * carriers[i])\n", - " return res" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 2d case" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:51:13.233975Z", - "start_time": "2019-09-26T22:51:12.672375Z" - }, - "scrolled": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0kAAAKvCAYAAAC/E4RrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOyddXgUV9uH75lZi5FAILhrobgWb/ECFeq0hbrL27el+tZpqTtfC/UWKlix4u4SLHiwkCAh7qsz5/tjQ0iyG0hCEkJ77uvqVVg5c2bYOXMe+z2KEAKJRCKRSCQSiUQikXhRL/UEJBKJRCKRSCQSiaQyIY0kiUQikUgkEolEIsmHNJIkEolEIpFIJBKJJB/SSJJIJBKJRCKRSCSSfEgjSSKRSCQSiUQikUjyYbrUE6gIqlevLho1anSppyGRSCQSiUQi+Yezbdu2JCFEjUs9jwsx5OogkZyiX7Ljb4tyLhZCDL1kE7gA/wojqVGjRkRGRl7qaUgkEolEIpFI/uEoinL8Us+hOCSn6GxZ3OCSHV+rfaj6JTt4MZDpdhKJRCKRSCQSiUSSD2kkSSQSiUQikUgkEkk+/hXpdhKJRCKRSCQSieQcAjAwLvU0Ki0ykiSRSCQSiUQikUgk+ZCRJIlEIpFIJBKJ5F+HQBcyklQUMpIkkUgkEolEIpFIJPmQRpJEIpFIJBKJRCKR5EOm20kkEolEIpFIJP8yvMIN4lJPo9IiI0kSiUQikUgkEolEkg8ZSZJIJBKJRCKRSP6FSAnwopGRJIlEIpFIJBKJRCLJhzSSJBKJRCKRSCQSiSQfMt1OIpFIJBKJRCL5lyEQ6EIKNxSFjCRJJBKJRCKRSCQSST5kJEkikUgkEolEIvkXIiXAi0ZGkiQSiUQikUgkEokkH9JIkkgkEolEIpFIJJJ8yHQ7iUQikUgkEonkX4YAdJluVyQykiSRSCQSiUQikUgk+ZCRJIlEIpFIJBKJ5F+IFG4oGhlJkkgkEolEIpFIJJJ8SCNJIpFIJBKJRCKRSPIh0+0kEolEIpFIJJJ/GQLQhUy3KwoZSZJIJBKJRCKRSCSSfJSLkaQoSl9FUeYqinJSURShKMo9xfhOW0VRViuKYs/93muKoiiFPnOToij7FEVx5v7/xvKYf1mTkZzJvo3RJJ9OBUAIgahgyz06LpElWw5yIDaBbQfj2LT3OA6Xu0LnUF6kZORwJjWzVNc0JT6N9KSMcpjV5Y8Qgn2botm8YAeZqVmXejqSfMSeSWXR5v1EHojDMLy/eyEEh08kcjA2Ie81ycWTmeNk15FTnEpKv9RT+dcihGD32v2snraB+JiESz2dywan28PaqKMs33aIjGwHdqebhNTMSrs+ZOY4WLf7GDsOnbyoOeoencglu1jx+zoS4pLKcIb/TIxL+F9lp7zS7YKBPcAvuf+dF0VRqgBLgTVAV6AV8COQDXyc+5mrgD+B14FZwChguqIovYQQm8vhHC4awzD4v//8xMLvV2C2mnEYBtYbOxAfoKIbgia1qvHiHVfTqWX9izqOI8dJQlwy4bXDCKoSWOA9u9PN05//xb6YeO9nXR5URcFmMSEEvHn/UK7p3Pyijl+RnEhI469VUcSnZNKqUU2W7TjEwbhEVAUiqoYw/v5hXNm41gXHObT9GO+N/ZL4Y4kIIWjeqTEvT3mKmg1rVMBZVH5OHo7nhSHjyUjKRNUU3E4PY9+6jVufHVnisVKz7Hz/92ZW7DxMoNXMrf07cHPfdigKnEnJxGI2Ua3Q7/afhBCCg1uPcHhXDLUaRdDxmivRtNL5p3TD4M0fFrN8WzSa6h2jWpVAxt1xDe/+uoyMbAcKEGCz8P6jI+jQvG4Znsk/m20H4vhh/mZOJKRzZZNaPHBdD5Zui+anRVsxmzTcHp12Terw4aMjCQm0lnh8l9PNT6/9yYJvl+PIcXJF9+Y88cV9NG3fsBzOpvIghCA2IQ1VVahXPZRCvs8LkngimXED3iDldBoooLt1Bt7Vl6e/eQhVLd59ZBiCNVFHWbj1ACZV5bpebejWsn6J53IpOX30DPMmLyP+WCIdrm7DoLv7EBBkK/Lz26NP8MxXcxACBAKH04MqBGZVI9Bm5tk7rmZIj1YVeAb+SUjN5KOpK1m16wg6YNJULCaNoAALE/8zika1qrHgu+XM+2YpTruLvjd157Zx1xEcFuR3vJh9cTx560dk2zS0LCem2FRufGIYD75/12X17y2pHCjlHdFQFCULeEII8dN5PvMo8D5QUwhhz33tf8CjQD0hhFAU5U+gmhBiUL7vLQMShRB3nG8OXbp0EZGRkRd/MiUgMT2L8R/OZOuWQ2gn0rAcSSbjuivRQ21wdoOUe+lv7dWWF8YMKPENLIRgyjuzmP7JfFRVRffoDLq7L49/OhbNpAHw7i9Lmb9hHy6Pnu+L5/5oNZuY9vZY6tYILTD2ztX7+PmNGZw4dJoGLesw9o2badfnihJfh7JkfdQxXpw4D49u4NYNMCs+wpWBVjOz37mX8Cr+F1CA9KQMxrZ4mpxMe95rqqpQrU5Vfj30Zd61uxDp2Q4OnUwkIiyEBhFhpTmlMsft0flpyVZmrt2N0+WhT7smPHl9L2qEBRd7DCEEY1s8TfyxhALROWuglXfmvUD7/m2KPVa2w8Wtb/5CYno2Ht3rN7JZTHRuXo8TcSkkpWVhCGjVMIJ3HhtOrfAqxT/ZywCXw8Ur131A9LajGIaBpmmERYTy8fJXCa9dtcTjzVy1i0+nrcbh8uS9pqkKQuDjeQ2wmpn3/gOEhQSUau7xxxNZNmUtaYmZdBncjq5D2pfauKvsLN8azevfLcKZe11VRUEzq2BScLrPrZ1mk0qP1g357IkbSnyMt279hC2LduKyu/JeCwi2MWnHB9RqFHHxJ1FJiE/KICPbQeO64USfSOT5b/8mNSsHBERUDebDh0bSvG71Yo/35FUvEx15BEM/53e2BVl5/Iv7GXrv1Rf8vhCCl75bwNo9x7A7vdkTARYTHZvUJTkxi/QsO93bNOShUT2pXb1yrj87VuzhtVEfo7s9eNw6tiArYRFV+GrjO1Sp5ru2251uBj83iRxHoWwRIVAMUPCuw5/950Y6t/LvpHVkOzm2N46qEaHUalQ+zkOH082ol34gKSPH29C00B6oRmgQfc442DRvB84cJwBmq4mIBtX5JvI9rAGWAp/PcbgYeN/HOIJyXzcEqt1FxNJo/vf94/S8rmu5nIc/FEXZJoToUmEHLCXt21vEwgXFvx/Lmrr1Tlfq61RZhBuuAtaeNZByWQy8DTQCjuV+5stC31sMPFEREywJe2LieeizGdjtLmgWDg3DULrWR6gaSv5NhgIIwZwNe+jZrjF9OzYt0XEWfL+CGZ/8jTPn3EN32ZS1BATbeHDCaIQQ/L1pf0EDKe+43j/qhsHfG/fx0HVX5b29ZdFOxo/+Emfuw3zPhmj+d/1HvD7tP3Qe2LZEc7wQbpcHk1m7oIGoGwZvfLvo3OZQ8T78Ci+qumEwb8M+7hla9GK4bMpaPIWuiWEIstNz2Lp4Jz2Gdz7vXIQQfDlnHVNX7MBi0nDrBlc2qsmnD19HcIDXw1xRHqvtB+L4fvYmTpxJo1XjmqR6nOyOicfp9l6nhVv2s2lfDDPfuIeQgOJ5v6O3HSUtId0nfdGZ42TO/y0ukZE0b8M+UrPseQYSeKOZ6/fGoDkFZ6/S3mPxPDxhGrM+uC8vQvJPYOqE2RzYchhX3mbFjcvh4qMHJzFh/oslHm/ayl0FDCQAvYi0FMMwWLzlALcN6Fji42ycv50JYyai6zoel87SKWtp2aUJ7857HpO5cjw2dN1g65Io4g6eokGrOnQZ1K5URpxhCD6aujLPQAIwhMCj6yAK3sduj8GmfbGkZdkJCy6+8Rkfk8CWhTvy/Q5yx3N6+OvLhTz68dgSz3vZjkNMXriJM6mZXNGgJk9d35vWDWqWeJyyIiUjhxc/n8v+mDOYNA2BwGkVOPOttbEJaTz4yXQWTniAAIv5gmMmnUzm6K6YAgYSeDfwc75aWCwjafvhk6zdfQx7vvRyu8vDhv3HvWuQgEUb97N251H+eHcM1UvgUCothiHIcjgJslkuuN4ZhsEH936dZySA9/yTT6by54dzeXDCaJ/vrNt9rMjxhAKK8K7DP8zf7NdImvXVIn56cwaapuFxe2jesRGv//EfQquHlOAsvbhdHmZ+sZCFP61Cd+n0u7k7o1+4nqDQQJZsOUiW3eUVDvDzyMyyO1m18QBKvnN3Oz0kn0plzq+r8TQPx+7y0OfKxrSoW4OnPpqOI8R6bl+ggWGykdy1PvO+XlKhRtLlgkB4DVSJXyrH0w5qAScKvXYm33vHcv9/xs9n/OZWKYryEPAQQIMGDcpsohdCCMEzX8/xLsha7o1q1hCait/foaLg8hhMW7KdhHWHOHkontbdm9F3VLc8L4kQgoS0LGwWM6H5wut/fjgXR77FA8BpdzF/0jLuG387qqrgdhcykArh0Q0yshwFXpv0/NQ8Ayn/uJOen8rk7e8V80qcn82LdvL1uKnEH08kIMjKDY8N5q6Xbyxyk3P0ZDIut8fvewXm6dY5kXj+2oFTR84U8OaeRXcbJMQmX/AYf2/Zz++rduLy6HkG6K6jpxn1+k9kJzpQFejXuRnPjbmGquWYRrYq8hCvfbMwb3MXn5qJ26YUeNjohiDL4WLehr2MHtCpWONmp+egqP6NvIykzBLNMTI6zmdTD4AAoYKSu/cxDEF6lp3IfXF0v7J06UeObCdul4eQqkVHEYuDy+1h8fZoNh44Ts2wEG7qdSX1qpcuUrjk59U+G2PdYxC1eh+ObAe286TL+MPhLH4dodOtk5yeU6LxwZsW9sF93xRYAxzZTg5uPcry3zcwZEzfEo9Z1qQlZvDfQeNJiU/H5XBjsZkJr12VT5f9jyrhJdvkpmfZych2+L5RhJ/DpKpk5jhLZCSdiD6N2Wry+S143B4ObS96Q1sU09fu4pNZa/Lurc0HYrn/6DQmPnojsSdSiI1P5YomtejfpRnmYkbGL5bnPpnNweMJeHQDl1tHN4GhKj7X0aPrrNx5mGu7XTgzwZHtRC3imZA/E+B8rCtkIJ1DeNcg3btO2p0ufl+0nSdvL9/f96z1u/ly3nqy7E4sZhNjB3TmwaHdi3SsxR9LJCs92+d1t8vD2llb/BpJdqe7WDW6/ursti7exU9vzijgfD0QeZS37/yCjxa/csEx8yOE4LWbP2Hvhui89WTON0u9z/9N73AoLtEb3VPwcXgCXuPY6mtMp0UE8sHOPWj7NHTd4NuFm7muR2u2xSX4XkdFwV2vKtn7U0o0d4kEKo+RVOYIISYDk8GbbldRxz0Qm0BSRo7vA1ZV8ObEFHpdeD1ZO9cfJHZTLC6Hm5XTNzH1/bl8sfI1olPSeG3KYpIzszEM6Ni0DhPGDiO8ShDpRWxYXS43bocLW5CNji3qsv3giYL2Wb6/BFjN9GrXON90BCcOxfsdN/bgqSLPe8fRk8zfvB/dMBjauSXdWzYoctHfvf4g79w9MW/RzMl0MPPLRTiynTz8nu+Cf3aeBWQqi/gXDbCa6disTpHzBLiyV0uWTVmDvZBxqGoKLbtcOJr367JtPht/j26QlGXHrHujI6siD3PweAJ/vn9PuURGhBB8PKWQ91sB74UpeN0dLg+7jp4qtpHUqlszdD/GtTXQQu9R3Us0z3o1QjFpaoFIUu4ZoBT6NzSE4ExKyYwwgLSkDD5+5Ht2rNwLQN1mtXj26/tp0anxBb7pS47TxZiP/uBkcgZ2lxuTpvL76h18/MBIerVuVOLxPEUY9gKvsVRSruncnD+W78BdODrshwCrmc6t6pX4GAe3HvFrIDhynCz/bd1FGUke3SDH6SLYZkUtwhAvDv83bgrxx5Pyfqf2LJ34mAS+eWEqz3/3cInGCgyw+DeIDMCPfWG1aNQpYVpWvRa1cTt9fwsms4lmHUv2O/XoBl/OWe+zBjlcHh75dAaBLhWH00OAzczkGev57o07CC2BQVcajp9O4fCJpAL3uShiPXJ5dJL8bPr9UadZLQJCAnBkF3QGmq0m+t7co1hjBNssmDXVm6J9Htweg8gDccUaUwjBjpX72LhgB0GhgQy8oyf1mvmvhTUMwdY9x4ncF0diTjYLdx/Ki/S7dRc/LN2Koig8ONT/2moNtPhE0s4SGOzfydL9igbouv+H5Nl1V1UU2jfzrVmc8eXCAgYSeOvADkYeJeFEMhH1wv2O64+D246yb+OhAg4Xt9ND4okU1s3eStN61Qmwmslxuf1mhhgCbMk55P/XNywamf2b5NWnAeiGhxlro4qeiAJ9brmq6PclkiKoLHkt8UDhPIGa+d4732f87+gvEZujYop+0xBeQ+ksuX/WPAJTTEqel9GR7STpZAqfvPknj06cxamUDJxuHbeus+3wCR76amae2IA/atSthjW3sPjFOwcQFGDFYvZ92gdYzHRsUZfurc957hVFKTKkXjUi1O/rX8xdx6MTZzFr425mb9rLM9/O463flxXpyZry7mzfSFWOi7+/X+nzMDxLvYgw6keE5Rleec/ffJhNKhFhwQzq0sLvGGfpPao74XWrYbae8xFYAyy06dmSll0vbCSl+/M6kzuh3DXeoxskp2WzcVfMBccrDdl2FymFowRFGI4Wk0bjWtWKPXZgSACPfDwGa6Al73pbAy3UaVqTofddOL0lPzf3bYepkCdYVbwP6MLzFQLaNLmw6EbB7wheGP4BO1buxePW8bh1ju8/yQsjPshTkzwfC9fs5YanJtPrrk+45b/f886UpcQlpeV5nj26gcPl4ZWfF6EbJTdqet3QDZOfe6/xlfUJCi15lPHea7sRUTUYW26qkklTsVlMtG9aB5vl3O/ZZjHRpnEturYqeRTdZDYVee9abBa/r18IwxBM/HsDfV78P65+ZRID/jeJOZv3lmosIQTr527zMeQ9bp21s7eWeDyr2cSIXq2xFkojtGkagVYz5tzf79k6jhdHX1Nix0etRhF0HdoBS6EaCrPVxKinhpVorJTMnCKNZDcGjlxjzO5wE5+YweQZGy44ZlpiBgt+Ws2cycuJP55YovkApKTn+N7nRdwuZk2jfZPzO7LyxlBVnv/pCayB1rz7yBpopXrdcG4dd32xxhjWrVWRAg9KvsuoKAr1i1FbahgG4++eyFt3fcW8b1cw/fOFPNbrdZb+tt7ns26PzpMTpvPiZ3P5dd4W5kbuzzOQzuJwefhpWWSRam7htavSrGNjn4iaNdDKdY8N9vudmtVCuH94d2wWU0G7Q5w7V5vVxP0jfQ2z1Hj/mRiaWSM90deJFZOQyo/LtvLT8khOJKUVeC962zEMP+umI9vJvs2HGdK9FQFWs/d5AAX2RzaLicdH9SSsSmCBc3fVC0Xxsz5516xCe6zcMc1ug+seHOj3vP71CNAv4X+VncoSSdoIvK8oik0IcXYHOgg4BcTk+8wg4MN83xsEXPgJUIGYNQ3VAEMt5BURgppulZoNq7P7pFeSUjEgUNXQUzOxHi+4uOTUCGZ+VhqeQNUnfep0Sga7Y+J56L07eW7QeFx2V96mxhpo4bFPx+ZtbhvXCWfWO/cyc/UuDhxPINBmISPLgUBwbY8rGNStpY9H9/ZxI33C7dZAC3e8cJ3P+R5PSGXqqu0FipvtLjdzNu4lcssx+rVrwpjruxMRfs7wOnHYv12rqArJ8WnUbeo/r/7DJ6/jkfenk5njBCFwGzotGkWQnuPE5dEZ1KUF91/bDcsFaibMFhNfrHub3yb8xeppGzFZNIbedw03PzPivN87S8/WjZi7aa9vLUjuGn0Wl1vn2Klkends4jOGYRgs+Gk1cyevICfLwVXDOnDn8yMJq1E8D7XNavaJ0ChnNTUL7clNmsqoPu2KNe5ZRjw8iCbtGzLv6yWkJqTT6/quDB7b36dQ9kLUqxHG50/cwOs/LiY1y44hBG0a1iQ5MZOk1Oy8zZ7NYuKqto1oWq9kBaT7Nh/mTGwSHp8Ns4eFP6/hrheL3kjNXbmbT39ekeeRPxGfxhF7Boaf6IHL4+HwqWRa1itZAfO9b97K9mVRpCdl4sh2Yg2wYLKYeO7bkkU7zlIlyMafb4zh7037iDwQR90aYYzq25aa1UKYv34fs9fuxjAMRvRsww1925YqWtOiSxNsgVbsmQWdAbYgK9feXzIj+SwTF2xgyqrtedc6JcvOu9NWUCXARp82jTmVkEZwoI1qxTQcRZF1WKV76j47+mqcbp2lmw9iMqkYQnDv8O6M7NOGqUu3s/VgLHXCQxkzpAttm9Qu1TFemvIUP736Jwu+W44920HrHi144vN7SyzaEBpkK7KCQCm0H3XrBss3RzPungFFjrd2TiQfPvI9iqogDMH3r8/gjmeHc8dzxVsPAZo3qOFT56nooAoFRVPzHAw2i4mOzerSrgTXsMvg9kza+SHzvlnCmZhEOg1sx8C7+55X2S0/daqH8ubYwbzxy5I849bhdKN5FIx8qR1Ws8adwy5cP7554S62rdib59DT3Tq6W+fLZ36h5/COBZwfc1ftZu/h03m/e0PxjawBOF0eHG43gVb/6+urvz/NuMHjCyj89b+lB0Pv7V/kPB8Y0Z2ureozZ/1eHC43EaFBbN4dQ0qmnQ7N6/DoqN7Ur+krHtN1cDtOHjmDp3CatICGVxQ0br9esIHJS7Zg5O4/Pp+7lqev6809A7y1PxH1w9HMGhRKM7UGWKjVyBtF+ul/o3l/ynI27IkBBapWCaR98zqMHtiJDs3qMmRZY96+43OORsWiagqW8GBcNjNOP9E1xeCcB1XJzd5B4cUxg7AWsyZXIslPuajbKYoSDDTL/esG4D1gLpAihIhVFGUC0E0IMSD386HAQWAVMB5oAfwEvCmEOCsB3hOvRPj/gNnAjcBbQO8LSYBXpLpdzMlk7njpZxyBFFgLNUPht5fvpEXDCE4kpDFvzR6SM3Lo3LwuX934Kbrr3APGsJrIvKYFzlANw+q7oAbZLLw+ehCDO7bg6O5YpoyfyeGdMdRrXps7X76RNj1bXtQ5CCH4/f05TPv4b3SPjmbWuP25kdw2bqRPCt3UVdv5fM46b3Hu2bcEIAQmOwQ6FQIDLPz6/hgiqnkNpVdv+oStS3b5OHxsgVamHf/qvBtx3TDYduAEyenZtG1am3qXQFXuTGomt787hRynG5dHP6u/gckhUPM9VwJtZt5+bLhfI+nTJ39i1awteYaoyaxRJTyYB966hejtMVSvU5UBt11F1YiijabPflvFrBVRBVLurBaNuo3CORLvra1qVLMab4wZROuGJYvQlDVCCE4nZxBgNVM1JJDMbAc/ztvM8q2HsFg0RvVvxy0DO/p4oy/Est83MPG/v2D3E4Hsd3N3XvrhkSLnM+Kxb3yicY4QBWH2vedsZhPTXr6bBjVK/ntz2l2smbGJ/VsOU69FbQbe2cevIlVlInrbUV4a8T66bmB4DIQQDB7Tlyc+G1tiURK3rtPnhf/D7qc2rXZoMMS7cLl1PLpBhyvq8vaTIwi9gCLfG7d9xuZFuwqkIWkmlauGd+LVqU+WaH75ych2kJyeTe3qVfKidZWRD6av4q8Nuwum3Akw5Qi0Qpc5PDSQvyf6vw8yU7O4s/U4n1opa4CFjxe9SLN2xY9E/jRvCz/O3ZxXN2c2qVQJsnH94PYs23EIVVW4oeeV3NyvHWatYuqk8pPjcLE1Og6TqnJFw5q898My1kcdQ1UVggMsvHTPIPp2unAmwYT7vmH1zC0+rweG2Hj26/vpNfKc8M/9r/3G3iOn8/7uDFEQJt/7JzwkkGXvPnTee0sIwd4NB0k6mUrLLk2o3aR8RDrSEjJ4pPsrZKZl5xlK1kALj7x/J9fmyyQ4cjqJm9779awdkjtJ759fHt6PUde0RxGCsW2eJeV0WgEHRmCVAH7a/VGBrJWze9GirkHy6VScOS6Ca4cy8KXJvtFUITBlA4ZAqAKhKZg0lZv6tmPcXddUuPz35aJu17adWcy5hOp2TevHV+rrVF5GUn9gpZ+3fhZC3KMoyk9AfyFEo3zfaQtMBLoBqcA3wFsi3wQVRbkZrxHVBDgCvCKEmHWh+VSkkfT7J3/z3cyNZDYMQ5gUhAaKW9CpahiTvnzA73deHPkBUesO5j3wnY2q4biiFu4gFXeggtDyFb8aYNNUZr0yttTF5MXF7fKQkZxFaPXgIhWtZq7fzbvTl+PxF+LOEpjt3kjGDQPa8ey9Xm/moR3HeHbIuz6RqlufGc5dL5VcWvdSkJyRzdQVO4iMjqN2tSrsioojI92eF10yaSp1aoTyx/tjfVJzzsQm80C3V3xqFBRVRTNreFweLDYzqqYyftpTXHmV//RBj0fno19X8ve6vXnHGDuyG/eM7IbD7cGjG8VWtLtcidl/kqf7v+WTvmkLtHDvm7dw/cP+UyzcHp1+Yz/zMdQ9FnAHqQVEzTRFoXGtasx8ZUxZT79S47S72LJwJxkpWbTvdwX1mpcsgpJtd4EQuAyDq1/5puBmCvIir4FJ5wwdk6bSqklNvn3Lf23iWZJOpfBU/7fIycjBnuXEFmQlOCyIL1a/TnityiHHX554dIPPZq9h5trdGEIQaDUTbrZxOi6tQITbYta4dXBHnrjDfx3Zsj828NVzU33SnBVVoUWnxqQkpKOoCgNv68ltTw+9YCR53Y6jTF0YSWqmnV7tG3PXtV3KVbzmYsnKcZJldxJRNaTYUdePH/2eZb+v91k7AkNsvPD9w3Qf0j7vtYfe/J2o6HO1vLoJ3MFKgSwTm8XEK7cNYGT31hd3MmVIWmIGM79cxLZlu6lepyo3PTWM9n3PCW1E7j7Oi78sJNFp9w2MCYHNqVBbC+Lb8aPRM+28d+/XHN4RAwrUbhzB8989QvOOjUo9v5lrohj/x3Lyp26oTjDlGFhjU9HS7ZhDrDz57HWMvPnS1CJJI6l4/CuNpMpGRRpJtzV/hvTkLDxVrLhqe6MAltOZhOiCWbFf+vVmJJ9O5bkhE0hLysTQDXIaViO7aTi6pmCvnvv5vJxdqBUWzJI3HqyQ8zkfGVl2/lqzm0+W+OZiIwSWNIEpd+/asE41/vj43ry39246xLcdV2oAACAASURBVOSXfudI1HHCqlfhtudGMOKBivf2lBVJaVl89MsK1m4/iqoqXN21Oc/edbVfj/i6edv45PEfycn0p6hV8PzDa4cxZc8H570u2XYXyWnZRIQHV2rvd3nxxu2fs2PFXpy53nDNpBJWowrfRr5LYBERCSEEwx/9htSMgpEkAYgaJuxnm4kAFlXj56dvo80llFe+nDidmMH4iQvZffAUAmjasDo77EV0vReCwMSCzyCrxcTPE+6mYZ3z19G5HC7Wzt7K8f2naNS6Lr2v71LqmqnLFZfbQ6bdq7SXlJrNg2/+QVaOE7dHx6SpNKkXzlcv3UKAzf+6sGTqOv7v+d99VFJRFVSTlue4s9jMNG1bn08WvHDZrtEXy+mYRKLWR5N6Jp3f3p+Ly1HQMRNYJYA/Dn+GJZ8a2/zVe/jo5+V5dWLgNZSMKhqKWaF6SBDP3tCXQR3PX0dbmViwei8ffbeMlCAd4SfTBSHQHILATIVenZrw/jiv4zM9KROPWye8dtk4MV7+ah4rdh3BYxioblB1AR6DKuuOojo8qIrCt5vfLjJ9v7yRRlLxqOxGUmWpSfrHkJXhlSU1ZTgxZZwrgHUo3nx5TfMTaq9dle92vMeOlXs5E5tEYL2qvPnbajJNHu+uLb+HS4GkzGy27I+l2xUVJ21emBkLt/PVlDW4ghQIwK86lGFWwOXdAEUUkuVt06M5n698rQJmWjFUDwvmvad8a7b8UaNuNf+1E342H9npdmIPnqZhq6ILnYMCLASVsFbockUIwfZV+1n911Y0k8qAW6/ilV8eZ/pnC1j442qcDhc9hnXgntdvLtJAAm9KxwM39+TLqasKbGC0QA2nWYH8vZcVwSfz1vD947eU56n9I3B7dB559XeS07LzfuPRMQkQofpdI/wV15hNGkmpWRc0kiw2CwNu71UGs758sZhNhOdG+WuGhzDrk/vYsOsYpxIzaN6gBp2uqHdeo6broLYY+lTfNxSlQCqjy+Hm2L6T7Fp7kA59W5X5eVRmhBBMemUaC39dh6IqqKqKYjFjMgSaSfFGoAS88ftTBQwkgGF9WrNh5zE27DyKbhhoJpWMYAOTTcNjGKS5HHw4bw1tG9emVljJexBVNB7d4ItfVuFweTCZwG3B73NLc3n3Oxu2H0UIcV5BqNIyfGBbojNTOR6fipHpwRyfReDBRFSn7p2TqrB46nrue21UmR73n4eCXlS/A4k0ksqalh0bsW/LEZ/XG7Wu57cHUHamnZkTl7Jm7jasNgvD7+lL3wHtuC0lk+/WRvrVH/R4BM98OIsPHh9Jr84la0BbFuw/Es/EqWtxuXXcLsCm+F0oz+Yj2Kwm7hrxz2viJoTA7vJgM5tKVCDfomMjajeqQezBUxeUgRZCYLbI2xS81+LTp39h7ZxtOHJcKIrCyplbuf6Bq7n31RsZ/XzxjNSzjBrYHk1T+G7GBpJSs6lVvQqmOjYOnikY9XDrBlExp/l1/lbWbzqMosCIa9oytG/rUjUv/SezLvII2TnOAk4AYXiVzvwJYpjd+Tpb5+Jye2jWsGQCGf9Glkcd5uslmziTnsmV9Wvx9PBetKobQd/OzS785VyqRoTyyITb+eblP7z1Z7qBoql+nThup5vonTH/OiNp06JdLJ663qduq2qNUO567loCqwTQfWh7AvzIcWuqyrtPj+TAsTNs3x/H2mPH2RpzEkeuwt1Z9czX/1jCpEduqpDzuRgSkzPz+hWa7eAOhjyRBPA+8wWYzrawKqeo45cL1jNlzXZvnaMGSjAQrhGcL4XfMARzv1vJ2FdukOu0pNTI3VcZ88iE23l+5Ie4nB7cJgURbCbAKXj8Q98ce5fDzTPDPiA+NimvPmXyazPYvfEQL3x9H6c9Oczdvj9POeYcAt1h8ObnC5j//aMXVHMra+Ysi8KVqyZmcoDLr7aAQojQ0GwqT9zZl27tGhU5nsvp5pf35rLw13U4c1y06dGMx9697bzRk0vN8qhDvDd7NYkZWVhNJu7s05HHh11VLGlgRVGY8Nd/ee+ByezddBhFUzBbzLic7gJ1SorijTrVbiw3jAAHIo+xZs62vFo2IQTOHBezJ69g8OiepUqruP7qdlx/dbs8b+dNH/zi93Me3eDbGevRs72/++hjCazdepgJ467/16Yf+eNUQnre2nAWBbCkGXhqmDCEwBACs6ZiM5sIdIJdc+WpNNqsJm4f1vmi+voIIUg5k441wEJwKWTWLwf+3LCLj+asydtsrz8Qw7ajJ/j1ydtpWbdk68W19/ajQ79WrJq5BZfTA4rC7EnLfVLwLFYzNeoWv5XAP4UFP6/FkePbfNyR46Rph4a0LEY/tlaNa9KqcU2+eXUrLr3g/WEIwcaDsQy6+3NaN6vN43f3o0U5iTJcLFWCbXn9lxQBAckCR5iCMHlfU91gTff2CtRUhd6dmpT5+hifmsnPq7blNXIHECYVd3UbjnohBMSdkyl3Od24HW60oH92be7FIPB2p5H4RxpJZUyLjo34aMmL/GfSbM6objRFQTNrbM5O5cpCn10zdxuJJ1MKbIyddhcbF+wk7lA8j4/oxZI9h8hx5vNgCa+CmuoGTLA3+jQd29SvkHM7S1a285wSjQGWVIGrKigoWCwaKPDfEX3p0rguDWpXvaARN+Gh79i+an+epy5qfTT/HfEh36x5jRp1fCVKLzVbDsXx4tRFeRuUHJebX9dsx+nxMO76fsUaI6xGFd6b8xwZKVnYs5yE1wljwv2TiVy2ByEEmknDbDXx6i+PlvghoxsGu2JOk+Ny07FRHYL+IXUam5dE4bL7blYAIpfvvajc87PX+KqWDYlJSPVpPKnrAk+OkZeU4HB62BoVy97o01zZsvIa8xVNy8YRmM2aT/PgENXMfUN7cTA5mWNnUujYtC539euIYsDPszezYecxQoMDuGN4ZwZeVXp1zp3rDvLpf34lLTEDQwg69G7Fc1+OJTS8cqsJFoeMTDtpmXYiwkP4/O/1eesPeDc6DpeHLxeu56sHSi5+U6dJTUaPGwl4m3vP/3EViv1cyxlFUbAEWOg1vGNZnMplRWFRmLMoqoLL4b9ZdFEULVEvyHa4idwdy6Ov/sG3E0bTpEHlc44FBVrp3705q7ccwuXWUXUITBaYrSrBATacDjcOw4PNZqJKcADPPeArnBN/PInUpEwat6qDrRTGy4Kt+/H4ycAQZg1H/eACRlJ4rTCsgf+M55/k0iCNpHLg+517iLcJhEfBg1eF7Julm6lTLZRhHc9tAHatO+jXQ6WoCge2HWPQ7Vfxw+O38ODn08n05EpF28GWKs7JTpsqPozcv0dzNu44hv2s1KsTTGcESqDCcw9czcCOzakSWLweFqdiEgsYSGdxOd3M/W4V9792Y5nPv7TEJ2aQkJzJl4sLblAAHG4P0zZE8cSwngSUQDyhSrXgPDnoV39+lMNRsezbfJiqEaF0H9rOJ8f9Qhw8lcij3/5FttOFqih4dIOXbryaUd0Lm+iXH7ZAC5pJ8+mJpKoKtsCy8RTee00X5kfuJ9PuzDOUTKqKlurJ61R/Frfbw459cdJIykenNg1oVDecI7GJ56LNJo0a4SHcOqADJpNvzt2z9w7g2TI49smjCbxx99cFNrU71h7g1dFf8cXiF8vgCJcGu8PFO18tYn3kEUyaijApOGr4aaYJ7I49c9HHCwyx8dH853n/4e84ccjb065R67q8OOkBLEUIQJyP9BwH6w/GoCoKvVo2uuwUN6++qRvRO48XUGMFQEDLzo1KNNbg9i2YvXVvQSeCEKhO8tYXp8vDD9M2Mv65kqUPlwWZqdnsizxKSFgQrTo38tuE96WHB6PrBuu2HcGkaQgED9zSk1uGdWL99iMci0umYd1q9OnctMD9npGSxZv3TOJwVCwms4buMRj74khufLjoHl6Fyci088u0jRjBRsFabQBDeOuRcrEGWHh0wm0y0i+5KKSRVMbkOF0s2XWoQCgYvJvo71dsLWAkRdQLx2zRcLsKpaeoCtVqhQLQun5NJtw6hDe+WIDD6SlQXme1mmjdrHSNDS+G/t1b8NeSXRw4cga70+3t3m3WePCmXozq1bZEY504FI/JrPkYSR6XzqGo42U57VKTY3fx6sdz2bEnDrNZ43Q13adhK3i9rcmZOdQLDy31sZq1a1Ci3iT58egGD02aSUqWvcDrE/5aSZv6NWlZp/J5JktC/1Hd+P2ThVDISBJC0HN4hzI5RnhIEDPG3c2PKyJZfyCGiNBgmoeEsWjhHhwUNIzNZhNhlVjeuCSkJmawa300gcE2OvZtVeo6OFVV+Or1W/lhxkYWrtmLYQgG9mzFg7f19GsglSVzf1iFp5DzQnfrxB6K58ieOJpeWbER97Li7c8XsGnHMdxuHbdbRyjgrqb5FcKoXbVsiuMbtarD16tfIzUhA0VVCCtl0f38bft5Y/rSvJoQwxC8e8dQBrVrXibzrAgG3n4Vy6dv5sjuOBzZTjSzhsmk8uzEe0rsxHpqeC+2HI4jKTPbmyFiAAICUs4ZTUII9h/x33C9PJn+1RKmfLQAk0VDGIKQqkG8++eT1G1SsNmxzWpm/DMjScvIISU9h7o1Q7HmOgb7dW1Ov67+/23HP/AtB3fEoLv1vOf9z+/No16zWnQd0KZYc1ywai9KjgFBvu+ZTRo9atYkobaLgCtr0nxoG9yNQvHoRon77/3bkMINRSONpDImw+5ELcJzcSYts8Dfh97Zk1lfLy1gJKmqQlCVQDr0OVcc26drM4b2ac2iNfu88pqaiqIqfPBC2RckHtlzgsN74qhVP5y2VzXz60kyaSqfv3oLKzdFs3JTNMGBVq4f2I42JeyjAlC3aU2fyAB4m6s2qySbmgkTF7F9Txxut47LraMFq3hsis8mRVEgItTP6l1BbDkch9PPtXR5dH5fu5MhzZtitZhod0W9y/KhUathdZ7+5C4+/+8UNJOKgoJhCF767gFCqpbdda9eJYhxN/RjHN7UyYxMO4sX7/X5nKrA1T0uH+neovjzi8VM/WQBJpOGoihoJpXxvz9Biw4NSzVegM3M43f15fG7/PfmKS9OHknwK4SiaSoJJ1IuSyMpNT2HjbkG0lkUAZYsA0+ISv6ztZlNPDq4R5ke/3zNrC/E6dQM3pi+1NtoPJ/T8KXfF9GpcV3CQyrWwRCblMbq/UcxqSoD2zajRpXipWCaLSbe/+sZNi2OInLZHkKrhzB4dC/qlKJWNCwogL+eH8OKPYeJOnaaWfN2oGQaPlHqehXc62vnuoNM/WQhLqcbV26GiCPHxf9GT+SHjW/4jcaEVQkstpMo8WQKB7d5DaT8OO0uZn69rNhGUkxcMi6XTmAC2CPy97NTuK5dK8bdcTX3fzOD3WdS2LE/mr+iDxMWGMCvT95W7H9viSQ/0kgqY2pUCcKsaT7pWAiBJ92FYYg8JbQadavx5pTH+eCxH8lKz0EYBg1a1OaV7x8qYPwoisLzDw/ithGd2LYnjirBNnp3aYqtkBcr8VSq1xscYqNL/9YlSo1wOd28ff+37N50BEXxHrNazVA+mPk01fw8KE2ayqBerRjU6+KUjuo2iaB975bsWnewQDTJbDVx3QNXn+ebFUOO3cW6rUcKdPe2pRlk1VQLNgQ0m3hoYDcspkt3S2XYHfjTVDaEYP6qPWyctx8hvA0mP3xlFFeUwqgtLYfjk1h38DhBVjOD2jYnLKh0hfnX3NKdboPbsmPVflRNpfPVrUuV114SqoQE8PHLo/jfx/NwON0IINBm4d1x1xF8mRcE79l8mN8/W4Tb6SlQG/nqnRP5bdcEtHKO/pQl7Xo2Z/fGQz5RabfLc1kaSACp6dmYTVoBIwnAliLQAszkBOQqiJpNPDuyD/3aNLkU0/TL4l3RfkSHvMzetIcrqoZTvVowzRtHlHtK1LcrtvDN0k0IvM+2j+av4fWbB3Jd5+I1cNVMGr2GdyyTmiyzSWNIh5YM6dAS44yLxWv24XSdu/esFhP33lKxDVDn/bDap/ZKCEFaUiaHo+Jo3v7i2o2kJ2dhsmh5Blh+UhPSiz1Oq2a1WLb+AHanm6CTBroVUCBImLi5Zzu+WrSB6NNJeZk8Lo+Ow+3h1T+X8M2DUgrcHwIZSTof0kgqYzRV5crgamzMOeWNNChKnixmQBps2HWUM247bl2nX6vGtOvVgl93vsupY4lYrKbzqgc1rBtOw7rhft/79cO/mf71MjRNQ1UVFFXhnd8eo2Uxu1pP/79lRG08XGCDER+bxMf/+ZV3fnu8JJegxPzv+4f4/q1ZLPl9A067myu6NObx9+8got6lV1LKynGiFMp91lwQdMar1qUEaFQPCeLBgd24sXvxvGHlRZcm9XwEBwAwQM0yyM7xPjhy7PDMWzOY8/2jWMtZXlwIwbuzVzJr614Mw8Ckqrw/dzWfjRlJ71aNSjVmcGggfa7vXLYTvQDtr6jH7EkPcygmEUWB5o0iSiT77o+s9Bx2rovGZNbo2Kcl1kvQ62rRlPU+TTHBm+4atfEQHftcPnLPw+7uw+zJK9E9el5EyRpgod8NXSrFWlIa6taqir+G7yZVYVir5jzz4ADScxyEhwQWS1mzInG4POj+pMTdHr79Yz1hDg1dF9SrHcYnr99M1bDyicJHn05i0rLN3ohWPt6csYxeLRsRHnzpUmaffWAANquJOUuj0HWDqqGB/Oe+a+jQumKN+sy0HL+vq6pCdqbd73sloUGL2gg/vwWTWaPrgOLXyw7ucwU/Tt+Ay+39bZmcXqdfq2Y1ad28Fo9Mne1T6qAbgs2H4nC6PVgrWAlYcvkjfzHlQKhuJuiMgSNURZi93adtaQJ3iMqT0+Z7i28FfDhvNU8O6ck9/br45P2WhF3ro5k5aYXXG5yvbuK1Md/w2453iuUNXjR1g48HVvcYRG04hD3bSUA5eswtNjOPvnsbj757W54Uc3HJcblZfzAGp9vDVS0alvkDr3rVYEICrSS7CkYGLW6FYU2a8fp/RpTp8S6G6lWCeGhgd75bsQVH7nxNqgpOHXN2wQeUYQg27zhG3+7lWxuw8VAssyP34cyNrJ414v7763zWvPEItsvooaVpKq3KqHv7kmmbmPjy9Ly0QYHgte8fpEOvik3fy8l2UoSz37dQvZITEhbIV8te4tcP/2bL0igCgmxc90B/RtxTsWl/ZYnVYuLBO3oz+be1eU2PVVXBZrMw9uYeWM0mIkIrZxpR39aN+X7lVp+sCsMANVMnOzc6FhOXzOsfz+eLt28r0+NnZTuJO5XC3D0HC2QCnEVVFFbuPcLN3UtWR1uWmEwaT997DY/f3Q+7001woLXYz7/sTDsHtsUQEGylVSf/IgvFpdfw9kTvjMFp990DtOrUqNTjnsViM/Pgmzcx6dUZeRErk8VEcGgANz3mq4BXFIEBFr7/4G6++nkV67cdxWxSGda/DQ/e3htFUfwa5eDNppAq15LScPnsUC4j+nZvzo69cZjOnHs4GCqkVhWgiwLe/i+XbKRni0a0qF291Mdb9NsGvzKlHpfO7k2H6dD7wpK6blfRUqYelwcqKK2oJAbSpkOxPPXzXBRFQQiBxzB4bkRfRvcsmyJ+8G5Ixj0yiNc/nY/L5fEqCmoqATYzD97Ru8yOU1Y8PKg7nRrXYdrGKDLtTvQUN/u2xPkE04UhyC7UB6WsEEKQlJKFzWpmTuQ+7C7fFAtFUdh8OJZ+V1Se9KCK4sTRBCa+PN3HKfHmvZOZun08gX6aUpYXfUd2Ysfq/T4qmx6PTturLp/i+rNUqxnK0x+NBnz70pWWpMxs9p1MIKJKMK0ugfjJrSM6U6dmKFNmbSEpNYuOV9bn3luuonZE6QViKoLW9WpyY7c2zN6y12soKd6WEeZ0Ay3fT9+jG+w5eIqUtGyqlUE0SQjB5ClrmTZvG2aTSkqQjh7m+1wRnE+Su2IxmTRCSpDaunDqeia9PgvN7BVZCKpiY/yUx2jYsnQp1EPu6MWiqRs4HZOE0+5CURUsVjMPv3VzmSmHDrurN3UbRzDzm+UknU6lc//WjHpkwAVFQbLSczgVk0RE3aqEVQ+hetVg3ijCOdmjSX2W7jlcsF5YCMxuBZNUuSsSQ8hrUxTSSCoHBvdrzV+LdnL8VArOs3n+oRpmk4rbKJgO5fHoLNh5gBa1S7/h9icjfpbCG7Gi6Dm0HUumbfYprKzXtGaZFsWXFTlOF0/9PJecQhvwT/5eS5fG9S7K6CxM767NmPjW7UydvYUT8Wl0aF2P0dd3pUZ42ShJlTVdm9WnazNvusbaLYd5Kyoee+EooWHQqe3F5Zn7Y/vuWCZ8sZCU9ByEIVCbFR3ZqywblIpmxcyt6H4824qisGnJbq4Z1bXC5tJnZEcWTV3PwZ3HcWQ7UTUFs9nEQ2/dRFCV0jd0/ScghOCD+av5Y1MUFpOGbggaVg9j8v2jih2xzs6wYxiCkLCLi3D37tqM3l2bXdQYl4KXbriawe1asGDHAVRVYdPSaNJTsn0+p6kKOXZXmRhJc5dEMWP+NlwuDy4XaDpQRfORjBZC0L/15eekORQVy6Q3ZuF0uCF3XbdnO3n5jon8svWtUok52QItfPb3OJbP2MzGRVGEVQ9hxL39aFlK8ZaiaNerBe3OEy3XDYNjCSkEWi3UDgvhu/FzmP/zWkwWE26Xh15D2/PMx6OxWP1vXWt7bKgegaHh/fc2BIqA0BSFyF3H6dH58vv3llxapJFUDlgtJr5+9w4WrtrLyg0HqRIcgK1uANN2+ipkGULg1n03TCWh73Wd2Omn55Lu0bmyR/EerGOeH0Hkqv1kpmbjyHFhsZrRzCrPfnbXRc2tvFi9/xj+HENuj87cbft4bkTZpti0alaLty9B34qLpWfnJrRpUZs9B0/jyC2atVnN3DqyEzWrl165yh8nT6fywviZeWlBAPqJbNRaKkbhlhaGQffmZW+kXQ7kZDn8qrAZhnFeh0d5oJk03vnjCTYuimL9wp2EhAUxdHRPmrSpV6HzqIzM27Gf6Zt34/LoeXUOh+OTeXbq3/z08C3n/e6ZuGQ+eOpXonfFAtD4ijqM+/xu6jcrm3TNywVFUejStB5dmnp/T5+nKMxevNOnGWhAgIU6NctG0e2P2VsKrEEmJ1jTBM4wUDUVVVHQNJVxI/pW2lTF87Hg1/UFRFbOYs9xsnfzEdr1LBgBdus6hiEuWI9jDbBw7d19uPbuPmU63+Kyct8R/jd9CS6Pjm4Y1DBZMf11FMPpwZV7vhsWRxESFshj42/2O4Y9y0VIrIE7WMFjFWhusGQKNKtGRpajIk/nskEKN5wfaSSVE1armRuGdOCGIR1wu3VufXwyehXfBmhWs4nBbS+uDqHvyI4snbaJ/ZHHcOS40DQVzazx+ITbip26E1Y9hMmrXmHlrEj2RR6lXrOaDLn9qlL3xyhvnB4P/gIRuhDMXrKTLXMOcO2AK7ljVDesFhNnTqQw+a2/iFx1ALPFxKBbujH2+eHYLkGxfEWiaSofvXozK9YfZNm6/QRYzVw3uD2dyyGKNHPBDp/Nj5plYMlWEWEaLo+OKVdY5N3bhxJYgqa7/yR6DG7L4t83+hhEhiHo3P+KCp+PZtLoPaIjvUdcvHLXP4lf1u7AXqiexmMY7Io9TVJmNtVD/Ec93C4P/73hM9KSMvOipYd3x/HsjZ/x86Y3yrW+s7Iz5uburNoYTWaWA6fLg6oqmM0aLz4+5KKFUM6SnuErNBCYbBCYo3LTnZ0JCrQypF0LGlSvWJntsiItOavIKHxm+jkBhtRsO6/PWsrq/ccwhKB9g9q8ddMgmkRUPhGTI2eSee63BQXq1064sjH1Cqf2/JN5W3iXw83iPzfx8Bs3+q217tmlKeu2HMae5caSde51XTdoLx0/klIgjaQK4Lep60k7k0WAU8FeQ83Ll1WEQrd6dWnf8OKkmDWTxttTHmPr8r1sXOT1tAy6vQcNW5RsXFuglWF39WLYXb0uaj4VQa8WDdENf0puAj3RRXyOkykzN7N9dyzvPn8dT4/8hIzUbIQhcDnc/D1lPUf2nuCDaU9W/OQrGJOmMrjvFQzuW74b8BOnUgt2ksf7Uw9PUxg9sjuZmk6wzcqwDi2peRl6cMuK9j2b03VAG7Yu34sjx4WiKFhsZm56+BpqFkOFLTo+ic8WriMqLp4aVYJ4+JpuDG134bpDScnIdPj3PGuKwsJVe9gVGYvVambkkHZ079Q4r55y09I92LOdBTayQniNp9VztzP0joqVd65MVA0L4pcv7mHekigidx2nds1Qbh7eicYNyi49ul3remyIPOIjSFIzKIinh/cud7nx8qbnsHb+M0fcOld2bwp4HS5jJk0jNikNT+5zcufxU9z59R8sGncfoYEVV/dYHH7fuMtXXENV0AM0XOFWrMnn6mc9bh2X00OAHyOp/1XNmTF/G0eOJ+ZFE21WM6Ou7VjmmROSfwfSSCpnVs3dxm/frkAPDyQgDSw5Os4QBRQFS5ZOty61yuQ4mqbSY3Bbegz2Veo5lZbBlPU72H8qkTZ1I7irV0dqhVbOCFFxqVElmCeH9GTiko04PV5BBcUAc5bAlON9OrpcHg4cjuenb5bjyHEWkCB1Oz1E74rj8O44mrW9PHuoXApiDp5m2Ywt5GQ5uWpwWzr3a5mnqtTxyvps3x1boOcHgMdjMKRLK2pV8kLzikJRFF6cOJbIlftZNWcbFquZQbd2p03XC+fLHz6TxJ3/9wd2l7dfU2qOnVemLyEhI4sxvStWFv2fTv8rmvLnxl0+daS62+CXXzbk1Ztu3RHDDdd24LF7+wPeVDt/6VCOHBfxscnlPu/KTkiQjdE3dmP0jd3KZfxHxvRjx544nE43uiFQFLBYTPz3kUGXvYEE0O+6Tsz/aS3Ho+O9IgsKWGwWRj89hNBqXufTlqNxxKdl5hlI4E2rcnl05mzbx5g+nS7R7P1zKi0DvQiZTU+ghjXfklRgMQAAIABJREFUbVOzXrUio7Emk8YX429j4Yq9LFt3gCCbheuHtucqWYtUJAIFncrVPqAyIY2kckTXDb5+/S8MXYeqAaApaC4ITBaAIDDAQqMG/vselRX7TyUwZtI03LqOWzfYHnOS3zbu5M6eHbi5azsaXqYpBwD39utCt6b1mbttH9v3xBK3JxFzjiiQXat7DPbsO+EjbQreFlbHDpyWRlIx+XvKeiaPn4PHrWPoBivnbKNTn5a88vU9qKrKiEHt+HNuJB7dQM+NKNmsJq7uJQ2kwqiqSrcBbehWzE7zZ/lq6UbsbncBOVuH28NXSzdxe4/2l7SZ8T+Nh67pxpLdh0jPseP06GiqgqoohJw0zgnyAA6nm1nzt3PjtR2pXTOUZm3rY7Ka8BTyjAcEWWneTq41ZUVKQga7Nh4iMNhGx94t84r5G9UP58dPx/LrzE3sPXia+nWqctdN3Su0eXZ5YrGa+XDW06ycFcmaedsJDg1i+JhetMtVo8yxu1i366jfTAuH28PhhMpnqPds1pDNh+N85OKFpqBXD8VIcKG5dCw2E4+/c/56QIvZxPVD2nP9kPblOWXJvwT5RC1HkuPTcTpcaHY3isdAKMq5miQhCK0SQO9u5atYNH7OigIKcB7DwGPAj2u2MXXDTh7s341HB/Qo1zmUJ23q1aRNvZr8Zd7B17tW4aDgIms2azRoWIPT1qN+u33Xa1Lxkr6XIxmp2Ux+e3ZeAS14PePb1x5ky4p99Bh4JSHBNr7/ZAw//rGB9VsPExhgYdS1nbhxmKx1uRiEEBiGQNNUouLi/fY1EkLw/+ydd3gU5fbHPzM7W9IrkEIPJfTQe+9NaUpRVLCiXPV6vZaf7dr16rVXsKAginQVFFA6hF5CKIGQBAikkd62zczvj0Ag7AYQdlPn8zw+Puzu7ByW3Zn3vOec7/dcdj5WuWTouWVIHZfNeNRWgrw9WfnEdH7ecYjo+FOEB/hhO13EttjjDq8VRZG9B08xZlh7OvRqTqMWISQcOVtaUdIbJOqEBdBj6PUbZ2qUz4+frOPHT9YhSToEoUSQ4bXvHqRlh5JZy7AQf55+ZEQlR+k+DEY9w6f2ZPgVrZvLV+3n8282InsJWENUriwQeBgk2tWveuIh47u2Yf62faRk51+qKCkqxmwVHTqUlvXoWz+YKbOHXvemZmFeMcVFFoLq+dWICqI70STAy0dLktyIl68HilxS2fA4cR5LuC+yX4msrh8CX/z3DqS/4Y3wd1FVlYNnUpw/B1jsMl9t2s2AVk1pFXbjZrZVgSH9WjF3/ha4LEkSBAEPk54HZg9l37K92Ky20gWmpNdRP6KuS4zyagMHt59Ap9fBFW1E5iIrm387QI8hJYu/oABvnpw1jCdnDauMMGsUdlnhmx+2svy3/RSbrTRqEIRfUyNpFDi81irLzPxmCbnFFgTA06Dnvamj6dJEG1a+GXw9TNw3sCv3DSyRZZ/z/WZ26kSH2TtRFPC5IJIjCAJvLZrNjx+u4a8lu1EUhf63dubOf464LmNvjasTuyuBRZ/9VWKeftn16MUZc1m46z+19jOOPXqWL77diMVqR7WCzk/E7nFpY1YnCviYjIzuWPHiMNfCy2jg53/cwdin55KtsyEqKqYsFUNeyQ1bMUk88vZkAq5DIj4/p5B3nljI/q3HEUUBvyBv/vn2FDr2qVijbo2agdaI6Ea8fEz0HtEOg1FCkBVMp3PwOpRCUHwmTz4w5Lp+8DeKXVZYve4QghNtg8ux2mVWHTjmtjgqCh9vEx+/OYWmjYLR63XoJR2tmofw6VvTCK7rx3vLH6N1l6YIooCk19FnVBRvLnxY22G6TvRGyalIqCAKmDxrtkJgZfHeZ2tZ8steioqtqCoknc4k80AmxisWgQZJByKk5hZQbLVRZLVxvqCIh75bQVZBUTnvrnEjjBrSzqkPjSgK9Ohyae7B5GFgxjNjWbDnFRbue40HXxpf632nrpeMlBw+eOZnpvd6lUdGv8f6FXtRLyuf/v5jNFazE/N0m52YnScrMtQqxbLf9pXOgwqA3ykFU5aKKINR0tEyrA6KAL1f+5w7v1zEoeTUyg34Cvw8TdQrMhCQIOOXpGDMu6x1/m9Y6r0wYy77tx4vFXjIOJfDyw98TfLJdHeErVHD0SpJbuaxtyZjNdvZvekoer2ELMtMmT2E/mPd14KkqirPv7ac/QdPYwiSMQeKDtLjF1FUlaQzmVgsNozG6i3JHNG4LvM+mkFWTiGiKODve8nAsWHzEN5d+iiyXUYQhVKxAY0S8rIL2bByH+nncmjduTE9BrcusyPbqU9LcJImGYwSw27rXoGR1g7y8otZu+EItivMncVcO+31dYg3FlBkKVkoRjUOIyY5FdsV1Q1ZUVh18BjTe1etIe3qTP2wAJ59fCRvffRHSTujWuKL9/aLEzEatNvpzZKdkc/sMe9TmFeMLCucT83l4+eWcup4KjOeGg2UGKeWM+PvdPa0tpCVU1TmcxFU8E5T8MqXaDs6gr9OJpRK2u87dY575i7mx1lTaRHiOmXBG0FRVDbGJbDq4DGUliZUayFC7qXrniBAk4ZB17WpnHjsHElxqdivuG7arTIr523hkVcnujz+6o7mk3R1tKu6m5H0ElMfG86khwZjMEqENwnG5Olen4yYw8nsjzmN2WLDKwVkg4DNm5KrzRW/BUFRObI5iYnrP+f9NybTPKLq9Sv/Xa7m2l5bWzGuxvGYMzx75xfY7QpWs43VC6MJb1KHdxc9XPpdNZj0/Ofr+3hp5lwQQFVUZLvC1NnDiOzoWld2DUhNz0Ov1zkkSaoKcqqFzZ8+SGZhET4mIz/tPMjeU+cc3sNil8nIL8Rqs5Ofb8bPzxPJSRVE4xJHz6VzNiePVqF1CQ9wLhk8qE8kvbpGEHv0LAaDRJuWYU6rSxp/nxXztlBUaC4VfgEwF1tZ8e0WJj0wEB9/T/qNiWL/tuMOEth2u0K7brVXxaxP92YciTtXRlQEwCIrrIs/ieUKIRGLXeaz9Tv4YNqYigyzDKqq8uSi1Ww6nkix1YYACA0EvLwlTOfsmIx6DAaJF568vhjTz2ajkxx/i7KskJyoVZI0/j5akuRGtq09xAfPLEa+oPYV3jiYl764x+1J0oFDJfKnULKb5JckIxugOFiHrU5JkiArKqgqxvMKco5CPjaee205i755UGtBq0Woqspbjy2gqOCSD4W5yMqZ+DSWfrWJOx69NFvUrnsEC/e8wu4NRzAXWenUtyVB9TTVOncQFuLvoI4GJW1dzZvWRRQF6lwwM+3SOBxJFLgin8LToCfteBZjvvwI9ULF4/67+3LraE1I40pyisw8MG8ZJ9Oz0IkCNllmeNsWvD5xGDonVWeTUU+XqMYVH2gNJyY6HrvV8XuvN0gkxqXQvnsEfUd1YM2inRw7cApzkbXEkNYg8eCL42p1S+PoYe345Y8DpKbllbbdmYwSYyd05LsThx2SJEVVOXquchOHXYnJbIpLpNhWsl5RAVUAc7DItF5diKxfl/69WmC6zi6XiDbh2KyO8vsGo0T7Hu4VydKomWhJkps4HZ/GO0/8iMV8qfx/6ngqz0z/kq//etqt7V5+vh4YDFKZHSWdFYKzRKaN6sGS6BhSMvIw5ilIl/kl5uUVk5CUQUSTqinioCgqX2/Zw/fb95FnNtM2PIRnR/enbfjNe01ZLTaWzN3EuqW7URSVAWOjmDJrcLl+DDWF9LPZZKblOTxutdhZv2JfmSQJSmYt+o6Kqqjwai3eXkbGDu/AqrUxpaaIAAa9jum3l1W0als/hD4tGrP1eFJpO41JLxEgGtm/4WTpdcBqtfPpVxvw8/NkQB/NfPZynlu6hrjUjDIti2sPnyAytA73aP5TFUZIwyDiYs6U8bQDsNlkgi9syOgkHa99/yA71sUSvfYQ3n6eDL+9O01ahVVGyFUGD5OBL9+bzm9rYtgcfQJ/Xw8mjO1Es+b1+OqNGKfHFKQX8e7Ha5g6qRvhoQEVHDFsOpaA2ebYIimKAnWbBzK8x9+zSAgO8WfIhC6sX7G3tPVSpxPx9DYx+o5eLom55iEgq1olvDy0JMlN/LZgO7YrNP8VRSU3q5ANW45w2JZHZkERfVs0ZmBkU6e7lTfKoH6RfPHNRofHBUFgwtAoojefIC89x/FAQcBuv4bSQyXy5uqNLN0bW+qlsP/0Oe7+ajE/z5pGRN0b95tSVZXnZ3xFXMwZrBeS2uXfbGHPxmN8tOKxGt2iJ+pEymvw11qIKpfZ9w0iONCbn1fuIb/ATMtmIfzj/kE0aeQ4Q/De1NGs2HeEJXtikRWFWzpE8v27mxxbbyx25v2wTUuSLqPQYmXriSSHmS6zzc7CHQfcniRZzDai18WSfi6HFu0b0KFHRK2t5k+4tz/Ra2PLbC5Keh0t2zcgrPGl771OJ9J7RHt6j2hfGWFWWTxMBm67tQu33dqlzOPjOrXml/1Hy/oQKSpqQjGrj8bw54YjfPLuHTRrWrEbpHpRhygIDkayOlHE6wZnpGe/NommrcNY+e0WigosdB3Yiun/HIFvgPuEsjRqLlqS5CbOp+aiyI6Lz/wQPU/89ReqUKJAtyrmGK1C6/D1jEklKlUuwNfHg7dfnsSLb6zEarWjUlJ2f+258fh4mxg+uA0JpzIcFlBGg67CL5LXS26xmSV7Djntq56zaRdv3zbyht/78J5ETsQmlyZIADarnZQzmezccJReNdjbpE6oP+FN6pAUl1pGQcpo0jNisnsFGVRVJel8Nga9RLi/8/mP2owoCkyb1J1pk67976ATRSZ2acvELiXf1ZzcIuYpG5y+NuN8vkvjrO5Y7HaEcgaXL/eYcwfJiRk8OeUzrBY7VrMNvVGiaWQob3z3AEZT9RbSuRGat6vPUx/cwcfPLaG4yIIiq3Ts05x/vzetskOr1jw/dhA+JiMLdxyk2GJDtKj4nJXRF6nIQLFs49O563n/zSkVFtOfG47w61e7UJoLDjrLKjCoVYTDMaqqUlRgxuRpLHcTTxRFxtzZhzF39nFD1DUPFVA0oety0ZIkN9Glf0v2bTteRm1HFeFclDfqZU7YRVYbh8+ls2L/YW7v6rpdsQ5tG7Bs/sMcP5mGIEDzpvVKLyqjh7dn/eZjnDiZRrHZhl6vQycKvPT0LVW2epCclYtep3PaV33kJvuq4w6ecVDDASgutHJs/6kanSQB/N8n03ly8mdYzTbsVjs6SUebrk245W733WR2JSbz78WryTdbUFSVxkEBfDh1LI2C/N12ztqEr48HXp5GcnIdJcBbNKv+4iw3g9ls4+vvt/DHulisNjvdOjch2NuTc7llk0edINCvRRO3xvL2P38gL7uodINCLrISf/gsS+Zu5I5/DHXruasqvYa1pceQ1qSfzcbLxwMff89rH6RxVSSdyL9G9GX2wJ4Mm/g+KI4buLFHzlZYPKlpufz3gz9QrHa8TwnkN9QhqIAAnh4GPr7zFrxNZVvd1y7Zzbfv/k5BbjEGo8TE+/oz5eFBmlKthlvRkiQ3MWhcZ5Z/u4X0s9lYL1Rs1FBP9Hod1itE/802O78dPObSJAlKWhJatQh1eNygl/jgzSns2pvI3oNJBAV4M2xQG4ICvV16fldgt8tsjT7BoRPnMDsZyBQFgeb1bk7CtG6YP3qD5JAoGT301KsfeFPvXR2o37Qu3299np1/HeZ8ai6RHRsRGdXQbS0/aXkFPDR/RemwLsCJ9Ezu+vpn/vrXfZoCmwsQRYGHZvbn/c/WlakYG40SD8zoX4mRVT7PvLSEI8fOYb0gELAtOh6prhFTYwm7rGBXFIyShLfRwGNDe7stjpzMAk6dSCtTwYWSecB1y/bU2iQJSqoBIQ1uvIX6etiRcJrvtu8jI7+I/i2aML1nR/w9TW4959/BarWzZXUMOzceJTDYh1FTutPwJjc4DAYdRr3OoYsEwKsC52/XbzqKcmGz2CNbxZhrx+YjYNBLPD6lFz0iGpZ5/bY1h/jslRWlm852u8zPc0oq5dNmD6mwuDVqH1qS5CZMHgY+WPoov3y/lS2/x+DlY6L9uHZ8GHcQq5MWDpO+YlsrdDqRnt0i6NnNsaRdVcjLL+bhx+eTmVVIcbEVY1M9sr/A5TOGBknHA/273dR5ug9ug9G0AnOxtczAsKTXMWBs7RApMBgl+o7qUCHnWrq3ZG7mchRVpdBqY9vJU/R38+69u8nJLCAvu5CwRsFI+sqbZxs5tB2+vh7M+2Ebael5NG9al/vu6ed046S2cDw+lWNxKaUJEpR894QcOw9FdiPLx05iRjadG4dze7f2bl00K8qFrXMnlOcDpOEaFuzYz//Wbi2d0YlPP8+yfbGsmD0dP4/KT5QsZhtPTvmc5KSMEgU/ncAfP+/iibduo99NXKcFQWDM8A789sfBUgU8KNk8mXRrxQmUFBVby8i8iwoYc1X0egW71XEuev6Hax08sCzFNpZ+tYnJswaV6YBRFJVDZ1MpsFiJahCKl1EzO78Wmk9S+WhJkhvx8jEx9ZEhTH2kZKdDUVS+fveoQ5+7oEBjURsqvJIvvtpIalpuqZiEZ6INNVyHpZ6EIqhE1A3ihbGDiAytc1PnMRgl/rfoEd56/AeSjqcAAqGNgnj6vWl4+dReSVl3kZKbj1V2bG9UFIX0vIJKiMg1FOQV8/Y/F3JwZwKSJCJKOma9MJbBFbj4uJLe3ZvRu7smfXuRhMTzTiukFoud9NM5PP9UxXnGBNbxIbxxMKdOpJZJigxGiUG3aDLt7qLIaiuTIEHJbGt2UTHzo/cze1DPqxxdMfy+aCdnEtJLBSwUWcUi2/jguaX0GNIGw00YFz90b3+ysgvYtjMevV7CZrUzdGBrpl7H7KOr6NW9GUuW78VsuWItJAj06OrodZV+zonQFCXVNnORpfQ+fTIjk/u/X05esRlBELArCs+NGsCkzu1c/5fQqBVoSVIFIooCn08fx51fLMJstZU03QlgylDYdDiWTiFhjLjO+ZfkhAxWzt9GcuJ52nZtzNhpPWucesumrXFl1PYEFbyTZXxTVH5b+hieJtftEIU1DuajFY+Rk1mAIisE1tWEBNxF9yYNWB0TR9EV0q+KCv6Kni/mbgABBg9oTfNqND/z+j8WELsnEbtNxnbB5/LjF5ZTLzyQtl2qd3WsptCgfgDOijQGg0STxjfXtnsjPP3+NP497XPsVhlzsRWTp4HwxsFMfmhQhcdys6iqyroj8czbvpfsIjP9WjTmgb7dCPKuWjNFx1LSkZzMsVjsMhvjEqtEkrTl95gyCn8XEQQ4cSiZNp0b3/B7G/QS/3n2Vs5n5nMuNZcG4QEEXMWA3R20ahnKwP6RbNh8DLPZhiCA0aBn3NiONHDS4t6oeT2OHTjt8LiXj6nUpkNWFGbOW0pGfmGZ3/jrqzfSKrQubcKqz72kIlFVTQL8amhJUgXTKrQu4XEqGdhRJAF9voJkAQsy387fcl1J0sEdJ3npoXnYrTKyrHBkXxK/Lojm4+WPUifEPeaehQVmzqfmUjfUv8K8g8qbiRHA6U3OFfgHVb25rGuRkJFFck4uLeoGE+LnU9nhXJNhbZozZ/MuTmfllApxeOglGhq8+d9rq7Ba7QiCwIpf9jHltu7cM73qqxSlnc3myL4kh7k2i7mkJURLkqoGrSPDaBAeSOKpjNINGEEAvV7H6OEVLyfduEUI3218li2/x5B+NpsW7RvQpX9klRXQuRqfbdzB11v3lHp1nd2Vy+pDcfwy+y4CPKtORT7AyxO74tzqoo6PJ8VmKyajvlJl2Mu7xyqKisnDNZuDwUE+BAdVzv1CEASeenwEg/pF8tfGo4g6geGD29KhXQOnr5/571G8cO/XZRJHo4eeGU+OLBVu2HvqLIUWm8MmiNUu89OuGF4dV3tn/DRuHC1JqmBkWSEvp5iSS2DZn/P5zGu3Gqmqyvv/t6RMf67VYke2K8z/cC1PvHmba+O1y3zx+q+sXboHnaRDlhXG3dWbe54Y7vabyOABrVj9Rwy2yxTtRFGgU1Sjm2o3qCkUWKw8vHAlMcmpSDoRq11mVNuWvD5uqEt9t1yNQdLx4wNTmLdtH6sPHcMoSQxu2oRfPt9xSeREVbFY7Pz4804GD2ztdHexKpF9Ph9JryuN/3LSU5y3imhUPIIg8N5bk3n/k7Vs3nYcRVFpHRnGk48Ox9+vcioent4mht92c3OVlU1usZm5W3aXUR+1yQp5Zgs/7DhQJaozF2kSHEBEnUCOpWYgXzaDqhdF4tclMfaHD/D2NjHjrj7cOrZTpcQ4ZlpPDu9JwlxsLX1MEEo28Zq2qhkzhYIg0LVzE7p2vvYGUrtuTXn163v55p3VnDqRRp1QP6Y/Npw+Iy610eUWW3C2JFFUlawiR5VPDY3rQVtpugG7rLDy4BEW74tFVVUmRLVhQqc26HU6dDqRenV9SUvPczguPOzajtfZ5wvIynD0OZFlhd2b4lwS/+XM/2gd65btLVn8XVgArpy/jYBgb8a5USIa4P4Z/TkUm0xKWg5Wq4zBIOHtbeTfj49w63mrCy/9+icHzqSUzPdcWJv/ceQ4zesGcW+fLlc/uJLxMhp4ZFAPHhnUA4AFC7c7lWFXVYXoHfE0mFS1F5ENI+o5NWKW9DqietacmaDCfDPrlu8hdk8S9ZvUYdTk7tQNq16y7T7eJl585hZkWUFVVaQabBZdUcSlZji1aLDaZbadPFWlkiSAz+68lUcW/EJ8RiaSKGKx2fE5LWNPLdl8zM0r5vO5G9DrJUZVgmFt90GtGHNHT1Z+v61U/MXDy8grc2bUWqPhdt2a8v7i2eU+37lRGFa74z3EQy8xpFXNuQa7A0UTbigXLUlyMaqq8uiiX4lOOF3adnA87TzrjsYzd/p4BEHgofsG8Nb/VpeV5jVIzLpv4DXf3+ihd5CMvYint2vb4FRV5ZcF0Q690ZZiG0u+3uz2JMnby8hXn81g994ETiZmEB4WQK/uzdBXomLY1VBVlYTzWQiCQJOgALfezCw2O2uPnMAml12Ym212Fuw8UOWTpCuR9DpEUbig+HUJQRArVSHuevH0NnLH7MEs/PSv0iqvThLx9DYycWa/So7ONWRl5PPopI8pyDNjMdvQG3SsnL+dN76eSauOjSo7vL9NdWxpq6rU8fFy2sImCBDmX/VagOv6eLN41jQSz2eTU1TMK88sJSfTXOY1FoudeQu2VkqSJAgC9z41ilvv6k3snkT8Arxo3yOi0r6ziqJQmG/B08uAropuKnhJesILDCQZikoUcAUBQYbwIF9Gt2tZ2eFpVFO0JMnFHEhOITrhTGmCBFBss7PvzDl2JSXTvUkDBvZrhUEvMXfeZlJSc6gfHsgDM/vTvYujqsuVeHmb6NK3JXu2xJXZeTd66Ll1ei+X/l3sNhlzkdXpc3nZFVO+FkWB7l0j6N616kqVAxxMTuHxxavIKTIDKkHeXnx8+xhahdZ1y/ksdnu5MsEFFotbzulOBvSN5NvvtwKOC61+fVpUfEA3wO0PDKR+k7os+Woj2ecL6NynOVNmDa4xIiDff7iWnKxC5AsVM5tVxobMe88tZe7qJyo5Oo2KZsOmo3z73VbS0nOpHx5IeEsfTuXllkmWjJLE3T0rT93xWjQJDkBV/cnJdH4/y7yOFnh3Ehzix4AxlWtD8fviXcx7fy1FBWb0BomJM/oyddbAKmfiunjpbuSjhfibZIrqiag6AWOWgn+qBYOkLXXLQwVkqta/ZVVC++a4mN1Jydhkx7mEYquN3ReSJIDePZvTu2fzGzrHE29N4sX7vyUxLhWdJGKzygwYHcWYO1zb0qA3SIQ1CuJs0nmH55q1CXPpuaozucVmZn6/lMLLpN2Ts3O5+7slbHzifjwNrvPAysjI41BsMn5+njQI8CMxM7vM86Ig0LtZ9dvVDwnx49GHh/DRZ+tKb76KovLk4yMqbbj4Rug1tA29hrap7DDcws4NR0sTpMtJO5tFTlYB/lXQjLq2kZFfyJJ9hzh5Pouo+mGM69AKb5PrhXb+WHuIDz5eW9oNkZCYgZQu0WxAHRJyc5BEEUkUeXHMINrXD3H5+V2JIAiEhviRkprr8FxYaPVqJXU1m3+P4cs3fivtJrHbZBZ/vQlRFJg66+oKjKqqcizmDId2JeLj70nf4W3x9nWfgMeadbFYrXaMVjDmXdpAzjDkk5qWS0g994haadRstCTJxQR5eWLQSdiVsi1qRr1EkJdrBoN9/Dx5/+dHSIxLJf1cNk1bhblN1W7WC7fw6iPzSy+SgiBgMEo88GzF+YlUdVbHxqE4KevYFYW1R04wLqr1TZ9DVVXmzN3AshV70Us6EEAIkDBFSNgVGbuiYpB0eOj1PDm0702frzIYPbIDvXo2Y/uOeARBoHePZvhV0jC9hiNGDwNQ6PC4qpZsqGhULkdT0rnz28XYZRmLLPPXsZN8uWUXSx+cRl0f1yWwqqoy95tNZdrFAeyFdgKP2PnynZnkFJtpEhyAXlc1W7Ou5MH7BvDmO6vKtsAbJR66/9ot8DWZ+R//6bzd/pvNTH5wQLnVJFlWePOJH9mz9Tg2qx29QWLO26t49ct7aHsT8uXloapque3tKuUr5WpoXAvtzuZihrduzhu/b3R4XBQERrZ1bV9sk5YhNGnp3l26zn1a8Nb397Pw0784czKdiNbhTHtkEE0jtUrSRTLyC8u0V17EYrdzvsBxUXkjbI8+wcpf92OzydgutFkKxVaaib60n9CGk+ez6NQglDu6RxHsXX39sgL8vRg94sYd5TXcx+ip3fnhk7/KLJokSSSqRwRe3ia3nVeWFVYu2M5vC3dgLrbSY1Brps8eTEBw9akwVgT/t3IthdZL7dHFNjtWu8z/1m3l7QmuE7ux2WRycpy3pyWfzaKurzd1fatXVbF/30j0eom532wiJTWH8LAA7p/Znx7dqnabt7s576S6BiXWBpZiW7kKd2DRAAAgAElEQVRS5ZtWHWTv1hOl85kX///6Yz+wYNOzLpmtssoy7/+1lUV7D1FssxHW0hsKJci+dC8WhJJqYL0a0vLsHjSfpKuhJUkuRrWphMVDQoiKogNUEBWY3qwV/p7uW0i4k8gODXllzgyXvNfpk+kUFpiJiAzFYHRdG1pl0qlhOJ4GPUXWsjtuBp1E+7AQMjML8Pf3vKkbw8pf92O+YkdPVaEoo5g7WrUhIkIzytNwLxPu7sOJ2LPs3HAUnaRDVVVC6gfyr7dcaztwJe8+s5jov46UJmdrl+1h18ajfPnrP/HyqZ7XVFdTYLFyIj3T4XFZVdlwPMGl59LrdXh5GcnPNzs8V7dO9V2M9urRjF493KOClpyTy/7T5wj08qRHkwZV2qLhcho2q8vxQ8kOj/v6e2LyLN+vac2yPWXkyy9isdg5EXuWyA7O/ZD+Dv9e+jsbTyRisV/w5TIXoGulIzRWgEI7BoOEJOl46flbb/pcGrUXLUlyMctX7MWWWkzd0zJ2r5ISr1Sosvbgfh6a1BcPFxnBVTfSzmXz0sPzSTmThU4SURWV2S/ewuCxHSs7tJumV9OGtAmty6FzaZgvVJRMeok6qpEXHlmEAHh46HnowUEMH97u6m9WDkXlCGiIokCxE2d2DQ1Xo5N0/N/700hOzCD+yDnqhQcQ2aGBW1tZzp3KZPufh8v4T8l2hYI8M+uW72XcXb3ddu7qxNXMtY0uHloXBIHp03rxzbwtmC2XmXsaJWbeXT1bfd2Fqqq89vsGFu+LRdKJCAh4Gw18d/ckGgdd2/Kjsrn3yZG8+OC8siauJj0z/zXyqr/78kSFAFQHu9e/T0puPhtPJDhIzgs6aH1LBD2N9QgO8qFvnxaYTDVjM9ZdqICiCTeUi/bJuJjo6BNYrTICoC9U0ReqCJTIzZ48mX7N4+MOJTPnv6uZ89/VHI913MGpjqiqyrP3fcvpk+lYzDaKCiwUF1n56D8rOXHkbGWHd9OIosDX0yfwz0G9iawXTOuQunTWBSFG52Kz2rFa7eTmFvPBh2vYtevGdnUH9o/EaHS+2GnRvGoPRldlTmXl8PLq9dwx72feWruJ1DxHDzKNstRvUocBozvQKqqh23v9Txw561Ry2GK2EbM70a3nrk6Y9BJ9mzV2SJaMksSkjm3Yvv0ES5fuJubg6XItJP4OkyZ04d4ZffH1NSGKAoGBXjz+j2EMHNDqpt+7JvHHkeMsO3AYqyxTZLVRaLWSnl/Awz+tdMm/g7tp360pr86ZQWRUQzy9jDRuEcJT70xmyLirm+wOm9AZk4djcqI36GjRtv5Nx5WUmY3BybybXVHJsBUzdXIPhg5poyVIGjeNVklyMUHlKDzJsoKv39WVXb55fw0rF0RjtdgAgVWLdjFuek/ufmwYYjUePIw7lEx2Rr6DB47NaueXH3bwr9cnVlJkrsMgSdzdsxN39+xEcbGV8RM+xGotu8tlsdiZP38b3bpdW+r9SsaMjmLNuliSk7Mwm23odAKSpOPJJ0Zi0Ibmb4gDySnMmL8Uq2zHrqgcPJvK4v2x/HzvVCKCAys7PA2gXliA08WkpNcR3jioEiKqurxx6zDu/m4xyTklRuWKqtIhNITor/fzZ54Fm01GpxNp2rQu77w79aYWkIIgMGlCVyaO74LVWtLapA3HO7Jw90GHeVWVkkpIwvksIupU/e9wu65NeP/HWX/rmIFjoti29jAHdsRjMdswGPUIAjz3wTSXzCM1CQ4oMVG/AkkUae0m2w2N2om2unIxEyd2Ze++U1gua0MQRYGGDYNo2KD8C2JSfBorLzNutZsEUruYeCvnMO+8dpR+zRrz8ugh1KtmA7EAuVmFCKLjDVRRVDLT8yohIveSk1N0QfXH8SKemuZ8EPZaGI16PvlwOhs3HWPHzniCAr0ZMzqKRo2CbzLamomqqmxYdZCfvtpITmYhraIaMvPx4TSKuHQDfWnVnxTZLv1ObbKMXZZ5a+0m5k4bXxlha1xBy/b1CakfyJmE9DLy45IkMmZKj0qMrOoR4OXBylnT2Xf6HMk5uUTWq8Pcd9ZxML2gdIPKZpOJj0/j+++28sCDN6/cJggCxhoyW+oOCi3OW6FFQXAq9lNT0OlEXvzkTg7vO0XMrgR8AzzpN6I9vv6uUSsN8fVhcMsI1sclYL4wk4QKggq3ta+ZFgzuRFa1DY7y0NrtXEzHjo144P7+GI0SXl5GjEaJiIi6vPH61Yebd244VmoOq4pwbogfRaEGEAVkVWVzfBK3f/0jVnvVv7CqqkrMuVTWHz9JRkEhkR0aYLM6JgxGk57u/WueE3adOj6ITpJCQYDIyNAbfl+DQWLY0La8+Pw4Hnl4iJYgXYVFX23iw1dWcPpkBnk5RezadIzHpn3OmcQMAKx2O8edDLqrwK6kmtHmWhMQBIE3v5lJVPcIJL0OvUEitGEgr86ZQb3wqj/TUdEIgkDnRuHc2qE1Df38iIk541DBt1rtrF17qJIirLkoisKZxAwyUnOAkoR0ROvmmJzMhEmiSGRInYoOsUIRBIG2nRszbdYgxkzp4bIE6SJvjx9BN/8QRLsKqoohVyb4gIWXn1xCfn6xS8+lUXvRKkluYPz4LowY0Z6TJ9Px9fO4agXpIpJeh6gTkWWFojADikGAyxbasqqSb7HwZ9xJRrWpuolFWl4BM35YSkpuPqIoYLXL3Nm1A7fd25dl87ZiviAFajBKBIf4MXxCl0qO2PVIko6ZM/sxd+6m0oqiIJRUg2bcow02uxtzsZUf52wsM2ysqmA12/jxyw089dbtSDodep3oMPgL4GWsneIqVRX/QG9emzuDgrxirBY7AcHeWmvXdXBlcnQ5suJoCqxx4+zbHs87zy2muNCCDcDPE4usIBpEPHp6Ieglim32UpPdN8cNv6rYhsa1sVnsJP+WROgVfl05hkJWLN/L9Lv6VFJkGjUJLUlyEx4eBtr+jQHFfsPb8t1H6wCw+ehQJcdFQJHVRsL5bJfF6A5mL/6VpMxs5MvmCH7ce4g3xg7l2XZTWLkgmvzcYvoMbc2YqT2uKiNanZkwvgtBgd4s+GE7GRn5tIoM5b77BtC0qdYv7W5Sz2YjOul7lxWVAwdOsXNHPK1ah3Nr+9asjDlSJlEySRJ3dGlfkeHWamRFYcOJRLbEJxLo5cmEDq1pEODv9LXevlef6dQoi5eXkebN6xEXl1JGbUySRPr3j6y8wGoYKWeyePmxBVjMNlRRwO7vCReuKbJFxnd7IQ3aB9KgRxj1fL25vVO7aqFsV9WJj09HJ4lgKfu41SqzY2e8liRdJyoCstZUVi5aklRFqBPqzz9evJWPX1mJl1kg1w7KFa3engY9LepW3UHPc7l5xKVnlEmQAIptNr7ftZ+fZkyhey26OffvH6ktRiqBoDo+pa2rF1FFAbufBxlWmddfW4nNpjD1rp70aNyAHUnJGHQiVllmWKtmPNi3eyVFXruwyTIzf1hGbEoaRVYbep3IN9F7eXf8CIZGNq/s8GoETz09hscenY/NJmM22/Dw0BMQ4M3Mmf0rO7Qaw+olu0rn5WSTHq7Y37RbZQoO5fDIrFu0TTIX4u/vWWZOsRRVRbVplVIN16AlSVWIoeM60bVfS9atOsjbCXspxM7Fn7okigR5eTKwxd9XRqso8s3WCyZ5ji1MeWaL4wEaGm7Ax8+TvkPbsvXPWKwWOypg9/UAUUBRVQoLSzynFi3YwauvTeKFkQM5lZVLRHAgoX4+lRt8LWLloaMcOpdaOsBukxVsKDy9ci39mzXB4GJ/n9pIo0bBLPhhFuvXHyE5OYvIlqH4eRr49LVfKCww0294OwaOao9er33WN0ra2RzsF6vROrGkt/oKdDqBs8nZNTpJOpySxiebdxCXdp6IOoHM7teDDuE3PoN7LRo2DCKknh+nTp13+MxPxZwhJ6sQ/0Avt52/JqGoWiWpPLRPpoqRkJjOvB+2EbizCFOKHUFW0akwqnULFt07Fb0Tb4CqQkSdQKfxGXQ6hka6x8lcQ8MZj78ynkFjojAYJSSTHiTHxYvZbGP58j00CPCnT0QjLUGqYH45dNSpwpcgwIGzqZUQUc3E29vELbd04uGHh5B2MoNXHl/Ipj8OsWfrCT5941eevvcbh8qrxvUT1b1pqSeQYJedOqnaZYXGTWqW0I6iqphtdlRVZc/ps9zx3c9sOJ7A2dw8Nscncdf8JWxPOOXWGLq1qY8gKyWfuVIi4CAWWJFUiN5w1K3n1qgdaElSFUK2K7z68grMZhtykUzgURvhm8w0jrbTT6pHoGfV7smXRJHXxwzBpJfQXViQGnU6jKpIzI9H+M+LSzl29FwlR1k5JJ1I44VH5jOpz+vcd+uHrF25r1qYCVZXDAaJx/8znsVbn+fZd6fi6WV0+rq8XE0FqbJwpvoFJeqYzowiNW6O7MwCFs7ZiKX4kqCJpdhGQlwqW9bFVmJk1ZtBY6IIquuL3qBDNNtKJDIvu7YbDBJdOjehwXUIOFUHZEXh/Q3b6PzOp0T99xMGffINz/yyhmJbSdX+ImabndfWbHRrLAaDDn2uBSm7GF2eGSmzCJ3FDioostZyp3HzaEmSm1AUhZzMAqzW65fsPnEi9VLZ/jIsZhtr11QPydahkc1ZPHMqkzq2pWv9cPxO2vHfVMDp4xls3RLHE48vYMvmY5UdZoVyJjGDx++aw55txynIN5OcdJ5P3/iNH+duquzQajxGk57O3Zo47V03GiX69qu6SpHVGUVVOZySxqFzqeUqqd3eqT0eekePHU+DgfbhIe4OsdZxaE8ikuR4yzcXW4neULuuya7E5GHgw4WzmHh3H+o3DCYi2IdmEXUxGiX8/DyYdFtXXvxPzfFde/vPzXy7cx+FVhuKqnI2N4/TObk42/I7eT4LxY2bgX2GtMVglBAUFdGulI6DqYpKN+3afl2ogIxYaf9VdbRGZDewduU+vnpvDcVFFkRRZPRtXbn38WHopKvvjoo6sdzqgl5ffXZWW9QN5pXRQ3jz9V9IS7SUStGqKlgsdj58fw29+7R06iVUE1k4ZyNWs61MF4bFbGPR15uZML0XJo+aqfBXVfDwMPDI7CF8+sk6rFY7qlqSIIWE+DN6TFRlh1fjOHg2hUcW/0qB1YpAScXoo4lj6NqorNrn4BZNmRjVmsX7YxEFAVEQkESRTyaMYe4X61mzOgar1U7nrk14ePZQ6oX4Vc5fqIbg5WNyOi8jigK+/lW7S6Gq4+3rwT2PDuOeR4dVdihupchq48d9h7Bcp1+jj9GI6Ea5/hZtwhl9e1dW/bwbq8WOKAroJJF7Hh1KHe16oeECtCTJxezcFMenb/x2mUeLzKrFu1EUlYeeGnXVY5s1q4eXl4ni4rIu3SaTnlGjq95i7mxuHtsSTuFlNDCwWVM8DWV3hfftS3Lq1VFUZCEjI4969WrHRexYTLLTz0HUCaSezaZxs3pXPf7UyXRysgpoFhlWstDR+NuMHhNF06Z1WLliH1nZhfTq3ZwRI9pjMjlWMjRunAKLhRk/LKPAai19rNBq4/6fVrDhH/cScFnLsCAIvDBiEHd168SuU2fwM5no37wJLz69mEMxZ0qr8Nu3Hic2Jpl5PzyIj4+2mL9Roro1xWCUKC4sK6KjN0iMnNi1kqLSqE6cLygsbaV3QIDLy0kmSWJKVFu3x/TAk6MYMLID2/48XCJvP7I9DWuwQIarURGQ1dqxYX0jaEmSi1nwxfoyJpZQUjX4fekeZj42FIOx/EWZKAq8+voknvzXQhRZxW6XEUWBXr2bM2hwG3eH/rf4cNN2voregygKpTtFcyaPo2vDS7vFfn4eZGUWOByrKCre3rVnsR/WMJCU5CyHx2WbQlBd33KPyzqfz4v/WMCZxPPoJBG7TWbq/f2Zep8m33sjtGodTqvW4ZUdRo1mzdF4p+01sqrw2+E4pnd13OxpFOhPo8ASb6SEk+nEHjpTpk1ZVcFstvLHqoPcNqWH+4Kv4egkHW9+eQ/Pz/qe4mILgiAg2xVmPTOaiEj3qZBp1Bzq+Xo7fVwA6vv7cT6/EAGwWO0YEi2s+zOaxKUn+b8XbiW8fqDb4mrRJpwWbbRru4br0ZIkF5OekuP0cVVVyc8tJqju1XeuW7QMZdHif7B92wlycgqJimpERLN6KKrK5pNJbDmZhL+nifHtWhPmV/4C253sPp3MNzv3YpHlMmrfD/38C9GPP1Aq3Tt5Sk8+eO93zJcljXq9jp69muNVziB9TWTq/f2J3XeqTPJsMEr0G9YWn6sYZL7yxI8kHk9DvmwA9aevNtO0RQjdtX5rjSpIVlERVtnJXKVdJquw6JrHJyZkIIqOfeoWi52jtVT0xZU0bRnK/HX/5ujB05iLrLTu2KhcURNnpJ7NZv/OBDy9DHTv11JrFa5lmAut3NKkGSsSTmC+rOXOKEl8OHE0oZ5e3HnvHKzZZgR7yfLgeFwqj83+noU/z8Zg0JacGtUL7RvrYpq1CmPv9niHxw0G6bo1+z08DAwecqlyZFcU7v9pOfuTUyiylZgufrltNx+MH8WgFhEui/16WXIgFrMT6V4VleikM/Rv1gSAIUPbcOZMJosX7USv12Gz2Ynq2Jh/Pz26okOuVNp2asy/35jI52+tIi+7CEEUGHprRx78d/ntl6lns0mISy2TIEFJVXLZ/O1akqRRJenWqAF6nYj9CrEGT72e7o0bXPP48PoBTucyDQapRnvMVCQ6nUjbTo3/9nHfffInS+dvRxCE0nnSVz+dTtuOjVwcoUZVQ1VVvvpyPcsW70ZvkPAJVhGaGVCMAq3r1eWZof1oG1qP1b8dQJcvI9jLHmsx29m29TgDB7WuvL+ERrko1UBAobLQkiQXc8+jQx2qBkaTnnseHXpN4Yby+DX2GPuSUyi2lbznRdPFf638g53/fLDCTRfNdtmpkg0qZXaRBUFg5r39uX1yd06fyiS4jg91r9JeVpPpM7gNvQe1Ji+nCA8v4zV31PLziku+LxbHZDQnq9BdYWqUg6KqrIuLZ/mhIwjA+PZtGNoiAsGNQ8nVhVNJGRw7eo7gOr50iGpIn6aN2ZqQVOqB5KGX6NwgjO5XCDc4o2VkKI0aB5NwMh3bZd49kl7H6LFVby6ztnBwdyLLfojGesX16D+P/sCP65/SzGhrOH+ujWXlsr3YbDI2m4xHIfictdO5a1Nef2Zy6etSU3PKdI5cxGq1k56a6/Y4MzPy2fB7DLnZRXTs3pSO3Ztq12iNm0K7srmY5q3CeOebe/n2o3WcOHKO4Hq+3PnQQPoMufGZopWxR0sTpMsRgP1nU+je6No7tK5kdOuWbIxPdIjJpij0dLJb7O1torXWL4wgCPgFXF81sVGE811zvV5H9/5aFakiUVWVp375gz+Pn6Townc+OukMQ1tG8M4tIys5uspDtiu8/spydkbHIwgCgijg5+fBOx/cSXSLpiw+EIuiqkzs0Ibx7Vtf12JFEATe/t9UPn5/DZs2HkVRVFq2CuOf/xpJQKDzeQgN97Nmxd4y/koXURSVg7sT6dKreSVEpVFRLP5ph0PyY7cr7NuTSF5eMb4X2sZbRobh4aF3EJ8yGHS0dPPc297oeF5+4icURcFmlfll0S7adWrIyx9Mu+EN6tqAqoKsapWk8tCSJDfQok04b355j8verzxjRRUVyUn/vrtQVZUjsWfJ2J9KhIcvJ8ml2GZHEkUkUeQ/Iwbhbaw9s0buxGCQePjpUXzyxm9YLSXy4QaDhG+AJ5Pu6l3Z4dUqYlLSWHc8vrQyAlBks7HmWDzTu6TSPqx2evqsXL6HndHxWC6rLlgsNt56dSUffnY3Ezrc2MaQt7eJZ1+4laf+byyKolYr+4OaypUVpMuxWR1n0Ko7RYUW1v6yn5i9SYQ3CGL0bV0ICQtwy7l2JJ1hzo7dpOTm061hfR7s1bXS5o3LIz/Puem2KIoUFlhKk6QePZoRGhbAmdOZpZVgg0GiSdO6dHBjW6bdJvPG04vLdPCYi60c2neK9asPMfQWrQqtcWNoSVI14Paoduw8lexQuTFJeqLCK0aVSJYVXn5uCfv3JGG12tEbJIKDRNrd0oKGYUGMa9+KpkHuU6+pjQy9pSP1GwezbP52zqfn0bV3c8ZO6X5VsQcN17M98RQWJybPNllmW+LpWpsk/bpyX5kECUCRVY7HpZCdXUjAdVZNy0OnEylnf0ijghkwoh17tp3AfEWFQJYVOnRtUklRuYecrEJmT/uCvNxiLGYbkl7HL4t28ton02nXybUL/WUxh3l5zfrSDZhT2dmsOhrHipl3UN+/6lhkdO0ewZrfYxxmZD09DWX8y3SSyAefTGfh/O38ua7E/2zYiHZMvbOXW9vejsU6t9kwF9tY99sBLUnSuGG0JKkaMLhFU8a1a8WymMMljtKqgCIr3OrRkIy0PEJC/d0ew7o/DrF/T1Jpyd1itiGehZSfE3lv+dhaYwxb0bRq34Dn3pl87RfWQMzFVn5fsZdt64/i4+vBLZO707Fb0wqPw8doRK8THRIlvU6Hr6n2Vk7Lqy4IolBGwluj+tNzYCs6dGvKwV2JmIut6CQRnU7k0efH/i11vOrAgi83kJ1ZgN1ekhDYbTJ2m8y7Ly5j3q+Pu2yxb1cU3vhzU5kKtV1RKbRY+XTrDt4cM9wl53EFd83oy7YtxykqsmCzyQiCgMGg4/EnRzrc+z09jdz34EDue3CgW2LJKTbz8ZZo/og7gUGnY3JUO3p51HMq+AIgSVor2dURUNDWb+WhJUnVAEEQeHnkYKZ3juK5D1eQmpSFdMbKJvEgWxfH8PQLt9B3QCu3xrBm1QGnA5nFRVZOnkileUvNZ0PDdVjMNh6/5yvOJWeVtlDs23GSaff1Z/KMvhUay6hWLfjv+i2OTwgwslWLCo2lKtFvQCTLl+3BbiubPAYGetdagZaaik4n8tL7U9m/I4Hojcfw9jUxZGwU9RsFV3ZoLmf7xmOlCdLlZJ0vIDMjn2AXfbeTc3Kxy47nkVWV7UlnXHIOVxFcx5evvruf5Ut2c2D/KULD/Jl0e/cKv++bbXYmzFtIan4+tguf3afbdrK3YX2MJj3FRdYyrzd56BkxvnOFxqhRs3BbkiQIwsPAv4FQ4DDwuKqqTlYaIAjCPOBuJ08VqarqdeE1A4ANTl7TSlXVY66IuaJJPJHGT99uJjE+nWaRoUyZ0ZeGTeqU+/rTB1Mp2pmJ4cKi0a7I2O3w39d/pVvPZhivYlR7s5SzSXPN5zQ0boR1vx0okyABmM02FszZyMjxnfH196ywWAK9PPls0i08uvy3Ekf5C5tuH08YgzmrmK/mbScjPY8u3SPoP7h1rfECmXZXb7ZvPU5mZgFmsw29XockiTzz/C2aopQLUVWVhftjmLtjD9lFxXQMD+XpQf1oVa/8e4U7EEWRzr2a0blXswo9b0VTnveToqgYjK77bft7eDhI5V+krvfNtaq6g4BAb2Y+4J7q0PWy6mgcmYVFpQkSgNluZ8fpZN76z0i+evZXVFXFbpMRdSJ9Brem7xBNdvxqqGjCDVfDLXdzQRAmAx8CDwNbL/z/d0EQWquqetrJIY8Bz1zx2DZgs5PXtgGyLvtzxs1HXPHE7j/F/82ej81qR1FUziRmsG39Ef775QxalqME99faWKfVHFEUOHTwDF3c2Io0fHQHTp5Iczi/yUNPsxa1cyZDw31EbzpWJkG6iKTXcSTmDD0q2CeqT9NG7HjsQfacKTE07dIwnAO7Enng+bnIdhm7XWH71uMs/jGaD7+YgYdnzTfZ9PHxYM68+9nw1xEO7k8iNDyQUaM7EFxHqyK5kv9t2sb3e/aXtmVtSzrNlPmLWDZjGhHaHKjLGXt7V779+K8y1x+dTqRtx4b4+rluc8bfw8SAZk3YGJ9YxjrDQy/xQM+uLjtPTWLPmbOlCqOXIwAFPvDDmn+xbf1R8vOK6dClCREttbWJxs3hri3PJ4B5qqrOvfDnfwiCMAKYBTx75YtVVc0FSkX0BUHoDTQFpjt573RVVc+7PuSK5ZO3V5W5CCuKirnYxufv/s4H397n9JhyVZ5U0LtZ4nLYiPZs3XiMmAOnsVhsGIwSgiDw4qsTXT6PlJtTyM/fbSN6Uxw+vibGT+1B/2FtK213+uJO7hfRu8gsKqJlnWCeHdSfbg2v7fuicWMEBHojCIJDn7mqqvj4VY5whUGS6NWkIVAif/32qyuvUFOycTY5m5VLdjOlligQGo16RozqwIhRHSo7lBpJgcXKvN37sdjLzniZ7XY+27aT/9ViCXp3ccvk7sQdOsu2DUfR6URUoG6IH0+/PtHl53przHD+tXI12xJPo9fpUFSFCY1bsHdRLLvkGIYMb0e3ns20yuwFGgX6Y9TpsMhlW3xFUSDM1xdPLyNDNT81DRfi8iRJEAQD0Bl494qn1gK9rvNt7gcOq6q63clzewRBMAJHgNdUVXXWglelURSFpPg0p88dP3K23ONGjo1i764Eh2qOJOlo2969Xkk6SeS1dyYTc+A0MQdO4+/vSf/BrUulP11FYYGZR+74kuyswtJZh/df/YX4uBTue3SYS891vXwRvZvPo3eW7uTGpqZz78/L+X7qJDpWkLpgbWPs7d3Y/OfhMkmIIAj4+HnSql3lJ6eJCWXNTi9itdjZ8NfhWpMkabiXMzm56EURyxWPK6rKoRTn9xCNm0OnE3nmzUmcPZ3JiaMp1KnnS+sODdySqHgbDXx5+zgyCgrJKCxk7cK9/PXFpfnfHdtO0LtfS55+4VYtUQImtW/DF9t3lUmSdIJAgIcHvZx4NGpcHzJau115uOOTCQZ0wJVX8DTgmrVPQRD8gNuBuVc8lUJJJWoiMAGIA/4SBMHpFLcgCA8IgrBHEIQ9GRkV35F3MjOLe35aSuv/fkin9z/lzfWbSncDBUEot+/Z28dU7nt26xHByLFRGAwSBqOEh4cBT08Dr/73dnQVoLfZ9SAAACAASURBVOAiCAIdOjZi+oy+jB3f2eUJEsDqpXvIzSkqMwxuNttY8dMucrIKXH6+a2Gx2/kielcZBSIo2cn9YIuzHF7DFbRsE86sJ0diNOnx9DJi8jAQEu7PW5/dhViB3mDlYTLpUWTnw3jl/bY1KgdFUfhj5T4evuMLZk74iG8++ZOCfOe+L1WNUF+fMq1YFxGAJoHuVzWtzYQ3DGLA8La0iWro9gSljrcXXkUCf/56sMwmqLnYxrZNcRw9XP7maW3CKIvc5d8CH7sOHQJ6UaRT/TAW3nk7uipwX9CoeVTFCeM7KUne5l/+oKqqcZQkRheJFgShMSXiEA6CEKqqzgHmAHTp0qVCpQUyCgq57fsfKbBYUQGbxcoP+w6SmJnNnNvGIQgCY2/ryspFO7GYLy2+jSY946b2KPd9BUHg4ceGccuELuzfk4iXt4mefZrjUYMWZXt2nHQqLWww6Dh+5Bzd+lSsmtj5wiJUnH99jmdU+67PKs3I8Z0ZOKIdx2LP4uVtpFlkaJXZTa3fMIiQUD9OnzpfRrjEZNJz64QulReYhgMfvP4rm9Zcmudc9kM0m/88zBc/zcJkqtrXTn8PE6Nbt+T3o8cxX9ZyZ5QkZvXqXomRabia3TtPOvX6sVhs7NweT+u2lV9Br0zOZ+Tz8IyvKCq0EGSxE+Qpohd1/N87PQj19ans8KotKgKKWjXuq1URdyRJ5wEZqHfF4/WA1Os4/n5gqaqqWdd8JewEpvy98NzPgn0HsNjlMktri10m+tQZErOyaRIYwN2zBpOTXcSGPw6hN+iwW2WGjOnA5HuuLW9cv0Eg9RvUzIHdeqH+iKLgcLOQZYWgOhV/IQz28qScHIkmge5xYHc15/LyePXPjWxOSELS6RjXJpKnBvTFy1C1F4hQUpWJqqJmlS+/fTtPzp5PYaEFVVVRZJXBw9sxcGibyg5N4wKpZ7PZ8PuhMr5NNptM1vkC/loVw+iJVS+hjT+eyqoV+8jJKaRX35a8NGQgPgYDPx+MxabIhPr48tKwgRVmJK5RMXh6GtDpRIc2XknS4e1ds7yoDh88w4/fbCb5dCaRbcKZdl9/Gja+upz8vDkbycstQr5YwS9SsKHw7uu/8u2ih6vMBppGzcLlSZKqqlZBEPYCQ4HFlz01FFh6tWMFQegGdAAev87TRVHShlelOJSS5rRFQhJF4s9n0iQwAEmv418vjePeR4eSejabsPqBFSprXFUZN7U7G9fGXqEsJBBWP5CmlaCiZ5Qk7urSsURd6rKdXJMk8VifnhUez98l32Jh/Hc/kl1cjKKqWGSZxTGHOZyWzuI7p2g3lpsgvH4gC5b+gwN7k8jKKqBtuwaEhlePxLm2cCw2uaQVuax9Chazjf27E6pckrRm1UE+fvd3bDYZRVHZszOBFYt38/7nd/PskP6YbXa8DHrtd1sD6TOgFZ99uM7hcUEUGDCk5my87Ngcxxv/twTLhY6RtHM5RG+O472vZhJxlXv8zu0nLiVIl5GelkdOdhEBgVVPNl2j+uOudrv3gPmCIOyiRMr7ISAM+AJAEITvAVRVveuK4x4ATqiquvHKNxQE4XEgiRLPJQMlbXnjKJlRqlK0rleXnaeTHRIlu6I4VB/8A7zwD9B+3Bdp2jyEp14Zzwev/YrdJiPLCs1ahfLC27dX6MIgIT6NNb8eID/fTM9+LTD16Mo3u/dRYLHQwN+P54YMoHujqj8ouvLwUYpsVpTLesKsskxcRib7z6XQKTysEqOr/uh0Ip3dKL2vcXOUV32WJJF6oVVrpqe42MrH//ujdPEIJTMpp5LOs+6PGEbf2glvY9Wv/l6LmJRU3ly/mdi0NAI9PHige1emdWxf6xM/X18PXnpjEq8+txRBgP9n77zDoji7PnzP9qU3ETuoIKKigr232KMmdqNJTO/9TeKb3uubGFO+9J5oYhI1lkRj7LEXrFhARVEQ6W3bzM73xyIRWRSVZZdl7uvaS5n2HJbdmec855zfQQBJknnimTE0cFFz5sL8Un79cROb1x0mKNiX62/oQXcXprTLsswHby6r8Bk/p+z7xfsrefX96VWeazDqIK/UyTVBX4P9q+ojinBD1bjkkyXL8k+CIIQCT+NoJrsPGCnLclrZIc0vPEcQBH8cqXMvVnFZHfAW0BQw4XCWRsmyvKyGzb9qpid24vudSRWcJL1aTULTxrQOC3WjZXWDPoPi6NmvDSfTcvD109OgYWCtjr9s4U4+mr0csWw1d8OaZNp1aMbW/92JLIBW7Vq59ZpkX2ZWJdEJAGSZI9k5ipOk4NW069ScoBBfLGZbhRRetUbNaA+LIiXvS0etrjxZsZhtrFl5gFFjE9xgVc1y6Gw2N8ydX35POmUr4vXV6zhbUspDfT0/Mu9qunZvxfylD5O04ziSJNM5MdJlPdeKCk3cM/0T8nNLylP8kvelM+2WfkyZ2cclY5YUW8jLcS7AlLw3/aLnjr2+C19/tqaCg6XRqEjsFoWPr3elIyp4Di5zH2VZ/kiW5UhZlvWyLCfKsrzuvH0DZFkecMHxRbIs+8my/GYV13tTluVoWZaNsiyHyLLc1xMdJIAIfz/mTZ9Mx4gIBECDQPfgRrw/ZpS7TaszqDVqIluF17qDVFxk5qPZy7FaxPJJldlkY//ek/yz5mCdcpAAYsPDMGgqr4UIgkDLOlJTpVBznEzL5t1XFvPgrV/y8bvLycosuPRJdRiVSsWbH99MdNvGaHUaDEYtIWF+PPf2FBo19ay6TqOPHtlJ4T6An793TALnbNiE+YJFG5Mo8sXW7ZicNAmtj+j1Wrr3iqZX3xiXNqVePH8bBXklFWqgLGYbP3yxlpJis0vGNBi1qJwsBAAEBV+83OC6yd3o3T8WnU6Dj68Og0FLZMtwHn9mTIXjckpKeXr5Srp/8DH9Pv6c/9u0FZuT8gcFheqgxChdhDZfQvz+JDGihMUictaYyiN/f8V7n83E169qmW8F97J753E0GnUlhT2zycbavw/Qv47lhl/XPo73/9mCRRTL9Se0KhUtgoPo0rSJW21TqF327z7Bk/f/gM0mYpdkjiSf5s/fk5jz5S00j2rgbvNcRnhEIHO+uZ2zZwqwmG00bhbiETLyF9KmbWP8AgyYzdZKionXeoli4v4zWU51cFQqgVMFhUqmRS2ybWMKVquT2mmtmiMHM+jUpeYFczQaNcPHJrB80c4KESG9Qcukmy4evVKrVcx6fhwZp/JIOXKGhhGBRLeJqJCmWWK1MvbbH8guKUW02wH4cNMWdmdk8vH1Y6q6dL1GBuyy590PPQXlnXERrz+/gNISC1aLiIBjkp1xKo8fvtrgbtMqYLNJ/LV0Ny88/jPvvbaU1MPVESD0XvQGrdPtgoBLV/VcRaDBwK8zptCjeTNUgqOvxIjYGL6bMp59u08w79sNrFiSRGnJhe0qFbyN2a8tdaSdlRU/i6IdU6mFT2ZXLhb3Rho0DKRpizCPdJDA4Si89s5UgkP88PFx9MDT6tRMntGLBBdMWN1BVYqgot1OQ3+/GhsnJSeHB39fysBPvmTm/N/Ynq70GbqQsPAAnJWBSaKd4JCa+1tcyB0PDaXfNe3KIkJ69AYtE6b3ZPjYztU6v1GTYPoOiCXGSUuIBfsOUGA2lztI4OhpuOF4GoezlZYdCpePEklyAbk5xZxOz6u03WaTWLNyH3fcP8QNVlXGahF55I6vOHE8G7PJhkolsPKPPdz72HCGj6neDcvb6JQY6bQuQKfXMnJM3awJiAwJ5rupE5DsdlSCgCTZeebReezffQKbTUSr0/B/s5fz1oc30rqNIivsjVgtIiePV54kyDLs2ZXm5AwFd9AiqgE/LnyAPbvSKCo006FTM5dOWGub+3v1YNvJUxV6Phk0Gq5rH4e/vmZSCpOzzjL5h58wiyJ2WeZkQQHbTp7i3dEjuCamdY2M4Q1cN7U7WzYcrqQk2ywyjBYtXRdZ1mrVPPbcOO54aBi5Z4uIaBJUYw24t6efdlqDqxIE9p/JIibs4jLj9RMBifotmnIxPHNJrY6jVquqaq2DRuP6mpbM0/n88OU6Pp3zF7u2H0OWnVuzYkkSacccDhI4VGYsZhsfvv0nJpPV6Tnejkaj5uV3puLnb8DHR4fRqEWrUzPtpj606+j5anYXQ61SIQgCSxfsZN/uE5jNNiTJoSxUUmzhxVnzq/ysVBeTzcbXO3Yyde5P3LVgERuOKxNwT0CtUVV57/H1sh4sdR21WkXnLlH0G9TWqxwkgISmjXl/3CiaBgagFgSMWg0zEjrx9KD+bN+Syoqlu0k/kXNVY7y5Zj2lNlsFRU+zKPLC36uv+v7mTcTFN+O+x0fg46vDx1eHTq8hpm0TXnp3Wq2MHxBoJLJ1eI05SAAtQ4PRV1E33DTANQqBCt6NEklyAYFBPkS3ieDQgdMVFJV0eg3Dr+3k0rHXrTrAWy8sRLLbEW12lizYTmK3Vjzz2kRUqoqrBev+PlBhFekcarWK5L3pJNRTaeO27Zowb8nD7NiSSmmplU6JkYSGeU9H7z8X73T6d8/PLeFkWs4lm/pVhdlmY8L3c0nLzy9fKf7neBr39OzO3T26X5XNCleHWq1iyMh4Vi7bU6Gxqt6gZeykbm60TKG+MbBVSwa0jMIsiujUarIyCrhl/AcUF5sdTZntMn0HxfGfZ8ZWemZVh6QM560Tc0pKKbRYCDQoNcHnGHptZwYM60Baahb+AUYi6nift8nxHfh86w44T6hBoxJoFOCv1OAqXBFKJMlFzHrxeoJD/TD66NDpNBgMWhp2acj+CBPPrFjpkhxpk8nKWy8uwmIREW2OnFyzycaOran8s+ZgpeP9/J0/LGRZrpP1NzWJTqehZ982DB7WwascJICLLqZexUrrwgPJnCjIr5BKYxJFPti4mdxS0xVftz5htYisXbmfRfO3kXqkZusD735kGAndW6LTafD106PVqel/TRyTZvSq0XEUFC6FIAgYtVrUKhUvzvqZ7LNFmEqtmE02rBaRDauTWbE06YquHeLjXCVNrVJh1DqvOa3P6HQaots2rvMOEkBDfz++nzKBViEhaAQVGkGge7Nm/DBlQr3vw1UV54Qb3PXydJRIkouIaBTEd7/dz7ZNKWRlFrKeMyxPP4p57z4AFh44wJT4eJ4aNKDGxty7K81pPY3ZZGPV8j30HdS2wvbR47uwfVMq5guiCn7+BtrEKasu3srQUZ346uO/sZgr5m4HBPnQ7AqjSAArj6Q6zQfXqtXsPH2aIa1bXfG16wPHUs7w2L3fIYoSkmhHUAn07BPDky9cd0Ur6heiN2h58X9TyDydT8apPJpHhXndAoBC3eJMRj4n03IqpcFZzDYW/7KN4ddefm3sXd278uLfqyvciwwaDRM7tENXx1o4KFw+vkUQujAXjcWCCoECMYXDEemE9W/jbtMU6iCe78bVYTQaNT37tqHtgCj+TD+KqUyGWQZMNpG5u/dw6OzZGhvPmYNUbou2sj+c0K0lk27sjVancdTf+OgICfPj1fduqJFJmTs4mV/AVzt28u3OXWQWFbnbHI/k2vFdiI1rgrEsF1xv0OLjq+OZV69utS3U1weVk/NlWSZYSXG5KLIs89wTP1NUaMJUasVqFbGYbWzecJiVf+yp0bEiGgfRuWuU4iApuB2LRXR6zwAqLd5diuIiM9lZhYxvH8cd3bpi1Gjw1WnRq9WMahvDrEH9a8Jkj0SWZQ4fzGDrphQK8kvdbY7bsFpFHr/nO7KzirAVWLEUWCgtsfDaM7+ScaqymJaCA6lMvMEdL09HiSTVAquPHq0gSXkOmySx+ugx2jSoGSWZ+IRIp5KeBoOWYVXUQk2/rR+jrk9k7640/AOMxCe0uKiz5cl8tnUb7/6zEXCkc7y+dh0vDBnMxA7t3WyZZ6HVqnnzwxvZte0Y+/ecJDjUjwFD2lWZflldZnTuxNKDhyqk2wlAoNFA5yaNr9Jq7ybt2Fnyc0sqbTebbSxduJOhozq6wSoFBdfStHkoPr76Sg6RTqdhwDXVu28X5JfyxnMLSNp+HJUgEBTiw6PPjOW2++8ivaCQcD9fr65DyjpTwKwHf+RsVgFqlQqrTWTyjN7ceJv3OoVVsW1jCpKTxrGSaGf54iRuvmugG6xSqMvUzdlwHcOg0aB24r2oVSoMmprzU7VaNS+8ORmjUYfR6FCr0ek1jBibQOJFRBiCQ3zpNziOzl2j6qyDlJKTw+yNm7BIEhZJwiyKWCSJ51b+zZniYneb53EIgkBCt5bMuK0/o69LvGoHCaB9REOeGzwQo0aDn06Hj1ZLs6BAvp00ocrVYgUH59LrnGFzksKo4F5KrTbWHD3GxhMnsDmZlClUD5VK4Innx2EwaNFoHalwBqOWRk2CGT+1xyXPl2WZ/z7wA7u2HUO0SVitIlmZhTz76DxyMwqJDgv1agcJ4LnHf+ZUeo5DpbTEgs0q8csPm9i0/rC7TbsqZFnmbFYhebnVf34XFpjK+8CdjyjanS5CKShcCiWSVAsMbxPDm+vWV9ouCDCiTUyNjhWfEMncJQ+zcd0hSkssJHRrSdPm3t/FfNmhw04nK4IgsDIllRs6KSvxNc2pk7n839t/kLT9GHq9lmFjOnPz3YMYFRvLnsxM/PQ62oWHKwWz1SCyVTh6vRZTaUXpfb1eyzUjlM+uJ7E4+SCzlq9AXdaUVqNS8dn140horERLr4TOXVvy+bx7WLZoJ1mZBXTuGkX/IY5mo5ci9XAmJ9OykcSKmRqiTWLhz1u597ERrjLbIzh1Mof0tOxKjoHZbGPhz1vp2bdm5xe1xaEDp3nt+QWczSpClmVaRTfkqZeuJ6JR0EXP65jYooKi8DkMRi1deyk9spwhy0KdEFBwF8o7Uws09PPjrRHDMZTlSPvqtOg1Gt4YPoyGfjXfB8PHV8+QEfGMmdC1XjhI4Fh1qkqXTXKS6qhwdRTkl/LgzZ+zfVMqNqtEcZGZxb9s44XH5uGj09KjeTPaN2yoOEjVRK1W8dTL12MwOPpyARiMOlrFNGTUdXWzibE3cjwvjyeXr8AkihRbrRRbreSbzcz85TdMtsuroVH4l/CIQG6+cyCPPzeOa0Z2rJaDBJCVWYhKVXkaI0l2Tp3MrWkzPY7iIgtqjfNpXGFB3axNysst4bEHv+d4bj5mScRmkzh8MIOH7/6mkjN8IY2bhjB8bGcMxn9VDA0GLa1jGtGjjjqMCu5FiSTVApJox76/iF5Jes76iXTo3IIHJg8kzN/X3aZ5DcNjovl02/YK9TDgcJ6GtFZWkGqapb9ux2K2VVClslpE9u5KI+3oWZd2bPdWOiZE8vUv97Hyjz3kZBfRKTGSbr2i62wKrDfyy779TutLzWYbQ298hwZ5akaO6cyNt/Wv9kRf4cppHRuBzVY5g0Cn19AxIbL2DaplolqH42x1UKdT03tAbO0bVAO89MsKDg5WIwtqEMDvhEhYkqPh+fYtqXTvHX3R8+99bDidu0axbMEOzGYbg4Z3YOjoTsp99CJISiSpSpS7uIuRZZkX/jOPpO3Hyxt4Ju3O57UtZ3nrk5vrrIqcp9GmQQNu69KFz7dvxyZJCIKAWqXi8X59aBygqHjVNIcOnK7QlPQcao2K46lZipN0hYSE+jFputK3yFMptFicOkmS3Y5VsFNUaGPBz9s4cTybF9+c7AYL6xfhDQMZPKIDq5fvK3++qtUqfP0MjKwHEVidTsP9/xnJ7NeXYLVKyLKMXq8hJMyf6+pgk+iVKaksLTyOXfvvvKi4mQZkaHrATlZW4SWvIQgCvQfE1lknUcGzUJwkF3Nw3yl2n+cggUP2NOVQJju3pNKlpxLlqCke7tOLUbExrDiSgizZ0Rwzs+aNzfxRupbo6AhuuXsgLaMj3G2mV9AqpiE7Nqdgs1ZcxZUkmaYt6keKp0L9Y2DLKBbsP0Dphal1AhizHM6T1SqyY9sx0k/m0rRZiBusrF88NGs00W0asfCnLZSWWOnRN4bpt/fDP8DobtNqhcHDO9AsMoyFP2/lWF4uJ1rJ7JCKGfzd10zv1Im7undD4yQl0RP5YPNmRKFiaEzWCBS30GA/LBIbp9T9KdQuipPkYvbvPoEoVk4HMJus7NmZpjhJNUxMWBgtg4J54I6vSD1ypryIc8vmFLZvSeWeB4cypg6usHkao8d3ZcHcLRWcJK1OTXRsI1rFKI6ognfSPyqKLk2asP3UKYejJIMgyQSkimhL/53caTRq0o6eVZykWkClErh2QheundDF3aa4jZjYRtz48CCGf/MNxRaroxejKPJ/W7dyPD+Pt0fUDQGLKnsbytCmcxOi2zSqXYPqATJgrwP9itxF3VheqMMEh/ih0WocH0SVgF0tIAuOnOHQBkoamCvYsOYgaceyK6rcCAKSLPPJ7BXk5ylSoFdLSJgf73w+k3YdmyEIDgdp0LAOvDR7mrtN8yp2nT7N/UuWMHHuXN7ftIkCs9ndJtVrVILA59eP49Vh1zCoVUtiNYE02WYjdH/FyJIoSjRtrjhICrXH1zt3YRalCiVKZlFk2aHDZNSRxuqdGjVyOl03aDW89rKSvlofEQRBLQjCS4IgHBMEwVz278uCIGjOO0YQBOF5QRBOC4JgEgRhjSAI7S64TrAgCN8JglBQ9vpOEISLyyWiRJJcTu+Bscx5cymytswfFQRklYwV6DOwrVtt81Z2bD3qtF4GAJXAzs2pDBoRX7tGeSFRrRvyzue3IEl2VCpBUbKrYX7bv59n/v4biygiA/uzspi3dy9LZswg2Fg/Uok8EbVKxbWxsVwbG8vZrEJuXfoxpguOsVlEdmxKoUWUUpunUDskZWQ4bYOhU6s5kpNDI3/PX5R9pE9v/jlxApPNVu7sGTUanh44EINOe9FzFa4UwdOFG54A7gVuAvYC8cA3gAV4qeyYx4FHgZuBQ8CzwF+CILSRZfncCsGPQHNgeNnPnwPfAddebHCPfme8Aa1Og8qgdTRFOjeJFATUOjUb1h106dhp+fl8vn07X+zYwcmCApeO5UmENvCnqvm6I+qhrA3UKDLs2XWCbZtTMZmslz5e4ZJYRJEXVq/GXOYgAVgkiVyTic+3b3erbQr/0iA8gDfem4ZKAGTZ8ZLsYBX5+qNVHEk+7W4TFeoJMWFhaJw8+Kx2Oy2CLrlg7hHEhIXx2w3TGBrdmnBfX+LDG/JS74FMjItzt2kK7qMXsFiW5cWyLB+XZfl34HegOziiSMBDwOuyLP8qy/I+HA6VPzCt7Ji2OJyjO2RZ3iTL8ibgTmC0IAhtLja44iS5mKMpZ5xq+9usEiv/3OuycT/fvp0R33zD/zZs4O0NGxj29dd8u2uXy8bzJIaP7lTevb2cMqlqNShN5WqQ5P2nmDR2Ns8++TMvP/cbE6+dzaq/9rnbrDpPSm5uBXn1c1gliVVHj7rBIoWqKMgpwUejRrBJCDYJleTI8LdZRf5ctNPd5inUE2YmJqBVV3zu6dVqujdtWmecJIDo0FBmDxvB+MxwbB8d5asHFzHpmjdZrnyXvJUwQRC2n/e644L9G4CBgiDEAgiCEAcMApaV7Y8CIoAV506QZdkErMPhYAH0BIqBjedd9x+g5LxjnKI4SS5GrVY5neyc2+cKjuXl8c7GjVgkCavdjlWSsEgSr69bR/p5ESWTzcaujNOk5ee7xA53EdEoiOdfnYheryl3jgRBwKASeO7tKRiMOjdb6B1YLDZmPTqXwgITpaVWSkusWMw23nl9Ken1oJGjKwkyGJxKTQOE+PjUsjUKF8NssiLLMgJUqKew22VKS5TIam0iSXbM5vrZ1DcqOJhvJ06gTVgYKkFAp1Yzpm1bPhxz0Wwij+S9VxazdsU+bFYRi9lGcaGZD99cxraNR9xtmtchA3ZZcNsLyJZluct5r08vMPENHGlxBwRBsAH7gW9kWf6obP85pagzF5x35rx9EcBZ+bzJeNn/s847xilK3pGLiWoVTkCgD2ZzxXQ3g0HLyDGdXTLmiiNHsDuZYMnAipQUbklM5LukJF5fvw61SoVotxMbFsYnY8bSwNc7Gtx269maRSseZ+e2oxw5eJpGjYPp0TcGo4/e3aZ5DVs3pSLbKy8AiJKd5ct2c+udA91glXfQJCCAuPBw9mZmIp63yGLUaLglwfv7v9QlOnVr6TRbwGDU0XewkiZUG9hsEp9+uJI/Fidhs0lENA7igUeHk9i1pbtNcznb/jnC95+uIfNUHq3aRDD73sE0jw5Ho1bXGenv8ykpMrP2r32V2ktYzDbmfrGOrr0u3kxWweuYDNyII3VuP9AJeE8QhGOyLH/h6sHr3jeojiEIAi+8PhE/fwNGHx1anRq9XkOP3tEMGe4a8QAZp024y/dtPHGC19evwySKFFutmEWRfWfOcOfvi1xij7tQa1R07dmaaTP7MXBYB8VBqmFKis0VFQTLkEQ7xYUXlrIrXC4fjxlDbHg4Ro0Gf50OvVrNPd27M7hVK3ebpnAeQcG+3HL/EPQGbXlzcINRS3xiC3r0i3GzdfWDd15fwh+Lk7BYROx2mdPpeTw36xeOHMpwt2kuZdWfe3jp8Z84uC+d/LwSdmxO5bHbv+L4oTN10kECyM8rQX1B2uA5sjLrT211bSKhcturGrwFvC3L8jxZlvfKsvwd8A4wq2x/Ztm/DS84r+F5+zKBBsJ56lJl/w8/7xinKJGkWqB1TATzFj3IxvWHycstIb5Tc1q7sJfMsOho5mzeXCldRwCGtm7N86tXYRIrqr+JsszB7GyO5+cRGRTsMtsUvIdOiZFOnSSDUUuP3spq3+UgyzJ7dhxn7Yp9aLUahozqSHRcYxbdcANHcnLILimhXXg4AQbDZV331/37mbN5E2dKSmgVHMyT/frTt0ULF/0W9ZfrpvagfefmLF+0i9ISC30GxdGjXwyqOjpRrUvk55WwdnVypciD1WJj7ncbefbl8W6yzLXIssyn7yyv0KgeJdeZGAAAIABJREFUHBGXL+b8xVufznSTZVdHeKNABCdfG5VKIC6+2SXPlyQ7K/7cw9LfdyFJdoYM68C1YxPQKYJNdRUf4ELZRol/gzzHcDg61wDbAARBMAB9gf+UHbMJ8MNRm3SuLqkn4EvFOqVKKJ+aWkKv1zJwSLtLH1gDRAUH81DPnszeuBF7WaqOSqXiP3360CwwkKwS532CtCoVuaUmxUlSqBYRjYIYN7ELv/+6o7wOwGDU0q59U7r2UMQxqossy7z74u+sXb63bMIj8MdvO5hya1+m3daf6NBQokNDL/u651Jqzy2IJGdnc+fvi/hi3Dh6Nmtew7+FQnRsY6JjG7vbjHpH1plCtFpNJSdJliHteLabrHI9xUVmigqcR+xTDtbdCJpWq2HmPYP54v2V5Q6gIAjoDVpmVCOF++XnF7BtS2r5M+lEWjbr1yTzvzkzXFYHruBSFgNPCoJwDEe6XWfgEeBbcNQWCYIwG/ivIAgHgcPA0ziEGn4sOyZZEIQ/gU/OE4b4BFgiy/Khiw2uOEleSFZGPp3MgXw+aDR7TTkgCAyPji5XuBkYGcWRnBysF/RUEO12YhsofT0Uqs/tdw8moUsUfyxOwmy2MWhIO/oPiitPO1K4NAd2n2Ttn3vPKziXHfn3n61jyKiOhDe6fGUqyW7n3U0bK0WMzaLIWxs28NtUpemvgmeQn1/CujUHMZVa6dq9JS1bXZg1c3EaNwlGtFXuD6RSC8S29V6n1cdHh0arQhQr/+6XalSfUVTE93uSSD6bTYeGDZke39Gj6pHHTulBWHggc79cR87ZQuI6NuemuwfRLDLsoucdPpTB1i2pFaJrFotIakoW27cepXtPZfHuQmTKBRQ8lftx9EP6CEd6XAbwGfDiece8CRiBD4FgYAsw9LweSeCoaXofWF728+/AfZcaXHGSvAhJsvPO8wtZu2I/Wp0a0SYR26EpL8yeho/vv/U4MxMSmH9gP3kmU7mjZNRoeLxPX3y0SsM2hcsjsWvLelEg7So2rk7GbKmsyCWoBLZuOMLoiV0v+5pFVgslNucqXym5ivKggmewZVMKLz77KwCiaOfbr9YxdEQ8Dzw8vNrNqf38DYy5PpHFC3dWmBzrdBqmzrioum+dRq1Rc+2kbvz+09YKv7feoOWG2wdUed6BrCwmz/8Zm13CKklsOnmCb5J2sWDKNKKCPSeLpPegtvQe1Payztm35yR2qbKIislkJWlXmuIk1UHKHJ2Hyl5VHSMDz5e9qjomD5h+ueMrsUcv4tfvNrJ+5QFsVpHSYgtWi0jynpPMeXlxheOCjUaWTZ/BHYldaBcezoDIKD4dO46bOrtGbU/B8zCVWkk7no2pVJEodjd6vRa1k9oVQRDQ669s0cJfp0dfRfFzs8DAK7qmgkJNYjbbePn537BYRCwWEUmyY7GI/PXnXnZsO3ZZ17r9nsHcescAGoT7ozdo6ZQYyeyPbqJp88tPU61LzLx3CKPGd0Gv16A3aPH103PLfUMYOLxDlec89fdKSmzW8gVSiyRRZLHw0to1tWT1vxRZLCxPOcLK1FRMVSzqXA7BIb5oNJXvezqdhrAwv6u+vkL9Q4kkeRGL5m2pVMRps0qs//sAj1jFCoWLwUYjj/TuzSO9e7N62R4+vesXXsksoGHjIGY+cA39hravbfMVagG7XebT//ub3xfuRK1RIYl2xlyXyB13DVJS5NzEwJHxzP/2H6QLVkBlWabHgIs2A68StUrF3V278cGWzRVS7gwaDY/06n1V9ioo1ARJO48jOLnnmM02/lq+hy7dqh+dVqkErpvUjesmdatJEz0etVrFnY8MZ+a9gyksMBEU7Fu5kfp5iHY7e85UFvOSgU0nT7jQ0sosPnSQJ/5aUa7CZ5dlPhw1mv6RUVd8zV59Ypjzzp+VtqvUAoOuUeY0VWFX4iVVorwzXkRpicXpdtkuY7OITvf9vSSJ2S8u4szpfOx2mYz0PP737ALWrdjnSlMV3MTcHzay+PddWK0iplIrVqvIkkU7+WnuJnebVm9pFhnGXY8NR6fTYDDqMPro0Bu0PPXmJPwDjFd83bu6duXBnj0J1OsRgEZ+/rw1dBiDWyqpkeCYlElVNOxVcD2yLFfZq6KK/usKVaDTawkLD7iogwSgEgS0VUSYjbWYan+qsJDHV6zAXNaGpNhqpdRm454li8k3X3n7CL1ey//mTKdRoyAMBi1Go46QEF9ee3MKwcGeU3OlUHdQIkkuQpZlVq3cz/yftlBUaKJrj1ZMv7EPYWEXL6i8GhJ7tOKfVcmVZJkbNw/B19+5dPBX5ynInMNitvHVnL+UaJIX8uvPWyv9vc1mG/N/2sLUG7w3f9/TGTWhK70HxbF9YwoajYqufaLx9bs8ue8LEQSBO7p05fbELtjsdnRlkyObTWTtn/vYuDqZoGBfRk7sSuvYRjXxa9QJCi1mnlu9imVHDiPJMt0aN+HlwUNoGRzibtPqFZ0SqmghYNAyRHn2uASVIDAuti0LDyZXEG7Sq9VMaV91il5Ns/jQQexy5QUKQRBYnpLC5KuwpWWrhnw77x5OpGUjSXYio8KVLImLIMsgebZwg1tRnCQX8fknq1m04F9p5D+W7GbDukN8/vXtBAW5ZkXjtoeGkrT1GGazDZtVRK1WodWqefjZsU6Pl2WZ7DOFTvedych3iY0K7qWoyPkqXVGhuZYtUbiQoBBfhozuWOPXFQSh3EGyWmw8OvMLTh47i9lkQ6US+Hvpbu5+YiTDr0us8bE9DVmWueHXXzick42tLIq05VQ643+ay+qbbyHIcOWRO4XLw2jUMeuZsbz64kJkGURRQqvTMGBQHF27Kw2TXcUz/QdwoqCApMwMNCoVot1Or2bNebBHz1qzodhqrdTHERzpgKU1UJskCAItIhWlXoWrR3GSXEBBQSkLft2G9by+DZJkp6TYwsJft3Pzrf1dMm5Ek2A+++0+fp+3hQN7TtIsqgHXTetBkyqKVwVBILSBPzlniyrtC4+4fNlhV5BdWsp3u5NIysygTVgYN3XsTJOAgFoZu7DQxNo1yRQXmemcGEmsF/RAiYpqwNGjZytvb6k8UOoDfy1O4sTRs+XRRLvdITf+f28so/+w9hh99Je4Qt1mR8ZpjuXnlTtI4Mj4skgSv+zfz22JXdxnXD2kd982fDv3HtasSsZkstC1eyvaeMF91pPx1en4ccJEDudkcywvj+jQMHxKYPemozRpHkqTFq4XuxgY1ZKvdu3CJFZ0iARBoH+LSJePr6BQXRQnyQUcTc1Cq9VUcJIAbDaJXTvTuPlW140dHOrHTfcOrvbxN947mI9eX1pJQvSm+6p/DVdxPD+P6+b9iFkUsUgSm9NP8uPePfw4fiLxDSNcOvauXcd5etZ8ZFlGFCW+++4feveOZtZTY+t06P6eB4by1BM/YbWKyDIIgkP5594HrnG3aQq1wPoV+yulWwKoNSqSd58kwcslco/l5TlqYS7ALIokZ3tv41FPJjTMn/H1THDBE4gJDSMqIIg3nvyFresPodE62obEd4ni6f9NwWDUuWzshEaNGN66NctTUyi12RBwiMrcEN+RliFK2mtt4+F9ktyKItzgAsLC/J02eBMEgYhG/8rvHs3L5e+jqaTluy+1bdi4BO6dNYqwcEd0pkFEIA88cy0DR8S7zaZzvLJuLUVWK5ay3GlbWSj+v3+vdOm4oijx/LO/YTbbyqRpHavtG/85wvp1B106tqvp1LkF78yZQfeerYmICKR7z9a8+/4MOnZq4W7TFGoBvwDndU6yXcbHz7ujSAAxoaFA5QmBUaOhQ8PLa2LqaqxWkbNZhdicNEpVUKgJfvxkDVvXH8Jq+bdtyJ7tx/j0f5UV4moSQRB4e9hwPhg5mrGxsVwfF8fnY8fx336uybJRULhSlEiSC2jWPJRWrRty+FBmBWdJp1MzYVJ3zKKNu5f+zpZT6WhUKmySnX4tWjBn+Gj0mtr/kwwdm8DQsQnIslztBn61wab0k9idrPoezD6LRRRd9l4d2H/KaUM6s9nGn3/sof+Ay2tw52m0iW3Ey69NcrcZCm5g9KRubPvnCBbTv9EkQQC/ACMx7Zq40bLaIb5hBHENGrA360x54bpKEPDRarm+bZybrXMgyzLffLWe+T9vQZZlVCqBKVN7csOM3h51f1ao+yz5eSvWC5RvrRaRlb/v4v6nrnXJ5y0nq5AvZq9g89qDaHUaho7tzPS7BqE3KI3s3YGMgF1W4iVVobwzLuLl1yfROaEFWq0ag0FLUJAP/31mLNExEby6fi2b00+Wy19aJJF1acd5d/NGt9rsaQ9gYxVOkEalKu+t4AqcpeNUZ5+CgqfTqVtLJs/si1anwcdXj9FXR3CoHy9/OAOVC79TnoIgCHxz3XimtO+An06HXq1mcFRLFk65gQC9Z0TS5v+8hfk/bymPZJtMNub+uImFv+1wt2kKXoa5imbiNqvkdKHwaiktsfDAtI9Z++deSostFOSWsOjHzTx3//c1PpaCQk2gRJJcRECAkdfemkJBfiklJRYaRgSiVquQZZlfkveXp5CdwyJJzNu3hyf79HOTxZ7HtA4d+WzndsznNcPUqdWMjm6D2oUTurh2TZ3WHRkMWoZ7QBqit1JoMfPWxg0sPXIIARgdE8ujPft4zOTVW5h2xwBGTujCvp1p+AUY6ZAYiVrtnQ6SLMvs2XaM7f8cxi/AyKBRHWkQEcTzAwbx/IBB7jbPKfN+3FSuinoOs9nGjz/8w3XjFWEJhZojvksUOzenVOpLFR3XGLXm4j2XroRVS3dTXGyu0DjbahFJ3nOSIwdOER138Wi2KEqsWL6PFSv2otWoGTW6E/0HxHrcAq+C96A4SS4mMMiHwCCf8p9lwCI6b+x6odJLfee+bt05kpvN6mPH0alV2Ox2OjaM4PmBrp3caLVqnn3+ep59+hdkWcZqFdHrtXTv0Yp+/et2qp2nItrtTJg/j7T8fGx2xwLCT/v3suVUOkunznCpU1wfCQrxo8+Qdu42w6VIkp1XHp3Lzk2pWMxW1Bo1P36yhiffmETPgZ75PZZlmYIC5zL9+fmltWyNgrdz1xMjeXD6J9gsIjabhEarRqNVc//T17pkvIN70yuk+p5DEODY4TMXdZLsdpknn/iJ5AOnyxcRDhw4xdatqTz+xGiX2FtfkJzUaSo4UJykWkYlCHSKaMSuzIwK2wWgW5Om7jHKwzjXv0mr0/DRqDGk5edzOCebyKBgokNdL08KkNglih/m3cOa1ckUFZlJTIyibVxjZcXKRaw+dpSMosJyBwnAKkmcKixgbdpxBkW1dKN1CnWRf1buZ+emFMxlkzLRJiHaJN6cNZ95a2Z5ZA2EIAg0bRZC+sncSvtaRIa5waL6g0UU+SP1CCm5OUSHhDK8VXSt1wiLokRS0gmsFpH4js3wu8qG0peiWVQDPv3tARbN3cTh/adoGRPB2Gk9iWgS7JLxmrdsgE6vqVQHJQgCEU0vPua2bUc5mHy6QpTVbLaxZnUyEyd1JypKaWOhUPMoTpIbeHngECb98hNWScJml9Cp1Og0ap7t55npH7XJgd0neGvWL+ScLUS2y0S3a8KsNydxTavalyYOCvJl3HVKekttkJx9lhInTQRNNpHk7LOKk6Rw2fy9JKncQTofQRDYt/M4ib2i3WDVpbn3vmt4/tlfsZw3kdTrNdx9zxA3WuXdnCkuZtz8Hyi0WCi12fDRanlj43oWTJxGQz+/WrEh+cAp/jtrfrnYkyjaeeDBoYwYWfMNps8nrGEAtz40zKVjnGPYuAR++nxtBSdJrVER3iiIDomRFz13547jmJx8n2UZdielKU7SFSKjSIBfDCWHxQ20bRDO8uk3cXOnzvRu1pxbExL5a/rMWouSeCo5WYX8986vyUjPxVoW/j+4N53HZn6O3Ul3bgXvoXlgED7ayiv7Rq2G5oGBTs5QULg4movUVHhyDVa37q147Y3JdIhvRlCQDx07NueNt6aS2CXK3aZ5Lc+u/ZuzJSWUli3UlNpsZJUU8/y6VbUyvtUq8uQTP1FYaKK01EppqRWrVeT9OSs4fqxy8++6SmCwL//7+nbatG+KWq1Co1HRvV8b3vzilktmaQQF+aDVVv5Oq9UqAgKM5T/bZZmzpSVVljUoKFwOSiTJTTT2D+DhxJ4s/207Gz7bzycBaYye0sPrmzlejOULdiCJFZ0hu2SnML+UpK1HSehRf98bb2d469a8umENZlEsl31XCwK+Oh1DW3re372w0MSXX65l7dqDqASBoUM7cONNfTC6sAGjwuUxdFwCOzamYDZVVPBSqVW0T4h0j1HVpGOnFsyeM8PdZtQbVh0/inSBeoEky/x9LLVWxt+29Sh2e2XlVJtNYtmy3dxzr/dEESOjG/LeD3diNllRqVXodNWbhg65pj3ffbuh0naVSqBX7xgAFh1K5qX1ayi2WgCY0LYdz/YbhE5d8yIUCvUDxUlyExazjYenf8LpEzlYynJsd2xKYdodA5h82wD3GucmTp/MxWatvPoj22WyMwvcYJFCbWHQaPl14jSeWPkn206fAhw1em8OGe6W3mEXw2aTuO++bziTWVieGrNgwXb27DnBBx/epNSteQjd+rVh8JhO/LVwJ8ig0qgQgOdmT0PjZEVaof4iVFG4Xlvf5dJSi9P2Ena7THGxuVZsqG0Ml7mg1KCBPy+8OJ6XX1qE3W5HlsHHR8dLr0zAYNDyz8k0Zq1agem8CNKvBw9gtdt5c3DtpBPWTZQ+SRfDs2Yf9YiVi3dVcJAALCYbP3y8mhETuhFwniLe+ZhMVtavP0Rubgkd2jclrl0Tr5mUxXeJZMNf+yrVEcgyxLRXRC28nWaBgfw4fjLmMpVHg8aRfme32zm0Nx2rVaRtfDN0evcW3G/YcIjcnOIKjaJtNom0tGySkk7QuXMLN1qncA5BELj/qTGMndqDnZtS8fXT02twHL4uLoavL5wuKuTbPUkczDlL54aNmR7fkVCj8+eWpzO8VWv+SD2CeF5at1alYnir2qlb65wQWUEW+xwGo5Y+fdrUig11ga7dWvLrggc4dDADtUZFTEyj8nYd72/bXMFBAjCLIr8fSubpPgOUVhIKV4TiJLmJzauTKzhI59Bo1ezfleZUojYl5QyPPvIDkmTHapXQatXEd2zOSy+Nv2j+fV2h//B45n62luzMAmw2xwRUb9CS2DuayNYN3WydQm1xzjkCSD14mmfu+RZTiQVBJSDbZR55aTx9h7Z3m32HD2U6LSC22eykppxRnCQPo3nLcJq3DHe3GV7F3qwzTPntJ2yShM1uZ3P6Sb7evZOFk2+gRWCQu827bJ7rP4i9WWcctSyShF6tJtzXl+f6DayV8cPC/Jl2Qy/mzt2M1WJDlh19+dq3a0r3Hq1qxYa6gkajpp2TRdOTBc6zTdQqFTmmUsVJugh2RQK8ShQnyU0EhfqVT/rOR7bLTqNIsizz/HO/UlxsKd8mSXZ2J6WxdGkSY8cmutzm87HbZacNV68GvUHLnLl3M+/TNaz/ax86vZaRE7sydmqPGh2nLmMRRQ7n5hBsMNI0IMDd5rgUq1Xkydu/ouiC/jBvP/ULrWIb0bi5e4ROmjQNxmDQVmr4qdWpiWhU9yaICgqXy6xVK8pFDsDRDN1mt/PKhjV8OmqcGy27MkKNPvw1fSZr0o6VS4APaBFVq/3ZZtzYh44dm7Ns6W5MJisDBralX/9YjxYZ8SQ6RTQiM7W4vKb1HALQ2N/fPUYp1HkUJ8lNXDulO+v+3FshmiQIAv6BPrTt2KzS8SdP5pKXV7mZoMUi8sey3bXmJC1fvpcvvlxLdnYxISG+zJzZj1E1KFHqH2Dk9sdGcPtjI2rsmt7CvP17eXn9agRBwCbZ6RAezsejxtbZFJdLsWPDYSSbVGm7JEosX7CdmQ+6J8980KA4vvh8LZayFV9wFA/7+erpoaz6Kng5FtEhy38hdlnmnxMn3GBRzaBRqRgS1YohUY7vcGZ6Lj/832r2bj9GWMMAJt3Wn279XJv6Ft+xOfEdm7t0DG/loe69WJN2DJPNxjk3yajR8GC3nujVylRX4cpQlijcREy7ptz95Gj0Bi0+fnoMPjoaNgnitc9monKyeuVM+aZ8n5OCT1fw18r9zH5vOdnZxQDk5pbwwQcrWfbH7loZvz6z5VQ6L65bRYnNRrHVikUSSTqTye1LFrrbNJdRVGBy+rkXRTv5OSVusMiBj4+eOXNmEBvbGLVahVqtomPH5sx5/0avSHtVULgYapUKjeB86mDUesdkNDM9l3vHf8jfv+8iMz2PfTvSeOXhuSyZt8XdpilUQXRIKL9OmMrAyCiCDQZiQkJ5Y/Awbk/o6m7TPBpZBkkW3PbydLzjjuZhFJjNfL1nF6uOp9LAx5dbOyXSs2nl1aHh47swYEQ8h/al4+Orp3Vc4ypFGFq0CMXf31ApxUev1zBsWLxLfo8L+eqrdRUaHAJYLDa++mo9I0e4tuFdfeeLXdsrFaWKdjvJ2Wc5np9HZJBrOqS7k/iuUU77YxmMOrq6eEX3UjRrHsoHH95EaakFQRDqvfR3auoZ1qw9iCAI9O8XS6tWSg2Qt6JRqRgV3YalKYewSv9Geg0aDTe0947nwI8fr8ZkslRYpLGYbXz5znKGXZ+Itpqy1Qq1S0v/IO7wj0M0tqFT91b4+isiLQpXh/JNr2EKzGZGzvuWbFNp+QNkY/oJnujZl5s6JlQ63uCjo2O3lpe8riAIPPf8dfznsXnY7XYsFhGjUUt0dATXXtu5xn8PZ2RlFTrdnpNTjCzLXqOy54lklhQ73a5VqcguLfVKJymiaQijJ3dn2fyt5YqHeqOW1nGN6Tkg1s3WOfDxUYqBv/56PT/9vKVcbGX+/K1MndKDG2/s42bLFFzFiwMGc6qogL1ZZ1CrVIiSnX7NI7m3q3fUj+7Zdgy7VDmKLcsymel5NGvZwA1WKVyMpM2pvHT/9wDIOHos3vfsWIaMqzzvUlCoLoqTVMN8vWdXBQcJwCSKvLFpPRPatsdXd+Urzm3bNmHuvHtYvSqZ7Jwi4uObk5AQWeMCClXRKCKIU6fzKm0PDw9QHKQqSCvIJ6OoiDahYQQbjZc+oQr6t4jkUE52hc8VgM1uJzbMex/Ytz82go7dWpY7SgNHdWTwtZ1QK2ltHkFaWjbzftqC9bz+ZhaLyI9zNzNwYBzNmoW40ToFV+Gn0/HT+CkcysnmeH4ebULDvGqhJqxhAJnplZ91oigRGOLrBos8B4e6rojBoPWY535piYUX7v2uUuPo919cSFznFjRu4R6Rn7qC0iepahQnqYZZdTy10kQWHHncyTln6dKoyVVd39/fyJix7lkZue22/rz+xpIKKXd6vYbbbu3vFns8mUKLhTuXLSIpMwOtWoVVkrgxvjOzevW7ogfLzI4J/Lx/H/lmM1a74/Nl1Gh4uEdv/K7C8fZ0BEGge/9Yuvf3jMiRQkU2bkxx2t/Fbrfzz8YjTJnc3Q1WKdQ0x49k8s3svzi09yQNGgUx7e5BdB8QS5vQMNqEhrnbvBpn8u39ObJ/bgVhJa1OTff+sVX2MPR2JMnOZ5+v4fclu7DZJEJD/bj/3iH07hXjbtPYvCoZZ49VSbTz9++7mHH/kNo3SsErUNzHGqaBj/NVJtFuJ9hw5ZEET6B//1hmPTmapk2DUatVNG4cxOP/GcmQIe3cbZrH8djKP9iZcRqzJFJktWKRJL7fu5tfD+6/ouuFGH1YNu1GbumcQJvQMPo0a87/jRzD7Z271LDlCgrVR60WnEayBUFAo1EeL97AscOZPDz1Y7asOUhedjGH96bz2iNz+fOXbe42zWV07duG2/8zAqOvHqOPDq1OTbf+sTz26gR3m+Y25nywgkWLd2I225AkO1lZhbz86u/s2XvS3aZhLrU4FfmRRDslxWY3WFR3kBGwy+57eTpKJKmGubVTIhvTT1QoslcLAq2DQ2gVXPdTT/r1i6VfP2VV/2IUWiysPXG8POJzDpNo47Nd25nQtnqNUK1WkaRNKVhMNuK7tyQ02JcnevXjiV79XGG2gsJl069/LF9+tb7SdkEQ6NfXveIaCg72n83i411bSc3LJSGiMXd27kqzgMBqn//te39hMVk5X0TVYrbxxdt/cs24BK9NfR09pTtDr0/kTHoegSG+9TaCBFBSYmH58r1YL2jJYLGIfPf9P7z1xpTybYWFJv5auY/TGfm0b9eUPr1j0Gpd+xlJ6B2N7ETl12DU0XNQW5eOreDdKE5SDdOzaXOe6NmXNzatR6NSIdrttAoO4YvR17nbNIVaothqQVVFSl2+pXqrWslJJ3j2zq+x2+0OiU5RYuYjwxl3Y++aNFVB4aqIaBjIffcO4YMPV5aluwjIsswDD1xDeLh3NzuuC6w/eZzbly3CKknYZZnDOdksPJTMwgnTaB1SvTqNg3tO4qzLhM0qkptdTIOI6jtcdQ2dTqOINAA5ucUOZ9hJ37r09Nzy/x9JyeThR39EkhziUn8u38M3323ggzkz8PN1ndJcRNMQrr+5Dwu+/Qer2dG/zuCjo2vfGOKrIYyloFAVipPkAm7qmMDEuA4cyM4i2GD0igiSQvWJ8PMnQKfHfIFkt1oQ6Ncs8pLnW60iz975FcWFFR2qr99dTrvESKLbXV1dm4JCTTJ6dCd69mzNxo1HAOjdO5qQEL+ruqbVKpb3oFK4MmRZ5qk1Kyvch0RZpsRm5bWN66q9cNcgIpD8nMrqmrIs4x9Yt1PIFapHw/AAp+lsggAxMRHlP7/62mJKS/8VTzCZbGRk5PPDD5u4846BLrPvaH4u7Se3J65HFOt/34PNKtJ/ZDzd+rfxGHEJT8aO8h5VheIkuYBjhzP5/v2VHNmXTkSzEKbdM5hOPVq52yyFWkIlCLwy8BruX76kfAVXq1Lhq9PxUPdelzw/aWOK0weSzSqy/JdtipOk4HGEhvrYSdFEAAAgAElEQVTVSCuC5IOneee95aQePYtWq2LokPbce9dgDAZtDVhZvyiyWjldXFRpuwxszUiv9nWm3jWQN/7zUwURA71Bw+AxCRjqeX+w+oJer2Xq5O7M+2kLZst5nwOdlpvKpP5zc4s5nZFf6VybTWLVmgMucZKySoq5delCjuTloC3L3PnvlP7M6NCpxsdSqJ8oTlINk3LgNI9N/xirWUSWZc5mFvD83d/w6OsT6Tusg7vNU6glhkS14pfxU/hs13bSCvLp0aQZt3RMpIHvpeVjzRfk/5/DbpcpKba4wFoFBfdzOiOfRx6fV94w22qVWLFyP1lZhbzx6iQ3W1f3MGg0qAUB0cm+QH31U596Do7j9idG8tU7yxFtErIsM2hMAnf9d3TNGavg8cyY3pvQUD9+nLeZ/LwSYmIiuOuOQbSMcjSOVqtVTp9bABoXRYRvWbqA5OyzSLLMubyLVzeuJToklB5NmrlkTG9DhjohoOAuFCephvny7T+wmGwVtlnMNj5+ZTF9hravV6FfWZbZf+AUO3el4e9vYGD/tgTVo+LXdg0aMnvoqArbigtNJO86gV+gkdiOzZx+HuK7t0QSK+d+G3x09BlaPdEHBYW6xi+/bStvSHsOq1Ukac9JTp3Ko0kT7+nDUxvo1GrGxbRl4eFkLOe1pTBqNNzWKfGyrjVqcneGXd+FnKxCAoN9MfgoEaSqMIs25ifvY9nRwwTqDdzYoTO9mjR3t1lXjSAIjBrZiVEjnUdpAgN9iI5uyKFDGRUyIfR6DSNHdKxxe1LyckjNy0W6wDMziSJfJO1QnCSFGkFxkmqYw/ucpzEU5pdSXGDCv544CXa7zEuvLmLz1qNYLDZ0Wg2ffL6GV14YT2JCpLvNcwsLvt7A1+/8iUarRrbL+Af58MqXt9I0qmJhcFCIHzc/NIxv3luBzSpit8sYfHS0S2hBD0WpR8FLOXrsrNOeS1qtmlOnFSfpSnih3yAKLGZWpx1Dp1ZjlSSmxsVzU4fLT43UaNU0VP4GF8Usioz/bS5H83PLFW7XnjjG/V16ck+C9/cMe+a/Y3jg4e8pKbEiSRIqlYq4tk2YOKFbjY+VazKhUTmPUGWVltT4eAr1E8VJqmGCw/woKaqsYKZWq+rV6tuadQfZvPVoeeqMxep4YDz/8kIW/Hw/Gi+Vja2KfduP8827y7FaRKxlzXjNJhtP3/olX/39eKWI0nU396Fdl0j+nL8NU4mFPkPb02NwnFLIfpXkmEopslpo5h+IuooHrIJ7iG3TiP0HTiNeEEW1WUVatPC+hqW1gUGj5eMRY8ksLuJUcRGtgoIJquP9+jyZRYeTKzhI4IhsvLdtI1Pj4ut8r8RLERERxI/f3c2WrUfJyioktk0jYmMbuSSDpl1YODZ75UUVvVrNoEhF0e5ysMvKs7AqFCephpl850A+eGFhhZQ7vUHLiEnd0Oqq93YnHzzNR5+s4nBKJgH+RqZM6s71YxPrVKre8r/2ljtI52O3y+w7cIpO8XU//eByWPLDRqyWiu+HLMsU5pVwaM9JYjtWfj9i2jclpn3T2jLRq8kzm3jgryVsyUhHLagwajS81v8ahrV0f7d4BQfjr+vCkmW7kSSpvLZBr9fQu2c0DRU58asiws+fCD//8p8P703nly/WcTotm/Zdohh/az+vlvKuLZYfO1LBQTqHVqVmW8Yphka1doNVtYtGo6Z3r2iXj+Or0/F4j768tXl9+XuuV6sJNfpw8xVEShUUnKE4STXM4DGdyTtbyI//txpwdHweNKYztz42olrnpx7N4pHH52IuizZk5xTz+Zdryc0t5vZbBrjI6ppHUCQlK1CQV+q0qFVQCZQUKh3BXc0ty35j39kzZSuPEibRxoN/L+MX/0DaN2jobvMUgAZh/nz03gw+/GQVSbtP4OOjY+zozkyf1tPdpnkVm1cd4PWH52K1OMSFjh8+w8oFO3h/wf00ala93kkKzgk1+qASBOwX3OxlIOgyxDIUqsctHRNoExLK57t3kF1ayuDIlsyMTyDQoLzX1UYWFOGGi+AyJ0kQhHuA/wCNgP3AQ7IsV27N7jh2ALDaya62siwfPO+48cBLQCsgFXhKluUFNWz6VSEIAhNvG8DYGb3JziwgKMwfH199tc//9seNFGtEClsKSHrwyZKRM0R+XbiD6VN7YawjkqvDh3Vg996TlaJJapWK9nH1T8K699B2JCelVRL1EG0SsZ3rV1SttknJy+FgztlKqRlWUeKLPTt4d/BIN1mmcCHNm4fyxisT3W2G12K323n/uQUV5LwlUaK0xM63s1fwxP+mutG6us+M9p1YknKwQjRJAAJ0ero0qn/Pvdqgd7MW9G7WgrMZ+ZSWWPDT1o05kkLdwCWJiIIgTAbeA14FOgMbgT8EQbjUbLAdDqfq3OvIedfsCfwE/AB0Kvt3viAIHlkNqdNradwi7LIcJICt2adIH6iioLVAcaSKswkqMvqqEDQCmWcKXGRtzdOvTxt694zGoNegVgvo9RoMBi0vPDuu3tUjAVxzfRcaNw9Fb3T0exEE0Bu13Pqfkfj6KateruRMSTEaVeXPnB2Z1bsOU+SkhlBBwRvJO1tMcYGp0nbZLpO0KdUNFnkX8eERPNtnEAaNBn+dDl+tlib+AXw/ZiKqOpQuX5fIzizgoYkfcNvQt3howgdM6/USW1Ynu9ssBS/BVZGkR4CvZVn+rOzn+wVBGA7cDcy6yHlZsixnV7HvIWC1LMuvlP38iiAIA8u2e8Xyl02SSIm2IKv+vZnKGgFrgExOYzsNGtSdvHyVSuDpWddy8FAGO3el4eOrZ1D/WAICvLtwtSr0Bi3v/nwvKxfs4J+/9hEY4se103oSl9DC3aZ5PW1DG2CVnHSLkWRU6TZmf7CCZ2aNqX3DFBRqGR8/PbKTRtUAAfVEedXVTI2LZ0x0LLsyM/DX6YgPj6hT9cR1CVmWmXXzZ5xOy8FepoxpLrXy2oM/MOe3+2neWkmlvhQyYFfKI6qkxp0kQRB0QCLw9gW7VgC9LnH6dkEQ9MAB4GVZls9PwesJvH/B8cuB+6qw4w7gDoDmzetGOtPes2fQalXYLpDBlTUCqrZG/C4zKuUJZBhNfCbv59ipPAJ//Yfb47twT0L3ermqpjdoGTW1B6Om9nC3KfWKEKMPMzsk8sn2rcjn7niSjNoGfqn/z959h0dRbg8c/87ObElCeoAQeu+9gzRpgg0p0uxdvJZrvd7r1Wv74VWxixd7RaVJEaT33kKHAKETAikkJNk+M78/AjFlg6C7mWTzfp6Hx4fZMgcJu3PmPe85GqsPJaFpOiZT5fuZFCqXkDAr3Qe2ZMPSfXjcv984sIaYGX5PLwMjq7g0TWPr6oOsW7yb0DAbg0Z0IiYhij2LjrB2wyHCwqyMuqUz/fs2F8mSnyXtOkl6anZBgnSJ16Py69QNTHhxmEGRCcEiECtJcYAMnC12/CwwoJTXnCF/lWkLYAFuB5ZJktSn0D6m+FLeM97XG+q6/inwKUCnTp1KmQNdvlgVBUwSlJwjSoNaFa8F7uYzp3h48VycF+uzs11OPk7ciN3r5tmuvQ2OTqhMnuvWiznfbCajro5mhpCzOlFJOrIbNJOOrusg7qYJlcATr43Anuti16ZkFLOC1+Plptt6MGhEJ6NDq3A0TePVR75lx4ZknHY3JpPE/Gmb0BvG4XR78VxsZ//2+wtJOniGRx7sb3DEwSXj7AWfN7dUVePsqfMGRFQxicYNpSsX3e10XU8Ckgod2iBJUj3yGz/4bPYQjFrEViXGFoLdU3Rzf6hi5rZWvqdcl2fvbllXkCBd4vB6+Wr3dh7r2B2bYjYoMqEy0C92mJIkCUmSGJjQkHWrDxWZBi9J0KpFTTF/qpzJdbtZeuwweV4PvWvVo3aEaE/tLyFhVl797G7OpWSRdiaLOo2qER4pSu3+jM0rDhQkSJA/4sJlMeHNdebf8LzI6fQw+9dExozqSmxMFaPCDTpN29TG7VbRzPl7TiWPikR+1UaHawLfhlwIfoFIktLJXwspXgxaHUi9ivfZBIwp9PtUP7xnuSZJEl8MuYWxc6fhVlVUXUfTNYY3bcGQCjjPJTkrs5RHJNLsdnHhIwTE+fQcPv7PbDYu2ws6dO7bjL+9fAt/e6g/e/aewuHw4HR5sFoVLGaFpx6/zuiQhULWnz7BfQt/QQJUPX+V74G2nXmqyzVGhxZUqiVEUS0hyugwKrTVv+0qSJAu0ULMRRKkSyxmmaSDqfToFvyzksrKuaw8vA1ifu/WqGmEZTiIig5j4PDOxgYnBAW/J0m6rrslSdoGDASmF3poIDDzKt6qHflleJdsuPgebxV7z/V/MtRyqWlsVTbe8RCrTx4j02mnS41a1IuMNjqsP6VpTBzn7HkljktA1dCwsg9ICHpej8qTt35M2pksVG9+nfrmlft5YuRpvlj6HN99+QCLlu4h6eAZGtSrynWD2xBZSZuJlEdOr4f7F84usZr++a6t9K5dj841xHBlofywhVqQTFKRZhiSVwNdz1+mLkTVdGJjxSqSv+TkOnn6nz/jcHt/T0pNMu6ECN74/D5Cq1S8PdxG0BHldpcTqHK7d4DvJEnaDKwDHgISgP8BSJL0LYCu63dc/P0TwDHy5ylZgNuAYcCIQu/5PrBakqR/ALOBW4B+QFDdXjy05xTJ+05TvVYM13ZriclUccuA/t65J1tSTxcpuQtRFB5s1xmbUi4qPYUgs2nFfrIz8woSJABN1cnLcbJ+8W763tie4Td3NDBC4XLWnjrhc2eY0+tletIekSRdpSPH0liweBe5OS6u6dGY7l0aitJSPxo8ojPLZ28vMndKyXahhhe9QJdliRrxkTSpBN3WnF4vX+3exsyDezFJErc2bc0drdpjkf07+mPVmgNFSqcvkc0y23ae4PoaFfPmslC+BORKVdf1nyVJigVeIH/e0R5gqK7rxy8+pXi7OQv5K0S1AAf5ydL1uq4vKPSe6yVJGgO8BrxC/jDZ0bqubwrEn6GsuV0eXrr/K/bvOA46mGQTMdXCeWvqw0THhRsd3p/SoXoCXw0ZzqvrV3LwfDqxthAebt+VO1u1Nzo0IUidPpqGy+kucdyR5+Jk8jkDIhKuhlfz0bWG/LudLq/vxwTf5i/axQeTl+Lxqmiazqq1SbRsUZM3XhmJIhIlv2jatjbjHx3Ad+8vQTGbAAlJgjvHX8PPv27H7faiahqNG8Xzyr+GBX13O03XGffrNPaln8N5cezCpK1rWXHyCN9fP8qvf/7M83ZcLk+J4263l8zzJStYhNKJlaTSBex2vq7rk4HJpTzWt9jv3wTevIL3nAHM8Ed85c1Pk5exb/sx3K7fV11ST3p457lpvPrFvQZG9td0r1mHBaPuMDoMoZKo06gaVpsFR56ryPGQMAt1Ggf/XdyKrnvNOng1rcTxUMXMjY2aGRBRxZSb5+L9yUtxF2rz7XB62LPvNKvXJnFtn+YGRhdcRt3fl/7DOpK47hC2EAud+jTFajMzenwPTp3OJDTUStUKeqOzOIfHw5Hs81QLDfNZMr/65DEOZKQVJEiQ36wp8ewZtqSeposfV4LbtKqF1WrG6SyaKFksCm1b1/bbeYTKTdQ8lROLZmwpkiABqF6NxPWHcDrc2EIsBkXmP6qqsWTGFhZO24Tq1eh/S0eGju2OxRpcP4ZeVePbH9bxy7xE8uwumjWpwWMP96dZkxpGhxb0OvdpRkzVcFJdnoKSO1kxER4VRo+BrQyOTvgjkVYbr/UawAtrluLVNFRdI0Qx07dOfa6t28Do8CqMnbtPoCgm3MUWVZ1OD8tXHRBJkp/FVA2n/7AORY7Jsom6dSre6I7SfJK4mQ+2r0eWTHg0lT616/PetUMJNf9+bbL97GnsXh+rO6qX7WdT/JoktW1dm9Yta7F7z0mcF6+dbFYzbVrVonVLUZYr+EdwXZ1WYB536aUkapCUmUx87Du2rk7C5cj/ED2ZfI61C3fz5tSHKvTeq+Lefm8hK9YcwHXxg3vfgRSeeO4nPvvwTmrXijE4uuAmKzKTfn6EKa/PZd2i3QB0H9CSB1+4CbNFfNxVBCObtaJjfE1+ObiXXI+bAfUa0T2hdtCXKvmT1eJ7vIIkQWioGL0gXJ35yUl8sH09jkL7i1edPMpzqxbx4YAbC45VDwsnRFGKPA/AoihUD/Vv0wpJkpj48gjmL9zJgsW7kYAhg9tw/eA24rPiKuhIotzuMsRVQ4Bkns9j556TVAmz0r5t3T+sAe8+oCXLZm8rsuEcoF6TeMLCK373rUN7ThVJkABcTg/J+06zdXUSXfoGx53NzPN5LFu1H4+naGLrdnv5ccYmnn1iiEGRVR6RMWE8O2ksTBprdChBz62qLDx6kG1nT1M7PIoRTVoSbfvrn1f1o6J50kfLb5fTQ+a5C8RWj8BiFRf7pWnXprbPBg1Wi8L1g9saEJFQkX2yY3OJxMelqiw6epgLLhcR1vxGFTc0bMobG1eVeL3FJHNdff/PLVIUmZtv6MDNN3T44ycLwp8gkqQA+O6nDXz743oURQYp/4tp0uu30rB+tVJfc/fTQ0hcd5CcbAdOuxuLTUFRFJ767+gyjDxw9mw5iqaW3GvgtLvZtTE5aJKk0ynnsZiVEkmSpukcOnzWoKgEwf8uuF2MmPMDp3MuYPd6sMkK721bx483jKZ11Xi/nkvTNL6ZtJA536xFMkmgw/B7e3Pb44PEXWMfFEXmv6+M5NkXpqNdnDWlejXG3dpV7NcQrlqaj1EeALJJItvlKEiSIq02frxxNI8sncfZvFwAaoZHMHngjYSYxU0NoeIRSZKfJe46wfc/b8TtUXFfvFC229088+/pTP/m4VLbr0bFVuHThc+wYl4iSTtPULN+VQYO70RUkMxViI6rgmJWSpQVWqwKsdUiDIrK/xJqROH2eEscN5kkGjYoPUkWhIpmcuJGjmdn4b7Ykc6pekGFJ5bPZ9lo/zabmTZlJXO+XVuk1fLML1ZRJTKEW+7u7ddzBYsWzRKYNfURNm87it3upkO7OsTFBkcDAaFsdUuoxa/JSah60ZbbVlkhoUrR7+9WVauzcsy9nLiQjUmC2hFiYHF5p/kcvCCASJL8bu6CHTh9tKV0ODzsPZBCm8tsKLSFWhgyuitDRncNZIh/2uGsDCZuXMXm1FNEWW3c17oTd7Rsf0V3crsNaMnH//mlxHGTbKLfzcGzVB4bU4W+vZqyet3Bgj1JABazwthRf/3v1atqLF25jyXL92G1KtwwuA3duzQUd9OFMjc3+UBBglTYqdxsUvNyiA/z3wX5rM9XFSnVBXA5PEyfslIkSZdhsShc093/ZU5C5fJkp2tYfuIodo+7IFEKURT+0/NaZB/7iSVJom5kFE67iyUzt5CReoGmbWvTrmdj8V0lVCgiSfKz3GKthy+RJHA4Ss5vqShO5mQz7JfvyfO40YEct4s3Nq/iRE4W/+5+7R++3hZi4b/fP8SrE77hfHpu/gbiKlaef/+2oFktu+S5vw8hLjacufMTsTvcNGkUz+MTBlC3duxfel9N03n239PZeyCloO3pth3HuX5wGx57sL8/QheEK2YupdmKroPix0Ysuq6Tk233+dgFMQ9FEAKubmQUC0beweTETWxKOUWt8AgmtO9Kt4TSSzePH0zlmTGT8bi9uJwerDYz9ZslMPH7B7HaROlduaGLOUmXI5IkP+vXqxm79pwqsZrkVTVat6jpl3M4vR52p58lwmKlSXRcmdyZmbJjM06vl8KL7Q6vl+/37eDR9t2JuoLN2g2aJ/Dl8n9w4vA5NFWlbpP4oOpqd4miyDx4Tx8evKcPuq777e9n09Yj7CuUIEF+S995v+1kxE0dqCkmjAtlaEyzNry/bX2RmSgmSaJFbDXiQkrOUPmzJEmiVoOqnDqSVuKxuk38u/dJ+HPcHi9mRRarBEGsdngkE3sPuuLnT3zse3Kz7Vyq0HPa3STvO8WsL1Yx9pEBAYpSEPxLJEl+NrBfC35dtJPko2k4nR5MJgmzWeaxh/oTGmr9y+8/PWk3L61fhkmSUHWdmlUi+Grw8IDX/W4/dxqvXrLxgsUkk5ydSUfblSWAkiRRtxIN9fTnRcPGLck4nCVLOSUpf0VJJElCWbq3dSfWp5xga+ppNF3HbDIRZrbwYf8b/H6uh/59M68+/E2RPUlWm5kH/3WT388lXLkNm5P5YMpSUs9dwGY1M+Kmjtw9vmepe2+FiiUn287Pnyxn/cLd2EIt3HhHTwbf2uUPb26mp2Zz5ng6xbYw4XZ6WTpzq0iShApDJEl+ZjbLfPDfsaxcm8Sa9YeIjAzhxuva0rjhX08MdqWl8uL6pUVacR7JyuS236az8tb7AnoXr2FULAcy0tEo+qnn1lRqVgmexgvlWWREKIpswlusS6BsMhERBG3ihYrFIst8O2QkO9NS2ZV2hhphEfStUx+zSfb7uTr2asrr39zP9+8v5sThc9RrEs/tTwymWbs6fj+XcGV27jnJS2/MKdh7aXe4mTZ7Cw6nm0cfEOW/FZ3T7uKxm94j/Ww23osNl6a8OocDiSf4+39vvexr9eLZ0RU+JpQ9HVFudzkiSQoARZEZ0LcFA/q28Ov7frN3O65ig2U1dNLseexIO0P7agl+PV9hD7XtwpLjh3EWStCssky/2g38ukFbKN11A1rx08zNJZIkk0mie+cGBkVVuaXm5TD94B7O5OXQM6Eug+o1CkiSUF7kZOWRk+2geq0YZNmEJEm0q1aDdtVqBPzcLTvWZ+K3Dwb8PMKV+XrquiLNaQBcLi9zf9vJfXf0IsRmMSgywR+W/rKN8+m5BQkS5DdLWTF3O2P/1p/4y+yxrVojivg6sZw4VHTshcWqMGB4p4DFDJCekcOO3ScJr2KjY7u6+aNYBOFPEklSBZJmzyuxkgP5+wCynM6AnrtlXHU+HTSMf61ZQmpeDiZJ4uZGLXilp7hjWFYSakTxr6evZ+I7CzCZJHQdrFaFif8ZgVUM1ixz61NOcO+iWaiahltTmX1oH//bGcO0G8cSogTX30feBQdvPzWVbauTMMkmrDYzj7w6gt7XtzM6NMEgJ0+f93lcNklkZOZRK0EkSRXZjnWHcPloNqUoMgd2nLhskgTw/Ae38cyYyXg9Kk6HG1uIhXpN4xlxf98ARQxffr+WqTM2Yb6YGFksMu+8PvqyMyoFsZJ0OSJJqkCurduQLamncKhF7955NLVM7uT2rlWf1WPu54LbhU1RsMrix6es9bmmKd26NGTvvtOYzTItmiWI+n8DaLrOY8vn4fD+vkfG7vVw6HwG3+zdzkNty2cb/z/r9Ue+Yc/m5II5Zy6Hm3ee+ZGqNaJo3qGescEJhmjUoBrpGTkl9p3oQNUg61haGcXXiUUxy3iLDUZH14mrHvmHr6/XtAbfrn2BNQt2kp6aTdN2dWjfs3HAmjVtTTzGz7O24PGoBcPc7Q545sUZzPjmYUwmkQgIV09cXVUgtzZpRY0qEdgKJSchiplH23cn+gq6y/0Zuq5zIPE4Mz9byfLZ23A5PURabSJBMpDVotChXV1at6wlEiSDHDyfjt1TsomGU/Uy+/B+AyIKnHOnz7N3y5ESg6DdTg8zPl1hUFSC0e4Z3xOLpej3gM2qMHZElwq3sn3mbDYff7GCp1+czldT13E+y5jW8g6vh/9uWU2XqZ/Q6YfJvLxhORfcvseKBNrQsd2QlaLfLybZRHS1CFp2rn9F7xESZmXQqC6Me3QgHXs1DWg32zmlzqh0s+9ASsDOKwQ3caVbgYSaLcwddjvf79/BomMHibKGcFfLDvSpfWUfWFdL9aq89vDXJK47hOpVMVsUPnlpFm/8OIGGfmpnLggVkcUko/sofYX8vXrBJONcNopFwV1s/4muQ+rJTIOiEozWpFE877w+msmfr+BQ8lmiokIZP6obNw+tWCWY+5JSePKFaXi8Kl6vxs49J5k5bzufvns7CfGB7RpbmK7rjJ3/M/syz+FS829IfL9/B6tPH2Xh8LvKfK9jQt04XppyN28//RN5OQ40TadRi5o8/9Ht5bLVu72UOZSSBA4fyZOQT0cS5XaXIZKkCqaKxcJDbbvwUNsuAT/XommbSFx3sGDS/aVl91cf/IqvVv+rXH5QCkJZqB8ZTY2wCI5mZxZJlUIUM+ObV6yLxD9Sp1E8Xre3xHHFLNO2eyMDIhLKi1bNazJ50m1Gh/GXvPXhoiKjFdweFa+q8clXq3j1+ZvLLI6NZ06SdD69IEGC/O6xZ3JzWHL8MEPrNy2zWC5pf00Tvlv/AqknMrGFWoipZnwnW7vHzbELWcSHVSHGFlpw/Nrezdiz73SJ1SRV1WnVPHBNrYTgJpKkCsrt9LDgx/Ws+GUbFquZIeN70Pem9n5dzl7486aCBKmwrIxcTiafo06jyjPvSAis/QfP8MGny0g6fJbwKlZG3dyJcSO6lts6ckmS+HTQMMbM+wmn6sWr5XccHFS3ESObtDI4Ov8KC7dx68P9mTFlBc6Ld2tNskRImDWgm7AFIdAcTjfHTmaUOK5pOlsTj5VpLLvTz+LR1BLH87wedqWlGpIkAZhMJhLqxRly7sJ0XeeDxA1M3rkJxWTCo6kMqtuYt3sPwaYoDOrXkgWLd3G48IxKRebJRwaKTot/QBcrSaUSSVIFpHpVnhv7MUf3pxQMVzy89xQ71h3kybfG+u08mlpyeCzkL1+X9pggXK2jx9N54p8/4bxYzpWV7eDbnzeQkZnL4w+W36GDjaJiWT/uIVaePEKaI4/O1WvRJMb4i4lAGPfYIGo1rMbMKSvIysyl/TVNGP/YYGKvYAO3IJRXZkXGZJLQtJKlsyG2st1XVSs8Aous4NGKlo2FKAp1AjwsviKYdWgvn+zahFP1wsVccsnxw7y4filv9r4Os1nm/TfGsnr9QdZuPERURCg3XNeWBvWqGhu4UKGJJMkguq5z6KT/fBQAACAASURBVMg57A43zRvHX9VG141L93Is6UyR6fNOu5tV8xIZ+eC1flvhGTC8E6eSzxU5D0BYeAh1GotVJME/vpu+AXexDkoul5dfF+3invHXEF7FZlBkf8wiywyq17jEcV3XcTk9WKxKQDcrlxVJkuhzQ3v63NDe6FCCXrbLyfyjB8h2ubimZl1ax8UbHVLQUhSZvj2bsmrdQTyFZhBaLQo3Dy3bn/UBdRpRxWzB4fWgXWwZKAFWWeHGBs3KNJbyaPKuTTi8Rct+naqX2cn7eKVHf2yKGUWRubZ3c67t3dygKIVgI5IkA5w8nckzL8/kfFZewV2sJx8eyOB+La/o9YlrknDaS25SlCTYsynZb0nS0PE9WbdoN4f3nsKZ58ZiMyPLJv758R1BceEnlA+Hk8/5vJNrNsucSc0mvFH5TZJ8WTVvO5+9PpfzaTmEhFoY+eC13Dqhv/g3I/yh9SnHuXfJLHTAo6q8n7iOIfWb8k7voWIPaIA8OWEgaRk5HDiUiiKb8HhVenRpyPiRgd/3W5hFlpl54zj+vnI+O9LOANAspirv9rmecIu1TGMpjzIcDp/HdSDH7cYWZLPpypKG+GwpjUiSypiqajzxws+kZ+YWmS/x9uTFNKxXlUZXMPQspnokZouCp9hmapPJRKQf51NYrAr//XEC29cksWfTEaKrRdDv5g5ERIf57RyCUL9uVU6cPo9ebOCK26MSX934jcJXY/Pyvbz7zE8Fq695OU5++ngJXq/KbU9cZ3B0Qnnm0VQeXDYbe6HZW15VY+Gxgwyq25gh9ZoYGF3wCgu18sHEsRw9nk5KahYN6sZRowy72hVWOzySGTeO44Lbha7rRFor1g2iQOpUvSZLTxwu0VM0ymIjNiTU52sE4a8StzbL2K59p8izu0sM4PN4VOb8tuOK3mPgyM6Y5JKZv9mi0LlfC3+EWcBkMtGpT3PuevZ6br6rlyEJktPl4bdle/j4yxUsXL4HV4Dbeeq6zqJjhxi34CdunPMtn+zcRJ7Hd3tR4a+7fXQ3LJai7W2tVoVB/VoQER6Y+V+B8u2k30qUp7ocHmZ9trLkUEZBKGTr2dMFZVaF2b0eph/cbUBElUv9unH07NrIsASpsAiLtSBBSjuTxYbFuzm480SJG0mVyXOdexNqtiAXWlENURRe6dEfk1hl/dN0HTRdMuxXeSdWksrYhRynz+OappNx/soG2FVNiObfU+7hzce/w+NW0XWdqNgqvPT5fViswfVXejbtAg8+/T0OhxuH00OIzcxn361hytu3Exegqe4Tt6zi2/2JOC7e0T14Pp1Zh/cy7+bbxZJ+ADSqX423/jOS96Ys48ixNEJDLAy/oT13j7/G6NCu2tlTvucGeb0quRccRAXoZ1ao+C53AVzaTC4heOm6zscvzGDxtE2YLQqaqhFfO5b/mzqB6KrhRodX5hpHx7Jg2B18uGMD286mUC8iikfadaNzfC2jQxOCWHBdUZdDuq6j6Tryxf0IrZvXxOsteUfZZjXTs8uVzxzp2LsZU7e8wpH9KZgtCnWbxAdlzfo7nywhK8tecIfV4fTgcnt5/7NlvPoP/8+wOJuXw9f7thWZVeFUvZzMzWJ28n7GNG3j93MK0LZVbb768C40TS+3bb+vRJ3G8ezberTEcavNTHiUKAkRStexek2fOwNCFTMjGwdXW3nhjy3+eRNLZ2zB4/Liudj582TyWd742zf89+e/GRydMepFRjOpz1CjwxAqEVFuFyCarvPRjg20/eEDGnz9Nr2nf8qyk8nERIcxdngXbIW62VmtCrUSohjQ5+o6ssiKTOPWtanXtEZQJki6rrNp+9ESJSiaprN+S3JAzrn1XIrPyeYOr5flJwNzTuF3FTlBArj72euxFmsdbA2xcPuTQ5Bl8XFbnuxLSuG5V2cy7qHPefnteT7n5ZQlq6ww+dqbCZEVrLKCRH6C1LdWA4bUM2ZGjmCcOV+uxuUoWuatejX2bztGVnqOQVEZ58zxdL55az4f/ONnNizejSrGkPiNrkuG/SrvxEpSgEzavoYv9m4taFl5PCeLCcvn8PWgkdw7/hpaNkvgl/mJ5Oa56HdNU24Y1AarRfx1FFfaDItAXUzH2kJ8FrbIkkR8qCiVEi6vVZeGvPL1A3zxf3M5djCV2OqR3PbEYK69pZPRoQmFbNx2hH+/MQfXxeY3KWezWLclmY8mjqVJA+PGG/SuVZ81ox/k1yMHyHY56VWzHh2qJQTlTTDh8uy5vkvzTbKEPddFVFzlKblbt3AXbz72HaqqoXpUVszeRuM2tXn9+4cxi+smIYDET1cAOL3eIglSwXHVyzvb1zL9+nF069iAbh0bGBRhxSBJEr26NWb1hkNF7hopiol+PQNzZ7VLfG2iLDYcHg9aoXTJbJK5rXlwzYjJdjnZnXGWqiGhNI0WA/f8pU23Rrw/90mjwxBKoes6705ZWpAgQf7qtNPp4eMvV/L+a6MNjA6qhoRxd8uOhsYgGK/boFbM/25diYYvYREhxNeJMSgq/9M0Hacrf7+xr5sBbqeHd578AXexuZAHd55k+S9bGTy6W1mGG4QqRgMFo4gkKQAynXZK22d7JNv3xm7Bt78/NIDDR86RcT4Pj0dFMZuoXjWCR++7NiDnM0kSPw4dzT2LZ3I6L6egk86b11xHk+i4gJzTCO8nruPjXZuwmEx4dZ0GEdF8PWgk1cRqmRDknC4P59Iu+Hxs/6EzZRaH16My9+vVLJy6Aa/HS+8bOzBqwgDCwkXbZwHGPDqIdb/tJOe8HZfTgyybUCwyT749Nihmrum6zrQ5W/l22gbsTg9Vwqzcf1svbhrctsjzDuw4Dj5267kcblb8sk0kSUJAiSQpAOJCwkptSdk0iC60y0JURCjffnwPW3Yc48SpTOrVjqVTu3oB3btSNyKapSPu5VBWBnavhxYx1bDIJfcpVVRLThzik92bcaleXBdvUh44n8aDy2fzyw23GRucIASYxaygmGVUl7fEY1Fl2HL+9Qe/JHFNUkHL+FmfrWDD4t189NszooRIICq2Cv9b8g8W/rSRnesOEl8nlhvv7EVtPw2LN9qMedv4YupanBf/HWZfcPDhF8uxWRUG9W1Z8DyzRSm186PFJrrNCoElPokDwCLLTGjTjY92bSxoIw1gkxWe6tjLwMgqJlk2lUl5oiPPBUBImBVJkoJq5aiwz/dsLfJzCaDqOvsyznEqN5taVSINikwQAk+WTdw0uC1zF+4sUnJns5oZO7xLmcRweM8pEtcmFZmp5XF5OXc6k3W/7aLvzR3KJI7yItvlZN7R/Zy159KxWk1616wvZt+QX1o34oF+jHign1/fNzMrj4PJZ4mNDqNR/WplvudN13W+nbaxIEG6xOXy8uXUdUWSpKbt6hASZi34fr7EFmph6LgeZRJvsKsIDRSMIpKkAHmkbTciLFY+3rWRdKedplFx/LvrtXSsVtPo0IRiUk9kMOnJH9i/Lb91c7MO9XjqnfHUqBucSVKG0+7zuGIykeVyiiRJCHoP39mHPLubpav2oSgyqqox4oYODBvSrkzOn5R4rMRAcQBnnps9Gw9XqiRpd3oqYxf+hKprOLxewhQzTaOrMvW6MdgUcYniT7quM/mbVcyavx2zWUZVdWrViGLSf0YRE1V2g+JVVeNCrsPnY+cyinbuM5lMvPzV/Tw/7hNUVUVXdTRNY/DobnQd2NLnewiCv4hPoACRJIk7WnTgjhaV58uuInI7PTw57F2yM3ILuujt33qUJ4e9y9frX8IaYjE4Qv8bWKcRxy9k4dZKzutqEhWciaEgFKYoMv949Doeubsv59IvUKN6FKFl+G89rkaUz5bwFqtC9TqxZRaH0XRd55GVc8j1/N7qOs/rYV/mOb7et42H2nQ1MLrgs2zNAWb/tgO3R8V9sSHE0ZMZ/PvNuXz8f2PLLA5FkYmLCScto2Qr89oJJZtSNGpdmx+2vMyWFfvIybLTpntjEuqJ7yoh8Cr+7r9yzu3ykLgmie2rDxTpziKUD+sX7sJpdxdpM65pOi6Hh3W/7TQwssB5oFUXYkNCsV7cZyWRXwr6WveBQbX3qrI7mXKeb6dv5Muf1nHo6DmjwymXwqvYaFivWpkmSACd+jYnNNyGVGxvpazIDBjZuUxjMdKJnCzO2fNKHHeqXmYc3mNARMHt57lbcbqKlVqrGgcOnSE9M7dMY3nozj5YrUXv01stChPu7uvz+RabmZ5D2nLd2O6XTZB0XWfekf2M+e1Hhv36HV/v24bTW3L/oZBPBzRdMuxXeSdWkgIocU0Srz/4BbpG/pWoDs9/ched+rUwOjThojMnMnAWG9gH4LC7SD0RnJ0Io20hLBp2N9/u386q00dJCIvgnpadaFe1htGhCX7yy2+JfPT1KlRVQ9d0pv6yheFD2zPhzj5GhyaQnwy9NeMxJk74mmNJZ5AkiZhqETz74R1EV40wOrwyc7l9R2JPkv/l5vmevSTLJnLzXMTFlF1304F9mmOzKnz2/RpSz2VTOyGGB+/sTZf29f/S+/5z/WLmHNmH/eK+26TMNOYc2c/0oeNQgqAroFC2RJIUIBfO5/HKPZ+VuAB/7YEv+Gr9S5Xqi7A8a9iyJrYQS4lNoSGhVhq0SDAoqsCLtNp4tF0PHm0nNr4Gm/TMXD76amVBOQ2Ay+1l1m+JXNuzKc0axRsYnXBJjbpxfDD/aTJSs/F6vFSrFVPphsbWqhJJQlgERy4UvSFlkxVubdzaoKiCV88ujZg5fzter1bkuMWsUDshuszj6dWtMb26Nfbb+x3NzmRW8l5c6u8rRw7VS9L5NJacOMSQeoGZr1ih6fjcHynkE2l1gKydvwPdx7AkXYfVcxMNiOh3mqazZPV+/vbCTzz03A/MXrgDj6fk/pTKoGPf5sTXiS3SclexyFSrGU3n/pVrU6jH7WXV3O18NXEeS6Ztwml3/fGLhHJn/dYjPlvku91elq9LMiAi4XJi4yOpXju20iVIkL93d3K/m4my2AhTzCiSiVDFTKdqNbmj+ZXt53WpXv5vywraTH2PJt++zR2LfxbzCEsxfnhXoiNDsV78vjOZJKxWhWcfGexzj1xFs+nsSZ8rkHavh9Wnj5V9QEKFJ1aSAiQ3215iUjbkX4jm5fju6lJWJn74Gys3HCqoTU4+nsbSNQd4/5Vbg+KD8mrIsom3Zj7Od28vYOXsrehA32EdueOpoZXq/0V2Zi5P3DCJ7IxcHHkubKEWvvy/ubw790niK9FG8mAgmyR8DV+UJAlZlJsI5UyzmKpsGP0wC48fLGgB3qlazStOGh9eMZt1Z44XrB6sSTnGsPnfsuyW+6kaUnYd2yqC6MhQvnn/buYs2snWnceIrxbJyBs60KheNaND84sYa2jBAPjCzCZZ/CwIf4pIkgKkQ+9mTH1vIWrxZW2rmfa9jFvyPXI8jRXrDxaZD+J0eTl45CybEo/So1NDw2IzSli4jYdeHs5DLw83OhTDfP7qbNJSzhf8vDrtbtxOD+898yNv/Py3Ul+nahpHL2QSaraQECZKSMuDHp0b8u5ny0ocNysyA3o3MyAiQbi8EMXMLQ2vfuU+OTuD9YUSJMjfiO7yevnuwHaebC/mEhYXXsXGbSO6ctuI4Osc2LdWA8ymks2HFJPEKFG+WSrNx001IZ9IkgKkUeva9L6xA2t+TcRpz9+XZAu10G1gK5p1qFcmMWzffYIPv17JsZPpxESFcdeo7rjcHh9FgOBweti683ilTJIEWP/brhIJvabp7N54GI/bW6Qc8ZIVp5J5eu0CHF4Pqq7TPLoqn/QbRg2RLBkqOjKU5x4ZzBsfLUSSpIJp9Xfe2p2GdasaHJ0g/DG3y8PinzeyYtZWrDYzQ2+/hp5D25ZYXTqUlZG/Gb9Y0YZLU9mVnlqGEQvlgUWW+WnIGO5bOosMpx2TJCFLJt7rfT21w8X8P+HqiSQpgP4+aRw9rmvD4p83ouswYFQXug9uXSa15zv3n+KZ12cVrBidTc/h/S+X06drYxTZRPF+bhazXKbD5ITypXgr4oLjEj5/XpOzM5iwYjaOQndwd2ekMn7Rzyy75b5Kub+iPBnYuzkdWtdh9aZDqF6NHp0bkFA9yuiwBOEPqV6V52/9kOS9p3A58kvC9287yo61Sfxt4ugiz20QEY1X00q8h8Uk0yImOErIhKvTNLoqq0c+wP7zabhUL61iq/tcXRLy6YBeAVpxG0UUqAeQJEl0G9SaF7+4nyf/dwfnm1uYd+wAWa7A70n69Ie1RUrqIL+sbu2WZHxtSzCZJK7rV7kaFQi/63tzBxRL0S8SWTbRsU9zFHPJL5hv928vMYxW1XXO2nPYnpYS0FgrI7fHy4nTmeSU0sLXl9joMG65rh0jb+ggEiShwti4eDdH9qUUJEiQX/675OdNnD5SdN5Xk+iqdKiagLXYRbBZlrmjmRjkXllJkkSLmGq0r5ogEiThLxErSWVg0fGDPL5mXv6maR28usbE7oMZ3rBVwM559GS6z+NeVeON54fxxkeLyMl1IkkSimLiP0/eUKYzEoTy5e7nb2Lf1qOknsjA7fJisSqER4fx2H/H+Hz+6bwLqD76hkqSxDl72Q4lDHbTf93OZ1PXoJM/+LFv9yY8N2FwQYcqQQgm21bu99lZ02SS2LMpmZoNiq4Qfd5/BK9uWc6s5L24VS8dqtXktW6DiA8LL6uQhXJE9apMn7yUeV+txp7rpE2Pxtz/4nBqNRQri8LVE9+yAZbhtPP4mnk4VW+Ruul/blhEl+q1qVUlMHWyteKj2JdTsibbZJJo16I2Mz59gMPH0vB4VJo0rI5SiTq5CSWFRYTw0aJnSVyTxNH9KdSsX5Uu/VsiK77vwl2TUI91KceKlNsBeDRVDKX1o1UbDzHlh9U4Xd4ix2STxL8eG2pgZIIQGFFVw1HMconusJJsIjw6tMTzQ80WJva4jv/rPhhN10UHx0ru3ad+YO2vO3A581cityzbx97Nyfxv+b+IqyFW1EuS0ES5XanEp0mALTx+EMlH5xBV15h3dH/Aznv/2GtK3Gm2WRXG3NQJs1lGkiQa169GiyY1gjZB0nWdDakn+M+mpbyxbSUHz6cZHVK5ZjLll9eNfKg/3Qe3KTVBAhjVqDVxIWFYCpUyhChmRjduKxo3+NG3MzcWSZAgfzDssnVJ2B3FdxYKQsU3aHQ3n+MXFLNMp34tgPwV1bTT57Hn/l5+Klrc/+7UmfMsXXuAPUkpBY1bKoO0lPOsnpdYkCBB/nWA2+lhzperDIxMqKjESlKAOVUPql5yY6mqaTi8Hh+v8I/O7erx4hND+fCrlZxNv0CVUCtjh3XmtluCr+2nL7qu8+Ta+Sw6cRCH14NJkvh6/zae79SXO5t1NDq8Ci/MbOHXG+9iyp5N/HY8iXCzlbtadOSWBmJfmz+lZ/ouXTSZJC7kOAgNsZRxRIIQWPF14nj+f/fw1mPfoms6mq4THhnKy988iMVqZvXcbUz+5zQceS50Tafn9e14/O1x2EKtRoduOFXVePWDBazedBhFkdB1iK8awfv/ubVSNGY6cSgVi0XBU+zGksetcmDbUYOiKv8qUR591USSFGD9ajbkze2rSxy3ygoDajcK6Ln7dGtCn25N8HpVZNlUqTqOrTtznEUnDmK/mIiquo6qevm/rSu4vm4z4sRgub8s0mrj2Y59eLZjH6NDCVptmtVk5cZDJe4GW8wKcbFiz4UQnLoObMVPuyZycMdxLDYzDVvVQpIk9mw6zDtPfFekqcP6BTtwOz38+8sHDIy4fJixYDtrNh/G7fHivvi/6ERKJq++v4B3XxplbHBloEadODzFyjQBZMVEvWYJBkQkVHRibTrAGkTGcG/zToTIChIgkV+WNKJhK9rElc3eDUWRK1WCBLDgeFJBglSYIplYnSLuKP0ZTruLrcv3smv9QVRvyS8iwf/uH3cNITYzpkIt2m1WhUfu6hu0ZbIVka7r/HY8ibGLf+TG+d8wZc8m7B5RDvlXKGaZFp0b0Kh17YLvr2kfLi6SIAG4XV62LN/L+bQLRoRZrvyycEeJrraqqrNj30ku5F55Z0wjpWXkMHX2Zv73wxoS9568qnLBhPpVad2tEWZr0fv/ZovCsPv6+jlSoTIQK0ll4NmOfehfuxG/HNmLqmvcVL8F3arXNjqsoGaVZUxAyUJHCYssWoJereWzNvPB0z8iKyZ0HSxWhZe/n0DTdnWNDi2o1akZw+dv3cZXP69n94EUqleN4M6R3ejSrp7RoQmFvLZ1OVMP7SwooT6Ulc6sI3uZc/0d2GTxNesvqSd8d201mxUyUrOJrlq590M6XL5L+CVJwu3x+nysPFm7JZmX3pmXv4/IozJj/na6tq/Hq0/dVORG0eW88Nm9fPLvGayYtRVV1ajTuDqPvjGmRFdE4XdiTlLpxKd3GelYrSYdq9U0OoxKY0TDVvx4cGd+V8FCdHT61WxoUFQV06nDZ3n/qam4C22GdeTCC2M+5IcdE7HYzAZGF/zqJMTw0t9vMDoMoRQpeRf4/mAiLvX31VWn6uVkbhbzju5nVKPWBkYXXFp2bUTKkTRUtejtL9WrUrNBVYOiKj96dW7EvGW7S/z/iYupQmw535Pkcnt55b1fi6yEOV0eNiUeY+XGg1zbo+kVvY8t1MrfJ43nsf+OweNRsYl9m8JfIOo1ykB2Ri6/fr2a6R8t5uj+00aHUym0io3n8bY9scoyIbKZMMVMiGLmk77DCDOLD82rsfinDT7L61RVZ8vyvQZEJAjGyMzKY8oPq3nk3z/x5pTFHD+dwdZzp1CkkqvTdq+HlaeTDYgyeI15bDDWUEvR8tNQC6MfH0xImM3AyMqHe8f0ICYqFNvFcjOzIhNiM/OvR4eU+5L7XftPFcSoo5NbR+PMAJXkAU6e27uInelnrur9ZEUWCdIV0PX8lSSjfpV3YiUpwDYv3cP/3f85oOP1aPwwaT4Dbu3OI2+MLvcfWhXdw627cUuDlqxKOYJNNnNtrYaEW0QHpKt14Xweqrdk4aKuadhzHAZEJAhlL+VsFvc+9z0OpwePV2VX0mkWrd7HbY/0wNdHuSxJVA8VzTX8qXrtWD5c9BzfvTWfnesOEh0XzqhHB9F3WCejQysXoiPD+OH9e5i/fA+Je09Qp0YMwwa3Jb5aYOYx+pOpUPv2nMY6OU109ItXqGmKgzGLpjJjyG20jKluUIRCZSSSpABy2t1MfPALXIXmmahelWXTN9J9SBs69m1hYHSVQ3xYOKMbtzU6jAqt68BWrJq9Fae96EZ0VdVo2/PKSiAEoaL739S15Oa50C5uJNc0HafLy4KpO6nS10qex03hLeZmk8y4Ju2MCTaIJdSvxnOT7zY6jHLFrapccDuJtoYQGmJh1PUdGHV9B6PDuiptmtVEMknopqIJ0iVO1ct7O9by2bUjjAlQqJREkhRAO9cm+dxs6LS7WTZ9k0iS/MjucPPtrI0sWrMfCYkhfVtw+y1dsVl/3y+jqhobF+5k9dzt2EItXDe+J807NTAw6oqhy8DWNOtYnwPbjhYkStYQCzff15dqtWIMjk4QysaWnccKEqTCzqXn8GnPsfx906+k5OUgmyRMSLzVYyiNImMNiFSoLFRN460dq/hm/3Y0NEJkM8926Fshk3OzWWbic8N44r0ZQMnOkDqwJ/NsmcdVGWgVoOzNKCJJCiAdHUrpXqlrYnqXv6iqxoR//8jx05m4L85ImDpnC1t2HWfK6+OQJAlN03j5jk/YveEwTrsLSZJYNXsbY/8+hNGPDTb4T1C+ybKJ16Y+wqo521j5y1ZsoVaG3NaT9r2bGR2aIJSZsFCrzzbKEhJNY+JYevN9HM7OIM/rpmVMdcwm0UVTCKxJO1bzzf7tONT8pjouVeXVLcuIttoYUrfifT63b1mb6W/fR885/8PjozdtvYhoA6ISKjPRuCGA2vVsWqLLjA5YQy1cO7KrMUEFoQ2JRzmVmlWQIAG4PSpHTqSzbfcJIH9v2KUECfLnmrgcbn6YNJ/Ms9mGxF2RyIrMtSO68Mr3E/jnp/eKBCkAPJrK9rTT7M5I9bliIRhr1ND2BRviLzGbZXp3aYTVakaSJBpHxdEuLkEkSELAeTSVrw9sK0iQLnGoHt7budagqP66apHhjG3ajpBirfNDZIXH2/Q0KCqhshIrSQFkC7Py7OS7+O/DX6Ghk5sQiTs+ghzZxIcLtvJUQiRtm9cq87h0XQPtHEjhSKby3Rb0ShxITsXhLDkfwu32sj85lU5t6rJ+wc6CBKkwRZFJXH2A/qNE0ioYZ+XpIzy+Zi6qrqHpOhEWG5/3G0Gr2HijQxMuGjW0I8dOZbBw1T7MZhmvV6Nlkxo899CgUl+TuGo/0z5aTPrp87Tu2YQxj18nSlQFv8h1u/FqJVdbAFLycso4Gv96sXN/QhSF75LyW+tXD63CS5370y2+jtGhBSVxT650IkkKsB5D2vHVpld49rWZXDiXBRfL7JJPpPPk6zP5bOJ4GtSOK7N4NMcSyPkPaBcAHd02GCniVSRTaJnF4G/V4yIIsZpLDNKzWhTiLw4XDAu3YZIlNLXYp4EkEVJFdLwTjJOSd4GHV87CUWiml93rYfySn9g06m9iGGk5YTJJPPfQYO65tSdHTqRTo1oEdRJKT3gWTV3HJ/+cVtC458zxNNbM2cZHy/5J9dpir5Lw10RabYSbrWS47CUeaxFTsQenKiYTz3fsx7Pt++BUvYQqZtENWDCEKLcrA7pFITnjAt5i+5DcHpUfZm/x+/l2ZZzhp0M7WZ96vEjZju7eAdlPgZYGuAA3OBejZz/p9xjK0rU9mmI2y0Xa8EoSWCwKvbs0BmDQuB4o5pIXm5IEnfq1LKtQK5XUtAucPHMeXdymuqyZyXtQffw/8moay04eNiAi4XKqxlSha7t6l02QvB6VT1+aWayzqYY918mP7ywoizCFIGeSJP7Rsa/PZSm1lQAAIABJREFUsrTnOvQ1Jig/k00mwswWkSAFmJiTVDpxi7IMpJzLwmyWi+yZgfwWsskn0vx2HqfXw93Lp7Pj4tA1kyQRHxrOz4PGERcShp73KfnJUWEucK1DV88iyRVz/kBYiIXJr47h5ffnc+xUJgAN68bxnyduwGrJ/xGv37wmD74ykikvzkBW8hMqk0ni5e8nYLGZL/f2wlU6lZrFC5Pmcvx0JiaTRGSVEF56fKghpaUVQbojD7fmY1ivrpHp4y7xH9G9J0BNAXNTJJPY6GyEM8fS0NSSpVCaqrFjbZIBEQnBaFSjNkRabLy3cy0peRdoHlONZ9v3pX3VBKND84us9By+eHkW63/bgWSS6HtLZ+5+YRhh4SFGhyZUEiJJKgO14qPxeEpeBJlMEs0a+C8x+WDXOranp+AqVLZzPOc8z238jS/6jQT1BD7b7UlmUFOhgiZJAPVrx/H123dyPjsPkIiOLFk+OPSOXvS6qSO71iVhsVlo16spZov4J+BPXq/KhBd/IjPLXrCC5HTl8NTrs/jpw3uIi65icITlT++a9ZmevAu7t+S+uu7Vr7wGX9dy0M9PAM/O/H/Tuhs99Hak8GfEndgyFhFTBdXHZz5AbHz5H+wpVByD6jRhUJ0mRofhd26Xh78P/S9pp7NQvfn/lhZNXc/+rUf4cMnzRYbPCkKgiJ+yMhAdGcp1fVoUrGpcYrUojB/WxW/nmZa8q0iCBODVNValHMGpesHcCV95sa57mHPSyevbljMzeTdOHxdrFUV0ZJjPBOmS8KhQel7fns79W4oEKQA27TyGw+kpUWKnahoLVuw1KKryrW9CA1rHxhcpmwlVzNxcvwWNoq58v6Ke/Sx4tgNO0HMAF9h/AMcv/g9auKzI2Cr5nzHFuuFZQyyM+psYOVBWMrLyWLctmQNHzgZd2a8914nDR0v6YLF+/g6y03MLEiQAr9vLmaNp7FgjVmP9Rce4UjtRbicUePq+AcTHRTB9wXZy7S5aNE7gibv7UruG/8ph3KrvO5fo+RepUpX70Z3zQM+DizMINGx8daIN7yavxe71EKqYeStxFXOG3kn10HC/xSZUDumZeSXa3kP+/rvU9AsGRFT+ySYT3w0Yw/TDu/jlyF6ssszYJu24/irmnOhaDrjWAMVvcDjQ7V8ghQ73a8zCH3vqo7t48+Ev2b5qP4pZQdd17vzHTXQb3Mbo0IKerut8+N0qZi3agVmR0TSdGtUieO+FkRV+NftU8lkm/e0bDu04DkDLrg158sM7r6gZiK7lobtWAi4kSy8kuWpgg/0LkvecxJFXsiOtx+3l6L7TdOjT3ICohMpGJEllRJZN3DmiG3eO6Bawcwys3Zg5R/fh1X+/SJWAFjHVCTNbgJoQOws9513wbAJTDF8fb8vEpAi0ixdXdq8Hl+rlpS1L+F8fcWElXJ2WTWr4PB5iM9O+hdiTVBqLLDO+aXvGN23/595Az6XUwgBNzAEzQmgVG//5bgLnz10gK+0CCQ2qYQ2xGB1WpbBk3QFmL9mJ26MW7AU+fjqTf02ax5TXxhoc3Z/nyHXy1NC3yDmfV7AytmfjYZ66/i2+2vraZasjNNc6tKyHyb8q0EFXkcKfQg67p2yCv0q1GsVjC7XgtLuLHDdbFWrWL7/JXUUUXGus/hWwcjtJkiZIknRUkiSnJEnbJEnqdZnnDpckabEkSWmSJOVIkrRJkqSbij3nLkmSdB+/bIH6M1Q0z3XoS1xIGKFKfiMCm6xQxWzlre5DC54jKXUxRb+HqdoGpNhfmbg/qsRca1XXWX4quQwjF4JFo7pV6d6hfpGhmxazTI2qkfTtGnx1839W3gUHB3cc5/w5P62umaqDydfKrwzWa4occaleph3eyT0rpvH0+l/ZkZ7inxgEn6KrRVC/ZS2RIJWhaQu243QVLT1XNZ2ko2dJy6y4M4RWz96G2+kuUjqoqRr2HAebFu0q9XW6lpefIOn2/EoS3Q640HPeQffsK4PIr16fmztiDbVgMv1ekiUrJiJiqtB5QKvLvvbY6QxmLEzkt9X7yHO4L/tcQbicgKwkSZI0GngfmACsvfjf3yRJaqHr+gkfL+kDLAdeADKB8cAvkiT11XV9TaHn2YGGhV+o63rwFuVepWohVVh+0/38cnQvO9JTaBwZx8iGrYmxlb5HxyRB8dFB+cfLf62oUD698sQNzFm6i9mL8+/kDrymGWNu7ITZLBsdmuF0Xefr12cze8pyFLOMx+2l6+A2PPPx3X+py6IkmSDiNfSsxwE3+eW0FpBCkao8VvA8l+pl5KJvSb6QicPrwYTEguP7eb7jtdzepONf/vMJQnmQ46NMC/IrOnLtbqpW0Hm+Z46llVhZAfA4vaSeSC/1dbp7FfkrSMW50Ry/IJtb+C/Iq6TrDnTnb+jeI0hKMyTbQCTJii3Myrvzn+X9p75n94bDSBJ07NuCx9+5DVnx/V2i6zqTvlzG/BV70XUdWTbx9udLmfT8cNqJ7qrCnxCocrsnga91Xf/s4u8flSTpOuBh4PniT9Z1/fFih16WJOl6YBiwpuhT9dRABBwsQs0Wxjdpz/gmf1y2I0kS19VuysKTSXgKTe42m0wMvYr9EIJQmCybGD64HcMHtzM6lHJn4XdrmfPZCtxOD25nfonr5sW7mfyPn3jivdv/0ntLtn4Q+xN63pegHgdLV6TQO4vsO5iZvJvk7IyCwbUaOg7Vy/9tW86weq0It4jBykLF17tzQ6YtSMTjLbpP12JWqJNQcdviN2xTm5Awa4m9OopVoWGr2qW/UHfhu6hKA93h1xivhq6eRs0YCVoeYEeXQiH3beTYmUimWGrUq8obM/+Ox+1FkiSUP7jRtj7xKAtW7sPlvriKeLHU8h9vzeHXTx9CKSW5qtR0KkQDBaP4vdxOkiQL0BFYXOyhxUCPq3ircOB8sWMhkiQdlyTplCRJv0qS9CcL+IOPpmnsXn+QtfO2kXn2yvcgvNxlILWrRBGmWDCbZMIUM/XCo3mxU/8ARisIldOMjxbjKnYn2O30sHzGJtyuv95VUjK3wBT1NqbY6ZjCny6xMXvhyaSCBKkws0lme/rpv3x+Qbgcu9PN7CU7efuLpcxZuhO7MzClULfd3IWYqNCCjrKyScJqUfjnw4OQK3Dr6O5D2hGXEI1i+f1i32xVqN2oOm17NS31dZKlF+i+GjuFkOLtzmNr59B/7hQeWjWT3RlnAhC5b2r2v0DLIL9IiPwyQDUV7cLEIs8zW5Q/TJAAfl2+G6ePz1GvqrHzgPh8E65eIFaS4gAZOFvs+FlgwJW8gSRJjwC1gO8KHU4C7gF2kp9APQ6skySpra7rh3y8xwPAAwB16lz5rJGKKOXIOf4x/B1ysvKQJAmP28uICQO561+3/OFrY2yhLLnxPlafOUpydgaNo+LoVaO+KLcThAC4kJnn87iu6TjzXFisgR1sHGmxXdq2XYSm64SbK++eGc29GdU+HXQHJtsNmGwDkSRx19mfzqRd4L5//YDD6cHp8mCzmvls2no+/7/xxMdF+PVckeEhfP/2ncxZtpvNu44RHxfBqCEdaFjnylvql0eKWWbSgmf47o15rJ69FUmS6H9rV8Y/c8Nl5wZJchxS+DPoOW/zezluKFlSdwYv3odLU9F0naMXMlmVcoTP+46iZ416Af2z6LoX3BugxK5oL7qr+D32K1N85fASifwOv0IpROeGUpW77naSJI0A3gJG67p+/NJxXdc3ABsKPW89sAN4FHis+Pvouv4p8ClAp06dgvZHQNd1Xhz3IWmnzxfZzDl7yjKad2pI1ytoNyubTPSr2ZB+NRv+4XMFQfjzWnZrxObFuyg+siW6WgTh0WEBP/9tTTqw7NRhHOrvd1sl8pOndnE1A37+8sib8z5q7qeAE9DRnCsxWXugRP8vf6+X4BeTvlhKdo4DTbs0ZNqD2+3lnS+X8+azw/x+vrBQK+Nu7MS4Gzv5/b2NFB4VxoQ3xjDhjTFX9To57C50S1c0xy+gO5Bsg3l45REc6u8rLDrgVL28tGUxS296wM+RFyfhe58Ulzl+edf1bkHi3lM4iq0mabpO22aV8/NN+GsC8Q2QDqhA9WLHqwOX3U8kSdJI8leP7tB1fd7lnqvrugpsBRr/+VArvuMHUkhPKZogATjtbuZ+vvyq38/lcLNy1mZmfbKE/VuPBN0Avsomz+7ix1+38vjrM5g4ZRGHj6cZHVKlds+Lt2ALsyEr+R+9kpQ/YPRvb45DKoPV267V6/BEm2uwygpVzBbCFAvVQ8P5pv/oSrl6rKunUXM/ARz8fjvVjuZej+5ac5lXCldD13U27TxWkCBdouk6G3ccNSiqykcyN0eO+Cdy5KuYrNewK8P3JdmRCxl4tFLmLv5FaY5cnl7//+ydd3hU1daH333alPRCCr13QaSrgCBNBRFRxIa9ole9tmtBr+VTr71f9KrYFSuIYsFCESlK7713EtIm0075/pgQCDMIhEwm5bzPMw9kz5x91kwy5+y111q/9S2dvniZmTkNMKzDl6EqwjmkXHP37dGSLic1xFUigqMqEg5N4aFbzsKhRTdKb1MzqfBIkmVZASHEfGAA8PkhTw0AvjzScUKIkcB7wBWWZX1xtPOI0IqiA6H0u1pLcaEPSY7s6xblFx/XXJtX7+Cuoc+gB3T0gIGsSJx0akse/uBmFDX2QUczsBCjaByWsRWhdUOJvwEhR+7LYwMFRV6u/NeH7M8vxh/QkSTBT7+v4qFbzqKvLccdExq2zOa13x7gs5d+YOVfG6nXLIORtw2mVafGlWbD9e16MLJ5R/7cs40kh5MuderXSgcJQr1jEHJ4uolVjOGbiuTsExO7aiKyLGFEWHjLR7h/2USfZIeLPd6isHGXoqJEIYrqCQY49/t32evzYFgm96zowedd9pGmBXDJBggHyPWQEu4t1/yyJPHU3cOYv3wrsxduIDHexeBebcis4HTOmoYt3HBkorXyfR74QAgxD5gF3AjUBcYBCCHeB7Asa3TJz6MIRZDuAmYIIbJK5glYlpVb8pqHgTnAWiCRUIpdB0KKebWW5h0bhqXuAGguld7ndT2uuR6/6g2K9ntK5wsGYMmsNXz37nSGXRdbIQfD+z163p1ASKXH0tcR8E5CS5+MUGxpz0h8PPkvcvZ7SvO0TdPCH9B56o2f6NWlOYq9OIkJ2Y3rcNsLJ6Zkd6IkO1wMaFB7gvCGaSIJER6tE/FETqhQjtB3yqY8CCHo16MVP89eha4foqSqSJzZ096wiRXXtunGC4tnlkm/dcoKl7fsHJXI9sSNy8gP+jBKGt7vC7jpP/sCBtbZwdiTm5GZ2Bmh9TqhNFchBF3aN6RL+5pdi16bEEJkA08BZxPSJNgA3GRZ1vSS5wXwMCEdghRgLjDGsqzlh8yRArwMHOjB+g1wq2VZeX937qiskizLmgDcTqjv0SLgdODsQ2qMGpY8DnAjIYftRWDnIY+vDnlNMqEao5WElPLqAb0ty5oXjfdwouzJLeS58b8w6s7x3Pr4Z8xdsikq59EcKrc9fxkOl4okhy5qTrdG3cZ1OOeK3sc8z85Ne9m9ZV+Yw+X3Bvjhg98r0uTjxrIM9IKxHKgZCKGDVYhe+EIMLavazPhzXcRCVsMw2bQtJwYW2VRldm3ey0dPT2bcfZ+y4LcVNSLVdlnuLob98C4tPvkPbSc8y4Nzf8CnH1wQSs6+RzhSQXZfUDlG1hJuv7IvjbJTcTlVNE3B7VRpVC+N26440u/AJtpc06Ybl7Q4uTT91iHJDGvcjjtPPva1w/GwYN92vHrZeiHDkpiW24TfC/ojOfrYdYAxwLJi9zgaQohkQsEWAZwDtCGkRbDnkJfdA9xZMt615LmpQohDd7o+Bk4BBpc8TqGsOFxEopZDZVnW68DrR3jujL/7+QjH3AHcURG2RZs9uYWMvvd9PN4AumGyeUcuy9bu5NZL+3D+wJMxAvPRC5/D1NcgyU1QEv6J7OhZ7vOdcX43GrWuy7fjp5O7K59uA0+i3wXdj6vDu2mYR9w5OnTnLyYYO8GM1MvBxAz8UenmVBcS4pwRx3XDJM5t98OxOciMSX/x7E3vYBometDghw9/p+PprXjowzHVNh1qW1EeF039kOKSRZnP0Pli41K2F+czvu9FAAjhQk19h2DutZRuwFg6ctKjSIotZFORJMY7ef+Z0cxfvpXN23NoXC+NU9o1qJRaPJvISELwYJf+3NahF1uL8qgbl0iywxW18zVLTMchK/gPa0MgAQ3ik6N2XptqzT3AzgOZZyWUFjKWRJFuB56yLOvLkrErCDlKlwBvCCHaEHKMTi8RgUMIcQMwUwjRyrKs1Uc6eewLTWogH0yaV+ogHcAX0HntkxmcfZoHq+BaQlERMM0cArlXo6W8iuwsf0pbk7b1ufWZS8t9fN2mGSSlJ7Bna9kIg+ZU6T+yR7nnLQ+WZbHHW4RDVkIXbCmRkBZIOEKqpq3TK4GRZ5/C2s178PkP3pAkSdC8UR2y69g52jYhfB4/z40ZX9rc9sDY4pmrmfXNfHoPP7603arC+FV/ETDKXjf8hs6c3VvYWJBLk8TQtUPSuqJlzsP0zwb8SFpPhGR/P6KBnQpVNUnQHLRNPVxrq+IZ2bwD/13xB/5DvpaKkMh2J9I942+a4drUZs4DfhBCTAD6AjuAt4DXrFC6QxMgi0N6s1qW5RVCzCDUm/UNoCdQBBy6qz4L8JS85ohOUvXcIqzi/LlsSxkH6SACf96jHHCQDuIjWPBYJVh2ZIQQPPD29bjinThcIRUYZ5yDJm3rcd4NlVePtHDfds78dhxnfPM63b9+mVFTP2C3TyA5+wGHRcaECzn+hkqz7fcF67nivg8YeO2r3PTIpyxeXbWb0/Xr0ZIRgzqhqTJxLg2XQ6VR3VSevHNYrE2zqUIsm702YrTIV+zn1y/mxsCiimFl3h50K/w6rEoymwpzy4wJ4UB2noHsHGQ7SDa1ltzd+bz/5CTGjnyJ8Y99xb4d+yt0/nRnHBMGXEa7lEwUIaEIid7ZTfhkwKV2RDFGWISEG2L1ANKFEH8d8jhce74pcDOhOqRBwEuE6pPGlDx/QMMgUm/WrENes9c6JIe85P97DnlNROxIUhSokxLH5h25YeO6YaBKGyIeYxlbsCwdIWL3K2l1ShPeW/AEv305j73bc2nfowVdB5xUaek2u4sLufzXj0vTYwDm79vGqJ8/5Jdz/oNl3ooVmAtCAyuAHHcdknNopdj206yVPPHmT/gDoajMolXbue2JL3jxvhGc3LpqCkcIIRhzaW8uPqczK9fvIi0lnlZNMqrMzciyfAQLnsXwfgaWD0k7FTXpISSlaaxNq1XIf9PJXo1yc9to0iEtm/l7txE4TFEtYBo0T6reTUVtbCqarWt3cfvAJwj4ggT9OotmrOKb//3Gc9/fS9N2FXePa5uSybdnX01h0I8qJJxK9b3G2FQI+yzL+rtmZhLwl2VZ95X8vFAI0YKQk/RqtI2zI0lR4NKh3XBqZZ0dVZHp1KYBQop8czaJB2Lf4T0xNZ5h1/Xj2n9fQI/BHSu1HmHC+kXoh3XFNiyLHJ+HuXtz0NLeRcv4BTX1PbTMeSgJt1fKgt+yLF75cHqpg3QAf0DntY9nRP38J0pqchyndW5G66aZVcZBAgjk3oBR/CFYhUAQMzAD/77zsYx9sTatVnHSqS0ithFwujUGXXpaDCyqGK5s1QVNlsu0pXTKCn3rNrPrH2xsDmPcfZ9SXOAjWJKeHQzoeIt8vHrnh1E5X4LqsB2kqoAFWCJ2j6OzE1hx2NhKDoq/HWj29Xe9WXcBdcQhC6CS/2dwlP6ttpMUBXp0bMwtl/bG5VRxuzQ0VeaUtg147B/noMTfApQtjPQaCq9vaMPZ37/Fdk9+bIyuAmwq3B+26wshJ2VncQEAQq6LpJ2MqER53mJfkP2FkYQjYN0We0FfHszgWszAPEKS7gewwPKhF38cK7NqJYqq8O+Pb8EV78QV70BzqWhOlbOu6E3nfu1ibV65yXIn8PWgKzg9uwmqJJOsObm6dVdeOq16ppv+vmA9I//5Dqde+hxDbh7Hl1MX1QgFwlhjWUEss7DWf5aLf18d8TNY+ecGTPPExZvycwp597GvuLn3Izx4wYss+G350Q+ysQnVDrU6bKwlcEAteyMhR2fAgSeFEE6gFwdrkGYD8YRqkw7QE4ijbJ1SGHa6XZQYMbATQ844ic07cklNcpOeEg+AZV0CVgGBwlfxGwEs4L0tJzFucweE2Mdlv3zMr0NvrFI7/pVF94yG/LhtdZhEqInFSamxaxrrdCg4NIVibyDsuTolv1eb48PS14JQwpt44scMLomFSbWa9j1a8PGKZ5jz42I8+V5OOaMtdZtmxNqsE6Z5Ujrv9xsVazNOmLlLNvHgy9+WRrNz8jy8+vF0AkGdi8/+u0wVmyNhWQECBY+jF08AdISUiZb0GMoJCChVZxxOFf2wbAkA1aGc8HqkILeIm3s9QkFuEUG/zga2svSPNVz98PkMu6H/Cc1tU+N5AfhDCPEAMAHoRKhP6v0Qqi0SQrwI3C+EWAWsIdR+qIiQ7DeWZa0UQvxASOnuQM3TG8C3f6dsB3YkKao4NIWWjTNKHSQI1Ymo8Tfx5NYnOXfeSE6dOZpXN3XBQmBaFvt8Hhbuq9qCANHi3MbtSHO4UaWDaYdOWaF3dlNaJteJmV2yJHHJOZ1xOsruKTg1hWsuKL90e21GKE3ACr8hgwNJbVtmRDdN/ti1iV+3r6Mo6I9wjE1F4Ip30ndEd4ZcfUaNcJBqEuMm/B6W7uvz64z/ag5GBezy10YC+f8qcZB8gI5lbse/fwxGYEGsTYsJgy8/Hc1ZNv1NdSj0H9XzhJ2kr/87lYKcotJUPgj1YHznka/wecp3TfcFguQXeWt9BLAiqMp9kizL+pOQwt1IYBnwf8BYyrYYepqQM/Ua8BeQDQy0LKvwkNdcAiwGfix5LAaO2tXdjiTFiB3FXnb44sLGBZDjL658g6oALkVl0uCreHXZLL7fsgqHrHBpi05c2apbrE3jquE9MQyLT7+fj2GYODSFG0aexqDT2sTatGqJpLZB0jpiBhYCh0TohIrivqz0x8U5O7h6+gSChgEIdMvgsS6DGdG0Q6XbbFO1WT5nLR88MZEtq3fSqHVdLn/gPNp2ax5rsyqErbsiq4z5AjpFxX6S4qPX26YmYpl56N7JlLn2AOAjWPQKcur4WJgVU654YDhb1u5i8YxVKKqMoRu06dqM6x8becJzz/tpKcEIUSpZkdmwbCttux/799TjDfD021P5bd5aLMsiMz2R+64dQOd2B2XlTWMnWDpCrl8rs3JqGpZlfQd89zfPW8C/Sx5Hes1+4LIjPX8kbCcpRvSt24y5u7fgNcqmlgVNg5PT6sbIqtiT4nAztvMAxnYecPQXVyKSJLh+5GlcfX4PCjx+khKcyJIdiD0RtJS3CRY8juH9GgggqV1Qkx5DyKEoht/QueK3TykIlpXMH/vXD3RIy6ZFUuyiizZViwW/LueRS17BX5ISm7srjxVz1/HIhNs4uU/138ion5XC6o2HK9yGshXi7cbQx41p7ChVSS2LhalvjHhMTUdzqjz26T/YunYXW1bvoH7zLBq1rpi1SFpWMuuXbAkb13Wd5DrHV1/8r+cnsXj1doJ6qH55++487nr2a8Y/fhkNM/Px7b8ZS98ACISchSP5FWTN3lSzKR/2Ki9GjGjagSx3Ag75oJ/qklWubt2NOi67zqWqoigyqUlu20GqAIQUh5b8JM6slTiz1uFI/wxJPVifOWPnBswIfW6CpsHnG+y6JZuDjPvXx6UO0gH83gBv3PdJjCyqWG666HQchymmOh0KVw3vbl+LyoEkNzpCuq+MpHYs/WlN/l4mrF/Er9vXhSmvVhQFHh8bt+XgCwSP/uJKoEGLLE4bckqFOUgAI24ZiMNdts+hrEg0bdeAuk2PvYnt1l37Wbp2R6mDdIBg0OCzH+bgzbkAS19FSBDIh2Vswpd7MZZZsf2eahxWDB9VHDuSFCPcisbEwVfx/uq/+H7rKpI0J1e07MKA+i1jbVqlYlkW3/zvV7546XsKcoto3bkp1/3fRTTv2CjWptlUEqF0iPCUiKKgHzPCVdSwLPL8kdUGbWoflmWxZfXOiM9tXlkz6ju7d2jMY7cO4ZWPprFtdx6pSXFcNbwH5/fvePSDKwnLMjH1FWDpSGr7mPb8OxpCikONu5ag523gkGuJcKAl3Iphmtw+exK/bF+LACQhEa9qfHrm5TRKSKkQGwJBnSffmsovc1ejyDKmZXH1eT24fGjXGpci1rFXa6579ELeeugLJFmgBw2atm/Avz++5bjm2bk3H1WWOLyKyTAt4tVZYPkIW3lbOsHiiWjxV53Qe7CpnVTdq1gtIEF1MKb9aYxpX317kZwo7zz8Bd/87xf8xaFd4MUzV3HX4Kd4edpYGraqvWmHNtAjs1HEonS3ota6zQSbIyOEIDE1noLcorDnEtMqr1VAtOnVuRm9OjeLtRkRMQJLKc69FssqadWAhiv1VRRHrxhbdmTUhLsQcjZBzxtYZg6S2gkt8QEkpTkfrV3Ar9vX4TMORpu8epCbf/+S7866tkLO//z7v/HrvDUEggaBYCgy8s7EOWSkJTC4Bta6DrmmL/0vPpVNK7aTlJ5AduPjT5duWj+dgB7eJkRVJNo0FmBFisb5sMyasVkSHQTWsfUrqpXYcfoqgmVZGEbtUikqLvQy6Y2fSx2kAwT8QT559tsYWWVTVch2J3JD25645IOKSy5FpVNaPfrVrdyC/EKPj0WrtrFtd16lntfm2LjgtsFh6TwOt8bIO86KkUW1B8v04sm5GMvcCZYHLA+WtZ/i3GsxjfA6qqqCEAI17jLcGTOJy1qBK+0j5BJlzY/WLQirFzax2FCQWyG9DP0BnSkzl0dQLAzy3qS5JzwhqosEAAAgAElEQVR/VcXpdtC6S9NyOUgA6SnxnNWrLc5DUk8lIXA6VDq0OxsLOfwgEYes2TL5NuXDjiTFmGBA591HvuC7d6bhL/bTpF0Dxjx/Oe16tIi1aVFn16a9KKpMwHfYzcgwWbOgdhbPHg+WZWEE52MZuchaJyS55gkZ3H5Sb7pnNGTC+kV49ABDGrbjnIZtKq0Ow7Is3vpyNh98Ow9VkdF1kzbNMnn6n+eRGOesFBtsjs4F/xhMcYGXr/87FVGSunn+LQMZfvPAGFtW89H9U4Hw3X0sk2Dx1zgSbqx0m06UgBHh/RBakPuNSLVMx4cnQs+9A+Tke054/prMPVcPoEn9NCZ8v4CiYj9d2zfi5lG9mF6wmURvOicn7sQlh35HuqWiKY2RHXYvJpvyYTtJMeb5m99m1uQFBEoumhuWbeX+857l5WkP0ah1vRhbF13q1EslGAi/GQkBDVvGrnlsdcDUt5bs3uYAoTQDLf56nIl3x9q0CqdnZmN6ZjaOybl/nrOaj6b8VSYlZvm6nTz06ne8eO+ImNhkE44kSVz50Aguvnso+3fnk5qVHNbzxSY6WEbOEUQQ/Fjm3kq3pyIY2qgt41b8gd8se39K0pw0SUg94fmTE1zEux3k5pdt9yGA9s3te9/fIUmCiwZ35qLBnUvH9vk8PDj9RwxrEJfUXcb52atQhMkPe1sxqOUztK3C9XFVgmogoBAr7HS7GJK7O5/fJ/1V6iAdIOjX+fyFKWXGLMti6dodTPtzLXv3h+feV0cSUuPpe2F3HK6yixnNqTHqrqExsqp6UJx7FZaxrSS9pQjwE/C8RdA3NdamVRp+b4C8fQVRbSb48ZT5+PyHyfTrJgtWbGV/Qe3sZ1aVcbg0shrXsR2kCsZv6HyzeTmPzP+R8avnlRFOkR09iCS8gnAjO6pnve21rbvTKCEVtxL6O9IkGbei8uKpwypEVEGSBP8c3S9i2tiYUb1PeP7axm871iEJgW7JvL+9I+f9dRFD/ryY1zd15tstm2Jtnk01xnavo4Rp5KD7p4FQUB39EFJ4AfGuTXtQHWqZLtShY002Lt9W+vPunEJuefJz9uV5kIQgqBuM6N+Rf1zSp9qr4PzjxdHEJbmZ8s40ggGdzIbpjHn2Mlp1bhJr06osRnAtprEVOKyGzfISKBqP6qxaPaYqGp/Hz8u3v8fMr/8EIDkjkdtevpIu/U+q8HPlFUZ2hGRZotDjIyXRXeHntLGpShQEfJw/9V32eAvx6EGcssJLy2bySb/LaJOSiay2QXGdhe77EayS74twIasdUBxnxNT28hKnanwz6Gq+37qKOXs2U8+dxAUlbTsqiv49WpGa5Gb813PYvief9i2yuWZ4TxrVPfFIVW3jyBtlVkSFVJtDsLCFG/4G20mKAn7Ph3jz/w0ooQ02yyQu9XVUZ9m82LrNMsMcJABJlspIYN/zwiS278nHNA9+2b/+dSntmmXTv0ersOOrE4qqcMMTo7j2sZEE/UGcdmPEo2JZRRCpQBWwrBMvKq7q/N8Vr7No+srSDu57t+Xy2KWv8sLPD9D0pIZHOfr46NmhCZOmLQ0TVdFUmboZyRV6LhubqsjLy2ay3ZNPoCT1zGfo+Aydf875hu/Pug4AV/IL6N5vCBR/DFYQxTUCLW4kQlTfZBVNlhnWuB3DGreL2jlOadOAU9o0iNr8tYV+9Zrz7wU/hY1rssI5DWueUqBN5VF9r2BVFENfjzf/EULNzEJKP+DFs/9mTLOsMlZyeiJnjjoVh6usKpPmVBl5x9kA7NiTz6btuWUcJAip4Hz248IovpNjY+2WvUz6bSlzl26KKNd8rMiyZDtIh7EkZwcP/jWF22dP5Mdtq0o/3wMKTOE4UZ3nVJ6BMWD3ln0snrGS4OEpcP4gn7/0fYWf76rhPUiMc6AqIadUCHBqCvdc3R9Fti+fNjWfKVtXljpIh7KxMJccX0hkQAgJ1X0ecemfEVfnaxzxlyGEFnZMdccwTLat3cX+3TV/M6o6ke6M59+nDMQhK6iSjCwknLLClS27clKqXeNlU37sSFIFEyieCEQoYrUEQd9PONwjywzf+uJoMhqkMmncz3jyi2nVpRk3/udi6jXPAkIqOLIsIIL8f1Hx4S3VKg9dN7j/5W+Zu2wzQoTyqZPiXfz3wZFkpSfGzK5YYxq7Cfp+AEtHcQ5AVsoX2Xhr1RxeWDadgGFgYvHL9jV0rdOQ//UaiSw5cCY9iS//HrAChJSlXAi5Llrc6Ap9P1WNvdtyUTUlXBHRtNi+bleFn69OSjwfPXUlE36Yz5/LtpBdJ5FLzulCu2b2jdemdiD/TTRIqSSVyarA7O8W8MKYd/AXBzAMg7bdW3D/uzeTnFF773dViZHNTua0rCZ8v3UVQdPgzHotaJlU8xRfo4KdkXhEbCeporG8RJRDxSzpBl0WWZa45J5zueSecyNO16R+GnKEHWtNlenbLXYy4RN+WsjcZZvL9HnwBXTGvvYd/3v44pjZFUv8xV/hzbuHkhxLKHgSZ8IdOBPGHNc8+3wenl86rYyyUrER5M99W/h151oG1GuF5h6GrLYg4Hkf09iJ4jgTzX0hQnJV7JuqYjRsVTdiiqqiyrTtHp3vQ2qSm5su6sVNF0VlehubKs2FTTvyxorZ+MyD3ztZCDqmZpOk1ezrzQE2LNvCk1f+F/8hIkvLZq/hgfOf5bXfH42hZTaHUi8uiWtbd4+1GTY1iNqzDVRJqM5BWETqn2KhOvod93yKLPHgdYNwaAqyFCquc2oKGakJXHxW56McHT0m/rokrBGeaVqs2rinVqp+mca+EgfJD/hK/vXjK3wRI7jquOaavWcTihRec1SsB/lh68G5ZLUtruSniEt7D0f86BrvIAEkpsVz9jVnlGkcKiSBw6Ux4h+DY2iZzeEEgjo/z13NB9/9yfyVW6OqQmgTPW5o05NO6fVwyyoOSSFO0chwxvNCz2GxNq3SmPja1NIayAMYQYNta3axYdmWGFllczh7t+eyYu46PPm1bw1yYogYPqo2diSpgtkRaMbve5pwRtoanJKOBQRNhSL1CpKV+uWas0+X5ox/9BK+mLqYXTkFnNqxCef0aocrhjK3wWDkZntCHPm5mkzQNxWEFCFsHSTgnYxLbX3Mc7llNeKlQxaCBNWu27rhyYup3yKLr175kcI8Dx17teHqf19AnXq2KlRVYdvuPK577FN8/iCBoIGqyLRoWIdX/jUCp2bLc0cTyzII+n8hGFiALNdFcw1DkpLKPZ9DVvig7yUsytnB0tyd1I9Lond2s1qVardry15MI7zmVlYkcnbm0bR9xQrG2Bwf3iIf/3fFayyevhJFU9ADOuffMogrH76g2isA28QW20mqYJ5e+hs/be9Fp93N6J++Dt2U+W5PS/xSGlPPKv+8Teunc89VZ1acoSdIv24t+XzqIoJ6WYcoIzWBOqnxMbIqlhhHyOu1iJx+eWROz2oa8cKuSjIXNj25XNbVJIQQDLmmH0OuOf7IrE3lMPb1Kewv8JZGj3TDZNWm3bw/+U+uH3FqjK2ruVhmMfk5IzD0DSHRIOGiuOBJEtM/R1Hbl3teIQSd0uvRKb1mNzg/Ep3OaMvKuevCaiEDfp0WHRvHxiibUl685R0WTQsJ+hz4HU18/SfqNc9i4GW9YmydTXWm9mwFVRJ/7N6AacH8/Lr8Z31vntt4Gqs8ddhalEd+wHv0CaoJV53Xnaz0BFyOkmZ7qozLqfLvmwbXyp0b1XkmYX2LAHCUKs4VBny8tmIm5/70P0ZP+5Bfd6yNOJdDVnin9ygSVAfxika8oqFJMvd06Ee7lKzovQkbmwogv9DL2i17wtLrAkGD72Yuj5FVtQNv0X8xgmtKVFUBy4tlFVKYe3x1kYX7Pfw1dQlrF2600ySBIdeeSUJKHIp6MA3a6dY494Yza61ww6YdubwzaQ5vfT2b9Vv3xcwOb5GPWd/MD1M89RUH+PzFKTGyqpphxfBRxbEjSRVMvOqgIBiuOieEwCnXnDSThDgnHz4xmp/nrGbhqm3Uz0pmaO/2pCXHxdq0mCDJ2TgTx+IreJyQuqEFqDjirkLRTsITDHDez2+zq7gQf0kB9MKc7VzXqif/aB/eYf2U9PrMHXY7f+zeSLEe5NTMxqQ47MalNlUf07I4Uq65aS+4o4rf+xWhesiymMZ2DH07snL0SNCE5ybz4RMTUTQF0zBJy07hiW/uJqtR7VUKS0iJ4/VZj/HJs5OZM2UhCSlxnH/LIPqO7Blr02LCR1P+4o0v/kA3QlkS70/+k9FDu3Lt8Mr9PEwzn+KC97nmsWVsWBLHzInZeD0Hl7UFOUWVao9NzcN2kiqY0c278eLy6fiMg7samiQzqH5rHHLN+rgdmsI5vdtxTu/oNdurTjjjr0B19ibg/RYsHc01GFkNNbL7fOMidnsPOkgAXiPIuFWzuKxFF1IjOEAOWaFv3dgpGNrYlIeURDeN66aybsveMhuFmiIzqOex1+bZlIfITabBCtVMHoX5vyzjo6cmEfAdTFvauWE3D414njf+fKJWZgkcIDkjkZuevpSbnr401qbElO178hj3xSwCh9QeG6bO+5P/pF+3ljStl1Ypdhj6JvL3DgV89B7updsgieG3bGTsiK7s2+FCkgQnn3GknoI2ZbD3ro6InW5XwVzVshvnNGiLJskkqA6cssIpafV5rPPZsTbNphKQlSa4Em7FlXhHqYME8OuOtfiMcOlqTVJYkrsDgM0rtzPt8zmsWbDBTnGxqdY8etNZxMc5cDpCG0Nup0rD7BSuGtYjxpbVbBzuURCmriqQlWbI8sHeXnP2bGL09A/pO+VV/jn3azYU5gChOg5/caDM0aZpsXvzPjav2B5l622qAzMWbIi4qA7qOo//70de/nQ667dFP/3Ok3cflpUPhMoYnG6T+KQgV4xdg6LKuBJcXDH2/KjbYVOzqVmhjSqALEn8p9tQbm/fhzX5e2kQl0zTxMrZWbGpumS5E5AQmIfdXUzLJFFojB3xPIumr0BRJEzTolHrejzxzd3E19L0RZvqTdP66Ux64TqmzlnFjr0FtGuWxWknN0WJ0PPNpuJwxV9D0D8NPbgIrCAIDSEcJKT+t/Q1325Zzn1/TS7dtNnhyeeXHWv5ot9VFOQWRpxXUiQK8zyV8h5qEpblx++diN83FUlKxxU3GkWt3tENWRIRs2lNYPmGXazatJsvpi7m1lG9uHBAp6jYYFkWwcAsDq8DlhXo2DuXs6/uy4V3nE1GfXvtZXNi2E5SlMh2J5Ltrp0FnTbhjG7elSlbV5SJJklCkOVOZNG4OSyatjyU4lLy3PqlW3j5tne5/73jK7i2sakqxLk0zuvbIdZm1CqEcJCYNgE98Cd6cCGSnI3mHIgQoeiSaVk8vujHMtchEwuvHuDZZb9y5tDObFiyJUzFzTRMWnRqXJlvpdpjWV7y9p6HbmwAqxiQ8Xm/ID7pP7jcI2JtXrnp06U5r3w644jPG6aFYeq8/OkM+ndvRUpitGppFSAQPqo5GfPc5VE6Zw3EAqzam0Z7NOxtvShgWRbL/1jDlLd/Zenvq+zUKRvap2bzf13OIa5Erc4lq7RKzODdPpfw47szwhYlekBn1jfz0YPhKXo2NjWBPbmFjPtiFve+9A0fTvmLAo8v1ibVCIQQqI5uuOJvwOE6t9RBAsjxeygKhi8sLWDhvu0Mue5MMhqk4XBppXM5XBo3PnMZTrfdo+148Ho+QdfXlzhIEGoT4aUo/19YVvVVus1MTeCu0f3QVBmHpiBJRIwsKbLEnKWbomKDEALNNRQoK4YVMGUWF3VmjzdyRNTG5nixI0kVjKfAy7/OfpItq3ZgmRaSLFG3aQZP/3i/nTpVyxnW6CQG12/DqrzdJGpOmiSEUgH8vvBFC4R2b/WggaLaX9NoEPDPwut5D8vMR3MOwRV3YZkFpU30WLlxNzc/+RlB3SCom8xeuokPv/uT9x69jMy0hFibV2P5u2bUdZxxuBNcvPr7o/zw3nRmf7eAtKxkht00kNZdm1WilTUDv+9bDtTLHIpAJhhYiOaovv3Chp1xEj07NGb6/PX8Mm81i9fsiKBcKdCieO+KS3oUQ1+N178WwzKwgI3Fqdy1vBWOVW/xw6AbSdJcUTt/TcLexz8ydiSpgvnffR+zcdlWfB4/fm8Ab5GPzat28PqdH8TaNJsqgENW6JhWr9RBAujS/yQkKXwrrlmHhvbubZTwFL5Mfu4VBHzfEQz8jqfwEfL2DceywuWTbSqe/3v7J4p9QYJ6qKbAH9DJ9/h4dcKR03hsThynrDKsUXuchymtumSVG9ucFvp/vJPhYwbx9JT7uPedm2wHqZxIInK6vYWBENW/4XpGagIXDjiZ2y89A1UNV1W0LItTOzaJ2vklKYki1yfcseJsXth4GrctH8I1S4ZTaCgUBf1M2LAwaue2qT3YTlIF89uE2QT9ZVOk9IDOjC/n1oq0ux1783nh42nc/NTnvPrZTPYcoRDYJoSv2E/7ni3RXBpqiRKY6lBwJTi5/dWrY2xdzcQ0cigufBEOTXmxvOj6OvzeSbEzrJZQ7AuwYVtO2LhpWsxavDEGFtUuHuo0mMH12+CQFOIUDbeicmvbXgxt2D7WplVLdhbnc8fcrzhl4tOcOvl5Xlw+jYBp4Iq7EsThkQyBJKWhqCfFwtSo0LpxJtef3zPUUN6h4naquBwqT992bmmz+WixIn83qz0N+GZ3G5YWZnEg789n6MzbuyWq57apHdh5PBWMfkjvgEMxdDPi+BHn0bfgKXqdYGABitKSuIQxqIdISldFVm3azQ1PhFJodMNk8dodfPnrYt4ee3Gl9U6oTmxYuoW7Bz2BETRKu4WnZCQx+Mo+DL3+TNKyU2JsYc0kGJiHEFp41Mgqxu/9Aad7ZGwMqyUossSR2u04tZrTcLuq4pAVnuk2jAdPHsgeXxEN4pJrVKPzyqQg4GP4L2+R5/diYlGk+3l79WxW7N/Jm6dfjDvuJoqLXgMR+nwlEU9S6oc1rt/UZWd3ZVDPNsxZugmHpnD6yU1xO7Won7eeOwkjwuazKiSaJKRG/fw1hpq/f19u7EhSBdOl/0lIh8ncipKmZsd6YQwGV5Ozpz9ez8fowWX4vBPJ2TsEv//3aJhcYfznvV/w+oPoRsghDOoGxd4AL3z8W4wtq3pYlsWjF71E0X4P3iIfRtDACBoUFxSTnp1iO0hRREhJWBHvChKSbDvz0UZTFXp1aoqilL1OOlSF4X1rzg57VSdJc9EisY7tIJ0An29aSLEeKNPawW/qzNm7iXUFe4lLvJO0zDkkJj9HUuo7pGbOQ1Gbx9Di6FEnJZ6hvdszsEfrSnGQANomZ9E0IQ3lsEbJiiRzWfMulWKDTc3GdpIqmJufH01iWjwOd+gi4XBrJKTE8Y9XrjrmOQrzH8WyPMCBtD0TLC+5++6peIMrCMM0WbFxV9i4BSxYZTchPJxta3eRuzsvbNxfHOD7d6dVvkE1iFx/MXfPm0iHr5+kw9dPcufcr8n1H+zxomrdkUQc4ZJMGk736Eq1tbZy/zUDadGgTkl6joZDU+jWvhFXnts91qbVWjz5xUx561c+/L+vWPDLMkzz+LIfaiOLcrZHbBIuC4lV+bsBkOQMHK4haI7TECK8dsem/AghGN/7Ek7LbIIqyWiSTIO4ZP7XaxSN4u1Iks2JY6fbVTCZjdIZv+xZfvl4FmsXbaJZh4b0v+R04pKOvVdAMDCPSPFP09jMuq2bad6gUQVaXDFIQqApCv6gjhAWfTst55weC3GoOvNWtcM0r0GSqn+xakVhGuYRI4vmcaZm2hxEN01G/TaebZ48dCv0OX6/bQWLcrfxw6CbUSUZIWSS0iaQn3sJlpkPSFjoxCc+hqrZkYzKIDHOyfh/X8KqjbvZtief5g3SaWKn5MaMtQs3cs+g/8PUTXxeP063gxanNOGJb/+FFuW6kupMi8R0ftspEzDLptlblkWDODsboDJIcbh5q9fFFAb9ePUAdZzxNS6dMerYfZKOiO0kRQF3gouhN/Qv9/FCJJZEksqimzIvfjaHV++pek6SEIIhvdoxeeYybhj6Hb1OWo3TEaqzqZc+m5y9Q0jP+MGWWC6hQats4pLc+Dxl62IcLo3+l50eI6uqP7/tXMNeX1GpgwSgWya5/mJ+2bGawfVD3e4VtQWpGXPRg4uwzCJUrTNCsiX6KxMhBG2aZtGmaVasTanVWJbF4xe/RHHBQSETn8fPmr82MHncVEbcdnYMravaXNS0M+PXzi3jJKmSRJOEdDqk1I2hZbWPBNXxtxL3NjblwU63q0L8uG0lfb9/mZc3NMFrlPVf/QGFX+e3Y9Ga8JS2qsJtF/emf1cnfTquKnWQAGQ5iGlsw1f8TQytq1pIksSDH92KK96JVtK40RnvoPnJjRl6/Zkxtq76sq5gH149vO+URw+wtmBvmTEhJFTtFIqDXXntywWMeuA9bnzyM6YvWFdZ5trYxJzt63axf09+2LjfG+Cn96fHwKLqQ6YrgY/6jKZdchaykFCERL/sVrzb+1I7mlHJrF+8ifEPf8b4hz9j/eJNsTanWiGs2D2qOnYkqYowbeda7v5rIj5D5/PiDmQ7CxiSsZJgUEaTDP5a04S3pvQloQr3zXFqKvdenkrefo2D9VQhLKsYv38GrjhbOewAbXu04P1Vz/PbhNns25FLh15t6DzgJCTJ3rsoL00T0nApGp7DHKU4RaNpQnrY6ws8Pi576AP2F3oJ6qHd4BUbd3HFkG5cc26PSrG5tmNZOp6idykufh/L8uN0DiEh8R9IUlKsTasV/N1i3l7oH522Kdl83f86ivUASkldjE3l8sHjX/L5c98SDITWHV+//D0X3jmEyx8cEWPLbKo7tpNURXhxxbTSAlALwUubevHuti40UPPwTMlkf2ECTk3hogGdYmzp3yNJGUhCitDBWUWW6wHgN3RMy8Kl2LnuiWkJDLt5YKzNqDH0q9uSZM2FzwiWSsPKCBJVJwPqtgp7/Re/LCKvyFfqIAH4AjrjJ8/lwjNPJjHOTg+NNvtzb8Lv/xWrpG+Vx/M2Pt8PZGT+YqfnVgJ1m2WSmpXMzg17yow73BqDruwTI6uqH26lchTdbMqyZfUOPntuMgHvwewVvzfAZ899yxkje9KgpZ32aFN+7C3rKsKWov1hY/m6i2WebPykoikyg3q0ZvSQrjGw7tjRHKcjRCJhf1pCwSOfxzW/f0ynSf/hlG+eZtS08WwsDG8qaWNTXlRJZkLfq+iX3RJFSMhC4ozsFnzW72o0OXxPaPbSTQSC4epUqiKzevOesHGbiiUYXIXP/0upgxQigGnuxltsN/atDIQQjP3kNuKT3TjjHEiyhDPOQdvuLRly/YBYm1flME2TuVMW8sJN/+N/933MlpVHV2/dsTefR9/+kWF3vsU1j31ip/RWILMnz4/Yh9LUDWZPnh8Di6oZVowfVRw7klRFaJaQxuL9O8LGEzQHz44ZRpO6aaQlVf3CciFkUut8SV7ONej6BoSQEMJNfPLLDJn2MzuL8zFKvhkLc7Zx0bTx/Dr4VuLtgkubCiLDlcBrp47EKokk/V3KUGZqAkIQFvk0DLNafN+qO8HAIgRS2L0ylJ47B3fcRTGxq7bRrGNjPlz3CjO/mkvurjza9mjJSb1a2+l2h2EYJg8Pf5YlM1fi8/iRFYlJr//Era9cxaDRkaNuu3IKuPzhDyn2BTBMi505BYwdN4Ubzz+NSwZ3ruR3ED0sK0hh0RsUed7Hsry4nINISrwHWc6I6nkVVUaSBMZh40ISyIqd+mhzYtiRpCrCP9v3w3nYTrdLVrmt3Rl0adOwWi3YFKUR6Zk/k545jdQ631EnaxFz8+qyP1Bc6iBBaBPBb+h8u3V57Iy1qbEIIY66yBs18BQcatnvnSwJGmen0tSWpI46kpxN5NuQA0VpGDYaMA32eAsJmocviWxOFFe8k4Gj+zDqnmF06N3GdpAiMOvreaUOEoChmwS8AV65dTyeguKIx4yfPLfUQTqAL6Dzxtd/4AsEIx5THcnJvZGCwucxjK2Y5j48xRPYvWcQplkY1fP2Gt4t4t+qEIJew7tF9dw1AxGSAI/Vo4pjO0lVhJ4ZTXitx0haJmagCom6riTGnjyY0c2r75dcURqiqq0QQmJL0f6ICxuvEWRjkZ1yZ1M+DN1g2mezeeKyV3jl1ndYt3DTcR3fvlk29105gHiXFmpqqiq0a5rFC/8cHh2DbcrgcJyOJCUDZXd8hVBwx11c+rNlWby+agY9vn2G/j++Qvdvn2Hcqpml0UIbm8pg2udzwto2QCiasWT6yojHLFi9rYyDdAAhBFt2hTcUr44Eg2vx+X49LG1WxzDz8RR/FpVzFnn9vDRhOle9+BW+UV3wd6yHFu/A4dbQnCpjXrySjIbhYj02NseDnW5XheiV1YxeWc1ibUZUaJ2ciSJkAocFxd2yRvtku0+KzfGjB3XuHfwEaxdsxOfxI0mCn96fwQ3PXMaQ64+9T9lZp7ahf7eWbNyeQ2K8k6y0xChabXMoQsikpX/F/tybCAaXAhKynEFKyivI8sHrwnvr5vLm6ll4jdDOu9+Ecat/J05xcHk13kiyqV443FrE9FwA1RlZiCgzNYHNO8NrjnXDIDXx2JvMVxUCgWXk5T9KILgASUolIX4MkogHoUSoMfHi988hIf6aCrVB1w2uefxTtu7JKxXd0TrUw9GjGVe1b07PoZ1JzUqu0HPa1E7sSJJNpdAtvRHNEtPLyKMqQiLV6WZQvTYxtMymujL98zmlDhKAaVr4vQHG3fUBnvzIqS9HQlVkWjbKsB2kGKAo9amTMZnMrD/JyJxBRuZsNEdZgZo31xx0kA7gNYK8sfr3yjTVppZz1tV90Vzh9bOSLNGxT9uIx1xxTjecWtn9aFWR6d6uEenJ1SeNHiAYXMOefcPwB2ZiWR4MY3Uxc5IAACAASURBVCv5+Y/g808nchW+hiU1ZqsnciZJeZm+cD27cgrKqJIGdIM9/gD1+rS2HaTjxRZuOCK2k1QFsCyL2d/O59GLnueRC59j1qQ/a1waiRCC93tfziVNu5CiuUhUnQxv1IEv+l4TUXXMxuZozPhy7hFSXxSWzIyc+mJTdZHlOihK/bD6AsuyyPF7Ih5zpPHahq4bvD15Duf88036jXmNsW9MYXdudGtBaiMderXhwjuHoDlVnHEOXAlO4hJdPD7pblQt8n2sa9uG3H15P+LdDlwOFVWROa1DEx694exKtv7EKSh8EcvylRmz8FLs/aakxUfZzyBgCi78o4ihP4+j57fP8umGilGbW7ZhJ8X+8Hou3TBZsXFXhZzDxgbsdLsqwQs3vMm0z/4oXfDNn7qEU4d15d53x9So4tk4ReP+jgO5v6PdF8jmxIlLdB8h9cXCafc3igl+/wJy8x7CH1yCJCWTlHATifE3IET59+OEEDSNT2NDhNrFSA2CayMPvPEdsxZvwl8iZz913mrmLt/MF09eZff6qmBGjx3BWVedwcJfl+FOcNF18Mk4XH/fI2lor/ac1bMN2/cVkBTvJDneVUnWViyBwCIgXG5bCJWU5KcoLHwVn38mIMgNpvD42t7s9LuAkEPz1NKfyHIlcEZ2yxOyo26dJJyagi9Qtn2Dqkh2NkB5qFl78hWKHUmKMesWbuS3CbPK7Ij7PH5mTfyTVfPsXgo2NkfinOv6oUVYnKhOjQ69WsfAotpNILiKXfsuwB+cDwQxzb3kFTzD/vxHT3juf3UYGKb+6ZQV/tXB7uOzZdf+Mg4SgGFaFPuDTJy+NIaW1Vzq1E9j4Og+nD6821EdpAMoikyjrJRq5yAZlsn6gr3sKM5HUSLXTFtWEFVtTZ30j6iXvYKk9DlctuhClhSW3cTwGkH+u3rmCds0uEdr1MPkvSUhcDs0ep3c9ITnt7E5gB1JijELflmKHgjP1Q34Asz/eQltureIgVXHxrH0ofk79mzZx1cvTWH1/PU0PakRI24/m7rNqp+Iw67cQl78dBqzlmxEUxWGnt6OG4efilOLXMhrUzG0O7UVlz4wnA8f+wpZVRAipDL1xLf32v0xYkBewQtYVtn0R8vyUlj0HsmJdyJJCeWeu09WC9449WJeWjGNDYX7aJqQzh1t+9KtTuMTtLr6s2brXhRZ4vDsI39AZ8m68N57NjbHysxd67h3/kR8ehDDsuif0YTbGzmBQ1PunLhdw5ClFAAkKZ58PYgsJAjrXgS7vCeeBprgdvLmv0Yy9s3v2bxrP2DRulEmj91wdpjzZGNzIthOUoxxJ7hQNAVDL3sxUTWFuISqueNkmHnk7n8Qj3cyYOB09CEt5SlUpcExz7Fp+VZu7/0QAW8QPaizau46pn4wnad/Gkvrbs2jZ3wFU1Ts54pHPyKv0ItpWfgCOp//uojVm/fw33sujLV5NZ5Rdw9j0BVnsGT6StyJLjr1a4ei2pe1WBAILCVSKg5CQde3oGntTmj+HnWa0KNPkxOaoyZSr04SZoQaVlWWaVw3NQYW2dQENhXlcOvcz/EdIpjy/W4VrHO5s9lsDGMnQqjEuS8lOWlsmWOzXIkokhTmI0kIOqXWrxD7mjeowyePjSa3oBhJEtUuQlelsNPtjoidbhdjel/Qg0h/oUIIzrjo1Mo36ChYlsmuPefj8X4DBAADn38aO/echWkWHfM84+58n+ICL3pJioihG/g8fl659e3oGB4lJv++nGJfoMwiJRA0WLZhJ6s374mhZbWHlIwk+lzYg66DOtoOUpTZ5tnP0tztZRZOB9DUVkCEqLIVRFbqRd+4WkrrRhk0zk5FlcvezhVF4oK+J8fIKpvqzqcb5qMfpkhnWhbTcrLZIX9GveyV1MteQ0ryYwhRNuVQlWTuatcfl3wwm0JC4FJU/tH2jAq1MzXRbTtINlHDXlFEgSKvnxcnTOfHOavRDYPu7Rpxz2X9qJueFPbaxLQEHv7iTh4f9WLpmGVZPPDRbaRkhstYWpYJBBEiXIa0MvD5/0A3tnCgEDOEiWV6KSr+isT40cc0z9LfI6uPrVu4CUM3qk261IqNu8KKRyHk5K7dtpdWjTJiYJWNTcWS4/MwZs4EVubtQpVkTMvk3pMGclHTzqWvSUq8A69/epmGkkK4iHONQJaiI8nrLfLxzes/MuPLuSSkxHHeLYPpMaTz0Q+sQQghePWuETw+/idmLtoAQKOsFMZePZCstPKnONrUbrYX56FbESLDwF6/B0lq/LfHj2ramUxXAuNWz2SXt5BTUhtwa9s+tthKVcMCrJojEFbR2E5SBWNZFmOe+YK12/YS1EMXmNlLN3HFYx/z9VNXEx+hx0KXAR35bMebLJmxAizo0LsNmlM7bN4AOfmPU+j5EMvyoypNSU9+CpfztEp5XwcI6uuwrPA8Y4tigsFjl112xbsI+sNzk1WniiRXnwBns/rpaAtkAsHwz6RhZkoMLLKxqXhunv0py/N2olsmfjO0KfDU0p9omphO1/RGADi0DmSkvU9u3gME9bUIEUdC3NWkJN0dFZt8xX7GdL+P3Zv3EfAGAFj+x2pG3HEOVz5yUVTOWVVJjHPy9C3n4g/qBIMG8e7YbKLZ1BxOy2jK77vXh/UnC5oGHVKPLTLcN7slfU9Qyc7GJpZUn9VoNWHJ+p1s3Jlb6iBBKETt8wf5btaKIx6nOVS6DOhIl4EdwxwkgL3776Sw6IOSXVqToL6OXTmX4w8si8bbKENh0Me8fZvYWLgPVWmJEOFRHiHcqOqx1xwMvXFAmCqQ5lQZdOUZ1Ur2fFiv9miKUibJSJElGmamcFKz7Eq1xTAj7/rZ2JwIm4pyWF2wO2xX2WsEeWfN7DJjLufp1MuaTqN6W2hYdw2pyfchRHT24qa+P509W3JKHSQIKYN+/uxk9u/Jj8o5qzoOVbEdJJsK4dyGHchwJpRpAO+SVc5vdDL13HazVpvagR1JqmA27siJ2AjWF9BZvaV8NSqGsQ9P8WQsDleO8pNX+CqZaePKNe+x8Obq33l91XRUSUa3DFomZvBMq4ZYxnpCNUkAEkLEEe8efszzXvrA+exYv5vfJ85Dc6gE/UFO6d+B6/9zWVTeR7RISXTz1v0X8cR7P7Ns/U4kSdC3c3P+dXn/SnP2fluwlhcmzGDHvnwS45xceXZXLh/UpVo5m9UJywpgWsVIIqlWfMa5fg+KkIHwtNI9vshKVdFyjA5lzncL8BdHaCasKaycs5ZTz+0SdRtsbGoqbkXj877XMn7tbH7YvoJ4xcGlzboyrGGH455r16Y9fPrURJbOXEVW0wwuvvc82p9ut2moKghbuOGI2E5SBdM4KzXiwsmpKbRsUKdccwaNbSA0sA5fEJgEgqvKNeex8NvO1YxbPQO/qZem2KzI28Uj6y7kP+3W4PFOwrJ0XM5+pCX/H5IUd8xzK6rCfR/cyp6t+9i6egd1m2WR3aR61u80q5fO2/ePIqgbCCFQKjFdcM6yTYz93/eldVEFHh9vTppNIGhw7dAelWZHbcCy/OzL+zdFnk+wMFHkDNKSnyTOVbN79bRKykKPkGKrSTJ9smKnRJmWnYIkCUyz7B3eMi2S0u1aHBubEyVRc3Jbu77c1q5vuefYvm4XY7rdh8/jw9BNtqzazuJpy7n7nZvpc2HPCrTWxqbisdPtKpiOLerSMDMFVTn40QohcGgK55zWtlxzqkpjsAIRnpHJN5vx177NGEcosDwRxq+bHZaPrFsm83NzCTj/TaN6G2hcfwuZ6e+ilFO9KqNBOp37d6i2DtKhqIpcqQ4SwH8n/hEmHOEL6Lz/w5/oevjC1qb87N1/F0WeT0siukF0Yzt7cm/A558fa9OiSpyicXvbfmWUqlRJJllzc3mz7jGz69ybB6E6yvYiE5IgOSORtj3tOohYYpoWxb5AxKwKm9rFuw9NwFvoxTikBMFfHOCVW9/BMOwU8SqBFcNHFcd2kioYIQTj7rmQQd1aoykykhB0b9uQdx+8hAS3s1xzylIyCXGXIcRBmUsL8BoS9y9L58bZn9B7ynMszt1WQe8ixH5/ccRxRZLID3gjPmdTuWzdvT/iuG6YFERIRbIpH4aRS1HxN1hlmiiCZfnIK3wpRlZVHle26MHLPS7ktIxmtErM5KrmPZh05g2kONwxs6n5yY25483rcSe4cCe6cLodNGhVl//8NLZWpEFWRSzL4p3v5tLvH6/R99bXGPzPN/j2j+WxNqtaYJoedGN3zB3LgC/Ap/+ZyNVtb+eadrfz2bOTCBzeqfg4WDJ9RVi0F8BX5CNnR+6JmGpjE3XsdLsoEO928PC1g3n42sFYllUhN+y05EdRlHrkF76JYe5nWUE6b27pyQavG/DjAa6d9SEzz74Tp6webbpjondmCzZ7cgke1itBIGiWWL7UQZuKpUndNBav2xE2rikyiXHlc8ptwtGNnQihYYVFdC2C+oaY2FTZ9MpsTq/MqtXo+cxLetFrRA/WLdhIXJKbhm3q2Q5SDBn/3Tze+W5uaXQ7p6CYpz74hTinRt9TWsTYuqqJaXrYvf8uirzfIxBIUjIZKU8TH4M0XtM0uaf/o6xbuBF/iSDK+w9/xrwpC3nml4fL9d1Kzkgid1dexHPFJx97ir6NTSywI0lRpqJu2EJIJCfcRKO6C/mucDwPrhnGBm/ZfgMWFr/tXFMh5wO4puWppGjuUnUbAThllQc6nFVG8cYmdtx8/mk4tbJ7HU5N4dqhPSo99a8mE0p5DRcuABmH1ilsdJtnP1N3rGBF3s6o21bb0RwqbXu2pFHb+raDFEMM0+S97/+MmP477us/YmRV1Wdnzg14vN8DASz8GOZuduXeiC+wpNJtWfDzUjYs2VzqIAH4vQFW/7WeJdOPrM77d1x0zzCccWUVFzWnyunnd8edEJ0msP6gTl6hN+ZROZvqT9RWUUKIm4UQG4UQPiHEfCFEr6O8vk/J63xCiA1CiBtPdM6aSp7fG7HJm2GZFAQrLg0u1RHHxH43cl3L0+mQUo8BddvwzmmXc16jjhV2DpsTo3OrBjx36zCa109HkSUyUxO4c9QZXDqwdjXUjDaSFEdSwo0IcWh6mUAIJ8kJt5WOGJbJPX99ydBfXuOBBZO4fOY7jJz2JnmByKmrVZGCnEJeu+0dLml4I1e1/gdfvvgthmHXt9n8PV5fEH8w0kYC7Px/9u47PKoqfeD499w7PZNOOr33DqKA2LCtbXXtZdVVV9e+uvpbdfu6Vd1i7x3X3lBRERtKVzrSCSUhvU6fe8/vjwkhIQNoyJQk5/M8eTR37tz7BsjMnHPe875V9XGOpnMIhXfiC3yFpPUKtZR+ahoejHs86xZswNfob3M86AuydkH7JmCPPu8Izvu/M7A7bbjSnFjtViadOJabH72q3XGaMkAwvA3T9LQ67g+G+MPTczj6+gc56VePcsptTzB/ZfdY6VdiIybpdkKIc4H/AL8A5jf99wMhxHAp5fYo5/cD3geeAi4CpgEPCSEqpJSvt+eaXdmR+YN4a/tyvPsUVZASpuT079B7ZdpdXDfsKK4bdlSHXlfpOIcN78P//nBJosPo8jLTbsOiF1Hb8CCGWYXDNonsjLuwWfemoD23aQFzS9e1qgi5vm43d37zNg9OOT9RoX9vPo+fX0y6naqSGsJNKwJP3/US6xZu5K7/3Zzg6JRk5nLYSHXZqWloO1HXrzA7ARElv7BR0pTGu+/+UUkovDXu8WQXZuJIseP3tI7H5rTRoyirXdcUQnDhHWdy5g0ns3NDCdmFWWTlt7/PUnXDI1TX3wtIkAaprrPJzbwbIaz85okP+GrVVoJNRYvKahr4v0dm89ht5zC8b36776l0X7FaSfol8IyU8nEp5Top5fVAKXDNfs6/GiiRUl7fdP7jwLPArYdwzS5ret5AxmX3blVtyqlbOaffBPq42/dCpijKgQkhSHNfRO+CBfQr2kBBzovYrMNanTNr6xL8RuvZ9JA0mV+2CU84+QtpzH3+C2rL65sHSBCpRLXg3aXsWL8rgZEpyU7TBNeeOa1N+q/dauH6nyRf0ke9x095TUNCU7Js1sFR9jkCWHHaIi0cihurmL1jJUsqt2HGoIptSzPOPQLd0jaV3mLVmXbWoVWydLodDBrf/5AGSA3eN6mu/ydSepDSiyRAg/c1Kmr/SGVtI1+t3Eow1HrVOxAK8+wHSw4p9q5OyMR9JbsOX0kSQtiACcA9+zz0EXDEfp52eNPjLX0I/FQIYSWyHeaHXrPL0oTgkcMv4P2dq3l3x0psmoVz+k3gyCTbVK0o3Y03HO0DDyAgYIRJsdijP54kVn6+JmqDVl3XWL9kM72GtK/Uv3JoguFikGGslv5Jve/qjCNHkeK08chbX1Ne00jfgixuPHs6E4f2SnRozWobfNz1xPssW78DIQRZqS5+d9kJTBrWO+6x6FoGGe4rqPU8hZR7VuA0NOEkPfUq7lj2Jh/sWoOuReazs+0pPDvtUvKd6TGJJyXNxT2f/p4/n3sflTurkUBu7x789pVf4kyCQkBV9f9u8ecUIfFT752FP3AlVqvevIrU/LiE4v1UgVWUg4lFul0PQAfK9jleBhy3n+fkA3OjnG9pup74odcUQlwFXAXQu3f8X/xizaJpnNZ7NKe1o/u1oiixcWTeIGbvXImxz+x0oTOdTFviymV/X0WDCrDaLIT22XyPgJxeKmUq3gKhDZRUXknI2AEIdC2LwuxHcNqTd8/hzElDmDlpSKLDiEpKyXX/ep1NuyoJN/Xo2V3dwM33v8Ws311M77zMuMeUnX4HVkt/ahofwTRrcNqnkp1+O2/u2M2ckrWRtN2mBaSAEeLmxa/y0owrYhbPwLH9ePq7/7J7WzlCCPL7Jk8PQ8Moj3pcSoPeuVZCUXoD6ppg9ICCWIfWucnknXhJtC5b/kpK+ZiUcqKUcmJOjipXrSjJJBgKs213NQ3etpuEO7Obhh9Lhs2FXYvMP1mFhlO38ufxpyf1CsAeJ195HLq1dbqNpmtk5Wcy+sj2NcNW2seUfnaUn0kwvAkp/UjpI2zsYkfFeYSNqkSH1ymt31FB8e6a5gHSHqGwwSvzlickJiEE6e7z6Zv/Of0LV1KQ/TA2S19mbV2Mf599x4aUrKsrpcLfEPOYCvrlJdUAKWiE8chBRMuO1LV0UlPyOP+48a3SPYUAh83KT0+cHMdIla4kFitJlYAB5O1zPA/YvZ/n7N7P+eGm64l2XFNROkwoXELYKMduHYymJf+KQDJ7ae43PPTW10jAMEyOmziIuy6Zid3a+du25TnTmH3sdby6bRnLqorp5+7BBf0n0zMl/jPU7ZHbqwd/nXMXf7/kfqpLazBNybApg7lj1o2dYpDXlTT6PsKUAdq2pTeo97xBVtqViQirU9tdVY+mtf13bJiS7UmWkuUPR2/gqgkNn9H+5q6d0braUn729XMU2Qfw24HLsWlh9vw1CuGkR/ofEELjujOnUdQjnec/XEpdo49xg3ty/U+mU5QTm/REpevr8E8lUsqgEGIZMBN4tcVDM4HX9/O0BcCP9zk2E1gqpQwBtOOainLIDLOeksqf4wssRAgrEoPstFvJTut29UI6xCfLNvLAm1+16qXyybKNWHSN3116QgIj6zjpNidXDJ7GFUxLdCjtMnLqUJ7b9ACVu6qxOayk90hLdEjdkmGU0/T214qUfsKm6r/VHkP75BGKUs7ebrMwYUjy7JsCmFk4jOc3LyIkW8ebbnXQy9U5Jl06gilNrln4InUhH3WhdG7/7gzOL1zKoJQK3LYB9M++A5fjSCCyAnbmjNGcOUNtQ/jeJG3nYZRmsUq3uw+4VAhxhRBimBDiP0Ah8AiAEOI5IcRzLc5/BCgSQvy76fwrgEtpXajhgNfsTExTsqWkiu1lNarZWZIrrboWb2ABkgCmbERKH1X199Lgm5Po0Dqlp95f3KbZZCBkMGfRerz+/RQ9UOJOCEFOz2w1QIqx2qCXN4u/5ZVtS9ntq2v1mMM2ASHaVhoTIgWX/dAqjXVX+VmpnHTYsFYpWbqu4XbYOHPGqARG1taVg6eT70xrrmJrFTpO3cpfJ/y4W63qrqrZhcfY+95Q7M/ib1uO52erLuShnZc0D5AUJRZikt8ipXxZCJEN3AUUAKuBk6WUxU2n9N7n/K1CiJOBfxEp6V0C3LCnR9L3vGbSkTKML7gCISw4rKMQQuObDTv59ePv4/EHMU1JQXYq91xzKv0K1KboZBM2KvH6v4Q2jf68VNc/TKrzxMQE1olV1DZGPa5pgjqPH5fDFueIFCUxPi5Zy+3L3kATAinhr6vmcOOwY7h0YKRgq8M2Fpd9Kt7AV80VvQQO7NZBpDj2VwNJOZg7L5nJkN65vDzvW7z+INPH9OeqUw8n1ZX46m0tpducvHXMNby7YyULK7fS05XBuX0nUtRJUnc7SsAMoxF9ULjvni2lndRc/X7FbBOAlPIh4KH9PHZUlGOfA+Pbe81k0+j/gl1V1yBlGJBowo3b8SA33L8MX2DvL3bx7hquvOdVPvj7lVij9CdQEscwa0BYIUofi/1V2VEObOzAQuZ9u7nNCqrdaiE3052gqBQlvmqDXm5f9kZzw+E97l83j6m5AxiUlocQgqIeT1Lb+Dx1nllIGSYt5Swy3VdEXWFSvh9NE5xzzFjOOWZsokM5KKfFxjn9JnJOv4mJDiVhxmT2xIyScePUrfyoZ3Kt/ildT5etbpdIIaOMHZWXY5i1mLIRU3oIm2VUNlyCrreu5iWJpBt9uSr+3bU7s+1lNazaUtomdau9Sr21vLhlIS9uWUiptxYAm6UvIuqviEUt8bfTNWdMxWm3oLVIF3HYLPzynBnNvUAUpav7dPd69CgpUyHTYPbOVc3fC2ElM/Vy+ubPpV/BZ2SnXY+mOeMZqqIklF23cve4M3DoFiwi8h7h0m2MyCjk1F5q75ESW52/nFQSqvO8AVE6Y0tpMrr/er5e3bqUrmEYVNZ54hVep1ZR28jND77NlpJqLLqGaUp+ec6RnHlk+18sX9q6iHvX7O1l/K+1H3PLiBM4v99kcjP+RFntr1s0sLOiaW6y0248xJ8k/qQ08AWXYcoALtvEhHzY6leQxQt3Xsij7y5g5eZSCrJS+dkphzFleJ+4x6IoiRI2zagZLiaSkNm2sICidGfHFw1naEY+bxZ/S1XAw4z8QRyVPwRdqIm1jiBUut1+qUFSDITNSiRtu9ZbdIPM1LapW0IIxg0sjEdond6N97/Fpl2VGKZkT9biva98Tr+CLMYN6vmDr7fTU8O9az5qk/Zy75oPOTJvEEXuc7BaelHd8BCh8C5SHNPJSrsGi57fET9O3PgCy9le+VOk9BOpqG9SkHkf6SmnxD2WPvmZ/OXKk+N+X0VJFjPyBvGXVe+3Oe7QrRxfqPpRKcq+eqdkcePwYxMdhtLNqGF4DLjt0xGibS8dIXQavKNa9YNx2CxMG9WPQT1Vw9uD2VJSRXFZDYbZetojEAwza+637brm3NK1RJvTlUjmlqwFwOU4nJ45z9OvYB65mX/odAMk0/RRXHEBhlnZlP7ZgCk9lNTcSDC8LdHhKUpSafD6WfzddjaXVMbsHrnONH45fCZ2zYIuBILIAOmM3mMZm5VcpagVReniZAK/kpxaSYqBFMeRuGzjqfcvQReRFSW/YWFpXW+GHDORMQNS+GDRd1gtGmcdOZrTpo5IcMSdQ02DD4vedlwv2X/VtIMxpSRaC28piTp46owa/fOI9GJuTUqDWs+r5Kb/Kv5BKUoSevL9xTzx/kKsuk7YNOmTl8l/rz+DnPSOLypy8YApHJE7gNk7VxIyDGYWDmNMNxoghY0avMHFaCKVFPthqhiFoihJRw2SYkAIDc11P09vuompmd9hSI1PKoeyoLY/Nu1LXj3qGi49cVKiw+x0hvTOIRRuu9fLZtWZNqpfu655TMFQHlr/aZs9ZJoQHJs/rF3XTDaGWYeMMkiCEGGjKu7xKEoy+mLlFp78YBGBkEEgFPl92bSrklsefpfn/u/8mNxzQGoONw7rfilEFfWPUVb3d4SwEan+6qRfziwctq7xmqsoSteg0u1i5NOyTXxePYzfbzyVP236EV/XDkAiCEuTT0rXJTq8TsnttPOL049o1QjQZtHJSnVxbjvLufZ19+CaIUc1pb1o6ELDrlm4ZshR9HZ3jd5VKY4johYSESKFVOfeD2gBI8TsnSv455oPeK14KZ5w2311itJVvTj3mzbVMg1TsnFnJbsq6/bzLOWH8gSWUlb/z6YG3Q2YspGwWcHWiguRUhWt6K42l1Ty+YrNlKjftfhT6Xb7pVaSYiSSZd72X4Bo+lLa56LjJzCwZw9enPsN1fVejhzTn/OPGXdIjQB/Nmg6R+cP5eOStQgBxxUMp39q19kjZrP0JdN9CTWeF5HSC4AQLpy2cbgdxwBQHfBwwZePUhv04jWCOHUr//1uLi9Mu5LeKV1jsKgoB1Lb6It63KJr1Hn8FPVIj3NEXVN14/NNBWRaM6UXb2AJKY4pCYhKSZRGX4AbH3ybdcVl6JpGKGxwzLiB/PGyE6Om138fhllLg/9LBBZSHTPQtLZ7xBXl+1CDpBg5tmAY/1k3t81xXWjMLFDViw7FlOF9OrxkdP/UHH4+ZEaHXjOZ5GX8jhTHdGoaX0DKAOkpPybddUbzPoD71n5Iub+ecNOKk88IETDC/G75Wzw99WeJDF1R4uLI0f0pLqshGG67mjGwUE0UdBTDrGV/U8iGbN/eUqXzuvvFT1i9dTehFr93n63YzHMfLeXykyb/4OtVN75CSc0dTe9tkUquvbMfI9V5VIfF3JUIqUqAH4hKt4uRQlcGt4w4Abtmwabp2DQdu2bhhqHHdpk0LqXzEEKQ6jyW3jlP0yd3FhkpZyOEtfnxebvXNQ+Q9jCRLK/ZTsAIxTtcRYm7i2ZOIDPVid0aPSwVdQAAIABJREFUmTgQIlJ99LbzjsJmVfOJHSXd9SO0KNVfpQyRYp/U9P+SxZVbuHvVbO5ZM4f1dbvjHWaXs2FnBR8uWc/6HeWJDqVZMBRm3rcbWw2QAPzBMK98tuIHXy8Q2kpJ7R1I/JjS01TN1Utx1ZUYZn1Hha10I+qVP4bO7zeZGXmD+aR0HSaSY/OH0TMlM9FhKUob2n6b8gk0oRJEla4vPcXBK7+9mFc+X8H8VdvIz3Jz/jHjGd2/INGhdSnprjOoanwef2h9U/qvhhB28tPvQNfSkVJy1/I3mVu6Br8RQiB4Zdtirh1yLD8dOPWg15dS4gksIhDahN06sKlyXvd9DfMFQ9z04Nus2lqKpmmYpsmw3nncf90ZuBy2hMYWMsxo22UB8Aba9pQ8mFrvm0gZjvKIoN73EZkpP/nB11S6NzVIirFCVwYXDzg80WEoygGd0nMMrxUvIWjundHT0TgiZwBWTb1MKN1DqsvBz046jJ+ddFiiQ+myNGGjf+7r1Hnfoc77PrqWQbb7Ylz2cQAsrdrG3NI1+JpWsCUSvxnm/vWfcFLPUeQ60vZ7bcOsZ3PZOQTC25AYCDTsln4MyHsFXdv/87qy/77xJSu2lBAM7X1tX7NtN/e+9gW/uei4BEYGKQ4bvfMy2FJa3eq4JgSHtyOl3pQeorW7QJqYTftxlShk951EOBiVbqcoCjcMPZaBqXm4dBtWTSdFt5HvSuf3Y85IdGjK97CltIq3vlrNV2u2ETb2MzWrKElCEzYyU35C35yn6JV9X/MACSINvv1RUnx1IfiqfNMBr7ur5g/4QxswpQcp/ZjSiz+0gV01f+jwn6GzeHfB2lYDJIBg2OD9ReuQUXoExttvLp6J02ZtLtJgs+qkuuzccOb0H3ytNMdMNOGM8ogk1XHUoQWqdEtqilhRFFwWOy9N/zlLqrayob6M3ilZTM0dhL7fNLy2ImkuC/EFV2Gz9CbNeWyrfU9KxzNMk988PYdPl29GEwKhgdth5/FbzqZXTkaiw1OUH8yuW9AQGPsUdxAIbAdZ1a71voOk9QBLEqLW+w69s+/t8Fg7g2iFSIA2+4ASZcyAQl7+7UX879PlbCmtZkz/As6eMYastB9ekc5lP4x6ORmLMR+bFkm7C0kLS+snM7iwsKNDV7oBNUhSFAWIFHeY3KM/k3v0/8HPNUwvW8rPwx9ajynDaMKGrqUyMO9NbJaiGESrALz11Wo+W7GZQGhvHr4/EOZXj87mf3ddlMDIFKV9Tuk5lpe2LsIwW+8tMZHMyBt8wOdG348SKQrRXU0e0osF64ppuWgkgHGDipJmr1bPnAxuPeeoQ76O3wxx8+pBDHTZmJK+hbDUmF8zkJ2BnuSkLeesPhMPPdiuKPELiklLpdt1I2HD5POVm5k17xuWbdyZFEvtStdQVvdvfMG1TXnfQUzZSMgoZ3vVzYkOrUt77fOVbRqgmlJSXFZDSZWq5qR0PoPT8rhp2ExsmgWnbsOl23DqVu6dcC5uqwNPOMALW77imkXP8seVb7Oxfm/lu1THkbT9WKOR6ui67R0O5rZzjybVaW+u2mi36qQ4bfz6/GMSHFnHW1mzE13orGoo4vGd03l611Q2evPwGSHmlKxKdHhKJ6RWkuIkkoq0hIbAQixaJpmuU7Ho8UuHKa9t5LJ/vkydx0fYMNF1jUGFPXj4prNw2lRKlHJoajyvIQnsc9TAE1iCYXrQtZSExNXVtVxBakkTkfK6itIZXdj/cE4oHMlXFZuwajoz8oaQYrFTH/Rx3pcPURXw4DdD6ELjvV0r+MvYszi2YAQ9s/7Mht2nYppeJD4ETjTNRc+sPyf6R0qY3nmZvPnHy3jjy5WsKy5nSK8czpo+ul3pbMnOqVsx9zP567ba4xxN56H6JO2fGiTFgZQGWyqupCHwNab0owk7u2rvZmDOs7gd8ami9Ntn51Be24BhNv02hA2+21HOo7MXcNOZR8YlBqXrkhyoWIAqJBArx08cwjMfLmmz7yDVZadPnmo3oHRePRypnN5rXKtjz26ZT0WgobkKpyFNDMPkDyvfZkbeUGyWXgwr/JIaz5v4QmtwWkeQmfJjdC01ET9C0sh0O7tFxcaRGUWkWh14jdblw526lbP7/PDGtIqi0u3ioNrzRtMAyQuYmNKHKb1sqbwaKWO/edIXCLFsw869A6QmwbDB7IXrYn5/pfMLGmFeLV7M5Que4NrFz/FZWevKSBmuUxDsuyIpcNlGdvsPKLF08cwJ9MrJwGmP/NlbLRpOm4U/X3ZS0uw3UJSOMm/3ulZtCvYImQZbGysA0LVUeqReQq+sv9Mj9RL1+tONaELjwcmXkGVLIcVix6XbsGkWLup3BEfkDEx0eEonpFaS4qDS80rUGv2m9OMNriLFPjam9/9sxaY2A6Q9DLPrzfJ7Asup9c1BYCUr5XQcVvXieCjCpsGVC59iff1u/GZkA/Q31ds4q/ckbh1+EgD5GbfS6P+SkFGGKT0I4UQTDnpl/yuRoXd5KQ4bL95xAXO/2cji9TsoyErljKkjyctUHwyVrifV6oh63JAmKZbojyndy6C0PD4+7lcsrtpKXdDLhOy+B+ytpaAKNxyAGiQlXGxnez3+IH96cW7UxzRNcNz4QTG9f7xtr/4N1Z5XMKUf0ClreJSijDvITb000aF1WvPK1rGhoax5gATgM0K8UryY8/tOociViUXLYHDBx9R7P8QbXIXd2ocM1+nomjuBkXcPNquFkw8bxsmHDUt0KIoSUxf0O5wN9bubG80C6EJjUFoehS5V8l6JsGi6WjlSOoQaJMWYlJLS0FSc5jdYtdZlSDXhxGUbGdP7L1xXjK5Fz6p0WC1ce9rUmN4/nhoDy5oGSL6mI2GkDLOr5m4yXSdh1fMSGl9nNb98A759crwh0txxWfU2ilyRvS+asJGRcioZKafGO8RuaVNJJR9/swEpYeb4wQwq6pHokBQlpk4oGMna2l28tG0RNk3HkJJ8Zzr3Tbgg0aEpSuckVeGGA1GDpBi7d90c3they2VF+Yxwl2LVDAyp49Dt9M95FCH02AZwgH/8x4wdSIY7WnfqzqnW837TClJrQujU+T6lh/u8BETV+WXbUrAIjbBsnZqpCY0Ma9erkNQZPPHBIp6Ys5iwEdmf8dzcZVx2/ER+/qPDExxZ9+EPbaGi8WXCZjUZzmPJcM6M/et5NyeE4JfDT+SS/lNZXbuTHo5URqQnT78fRVG6FjVIiqEyXx2vFC8maBo8uP1oBrgqGJqyG7+ZwrSCqxhrnxTzGA4b1hvDaLvvyGmzcsqU4TG/fzwJzUKkFknbjb0C9eGlvc7oPZGXti1qM0iyCp3DcwYkKKruq7ishifmLG5V/tswwzz90VKOnzCEfvlZCYyue6jyvMvW6l81NS8NU+19D5dtFENyX0ATqqVCrPVwpHJUvkovVRQltlR1uxhaUbsDa3Oqm2CzN5f3KkbzSdUAvqgoiUsMbqedP116InarBZtVR9cEDquFU6YMY9KQXnGJIV4yXWcgonxAkZikO2fSEPLzwPqPOO2z+zj7i//yyraFGLLrFa7oaH1Ssvnz2LNIsdibKwblOdJ4bMplWDU1zxJvn6/cjBml4Iphmny2cnMCIupeDNPHturbkdIPRAaqpvTiDa6kyvNmYoNTlBaklCz8rpg/vzSXe177nPU7yhMdkpKMZAK/kpz6hBND2TY30fqa6WjkO9PjFsex4wcxun8BH3+zAV8gzLSRfRnSKzdu948Xl20YBek3UVr7LxACgYbEpG/WvzFI4ZKvH6TUV9NcQvbf6z9kWfU2/j5epeEdzHEFIzgydwir63Zi16wMSy9AE2qOJRE0TYuaXiQE+91/qHScxuAyRJT5RVP6qPa+S477nAREpSitSSm585k5fLZiM75gCE0IXvtyJdeeegQXHzch0eEdUL3/ayoaX0bKENkpp5HhPB6h3m+UBFCDpBgal9WbDJsLvy+E2WLIbNV0zu4d+1S7lnIy3FxwzPi43jMR8tN+QZbrdOp88xDCSobzBCx6Ju/u/IZyf12rHht+I8QX5d+xpbGc/u6uN2jsaDbdwvisvokOo9s7dtxAHnhnfpvjmhAcN05VdIo1TTiQ+5kC1YTao6ckh8XrdzQPkABMKQmEwjzwzlecOGkIOenJWXl0R81fKW98rrkAU53/M9IdRzKgx8Nq71msdIIVnURRQ/MY0oTG41MuZ0BqLg7Niku3kWpx8JdxP6F/qvpQHis2SxE5qRfTw30eFj1SeW1p1dZWZWP30IRgVc2OeIeoKO1WkJXGbWcfjd2q47BasFst2K06t5w1g8Ls+K1Qd1du2zj0KIMhTbjIdZ/f6tiHJas4+4v/cszHf+HGJc+zqaEsXmEq3dwnyzc2D5Ba0nWNBWuLExDRge2srOWh2a+ws/bJFhVqI6msdf4vaAgsSGB0SnelVpJirMiVyatHXkexpwpvOMDA1DysmioiEG+Frgysmk5on27tAqEazSmdzpnTRjFtZD8+X7kZCRw1egC5Gck5M9zVCKEzKPdp1pdf1FS4wURKg1z3JaQ5ZjSf9/yW+Ty88RP8TZMz8yvWs7R6K88fcbWaJFM6nJRhKj3vUNn4BkLoFOWOQhM6+/aRF0R6qyWTpRt2cP3Db3H4yG8ZPRQs+3xEMqWXWu8npDmOSEyASreVXL8pXViflOxEh9CtndFzIs9tmU+oReU7DUGa1cnkHqpCm9L55Ga4OfvIMYkOo1tKsY1kbNEi6nxfYJi1pDqOwG4pan48YIR4ZOO85gESRDJaAkaIRzfOU/sglQ4lpWRDxc+p9y/ElF4Ahg5Ywk9/1I+nZx/X6lxTSqaP7JeIMKOSUvLb5z/CHwzj9VsxZbSUOiu6lhr32LoL1Sdp/1S6ndIt5DnTuX/ST8l3pOPQrNg0C0PTC3liyhXoakOooij7WL+zgt88N4fL73uZh2d/TU2jr9XjmrCT6ZpJD/fZrQZIALt9dVGvaSJZXbszZjEr3VO9f0GrAVKEj0nDNtG3oBqn3YrLbsVps3LfVaeR4rAlLNZ9ldc2Ut3gAWD5pv5E23YUNE2uW76Lv6x+m4ZQ216IihIraiVJ6TbGZ/XlvaNvZZevBrtmIUel2SlJypQByhpmNafO5LjPJdd9NkKol+x4+GzlZn791PsEwwamlKwpLuO1+at46f8u/F5pjVl2937bCxS6Mjo6XKWbq/fP32eAFKFrknt+XsjaLTOwW3WmjeiHK4kGSAAOm7U5JTAQtPHAa6fxizPfjQyWLBJdmDy3awrbfTZKdy5jZe12Zk29VlVXVeJCveMq3YoQgp4u1WxTSV5SmnxXdgme4EpMGZk19dVsoM73KYNzH0twdF2faUr+NGsu/hbNeoNhgzqPn8fnLOLO84496DVSrQ5OLBjNh6UrCZh7r+PQrFwx8OiYxK10L7Kpv4gQAouehcCOJNDqHCGspLlyOTWJG8enpzgYP6CIZRt3EjZNNuzoya0PXMmIicVY+nlZ6+1BwIz0PwxJg13eahZVbubwnEEJjlzpDtQgSemUpJTMWbae/32xHF8gxPHjB3PBUeNw2ZNrlkxRfqg6/5d4gmuaB0gQ6cFT559PY2AFbrvahxRLJdV1eAPBNscN02T+6q3f+zp3jjoNm6bz7q5vkYDbYueWYSdzmNoDqRyC3b5a/rrmHb6u3ICG4Ki84dw65DiEuDdKX0ZBluuERIT5g/zlspP4+X9fY1dlPZomCBs6npQRbGlsm5oaNA02NZapQZISF2qQ1EXV+L5gZ939BMK7cNvG0DvjZly2wYkOq8P89ZV5vLtoLb5gZJa2uLyGD5auZ9ZtF2BPsso9ivJD1PsXYUpPm+NShmkILFaDpBhLcdgx9i0J1iQ9xfG9r2PVLNwx6nR+OfxkGsN+smwpKkVIOSQ+I8hPFzxCdaARE4mJ5NOytayvL+WpSQ+zpfJGaOrKqAkbg3MeQ9eSv+plVqqLV+64mDXFZZRW1zO8dx7L/Jv4x9pyfEbrCQubZqG3SxXC6lCqcMN+qU+TXVB54xtsqb6reSa62ldGrf9LRuW/RoptaIKjO3S7qup4a8EaguG9leoCIYOSqnrmLFvP6VNGJDA6RTk4U5rUh/y4LXYs+7QEsOm5COFAytYblIWwYdVy4hlmt5TpdjJxUE+WbthByNi7r8hhs3BROxpyO3QrDt3akSEq3dQnu9fgCQdaNac3pElVoIE1jflM7bWExsByBDpu+1iE6DztRoQQjOybz8i++QBkhUdx//oPCRih5p9XRyPD5mJqTteZ8FWSm5rWirPPV23m4nte4vi7HufWJ2azdXd1h15fSoNtNXe3StUBiSl9bK+9p0PvlSgrtpRi0dv+0/UFQ3y9dlv8A1KUH+CtHUs44ZO/cPKnf+WYuX/i4fUfYbbY5J+dchqCth9uhNDJdB0fz1C7rb9cehLDeufhsFlwO2zYLDrnHDmGH00elujQlG5sS0NZm5UViKSgbfNUogkbaY7JpDomdKoBUjROi41nDr+aCdn90IRAFxqH5wziqSlXtZlYUg6BjJQAT9RXslMrSXH06vwV3PvGF/ibUsTmrdjE1+u2Meu2C+ib1zHFBEJGVdQqNyBpCCxvdaTS8z476v5DMFyKyzaEPhm3k+aY2CFxxFJ2motonRQsmkZepuqloCSvebtXc+/a2fjNSP+cEAaztn2FEIKrB88EwKpnMTT3GTZWXothNgISq5bNoNxH0TVXAqPvPjLcTp699Ty27q6mrLaBwUU5ZKWqP3slsQam5uPSbXjbpKDpDHB3vQbFRa4sHpn8M0JmGIFQgyMl7tRKUpyEDIP/vP1V8wAJIk3d/MEwD723oEPuURv0UOoPRdm8GWHT85r/v6zhf2yq+hW+0EYM2UhDYBlryy+h3r+0Q2KJpYmDeuJ22tH2aahg0TV+Mm10gqJSlIN7dOPc5gHSHn4zxEvbviJs7k0fTXVMZFzRAkbkv86I/LcZU/QFKbbkrVDVVfXLz2LK0D5qgKQkhWPzR5BqdaK3+OhmETp5jnSm9BiYwMhiy6pZ1ABJSQi1khQnu6sbMIy2fTNMKVm+peSQrl0f8vGb5S+zpHozFqFzVl4/JmZsQbB3tkkTTnqmXwtESgwX1/4DU7ZujmhKP8W1/2RU/suHFE+s6ZrGEzeezU2PvcOuyjo0TWDVdf50yQn0zlE9SJJRIBTmsQ8W8ubCNQTDYY4c0Z+bzphObnrybyruSGX+6E1GQ6aB1wiSpjmbjwmh4eoCewgVRekYdt3Ks4dfzT3r3uOL8u/QEBxXMJJbhv1IFQVR2q8TpL0lihokxUmm24lhRm8umJ9xaCliv/rmBVbVbCckDUIYvFQyCpMwh2VsRxMWBBq9Mm6mR8rJAITNOgwzWkoeeIPrDymWeOmVk8Hrd15CcXkNvmCIgQU9ou5TUpLDjY+9zbebSwg09Z758Jv1LN6wg7d/c2lSdX+PtUGp+SyvKW5z3G114LbYY3LPhsBKNlX9gcbgKnQthQL3RfTJvAFNqGICitLZ5DjS+Pu48xMdhqJ0C+pTZZy4nXZOmDCkTXlqh83CFSdObvd1d3qqWFO7g5Dcm6pjoPNSySTer76dsQXvManXUgrTLmt+XNfcCBF9fGy3FLDLW83Kmu00hv1Rz0kmfXIzGdozVw2Qkth3O8tZ3mKABGCYEo8/wLuL1iYwsvi7bsiJOLTWgxOHbuW6ISfEZCbYG9rCyt0X0hhcAZgYZgMlDU+zsfLODr+XonQGUsrmRqyKoigHolaS4uiupk7tHy5bj65pWHSNG0+fxpEj+7f7mhWBeqyapVVXdwCJZLvPi8Pap81zNGGlMPVyShqebJVyJ4SD98pH8sHqf2PVdMKmyeUDjuKyAapDvNJ+63dWIETbUhu+YJhVxaWcx9gERJUYYzL78MDky3lg/YdsathNvjODqwYey9H5sSlbv7PuMUwZaHXMlH4qPLPpl3UbNr1HTO6rKMmmusHL3179lHkrNyElTB/Rl1+fcyx5Gd0r5bclUwYwZRBduKO+RivdhJoz2C81SIoju9XCny4+gdt/chR1Hj+5mW6s+qFtRhyYmk9wnwESgFXTmZy9/87uvTJuAgQlDU8hZRCLlsb82iN4v8xBSIabr/n0ls/o687l6DzVe0hpn1490olWjtBu1emX1/2aAo7J7MPjU66Ky70ag2uBtmm+mrDjC21TgySlWzBMk0v/9TIl1fWEm/YGf7lmK2vumcW7v70ch617fRQyTC+bqn5Hhec9wMRuKWJQ9p/JcB6e6NAUJamoHKUEcDvtFPVIP+QBEkCq1cnF/aa3alaoC40U3c75fafu93lCaPTOvJnDen3LpF5L6Jczl9dK01ql7QH4jRAvbP3ykONUuq9xA4ooykprkxJp0XXOPGJkgqLqHty24VF7LpkygNPSepXZkCbfVG/ly/J11AWj71lUlM7oq7XbqKz3NA+QYE/Kb5BPVmxMYGSJsa78eio87yMJIgnjDxezpvxKPMGu8WcRNkyqG7z73Qeu7CVQfZIOpHtNn3RRPx90HP3dubywdT61IQ9TegziioHHkG0/eEEIISxYRBoN4XIsQieE0eacmkBjLMJWugkhBI/fcDZ/mPUx89duRUrJkKIcfn/h8aq0coz1TL8q8mGoRe80TTjo4ToJmyWn+diWxjKuX/I0nnAAAYSkwS8GHc8F/aYlIGpF6VhbdlcTDLV9b/MGQmwurUpARInjD+2gLrAQyb5puCF21T/J4B5/i1ssYbORoFGOXS9Ab1HZs72klDzx0RKenruEkGFgt1q4+sQpXHjUOJVOqLSLGiR1AUIIji8cw/GFY9p9jV6ubCxRNo5bhMaUnEGHEp6ikOl28u+rTiMYCmOYEqddVVaLB5e1P6PzX9hb3U6kUJB6IX0yb2w+x5QmNyx5mspAfavU9Ec2fszwjJ6Mzewb97gVpSP1z8/CZtUJB1qvLLjsVgYUdK+UX394JwIb7DNIAgNvaHNcYpDSYFP13exueBUhLEhMeqZdRt+MGw9pMPP8p9/wxMeLmvtRBsMG97/3FU67lbOOGNVR4SvdiEq3UwCwaDq3DjsVh2Zt3j5iFTpui5PL+qvCDUrHsFktaoAUZ6n20YwrfJ3pfTdwRJ9v6Zd1a6vy36tqd9AY9rfZuxswQ7y2fVF8g1W6lLBhsrm0ivK6xGYjTB3el9x0d6uUX10TpDrtHDe2e00CumyD2hRzARBYSbWNY0XNNt7csYhl1VtiVgVwW+1/2d34GiYBDOnBlD521j9NScOLh3TdJz5e3DxA2sMfDPPYh+p17IBkAr+SnFpJUpqdVDSOQlcWL2z9kt3+WiZnD+CCvtO+V9qeoiidkyfsjzp7K4H6UNfbm2SYPhqCq7Fq6bisg1QaTozMXb6RP/5vLiHDIGyYjOqTzz8vP4XsBKTY6prGM788l3++/hlzv92IKSUzRg3g9p8c1aYtR1dn03uQ5z6Lcs9bLarbCjTh4F+bNNY0PI0pJZoQFDozeWjSVaTbOu7vTErJrvrnMGXrFiOm9LGj7jGK0i5q13UN06TOE71tSWW9p13XVJTu9eqgHNSYzD6MyWxbNlxRlK5pdEYfwmbb/RoOzcqx+V0rRaW04VU2Vf8ZgY7EwGEpZFTu4zisPRMdWpeybkc5dz4/B3+L3mgrtpZy7cNv8r/bLkxITBkpTu6+5CTuvuSkhNw/mQzM/gMuaz921T9D2Gwgw3EYc6snsKJuOyG59+9su6eSe9a9w5/GnNdh95aEMFq0HmkpZNS0+7q6plGUncauqvo2j/XNzWz3dbu8TlJAIVFUup2idCGhsMGyTTtZvrVEVfZRvhe31cENQ05qSrWNrKo4dCv93LmcVNh1eljVB1awqfpPmNKHIRsxpQ9vaCsryy5XzUU72IuffUMw3HrgHTZNtpZXs7GkMkFRKXsIoVGUfjmTe33BEX2+ZXjeI7y1a1erARJECrh8WrYaUx76e4mUkm+qt/DU5i8wyIl6jts+/JDuccsZR+LYZ2XQYbVw649nHNJ1le5LrSQpSc0bKqYhsBq7JZ90+3iVGnMA89du5fZn34+k+cpIH6L/XHk6o/sWJDo0Jcmd3edwhqf35LUdC6kL+jg6bwQnFI7BpnWdt4hd9c9H2YthEjDKaQyuJtXetVbNEqm0pgEzysDTomlU1HkYVKj6cyWbsGy7mgyRwi6HOoUQNg1+9e1zLK/Zht8IMsI9hHMLKrFpe+4p0ISdAZm/PqT7HDtmEPf9zMoD733Fjso6+udlcd0pU5k8uNch/gRKd9V13gGVLkVKg7WV/0eF90MEFkBit+QzLu9Z7JbcRIeXdMpqG7nlqdmt0ls8Abj64TeY+8crcdltCYxO6QxGZPRiREbX/TARNCqItlNYoB1Smo/S1pQhvVlVXEpgn7LbwbDBsF7q9TsZTc0Zwudla2k5JBIIxmf1R49S+faHmL1rGd9Wb8VvhgBY3ZhDw47pnJizgWGpkX5ufTOuO+SVJIgU6Zg6vO8hX6dbUQvp+6XS7WIkbJh8tW4b7y5ey87K2kSH0+nsbJhFhfdjTBmpfmNIL95QMWsqb0l0aEnp/aXros7cSimZtzI+ZV0VJZHqgh5e3PYFv1/1P14q/rJN0Yls59FowtHmeZIQqfb2t09Q2jp3+hjSXU6sLarJOW1WLj56PJnuQ++Ho3S8m4eeSqYtpbkxvUOzkmZ1cvvwMw752rN3LWseIO1R7M/m+ZIZZKS/yMi8hzpkgKQoHU2tJMXAtvIarrj/VbyBEBKJYZicdtgI7jz7mFbpYo3BDVT7vsaqpZGTMhOLpqrI7bGrflaLyjt7GNT5lxM0qrHpWQmJK1nVenxt9gBAZLBe541e8UdRuortngquWPQQQTNMwAzxedkant3yKU8ddh2FrshrRUHq2ZQ0vEQgXILZ1CNGE076pF+LVU/FhrixAAAgAElEQVRPZPhdTprLwcu3XchTc5fw+eotpLscXHz0eI4fNzjRoSn7ketI59XptzKn5FvWN5Qy0J3HSYXjcVvbTiz8UPvLkpfQvA9SSaBOtJIkhPg18BfgQSnldU3HBPA74CogE1gEXCulXNPieZnAf4HTmg69A1wvpTzgKoYaJHUwKSU3Pv42lQ0eWk7sz16ylgkDe3LS+CFIKVlf9Vt2e95BShMhLGyo/hOj8x4j0zEpccEnEUNGLz0shIja46G7mzKkDy/PX4kv2Hq2TgjB5EFdN4VKUQD+se6tpl5PkRddvxkiaIa577t3uGf8pQDoWgrjC1+ntOElKrwfYdWyKEq7hCzn1ARG3nVlpbq49ccz1Kb5TsRlsXNm7ykdft3TiiaxsaEUv9H6/SnFYmdQan6H30/pmoQQU4gMhFbu89BtwC3ApcB64LfAx0KIIVLKhqZzZgG9gRObvn8CeB449UD3VOl2HWxrWTVltQ3sm/nkC4Z55csVAFT65rHb8y6m9CMJYkovhvSyqvxaTBmKctWubU3dDn678iV+vvhhnt48j/qQlxzXcU17kVqzadnY9eR4UZVS8s2WXTzx8WLeWLiaRn/iBm+HDe7N+AFFOG17m4Q6bVZOGj9EbZJWujRTmnxbvaV5gNR8HMniqo2tjlk0N73Sr2R8wauMyntUDZAUJQ5OLhrPpKyBOHQbOhpOzUqKbudvYy9CO8T9Tkr3IIRIB14ELgdqWhwXwE3A36SUr0spVwM/BVKBC5rOGUZkcHSVlHKBlHIB8HPgFCHEkAPdV60kdTB/KIy2n7VlbzAIQGnD61FSySLFCur835DpPCymMSaTD0u+5a9r3yBohpFIvqvfxVs7F/Hk5Muo8H5CyKzBlH4EVoSwMCzn70lR4S5smNz45Nss27SLQCiM3apzz5uf89i1ZzGyd/wHcZom+O+Vp/PBsu94d8k6LLrGmVNGcuyYgXGP5VD5w6WUNLxNwKgg2zmVHNcMhNATHZaSpAQCXWiYUapzWbtQdT5F6ax0ofGPcRezum4731ZvJcOWwjF5ozoklU85dJ2kT9JjwGtSyk+FEL9rcbwfkA98tOeAlNInhPgCOAJ4FDgcaAS+bvG8rwBP0znr93dT9Q7SwQYX5mDRdaD1ipDdauGk8UMBkISjPJOmx6KX4eyKQmaYf373NoEWGzqDZpiaYCMv71jJzwe+R2njG9T6F+O09KEo9TycSdL08c2Fq1m6aSf+YOTv0tf0318+9S4f/u6KhAzkLLrGqZOHc+rkzrsBttI7nxXl1yOliSRIaeNbpNqGMqHgaTShKvQpbQkhmFkwho9LVxBqMVCyaRZOKhiXwMgURdlDCMGojD6MylDN6pVWegghlrb4/jEp5WMtTxBCXAkMBC6K8vw9s9Jl+xwvA4panFMhWzTEk1JKIUR5i+dHpdY5O5hF17j7ohNxWC1Ymir7OG1W+uZmcs60SAWlfPcZaMLV5rk+I8DjW4sp83ePanjbPOVRmziGpMH8iu+waCn0SruYUbn3MzDr1qQZIAG8tXhN8wCppXpvgE2lVQmIqPMzZZhVFbc2p6FCZG9afXAtu+pfT3B0SjK7echpDEotwKnbcOo2HLqVoWlFXDv45ESHFlP+YJg3F67mrhfn8MichZTVNiY6JEVROhuZwC+olFJObPG17wBpCJFCDRdIGf/9KGolKQamj+jHa/93Ma9/vZryugamDuvL8WMHY7VEUoZyXSdQ7nyfKt98DNOHITVMBC+XTmCzdyWflW/g+cNvpoc9LcE/SWylWlz7bWCXYU2JczQ/UJTBXfNDnalUTBJpCK5FyrYDT1P6KfW8Q6/08xMQldIZuK0OnjjsWtbU7aDYU05/dz7D0pNnUiUW6r1+LrhvFpX1XnzBEDaLzjPzlvLINWcytl9hosPrlPzhcnbUv0h9YCVu21B6p12E01p08CcqihIrhwM9gDUtMnR04EghxNXAiKZjecD2Fs/LA3Y3/f9uIEcIIfasJjXtZcptcU5UapAUI716ZHDTadOiPiaEzsic+9nt/ZrHN/ydxrCFlQ09aTCcgIk3HOCl4i+4fvAp8Q06zvKdGQxJK2Jt7Q4MzObjDs3KeX2j/9kli9Mnj2DT7qo2q0mpTjsD81WhhPbQsLK/WqQq1U45GCEEIzN6MzKjd6JDiYvHPlrE7poGQkbktTPSAsDgzhfmMPuuy5Ji72Zn4gluZXHJuRgygCRIjX8puxpeYWLBc6TZRxz8AoqixMJbwNJ9jj0NbCSywrSByEBnJrAEQAjhAKYDv2o6fwHgJjLg2rMv6XAghdb7lNpQg6QEEUJQFerJ/JqxeIzWVdHC0mBZdfdoAPq3MRfxy2+epthTga7phMwwF/WbwYzc5H5T+vHhI/l01Wa+3VqCPxjCbrOgC417LzsFTVMfTtrDbRuKVUvHMFqXf9eFk56p5zR/b0iTWds+55Xt82kM+xmZ3ocbh5zKwNSCeIesKAkzd8XG5gFSSxX1jZTVNpKfqfru/RDrq/9CWDayZ6JGEsaQYdZV/p7Dil5NbHBKGxtLK3n202VsK69mXP8iLp4xntx0d6LD6nz2pr0lpaY+Rq32oAghPEB1UyU7hBD/Bu4QQnxHZNB0F5FCDbOarrFOCDEHeFQIcVXTZR4FZksp91u0AWIwSBJC2IF7gPMBJ/AJ8Asp5c4DPOfXwJnAECAALAR+vecPoOmcZ4iU9WtpkZSy44v6x0kPe1qrjcZ7CKDAkRn/gBIgy57KM4ffwJbGMqoCDQxJKyTN2na/VrKx6joPXf1jlm3exdJNO8lOdXHCuMGkuVS1nvYSQjA270GW7b4MU4abiphI8lJOJC9l796Se9a9yYel3zYX/PimZjNXL3mIZ6bcRE9XdoKiV5T4slmjv31LCTarqgb5Q9X4lxDt02J9cA2mDKMJNaecLBasL+bGJ98hGDYwpWTdzgreXLiaZ26cRCMvUB/8Drd1AAMyf0G6fWSiw1Vi7x9ExhsPsreZ7PEteiRBpBz4/cCHTd+/A1x3sAvH4rf+38DpRAZJVcB9wGwhxAQp97MBBY4CHiKyVCaAPwJzhRDDpZTVLc6bC1zc4vtgB8ceV/nOTEZn9GVFzdZ9qjJZubBv92rA19+dR393XqLD+EGEEEwc2JOJA7v23od4SrUPY3qvz6n0fU7QqCbTMQm3bUDz4zXBRuaUfkPQbJ3mGDTCzNr2GbcNPyveIStKQpxzxGjuf+8r/KG9vwu6JhjeK48sd/JPNCUbXTijNirXhA1BZNAppaQh7MOuWbHr1jbnKrEnpeQPr8xt9e8+ZBgUZZSyoeFRdD0MSHzhnVT5FzI+70GynYcnLuBOoJOUAG8mpTxqn+8l8Pumr/09p4bo1fEOqEMHSU3Nnn4GXCal/Ljp2MVAMXAce0dwrUgpT9jnOhcDdcBU4N0WDwWklAfcZNXZ3D36Iv64+mUWV21A13RsmoVfDjmdkapMptJN6ZqdvJTjoz62w1uJVbO0GSQZmKyr3+9itaJ0OedNH8u3W3fx5dptaEIgBGS6nfzjp127ol+sFKWey/b6ZzGlv/mYJmwUpJyOEIIVNVv527pXKfXVIIDpuSO4bdhZuC3O732PDSWVPPDBV6zeXkZhZio/P2EK04f1i8FP03XVevxU1LWt4njq9K/R9ZbFzySm9LOu6m6m9ZwdvwCVLqWjV5ImAFZaN3XaIYRYR6RhU9RBUhSpRMqT1+xzfFpTXfNa4HPgTill+SFHnUBuq5N/jLuUuqCH+pCPAmcmFk2lSihKNIXOLEJm2wp4GoL+7vg38VWURLHoGvdediqbSitZs72M/MxUJg3sFfM9kVKabKt7muL65wmZ9WTYxzE0+3ZSbYNjet9YG5D5C7yhrVT6PkfDikmYDPsEhmT/Hzu8Fdzy7ZP4zb3JK1+Wr6Eq0MCDE6/5Xtdfv6uCS/77P/zBMBKorPdwy9OzufMnx3D65OTeg5tMHDYLRClKUpgTvfWGJ7QFKQ3VkFxpl44eJOUDBlC5z/EyDtKwaR//AZYTqUixxxzgDWAr0Bf4MzCvKY2vzRp50+asqwB6907+akfpthTSbUle9lpREqyHPY3pOcOZX7GuVRNim2bpdimqigIwsKAHAwviV1FzXdXd7Gp8s3nFpdq/gEUlF3BE0Ru4rMn/Xrs/mrAxJu8/eEM78IQ24bL2JcUaWeV5dfucNpMzIWnwXf1OtnnK6Jty8FTx/7w3v3mAtIc/FObed77glInD0DXVtvL7cNqsHDtqAPNWbW6q6BjhC9hJdfnanK+LFFRL0IPoZOl28fS9/uUIIf4shJAH+TqqIwISQtwHTAPOarmHSUr5PynlO1LKVVLKd4GTiBR6+FG060gpH9vTnConJ6cjQlOUhJLSJGjUYMa/n1pSuWvkuZxWNAmHZkUg6J+Sx73jf3bIK0lSSl788luO/t2jjLnlX5z+t2f4ct3WDopaUTq/oFHDrsbXW6WkARgywNbaJxMUVcdyWXuR4zq6eYAEUOypaNWmYg+L0Cn1Vbc5Hs3q7bujfhb1BUJUN3ijPKLsz+/OmcmEAUXYrRbcDhs2u2B1w0iCZuvVIk046JN2sSqHr7Tb911J+jfwwkHO2Q5MIdLkqQdQ0eKxPODLg91ECPEv4DzgaCnllgOdK6UsEULsBAYd7LqK0tntbHiL76rvJWw2IoSFPmkXMiTz+m6ZQmDTLNw09HRuGHIqhjSxah2zIP7UvCU8+tGi5g3BW8truOWZ2dx/xekcNqjzzpArSkfxhLahYcdsUzPJoC64OupzuoLRGX1ZVbuNoNx3NSlMf/f3az2Ql55Krcff9gEhVFXUHyjFYePRq89iZ2UtJTUNvNU4j/m1BVDfm4lpxZho6Jjo1mkMzLw20eEqndj3+nQhpaykbQpdG0KIZUCISFOnWU3HegLDOEjDJiHEf4BziQyQvvse9+oBFAGlBzu3s/P4g9R4vORlpGLVu9+H4u6uzPMpq6v+vHf2VobYVv8CYDI065cJjS2RNKGhiY5JowgZBk/MXdKqYhJE0mEe+OBrNUhSupUyfw0rareSYU1hfObA5n2yLkvPqBXgQMNtHRjfIOPozF5H8PrOrwmHDMym9SC7ZuWYvNHkOTK+1zWuOv4w7pw1p1UDcofVwhmHjcC+n3LuyoH17JFBVoaTX3/xHSFpMK96OPNrBpNu8VEfdlDg6smJvdVnpoPpbNXt4qlDfzOllHVCiCeBfzQVWNhTAnwlkfLdADQ1fHpASvlA0/cPEintfQZQI4TYkzfTKKVsFEK4iZT2e53IoKgv8FegHHizI3+GjhI2vTQEN2LXs3FZ21ciOhQ2uPv1ecxetg5d09A1wfUnT+X8aWM7OFolmW2ofbBNeosp/RTXv8TgzOvRhCpFe6jqvX5CRvQOBdvK960foyhdk5SSBze+y1u7FmBpWqV26Db+M/5q+qTkYrfkkOM6hgrvp5jsHSxpwka/jJ8lKuyYy7S5eXLyDTyy6QMWV20gxWLnrJ5TOafP9O99jZljBlFZ7+H+97/CME1MU3LqpOH86gy1l/JQeML+SB2H/2fvvqPjqK4Hjn/fzDatepcsuci9d+NuigsQMB2SUEILJUBCQsgvQHpCCElIAkkIJYEQWkLvHeOCC7Zx792WZat3bZ+Z9/tDsuy1VsZF2lV5n3N8OJrdHV0Za3fuvPvubbrQD0ob5aHGQco1QU/sAlO6hPa4ffF9wABe4vAw2W8dNSNpEI0leYfc1vTfeUed61c0JkcmMAL4FpBCY6I0H7jiqGFRHcLu2mfYXv13NGxYhEh2DGdc9iM49OO743TI796Yz3urtzZtTmz86/vLO5+TmRTPrJGqyrC78BuRF0slFiGrDqeuBqieqiS3C7uuh20EPqRPZvcY7Kwon5dv5O0DywlaBkEaVzx8ZoAfr3ua/07+MUIIRmY9yNbKP3Cg4XUsGSTe3oeh6T/v9N3tvkpuXBq/GnHVKZ3jm9NHc9mUEZTVNpAa78btVDe4TlW6M5EEWxxVwfBLQQ3B2NR+rbxKCaNWklrV5klSU6e57zb9ae054lhfR3i+Dzj7WM/pKMq8C9lR3Xjn/9A2z5rAOlaX3cWk3KeP+zy+YIi3V25ucdHmDxk88fHyDpkkefxB1u0rxu20M7JXbru3ou1KQpbB4vKN7GooJt+dwRlZo3DpDgCSHIOp9C9v8RpduHBoJ5Z4K5HZdZ0bZ07gn58ux3dUOcwdX5sSw8gUJXreLFoW1uYaGq+fqgL17G4opl9iDzThYGjGTxmSfi+WDKFraj/NibDrOnlpybEOo8vQhMZdgy7mN5v+S9AKIQGb0HDqDm7qd06sw1M6OVUI28Z21z6DKcPbUEoMagLr8BuluGxf3SoUoM4baLUjS3ldy0Fqsfb68o387vX52HQNS0oSXU4ev+Vi+udErzVtRySlpCqwBk9wH0nOgaQ4W87DqAl6uP3Lv1IVrMdnBonTHTyx8z3+Mf575MalMSjt+ywvvh7ziJI7XbgYmHpnt2zc0F5unDmBOIedf81bQVWDlz6Zqdx9welqP5LSbXjNSPuNQBMC31HJkxA6unr/UTqAGVkjeMSZzAv75nPAW8HIlAKu6nMm2S5VBaCcGpUktbGAGbm/hcBG0Kw+7iQpI8mNy24jcNRGcgGM6HV83XSiZUtRGb97fX7jpvem7tTeQIibHnuNT39xU7ed/xA0a1lWfCOeUBGH1rOTnUOZmPMPbNrhKe1P7HyXUn81hmxce/SZQQJmiD9ueYU/j72FFOcIJuY8zdbqv1AX3IpLz2FA6nfIjZ8Tix+ryxJCcNWMMVw1Y0ysQ1GUmJiZPZpdDcVhM8gABIJBiSe3t1ZRomFoci9+O/LaWIfR+UhUud0xdM+r13aUGTcNESH3FEC8o+9xn0fXNH54wQxcR3S9EQJcDjvf/drUtgi1zbyybH3EvRy+YIiVO4tiEFHHsKHifuqDezClF1P6MKWPmsAGtlX9Pex5i8o2NCdIh1hI1tbsah5gmOIayaTcfzOn9zJm5L+hEiSl09pSVMazC1fzzpeb8QaObiWtxNIFeZPoHZ+FS2ss9dXRcGp27hl6RZu12lcUReks1LteG3PY5+IxX8YhLGyahZRgSJ3sxFvQheOEznXRacPISIzniU++4GBVHSN753L7uZM7XAlbVYMPS0a6FSGo80WYC9ENSGlS7JmHJHwl0JJB9je8xbCMHzUfa62sUu3oUroSy5Lc88IHzN+4C9OysNt0Hnh9Pv+89VKG9zq1QcDKsVkyRIlnPlX+NcTZ8shPPA+n3rIUyanbeWz8HcwvXceyyq1kOJOY22MiveKzYhC1oijtTaCuNY5FJUlt7NGdi9heO4VxyXspcFdQZ7hYWVtASrWXcSfxOTNtSB+mDenT5nG2pZkj+rF02z58wfASjZBpMq5v9yzRkFjICBPaofGC5UhnZY/mg4MrCB3RAFJDMD5toLp7q3QZ763eyoJNu5tnUYXMxt+P7z39Np/+/CbV6KWdGJaHxQe+hdc4gCm9aMLF9upHmdzjKVKcQ1s8367ZmJM7jjm542IQraIoSsehyu3a2Iaa3XgsJ4uqB/GfA1N5o3QcRf40NtXuQ0Zcben8zhk9iP456cQ5Dl/Quxw2bpk9kfREdwwjix1N2El1jqTlPRqNLPe05q8Wlq1jc91eJBa6EGhC4NadZDiTuXvw5VGNWVHa0+vLN7a4kQLgCQTZcqAsBhF1Dzuqn8Jj7MOUXqBxxpohPawu/XGX/UxSoscbCPLfxWu5+9n3+Ov7Syip7nBTWRTlpKnb1G3MqTkwzJYlZg7N1mpZVWdnt+k8c8flvPPlFj5cu52kOCdfnzKK0wb0jHVoMTUq85csPngNlgxhST+6cGHT4hmW3lhq9799n/GfPZ80t9wVCOyazi39z+O8HhObp9wrSldgWpFXVgWNpXhK+zjo+QBLttz75TNL8JulxNlUqaNycirrvXzj4Rep8frwBw3sus4Ln6/hiZsvYXRBj6jE0BDax86aZ6kLbifZOYT+ydcSb8+LyvfuMtTbb6tUktTGzusxkTcPLCFoHd6L4tBsnNtjQgyjan8Om41LJ43g0kkjYh1Kh5Ho6MvMnu+xv/5N6oI7SHEOJz9xLnYtAb8Z5Jk9H4d1kZJITMtie30RF2pqNo/StVw4YShbisqay+0Oses6Q/LVnpf2EqmREABStnhMSokpLXWDRjkuj328jIo6D0bTDZCQaRIyTX7yv494957r2v3GcLV/A0uKb8aSQSQmNYEtFNW/y7QeT5PsHNyu31vpHlSS1Ma+3f9cinzlfFm1A7vQMaTJqJS+fKf/3FiHpsSAQ0+hX8p1LY4XeSvQRctqVxOLDbW7oxCZokTXBROG8tHa7azdW4wvGMJh09E1wUPXnodNV5Xf7aVX0iVsq34MSx5Z4aCR6OiPy9bYBMi0TJ7e8yFvFC3BbwbJi8vgzkEXMz5tUGyCVjqFD9dua06QjlRSU09FvYfMpIR2/f7rK34XNpdSYmBIgw2Vf2Raj6fa9Xt3JUKtJLVKJUltzKHZeGDUDRR5Kyj0lNIzPoue7sxYh6V0MOnOREJWy7bpADmutChHoyjtz67rPH7zJXyxo5AVO/aTlujmvLGDu+2+xWjpm3w1Fb4VVPlXI6WJEHZsWhzjsv/Y/Jy/7niTj4pXNq9sF/nK+en6f/OXsbcxJEkNUz5ePqOMjZWPUOJdjC6cFCRdxqDU69GEPdahtblXv9hAnTfy8GEpJU57+15eSmlRE9wS8bEq/7p2/d5K96GSpHaS784g392xWnUrHUeqI5FJGYNZXrk1rDTTqdm5svdZMYxMUdqPpgmmDOrNlEG9Yx1Kt6EJOxNz/kFNYCM1gQ24bDlku6c3X7g3GD4+LF4R9j4EELRCPLfnEx4YdWMswu50gmY9nxVdScCsAUxCwPaap6kJbGVy7p9jHV6bChkmf3pnUcStLAIY1zePpDhXO0ch0IUrbCXpEJumbrwobUMlSYoSI/cOvZKHtrzM5+Ub0YTAqdm5Y+BFjE7tH+vQuqSAWYOGjl1PjHUo3Uqt18+Db87n4/U7sCzJtMF9+MklZ5GTov4/RIsQglTXCFJdLfeMlvtrsAmd4FEz3SSwz1MapQg7v311b2BYHuBwhYApA5T6llIf3EOioyB2wbWx/ZW1rTZb0TWN3115brvHIISgd+Il7K1/FUseXtEKWRob6nMpqN3HkGR1M+a4qHK7VqkkSVFiJE538rPh1+Ax/NSHvGS6UiLuU1JOTW1gOyvLfkZ9sHGvV7prFOOz78etunq1O8uSXPvoy+yrqMZomou0aMseNj7yIu/dcwNuZ9crQ+psclxpmLLlvhKBoH+i6hJ2vCoD6zFly862GjZqg9u7VJKUlhAXcS8SwIheOWQkxUcljqHpd1IfOkixdyGmFOhIdnozWViVy/Lax/nvlJ+TYIuLSixK16SuyJQ2UVJTz7aD5YTMyPtslNbF21zkxKWpBKkdBM1aFh28kbrgdiQGEoNK/1oWHrgBSxpffQLllHyxo5DimrrmBAnAkhJPIMRH67bFMDLlkDibk0vyp+HUwhNWp2bjWwWzYxRVx3bQV8m7B5fxWekafGbjKkaivQCNlkm/xMJt61rJZkp8HNMHF+CwhXdBdNlt3Dgzep18deGgWl7BS8XTeK9sJP85OJlPKodhoWFJycKytVGLpVOTMfzTwamVpA4oZNZT4V+PXYsn3TUS0YEvnivrvdz17Dts2l+KTdfQhMZ9F5/J+eOGxDo0RaGw/t0WyZDEJGTWUupdRm789BhF1j3sKasKS5AO8QVDbC+uiEFESiQ39TuPNEcSL+2fT23Qy8DEPG4bcCH9EqIz66Yz+efOd3n9wCIEAk1oiG3wwMib6J98ObtqX8SSh8c6COwk2HuT6hwWw4jbxwNXns29L3zI4m17sesaErjza1M5fWjfqMZRE2ygKmRHkhp2PGCFqA42RDUWpetRSVIHs7P2ZdZXPozADljYtQSm9/g7yY5+sQ4totufepNtB8sxLIuA0biK9KtXP6VXRgoje+fGODqlu2sI7Y9YAmNJA69xMAYRdS8F2WnYdI2gEb7CHOewMzBXNbbpKIQQXNZrBpf1mhHrUDq01VXbefPA4hZNLn664SlenforpvV4ktXlv6Q+uBeAnPhpjMv8RZccJO92OnjkhguorPdS1eClV0ZKu3e0i2RESl+cmr15KPshTs3OyJToJmxK16OSpA6kyr+R9ZWPYMoA0LiEb5heFh28jSTfo5TX+RjVO5cBOR3j4mJ3aSW7Sitb1CYHDIPnFq3mj9ecF6PIlO7CtEwWV2xgacUGEm1uzu0xiX4Jh0tb0lwj2Vf/dosOSELopDrVamd7m9S/F7kpSRRWVBNqWlHShCDe6eDsUWoGj9K5fFC8vMXFOIAlLdbV7GJ82jBm9XyFkFmPJuzoWnt3eIu99ER3TNv4j0juy4iUvqyv2dXcwt6p2RmR0pcRySpJ+kpSzUk6FpUkdSC76l5rSpDC1flreebTZ9hXnA3AjMEF/PHqr6FrsS3Dq6j3Ng6BDIUfl7Jxj9IhphWgyDOP+uBekp39yYs/s0vOjVCiy7BMfrzuMbbX78dvBdEQfFiynNv7X8y5PSYDkJcwiy3Vj+MNlSCb/qFqwkmqcxipzpadvpS2pWmC/9x+Bb9/awEfr9uOaUmmD+nDfRefpZo2KJ3O0StIRwod8ZjqoBk9QgjuH3EjHxQv58PiFYDknNyJnJs7sUuu4CnRpZKkDiRgVhNpJ5slQQov/lDjm/CirXt4edl6vjl1dJQjDDc4L5OQ0bJRg8OmM2VQHwC8Rinziq4lZDVgSh824Wa9/jdm5j2Dy5Ye5YiVrmRB2ZrmBAnAQhKwQvx95xvMyBpDvM2FLhycmfccm6se54DnEzRho3fihQxKuYHvrh8AACAASURBVEF9gEZJstvFA988hwe+eU6sQ1GUU3JW9li+rNrWYjXJlBajU9TohlixaTpz86YwN29KrENRupiO2xGgGwmYQV7Y+zEflHgwLL3F47pmUViS1fy1P2Tw8hfroxliRElxLr498zTiHIdzbbuukex2ceW0xgRuVflv8ZuVzeVOhvTiM0pZV9m1husp0bewfG3E0heb0NlYu6v5a4eezOjMH3Nen085t/eHDE37Drrm/Mrzr9pdxC9e+YSfvPQRS7btQ0pVk6Ao3dm0zBGMSR2AS3MAoAsdp2bnh4OuIM721e8pitIhqe52rVIrSTEmpeTH6x5nR0MRhuWmhzOOVLsXu2YBgmBIZ/7KkfgD4W/AgQgrOLFw6+xJDMjN4NmFq6n2+Dh9aAHXnzGeZLcLKS1KvcuA8D1LEpMDnvmxCVjpMty6E0Gk91mJ6ziSoGP583uf898la/GHDCTw8fodnDNqIL++fLZagYqSuuBetlY/S21wJ6nOYQxOvZoEe9dqpax0LrrQ+PWI61ldvYNlFZtIsMVxdu4EesR1jH3CiqK0LZUkxdiamh3sbjhI0AoBGm+XjWZAfCn93VUMSBzIW59ms3ZnQthrHDadr43uOJueZw7vz8zhrZUatHZB2biIaUkLj+HHbXOpOUHKCflaj8ksqdjQvFn3EIdmZ3jyyQ9u3FtezQuL14TdiPAFQ3y4dhuXTRzBKNW1sd2V+9ax8OAdWDKExKQ6sJ199e8zM/9fpDgHxDo8pRvThMb4tEGMT+s4n8GKcipU44bWqavSGNtWV0jgiJIhC41tnlzeKx9GlTWXu+bcQJzD3jy0ze2w0zM9metPHx+rkI+bEBq58dMRhJcQCmzkJ8zknQOLuWLpT/nGsp9x+ZL7eKnwU1XSpBy3USn9+WavWdiFjTjdiVt3kmRz88DIm9G1lmWrx+vzrXsiVgH4QwYLNu8++YCV47aq/EFM6UfSmKhKTAzpZU2FKtNVOr/1hcXc9M/XOPP+J7nhiVdZvedArENSFCUCtZIUYxnOZJyaHd9ReytcmoNMZwpjc/N47/+u480vN3Ggqo7T+vdk9oj+OGyd43/duIx7mRfYTtCswZQBdOEkzpZFtXkG/9z9VvMqgGGavLDvY3ShcVnPs2IctdJZXNVnDufmTmJdzU7ibXGMTR2I7RQSJGic4aNrLVdAbbqmOrJFgSUNaoO7Ij5W4Y/9XkxFORUrdxXxnaffaG7EVF7n4eZ/FfPwt+YyranhUWd30LOEdRV/pT60jzhbJsNSb6Jv8gWxDktRTljnuNLuwqZnjuLxnW/it4Jhd69tms7pmY3ND7KSE7h55sTYBHiKXLYMzu31OsXexdQH95Hs6EeOewrXrXigRZlUwAry38JPuTT/TLXvQzluac4kzswe22bnmzW8Pw++taDFcU3TOlSZa1cl0NGFI+I4BLsW3+KYx/CxsXY3bt3F0OQCVbardGi/f2dBc4J0iD9k8ODbC3j3R9fFJqg2VOL9gqUlP27+/fUaJayu+COm9DMg5Yo2+z5SSmq9fhx2G26Hunl1SlQBT6tUkhRjLt3Bn8d8l99ufpYibzkIyHNlcN/Qb3WZbjmasJMXfyYccX1TGaiN+FyP4cOUFjZxaqsBinKyUuLj+NM153H38++hNV1wG5bFry6bRV5acoyj6/qEEBQkXcSeujfDEiVduBiQ/PWw575zYDFP7n4Lm9CRSOJ0J78b+R36xKt9Y0rHtKOkIuLxveXVWJZEi7CK3RFIaVET3IVAJ9lR0OqNzPWVj7a4wWFKPxurnqR/8mWINriJsWpPET9/5RMOVtcBcPqQvvz68tkkxXX94b1KdKkkqQPoHZ/DkxP+j4pALSDJcKbEOqR219OdxW7PwRbHM5zJp1wupSin6vQhfVn481tYur0Q07KYMrA3iXFd46ZFZzA643v4jQoOej9vWlUK0ithDkNSr2t+zra6Qv65+y2CVohg06BgnxngnnX/4IXJv1IrSkqHlBbvprze0+J4UpyzwyZIZb61LC65D8PyIpG49FSm5/6eNGfLlfX6YGHEc4QsDyHLi0NPiPj48SqsrOHWf72B74jVuIVbdvOdp97ghTu+eUrn7q5U44bWqU+RDiTDmdwtEiSAb/e9EKcWvkTu1Ox8u6+qW1Y6BrfTwawR/Tl71ECVIEWZLhxMzX2Q83q/zrTcPzG399uclv0ztCNWmN8rXkLQMlq81m8F2VATeU9TVxE0TD7ftpdPN+6kzuePdTjKCbjxzAnE2cPvT8fZbR22GZPfrGHBwe/jNysxpA9T+vEYxcw7cDuG5Wvx/PhW2/Tb+deeD3i9aD61oYaTjufFxWsJmeFjRUKmxbbiCrYXR16lU5STpVaSlJgYlzaIXw+/iaf3vEuht5QcVzrX9fkakzKGxzo0RVGOg2H52O9ZgM+oINM1kgzXyDbfS+i2ZeO2ZUd8rC7kQUYophcIvGbLi7euYs3eg9z2zJuYVuPPbpgm91xwBldMHBnjyJTjcdXU0dR6/TyzaBXQuLfmyqmjufGMCTGOLLJ99R8hj5p1CCClyX7PQgoSzwk7PjL9OywtuQ9THk7eTamzvr4H6xoW49TsvLDvA34/8rv0T+x5wvHsLq/CsFrGY9M0DlbXMjBXzaxS2o5KkpSYGZ06kL+m3hXrMBRFOUHVgR3MO/AdLGlgyiC6cJDhGskZPf6MJqLzsTI1YyRrqrfjP6ozqCENhif3i0oM0RYIGdz67zdo8If/zL9/ZyFjevdgQI66QOzohBDcPmcy3z5zAuX1HjIS43HZO+6lmM+oiNhExZJB/EZVi+M94qczMeuXrK18BK9RjIWb1XW5bPJkATQ3bPrD1ud4csJ9JxzP2D49+HJ3EcEj5tgBhEyTgbmZJ3y+bk+iGjccgyq3U5QT5AkEeXHJWu753wf8c/4KKhu8sQ5JUaJGSsniknsJWvUY0tc0w8hHuX8dO2pfi1ocZ2SNpU98Lk7NATSOrXZqdq7tcx5J9pZd8LqCz7ftJdIouZBh8vrKTdEPSDlpTruN/LTkDpcgBcwg/yv8mJtW/pabVz7ANo+GTcS1eJ4QNrLiRkc8R8/Emczt8zaX9/uCDyrPYJMnm6MHy5f4K1tt4HQsX588ininA+2IVWuX3cackQPpkZp0wudTlGPpWL+dSodlSYM99R+ys/ZdhNDonzSXPolzwvYIdAdldQ1c8dcXafAH8IUMnDadf81fyXPf+bpa5le6hQbjAF6jrMVxU/rZVfc2g1K+HuFVbc+u2Xho9HeZX7aKReVrSbTFc36PKQxL7huV7x8LnkAw4sBtU0oa/C3v9ivKibCkxT3rH2W35wDBphWf/+6v5GuZiSTbZHMJnS5c5Lonku4aeszzaULH1urKskQ/ieuH1Pg4Xr7zKh75cDGLt+7F7XRw5ZTRXDN9zAmfS2miVpJapZIk5StJKVlw8MeU+FY1v0lW+jez3/M5p+c+EOPooutP739OVYMXs+lCJWCYBAyTn736MS9998oYR6cobacu5MGu2YjTw5tWSGlx9F3h8Meix67ZmJMzkTk5nXOO3ImaPKAXZoT9GG6HnVnD+8cgIqUrWV29jb2eg80JEkDAMviwfCB39O1L0FyBQKdf0oX0TTr/uM55ds4kXiz8KOycGoJ+CfmkOE6u011uSiIPfuPck3qtopwIlSS1ky92FvLXj5dSWFFDv+w0vnf2VMb1aa3ry4lrvJso22TmwFcp862h1Lc6bCOmIf0c8Cylwr+JDNewdo+ho1iwZXdzgnSkLQfL8AVDxKmhdkont61uH3/Z/jzFvsZOUaNTBnHXoKtIdiQCkGjviUtPxWMUh71OF04Kks6LerzdSVZSArfOnMQTny0nYBhICXEOO+MK8pg+qCDW4Smd3Ja6PS32+AEELYsKcxBX9Lz9hM95Sf6ZrK/Zyaa63UgsdKETr7u4Z8i1bRGyorQrlSS1g/mbd3H3f99vnqr95Z4D3PzU6zx67YVM6t/rlM5tSZMNVf9mS83/CFkNJDsKmJD5Q3Ld7dc+tMS3GkO2bDNryRAl3lXdKkly6JHLA4QQ6B10xoWiHK+KQDX3rf87futw6dbamm3cu/7vPDruHoQQCCGYlvMA8w7cgcTElH5sIo4U5wAGJV8ew+i7h5vPOo0JffN5feVGPIEg54waxMxh/TrsjB2l80h3puDUHASOSpTsmp10x8kN0rZrNn478jtsq9/HjvpCMpypTEgbclKldkrbE6g5SceikqR28Pt3FzYnSIf4QwZ/fG8Rr9159Smde1X5I+yoe7t5Vac2uIf5B3/InPzHyPiK+uATUeqvZGPtLpLs8cRrSU0DHcNr3nXhwKl3j7lOh1wyYTjPLV5N4IjOOjZd4/TBBThsHevXqcbrp9rjJS81GYdNfSApX+3D4qWYMrxrlCFNSv2VbK3fy5CkxtWKdNdQLurzJvsaPsVrlJHpGk2u+7SorGwrMKZPD8b06RHrMJQu5vTMMTy1+60Wx+2aztSMUad07kGJvRmU2PuUzqEo0daxruq6ANOy2F8VuWPLztLKUzp30PSwo+5NTBl+l8eUQdZXPsVZeX86pfNDYxnfk7te54OSJehoCCFI0CWzI/YkEPROOOuUv2dnctusSWwqKmXNvoPN3XXy0pL41aWzYxzZYd5giJ++8hHzt+zGpjX+P7zr3Gl8Y9KpfcgprfOESqkK7CTBnkuqs/M2DtjvLSUkWw5oFUJQ6q9qTpIAHHoSA5IviWZ4iqK0o3hbHL8f9V1+t/kZKoI1AOS40rh3yHW4dEeMo1PajVpJapVKktqYJgTJcS5qI0xBT090n9K5vUYZAhtwdM2wpCa4+5TOfciyyvV8XLKMkGVwaJulz4Qva0cyOWUHlgwiAZvm4vTcB3HoJ7fxsrNy2m3866ZL2XKgjG0l5fRMS2Zsn7w2H6J5Kg4lSEHDJEjjqsAf31tEj5QkZgxW+xbakiVNlpU+yJ6GT9CEHUuapDkHMKvHnzrl78aw5H6srNrUPMvkEFNa9E1ouz2ViqJ0TP0T8vnXhJ9Q6q9CCEG2Ky3WISlKzKjaiDYmhODGM8YTd9Tsgzi7jVvOPO2Uzh1vz0ZiRnhEUBl08utNj/NZ6QoMK9Jzjs/7BxdH3LhZ6HMyNusJZub/ldn5f+PSgnfIiuu+E96H5GVx0bhhjCvI71AJUo3X35wgHckfMvjXgpUxiqrr2lLzCnsa5mHKICHLgyn9VPq3sqS0c3Z9nJV9GvE2N/oRHw1Ozc641CH0cufEMDKlM6vx+nllxQae+XzVKVdUKO1PCEFOXLpKkJRuT60ktYMbZozHFzT4z+ersKRE1zRuOfM0rph4akmFXXMzKOVyttW8elSnOcGiqnhqjE2sr9nBJ6XLuH/EHSe1MdJnRp61IYQgaJlkuoafdPxK+6v2eLFpWvMK0pFKautjEFHXtrXmlbDfRQCLEEWeJYQsH3at5RDGjsxti+ORsT/iuT3v8kXVBlyag3Nzp3FJfvcqq1XazpLt+/je828jAMOy+NsnS7l0/HDunXtGh7rB1FFIKSnxrWJP/adowk6/pHPJbMP9xopyNBFpQrUCqCSpXQghuGP2ZG4+8zRqvD5S4+Owt9IV7USNTb8Nl57K5uoXCJi1VIVcrK7rSY3RWMoXsILsrN/P8soNTMmIPA37WM7IGscez4EW5TYCwYDEU+vMp7S/vNTkiBceuhBM6Jsfg4i6tpDlbfUxUwaw07mSJIA0RxJ3DrqSO2MdiNLp+UMGP3jxnbBGRiHT4vVVmzhjSF+mDFAb+Y8kpWRJ6W/Z1zAfQ/oAwc669xiReg2j0q+PdXiK0u2ocrt25LDpZCUltFmCBCCExrDUq7i87/vkJz/M59VjqAwlhj3HbwX4onL9SZ3/7Nwp9Hbn4tIaN2nahI5Ts3PXoKuxayqn7ugcNp27zp2G64hyT00I4px2bj2rewzcjKYe7olEehuNt2fj1A63zC0PVHP/pie5ePH3uXTJXTy8/XkajNYTLEXpClbs2k+kwcO+YIg3V2+OfkAdXJl/PfsaPmtKkAAkpvSzofo/NISKj/lapVFJbT2/ffszLnzkWW579k1W7T0Q65A6NhnjPx2cuurtxNx6HJGqFTQ0Em3xJ3VOh2bnj6N/wNKKdays2kSaI4k5uZPJi8s6xWiVaPnGpFH0SEniXwtWUlJbz4S++dx61kR6pnevdu3RMDbjVg56l2NIH6YMItDRhZ0p2fc2r+j5zAA/XPMQNaF6JBKkycKyL9ndUMQjY36sSo6ULsuUVuuPWa0/1l3tb1iEISOVvAuKPMsYnNJ9u0lKKdlS8xobq/9HwKoj2zWSCZm3hXUTPVBdx6V/fx5vIIRhWeworeSLXfv59cWzOX/04BhGr3RWKknqxMakDsYWYd+RTdOZkzP5pM9r03RmZI1lRtbYUwlPiaEZgwtUJ7soSLDncFHvF9la+wZlvrUkOXozNOUKkhw9m5+zqHwVPtPfmCA1MaRJsa+C9bXbGZUyKBahK0q7O61vT6wIyVCcw87c0UOiGkuN14dd14l3dtxW1rrmQqAjCW/DL9CwaS6klGyr30OD4WVQYgGJ9pO7GdoZrSx/lC21bzbvAS3yLqe0cD0X9v43SY7Gzpv/mLcMjz+IecQeG3/I4IF353POiIHYdFU8pZwYlSR1YnbNxq+H384vNz1G0AohEJjS5JZ+V9A7Xg0aVJRocNlSGZ1+Q6uP72koitgx0sKkyFuqkiSlmSUNKvzb0ISNdOeATj+cN97p4IHLz+GeVz5EWpKQaeKy25g5tB+nR+kmzrr9xfzk1Y/YX1WLBKb2780Dl51NanzH2y/YN/FsNlW/iHnUrDKJxKEP4eYvf0Gd4UFDELIMvtnra1zac06Moo2egFnHltrXj5oRKTFkgA3VLzA1+/8AWLZrf1iC1Pz6kMHBmjp6qWqKiEQnKHuLFZUkdXL9E3vxn4n3s7l2NwEryNCkfrhtrliHpShKk97xPXBqDgJHJUoaOvlx2TGKSuloDnhWsKD4l1gYSClx6AnM6vEgGa7OnUTPGTGAkb1y+GDdNur9QWYM6sOoXrlRKTMtrqnnxqdewxs83IhoyY593PDUq7z+3as7XKlrsqMXp2V+nxXlf0Gg07ify+L0nPu5f8t/KA9Uh61Iv7T/Q/on9u7yN1pqg/vRhP2oJAkkJmW+Tc1fZyS4I3ZxNS1JsltdFyknTiVJXYAudEakDIh1GIrSgpSS1fsOsrW4nPy0ZKYN6I2ude674yfqjKwJvLjvfUKWgUVj6ZFN6GS70tTvrQKAJ1TOvIP3YRw52sHw8WHRnXyj71vYNGcMozt1OcmJXD9jfNS/78sr1mOY4eV+hmVRWFXL+v0ljOqVG/WYvsrA5AvplXAGBz3L0YSNvPhJ7PdWUh2sD0uQoLGb7XsHF3bpJClgBlldXUTQ9EfYgy1IdhzuunvDjPH85NWP8B3RTdGh60wf1IfkOJUkKSdOJUmKcpTqwB62132AYfnonTCDPPf4DnfHsTPwBUN8+9+vs7WkHNOysGsaKfFxPH/T18lOToh1eFETpzv50+i7eWzny6yu3oIuNKZmjOGWfpehdfJyKqVt7Kz7ECtCkwMpLQo9i+mbODMGUXV+u8urCJotZ8ZpQnCguq5DJkkALj2ZvkmHy+g8RhFaK59B9YYnWmFFnc8M8H/rHqLUX8mIhERynLXoR9SG2YSTUWnXNH99zoiB7Kuo4YkFyxvnBZomE/v25MHLz4lF+J2HKrdrlUqSFOUIm2veYHn5o1gyhMRiR92H9IyfzFm5v+z0+wOi7bH5y9l0sJSg0XiREsTEHzK477WPeOqGS2McXXRludL4xfBbkU318irpVo7kM6uxiLRvzcBv1sQgoq5hXJ88Pt++N2xOE4BhmgzpkRmjqE7cgMTeETsFOjQ7U9JPfB5iZ/H+wYWU+CoIyhCr63oxPOEAveKqEECSPYcp2XeT7hoY9ppbzjyNq6eMZk95FZmJCd3qhpzS9lSSFEU1Xj//WbyKz7buIiXOxTVTxzJraP9Yh6U08Zs1LC//e1jdsyH97Pcso8iznJ4JJ98xsDt6c/Wm5gTpEFNKVu4pwhsM4XbYYxRZ7KjkqHsq81fy5O6XWVuzGV3oTM8Yzw0Fl+K2NTYPyHOPZ3vtO0fMxzlEkBs3JvoBdxEXjxvGU4u+xDBNDKvxBoXLbmP6wD4UZKbFOLrj59KdfLvvpfxr92sErRASiVNzkO1KZ3bOlFiH124WV6whKBv3k1lorG/oyYaGfBJsNn4+9Afkx/eJ+Lp4p4Ph+TlRjLRzU40bWqeSpHa03/MFK8sfpzZURLyew3tL+rJuR0bz8v/GA6XcMH08t89UF98dwQHPl2jYMI+6o2tIP7sb5qsk6QQZx5iDEqktsKJ0RV7Dx4/W/YEGw4OFxJQWC8tXstdTxB9HNc7JyoufSIZrMBX+Lc37kmzCRZ+EM8LmwCgnJtHl5NXbr+SRT5Yyf8tu4hw2vj5xJNdPi/7+qFM1J2cqfeLzeL94ETXBeiamjeSs7Ik49Y7b0vxUxUdoQiURBC2NONWgSokClSS1k8KGJcwr/gVm02C4OmMfk8YVUeMbx459+QD4Qgb/XLSSqyaPJsXd8dqRdibl9R5MyyI7KeG47tY3GF421GzDrtkZmTIIh2ZHF6192AhsQr0hn6g5wwbw+qpNYcmSAIbkZpLg6twb0RXleC0oW07ACmKFzckyOOArY0v9LoYm9UcTOmfn/5kdte+zs+4DNGFncPKFFCSeFcPIu4bMpATuv7RrtMkemNiHgYl9Yh1G1Hwtdwbb6/eFdQYVCLKc6aozqBIVKklqJyvKH2tOkA6x20xOH7+hOUmCxs4rGw+UMm1AnyhH2DUUVtZw10vvsbO0EgT0SEnioSvOZWiP1t9APy5ZzFN7XkVvGsQrgPuG3MqgxNMiPt8mHAxI6pobP0OWjxLfBnThICduOJpou7eE78+ZytKdhVR5vHiDIeLsNuw2nd9ednabfQ9F6eh2e4patH+Hxtk3Rd4ShiY1llzrws7glAsZnHJhtENUlA5pcvpotuXs4b3iRdi0xs/rBJubnwy9RZUutyVVbtcqlSS1k9pQUcTjifE+Gv9FNv6Cm5ZFRkL3mZrdloKGydX/fIkqjw+raUP83opqrn3qVT754Y2kRJiLsN9bzFN7XiVohYDDszN+u+Ux/j3hQWbnPcDHB+5FIJBYSCxGp11HVtzQaP1YUbOj9lMWlf4RIXSQEl2zc27e78mKG9Im509xx/HOnd/io0072FhUQu/0VOaOHkySasWqdCMF8fkR52QJBPlxat+EorRGCMH1fS/hgryz2FK3mxRHIkOT+qmuoErUqCSpncTbMmkwSloc9wUcHEqQdCHomZbCoJyMKEfXNSzcthtfMNScIB1iWhbvrtvC1ZNbbnieV/oFhtWyJSzAquqNTMkYx1X93mS/5wsMy0ee+zTi7Z2nC9LxqgnuZ1HpHzBkoPkuUsiE94ru5pp+r7fZXBan3cYFo4dwwei2SbwUJRZ2lVXyyaadCAFnDx9In4zU437tGVkTeXn/B4SsUHPJnU3o9IjLYkhSv/YKWVG6jHRnCtMyx8Y6jK5JqsYNx6KSpHYyNv0GlpT9KazkzpCC3cEMsvrWU1uYysCcDP5+1Vy1bHySSmrrCZktGwD4QwZF1XURX+Mz/c0DPY8kpcRvNv6/smtu+nbxvQDbat/HlEaL4xKL/Z7lFCTOiEFUitLxPL5gOU8uWEHIMgHB4wtWcOesKVw3bdxxvT7eFscfRv2IJ3e9xNqarY3d7TLHcUPBZeq9/wj1oWI2VL9GVWA3Wa4hDEu9hHhbeqzDUhSlG1NJUjsZmHwuKyrX4At+gk0YGFJnpzeT/cE0coZW8tSF36FfRtdboYimkT1z0TUBRy0MuR12xrQyJHBi+kgWla/Eb4XvFzOlxaiU7rPa4TfrkEf/xdE4vDJgNYQdC1kh6kINJNuTmuvCFaU72FVWyRMLVhAwDt1QkJiWxSOfLmHW0P7kpyUf13myXRn8bNjtSClVYhRBuX8rbxd+H0uGsDAo9q5nU82bXNz7cVIcPWMdnqIo3ZRKktrRhgadnQ2D0ZGYCA6V2dk0nZCjDlBJ0qkYmZ/DmF49WF14sHlYoMOmk5+azFlDIpexjE4ZwsiUQayv2YrfCiIQODQ7l+TNId2ZEs3wY6p3whR21s1rMZdFYpHnbixrsKTFi4Vv837xAiQSXehcnn8uF/SYpS70lG7h0827Irayl8C8Lbu4duqJlQCp35vIFpX8Key9yCJE0DJYVvZ3zs3/fQwjU6JFSskrX27kqcUrqfb6GdMrl7vnTGdA9sltRzCsAJY0cOhqz/dXUuV2rVJJUjtKticCoilBOsyUJgk29Yt7qoQQ/OOai3h26WpeW7URw7Q4b+Qgbjr9NOx65BUPTWj8ePBNrKzawJKK1Tg1BzOzJzM4qXvNIukVP4nsuKGU+jaFzWUZnnoxifbGzeSv7P+A94sXHLHhPMRL+98jwRbPzOyuO8BQUQ7RBERKa0TTY8qpM2WIisCOCI9IDnjXRD0eJTYe/nQJzy1bg6/phufn2/eyau8BXrvtanqnH/8NTJ9Ry/ySP1DoWQFIUp29OTPn/8hyDWqnyJWuTCVJ7ej83LPYWLs9rKuRhkaeK5t8t+pq1BYcNp1vz5jAt2dMOO7XaEJjYvooJqaPasfIOjZN6Hwt/w/sqp/PzrpPsQknQ1Lmkh/f+PdoSYt3iue16MgVsIK8VvSBSpKUbmH2sAH8Y/7yiKtJs4YOiEFEXY+Gji5smDLU4jG7dnh+YMAM8kXlGg76Sugdn8+EtFHYNXUJ0xopJRsOlFLZ4GFkfi7pCe5Yh9QqTyDIf5auJmAcLgGXNM6SfGLhch645PjGRkgpebvoLqoDhVg0JluVgd287oWq0QAAIABJREFUtf8urix4Vu1xi0CgGjcci3qHaUcjUwbzzV5zebHwbWxCx5QWOa5M7h1yW6xDUxQ0YWNA0mwGJM1u8VjIMgiYLWe7AFSHIjfFUJSupk9GKj+YPZW/fLI47Pg9XzuD3JTEGEXVtQihMSBpDjvqPsGUh99zdOFkaPIFAJQHqvjJ+j/gM/34rQAuzckL9jd4YOSP0fGzq2EhljToEz+FVGevWP0oHcbBmjpufOY1yuo9aEIQNE2unzqO78+aGuvQItpbWY1d18OSJABLStYVtewS3JpS/2bqgsXNCdLh84TYXPMOEzKua4twlW5EJUntbG6PmczKnsruhkIS7Qn0cveIdUiK8pUcmp00RzIVweoWj/V258UgIkWJjW9NHcvMof35bMtOEILZQ/uTk6wSpLY0Jeu71IdKKPFtbF5V6h0/mbEZ1wLwxK4XqA3VN3cm9VsBQgGD53b/HrtYCTQ2nVlR8W9Gp17BxMwbY/azdAS3v/A2hVW1YeMxnl26mhF5OcxsZb9uLOUmJxI0WzYSEkCf9ONvt18XOkikAllThqgKFp5ChEp3pZKkKIjTXQxLHhjrMBTluAkhuK7PZfx15zNNg3cbOTQ73+pzSQwj697W7S/mmaWrKa6tY0q/3lwzaQyp8XFf/ULllOSlJnHNFDWnpb3YtTjO7/lnaoKF1AaLSHUUkORo7FBqSosNNVtbjG7QRAApl2IeuetcwrrqV+ibOJ1MV/f8zN1TUc3eyuoW8wN9IYPnl63pkElSWrybWUP6MW/L7iM6STbO2bv5BErpM5z9sSJ0bdWFg1zXsDaJtUuSqt6uNW2eJAkhnMBDwDeBOGAecJuUsugYr/kl8IujDpdKKXOOeI5oes7NQCqwHLhdSrmpTX8ARVEAmJwxBrfNxf8K36XEX07v+Dyu7HUBAxMLKPFtYWnZk1QEdhJvy+C0jG8xIOnMWIfcpb27bis/e/sTAoaBlLCluJxXVm3gjduuJiNBNYJROr8URy9SHMdXLpfuaAApWmyoMGSQHXWfddskyRMIoGtaxMdq/f4oR3P8Hrj4bB5wLuCttZuxpCQjIZ6fzT2LUT0jj/OIJM1ZgCmzMGURetO/C0tC0LLIiJvYXqErXVh7rCQ9DFxIY5JUCfwZeFcIMU5K2TLFP2wbcMYRXx/93P8Dfghc1/TcnwOfCCEGSSnr2yb0rsmyJPura4lz2MhKTIh1OEonMiplSIv5UaW+rbxZeBdG06DkYNDDvOI/4DNrGZl6USzC7PJCpslv3vusudU9QNA0qfX6+efnK7n33DNiF5yitCMpLfol9GRXwz4sJIfKqTS0pmSgZVON7mxgdkbEjowOm87sof2jHs/xctpt/OrCWfzkvDPxBkMkxzlPuGV+dbCWJVWJ5LrS6OGsRRcWFcEECn05JJcs5fqCK9op+s5NNW5oXZsmSUKIZOBG4Hop5SdNx64B9gGzgI+O8XJDShlxh17TKtL3gQellK81HbsWKAOuBJ5osx+inXiNKg561+PQE8h3j0ET0RnKuXTXPu557SMaAgFMKRmam8XDXz+f7CSVLCkn54vyp5oTpEMMGWB5+dMMT5kbtX/b3cneyuqIHdZClsXCbXtUkqR0SbsbCvntlkcwLAOERAOQ4NCc2LUCdFGBedQFnk046N+NV7UdNhu/vGAmP33zE4KGiSUlLruN7MQErp40JtbhfSWHTcdhO7nPkIO+Umyak32+TPb5wudQ7qjf0xbhKd1MW68kjQPswMeHDkgp9wshtgBTOHaS1FcIcRAI0FhKd5+UcnfTYwVAzlHn9QkhFjWdt0MnSSsrnufLyufRRONft004uLDnQ2S42nc2T2FVDbe/+HbY3ecNB0q47t+v8P73rlODDZWTUh7YGfG4KUN4jWoS7Cc3/E9pXbLLFTFJAkhTe5KULsiUJr/b8jcaDE/YcV3TuLTnucztMZvttR+xqOxhpJRILDRhY2Tqpd1+Js55IwfTNzONF75YS2ldAzMGFnDJ2GHEOx2xDq1dZTnTCVktW8lraPRUTbOUk9DWSVIOjWVyFUcdL216rDXLaSyj2wpkAT8FlgohhkkpK494bWmE80ZstSWEuJnG/Uv06hW7lqBFnrWsqnwBUwab25uG8PJO0Y+5rt9LCBG5drgt/G/Fegwz/MLKsCQHa+u56bnXGZKbzeXjhtMr7fgHtSldX9AK8UHxZywq/wKAM7Omck7OGdiaZpIk2XPwm7URX+vSk6IWZ3eSlZTA6PxcVhceDEuW4uw2rps6LoaRKUr72Fq3M+IFrylNCj1F6EJjSMq55MePZVf9QkwZoiBhKmnOPtEPtgMakpvF/RfPiXUYUZXpSmdUylDW1W4J+7dj12zM7TErhpF1YLLpjxLRcV2hCyHuF0LIr/hzxskGIaX8QEr5spRyvZTyU+D8ptiuPYVzPimlHC+lHJ+ZmfnVL2gnG2vealGaBBA0vZT4Nrfr9y6qrg27oDr0exAwTRbvKuTfS1dxwT+eY/623ZFP0AUETA/Lyp/h2d038N+9t7G55iOkVDXsrbGkxW82/4VXi96lyFdMka+Yl/a/xe+2/A3Z1AHntIxrsQln2OtswsmIlAuxaV37TmUsPfz18xmWl4XLZiPB6cBp0/n2tAnMUUNNlS4oaAUjdXMGwGcebkCQaM9mdNoVjEu/SiVICt8feCMzMiZiFzY0NPLicrhvyB3ku4+/AYSiHHK8K0kPA89/xXMKgUmADmQA5Uc8lg18frxBSSkbhBCbgEOf/of2KmU3fZ8jz3v8k8ZiIGh6Ih4XQiNo+dr1e0/u25PFO/fiCxkRHzcsC8OyuOeND1nyo1ux6e23qhULhhXkpX13UBcqaZ7mPr/0bxzwbWB27t0xjq5jWl+7hULPgbC230ErxI6GvWyp38nQpAH0SZjEGTl3saTscQJmfVOJy8VMyrwhhpF3fanxcfzvpm+yp6Ka8voGBudkkhTnavE8w7TwBIMkuU5847OidBSDEvtjRuj15NQcTMkYH4OIlM7AqTu4tf9V3NTvGxiWiVNXN+6+ilD3jVt1XEmSlLKCliV0LQghVgEhYDbwYtOxfGAIsPR4gxJCuIDBwPymQ3toTIZmAyuPeM504EfHe95o8xheNjf4iROiuR3lIZY0yHUPb9fvf8HooTy9dBWldR5Chwa1RbhmMkyLLSVljMg7VkVk57Ot7jPqQ+XNCRKAIf1sq/uMCelXkuJQNcpH21G/G7/VcuUzZIXYUb+boUmN9y0GJ89mUNJMAlYDds2NLtTItWgpyEilIKPlgEXTsnh43hKeX74OwzJJcrm45+wZzB01JMJZlPZSGzxIbaiEdGcf4m1psQ6n03Lb4rihzzd4eu//MCwDC4lTc9IvoReT01WSpBybLnR0XTURUk5Nm17ZSClrhRBPAX8QQpRxuAX4euDTQ88TQmwF/i6l/HvT1w8B79C4SpQF/AyIB/7TdF4phHgYuK/ptdtp3LfUQFMy1hE9tuvfbK4PMTjeQbweRBcSS4IubEzPvgOH1r4bruOdDl655Sr+9flKPt68g3KPJ+KqkiUlcXZ7u8YSC/u9qzFky7kQGjolvs0qSYog1ZGCU3MQsIJhxx2anVRHctgxITS1B6kDeeiTz/nfyvXNv+OVHi8/e+dTkt0uZgwoiHF0XV/I8vFe0a8o8q1HF3ZMGWRw0mzOyrlTdXw8SWdmT6VfQh/mlS2mwfAwIW0UE9JGo6u/T0VRoqA9bv9+HzCAlzg8TPZbR81IGkRjSd4h+cB/OVym9wUwSUq574jn/KHpfI9yeJjsnI46I8lreFlXswlDmmxsyCfd3kC6vYGQ1JGiH8NSzotKHCluF3efPZ27z57O+xu38ZM3Pw5LlISA7KRE+mV2vTueibYsNGxYtEwM3bY0DMtgQdkSFlUsQxMaZ2VNZ1rGRLR2bKbR0U1JH8/z+15rcVwXOqeldfz2sd1VIGTw35XrwzpZAvhDBn+bv0wlSVHwWclfKfKtw5Sh5iY92+rmkeboydj0y2McXefVKz6P6wu+HuswuoS9ldW8s34rvlCQmYP6M7ZXD1WSq6jGDcfQ5kmSlDIAfLfpT2vPEUd9/Y3jOK8Eftn0p8PzmQFEU22bRFARSqQilAhAij02F+HnDhvIyr1FvL5mE7qmIRC4HXb+ceUFXfKNcnjKeaytfgNLHpEUouHSE+kRN4IHtz7CjoY9jRuEgb2eQtZWb+B7A2+OVcgx57bF8fOhd/Hw9n9SHaoFJBmONH4w8GZcuvMrX6/ERo3P3+oH3YHquugG0w2ZMsSO+gVhpb3QOD9sbfUbKklSYu7V1Rv5zfvzMS0L07L478r1nDN0IA9cNKdDfv77zTpWV77BHs8K3HoqY9MvoXf82FiHpXQzaiNBO0lzpJBgi6c6VBN2XENjVPKwmMQkhOAX58/kxqnj+XLfAdIT3Ewu6NXlGjYckuzI5fz8X/PRwd9hWH4sLFIdPTk/75dsqtvOzoa9zQkSQMAKsqpmPXs8hRTEx65tfKz1TejFI2N+TWmgAgFkOTM65Ieoclh6vBu7TcdvtFw1HZwbu+6e3YVhBVvtmhmwWjbvqQnWUuIvJduVRapDjWBQ2le118dv3v+MgHG4oMcXMvhw8w7mjhzMlH69YxgdSCnxhwxcdhtCCPxmHc/t/g4+s6b5xkORdz1Ts65nbNrFMY1V6V5UktROhBDc3Pca/rLjiaZNpxY2YSNOd3F5zwtiGlt+ajL5qclf/cQuoHf8OG7q/zJVwf3YhZMkR2Nzik3FSwlEaFBgSYstddu7dZIEjf9+c1zq4rqzsOkad541mYc+WRxWcuey2/jBzKkxjKx7cGhukuy51ISKwo4LBD3do5u/NiyDf+55li8qV2LX7BhWiAlp47il73XNc8gU5UR4jWqqAvtJcuSQZM+K+JzFO/dh0zQChHcL9IVCvL9pe8ySJCklzyxbzeOLltMQCJLijuOuWVPpkb8mLEGCxqZLS8qeZnjKOe2+n7u7EarcrlXqXbkdjU4dwW+G38P7xZ9S4i9jSNJAzsk5i2S72uweTUJopDvDPwSS7UnYhY2QDL/zbhM6SfbEaIanKG3i6oljSHW7eXTBMkrrPQzNyeTuOdO7XNfKjkgIwczcH/DW/vswZQiJhYYNm+ZkatZNzc9748C7rKj6EkMaGGbje8+X1WtIK0rhm70ui1X4SickpcVnJX9jU+3H6MKBKUP0jh/LeXk/waaFl0bbW6kW0YTArsWukuSZZav56/ylYc1mfvP+fG6+aB2m1nKQsCZ0yv27yGvnzsCKcohKktpZL3c+t/a7LtZhKEeZmjGRV4rehqOSJE1ojE8d3cqrFKVjO2/EIM4bMSjWYXQ6O8sreXfDVkKmyazB/RnT88Q7X+a7R/HNPv9gddWrVAULyY0bypjUS0mwH+5R9Enp/LAZZNA4NHVe6UKVJCknZHXV62yu/bSpUUjjv6l9ntXML3mU2T3uCnvu9P59sGTL5QKHTeei0UOjEu/RpJQ8/vmKFh13/SGDwgqT7AiLYgHTz9sH53FhXgY94tTNnzYhgQj/NpRGKklSuqUURxL/N+gOHtnxBEErhATibW5+OPA21aBAiaraUCkrKl7igG8TKY48JqZfQW7c4FiH1W08u3w1f/50CSHTxJKSF1eu46JRQ/nFeTNP+Fxpzt7Myv1hq4/7zJYjCQB8lh8ppdr7pxy3NVVvYMjwknFTBtlSN4+zcr8XNrsu3ungkSvO53svv4uGwJISieTbU8czKj832qEDEDRM6v0tS94B1mzpyfnZFWE/n5TgNW2sqdzMyur7+emQu+mb0CdK0SrdlUqSlE4laJp4AkGS41xop3hBMTR5EI+Ne4g9nkJ0odPbna8uUpSoqg4e4Lk93yVk+ZGYVAT2sbfhS87Pu5f+iZNjHV6XV1rfwEOfLCZohm9of3PdZuaOHMLYk1hROpZ+CQXsaNjV4nhBfG/13qMcU6m/jDJ/OXnuHqQ5UvFbDRGfZ0kL0wqi6+GXdzMGFLDwrv9n77zjo7jOtn2d2dmi3ntFiI4opjcbMMYNN9zjXmInTnnzpr5JnMROb1+6UxzHSWzHdowruGEbF7DBmN5BgEAgCfUubZlyvj9WCIRWpmi1qzKXf/rhnZ05c2u1OzvPOc9zP59l1d6DeHWdeYX5Ya1Ndqg2kqIiqWntbmzikiM4P20aq6v+3jGJaeI27Bxwp2Ji4jW9/PvwMzw8/tthUG4xlLCCpEGGbvood+9CIMiKHIdNDI4msZph8Ku31/Dcph2YUhIX4eTbl8znsnG9Sy1ShMJwazbKIkysrn4czWxHdvp3S3Tp5Z3KPzI8eqZ149zHrN5/CJuigNG1oN2j6by1e3/Qg6Q78m/mR7t/hWZqmJgoKNgVlbvybwnqeSwGDx7Dyx/3/4U9zfuwKyqaqTE9aRpZEUUcbtvAqd7/8Y4MHLbIgGPFRbhYOjk87rqnIoTga4vm8tBrq7qazagqX79oHhMT8hkTu4gvbrkfXSp4za73MiVth0OsePBiGTf0jBUkDSIOt25iefmPOx8LBFdmPzgoegv8+I33eGX7nk6L45rWdr7zylskREQwq2BoO9FZDFzK2refFCCdwG000240EKUOvibP/QlVUQgUhwoheix27w3DovL4adH3ea3iTQ61HyEvMofLMy626isseuSp0mfY07wXTepohr/2aEP9Ji5Jm4VD2Ylu+jDRESjYhJ0L0/8nzIrPnKsnjcVlV/n9u2s51tRCfnICX180j7mFfqMlhy0Ckzi8Zvc01QibK9RyLYYgVpAUBhrdHp7fspMd5ZWMSkvmxikTSIoKPPNzprTpjbxc9nC3HOWXjz7MfYVPEqEOXEe9Vq+Pl7btxqd3n+19ZPXHVpBk0S+RUrKhfhOvHVtJi97ChLjxXJW1pEtfnAhbHG4jcLNXh9K7a4LF6Vk4ajgPv/Zut+0Om40lRWP65JzprlTuKbi9T8a2GFwY0uCj2o/RTzEY8pk+Vtdu5SdFj7KpbhnH3HtJcuYyJel6kp354RF7jlwybiSXjBvZ4/OLUhewsnIVPnmip6FDcbA4fWEo5A0NrJWkHrGCpBBztKGJ6x97Grem49F13ttfwuPrNvHMXTcyIjX59AP0wL7mDwj0TpdI9rWsZlLCkh6PldJkW8NKNjW8jM9wMzxmOnNSbiFKTThnPcGkrq0dVVHwndLjAaCsoSkMiiwsTs/L5St47dibeDsaFr9fvYZP6jfxswkPEWf31wJMS7yeVVWPdJncsAkHo2LmYVesmdK+Ji7Cxa+WXsI3XnwTRQhMJFJKvrxgFqPSzv16bGERDHRTx+yhSbHb8BBrT2VB+hdCrCq0XJd9JQ1aI+s7eotppsbMxGlck9XzPY2FRbCwgqQQ89OV79Pk8XbacXp1A59u8P3XVvHMXTee9Xjtejs7mnZS2rYXXXbvK2BIHa/RvTDyZN6ufISdjas6b9S2N7zJ/pZ13DP8r7hs4e8ZlBEbHXC7AMZnpoVWjEXQqWhqps3rY1hyImoYe3YEk3a9nRUVb6Cd9Jk0MHAbbt489g435l4LwPj4xTRqx9hU/wKKUDGkzrCoKVyU8eVwSR9yXDRmBO9/NYd39x3EpxvMHzmM9NjwX/csLJw2J5kRGZS5y7tsFwjGxg4Nq39VUfn88Lv5TO51VHmqSXOlWr0mLUKGFSSFmI9KSrv1K5DA1rJj6KZ5VjeJn9Rt4O+HHkdBwal4yHaBckp+vU2o5H9KTVKLVsOOxre7dLY2MfAarWxreJMZydefsZ6+wqGqfOGCmfzp/XVdeiq47Cpfmj87jMosekNVcytffG4F+6prsCkKDpuNn165mAtHDQ+3tF5z1F2OqqidNQTH0aXOruY9nY+FEMxLvZPpSddT7ztKjJpCtD0pJBq9RjsHWz9BMz0Mi55CrD0lJOftj8RHuFg6qX8UtFtYnMxdw27jl3t/22n2oQobdsXBzbk3hFtaSImzx1rBUR8gsIwbPg0rSAoxTlXFq3dPG7Mp4qwsrRt9jTxa8o/OmWqPKYnVXcSqHpSOd7xduBgRM4e0iBE9jlPpOYhN2LsESQC69HGkbVu/CJIA7pk9ldToKP6yZj01re1MyEzja4vmWSkxAxQpJXc99QKH6xowpAQM2tH46guv8/y9N/cq9bQ/kGCPRzf1gM8lO7oHQU5bVEh7I5W2beXFIw+DECBNTCQzk29kTspnQqbBwsLi9IyMKeTHRd/njWNvU+4uZ3hUARdnLCLR0T/S4S0sBjNWkBRirpk4jmc3besSKDlsNi4dN/KsgqRP6jeeskVw1JNIgqoxJiaGVFcK4+MWUxjz6SstsWoyMkDOs8BGg+blS5u/gtvwMCKmkFtybyI7MvuMNQabKyaM4YoJfVNMbRFadlRUcay5pSNAOoHPMHhqw1YevnxRmJQFh1RXCgXR+RxsLUGXJ33WFQeXZSwOozLQTA8vHf0hmvR0KWNcX/sc+VGTyYq0PmMWQ4ONR8p4bO1GKppbmD0sl7tnTiE1JnB6dzhJd6Vx17Bbwy3DYjAipf/HIiCDowBgAPHVhXM4LycTl10lymEnwm5nTHoK3790wVmN4zN9AQo6BU26i8yoy7km52FGxM45bZ+VVNdwEpxZKKfEyxLY2tRAs96CJjV2N+/hR7t/Rq239qx0WlgEorq1LeCkgCklFY0tYVAUfP535BcZGzsGVag4FSfRahT3DruTwpjwphMebtuCP8miK7r0sbPx7dALsrAIAy9t28U9T7/Ee/sPsa+qlqc+2cqVf3uSqubATVoHK6Y0WF39JL/ZewM/2305/yz5MmXte05/oIXFEMBaSQoxLrvKv267jr2VNRTX1DIsKYHxGWln3TRyUvwEXqlYjmF2Td1TFZWJ8RPOeBwhBDfk/pgV5b+krH0nAgWHEklxm0qb0XV2QTM13qhcyW15VuNDi94xITMNzeiedupS1c4eGQOdKDWKb4z+Ck1aM+16O6muFGzCFm5ZGKZGYM9XiX6SzS74P/MbGzZyqO0Q6c50ZiXPIsIWERKdFhZ9hWYY/GTl+12amGqmSYvXx18+XM9Dl10YRnWhZeWxR9jZ9F6ncVOl5wDPln6HO4b9lhRXfnjFnQNuTeOjkiMYpsnsYbnEuJzhlmQxgLGCpDAxOj2F0ennXiidHZnNgtT5vF/9QafFsENxMD/lfHIjc85qrEg1nhvzfopbb8Znuil317PzwCOAu8t+BgaHWg+fs2YLi+OkxkRz89QJ/HfTjk4zDofNRnJ0JNdOGh9mdcGlvxUc50VPxpDd66XswsXo2PM7H7fqrfxw9w9p1prxml6cipMXy1/ku2O+S0ZERiglW1gEldL6xm6pvgC6afJRSWkYFIWHdr2ZHU2rAtQka6ytfY6rsr8Z9HO26vVsa3iLBl8FuVHjGRN7AXYlOIHMmoOH+fKyV1GEQOL/e/5kySKu6KOeZ4MFy7ihZ6wgaQBzc86NTE2YwtradUhgdvJMRkb3bNJwOiLUWCKIRZdqwKJzBYWcMNYkWQwu/u+iCyjKTOff67fQ6vFy0ehC7p49lWinI9zSBjURthgWpT/Aqsq/YEoDEwO7cFEQM42C6Kmd+z1f9jz13nqMjv5kXtOLDx//OPQPHhz7YLjkW1j0mvgIF7oRuP9Qci8buw8kGrXKgMZNEpNqTwluw02jr5EkZxIOpffX5Qr3Pp4u/Q6m1DGkxt7mD/mo5lnuLPg9Eb1sN9Lk9vClZSu6OOACPPjqO0zKziQnIa5X41sMTawgaQAjhGBkzAhGxpx7YBSIJGcSRXHj2dG0s0ufF7uicmn6xUE9l8XQRQjBkvGjWTI+dK5uFn4mJlxCduQ4dja+g2a6GREzi9yoSV3SfjfWb+wMkI4jkRxqO4TH8OCy9f9mt+16M41aJQn2DCJUq/dRX9Hk9vCLt1fz+u5ipJQsHFnAdy6eT0p0VLilBSQ5OooZ+dl8fPgo2knBUoRd5Z7Z08KoLLTE29O7BUgAAoU2Q/A/W/4Hm7BhYnJZ+mVcmXnlWZcGHEdKyYryX6OZJzJUNOmhRavlw5qnuSj9/nP+PQDe3ncg4HbDNHlt114+N3dGr8Yf1FgrST1iBUkWAfl84f08e+Q5VtesQZc62RHZ3Jl/K+kR6eGWZmFhEQSSnDlckHZXj88rIrCvjxACEcD4oT9hSoM3Kh5hR9N7qMKOLjUmxC/ikozPofSDurDBhCklt/z7OQ7XNaCZ/oDjrb372VJ2jJVfuBOn2j9vM36z9DK+tOxVtpRVYLfZ0E2TL54/k0WDoE/bmRKpxlIUt4idTe921iT5UTjQ5kWTsnOi9PXK14mzxzE/df45natNb6BJq+623UBnb/OHvQ6S3D4N0wycQtnq9QU4wsLi9PTPq5dF2HEodm7Pv4Xb8j6DIQ1UxXqr9DeO1Dfy/Nad1LS2M294HheNLsRus24ALYLDnKQ5vFP9DvpJ9UsKCqNjRuO09e9i6NXVT7Oz6QMMqXXOlO9sfJcYNYF5qVYvqGDyUUkpFU3NnQESgG5KmtweVu7Zz5X9tB4k1uXi37ddR0VTMzWtbRSmJBHlGHqpvhdnPEC0msDG+uV4zTbSXIVsaWzqZtzkM328duy1sw6SjrYf5bVjKyhzHyJKCdw7ThW9f93nDc/nV6vWdNvusqssGFnQ6/EthibWna/FpyKEQBWheZtopod1tS+yo/FdACbEL2Rm8rVBK+ocTKzad5Cvvvg6ummimyZv7i7mX+s38+Tt1/fbmVuLgcXVWVdT3FpMubvcP1EiVKLUKO4Zdk+4pZ2WjfUrTpkZB0162VC/wgqSgsz+6rqADdLbNY29VTX9Nkg6TmZcLJlx/cdYJdQowsa81FuZl+rvw+Q1vHxY9wDQvWarRT+79gz7W/bz2/2/QjM1JJJMp4pL0Tg5Y08VDiYnXNqbXwGA/KQEbp9NQPIbAAAgAElEQVQ+mac2bMWj6Uggwm5n8ehCzsvO7PX4gxnLuKFnrLupPkI3TXYdq0JVFMakp55Vo9ihiJQmTx3+LtWeEvSOmd+1tc9zsHUzdwz7JaKH1J+hiM8w+NYrK/HoJ2bl2jWNfdW1PL9lJ7dMmxRGdRaDBafNyYNjHqS4tZgj7UdIcaZQFFfUL2zM3YabrY2b8RoexsaOJ9WV1vmclBKv6Q54nMdoC5XEIcOwpAScqg3d1/WmOtJupzAlKUyqLM4Vh+IgwZFAna+u23P5UflnNdYzR5/EZ55IdavyxZLlbEAV4BB2JDAsahLTkq7upWo/X79wHhcUDuOlbbvRTZMl40czb3jeOddRWVhYQVIf8OHBUv73Jf8sv0QS43Ty1xuuZFxGWrd9W/UGdjd9hNdopyB6MlmRwTVhGCiUtG2hxlvaGSCBv7lljecwh9q2URA9OYzq+he7jlUhA1RaejSdFTv3WkGSRdAQQjAqZhSjYkaFW0one5t388iB3wH+ehiQXJi2mKXZNwB+zemuAio9B7sdmxFRGEqpQ4LzC/NJjorCqzejd6TcKUIQ6bBz6diRYVZncbYIIbg171b+cvAvnQGOQGBX7NzQ8Rk7E6SUHGk/0mWbIW0c8SQRqWjcM+xOMiJGkOoaFlT90/KymZZnufBaBAdrej7IVDa38oXnl9Pk8dDm89Hu06hqaeWO/7zQpXEdwP6Wjfxh3328U/kv3q9+micOfYeXy36HDNC/YbAhpaRFa8Zr+FNiKtqL0UxPt/180kuFuzjU8vo1LlUNWKAK/vQCC4vBimb6+POB3+M1vXhNL5r0oUmNd6vfZl/L3s79Ls74HHbhRHR8xQkU7MLJ4l4Wh1t0x6YoPHPXjVw4sgBVUbAJwdyCPJbdc7N1PRqgTIqfxNdGfo2xsWNJciQxOX4yD455kILoM6/tEUIQaQtkpy4QIoGJCYuDHiBZnAMSMGX4fvo51kpSkFm+cw9GgD+8YZqsKj7I5eP8M7Ka6eWFo7/qkjevSS97mtYyNnYOI2MHrw3p7qadPFn6T5q0RgAmxU/hvPh87MKFJrsGSg7FSaw9ORwyzxqPpvPH1et4abt/qX/xqEK+unAuiZERQT3P6LQUEiIjaG/qat0aYbdz85QJQT1Xf8Ktt9CqN5DgSEcNQs8Oi4HHnubdAbf7TB9ra9cwKsZvJ58dOYY7C/4fa2uWUeU9RJqrgDnJ15Piygul3CFDUlQkf7j+CqT0r3Fb6eUDn5ExI/nGqG/0aowLUy9iZdUbXVLuHMLBorTFvZVnYRESrCApyNS2tuMzuhex6qZJg/tEnvzhtp0BbXQ16WFb46pBGySVu8v488Hfd7lobm3cRKvWhE1R0QzBCdN+gSLsjI6dExatZ4OUkruffpEdFZV4O/7+L27fzdpDR3jj83cE1UxBCMFfb7qKO558Hq+uIyUY0mTpxLFcNHrwpRPppo/l5X9kT/NabEJFIrkg5WZmp1wTbmkWIcaQ3a+tx9HNrpMGqa58rs7p3U2exdnht4e3sPBzReZVNOvNfFS7BlXYMaTO7OS5XJ5xRbilWZxM/1/QCRtWkBRkZhfk8tyWHbRrXb+wBYLpudknPe6Z4pa9bG3cxKT4KX2kMny8Xfkmutk17VCXOgfbDvKlEd9kdfU/qPdVAJDkyOLq7G/iUPp/08otZcfYXVndGSCBPzCub3fzxu5irp4wNqjnG5mazOqvfJYPD5bS4HYzLTeLnIT4oJ6jv/BaxV/Y27yui53zB9VPE+dIYVzc3DCrswglo2PHYsjurltOxcn0pFlhUNT/8Ok6r+8pZv3ho2TGx3H9xPGkx0aHW5bFEEQRCrfl3cHSrOuo9daQ7EwhSu2fDYYtLAJhBUlB5vzh+YzPSGPHsUrcHTVIEXaVS0aPYGTqibSx/KiigMX3poQ6zcPjh/7C4rTLWJK5NGTaQ0Gl5xhmAGtRv824i/sKH6FF87vqxNgHjjPS7qpqjAC1ZO2axrbyyqAHSQB2m23Q93/wmR52Nq3u1hVek14+rFlmBUlDjAhbBLfn38UThx/HlCYGBg7Fyfi4iRTFTQy3vLDT6vVy/b+e5VhTC+2ahsNm47F1G3nspquZlmsVs58tdd5jvFv1H0rbdxKlxjEv5TrGx80Lt6wBR5QaZQVHFgMSK0gKMooQPH7LUl7cuouXd+zGbrNx4+QiLhvX1R1KVRxcl/NNlh35OYbUMaSBBNymHa+pAl5WVr7KgtTFRKmDZxZwRPRIjrQf7tKgEkCTOpkR/i/xgRQcHScnPg5VEXhP2e5SVQqSEsKiaTDgMdoCpqUCtGoNXR5rpsa2xs3UeqvIisxlXOwEFMs6ftAxI2k2BdGFfFy3Fo/RzoT48xgZPcqy+QUeW7eRow1NnSnfPsMAw+BrL7/BB1+613qNzoIGXxWPHvwqPtODxKRVb+CVsj9S763k/NTrwy3PooMjDY2U1DVQkJRA7iDNpuhrrD5JPWMFSX2Aw2bjpikTuOk0RfSFMVP48qjH+FPx92j0HcNr2tHkiR4kNqFS2n6IsbFFfS05ZFyYdjFrat/HMIzOlTSH4mB20jxi7QO3od/cgjwSIyPxaM2dK0oCcKg2rprQv5sp9mei1Xgcigvd8J3yjCAn6sTrWu+r5Rd7H8ZjeNBMH3bFQbIzha+PepCIgA5LFgOZFGcqV2QGp7fKYOL1PcUBa2KbPB6ONDSRl2jdRJ4pa6qXdQZIx9GklzU1y5iZfMWASAMfzHh1nS+/+CprDx3FblPQDJ2F41wsnixw2lxMiJ9Pqis33DItBjjWNGuYiVLjSIkYQ5sR0SVAAjAxiVUHbuAQiHhHPA+O/SFTE6YTZYsmxZnKtdk3clPureGW1itsisIzd9zArPwcVEVBVRTGZ6bx7B03EuuyvkzPFUXYuDjjXuzC2blNoOBQXCxMu61z2xOHH/NbypseTEy8pocqzzFeKX8+HLItLMJCTwYxpilxquFvAjyQONK+u0uAdBxFKNR5/XWzpjQpdx+l1lv9qWMZpjkkWnuEkl+tWsPaQ0fw6jqtXi+Txu8iJnslG+pf46Pal3j04NdYX/dquGUODKQM308/x1pJ6gdcmHoxWxs3dnF8U1BIcaaSFTH4ZkKSnSl8dvgD4ZYRdFJjonn8lmtxaxqGKYl2WjbVwaAofj7RagJrapbR6KsiJ3IM56feRJIzE/DbPxe37OlW66ZLnQ3167gp9/ZwyLawCDk3nzeBX6xa3VkPC/4U8MKUJNJjY8553BpPGQ2+KtJcecQ5BkZLht4S70ij1lfebbsudWLsiexq2sa/Dv8VXfowpSTVmcb9w/+XZGdq575by4/x0Jur2FNVg8uucuOkIr6+YC6OILqdDlWWbduJV/evmqYmN5KbVYOq+r8DJCa69PF25b8YFzeXaNVaQbU4N6xPaj8gL6qAW3Lv5tmj/0ZKiSENsiJy+Nzwr1g55AMQq4Fi8BkWPZFh0T0V5vc8GxXIHMXCYrBy4+QiNhwpY1VxCUIIFAExLid/uHbJOY3nNdw8VfoTytr3YxMqhtQYFzebpdlfQhGDe2VqXsp1lLbtQjupl6Eq7BRGn0e77uHvJb/HJ09MbFZ4yvjd/p/ww3G/RREKJXX13PH0850Bq1vTeXbLDipbWvnD0nP7e1j4kVLiOWkiIDejBpst0KqfjQMtm5mUsDCU8iwGEVaQ1E+YkTSHKQkzqPCUEWmL7DIbZTE0MTrMLWzC+ph+Gg7FyfDokRxo3dclKLIJG1MTZgQ8RkrJW/sO8NSmrbR6fVw6ZiS3TplEpMMKcC0GLjZF4bfXXM6Bmjq2lh8jLSaa2cNysSnnllm/vPyvHG3fhyF19I6AYHfTx6S6cjk/ZXA5r55KXtQ4rsz6Im8c+3tHbZJkdOwsrsz6Aq9WvNitZ5dE0qa3sb91D6NixvGP9Zvw6V338eg67+0vobK5pVcre0MdIQSTszPZXOZPe1Qdeg/2PsL6/jwDLOOGnrHePf0IVVHJjcwPtwyLMNPoq+GV8kcoad0BQEH0RK7OfoA4+9BIczkX7sj/LL/Y+zA+04vX9OJUXCQ4Ergq64aA+/981Wqe3bK9c5b3QG09y3fu4YW7PhPUxr8WFuGgMCWJwpTeuYTqpsau5rWdkzXH0aSX9XWvD/ogCaAo/nzGxc2hWasnwhaFs8MEpt5Xh0HgxsbNWhMA+6pqAraFcKg2jjQ0WUFSL/nBJQu5+Yn/EhHTSExyW8B9pDQZETM1xMosBhOWcUM/xWcYvLGnmN988BEvbt+F+5TmtBaDE8308ejB/+Ng6w7Mjv8Otm7j0YP/h25a74GeSHam8pOi33Jz7p1cnnENd+bfz/fG/ixgb47K5hb+s2lbl7oNr65ztLGJV3ftC6VsC4uwopk+Pqp5lz/t/xmPl/ye4pZdnc/pUuvRbMBruEMlMewowka8I6UzQAIYGzsBh+Lstq8hDQqiRgAwPiMNVem+vuHTDfKTrBqZ3jImLYXX77udGZMbkIpCq+FASn+vSbPDE+C6nG/gstxNPx0Z5p/TIIT4thBigxCiWQhRI4RYIYQYf8o+QgjxkBCiQgjhFkK8L4QYd8o+CUKIJ4UQTR0/TwohTvtBtKZM+4AqzxHerXqOCncJyc5MFqReT27UqNMf2EFDu5vrn3iW2rY22nwakXY7v3zvQ5bdcRM58XF9qNwi3Oxu/hiv2d7FVUli4jHa2NP8CUXxc8Korn/jUBzMTDp9c9lNZRXYbbZuVsluTef9AyVcO3FcD0daWAwedFPnd8U/pNJdga+j7mZX8zYWp1/FxelX4bJFkuTMoMZb1uU4gUJh9KRwSO43TE2cxTvVr1HnrUHraHTtUJxMT5xDkjMFgLtnTOHlHXu6TG65VJVLx4wkNXrw9D4MJ5lxsdhcLWCA23TiNe04FB2JwJQuMiPO/L7Lot8yH/gzsAF/Z5UfAu8IIcZKKes79vkm8DXgTmAf8H3gbSHEKCllS8c+TwO5wCUdjx8DngSu+LSTWytJQaa8/SB/2f8tdjWto95XSXHLZh4veYh9zZvOeIxfvf8h5U3NtPn8F9d2TaPB7eY7r7/dV7KDjmGabDxazpqSUtp8p/a4sWjw1fFB9Ureq36ji31svfcYPtPTbX/N9FLnqwilxEFLUmQkgaawbEKQGmPdvFgMDTY1rKXScyJAAvCZXt489hKtuv++4qqsB7ALJwp+kwZV2HHZIlmccVvAMYcKDsXBN0c9zMXpV5LhymZYVCE359zFzTl3de6TmxDPM7ffyLTcLOyKQkKEi8/OmspPLr8ojMoHH8mOlM7/N1HwmA68ph1F2IlUrVWkgY6U8mIp5T+llDullDuA24AUYA74V5GArwA/l1K+IKXcCdwBxACf6dhnDP7g6D4p5Top5TrgfmCJEOJTI2lrJSnIvHHsX13ccMCfw/1qxWOMjDnvjNzq3txbjG52dWoxpWTDkTK8ut7vayb2VNVwz39fol3zIRAY0uShxQtZOsGaoZdSsrziST6seR0QGNLGivL/clnGdSxKX0KaKw+H4uoWKNkVJ2muvPCIHmRMz8sm1uWi3ad1CZXsNhufOa8nBz0Li8HF9saN+Exvt+2qUDnQspdJCdPIixrDF0b8lnW1K6jxlpETOYqZSZcTbbfSxVy2CC7LuIbLMq7pcZ8xaSn859bAdZEWwWFJ5nU8VvJHtJOcBh3CwYWpl1qmDWeAAMQA6Fd0EjH4F3gaOh4PA9KBt47vIKV0CyFWA7OBvwGzgFZg7UnjfAS0dezTY5699Q4KMmXtBwJub/TVoEkvDnH6xqKfFkj1d0twzTC489kXqG/vmrP+g5XvUpSRxoiUoWs+IKXkydJfs6tpPcdT1RWh4TNtvH7secbHn8eo2GnEqIk0+Kox6XC3QyXWnsSomClhVD94UITgic9cx/3LXuZYcws2oSAE/OzyxQxPTgy3vCFLq9eHZhokRESEW8qQIEqNQSC62eRLZJcZ+CRnBkuy7gu1PAuLM6IobjK35t3Li+XP0KI14VCcLEq7nEvSrwy3NIszI1kIsfGkx49KKR/9lP1/D2wF1nU8Tu/4t+qU/aqArJP2qZEnFVlKKaUQovqk4wNiBUlBJlKNpUmr6bbdJuyo4szshZeMGcXz23d1qZmwCcHsYbk4bP27N8Xaw0e62Z6CP3h6dusOvnfRgjCo6h8cbNvJvpZNnBrnOhQDj2GwtWE9l2Rcw33Df87KyifY1fQRIBgfN4eLM24f9H1JQkleYjxv3HcHB+vqafdpjE5L6fefrcFKXVs733jtTT4+fBQE5Cck8PPLFzMh41O/uyx6ybyURWyqX4OJDxOBKQUgcCouCqPHhFuehcUZMy1xNlMTZqFJH6qwowirkuSs6N5iKpTUSinPyIJQCPEbYC4wV0oZ2F4yyFhBUpBJduTR6KvpciMsJcQ7M8/4Jvfr8+eyqayCo01N+DrS62JdLn562eI+Uh08mjzegA08DSm7rS4NNXY1re9mp3scm9Axpf9KFanGsDj9Di7LuAen7fQrjxbnhhCCwuTe2SRb9A4pJbc+vYzDDY2dKcb7a+u47enneeu+O0mzasT6BN3UeLvyP7hUH7p5/JqkYBfJfGHEt62bTIsBhxACh+juOGgxOBBC/Ba4CVggpSw56anKjn/TgCMnbU876blKIEUIIY6vJnXUMqWetE9ArCApyBxsO4xP2nCc1ENBkwpl7hpMaZ7Rl0+My8nye27lw5JS9lTXkJ8Yz8LCAuwDYKZ7Rm42utF9WiLSbmfxyMIwKOo/2IUjYHoL+G1mJyVM53DbAZ4u/Su1Pv/ndmzsZG7OvZ8o1bpZtBh8bCwrp6K5pVsNpm6aPLdtJ1+aOzNMygY371e/TEnbbgypd07oCSR5UWlkRGSHV5zFgMRnGPx13Sf8d+sOPJrOBcPz+cb8eWSEqB9Uk1ZHo6+OVFcWEbburR8sBi5CiN8DN+IPkPae8vQh/IHORfgd8BBCuIB5wDc69lkHROOvTTpelzQLiKJrnVI3rCApyHhMD7pU0aQNBYnpL4tDoHfYOp/ZDJ0iBOcPz+f84fl9KTfopMVE89lZ0/jH+o2dfWgi7Cpj0lK4aNTQDpLOS7iAdXVvdikwPc75KUtw2SL4bfH3u5g27G7awp8P/ISvj/ppv69Hs7A4W8oamwNu9xkGh+rrAz5n0Xs2NqxCP+U6JJEcbt+Dx2i3estYnDX/8/JrrDl0GG9Huv3re4pZe/gob913B7GuvsuI8Bke/nPk9+xv2YYq7OhS4/yUK7g4/SbrO/MM6c/GDUKIR/A72l0NNAghjudht0opWztqi34HfEcIsRcoBh7Eb9TwNICUco8Q4k3gb0KI4wWWfwNelVJ+anNEK0gKMgVRoyhu3QmIjgDJT2ZE7pBxWvmfebOYlpPFM1u20+b1cdnYUVw5bjSqMrRTONIj8rgk/RbeqHwKgYKJgZRwZda9zExexIqKZ7ul4xkYVHuPcaS9hLyo4WFSbmHRN4xLT8UM8AUdYVeZkp0ZBkVDA1323Jja6Ej1N6TBe9Ur+LBmJR7TzYjo8VyZdSspTqtW7FS8hhtF2LArjnBL6XNq29r4xyebWFt6lIzYGO6dPoWEiAg+PFTaGSCBP8W+zedj2bad3DPjjEpOzokXyv7G/pZt6FLrfF+vqX2NJEc605KGbg30IOKBjn9XnbL9YeChjv//JRABPAIkAOuBxSf1SAK/HfgfgZUdj5cDXzzdyYfGXXsIWZp9O78t/j66qWOgo6CgKnZuyLkn3NJCyuz8XGbn54ZbRr9jTsrlTIifQ3HrVuzCwaiYyThtfjevKk95wJolgUK9r8YKkiwGHSNTkpmTn8tHh4/g0f3vfVVRiHO5uGrc2DCrG7yMi53Bhvp3Ox00j5PsyCRK9adHPXvkb2xr/Lhz5XtX8yYOtu3mW6P/H3H2hJBr7o+Utx9i2dE/U+k5ihCC0THncV3O5zpfw8FGVUsrV/zzKVq8XjTTZFdVNR8eKuXa8WOxBZgE9eg6m8uP0Vd3Pz7Dw/amj7t9b2qmlw9qlltB0pkgCdQ2sN8gpTztcmBHndFDnAiaAu3TANx6tucf2lP7fUBGRA7/N+ZXzE25iGFRI5mZtJBvjPoZ+VEjwi3Nop8QY49nSsJ8JsTP7gyQwL8KaRfdZyJNqZMVYfVIsuj/eDSd57fv4qsr3uA3qz+ivClwOt3J/PGaJTwwezqZsTEkRESwtGgsL935GaKdg39WPlwsTr+JOHsCDsVf6K4KB04lghty/BOrjb46tjau65IaLJFopo81NW+GRXN/o0Vr4K8Hf0CF5zAmBobU2du8iUcPPozsx+lLveHP6z6huSNAOo5H13l51x5Ms3stssOmUNiHbRU8phtB4HvoZs1K17XoPdZKUh+Q6Ehmafbt4ZZhMcCYmbSAd6tfxdB1zA5PTrtwMC5uMqmujDCrGxgccx9hRcWTHGrbR6QaxQUpS5ibfKnl1hUCWrxelv77GSpbW3BrOnabjX9u3Myj117FrLyeV5XtNhufnz2Dz8+eEUK1Q5soNZb/HfU7tjV+SGnbPpKdGUxNXEi0GgfAMc/RzvqOk9GlzuG24nBI7nd8XPdOwPToOl8VR9qLyYsaFSZlfceaQ4e7mawcJzM+liP1jV0CKFWxcfPkvmvQHa3GEWGLpkVv6LJdSvCYXt6vfp35qZf12fktBj/WnYPFOVHR3MzjGzbz6PoNHKyzZmyCQaQaxddH/ZQpiXOJtEUTb0/i4vRruD3/S+GWNiCo9VbyxwPfo7h1O5r00qTV88ax/7K84olwSxsS/H39RsqbmzsNWzTDwK3pfO3VNwPWHVmEF4fiZFrihVyX8wDzU6/pDJAAkp1pAVN/FWyku3JCKTPs+Ewva2vf5vFDv+D5o49S4S4FoNpb1mNtV52vOpQSQ0ZyVGBDD800eeSaK1hQWIBdUVAVhVEpyTz1metI70Mbf0UoLM3+LAq2zoyx45caTUpeP/YsXsPT4/EWANL/ooXrp59jrST1E6SUbG5Yw+ra1/EYbYyNncqFadcQrcaGW1o3nt++kx+88y5Sgiklv//oYz47fSpfmTsr3NIGPPGORG7N+3y4ZQxI3q1+Gc3s6tilSS8f173D4rTriLRs1PuU1/cWd2mAfZwWr5fShkaGJVp1LAOFFGcGw6JGUtK2r0sgoCoqF6ReGkZlocVjuPnD/u/Q4KtDk14ECpsa1nBjzufJjRzJ7qZNaNLb5RgpTTJdgzM9+rPTp/LVV9/onAgBsCsKM3KyGZ6UyCNLr8Cr6/gMgxhnaHoWjYubRpwjnTpvOeDvi2qiAAKbsHHMc9Qqd7A4Z6yVpH7C8ooneKHsMcrdh6jzVbO29i1+W/wt3EZb5z6aYfDewRKWbd/JofqGTxmt76hta+MH77yLVzfwGQa6aeLVdR77ZCO7qwbn7JlF/8CQOtsa1/FS2T95v3oFrXrXepej7Qc7bPa7ogo7Nd5joZI5ZImw2wNuN6XEpVrzcQONu4d9nUnxs7AJFQWFNGcW9xd8hxTn0En9/ah2JfW+ms5ASGKiSR/Plz3K5Ph5uGwRiJNuo+zCQUH0ONIjBqdp0UUjC/nCrBm4VJVohwOnauO8rEx+d9WJlDanqoYsQDpOsjMTAxs6Nkxs0FGnZEidmJNWSC0szhbrm6sf0Kw1sK7u7S4zdgY67XorH9etYkHqlZTU1XPLs8to1zRMCaY0WTJ6FD+7dDFKCHsBrDpQghKgUNJnGLy2t5ixaamferxuaqyvf5/NDR9hVxzMTlpEUdw0q5+BxafiMdw8cuD71Pmq8ZkeVOHg7arnuX/498iN9PffSnVmUek52q1Zry41Eh0p4ZA9pLhl8gR+8u4HXWaZFSEYmZwcsoaSFsHDaXNxS94D3JR7H7qp47T1Xa+b/sqOpo8DptRJoF6r5ssjf8HrFU+xp3kzdsXO9MRFXJi2NPRCQ8jnZk3n1vMmUVxbS0pUFDnx4Q9CFqRczv6WnV2MRmzYyIkcTpLz0+9JLED0/6y3sGEFSWHGkAZra1fiXyQ+/k71Bwya9FHcsoP5KVfw+ZeWU9vW3uX27/V9xczMy+GaEFrl9uazZEiDRw78iHL34c6L2aG2fUxPuIBrc+4OjkCLQcn71cup8R7rvGHRpQ9dwlOlv+fbo/+AEIKFaVezu3lTly9KVTgYHzeVGHt8UHRIKdlZVUVVaxtF6WmkRVspfMe5YWIRG8sqeGPffmxCIATEuyL409VLwi3NohfYhIrNNjRvFVy2qIDbTWngUiKIsydyc96XQ6wq/EQ7HZyX1X/6mBXGjOOarDt4peJJQGBInbzIQu4c9r/hlmYxwBmaV75+Qpvewp8PPEijrxZTap3rM/5ARKCgkORI4VB9AxXNLd0CFLem8/SW7SENkhYVFvCjd97rtt1hs3H56JGfeuyupk1UeEq73MT6TC8f17/HBamXkWw1KbTogS2NHwWc0W3RGmnQakh0pJIVkc/dw77Ji+X/oM5bhU2oTE9cwBWZtwVFQ3VrK3cse4HypmYUIfAZBjdPnMCDC+dbK6H4V41+veQSvjB7BlsrjpEWHc3MvJygrnTrps6WxrVsb/yEKDWa2ckXkRtp9Q+z6BvmJl/Ckfb9+MwTdUcCQaIzjVRXVhiVWZzKrOQLmZo4j0pPOdFqDAmO5HBLGjgMAAOFcGEFSWHk1YonqPdWYeAvdhb4AySBRCKwCZW5KZfS0Gz0eBPm1nrunN4XJEdF8fBFC/nB2+8i8dcb2BSFe6dNOW2q3Z7mrV2+bI6joHCwdQ9Ragy7mzaiS41RMZOIty5yFh3YROBLlUR2eW5ETBHfGv07vIYHu+IIqvX3F195lZK6eoyTvlCe27GDovQ0rrYan3YyLDGhT0wadFPjjwceotJThs/0IhBsbljLVVm3MSf5oqCfz8JiXOxU5iZdyura1zquMyMaxPAAACAASURBVJIoNZa7878Z9HMdqKujrr2dsampIa/pGSzYFQc5kcPCLcNiEGEFSWFkR9O6zgDpOMdDoUhbDDfmfp50Vw4pDhOXqtJ+SkDkUlWuGDs6RGpPcN2E8czOz+XNfQfQTINFhcMZnnT6hnExahw2bN1/Z6HQ4Kvmx7vuRwiBlCYSyUVpN7Ag7eq++jUsBhAzEhfyZuV/u6xCCgRpzizi7N3fe8Gun6hsaWFXVVWXAAn8q7n/3rzFCpJCwKaGDzsDJOhobip9vFL+JFMS5uCyBbYntrA4V4QQXJZ5M3NTLqW0vZhoNZb8yFFBXTmuaW3j3hdfoqSuHpuioJkmX5kzi89Onxa0c4QLKSV7W7aztWE9qmJneuL55EVZK78WAwcrSAojPa1wChS+N/avqIr/z2NTFH5zxaV8/sXl6FKiGQaRdjt5CfHcft6kECo+QWZsLHdPO++sjpmeNJ/3a17DkF2DJAWFNTUr/A5CJ70m71QtY2TMRLKsmaEhz9yUSzjQuosDrbuQmNiEikNxclt+aHLO23waNkWBABbXzZ7uq6MWwWdb4/qAK9E2YaOkbR9jYyd3btNMH8UtW/GY7RRGFxFnTwqlVItBRqw9nqK46X0y9v0vvcLe6pouEzB/WLuOkSnJXDBs4H73SSl5svQRdjRt7Fz5XV/3PovTr2Fxeu8mP5s9HkoaGsiMiSHVqgvtHRJE4P7AFlhBUlgZHzedbY1rMU9aWREojIqZ1BkgHWdufh4r772D57bvpLKllTl5uVw8agQOmy3Uss+ZZGcat+d/mf+UPtIRC0kibFHMS76I96pf6OYKoUuNzQ2rrSDJAptQuafgW5S1l1Davp84eyJjYif3mIYXbPIT4gOu5tptCotHFoZEw1AnoocieonEpUR0Pj7SVszjh36M7PjPlAYLUpdyYdr1oZJqYXFGlDY0UlxbG3CF+l8bN4c9SKryVPBJ3Qe4zXaK4qYyOmbCGa+iHWjd3RkgwYmV35WVLzItcR4JjrOfuJBS8ovVa3hi8xbsNhs+w2BBwTB+c9mluHpoQWBh0RusICmMXJF1B6Xt+2jTm/GaHhyKC5ctkqXZ9wXcPzM2lq/MnR1ilcFlfNxUfjT+7xxpP4hdsZMdMYzNDasDuuZJJLr0BXjGYqiSHVlAdmRByM9rUxR+funFfHn5q2iGgdHR+ycxMoL7BkFazEBgdvIidjRt6JJyCeBUIsiP8pvGGFLnn4d/isds77LP+9UvUxA9jmFRAy8tUkrJgfp6TNNkRHJySFs+WPQtjR43qhK4brK2rT3g9lCxvu4Dlh19HEMamBhsqF/DyOjx3FPw1TOq9dzWuKGHGmR4p3IZ4+OnMiJ6Iqpy5sHNM9u389SWrXgNA2/Hqv77hw7x0Kr3+Pkli894HItTsIwbesQKksJItBrHN0b/nl1NG6j0HCXFmUlR3IyzumgMRFRFpSB6VOfjkTETMWX3NCaH4qQoflYopVlY9MjC4QW8dNstPLllC2VNzczJy+WGCUWDvshaSsnBtn1sa9iAqqhMS5xDZhiaZQ6PHsOl6dfzeuVzqEJFInEqLj43/DudN20lrbsCXks06WND/aoBFyTtqqrm88uX0+B2AxDjcPDHK65gSj+yX7Y4d0anpGAGuEF12GwsHB76yaDjuI12lh19vJsTbXHrTnY0bWJi/OknhpyKEwUF86QG36owUITGtqYP2Nn8EYpQuHvY98iJPLPV+L9v2Ihb17ts8+oGy/fs4eFFC3FaTastgoz1jgozNqEyIX4WExi6wUCMPZ4lmbfzasWTmFJHIrErDibEzWJ41LhwywOgormZ2vZ2CpOSiLSW9YcsI5KT+OFFi8ItI2RIKXnmyGNsaljXUVeg8EH1W1yeeT0Xpl0Wcj0L0q5getJ8Slr3EmGLpCB6TJdZbc3saeVZ4jM8oREZJNo1jVuXLaPZ6+2y7a4XXuCDz95LQkTEpxxtMRBwqioPLpzPj1a9h0fXkYDTZiMxMpK7pp5dzW9v8Rpu9rdswcTEZ5oowtYtBd5netncsPaMgqRpSefzQc2bmB2BloKJKkyE8KfSH2/p8M9DP+G7Y/9+RqnTje7An2FTSto1zQqSLIKO9Y6y6BEpJSVtxWyuX4cQClMT55DfR840s5MvYXj0eDY3rEY3fYyPm0F+1Oiw959p8nj4wooVbC6vwG6zYZgmX5kzm3unTg2rLguLUHCwbV9ngAQgMdGkj1crnmNKwkziHad3tQw2UWoMRT3cpBVEj+1mDAP+VekJ8XP6WlpQWbl/P4bZvaLakJIVe/dy++TJAY6yGGjcMKGI4YmJ/HPTZipbWlkwfBi3Tp5EnCu4Dp2fxp7mDfy39DeIjgkHQ2oIAk8G2hXHGY2Z7srimuzbeLHsCWzChsAdcD9D6hxq3U1hzITTjjk9O5tVBw92S89PjY4iPoSv16DDyrbrkaAHSUIIJ/Br4GYgAlgFPCClLPuUYw4DeQGeel1KeXnHPg8BPzjl+SopZb/tQKqbJjurqrApCuNSUwdcLvkLZU+yru79ztnZtXXvcWHqZVye2TcF0GmubC7N+EyfjH2ufPnV19hYVo5mmp050L/7aC3DExNZUBC+dAgLi2AgpeRw225qvOWkuXLIjew6MdFjXYFQ2N28jdnJC0Ip97S4bFFclXUPy8v/gS51JCYOxUV+5GjGx80It7yzoratHV8AN0WPrlPd2hYGRRZ9xZTsLKZkh6c5bZvezH9L/58/te6km2Wb0BE4kJy4HtiFg5lJ88947DnJi5gYP529zTtYV7eCcveBgPsFusYE4pvnz+Pjo0fxaBq69Ctzqio/XLQo7BOqFoOTvlhJ+h1wFf4gqQ74DfCqEGKKlAGm+PxMA062acsANgHPnbLfPmD+SY97Gi/sfFRaypdfew3dNJFSEu1w8NerrmJCevBiumatiQ9r36PCfZS8qALmJM0nUg3sAHW2HG0/xNq697qkr2imj1VVrzEtcR6prn4bmwaN6tZWNpSVoZ0ym+vWdf62YYMVJFkMaNx6K4+VfJ96XyVSSoQQpDpzuLvgIZw2fyqXKuzd6grA389NDZGz4NkyLfFCciNHsqH+XdxGK+PiZjA65rygNhYOBdOys1BtSrfrT6Tdzoyc7DCpshhs7Gr6GOgeYNiw4VIEJq4Ol0iThWlLKIwec1bjR6uxTE2cgyokL5b9BZ/sGhAZ0qAg+sxqBYcnJfLq7bfx6IYNbK6oYFhCAvdPn05RetpZabLoirCMG3okqN9yQog44B7gLinl2x3bbgNKgUXAykDHSSlrThnnHqCZ7kGSLqWsDKbmvqCmrY37X3mlS4Fhm6Zx+wsvsPa++4JS01LhPsr/K/4hmqmjS40dTVt4u+pVvjXqhyQ5U85pzDpvBcUtm7ArTsraa9BNrds+EsnOpi0sdF3a21+h39PgdqPalM4VpJOpGYAzubppsrmiAq+uMyUry6qtGuKsKP87NZ4yDDquUxKOeQ7z5rEnuCr7fgCmJc7hg+qVnXUFxzGlZHxcaGsmzoY0Vw5LMu8It4xeMTE9ndm5eawtLe38LnGpKkVpaczJC5R4YWFx9mimt0sbkuNITOanXEZaRCEew82o2AkkOpLP+TxF8bPY2LCKI+378ZkeBAqqUFmSeTeuHuz9A5ETH8ePhlBdqEV4CfZU4BTADrx1fIOU8qgQYg8wmx6CpJMR/jXTe4CnpJSnJrEWCCEqAC+wHviOlLIkWOKDxct79gR0rDFMk7cPHOCqMWc3ExOI/xx5HLdx4uXRpA9D13mh7D/cN/wrZz3eO5X/4aPa5YBEoGBIHbviwGd2nWESKNgHufvecYYlJiICzLCpisL5w/JDrqc3bK+s5J6XXsJrGAj878WfLl7MlaNHh1vaoMSQBpsb1vNJ/Vrsip25yQsYE1PUb1JCpJTsbF53IkDqwJAaWxs/6AySMiNyuCLzBpZX/BdFKAgEpjS5a9iXgrZqbREYIQR/vvIKlu3YyXM7d2CYkmvHjePmiRMGXOq2Rf9lVOwU3qr8T7ftqnAwPn4m2ZEjgnIem7Bx97AH2d28kZ1N64m0RTEt8UIyIvKDMr6FRV8Q7CApHX8KXO0p26s6njsTLgKGAX8/Zft64E5gL5AKPAisFUKMk1LWnTqIEOI+4D6A3NzQ2tXWtbcHXH3QTZN6d+DixbNBN3UOt3XP7TUx2N+6gbcqnyQzYjhjYqefkWPMkba9rK1d0a0nUYTixmdGcOpS/KQzcLYZDDhsNh6cfwEPv/te50yuXVGIcTr53PS+6b7eF3h1nTtfeIEmb9c0h2+/9RbjU1MpSAx98f1gxpQmfz7waw62FXfm2u9u3s685IVcm31LmNX5Od5kNRCnbl+QdinnJcxkd/M2VKEyLm6yFSCFCFVRuHniBG6eePqi9oFGdWsra48eJdrhYF5enuVMFiaSnZnMTr6cdbWvo0tfh7usk4nx5wctQDqOImyMj5sx4OoDBz1Wul2PnNFVSQjxY+C7p9ktWBW8nwU2SCm3nbxRSvnGKZo+BkqAO/DXPXHK/o8CjwJMnTo1pO+A2bm5/GfbNtq1rulqihDMzMnp9fiKUFCEDUOemAVWMIlT3QhgTc2LOBQXq+xJ3Df8Fz12qj/O1sb3AzZttSsOXIpA4neNMaXBbfmfI8Ye1+vfYaBwfVEROfHx/H3DRo61tDAnL5f7pk0jJWrg3CR+cPgwegCXLN0weH7XLr45b14YVA1edjdvp+SkAAn8hcmra97hgpSLSHamhlGdH0UoFEQXUdK6A3lStbZAMCKmu2tanCOBWcnzQ6jQYjDzyPqP+eP69aiKgiIEihD8e+lSJqZnhFtav6PKU8nKyhUcbi8h05XNJelLyI4MbrrlxRm3MTp2KlsbPsCUJhMS5lEQNT6o57CwGIic6dTN74CnTrPPEWAmfgOGZODkOqM0YM3pTiKESMVv+vCF0+0rpWwVQuwCgjvVEQTm5uUxKSODLRUVnSsQEXY7iwsLGZNybvVCJ6MIhcnx09nS+ElnoBRt8yKA41kYPtNDg6+Kdyuf4fKsez91PFOaXW6UTj7P9Tl3IIlAEQrjYicNyRnkmTk5QQluw0Wz1xvQ4VOXsrNJpUXw2Nm0FW9AtybB3pZdzD0pSKrzVrCpfiVNei0joqcwLm7uGVvs9parsu7nrwf+D830oUkvduHEYXOxJPOekJzfYmiyobycP3/yCT7D6OLed9dLL7H+vvux22yfcvTQoqy9lF/t+xGaqWFicsxdzvamzXyh8GuMigluY+S8qDHkRfW+FKA/cLixgVUlJdgVhYsLR5AWHR1uSf0XCXSfQ7Xo4IyCJCllLd1T6LohhNgEaPhT5p7u2JYNjAHWnsGp7sRfb/TMGZzLBYwG3juDcUOKIgSPX3MNL+7ezYu7dqEqCjcWFbEkiPUfN+XcQY23kmOecgQSVbRxapq6IXV2NH142iBpQvxctjeuRjvFdcaUJkXxs4mwWReYgcysnJyA/VYi7XYWWg59QSfSFoUNG8YpxdCKUIi0RXY+3tf8Cc8f/TWG1DExKG7eyNral7l3+C9xKH3f8yPJmcHXRv+ZzfXvU+k5TFbEcCYlXNDpbGdh0Rf8d8cOPLrebbtumnxcVsY8y5Sik2VlT3eZcJFIfKaPp0v/xcPjfxlGZaGlWWtiRcULbG/agkNxcEHKIhakLsYmugfUj6z/mD+tX9/p2Pmz1av5yaKLWDo2uEGlxdAgqEnAUsomIcQ/gF8KIao5YQG+HXjn+H5CiL3An6SUfzppmwDuBZ6VUraeOrYQ4tfACvwrVqnA94Ao4N/B/B2Chd1m48aiIm4sKuqT8SPVKL456mFK20uocB9h5bE/dbPpBc6oUHxYVBET4s/vCJR8KNhQhMKVWZ+zAqRBQFZsLHeedx5PbN2KuyMFNNJuZ2J6uhUk9QEzk+axqvr1bk1NBYLxcf5UNkPqvFT2uy4TE5r0UO87xid1rzE35dqQaHXZopidcnlIzmXRneM3ckOJNs3XY+9Kj97dUfVsafJ4eGH3LvbU1jAuNY2lY8YS63T2etxwUNIauK9QtbeSRl8tmxtWcqB1M7FqErOSryYvavAFAh7DzU/3fJ8Wralz4ml5xfMcbivh3oKuSUd7amp45JNPutWEf/edtzk/P5/kyEgsLM6GvqiU/AqgA//lRDPZ20/pkTQKf0reyczHnzp3aw/jZuNfYTqeyvcxMFNKWRo05QMMIQT5UcPJjxrOnqZVlLbvQZ4UKKnCzqT405eKCSG4OvsBpiYuYm/zBuyKkwnx80hwWL0HBgvfnDeP2bm5PLt9O+2axhWjR7Nk1ChsysDqHTMQSHWlc3ve/Tx55O8oKIDEJlQeKPw6jo5UuirP4YCTGrr0sbPpw5AFSRbh4Wj7EZ4ufYKStgM4FAdzky9gafb1IUu1DCeXjxzFmtLSbjW7mmEyI7t3ac2HGxtY+uwzeHQdj67zenExf1r//9k77/iq6ruPv8+4O3svsoCwlyBLFHAh7lrcWldr7dDavW1tn1rbPq3aoT5qh1U7nAguRASRjewZIGTvndzkrnPO7/kjELjcBEHuTUhy3q+Xrxee+b3JzTm/7/p8N7D4plvIih14vbQu1UVLoId+YUnhucPfwaO70UWAKg5R5N7O5Rn3MiV+cMljr2tYTafmDsrM+w0/O1q2UOutIfW4mY1vHyjscQCzLEmsKCqKWNB6ICMhzDlJJyHsTpIQwgfcf+S/3o4JCZ0JIVbS00SzY/tvCouBg5TPD3uAZ4t+iM/woBl+FNlCsi2L+ak3nPI1spwFZDkLImjl2UuTvxEJiXjr4FV6m5OTw5wIlrJUtrXx3NYtbKuuYkRCIl+aOo1RSZ99rsZAZmrCTCbETeGQuxBVsjA8qiCoNMQi2xGi50Jwm2yWuw1mGn2N/Hb/r/AZXgB8ho/V9Stp8NXztZGnP75hoHHZyJG8vHsXW6ur6QwEkCUJq6Lw0Lz5Z5zx+emKFbT5fN0jODyahk/X+fmqlTx3zbXhML9PuShlAUurX8d/3FB3i2RhRFQirf7DQRL+AeHj3ernmBA7F3UQjOkwhM7GhiWsa/g38RYfXkOlXbNj0BXYUySFss7iICepp9Erp7LPxKQ3TM3NQUKcNYVvjnqawvZPaPHXke7II8919sxlOVup6CzjmcNP0ujrarlLsafypfyvkuHI7GfLBhaHm5q49j//wqtpaIbBnro63j14gGevuZbZw/pWgv9swSrbGBvTs3RzkjWTWGsyDb5KOK74yCLZODfx8j6y0CSSBAwfiqQin9A3saJ2Wcig7oAIsLdtN/W+OpLPAvXDSKLKMn//3HV8WHyYZQcPEWu3cf348YxOOjNRIyEE6yvKQxbDhhB8XFpyRtfuLy5KXUiTv5E1DatQJZWA0JgcNw2PtjdkxtlR6n3lpDsGfhn14orH2N+2EUP4UCRwygHsVo06fzQCCYEgwZoYdM7lIwv4x7ZtIT1vhhBcZJaW947pQPaK6SRFGEMYHHQfoLi1jjd21LKxvI5Ym50vnjOV2yZOCqsTo8oWxsXOCtv1Bjse3cP/Fv6aTr2ze1ulp4Lf7X+ERyf+AZsyMOvY+4Nff7yaDv+xXgNdCDyaxk9WfMCHd97dr7adjUiSxM3ZP+H54h/jNTrhyNyiKfEXMy7mvP42z+QMOOzewbtVT9Hkr0GRVKbEX8olaXd1R/fLPKUhoh4AqqxS460e9E4SgCLLXDJ8BJcMHxHW66qy3GO51UBVzJMlmRuzv8CVGddR56sh0ZpMjCWWfxT/hAZ/ecjxhtBwKtH9YGl4afJVs79tPZo4FkyQJJCEwKn48OouEqxJ5LuCxY3Hp6Zy15Qp/H3bNgK63i0v/9N580kxFe5MPgOmkxRBKj2V/OHA7/BoHjoDAaRkQWxnKuVVKTy6ZjXFLc08NDdc46VMjqfV60WWJKKPlG/oRoAa72Esso1kWw6SJPFJ00Y00YPKkgiwreUTZiaai9VTZWNlRY/N2BVtbbT7fN2/B5NjJNrSeXDUsxR37KJDayXbOYY46+BfIA9majyH+W/p/3QLcmjCz7bm9/Ho7Vw37NsA5DjzKHIfDBH20AyNdHtGn9s8WJAkiSsLRrG0cD+B49Q8rYrCNaPCpyzbH7jUKPLUYw7l7KRrqew8ECT8IqOQ4RhJrPXMx4z0NzXew8iSCuLEWZNgkwyyY8ZwV+59PQaZvzvnfK4aPZrlh4qwKDKXjywgOy6ur0w3GWSYTlKEMITB4wf+l9ZACwBHh4nnZ9fS1u6itd3Fv3ft5P7pM4l3mD0I4aKwoYHvvP8eBxobEMC0jAzun5PE2qanEQiEMIhSE7gx52c0+5uCar2PEjACtPhb+t74swAhBPvb97G/fR/RajTTE2YSY4n51PNibDbc/tCfpSJJ2FTzMdMbsqQwPGpyf5thEibW1L9K4ITB3Jrws69tHR3aPbjUOC5KvYSP61cGOUkWycK42Akk2Qb+Arc/eWjefAobGihuae5WDixITOSHF8ztb9PCSkH0NOal3MTKun+jSCq60Eiz53Fj9g/627SwEGdNQfRQAiajMCd5AZdn3HvS80cnJZ9x+eaQwiy36xVz9RIhityH8OhHB3UK7HIAq6QTkGWy0hpobXdhVRQONTVxbqbZ/xIO2nxebnz1v7T5jkXXCpsOsrz2JRTl2IKkOVDDiyU/Yn7qN7HJtpDBnxbZQp5r6NUv60LnTwef4KC7EJ/hwyJZeL3yVe4f8SBjYk4uLXv3lHP4/bq13cOTAWyKwtWjx2AdoKUuJiYnQwjBtppqWrxezklPJ87uoMFXDj3kVFXJQou/DpcaR4I1ke+N/gn/LnuBIvdBrLKN85Pncm3mor7/EIOMGJuNJbfcypbqKoqamhiZkMiU9PRB2Zt7XvJ1TE24jFpvCVFqHIm2wZOFTLePINGWQZ23FOO40lRFtjA76Zp+tMxkqGE6SRHCo3uQkJAQpFlbscrHFo/xmZ0Ul6Xi1xQyYz49Sm9yaizet4/ACfXoIzIqkKQTVcQEfsODUzFId2RS2VlO4Eha3yJZyHbmUhA9sMszPgvrG9dxwL2/O7sWEAEQ8HTRX/jD5D/2OLjvKHdOOYeSlhZe2bMbq6Lg13XmZOfw83lmOanJ4KOkpZkvLH6VZo8HSZII6DoPzphNetoI6n3lQaMYAHQRIOG4RWyWcxjfHf2jvjZ7QGEIg41NG/mo/iN0oXNe4nnMSZqDKp982SJJEtMyMpmWMfiDj3bFOShnI0mSxG25v+DNisc53LEdgFhLMldnfoM4czSJSR9iOkkRYkTUSDShEad2YpU15OMCWRZFZ964/XibryMjeuA3WZ4tlLS0BGUyAJw2H7LccyrZq7fx7YIfsLz2PdY3rkFC4rykC7g4dcGgjDx+Gusa1vRYfqgLnZKOYoZHddXEN/oq+KTpTZp8lWS7JjAl/gqcagy/uPAivjFzFkVNTWTGxJgBAJNBiRCCu958ncq2tqCc0R83refxy2djkdfiPyLvDV2KhVMSLjUHc58mzxx+hu0t27sz/WWdZWxu3sy3C76NLJnz3QYzh5ubWFK4H78+nwX5X2BMchxOJXZIvpcjjoAeRvaZHMF0kiKEU3VyfdZNrKl/IshBgq7mw7T4Fu6fPriGvvU3k9PTce7dHTSksKIxiZyURixKcIZJFxrDnOOwKTauzLiGKzPMFP7JFh7SkdkUJe7tvFr+MJrQEOhUePbySdNS7s7/I9GWJBKdThLNqeYmg5g99XXUd3aEFNV5NI1Xd1fxi4t/w/vVf6Oicz92JYpZSdcyI/GqfrF1oFLWWca2lm1BQRu/4afIXcTetr2Mjx3fj9aZRJJ/7tjGo2tXo+k6uhD8Y8dWbhw3gZ/NvfCUztcMjTcq32Bl/Up8uo/8qHxuz7mdbOfQHEVhcmaYTlIEuTD1IrY0PYPf6AjZp0gSdtWMhoWTBcNH8Nj6dVS1t3WrG1U1pBPw1+BwutGONFRbJDuT4y8l1lQSC+L8pAso7jgc0qNlla3kunIRQvB29eNBikqa8KPrOqvrX+SKjME/CNPEpN3nQ+4lot3i9ZBqz+P2vF/2sVWDi31t+zB6GLbsM3xsb9mOIqm4FCs7Wt7gQPs6JGTGxs5jfspd2JShGaTxagH+tn0riwv3oUgyN4+fwC3jJ6HKA2edUet28+s1H+E7rmzeq2m8vGcX14waw+S09E+9xrOHn2V76/ZuB/uQ+xCP7HuEX47/JcmmMEqPSKZwQ6+YTlKEGR97Adubl2MEDX6TSHeMHLIP80hhU1XeuOlmHlu/jncOHkSVJRaNHce9E7/Enrbl7Gv9GItsZ1riFYyKnt3f5p51nJswg+0t29nRug3d0FFlFQmJr414AFmScQea6NRCVf8EOkXtm/vB4sFPm8+LJElEW00J9bOFSWnp6EboAt6uqiwcUdAPFg0+otVoFEkJGdEgIbOibiVrGtaTYSvDIhscFcrY2fI+lZ37uDv/j0hDrBxPNwxuev1lChsbugepPrp2NatLS3juqs/1s3WnzqqSw0cqGoIrP7yaxjsHCz/VSWryN7GtZRsBEcAqaciSwGuoaIbGsppl3JZzWwStNxmMmE5ShJmX8gWKO7bToTXjN7xYJDuqbOGqTDPqHgni7A4enn8RD8+/KGj7zKTPMTNp4Lws+gNZkvny8K9Q0lFCYfs+otQopsZPwyJb+aRpK/vbdqP3MFcKMB3+MHOwqZFvL3+X/Q31XVL26Zn8/pLLyIg2+7z6G6fFwk8vmM8vV6/Eq2kIuhyk7Ng4bhg3ob/NGxScE38OL5a9GLJdYGAICYvUjCwdc5CgSxyjOVBFSccO8qKm9KG1/c+HJYc52NTY7SBBV/nnuooydtTWMCk1rR+tO3UUWaanJK0kSSinkBGr8dZgVwRZlnqsko6g62LVvhhKOkrCbK3JUMB0TDNj8gAAIABJREFUkiKMQ43my8OfpLB9PdWeQ8RbMxgXe0HEF5W6YbC2oozKtjYmpaYxNtksLTM5NXJdueS6cgHw6T4e3vMrarw1+Awfw+xWnIo36EVmkWxMSzB7usJFm8/L9a/+mzafr3sJuLmqgkWv/oePvnAPFlNSvd+5afxExiQl88+d22no7ODS/BF8fuw47Kqlv00b0OhC42D7emo8B7gufQbv1G6hU/cjIeEzfOgCQMImB1Ck0BIhw9Co9xUPOSdpc1VFUC/uUXRDsLW6asA4SRfnD+ehVStCtncNBB7zqeenWJNJtVSjSvqRd1TXdyTD1kqKPTbM1g4izHK7XjGdpAjj0TwsrlzK+qbNKMjMTU5mfFxkX6TV7nZueO0/NHu83XXdMzKH8X9XXGPOrDE5Ld6reZ8qT1W3RHqlN5Ysh45D1rArDjQRYGzsfM6Jv7yfLR08LC7ch1/Xg4QBdCFo8/lYVVrMJfkj+s02k2NMSkvn96fQI2Fyavj0Dl4s/iZtgXoCwosq2cixW5iX+i2iLan8bv8f6BRdswf9hoohCBFFUmQL8dbBL/19ImlR0dhVNSiTBGBRZJJdrn6y6vSJszv4/SWX8e3l7yEjYSAQQvCNGbNOaTis16jF0kPCSQISrd7QHSYmn4LpJEUQzdB4eO+vqfXWdddWL6l6hz1t+/jRmO9GTM7ywWXvUN3ejn5cdGBDZTnPbfuEr06b0b2tsLGBZ7ZupqipiXPSM/jilKlDopzHEAblnZVYZQtp9tQhLSuqC42tTW+wvfltNOFnZPRsZifdjlPtirqtb9zQ7SABGMiUeZJwKXBX7g2MjplKtCWpv8wflJS2hkrZAwQMnYr2tn6wyMQk8qypf5GWQA36keeNJnxows/25n9xR/6fGBM7hq3N2xAI2jQ7ydZ2JIzurLaEgkOJYXjUtH78FP3DtaPG8IcNa4O2SYBVUbkkb3j/GPUZWThyFDOyhrG86BB+w+DCvHwyT3Fd0qm1YpVt+I3OoO2SBFoPAlomAMLMJJ0E00mKIFuat9Pkq8cqdyAZCgGhEhABijtKOeg+REH0yLDfs8XrYVtNVZCDBF2Nj//Zs6vbSVpXXsYX33oDn65jCMHehjpe3bebxTfcSn58QtjtOlvY2bKbp4ueI2AEMBAkWRN4sODrpDsGRjlCuFla8StKOragHVGs29n8Lofdm7gz/xmssr3XAbJ+w0JelOkgRYJJqWk4LZaQ8hlVlhmX9NnKZoUQNHs9OFQLDotZEmZy9rG/9aNuB+kYgnpvCR69nRuHLWJf2/7usrtyTzJp9lacig8JmfyoqSzMeAD5JEOvBysJDicvXLuIr7/7Fs1eDwLIjI7hqcuvxqYOvGVegsPJjeMnnvZ5GY4xPXyHQJVs5EedGw7TTIYYA++vZwCxvfkNRriKEYCEoEO3UdKZhC50DrtLIuIkBQyj18yIX++KTgsh+NHK5UHR6oBhoPn9/Hrtap698tqw23U2UO9r4ImDTwbN3qj21vLIvt/x+JTf9uoQDFYafCWUdHzSLY0OYKDh0VrZ1/ohk+IvZ27yBbxS8VrQz0xCIsWeQpLNdJAiwWXDC3h84zoq2o5J2dsUhTFJyZybcfqlRB+XlfCjlcup7ehAkuCy/JE8Mv8SXFZruE036QG/EWBj42YOtBeRZk/h/OTZxFjMIeIncjJFOhmZVHsqv5rwC96rXsYhdxFp9jQuT19AuiMVkFCkob2cmZKWwZo7v0RxSzOqLJMdG9ffJvU5UZYEpiVcy9ampQREV3mdIlmJsaQwPu6iTznbxCSUof1UiSAH29fhDuxAPq651KX4yHE0Uu0bRpItMSL3TXa6GBYTS1FzU9B2iyyzcHiXPG27309lD2U7AthYWR4Ru84GVtV9jC6CpUUFAq/uY0/rPibGjafGc4C9re+jG34KYi4g2zV10Jbj1XgOdg+JPZ6A8FLRuYtJ8ZdzUep89rTuZV/7PnRhoEoqFtnC/SO+2g8WDw2sisLr19/CYxvX8fbBAyiyxKLR47h/+szT/i7ua6jny++8GRQQWXb4IC0+L89f/flwm25yAm6tg4d2/4qWQBs+w4dVsvBG5Vv8ZOx3yXWZwy2PZ3zsJWxueh1dHB+QkUl3jMKmdPXVJFjjuSXnpv4y8axHkqRTrgRp8/l4t+gAzV4PMzOHMSklbVC86y5IuYt0xyi2Ni3FZ7gZFXM+U+KvwiLb+9u0sxOBWW53EkwnKUJsbnwFQXDaV5YgSvXi1BQmx51+KvlU+cMlC7nljVfQDB2fruO0WEhyOPnGjFlAl1xtb8MQY2yD90HS5G8OcZKgy1FqCbSyqeE/bGx4CV0EEBgUtq1iePQsLsv4waB4eZxIjCW5q1j7hOejIh1rflYkhW+OeoBidwmH3EXEWWOZHDcJi2yWbEWSOLuDh+dexMNzzyz6+ey2T4IGMwL4dJ2NlRWUt7UyLMZUfIokr1UsocHf1P3c8YsAiABPFf2V30x8OOR4IYwhN+PnKLOSb6Sscwf13mIMoaFIFqyKkyszv9vfpg06tlRXcseS1zCEwK/rWBWVeTm5/GnBlacktd1XCCFYVruCt6reo01zk+3M4rbsGxgd0/s8MkmSKIg5j4KY8/rQUpPBiukkRYgOrbnH7RIKXxtxF6ocuR/9xNQ0Vn3hHl7Zu5uS1mamZ2RxxciCbnlaq6JwdcFolh7YH7SAcqgq90w+J2J29TcTYsexuWkLPsMXtN0QBsMcCSyt+HVQFDMgvBS1r6eicwfDXJP72tyIM8w5EZeSQKtRjeDYcEwZlYlxC4OOzYvKJS8qt48tNDlTiluaMHqIEloVmcr2NtNJijCbmrb0GJip8dbRFmjvLrsrdW9hZe2TNPvLsclRTE1YxPSkm4aUw2SR7dya+3vKO3dR6y0i1pLC8OjpKJIZkDker+5jR8tu/EaAiXFjibWcntiSbhh8+Z0ldBzX8+jRAqwqLWHJwf18btTYcJvcI6tKi3ls0zpK21oYlZDEd2fOYVp6cDnxaxVLeLvm/e5y7+KOUn5T+AQ/GfMdhkfl9YmdQ4LQ2dgmRxg6T+A+Js81FZnQHhe7Ymd49PiI3z/J6eQr06bzm4sW8PkxofM7fjH3Is4bloNNUYi22rApCovGjOOOSYPXSZqeMJVUezKW4166NtnK7KQZdGglyD2Wnvk42L42ZPtgQJJkbsr5HVnO8ciS2pVBsmRyfc6vibJEphzUpG+Zlp6JpYfIsF/XGZkwNH/HzV4Py4sPsbmqokcHMpyovfQ5CkR3D2RV516WVPycZn9XqbPPcLOp8d98XPdcRG07G5EkiWzXRM5N/BwFMeeZDtIJ7G7dz1e2fIeni57nb8Uvcf/WH/JudehcoZOxq74WrxYqbuDRAvx3765wmXpS3jlUyH3vLWFHXQ0tXi8bqyq4bcmrbDiu3N9v+IMcJACrpBGnNvBGxZ/waK19YqvJ0MbMJEWIGUk3U9i+Gp/eiUFXP4Aq2bgw9atnRYOpw2Lhr1d9jsr2Nira2hiRkECiI7IDbvsbVVb56dgfsrx2BRsaN2OTrVyUOo/ZiTMpbFvZY+mZhIxFGrwliFGWRG7M+R1evR3N8ONSEwZlaeHZwsbGLSytWkZboJ0JcWO5LvNKEm3xEbvfPZOn8vLe3egBf7dD4FBVbhw7YdD/vffEX7Zs5I+b12NVFAwhiLXZefGaReTHRUbRc17y+Sypfge/cWxRKiNTEDUcl9r189/Q8EK3uuRRNOFjR/NSZiXfjkV2RMQ2k4GFV/fx+8K/4D2hEuI/5W8wNqaAHNewU7qOODKQt/d9kUUIwS/XfhQy08mraTyybjVLrr8VgGZ/C9JxdqZZW8iyt3QJYYlm/lZ0C5emf5+RMRdE3miTIUv/r9YHKdGWJO7Ie5rNTa9R1rGNaEsy0xOvJ8s5ISL3aw20UewuJ84aS44z85QXupnRMac8g6CvqWhv5amtm9hUVUF2TCxfPWcGU9PPbFCgXbFxVcblXJURPPw0L2pGj28IRVIZG3fxGd1zIGBXoukh8WkSRhZXvsPiynfwHYmMflS3lk1NW/ntxJ8Rb42MElVaVDRLbryN363/mLUVZcTa7NwzeSq3jp8UkfudzawtL+XPn2zAp+vdZcadgQB3Ln2dj267JyLBgSszFlDoPkhh+yEQAllSiLa4+Mrwe7qPafSV9niuhIxbaxqSw1GHOu5AA2vr/0qJewOqbGdi/NVoIr/HYzUjwOr69dx+ik7ShJRUrIpMxwnJJIdq4YYxka9y8ekatR3uHvcVNjZ0/zvOEttdBu6Q/WTaW44TwhJowsey6kcZ5pqMXTk71zADBckUbugV00mKIFGWROan3tvr/s6An511tcTYbIxJTP5ML2khBP8qe4N3qz/EIlvQhU66PYUfjnmAOOvAfXCUtrZw5Ssv4An40YTgYHMj6yrL+N8LF3LFiFFhv59NcXFV1s9YWvEwEjICgYHGBan3kmjLCfv9TAYfHq2Vso5PUCSF7KgZWI/LAHh0L69XvB00mFfHwKt7eatqObfnXh8xu3Ji4/jzZVdF7PoDhX/u2h4ypFcADZ5OdtfXMSElNez3tMgWfjD6mxx2l3C4o4RkWxITYsciH9drlGTLw601hJwrMIhSh2ZJ5FDGp7v5d8lX8OhtCHQw2tnU8CJRlhE9Hm8gQrJLJ0OVZZ5ceDV3L30DIQQ+XcOhWpiZmcU1o8aE62P0ik1RcVkstPv9IftSXa7jjrNxccp8VtStItHShHximQddgYTD7g2Mjb00ojabDF1MJ6mf+M/enTy85kMUSUYXgvSoKP5xxedPe7bBusZPWFazioDQCByZg1TeWcVjB57h4fHfOW27dKGzonYNK+vWYgiDC5Jncmna3D5XM/vfTWvoOK5ECMCjaTz08QoWDi/oVZ3vTMiJmsqXC/5LsXszugiQ45qKUx16syZMTp/dzUv5uO4vR/oQJQQGCzMfJufIAMOKzipUWSGgB4dvNaGzp21fP1g89Gj1eXvcrkgS7f5TX2R+FvKjcsnvRfhkVvIdVHTuDCq5UyUb5yRcZ8oWD0F2t7yD3+jscpCOoAkf7YGDqFIGJ7aS22QbMxJOr5d4ZuYw1tzxRd46WHhEAjyb6RmnXoFyJkiSxL1TzuXJLRuDghYOVeWBabOCjr05+/M4VQdbGv/Z6/UMofW6z+QUMTNJvWI6Sf3A9tpqfr7mw6Ca3JLWFm5f+iqrbv30sg93oIENDX+jxL0et+Yj2RJFhS+eo3XGOgZF7hKa/C0knEYZjxCC3xc+za7Wwu5myaryN9nUtJ2fjftWUPQz0myoLO+xqbrD76fG3U5GhEoELbKDArPGuV9ZXV7Cs9s3U9fZwbxheXxp8jSSnK5PP7GfaPaVsabuSXTh53gds3crf8ZdI17GpkQRb41FM0JVzoCIzUz7rNR466nsrCHDkUq6I6W/zQkblw0vYEddTUg2STMMJqem95NVkOYo4HPZv+Kjmqdp8BVjV2KYlngD5yRc1282mfQfVZ7dIT1qAAoqF6dM5J3aAwSMAAKBTbYxOW48E2JPX5EuweHkCxOnhMPk0+ZrU2fg13X+un0LuhBYFZkHp89m0ZhxQcfJksznMq9kZkI+r5d9N+TnItC7SuVNTCKE6ST1A8/v2obvhBe1IQQNng6219Uw5SQvbJ/u5uWS+/DorQh0VAmyHT5cipfCzozu4xRJoVPrPC0n6ZC7hN3HOUjQNS2+pKOcbS07iFGa8entZLqmEG+N7CDERIeD+s6OkO0GYlDPchrq/GPXVn6zYXX3Qra4pZnXD+7l3eu/cNY6SvvblqP3EM2UkChxr2dU7CUk2RIZFT2C/e0H0Y471ipbuSpjQV+a2ysBI8BjB/7K9ua9WGQFTeiMjSngO6PuxaZY+9u8M+amseP5795dlLQ249E0ZEnCpij8/PwLcVr6V0UtyzmRW/Of7FcbTLoob2uhsKmR3Ng4RsT3fQAjwZpNCZu6BZ+OIjCYl7KA6UnXsrp+HT7dz/TEc5gUO27Aie3IksS3Z5zH/dNm0uL1kuBwoJ5kPlO6Yyzj4hayp+VdNOFHRkaWFOakfBmXWZJqEkFMJ6kfaOjs6KG6tisN3eL1nPTcvS3v4Dc6glLxiiRItHZg9/rxGl2LGVVWSXecXo191wIuNNqtSq1sqvshqiRjYEA9jIq5lLmp34zYw/krU2bwg1XLgqK+NkXh0ryRRFkH/oLNJBRPIBDkIAH4DZ1Wr5e/7tzC92eenRk+zfAFzZk6isAgcFzk85sFX+bPh/7K7tZ9yJKCKincmXszo6J77jXoa/5TtpQdLXsJiEB3WeCe1kJeKH2dL+bf1M/WnTl21cIbi25h8YG9vH/4EElOF7ePnxyRXiSTgUdA1/nGB2+zorQIq6IQ0A3OScvg2YXX4rL03TtnQvxV7Gh+M6iMTEYlzppFir0ASZLIc0UmSCmEYE3DZpZULact4GZS7Biuz76SZFtk1B+tikKK69SCX/NSv87omIspal+DIlkYFXMh8bZTE6swOQkCMMxyu94wnaR+4OK8EXxSUxlS9hHQ9ZNmkQCqPLt6TMULJKIUH37Djior3Jt/W/ccjlMlzhKLRVJPGH4omBBdiUAjcNzf0YG25WS7ziU/+vzTusepcvXI0ZS2NvPktk2oskxA17lgWC6/mW82aA5WCpsaUHoo6fQbOqvKis9aJyk/+jz2tLyFJoJ7XgSCHNf07v93qk6+N/p+2gLtuLUOUu3Jp/03Gkk+qF0TJFUNEBAaK+vWcU/ejd0BkQ6tkXW1f6SkYy0SEvlRc5md+sCAUJiyqyo3jZ3ITWMn9rcpJmcZf9qygQ9LDwepH35SXcnPP17B7y5c+Clnh48YSyrXZf+W5dX/S4u/EiTIdU3nkvTvRDxj9N/ypbxVteKYAmf9RjY37+QPk39KvPXMBk/7dD+fNO+kQ/MwMW40afbk075GmmM0aY7Rve7fUlPJ/25aw4GmRvLj4vnWuecxKzOyVS8mgxvTSeoHbhg9nhd2b6Oiva27L8mhqtw/bRZx9pPPxIi3DqMUNSQVb5EUMhwjGB+Xz8L0i8h1ZZ22XecmTObvxf8N2hateFGk0Ci5JrzsbX0rYk6SJEk8cO5s7pk8jcPNTaS6okhxRUXkXiZnB4kOJ5rR8+jvtLP4d5/hmMjw6PMpav8YTXiRkFEkC1MTbyXaEtrTE2OJJsYS3Q+Wnpzjy2yPJ2BoCAQSEprh443S++jUGruz2UXtK6n3FXJ97j+Q+9HpE0IgEH3aOzlUcfv9LCs+eKTpfxjjkwZ+Nu6lvTvw6sHvVb+h8+bB/Tw6bwHKScrBwk2aYwy35/8Vr96GIln6ZFZWh9bJksrlBI7LYBlHFDiXVC5mQoxEh9ZIlnMaOVGzkE9j3uOB9mJ+ufdPCCHQEQghWJA2hztzF4XN8VtXWcbd777evaZqrOnkrnde58lLr+LCnOFhucfgRJjCDSfBdJL6AYfFwpuLbuPfe3by3uEDxNsd3DHhHOYM+3Sp6Qnx17CreXFIKj7Bls1Xc39+Rg8cu2LjoXHf4veFT9MaaAMkYi0yVtmKLkKVoXQjdGp3uHFZrExISYv4fQYChhC0+324LNaT1m8PVIbFxDIhJZVttdVBzpJDVfnSpHP70bKTI0kSF6f/gFGxl3Co7SMUycLo2EtJPUnE82xkTMwIdrUWhmwfGZXX7XgUu1fj09uDyn0NNDoC9VR0bCY7amaf2XsUr+7j+ZLXWVW/gYChMSo6j3vzbybHZc4XigTba6u5/e1XMITAr+uosswluSN4/KIrIqI62ld0BnoOEmjCQBMGCn3/zO3L7GxZZxUW2dKtknuUGLWVTt8/2NKgYBDgQOsyEm35XDnsMVTZdtJrdgTq8BpuHtn3JJ168Bpiee1aJseNZUr8uF7OPj3+Z93K0AG1usYv1q40nSSTz4zpJPUTLouVL06exhcnTzut86ItqVyd/TtWVP+WNn81ANmuaVyU/v2wRGRyXVn8ccovqfLWYgiDdFsi/zj8efQTAg2qZKcg9pIzvl+jr4X3az6m3FPDqOh8Lk6d1T2J3uQYrxbu5tcbV9Pm82JVFL44YRrfmDZ7QC9KeuL/FlzDfcveZEddLRZZxhCCH82ey+yss7tkQpIksl3TyHad3t/z2cRdeTfy412/JWAE0ISOIilYZJUvDj/Wj9ToK0IToX2TugjQ7C8hm753kh7d/zT724q6I+D72w/z492/54nJD5FoMyX8w4khBPcuWxw04yZgGCwvKWLJwX1cW3D6KmtnC7Mzs/mw9HBIv/DYxGRsyuBfKiVa4wkYJwrQCMZHVSJLRlc/MqAJDw2+Q+xrWcqEhEU9XqsjUM8H1T+hyXcIgcT0GJ2d7ZnUB445fT7Dzwe1a8PmJB1oauxxe0lbC7ph9Gkm0GTwMPj/8vuBVp+XF/Zs46PyEjKiYrhn4lQmJp88G9Lib6Oko5IkWzxZzpMfm+4Yx235z+PRWlFka9DQynAgSRKZjmM2XJz+Y96vehhDGBgEsEgOUhxjKIg5MyepyF3GT3Y9ji50AkJja/MeFlcu53eTvk+SLf5MP8agYVnxQX665oPuHraAYfDMzs0IBN86d04/WxdeEhxOXr72ZiraW2nyeChISMSu9q/y2FBhmDOdx6f8jHerV1HkLiXXlcXl6fNJOq5pO8Gahyo5QhwlRbIQZw3/0GWP1syOphcp7ViLTY5ifPyNDI++uDsgVN5ZTWH74aASIQDN0FhWs5pbcq4Ou01Dmd31tXT0kHHxaAH+s3/XgHaSfjJ7fnevsF/XscgyFkXhV3PPPBg4EEixJzIqJp/9bUXdCpzRihdZCi3F0oWPg23v9+gkCSF4p+JB2gIV3YI2Vhkmx5SxrmUEHfoxdVpfGKtREp1OajvcIdtjbXbTQfo0zHK7XjGdpDDT5Onk8tf+SbPXg0/XkaUq3i85yG/mLuDqEaHTrIUQ/LX4Fd6vWYtFVtGETr4rmx+PvY+oT8moONQza6Q8VXKjZnFz3vMUtr6PR28m2zWdbNd0pNOo/Q8YHkrcq+jUGkixjyPNMYU/H3wxaFK4zwgQMHReKHmTb466MwKfpO8RQrCtrpri1iZGJSR/ptr9xz5ZGyLy4dE0ntu5hQemzh6UpXdZ0bFkRffN99vkGAnWOG7NubbX/fnR89jY8Ay6dkzRT0bBqSYy7DiRinDg09t5o/RuvHoLBhrtwJra39LoPcCMlK8BUOWpPSJ+ESo4UdxRHlZ7TEAXXb1pPe7rpZ9woJAXF8/ym+7mn7u2sb2umtGJSdw54Zwh9Rz6zqgv85dDz7OteQ+yJONUZKySikGoY6zIPSv+1Xv30qHVhyh+yghy7I3s7egqg7XJVuYmh++Z8fUpM3hkw0chA2rvm3z2lmqbnP2YTlKY+b8dm2n0eAgcGRxpCIFH0/jJxx+wMK8AixLc2Ly8dh0f1K4nILTuWuCSjkP85cAf+fboB1HPkonr0ZZUpiXd/pnObfIV8U751zGEhi58KJKNBFsBlT2onRsYbGnefYbWnh20+X3c9vbLHGruKgMQwISkVJ6/fBGO08iOVLrbe9yuGQZuv+9TxT4GK5ubdvF88ZtUextItsVzW85VzEk+vcnzJqeHKtv4XPZTrKl9nLKODUhI5Eafz5yUB8Mu2rC/9U18RnuQSE2XYMzrTEi4GaeaQJYz/QQ1zi4sksrwqPBntgYyQgjaA34cqopF/my/qwnJqVh6CMo4VJVFo8afqYn9TorTxXdmDK7s/OngUh18b/R9uLUOOjUvCdY4Xi65lfZAddBxqmRnTGzPWdpOrbErgHpCckKWwKl0/S3bZRujY4YzOyl8z+vbxk2mze/jyW0bMY448/dMnMqXJ4c3eGMytDCdpDCzorSo20E6Hl0YHGppYkxisOzl0qoPu+U2JQwmRlWQaW9BiN28VLScSQm3MynhjgE3LO4oQgg+rPopfuPYQl8THhp9+8mxJ3HIkxRyjrWXCNVA42drPmBfY33Q92FHfTW/3bSan82+6JSvMzohic01lSHbXVbrkB2su6lxF78r/Fu3ZHW1t54/HnwBTWjMSzFfipEkypLCZVmPII6UaETq2VTZ8Ql6D+MOZMlCo+8ATnUmmY5UxscUsKv1AAHR9V2QAItsYUFaZJQ3ByLLig/w83Uf0uDpQJVlbh49iR/OnHvazpIqy/zl0qu5593XMYTAp+s4VQtT0zK4bgCX2p0qAUPj4/otrGvYQbTFxcL0ORREDz5nPEp1EaV2zS9akPkrlpY9iIGGcSQgkRc9l5ExF/d4brJ9DIYILaOTsZHpnE6SczTT4sczKW5MWJUoJUnia+fM5EuTzqXB00GiwzkkesnCgllu1yvmNyjMxPcS1dcMg1hbqBJMh9bZ/e9xrioy7S0okgBJoAsfO5texKmmUBB7ecRsjiRurYYOrTZkuy58jHS5KfGmBg2wtUoWFqSd15cmRgRDCN46XBjiMPt0nVcP7DktJ+n7My7gtrdeCZKndagq35t+/qATbjhVni9ZHDLTx2cEeL7kzSAnqdFbyNbGp2jw7selJjMx4S5yoy/sa3MHJZEO3ERZ0pA8ckjZjhA6TvVYcOW7o+/lX6VvsqJuPX7Dz7iYkdyTf+MZz3UZLGysLucbH77d/fwIGAb/2tcld/3I+ac/d252Zjarb/kSbx7cR5PXw+zMbM7LzB6wgbxTJWAE+MHOJyjrqMZr+JGQ+Lh+C3fnf47L0wevQ55oG87tw1+jtGMDHq2JDOdk4m25vR7vsiQzKuYqDrS93T07TsaCU03gutzvYpXDK8y0p7GWv2zbQGFTHblJMvNzc1iQOdV0kEzCgvktCjP3TJjK7oY6PNqxBZwqSUxITiUjKlTOc0r8WFbVbUKgke1o6nKQjkMTXnY2vTBgnSQheq9Tj7fGkO3MoMpThyR1ZdsrM6uLAAAgAElEQVQmxY3muqwFfWhhZBBCoPfy2f36iQpCJ2daWiYvXLGIRzeuprCpgfSoaL45bTaX548Kh6kDkhpvQ4/bm/2t6EeU2Zp8B3iv4ivdL2q/v401tf+DV29mdNzn+9Jck8/AuPhFHG7/IGh4toRKrHUYibYR3dussoU78xZxZ17PSltDnT9uXR8y/8era7x2YDc/nDGXaGtw8M5vBGj0tRJvjcGu9JzVT3a6+OKkgavk+FlYVfcJpR3V3ZUfAoHPCPDXw28wL3kadsVKmfsjtjW8S01HANmYzZX5l5HZw3t/oKHIVvKjT32Y96yUb5DsGMOe5lcJGB3kRF3AxIRbwu4gbaqu4I73XkFSOsnJrqMawb/KdvBa1Vtcm3khd+SZwi2figAMM5PUG6aTFGYW5I3k3sZ6nt6xCausoAmD/LgEnrrkmh6Pvzn7Sj5p2o1muHtphwWv3hw5gyNMtCUDl5pMW6AiaLsi2RgVewWLcm/loLuUWm8Dua4shn2Kst9AQZFlpqVmsrmmIqg0W5YkLsjKO+3rnZuexWvX3hI+Awc4Sbb4Hh2lGEvUkUZ+2NrwTNACG0AXXrY2/h8Fsdec1jBEk74n0TaC+ek/4+Pa36IZPgQ6yfaxXJj+cH+bNqAoae35/aHKMnWdHd1OkhCC/5Qt49XyDwAwEFyePoe7869BMQf0srZhe7eDdDyqpLCn9RAtnr9S3rENRQ5gt4AmtvD9NSu5ueAbXJE/sGamnSmSJDEyZgEjYyIb8Hxo3Qd4tADTR5SS7mjDY1io8cWgCcHSqo+YGFfAlPih9bM3CS/mKiHMSJLEN889j7smnMPuhjqSnU5GJST3enySLZ4/nvMT3q5ahdtbiEzo0NZke3jmCPQHkiQxP/0XvFvxAIbQ0IQXVXKQYBvB2LjrkSSJguhcCqJz+9vUsPPIBZdy3eKX8Os6Xl3Doag4LBYemh1c7rW2YQcvlrxNva+ZbGcad+ZdzcS4kf1k9cDg1uwr+fOhl4IkZG2ylZuyj2VcG337COkeBjoDHjbV7mVm2sS+MNXkDMiJOp9hrtm0BSqwyq6gMjuTU2NichpV7jYEICFIcbURMBQ8gXgyo6K7j3unei2vlH8Q5Ai8W70Wh2LjttyBWckQTqJUBxKhTxSBwKftodqzA0Xueh7JMljRmZe7i5+uXcy8Yd/CZRkcvbZnC4YQFDbVcf24jYxNrKbrNyGhC5mVTaNp12FZzVrTSTI5I0wnKQJUeRp4veJDDrnLyXNlEuW4kExHSq/Hx1qiuSXnKkraY1hZ/TCG8CNJYBggsDA65u4+tD78JNoLuCHvNYrdH9Kp1ZNiH0+Gc9ppSYgPREbEJbLqpi/y8v5d7GuqZ0JSMosKJgap0X1Qs5EnD73cvdgvbC/lZ7uf5uHx95mO0km4IGUaAaHxQukSWvztxFhc3DhsIQuPa9aPUtN7ycIK7lu+nA03jzFnMA0AZEmJyAymocI3ps5mVXkxGdEVLBq7GYusIUsgk4TfuA47XYOaXyl/PyRT4jP8LK5cya05Cwd9z9GnsTB9Dhsad4bM9nEodgxjH4JQkRFdyOTF17O+qoyLc0aE7Df57MiSxIyMSkYn1KDKR0vbBYowOC/+IO81TMCrh2b+TE5EwEnaIoY6ppMUZg65K/ju9j8SMALoGBxqr2BV3RYenfR1Rn2aCo4xkZd2zWJ65l4SHB1Utsezrnw0S/ZvZem1AzebtLz0IL/95GPK2lvIi43n+9OSyHQNbgfpKAl2J5cNbyUt7hk6tTo+qIphXMJdjI69CYC/Fy8Jeen6jQB/L36Tx6Z8pz9MHjBclDqTi1JnEjA0VEkJWcRNSrybDyp/iCQd+/n6dZmt1bn4DZUVZYe5Ygj3dZkMDUYnJPPSFRexo+UrqPLxvUl1vF/xVa7LW4wsqbT4QwdxAvh0P5rQsEhDO6AwLnYEt+Rczkulb6NKKgKBXbbxi/Ffpcr9EoaQehy8GtBVs1wxQlySX41VCRZHkiVwygESLToXmCMhTM4Q00kKM08dehW73MzMuDKSLW46DQu72rN48tCrPDHl2yc99x97t1HcksDB5tlB29t8jexprGVc4ukPIu1v3inez7c+egfPkcbhfU313LdiMX+58Gouzh78kbUK90dsrPsV+lHxAKOVnY1PgzDIib6OtkBHj+eVdtT0pZkDGovc82MsyzWb2vYriLa/g1XREEhsqc5hWdEEbKqgzR8a+TU5e9jTWMuft6+nsLmBsQkp3D951klLl016R1bWYZU5USeQgNFJdedmMl2zyI/KpLC9NOTcFHsCFnloO0hHuS7rYi5OncWe1kO4VAfjYkegSDJRylUcbHsbccLQVSEkSlpSmZWR3U8WD25SnDZaekkW5bnSuCB5aImLfGZMCfBeMZ2kMFPt2cdlSbtRJANZAruiMTuuiB1tfoQQJy1ZONzaiNZD2lOWJCra2wakk/SrTR91O0hH8eoav9q0akg4STuanu52kI6iCy+7m/9OQeyN9NQzA13NwEdp8R1kX/PztPoPk2Afy9j4O4myZEXS7EHDuSmL+NoKC5LkxqtZ0EXXz9UQBnMyzRKus5WN1eXc8f4reDUNARS3NvNB+SH+tfAmzknJ6G/zBhwdWm3QUN6jCAy8etew69tyLueh3U8jTngm3ZZ7RZ/YOFCIsbiYlTQpaFuifTRTk77C5vq/4D+S2BBC4tW9F/DkxddhV4fWUqvR08mL+7ezpa6SkXGJ3DH2HLKj48J+n/zoS9nRVBYyT82qOPnRuB+hfsahySYmRxlaf7l9wOToym4H6SgW2WBSTAUGARR6b96ckTaMjdUVIXKtAUNnXGLvPU1nK0IIKtytPe4rbRu4in2nQ8cJk8qPohmddOjtGHCkofoYArqd5drOT/i4+ptHXgKCNv9hytuXc1HWc8TZzJ6lT2NeVh7npmWxuaYS/ciAQ6dq4bYxkxkWbc7ROVv56frleLRjz0EDgUfTeHjDCt68+vZ+tGxgku6cTpl7JZrwnLBHkGzvEjBZXb8TgYwh9G6BAllSWNewm/kpXRF5IQQ1nRsobl+KIQLkRC8kyzVv0PeXHo9fD3DIXYlLtZPtTO0OfI6Lv5HhMQvY3vARextbcSkTeO2qMcQOgoHfQgg+qa3k7ZL9qJLMNcPHMiGpZyXaCncrV775Tzo1Pz5dZ11VKf8q3MGLl93A1JTMsNo1Ku56Stwf0OIvxRBedCEhhMSa5lxstjUsGjY/rPczGXqYTlKYybT7epTyVmWZjkANMdbe0+63jp7E3/ZsIeDV0Y+kPx2qyuV5o8gagAs6SZJIsjtp8HaG7Et1RvWDRX1PtGUYzf4DIdutcjQWyUmX3lRw5FYANqVLlndL/aNBmSiBjiY62d7wOPMy/xJJ0wcEnZqXGm8TybY4oi2hMzhkSeJvl36et4sLWXxoL3ZV5eZREz+TDLtJMC0+D0sP76fO4+bc1CzmZOSGZbixIQSFzT3PwdrdEDqY2uTTyYm6kD3NL9IeKO+OuiuSndzoi7vfSavqt6IJARxzeAwhWNuwC0MYyJLMtobHONy2GP2Is1XTuZFS50zOS/vNkBB2WFH7CU8ceBUJCV0YpNnj+eWEL5HuSATArsQxM/UaZg68oo+T8tD6D3jl0G68WgAJeGH/dr42aSYPTJ4dcuyvN39Eq8+LceS9FhAGAc3gB2veY/l194TVLlW2syDrWb63/QGi5Cr8howqGQx3VrKu7lkKotOYGDcmrPccdJhzkk6K6SSFmWR7Hg3eHSHbFUnCoSb0ep7f0Gg32nnlyht4ascWVpQVEWWxcue4Kdwx9uxoPixta+b9soPIksyCnJFkRX264/aNKefxyOaVQVFhh6ryrXPmRNLUs4bJSV9jdfX3gsoBFMnOxMQv41BtTIkfxfbmA+gcaz61ShYuSZ2OZnhxB8p7vG6Dd2fEbT+bEULw3OG3WFz5MaokEzB0Lk6dxgMFi0JKLFRZ5prhY7hmuPmyDBdb66q4fdnLGMLAo2s4VQsTEtP454Lrz3jSvSxJRFustAdCmw1ibbYezjD5NBTZymXDnmV/y38paV+OKtspiP08+dGXdR/T2/DrowPB2/wlFLW9jnHcs0wXHmo6N1Dn2UKqc3D0f+iGl/KOD2jy7ibakkNO9BVYlRgOtVfyWOHLQUI7ZZ11fH/HUzw/48eD1kncXl/NKwd3dZfNC7pK5v+8Yz3XDh8bUka3urK420E6nsOtzbT7fSHDi8+UwvYKDnfGEiUHWJi8ExmBKgtyjQZ213+FUdFvYFPiw3pPk6GD6SSFmQkJ9xxZFB+L/iuSjZyoS7HIPWdPXi9fzd+K3wVAM3QuSJnIhvPuxar03Czr0QK8W1pIbaebKckZzEgdFvEH9NM7N/LY9rVdfVXAb7es5qfTL+S20ZNPet7tYyajCZ0ntq3D7fcTY7PxrXPmcH3BhO5j3JqHJl8bafaEXj/zQCXdOZPz037DtsY/0e4vw6GmMDHhS+TFLATgWwW38J0dT9AScKMLAxmJ4VFZ3JpzGbKkIEuWkHpr6MpEDWY0w0OF+31afYVEW/MZFr0Qi+zq3r+4YjVvVq7BbwS6W6U/rNtClMXBvcPNKeuRRAjB11a9SYd2zInp1ALsaKjmpf3buXvcmS+W7xw3led2bQ7qZ3SoKveMP/eMrz1UscgOJiTcyYSEO3vcPyNhbFfW6Dh5BxmJyfEFyJJMbeemHs/ThZfqzrWDwkny6c18UPEFfHozuvCgSHb2NP8fF2b+jSVVmwkYwaXwAkFLwM3ethLGxQ7O7PT7pQdDWgAAJCQ+LD/MnScEcZ2qpUdRHEkCqxL+HqFOzYuExNyEQqzyse+uRTYwRAd7mp7hnOTvh/2+gwpTuKFXTCcpzKQ7ZzA9+QdsbXgCTXQAEnnRlzM1+Vshx7b7i1lb+3f2NO4gUY2nwhcPSHxcvwtVUvjemJtDzilsrufGd/+F39Dx6Ro2RWVSUjr/uOTMI7i9UdTayOPb1+I74UH5y00ruDArn4yomF7PlSSJu8dN486xU/FoAZyqpduhCxgaTxx4jQ9qtqDKCkIIbsu9hJuyLxxUUbkM1ywyXLN63Jdgi+XZc3/C9uZCqr2NDI/KZHR0bvfnz4u+muL2JSGZqIK4W/rE9v7Ao9WxsuJ2Akb7kYWKg71NTzI/6wVclq6a9pcrVvYw0yXA0sq1fDH/SuQ+7JHQDYPVVcXsaqwhMyqWy3NG4RjE85cOtTbS6gsdeu3VNV49tDssTtKDU86jwdPJ64d2Y5EVAobOjQUT+crEGb2eU+NpwqP7yHalmpLLn4GvjLiOvW0ldGgevIYfu2zFplj5xsgbAFBlFzIyJ+abZFQsgyRos6vxz3i0OsQRkQtdeNGFj811P6fRN6fHDImERGugZ/n0wYBNUVAkOURUSpYkbD04PbePmcKftq8PcqwsssKCnBERWaOMjc1DlTxEqaHPJFkyqOz40HSSTD4zppMUAfJiFpITfSk+vRmLHIUqhzZulrQtZnvDo2iGn7FRggJXBaWeRJY1jsNvBFhZt42vj/wcTjX43K+tepNWv7f7Ud2pBdhWX8Xf927hvgm9LyDOhHeKCwkYPZViSLxfdpA7x0791GvIkhQycfzpQ0v4sHYrAaEROPJAfbFkOUnWWC5Mm0ilexl1nWuxq6nkxnyeKMvglFFVJJmpCT2Xgk1OfhCf0UJlx0cokhVd+MmLuZpRg9hJ2tnwe3x6I+JICaIuPOjCx7b6/2FOxlMAtAVC+9yga8aUZuhYlb5ZJHcE/Nz43r8obmuiUwvgUC38z+YPeXXhrQyPTewTG/oaCakXTUbC0pMEXSWSj85ZwA/PnUt5eyvDomN7bYCv9Tbz011/p7SjDkWSsMoWvj/mJmYljQ2LLUOFRFssf5v+Yz6q30ZRewU5rjTmp0ztfgdlRc1la/1vQs6TJJnc6IV9bW5EqOxY2e0gHUPQ7CtkRsId7GgpCgnOaEJjTExuWO3Y1VjDu6WFqJLMVXljGBmXFNbrnw5X54/hyZ0b0fTgNYAhBJflFIQcf9+EGexvquf9skNYZBldGIxNSOWR2Qsodlfj1rwURGdhC1PViEt1cHvOFXR61/a4X5HMEl2Tz47pJEUIWVJwqD0/2Px6G9sbHsUQvm4VPItkkO1oJMfeSKk3CQmJdq0zyEmqcLdS7m4NWaB4dY1XDu2MmJPURU/LohMlB06dgKHxbvXGkEGqXsPPf8reQ9F+Q6dWgS48SKiUtP2XaSm/Jc11wWe848BEkazMTnsEj9ZAR6CKaGs2NiX8UqpnEzWdH3U7SMcwqPNsQggdSVIYFT2MXa2HQ85NdyT1acnmX3au40BLA36jy95OLYBHC/Dgx2+x9Mo7+syOvmR4bAJJdiflJyhXOhSVm0ZODOu9Ym32k6qDGcLgW9ueosbT1B3l9+h+Ht79T56d/m2GOY/NVeoMVFLU+i9a/YXE2cYxPPYWHOog67A/Q+yKlQVpMyAt9F1ikaM4P+Mx1lR/ByEESCCEzoyUn+OypPeDteFHknovB7sw9VwWV22i1tuM/8h7yy5b+fywucRbw5dJ+9UnH/Li/m34dA1Jknhmzya+PeV8vjRuetjucTIMYbC7pYTmgJvxsbnkxSbw0IwLeXjjChRJRgJ0IXh87hXE2x0h56uyzJ/nX01ZewuFzfUMi4oj1iHzwLYnqPO2oEgyBgb3F3yOhemn95mK25r494EdVHe2My8znytzR2NTVK7OupR3yl7C7d/z/+ydd5hcVd3HP+eW6TvbW3azm95JDxBCDx3pPSCIgqigNF8LooiKDeRVmijii3SRJr0ESIOQENJIr5tNtu9smz63nPeP2czu7MwmMSQQMN/n4SFz99x7z7n33HN+9ftD9CroqwoXQ/znftpH8uXHwXC7fnFQSfoc0BJdhIKaEbbgUGyGe5rZFivCoegUOdKJEbr3pazYn3P81EEjuX/lh1hZ4pJPqto7GuqIFcfuJ1G43PEJYaMWm2SImcTEkiZLW27lFM+7KOK/b9q6taJ+le4vGwTZBRWBYOcXcM3Qs/j+8vuJ2waSZJ6cQ9G5bvhnuyE+v2VNSkHaCQmsa2+hPRbNKkQcyLClzdL2TWwNNVLpKeLQwlEZoWtCCB48/mwueeNpTNsmYVnoqsrhZQO5eOSEfq68f7Cqs4b2RCgjDMq0LV7a8QHXjjgLgI74WhbUfwNbJpCYtMVWsK3rWY6qeBS/Y+hn2uf+kLAsnt38CS9uWY1T1Zg1YiKnVI04oEKPS9xTOGvwW7REl2JLkxL3ZDTlizXHd4VBOWewseMp7F5FYQUqJe6p+HQ/902+kZfr3mde6wr8moezKo7i8KKx++z+nwQaeWzdsp5QNSmxLJO7ls3j9EGjGODtP7R9X6A+GuDGpQ/SZUQQgCEtzq88im+OPI2Tq4czZ8dWVEVwfOXQ3VKbV+XkUZWTlwyj//A3aYYMTVjMq78TJRHDpbqpzjmPKv95u9zb396+ke/OewnTtjGlzTs7NvHQmsU8f8pleHQHxw24izn1VxMzA93mW5tS96GMyLt0Hz6hg/hvw3+ftPk5wLRtNKVH0FCEg2zqji3BlApORedbw85E7cPSVenzU+bNoaZPjSGXqnHesHH7rL9ru7bzVsNSbGlxXOlEJuQN5roJh3Pvig+xbBshQBEKP5pyDBW7yEfaFfyahxzdQ1simPG30b5ASkHqDSlNuhIbyXPuP5Yy07b5+9qPeGLDcuKWyalVI7l+wgzynF8eQeBAR6XvFGqDL2PT42UUaAzwHpeqxzLSX8W9U27giZq32BDaQbWnlEurT2Kk/8sZkrmvYNoRpDTR1czvNmRGuf7jB6iPBjBtC13RyHP4uG/KdRQ40y3lYwtL+fDCb/PGto20RENMLa1kcvGAz1ygb0sEsxqOLGwaY22p3ytaf4Ule0I0JQamNFkV+D1HlP/lM+jprmHZNl+d/U9WtjakyCqWtNQxv76GX08/+XPuXTpUoVPm2Z9RC58fxuZ/k0BsJe3xdUhsFFScaj7TSn4OgEdzcVH1TC6qnrlf7v/Gtg0k+iFJeGf7Jr46at8x3RpWF2vb76E+9AYA5d6TuXezoDnWmRYj8sKO9xmTV81RxeM4f/h/Lmes7aqlPRFMKUgCyQVlSyjSQyQsm4QFa9r+QEt0IdPK/pi9r7bF999/NS3PKWIa1HS18+j6ZXxr3GG4tWJOGfg8LdElhM1GCpyjyT1YS3APIA96knaBg0rSfsSzmz7hzmVzaYqGKHR5uHHCUVw6YiIl7uwbjI2KIabwi0MuY2rByIy/CyG4/5izuPiNpzClTdQ08Go6w/LzMJ3NfH3R3ZS68rmk+ljG5+0Z046UNh3xT7BlnDznBB6tmctT2+YQt01A8nrDx5xSPoWbJpzLqYNG8ua2jahCcEr1CKr9e0+rKYTgumHn8Lt1T6VC7gQCp6JT4iolYmTWSZHYqGL/KivfmfsC8+q3phbjxzcsZfaOjbx15lVfymR8y7ZZ39GCQ1UZ6i88IKzWhxTdSHt8NWFjO7Y0UYSGSythYvGP09oN8pbzk7Gfb0jbuUPG8PDaJcStHm+SAEbnFx9QXqSY2cLKlp8QiCUZynyOoUwougO/c1SqzYMbX6E23Iwhk2MxLIt4zODudc/yqwlXZlzTozs4d9i+s6LvDUb7qzBl39BMcCk607rXUCktOuJrspwtCUSX7uce7hnerdvMJ4HGNDa/qGnw3JZVXDV2GkP8/ZeP+KJAGmuRkX+AWQfOGQjPLISyfz0j/ylUxcWxAx6iLf4J7fF1+PRKSt2H7TIMb19CVxQUoWREWQhIM7R+WkhpsaD+CsJGLbLbGLU9+AJH5bnYHD6c3rWyYnaCF7Yv4KjivTPEdhphlF7XG+xuoVAPo/diorNkjJboB3TE15DnzMwlXNvWjJmllk/MMnm1Zi3fGpeUqYRQKPF8NmGJB/HfgYNK0n7CS1tW89NFb6Y2vUAswh1L3kUAl46cxPTy/2Vhww1AUviX2IzLv5ILh31rl9cdW1jK+xd8m1e2rqUpEmRoXi4PbX+Ol+o2YEiLTaEGPm7bwM0jJjLc24Gu5FLqPQVHljyWzvgaljRdi2mHESjYWCxqGUPMLkm1idkJ3mhYwqkDpjE6dyDfGb/vktGPKZ1IrsPH4zVvUx9rZVROFZcPPhlNLmd5y+2pgoVJCNxaOT69ep/dvy/Wt7ekKUgAhm0TiEV4aesaLhr+2YYS7W+831DD9xa8RMw0saWk1OPjoWPP+1yThCGZ+3B85VO0xpbQldiETx9EifuwlBfpQMJ1449gfsM2tnQGiHYTNzhVjT8edUZGW9O2eLxmDs9uX0DYjDEur5rrR5zFsJz9m88hpc2HDVcQMeugO9crmFjPhw1f49iBr+PoriHybtPylIK0E5a0WRhYiyXtA5IxrtSVz6nlh/JmwxJi3Qn1DqFR6MzlpPKdLHsKinCk1ffZCVXJLED8eWBe/VYippFxXAALG2u/8EqSHX0LOr8PJAAbjOXIyJNQ9CJCSY7NlpIFDTVs6QowIq+Y6aVV+8RoI6UNdiMI3x4pZUIICl3jKXTt2/y6PcFXBo3iwVWZJAkSOHFgj1ekLhIgYZtUe4v3ismzKTKfqNmQUpCS9zDxaVGGeFrZHClJax8yM5nj9hRjcqsxZM+eWulqx6FkGjaktGmLLcuqJHk0PSuzICSNNWnXMbcgg3dB4iNQ8sDzdYTn4gPCAHgQXzwcVJL2E+5aPj/NKggQtQz+uGIBl46cRLF7GqcNepuG8FxMGaXUPR3PHia/+h1OZo1M1if6w7rnCJuxXvScklOLFpGIvcLGhEQRDta33cXk0gcocPdYWCyZYHHjVRh2V9q1Ty1aRn3sSDrMnpo0cdtkQfMqRvsH7sWT2DUm5g9jYv6wtGNSnkwgupTa0IsINAQCTfFyWNmf9utCtyLQkJWdK2IaLGra/qVSkhrCXVw15zmivQSzbcF2LnrrCT4877p9Us9CSguZWAhWI0Ifj9AzmZD6gxCCYvc0it0Hdl0cj+7g36dfzvz6raxsbaTSl8up1SNwZfE63rnueWY3Lk95Tpe1b+HbS+7nkcNupMLTY3yImc3UdD5Ce2wJHr2Kwblfx59FcNhTBGKLiFut0IcMw8ZgR/AFhuR9vft3diFESplK1D8Qcf2IcxmbO4gXdiwgYsY4pmQCF1Qdg0tNCk9CCKp8Z1EbfDEt10QRTgblnP95dTsNhS5Piuq8NzRFIX83uR8HOqQ0oetWoLegHQc7gAz9FeH/EW2xCBe8+QSNkSCmbaEpCtU5+Tx90iz8jr0fvx2fi915C9idgIVwHImSdxdC2X0h9M8Dw/KK+J/JR/P7pXMR3VmYEvj9EadS5PayPdLCLSsepT7ahiIEHtXJbeNmMbkgmVcnpQTkbg1KXYkNfYyQSWjCotgRTFOSHIrGsSV7rzDm6l4uqz6BJ7e9S8xOELKcmLaCpvSlFNdxqcUZ5ydsk+2xekr8Fg1Bg4ShsXMxcms6l4/sCUGUVh0ycD5ShpMcnFYnZtevUcxtaLk/2usxfKkhgazsxQcBB5Wk/Yb6cFfGsWJHhFkVi4kHPkLRx6B5L2fgp6RO/ahtQ1r9gjHeeoZ5WlKWGru7qO3y5us5tmp+KjGyJTIfO0uYiiIk43N2MK+9J9xPFco+o+vcEwghmFB8C8PzriAQW45TLaTYPW2fhjwkbBMFgdYr76vC6+/eltLhUFQGfYrQwgMRz25ehWVnWivjlsV7dZs5uWr3Ck0gFmFDRwuVvlwG+tI9ldKqx2qbBXY7SBuQCOexKHl/RHzJiDcUITimYgjHVAzpt01bPMjbjctI9ClGmbBMntw2l/8ZnSSciBp1LKw/P5k7hEFXYg3NkXeZUHI3JZ5j96p/EaMOMmhiwJZxQsa21O8ZRWOZ07QCq08x0UR8nWUAACAASURBVEn5w9K+kwMNQghOLJvCiWX9lyIYV3gTUbOelthHKOjYGJR6jmJUwa499/sCgXiQf26bz7L2LVR6Crmk+mhG+CvS2pw/9BAeXLWIvr4kVSgcXzmMLzSsGiCR5Q8GxN8BfsRPF7/FtmB7ai+L2xabOgP85uM5/Gb6Kf1eOpiI87tl7/HvmtVYtuSEgcP56ZSZFLt9SGM9dvu19FbOZGIBVvs30Qr/uS9HuE/xjTHTOK16JLO3b0ZXFE6sGk6hy4NpW1y75EHaE6FUzlDUSvCDFf/HY4d9m0D4fhpDryIxyXdNZXThz/E6sofde/VKVOFOy9MDUISLkJmDgsBG4lJ0St0FnF0541ON6auDT2SkfyDPb19AzCpGVbaRPicEitAp9RyTOhJObKU2tJxfr/2AxrgLXCZFDolp6IQ78zGlzYXDxnNadY+sIkMPYdkR1F4Md5qIkwg/Ct5r0LQv1z5+EPsfXy5p5QBCVU4eW7qSicNCWIzMC/D4xFdxCAsSNnZiEYnIE+iF/0TR995KnO/IoT7ak6A8IWdHVle2LU064yvJdyWtLobdSTbBSRUSt5q+oSlCMLN04l73cW/h0Svw6BW7bSetVsyu27FjbwMSxXUCmv/noCTDxnp7n2pCzfx69XOs7qxFEYIji0fzwzHnkufwMr2smkKXh1jYwOqVyKgrChcN++xDL/YnmiLBDFY2SIZXtcbCuzzXlpJfLHmbpzctx6FqGLbFtOKB/Pnoc1O1sKyO74FVT+85JuNzsCNPoHq/nNTYu8L2SCu6omUoSRY267p2pH5vbL8Hww7S89xsbBljTettFA98b69CDnvnHfWGKtzku3q8o9cOP5NVnTV0GWGiVgKX6sCtOPj+6APD2/JpoCouDi+/j5BRS9ioJUcfvEdrS29IaWMbS8BqRXFMRqhlJKwAdcGnCcZX43OMpiLnYpxajzW8MdrOlYvuIWrGMaTF2q4dzG1ezS/Gz+LI4p51v9KXy/3HnMUN819BkvTc+XQnDx9/Pq79VCT8M4PIAZlJRgCAkostJW/WbsgoVpqwLV6qWdOvkiSl5JK3n2BjZw8F/+vb1rGkeTvvnnkNeuT/yFTODDBWI83NCO3AYDXMhnKvn6+OmpR2bFFgPTErkVF4w5IWHzd+A5eoT4XPtcc+YnHDxcyofCMVTtsbZZ6ZrFb+gG3FkN1rjUDBofq4ccyveKluCa3xTg4vHM1J5VP2iZH00MJRHFqYXIvaYifzcdMPuiNZJC6tjGml/4uqOLHsGCuav0t77CNils0lZRZbo0U805g0lHqcFjNHDuDakadS6Uv3CJrxJWgiU66J2wof189lRtXZn3ocB/HfhS/46nvg4keTj+V7819COELk+mPcMmghXtVI1UUCA6SB2flTHEXP7fV9ZlUfyy9XPZWKx981ehbXQtehqcUxHS62x8rxqE4kEkva3DDibCo8Byb9tJQGicB5YDXAzirpsbeIxd5jSdwGNLaGxnL/unGYuPDndmJJM+lhlpIFLeuo/eivPH7EDShC8Mwpl/K9eS+xrLUeRQjKPTncfeRXKPV8OSrK78SM8mqe37oqax7EtJLKtN8tsS6eqJnHx21bKHflUaiW8szmT4jbFvFu4WRxcy23LHqdPx15FtJqAWMNmUp4LJmH8F+oJA1wF2DYmYKiKhSG+spSvwOxhWQzXhh2F3GrBdde1PXJc44jzzmJ9tjSFGukQENX8hjg7fFkFzhzePTwHzC3eSWbQ/VUe0s5rnQCbvXLU4zRp1ftVVFq29xOou1SpN1GskCQgXSdyZLgu1h2Akmctuh8tnf9gxz3vdRF/IzKL+Gf298jZERToYwSSdw2uHPN8xxx9Ki0fJKZlcP4+MLvsjLQgK6oHFJYts+K836eEGopUh8PxnJIK9TqRniu7CZrzh7qaWUpExGzDJpiHWzp6KAm2JZm7DGlTWcixqvb1nJWXg3ZviWEjrQaDmglKRsCiWDWshnFeisaPQpSEhIpYzR0/R9Ved9FiHQlR1WcHDngMVa03EYgtgSAQtcUJhT/Ao8+gBtH7V+W0ALXJE6oeouQsRVFaHi0gQghkHaQxpavM8BcRpEqqUcjKBUGu1uZWbCWNwPjMKTFlmhthoIE0JwoplTdgNrns9GFxZt1Xcw4SH6aHQfZ7frFQSVpP+GkqhHcdtgx/GnzczgVg6k5Tb0UpB5IY3mqQObe4KjicVw+eCb/2DobTaisDldT5e5AE703I4kiVHKdPVZjj15JVc6FbA8+m4pNVoWbXOdY7px8L4sCG7GkzWGFo8hzeDlQYcfeAbuN3puvwEKRFgWKSsCWVHhWct2oWv6w4UQM26S33GFKi4ZYO8vatzK5YAhlnhyeOeVS2mNRErZJidv3pUz4PHHgCIbnLmJ9R0uKqMKj6Zw8cAQj8nos4U3RDr668F4iZhxTWmwKNiDlOmzVCVZPwuwwfy1jcl7lw+1/osg5huqsgYuA3PsE4C8yil25HFk8hgUta9K8SbpQmTWoJ8REV3JJWJnMjkiJYm0nHnk02c79FRR9z6nwp5bdz+aOh9gefA4pDUo9MxlR8D3UPjVunKrOSeVTgP5D1/4bkWi/Gmmlhy3K2HP4UWjrZu6ySWDbCRY3/IgH1p+BKW2KC7qyKgBBM0ZLvItSV3qYqkNVmdrHSPFlgMi7B9l+FZhbQWggE+C5DFynoQjBjLJq3m/cht1LWFOFYGblMAw7adTShcojW97jH1vnoCCIWya6W4Ogk94Jc141QF1oHnbxeBRjJRneJBlHaPuvjMT+wrjc6qyqZLkzhqC3h1kyUDUYoEUQsfuIND2M7vsuuveatL3Mow9g+oCHsOyk4URVPltjiBCCHEdPiLK0u4i3nk6utQNVTcrtBWqCzaZGMxqTc2t5M5Bk2MumPAM0iovJtxfhVnvW2JilMi9QgaKVZT3nIA5iVzioJO1HREQnQsDpxZ9gQz8lMh3Ap2ONumzQ8ZxbeQQ14WbyNA+b239K1FgExKnSDEqVGIqIEGk9G3fuHSjaCIzYKwzRnRTnX8q2yAYs4gzwnU6F7wwUoXNC2aTd3vdAgDQ3Q5YEVBXwCEkA0BWbUlcXw/3NNNqZLH9SSnZEWplc0LNgH0j0zfsDmqLwz5Nm8dj6pby4dQ1OVeXSEZM4e/BYFrVuYnHrRvIcXjYE6wkbsVSeigQQEr8vTkubDgiOKtnAZUM+xKlaxC2oi9RR5rBxZUxrB8J92mc70AMIt469mD9vfI2X6xYRt02G+Mr4/qhz8Kle/rByDh+37GB68RRG5tQhe+VRCByMdJVgtF1KUuATmOG/ofuuxZHzvX7vJ6UFKAghUIWDEfnXMiL/2v0+zi8bbHML0qylr1dCwaZMlbTZvWvgwTB/IyEzDghipo2WZZe1pY33S+Sh2x2EWoQoehFprAO7GfRxCKUAKS2C8VX8fEoVF73dSMSyiJgGHk0nx6GBq42j3/45ABXuPFrind3lKZJwOBP4bAiFXbjUBN8d9Q4j/U1oipOlbQaTXAoqKinSEuFGuC9CqPuOpfWzwhBfGUcXj2V+y2pi3eQvCgrbwz6MAgtnt4BRoRoM0Mxub4oFMoQR+iNC5KF7L8647metHPUHM/wI0mpJeYGESO7jQzWTVktFE8l36FA0TizLHv4/ofwEbpp7LLcNm0eullSOl4Xz+MX2Kfzl8EM+i2F8MXHQk9QvDipJ+xEho4sx3nqm+WtotqBMJc0NbEnQveftE0+FR3ORMFUumPMUnYnBVHl9/Gz4e5Q5o90qmMQ2VhJuvQBwgDBBRnEKD6PUapaad/KzpavoSDzOyZUjuWzYFHz6gbF47gpCGwrCDTI9j8YCIrLnuQohKdJD1MfyMjx6NpJ5zetZ09HIqRUTmJC//2jGP290ROdT1/UQCauJPNeRXDHqW1w9NlljwrQtbvr4UZa11xC1EjiEiiWy5xIIIVEVG6Tg4sEf4VR75zdJNhgK45ygoAIJEB5QilG8397/gzxA4VA0rh95Jt8bcQaWtNEUle2hDk567a9ETYOEbbGk1cE5A4dxfPl6NOFMEgw4R1LIUkgrsGxhhO5Dc5+Jog1Ku48ZX0ik86fY5gYQHpyer+Hyfz+DMMOybZa0bidiJphaNJCcT8Ei9qWGDINQyGbG17IcNGyVnZ6NUEQnN8dK817rQmV60ShcqoMtwWZydBfFrgOrXtD+gtBHAcm8lM7YYta3fAcp40jggelutsdu4pP2QkblF/PP+tksbd+S8hpsjwTou1UKAR53glDYyZVD32dkbiMOxQYiGMDKuItRnvG4ZR0ouQjPlSjuCz7LIe9T3DruYl6r/4gXd3xIU7SL+i6LunAO2woKGORtxaFaVKQUpF6QUYzQPWieizATH2JE30Qobhzuc1H1z7/gqmXVY8Veg2xF5AE3ks3RAqSECnchXx18XNbr2NJmXPEalhsJvJaNBcS0Nr5a9SGbQrUcUrh/yy0cxJcPB5Wk/YSu2CImOX/MhHIbl2LSYoFHCHKVpD1SAEFbUOT7/j65X2ciyuXvPUXYTFpP2mM6Q10BFPom58eS/+3c22WYhLGBT+puZV5jkiJ8Q2czz25dwUsnfQOP5uBAgJQ2RuJDLHMbmj4W3ZEkUlBcMyFYAFacnSF3tgRTQpvds1NIBNu6CpGaQCoytdkqCBK2xZymtYDCy/VLuaR6OteNPBkpbRq6HqYx+DCW3YnPOYXq/FvxOLInwh/oaOh6jG0dv8Xu9rzFjBpawi8xccBrONRi3mxYwdL2rcSspJUyIS0UyBBMdkIVGrmOTtQsibJBKfnEKGFi3oVg7UA4DkO4T0eI/y5BPGjEUIXAo/UYHIQQaN3htXeueI9gIpbyUVgSnq2dyCedh/PYsUfh1srR4i9iBBdnubqNFXsLxffN1BHTWE2o7Yoe76oMEw8/jG0H8ObfmWq3tr2JK+c9RcQ0EEJg2BY/nXgilwyb3PcmX2h0JWLURTqp9OaRk8XoY9sdxEIPk4i9jaIW4fJejcN1TFoboY0iWxyAjUrATneXGpbCwuaeXJdo3IFbF+R4DXRFw7RNxuVWc0TRWE589zeYtoUpbcblVvK7SZdQ4PSlzpXSpjM2h87oPHS1gELveTi1/4xs4kCFYbWztvnr2L3Y1WwZpsJ5B2dNfZ9FgQbaa8L9hlX1hhACr2YztagmrUApQEzGWBcPMK3yg30+hs8KUkqWt29jY7CJgZ4CThswjTMqDmPGi/fSEQ0CcO+64zm3ainTi7agOjMjKwCk3UKk4waM2Bvd64NCPPQ3XP6f8kLTOB5av5D2eISpRQP5n/EzGZGbSce9L9CViPFy7Wp2hDs4ojDKOP1ObGsHTuxM5Y6krBSxNZ6vn0jC1IgmVBY0zWNGUQlufSSK0rOnrAy8wTBPUlHead5zKhbDvY3MCbzGOdWH7ZcxHcSXFweVpP0Ay46wrunrOJX0WOgaUyJQ8Ss2fsVAVySBponUxCfzZvslHFk+kWPKhu0yWdewmgjHFqOqeeyIDeLvW95nc7AZl+LC7pW4OdDdSUKqODOUpExowuTk4vXcvTWpJMUsk4ZIF//asoKjBwzi3ca1qELhhLIxVHoLuscYxrY70dSyfVbkM2TEeWDtAl6uXYVAcHb1IXx79JE4RScdgfOxrHpAgpTojsnkFj6KEC4chc/2YreDdqmw2ZApFiDDVmiI5LIpWIJDVSjNtVC1BAqCsGWSsCR0V6SIWQZP1nzAGZVTkLF7aQn/K6VUdMXeZ3XjeRxS/ioufdA+GfNnBcuOsa3jd6mxAEgMLLuLus6/MLjgVl6tW5ZSkHbClmRJglWZVDCEspJqVgY2offz+oVajrqLcLCdMBKrCIcewDQ2ojsm4825Fk1Lz7CVUrK6o5G6cAej88qo8n1+VK6WHaE9/Dzh+GKc2iAKfZega+kWyk3BJn624nk2dDUCMKVgEL+YeB6lfTwGC5q2ZqVP2RJMoCrj8OhuEnGNZEhu329ZSeZ39EI8eE+WvK8YRvQF7NxbUJR8TNvmirlPEoin0//+avnbHFJQzriCTGurYZus6qjDqWiMzh1wwOfpmbbN7cve4PmaFTgUFcO2mTVsCj8af0JqfbXtTjpbTsa2WoE4lglGYhGenJtx+3qowYXQ0XN/h9FxI2CQfA9uFLWcGANQ7JUINGJWjM2hQp7ZNjV1roLC4fnjueOwk9kSaqLYlUvQiHHVhw+lQqYAVnbUcv2SR3lsxneSfZMGG5qvIJxYji0jCHQauh5gaNED5LmP/wye4H+O3uGdu0Nr5GWykiogCYRfZ0d4IGaWEhVSZhptSpx+3jv3G3xc/yQyC9W4aWeW40i/ZpRY9HUsqw5dn4DDeWTanmZLSSAexqc5cWepf7YvIKVFc/BvtAT/gSXD+F3HUJH3QyyK+fbi/2NjsAm7u6BzkTOHh6dfTUei5ztP2DpP1xzG0zWH8sZhT1LhDmXcQ6gVJGJvQEoxtQCLUOfPeWDV5TTEk2vJnIZNLG6p5d8nXsWgnAIsaWNLG1359OLi2o4mZr33KKZtoxLi4vwnMM0EigADgSLVtPljSWiMu7l9w6mEbBc5WozzSx/DbwRY3ajjUBQq8m6hOOfy5BjtT9CEgVPYSClIIACBJiwqnHWfuv9fTsjkRn8QWbHPlSQhxDeBS4BJQC4wWEpZswfnnQf8EhgKbAZ+IqV8odffBXAb8E0gH1gEXCulXL2vx/Bp0Rp+EwcxvEoCG4hIlYTcaYm0yFcTvQRPgwr9I2Z4arjhwwuo8uVxzqDxHFkykAGOtUgZxe08Ck0tpKHjLlq6HkQIDUtKugzJ2rZT2BHPRQCqC5SIE9tWqI3mJenG9xCyT5p9zDJ5cusH3L/pNSxpIxA8sP4dbh5zHNNz/kVX5FUQCorwMSDvl+R5z8i4pmk1EYq+gxAaPteJqGo+UkqWtW1ndsNaHIrKVyrHM8xfgiVtZs15lM1dPXSuf9+wiA+at/K3iXOxzK30JmcwEksIB/8Xn//HCLUYPf++1N/yzTr8gV/SFl0AaGzsGsWf141BV1ROrhzFL6adTK7DzYMbZvO3zXOyPAvJwuYlDFGezth0bRmnvvNBhhT9tt9naZjbCIWfxZZBPK6TcDmnpy38TdEg85s2oSsqx5WNyCiWKKXko9ZtvLpjNQqCM6oOYXLhwF5/NwmFHiEUeRQp43jcZ+HPuQ5lF9Xko8amPsm9O8dq0BGbDySZ1jIhUFFQlGS4mGFbTMwfxK8nzMKnu4CZrGleSyD6DrbsCZVQhJuqvN3XoInH5tHediXIOGBjmuuJRV+gsPgVtO7is+3xCF+f/yRbgq2oQmDYNidWjOLOQ8/K2uetwVaWttVS5PQxo2Ro1ho/ltVGJL4QRfHicc7IYH/qD6bVxobG0zHtAFJGEThoCf6VISVP4XUm8/i6jChXfvA3QmYs5bBdEtjKlR88xMvH3ZjWZ5/upDORjcxCpKifNddpGMG7s/ZHdaXXWbOMDWSNCxM6trkDxZHPh801KbKO3kjYFk9vWcav+ihJc5rW8ZNlzyJJzk2/7ubeQy9jhL8nEVpKSXv4X7QE78G0WnE7DqEs9yd4nHteOsCWMUKR10iYNTj10fjcJ2aECMYtg5e2L+fthrX4dRcXDTqUaUWDMq51z+q5vLhtJQnbSq0nT29eSrHTx9WjpgMQCz+aUpB6BhIl0nUXTs+lKEoPo6XmPgVFG4oZeRxpNaA4j0XznMME4SaU2EDY2ERDJJfvL1nITuHTpWp4NAc/njyTXIeXSd05j7eu+FdaXg0kWdm2hJrZHGxiaE4pgfALhBPLUkYNiYGUBltar2di5ccoYvcefsOsIRqdjRAOPO7TUNVMhtItwVZWd9RT7s5lSmFVVgVnXWcTD2/4gC3BAJMLK/n68OmUe3qYxQxjPR0dP8RILAE03O6zyc37Zdrz6wvTak9bL3bClnEMu51hOdMyvm1JMudrZ4FVAKeic9PoM3DpRTi0YuJmX0FYIc91RP/9MDYRaD0bZBwpYwjhQtNGUVD0T4Ti4e26ddy+/HU6E1EkcFrlWG6fdFqastQWD7O4tQa3qjO9ZAiOXsqEaTUTiy9EUXJxO4/st0bctsD3aY++hux+3+2Rl+mKzWVu109Y19WQRvZSH23nl5+8wOSiCt5vqulzJcEjdSfxk2GvkV6814VUB4O5OePecQsm+LfS0JIMu5NA1DS4Z/VcnA6Lt+pXY0mbsXkV/Gz86ZRqC2gPPQ3Y5HkvIs97YWpcUhp0RV4lFHsXTSnB4TqPR7Zu5436VeiKQkfUIGjEAME5ZZtQhZ0Kf7eRGNjoqNg4iVsWO6I5XLf2BEKqAyHgO1VzGOJuQVMkYGFLqO/4JdKsoSZazdrAUo7Lj6Z2OgtBo+kkJHXG5O19qZWD+O/F/vAkeYC3gH8D/7snJwghpgP/JKkEPQ+cC/xLCDFDSrmou9kPgJuBrwHrgZ8BbwshRkopg/t0BJ8CHfEgNS0/pcwRQRFJy5dL2IRsm6DU0aVK0ircY0VzKDbDvAGqvfVsDsd5o/ZxJjteZ5tQcKoqSBOv5yJaw88jiSNlHAHkaHDjoLe4ef35SARCgNNpEI06aUl4ebt1CDOLtqYxvfS9N0DcUnm1OT0uWRWSZqMNu3dbCZHQLXSq9UACJFgyxo72m9C1UrzOQ1NN24OP0NJxO4hkfH4TP6I0/0/cvVHy6o5PiFkGihA8uvlDbhp7AuWuAraF2tPoXOO2SU2wkUR8DoK+Ql2cWOSf+Pw/zngHbq2CCaUPpn4fB3xrvI0iRJqXzqnqaELB6GOxVFDwKq0owokl+1omLcKJFRn33IlQ+HkCHTd3W1VNQuHHcLtOoLjgzwih8OimRdy1ajaqSFpcb5OvcPeh53F8eU9BvF+ueIPnty1PeXVeqF3BZUMP5fvjZgLQ2vZN4vE5qQ01GPoL0djrlJXMRojseWS6WpjmaewNh5qklT67cior2muI9vEmeTU3zx59E7XhFoqcfsrc6eQXI4t+w7rWHxKIvIvSrWwMyr+RIs/Mfp8TJAXrro4f9SHeMJEyTFfnLykoegyAHy95iQ2dzWnvaXb9eh7ZsIhvjJyeOmZLm1uX/pvX61an3rVbdfDoUV9jkK9HQGwP/o1A5x0g9KRpQOhUFD2Jy9HD/tgfGjvvxrCaoPtZShJImaA2cD2jyucihODVHSswpJWmqlhIOhNR3m/eyNGlPe/6iuFT+d9P5hLtpbQ4FJUTK0fi6hbEFK0Kh/82El23k/QeAdLGkXsHipqu0Kj6OGxrCxlWemmgdHvnQkYiK/OgLSXt8fRQnR2Rdn748T+J9RLSIlaCqxb+ndkn/ABHtyLXEryf5q4/peZkOL6QLS0XMLTkRdyOsbt+qIBh7mBb0xnYMoSUYYTwoqllVJe+jKok51vCMrl8wd/ZEmpJfRtzmzbwnRHHcuXwI3uGKiWPbvooQxGMWgZ/3/BhSkkyYu+QLQcCoWMZn6A404VrRR+OI/f2jOY+xwh8jhGUemH2V8bz5KZlbOhoZmJRBRcNnUieM50EpjHanlHrBkBTVFrjwW4l6fk0r29qbEjC8eXkuA7N+FtvtHfeRVfwfiQSIRTaOm6jqOA+vJ7TgaSn7fsfPcecxg0pZaTM4+eRI6+g2NUT8regaTPXfvgMCcvERrKus5EXalfw3HFXU+0rwLJaaG05i+QWLIEE0eiLmOYWikte6rd/ftfhKF0PpYXbASjCQa7rMCocgxnkLWZzqDG1J6hCJV/3ckh+JRu66qn0FPL1oTOZVJAsmDq88A7WNH+7W/myEeioiptB+f2HtHe0X4u029mpdkkZxjBWEQrdzxZzFjd/9ELaPHp9xxoiZoL7pifzmh7dvJC7V89G7zbEqELhL9MvY0JBJW2dd9MZvAe610QhXAwofgZHH1bKuLmD9sgryD45h7YdJRh9moSdLtyb0uaDlo387dBvsezdJ4hbJpaUKAgcqsrpw7+HK/88EsG7sc2tKPpoHDk3E4+8SLb9H5LRFr1hYzO3dRVgp9bdTzq289H2i5ic2wjsDNleQ1f0VaqKHkfKGDXN5xE3NyJlBNBIdD3E2sBxNESrup8vOB2CeEKn1BHqI5uAiY0pFTyes1kcOJwfLN+I6oygCINCPcQgd6BbQQKQuIWFA4PO0IN4LZWZ+VZazrGQkgFajC2GiyPKvpEx7oOgOzhn92Gt/63YN3FSvSCl/KOU8jfAgv/gtBuA96SUd0gp10op7wDmdB/f6UW6AfitlPI5KeUq4AogB5i1TwfwKfHiprsp0DpTH6oQSetXjmJh2Qoh0581h8OSggpXF7picsvQ1/CqCRxKDCnDSOKEIo93Lzw9UAR41TiD3YHUMVXtufbvNp/A/I5jQOQBGqrjUFz5f0UohSC8JAUuL9tjxTy8PZ3y16Hb9LVI+7UoE3K205dSVcooLV09npyEsZmWzl90K3SR5BhkjIa27zG3YTFRy0AClpTEbZM/rJ7N4pYaImZmqIRlJZD9Ma9kKDD9Q1OUjDDGk8vHZw1tlEgOLz4KO+v1FVz9JLradpBA+81IGSMpREukjBCNzSYam83GrmbuXv0OCdsiahlEzAQxy+Smxc/R1e1NWN3ewHPblqeekSQp3D26aRGbg60kjLXEYz0KUhIJLKuBSPTlfsfv1MrJcUxGkO4xUYSbCv/VABxfNo4Tyg7Bqeg4FA2P6sCrOblr8lfx627G5VVlKEgAquJmbMk9TB84n0nl/+KIqkVU+i/vty87IWUIy9qR7S8YiaRtJGwmmN+0JUORjVkGT2xeknbspe0rebN+DXHbJGoZhM0EgXiI6xY93XNeYjmBrt90z80Qtgxh2+3Utc5CyuxKZG90Rl+HLMqmYdZh2s0AbAu3ZoQtQpJuvj7akXbsayOm8ZXqsTgUlRzdiUvVmFRYwR3T0j1EuvdSOy6GqQAAHplJREFU3CXzcfh/isP/M9wlC9A9mQnorpzvQoai7MbhnYWiJK3/04oHYmQpJOxRdU6uTM+3+/f2jzGzfH+mtHm/ZSOQtP63dN3TZ06ClDGaOu/KODcbGttuxrJbkd0ELFKGMcxaWjruSLV5te4TtvZSkCA5D+5f/x4diV65LchUbmZfdCZ6+ijUEsiqLprJNXIvUOrJ4cbxR/Pno8/nmjHTMxQkgMOKhuHMErqUsE1G+gck+9avp0ju1usZTyynK/RAN0NiHCmjSGK0tl+HZXcC8NjmD5nbtJG4bRKxEkSsBNtCAX645PmeO0nJz5a9QswyUjTmprQJGwn+sOodACLhx5EyTvpekcA0V5NIrOy3j37nofidh6KInuejCDd5rqPxOSYihODPh17FOQMPxa+78WpOTimfwOMzvsvvJn2VF475IfdOuyqlIAHku49kYvmzlHjPJsc5iQH+rzF5wOu4+6mLZVmtmMZ6+u5zECca+RcPrltAvI+iHbdN5jRupCUWYlV7HX9c8y4J2yJsJgibCbqMGNcsfJyu6Bw6Q/el1hkpQ9h2Kw0tszIE0mhiTdb3LYkxyN2Qte8SyYi8Yl455RucN3g8I3OLObVqFM+eeAUzygajOo/EXfQ83rJluAufRHVMweE5nySbbjoUIfmgvSrjmCXNtHV3mKeFMb7t7FSQAKSMEIkvJhJfSHv4SeLmhl5yiolDMbl64BzU7qgWIUBRJIqwWRUqJWxmmcvCgeY+h1OHns2Sc2/klMrRqAj8WgxT9oisOhIHyfxiVUh8mono8y6T27tgXMG30NXMukoHcRC7w4GSkzQduLfPsTeB67r/PRgoI+mhAkBKGRVCzAOOAP7yWXRyT6BZc3CpmaEsEkjIYkbmnw2Jh0h3hSdpqjdHCpjk39FPoryNJsCQ6aFDEoFT6REavJqT6vwyBHDhkAmcPfSHaEq6Lqy7jsaMvo5t7UDVx1OScwgVm/7FjnAnqkguM8cNqGR+yzp6byD5egRTKuhZ8pzi5rbUv7siL2YVOC1bMiFnE7MD6YKYKgRBM4pH1Yn0ES6l4iEih+ATG/tcTcPhOinLc9pzDPDk85NxZ3PHqhdT1lRbSn414UKKPdV0eE6kPTI7jY5ZEU4qcrMztEXjC7prgKQflzJCKPICLzUoGFbms1NQeK9xA2dVjee9xg0ksoRC2UjmNW7kggFb0uNNUvcIE48vxOs5v9/xjiy+nw2t19EV/xgFHYlNdd4PyXMfBSQToH82/nxmDT6SjwKbydU9HFs6Jo10YFfQ1Xz0LNXd+0OSxEGFDC8hiG7vQTyLsrETfQXhp7d+lOEFk0B9pIOaUIBBvkI6U0JdH0iDSPx9vK5jd9lnZRfEEzu9eIfkVfLSjmVErPT+KUIw0p9eq0NVFH576OncMO5oNnS2UOnNZYg/u4CuqKUo3kt32T9VH4Gv8BminT/HMlYilFyc3qtw+nrmbKHLy/Xjjube1QuIdSvjblVnVF4ppw5M/zZbY6GsuSG2tGnvVkwMK7sgB5KYsWqX/U1eK04k/gGZOVcGwcjLlBUkCSfeaViT8X4h6YFZGqjl+PJk31WhMMxfxKauzHpTY/J7PG9u79UYsffoLfSBiqoORtNHZpy7r3Bh1eE8V7uY9kQk9Wxdqs6l1TPIc3gAKPbNIhRfkuFpUYUX7248nqHw89nnOCrR2Gx8nvN4auuSDEXekpIlgVq6EjH8DhcdiSjNsczcFhvJhy1bATCMNWT1xqFimptxdBPs9IUQgtElf6U5/ALNoWcRCEp8F1LsPSsV8ufRnNw8+gxuHp0Zyt0fvI6RjCz+/R637x+SbeH2rHWJHIpGY7SL57YtzbpWW1JS2/YgniyeQFuGiCc+xuWcljrm1KqRWdZA0HBoQ1GFkkZgIRCMyxuIS9UZlFPAbw87fY9GpDkm4Mq5nljwj0m2xm6v0rOtV2CL9HXYoSk4VJE2R8b4GlI03L0hZZRw7APC8XkZhpJkf2GIu5WNkZ5C2Ioi+aCtitpoLkM87b3YUV1o+gQ0R9JT6lQ1vjXiOOY1r6UulofS6404hJ0mKwnoJx/OgUvLNO4dxEHsCQ4UJakMaOpzrKn7OL3+n61NVrqf7tyobwJUVX12ZZaDlgtLClSRvrwmbJUpZT/H5z6UtqYnkDLBTrd3zFJZ3FlJXTyXoZ6Wfq8t0OgrHStItkSSLDQuReebw4/ha0OP2mUfhXChe85J/a4E3jj1GjZ2thA04ozNL2NHpI2F8zdg2T2Lc2Pcj5LFCwYqXmcPa4zsDnfIMgD0LOcLBFOKBvLWjk2IbqEtOTaBS9UpLbqPSPv53YpXDPCgKH58/lt2Oc49wekVkziqZBQftGxAEYIjikZ059rA0KK7qG3/NS2hZ7ClgUurZlDhr/A4shci7OulSRuh0ElYVtbCkhKZ2mxdqo6mKGlhh5AU/JyqjqqWkb3ilhNV3fU819U8xpY+TtxswLACuPVhqEqm0D8sp4xhOfu/8J4QOm7PBUQjz5JmNBBuvL5rAMh3eKjw5FITaks7VxWC48vTPXp9rb47oaCknq9tB8k2NyVg2+GM431R6PsqjZ139xEGVLzOqWjdit0J5WP588Z3SURNzG7hxqFojPKXMzE/+zsq8+RQ5uk/h+M/geaYSE7xi7tsc83oI5hcVMmTm5bSZcQ5feBovlI1NhU2tBMzSobzRv0nGQqfLSVTCwcl76cU9xuu4dB2T6cvupOrd4cCpxcFkeUbkhl5fbdNOoWrFzxN3EoWIlUQOFWNWyf2GFZ05zS8ubcT7vo5oIA0UfUR+Av+vtu+fBrkOjw8OeM6Htkyj/nN68h1eLh00AxOKBuXapPvPpkuz1m0Rl4ACUKoCFSGF/9tD4hyLLLmpQF0K2XZPJ2QfAsx28CPC7em9/tW/I6kB0h3TCAWe4e+Rj8pLfTdKJpCaJT6LqDU9/lQcqtqEZo2DNNcS/rzcuJ2n8eUwkq2Blux+nhSDdtisK+QkBHLup4nIwgylcskBHafchVux0g8+hgiiZXIXl5qRegcVXEL/7fjVTqNKFErgUvRcagaPzvknL4X3iO4cq7D4TkXIzYnKQe4TuSbJW6arbd4vmYlNpJil49vjT6cP657I+3cLtONKVW0PgqdEE40tQhFeLLeU0ESt3tETQE4FB1DCm5acz7fHbqB00s3IVBxei7G7bsqTdkZ5Cvmt5Mu4raVz/Ny8zTOLF2MU7EyvEY2Se9nX0VJCHA4pnMQu8BB4oZ+sUdKkhDiV8BPdtPsOCnlnE/do30EKeVfgb8CTJ069TObAXH1bAx7FWofb5KNkxzPTITQyS9+jVDXr4jF3qUjAa80j+MfDclCZyuCFWjZFAnhQVXKEFZLtztbwZQqjzccgUP1ImyLc6qmcPmQGXvd9+G9KD+H5pRwzfBj+cvGOdgpWgeNLi6iRLzQS0hUUISHEv91qXN97lNpDz2cYVVSBXwSGkxf2EhOGjCGiflV3LToRTZ2JRXF0Xml3H3Y2XhcBbhKPiAaeRrLTDKgudznIZTsi/J/Cr/u5pQBmdZZRTgZVHA71fk/w5YJVGXXBWZdriOzHhfCjc9zMSdVVPJMzdIMa7gtJUeXDQOSicH3rJmTcQ2J5OQBo3E5nSiKH8uK0FvYF0LDl6VQYDY4tXKcWiaD2ecBf94vsO124rFkgrmUCTyeWXi8VwJJy+Bvpp7JN+Y/QaKbLtmlavg0JzeMS6+VcWrFOGpCgYykeLemM9SfnNs+9+mEY+9khK4iDTzO3W+kxTlXEY5/TDA2p5sIQ6CpJVQV3ZNq41R1Hp/xLe5fP5vZjatRhcJZlZO5evixBxQr3LTiKqYV71qxPrZ0FCP8pazrakwJ1m5V58zKSVR5kx4vVfGS77uI9vAzad+8EG5K/Dfuth9COPA4ZxCJLyDdm6ST4zkr9evC6mm8XrcqTcAXJL3nkwvSx3F4ySCeOu4KHlizgI1dLYzJK+U7Y45iZG5JWjuXdxZOzzmYxhoUJR9VG8JngQKnj5tGn8ZNo7MXVxZCMKjwN5T6v0EwthBVySPPfcJu1yAAr+dMQpGnMq360sTtSjLjzSwfxbM1S1NK/E6Uu3Mp7qYhd6k6p1SM4c26NcR7GW3cqs6Vww7vvtcsQqE/I+0eox84cTinousHfqJ8XsEDBFrOThL0yAhCeFG1IXhzvsc1IxO8uj2Zg7RTGXKrOlcOPwyf7uSkAWOZ07ghI/rBtG1Kcs4nElqbuc5g4nJMoy+GljxCbeAHdEZnA+DQKqku+D0+1xieP2Y4b9avZE1nHUN8JZxWMYEcfe+LnSvqAJzeniwFJ/DLqafx00knEzET5DpcCCGY37KWJYGa1Hr6Ucdgrqj4MNsV8XvORFNLiCQ+ThuzLaHTdFMbK+huKSh0+bhx1Gk0RYNMKBjAlKKBu10Xjy4dxdszf8iWUAua/TFW4ilsYzXIVlKlP6CbXIZUukPM1sn3noV+ANSCOogvJkS/+R69GwlRBGRS46SjVvb6OoQQU4GP2AN2OyFELXCvlPLOXsf+B7hOSlkthBhCkvHuUCnlR73avAq0Simv2NX1p06dKpcsWbKrJvsMnYko9yy7novLX8eUCoqAhK1RUPA4Ff5MISxmGcxt2MSSQA3P1X6cFIaLlnFZxRJ0YaEIiRAe3M4ZlBX8ha7oa3RG3kJTiyj0XUqMKuojHQz0FuD/FAtnf6gNB3ivcR2qEJxQPpZSl5/OyL9pCd6PabXidR5Oae7/4NTThYum9lvpDD/VnZ+jIIROof8mXmqaxD3r3ksm1iOwsblr6gUcV9ZjdWyLRxBAvnPfKEGfJaKxBTQHrgBEymqb47uKgryfpGL8X9meFPQEAoeicv2Y47hyRM/ceKl2JbcufQVVURCAJW1+P+VsTq5MCh2muZ1A2zUkjNUIFBS1hML8+3A6MzffLwosqxHL3IGmDUFRCzL+vj3czhObl7A1GGBqURUXDp5EriN9vkfMBJfOe5jt4XYiVgJdUVGFwn2HXcwRJcnaNVKa1LVeQiyxLGVsEMJJof9H5Odcvcf9jSXWE0msxKFV4nUets9o8A9EJCyTF7cv5bW6FbhUnfOrpzGzbEyaYCOlSWPnbwmE/oGUBppaTHne7eR5sisBfWGYddQ2n4FlB5OsgcKNrg6gqvQlVKUnl+CZrR9x5+o30BQVW0pydBd/nX45Q3L2T02XLyKklLR1/IRQ5Olur37SC1WQ9ytyfMlwzbZ4mPPe+yudiShRy8ChqGhC5W8zLmNSLybNiJnghkXP8mFLDQ5FJWFbXDR4MreMPzn1/k1zG50dtxGPz0UIJx7Pxfhzf4gQ+34/2h+w7TCx6MvYVh26YyIO57GI7jpmW4MB7l79Lotbailwerh6xHTOqZ6AEAJL2vx/e/cepFV933H8/d1dYBdYiFwEDOHWRBNvhUi0WMFbSTI6qQEcMY1NSVtbYmKNyWhqk1Gmk9Fp2powTtKMph3G6ERb03QikZtEgg0EA06YekNbERQQBEQue9/99o/fWTl72Of+PPs8h/28Zn6z7Dnfc/vy23Oe3zm/83uWbnqU5w/vprW7Mxo4oYGvnns1fzpjFnvfuZ6Ozpei80w9ZkMZ+4H7GDVicZZ9aaXH26mvG131Gyrt3Z38YMcz/Gz387T3dDJn/Ie5/eyzaD9+Oz3RkzKzJiaPe5ARwy7B3dl/5B7ePf5INNqd0c1Q7t+5gO1HQy+Ys0dN5DsX3cDk4aV/hUOPt7HvwALaOndgtNLZU0e3GxsOncNHhh+isWEkZ4//CmeOWlyVXJrZNnefnTuyukY3jPc5zdflDqyQNUf+tabzlFcjqagVF9ZIehw4w90/GZu2Fjjk7p+LBm7YS2hI3RvNbwQOAHe4e9Z3kgaykQThTtLGfds53LKRsY3jmTNpEY15vNdxoPUoT+15gWOdbcwd18WkhmdwP8HIpmsZ0Tg/dR/EWtu3cqxlJVgDo4YvoDEa5ert1vfYuP81htU1cOWkcyrSuKumnp5jtLSupsdP0NR4BUMapr0/z93Zdmg3a/a8zLC6Bj4z5QLOGT3hlHUc6Wjl2f3/Sx3GvIkfpnnIqd3iurv3495Bff3kql9Qa0VHTxfr9r7M5gOvM7FpFAunzuKs4X37o7t3cbx1FcdbV1JX18zoETfRODT/oaolM/cueryVOhtZcJ3s8XaOt66mMxoCfETj1e9/WI070dXO7w6/yYiGYVx4xgepS9l5caC0d2ynpXU1Zk2MGH4dQxJdH090dfDz3dt57uAupo8cww3TZzOxqf+vEdjTcoR9Le8xo3k8Y1J486pSur2H9XtfYd2+lxjRMIzrp36c888IbwC4d3KidSUnWldRXzeW5hE3MSyPkR5rnXsPbZ0vAk7jkPNO+Rvt6HqTlvYt1NeNYWTjPKCe/W1HGVJXz9jYlyWXZ186OdG6ioPHV/PS0W7++90LmTbqQj47ZSZnNpWn+3Kx1EjKz6BrJJnZRMI7ROcCjwLXEho4u939cBSzHnjO3e+Kfr8U2Ah8C/gvYAHw98BlvUOAm9k3gL8Dvgi8GsXOA3IOAT7QjSQRERERGZzUSMpPrTeSKjFww1LC9x31+kX084vAiujfvwe82Rvg7pvM7Ebg24TG0f8Bi2PfkQTwHaAJ+D4nv0z2k7X0HUkiIiIiIqngDj39D7wjFWgkufsyYFmOmGn9THsCeCLLMh6tN+u6RURERERESlErQ4CLiIiIiMhAqtDYBKcDvfEqIiIiIiISo0aSiIiIiIhIjLrbiYiIiIgMQq6BGzLSkyQREREREZEYPUkSERERERl0XAM3ZKEnSSIiIiIiIjFqJImIiIiIiMSou52IiIiIyGDjQI+622WiJ0kiIiIiIiIxaiSJiIiIiIjEqLudiIiIiMhg5PqepEz0JElERERERCRGT5JERERERAYZB1wDN2SkJ0kiIiIiIiIxaiSJiIiIiIjEqLudiIiIiMhg466BG7LQkyQREREREZEYPUkSERERERmENHBDZnqSJCIiIiIiNcnMbjGznWbWZmbbzGzuQGxXjSQREREREak5ZrYYWA7cC8wCNgGrzGxKpbetRpKIiIiIyGDkPdUr+fkasMLdH3L3l939VmAf8KWK5SSiRpKIiIiIiNQUMxsKXASsTcxaC1xa8e27n/4vbJnZO8CuKm1+HHCwSttOI+WrMMpXYZSvwihfhVG+CqN8FUb5Kkw18zXV3cdXadt5M7PVhDxVSyPQFvv9QXd/sPcXMzsL2ANc7u4bY9PvBj7v7udUcucGxeh21ayoZrbV3WdXa/tpo3wVRvkqjPJVGOWrMMpXYZSvwihfhVG+cnP3T1d7H2qZutuJiIiIiEitOQh0AxMS0ycAb1d642okiYiIiIhITXH3DmAbMD8xaz5hlLuKGhTd7arswdwhEqN8FUb5KozyVRjlqzDKV2GUr8IoX4VRvk4P9wM/NrPngF8DS4GzgB9WesODYuAGERERERFJHzO7BbgTmAS8ANweH8ihYttVI0lEREREROQkvZMkIiIiIiISo0aSiIiIiIhIjBpJJTCzvzKzZ8zsiJm5mU3Lc7lFZvaSmbVHPxck5puZLTOzvWbWamYbzOy8ShzDQDKzYWb2gJkdNLMTZvZzM5ucY5k3otwmyy9iMcv6mV/xoSErrch85cyF6lefZe4ys9+a2VEze8fMnjSz8xMxK/rJ6W8qezTlZ2a3mNlOM2szs21mNjdH/OVRXJuZvW5mS0tdZ5oUcmxmttDM1kZ16JiZbTGzP07ELMlwLmus/NFUXoH5uiJDLj6aiMt6rUyzAvPV3znIzexELCavnKaRmc2Lzud7omNakscyF5jZr6Jr3B4zu9vMLBFz2tYvKQ81kkozHFgLLMt3ATObAzwOPArMjH7+h5ldEgu7E/g6cCvwCeAAsM7Mmsuz21XzPWAR8DlgLjAKWGlm9VmW+QThRb3e8nHAgX9PxO1IxF1Q1j2vjmLyBblzofp10hXAD4BLgauALuBpMxuTiHuavjm9pqx7XmFmthhYDtwLzCIMnbrKzKZkiJ8OPBXFzQLuAx4ws0XFrjNNiji2y4FfAtdG8U8BP+vng28LfevRJHdvI+VKqAvn0Tcfr8XWmc+1MpWKyNdtJOoN8DqnXgchS05TbCThZf3bgNZcwWY2ClgH7Cdc424D7gC+Fos5beuXlJG7q5RYgNmED+7T8oh9HFiXmPY08JPo3wbsA74Zm98EHAP+utrHWkKORgMdwOdj0z4E9ACfKmA93wSOAE2xacuAF6p9jLWQr1y5UP3KuZ6RhC+u+0xs2gpgZbWPscT8bAEeSkx7DbgvQ/w/AK8lpv0I2FzsOtNUynFswHPAP8d+XwIcr/ax1UK+CDcnHBiXZZ1Zr5VpLqXWL+APo/xdWkhOT4cCHAeW5Ij5EnA08TnhW8AeTg5YdtrWL5XyFT1JGnhzCE+f4tYQ7mQDTAcmxmPcvRXYGItJo4uAIfQ9rjeBl8nzuKJH5X8BPBLlJG6Ghe5jO83sMTObUab9rpZS8pUtF6pf2TUTnrC/m5h+mZkdMLNXzewhMzuz1B0eKGY2lJCf5HlnLZlzk+k8NdvMhhS5zlQo47E1c2o9ajKzXWb2lpmtNLNZJexqTSgxX1vNbJ+ZrTezKxPzcl0rU6lM9etm4EV37+/LNLPldLCYAzyb+JywhvDdOtNiMadd/ZLyUiNp4E0kPAKO2x9NJ/YzW0waTSTcoT+YmF7Icc0nfMh/KDF9C+Eu7acJF4+JwCYzG1vsztaAYvOVKxeqX9ktB34HbI5NWw18Abia0E3xYuCXZjas6L0dWOOAegr7P890nmqI1lfMOtOi5GMzsy8Dk4EfxybvAP4cuI7QJbQN+LWZfaTUHa6yYvK1j3C3fxGwkJCb9YnuibmulWlVUv0ys9HADZx6Hcwnp4NFprrTOy9bTNrrl5RRQ7V3oNaY2bcJXbqyudLdNwzA7tS8fPNVps3dDPzW3bfHJ7r7qsQ+/YbQX/vPCN/UXDMqna805SIfA1m/zOx+4DLgMnfv7p3u7o/Fwv7HzLYBuwjvn/xnObYtp4/ova1/BBa7+67e6e6+mVjj28w2ERrktwJ/M9D7WU3uvoPwIb7XZgsDH90BPFuNfUqRmwg3uOMNcOVUpALUSDrV94BHcsTsLmH9bwMTEtMmRNOJ/ZyQ2E48ppbkm68/INw9Gwe8E5s3gTxO4FH3puuAL+eKdffjZvYiUIt3aAckX736yYXqVz/M7LvAjYQbIK9ni3X3vWb2FrVZv/pzkPCULdt5JynTeaorWp8Vsc60KCZfAJjZ9cDDwBfc/clsse7ebWZbSU89yqTofCVsIfwN9sp1rUyrUvN1M/BTdz+cR2wyp4NFprrTOy9bTNrrl5SRutsluPtBd38lR2kpYRObCd3G4uYTRrcB2En4I30/xsIQsXNjMTWjgHxtAzrpe1yTgY+R33EtAdqBn+QKjPL1UUL3g5oygPnqXSaZC9WvBDNbTuj+dJW7v5Jrn8xsHPBBarB+9cfdOwj5yXbeScp0ntrq7p1FrjMVij02M7uBcHd/ibs/kWs70TuWF5KSepRJGevCTPrmIte1MpVKyZeZXQz8Pqd2tcskmdPBYjMw1/oOrz8f2Au8EYs57eqXlFm1R45IcyH0XZ0J/AlhVJlrot/HxGLWExuxhvBSYBfwt4QPr3cRPtxdEov5BvAeoV/x+cBjhD/u5mofc4n5+hfgLeCPCMOePkPoblIfi3kF+EpiOQNeJTEaUGz+PxGG4J0OXAKsJIxsM7XaxzzQ+conF6pfffL1/Sg/V0V/z71lZDR/ZJTTOYQXfq8gXFzfSlO+gMWE0f/+ktBwXE4YJWpqNP9h4OFY/HTgBOFJ3sei5TqARfmuM82liHzdSDiP35aoR/FrwT3Ap4AZhOvEv0XLXFzt461Cvr4KfJbwFO08whDzDiyMxeS8Vqa1FJqv2HI/Al7NsM6cOU1ric7DM6PSAtwd/XtKNP8+YH0sfjThZuBjhGvcQsJ5/uuDoX6plK9UfQfSXAjDLXs/ZUks5g1gRWK56wkf1joIo28tTMy3aN37CC/3/go4v9rHW4Z8DQMeAA5FJ7ongQ8lYhxYlph2ZTS93w8TnPyQ30EY4vOnwLnVPt5q5CufXKh+9clXf3+/78cQhkdfQ/guqQ7Cu0grkutNQwFuic5H7YQ72fNi8zYAGxLxlwPPR/E7gaWFrDPtpZB8Rb/3V4/iMd+N6k97VJ/WAHOqfZxVytedhCGvW4HDhC6x1/SzzqzXyjSXIv4emwkNqTszrC+vnKaxcHJ482RZEc1fAbyRWOYCwqitbYRr3T1Ew38PhvqlUp7SO168iIiIiIiIoHeSRERERERE+lAjSUREREREJEaNJBERERERkRg1kkRERERERGLUSBIREREREYlRI0lERERERCRGjSQREREREZEYNZJERERERERi/h87gINvnnBttwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "pa2 = make_pa_matrix(1000, 2, 10**4)\n", - "\n", - "plt.figure(figsize=(15, 12))\n", - "plt.scatter(pa2[:, 0], pa2[:, 1], c=np.arange(len(pa2)))\n", - "plt.colorbar()\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Nice and harmonic picture, isn't it?**\n", - "\n", - "That's because the curve plotted is [Lissajou's curve](https://en.wikipedia.org/wiki/Lissajous_curve):\n", - "\n", - "![](https://upload.wikimedia.org/wikipedia/commons/5/5d/Lissajous_animation.gif)\n", - "\n", - "Curve implicitly specified by harmonic coordinates\n", - "\n", - "$$\\left\\{ \\begin{align}\n", - " & x(t)=A\\sin (at+\\delta ) \\\\ \n", - " & y(t)=B\\sin (bt) \\\\ \n", - "\\end{align} \\right.\n", - "$$\n", - "\n", - "In our case $\\delta = \\pi / 2$" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 3d case" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:54:15.024781Z", - "start_time": "2019-09-26T22:54:15.019525Z" - } - }, - "outputs": [], - "source": [ - "pa3 = make_pa_matrix(250, 3, 2**3)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:54:15.422821Z", - "start_time": "2019-09-26T22:54:15.410521Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xyzc
01.0000001.0000001.0000000.0
10.0000000.0000000.0000001.0
20.5403020.8775830.9689122.0
30.8414710.4794260.2474043.0
4-0.4161470.5403020.8775834.0
\n", - "
" - ], - "text/plain": [ - " x y z c\n", - "0 1.000000 1.000000 1.000000 0.0\n", - "1 0.000000 0.000000 0.000000 1.0\n", - "2 0.540302 0.877583 0.968912 2.0\n", - "3 0.841471 0.479426 0.247404 3.0\n", - "4 -0.416147 0.540302 0.877583 4.0" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pa3_df = pd.DataFrame(\n", - " np.concatenate((pa3, np.arange(len(pa3))[:, None]), axis=1),\n", - " columns=['x', 'y', 'z', 'c'],\n", - ")\n", - "pa3_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "ExecuteTime": { - "end_time": "2019-09-26T22:54:16.196313Z", - "start_time": "2019-09-26T22:54:15.882954Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "hovertemplate": "x=%{x}
y=%{y}
z=%{z}
c=%{marker.color}", - "legendgroup": "", - "marker": { - "color": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - 30, - 31, - 32, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47, - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 58, - 59, - 60, - 61, - 62, - 63, - 64, - 65, - 66, - 67, - 68, - 69, - 70, - 71, - 72, - 73, - 74, - 75, - 76, - 77, - 78, - 79, - 80, - 81, - 82, - 83, - 84, - 85, - 86, - 87, - 88, - 89, - 90, - 91, - 92, - 93, - 94, - 95, - 96, - 97, - 98, - 99, - 100, - 101, - 102, - 103, - 104, - 105, - 106, - 107, - 108, - 109, - 110, - 111, - 112, - 113, - 114, - 115, - 116, - 117, - 118, - 119, - 120, - 121, - 122, - 123, - 124, - 125, - 126, - 127, - 128, - 129, - 130, - 131, - 132, - 133, - 134, - 135, - 136, - 137, - 138, - 139, - 140, - 141, - 142, - 143, - 144, - 145, - 146, - 147, - 148, - 149, - 150, - 151, - 152, - 153, - 154, - 155, - 156, - 157, - 158, - 159, - 160, - 161, - 162, - 163, - 164, - 165, - 166, - 167, - 168, - 169, - 170, - 171, - 172, - 173, - 174, - 175, - 176, - 177, - 178, - 179, - 180, - 181, - 182, - 183, - 184, - 185, - 186, - 187, - 188, - 189, - 190, - 191, - 192, - 193, - 194, - 195, - 196, - 197, - 198, - 199, - 200, - 201, - 202, - 203, - 204, - 205, - 206, - 207, - 208, - 209, - 210, - 211, - 212, - 213, - 214, - 215, - 216, - 217, - 218, - 219, - 220, - 221, - 222, - 223, - 224, - 225, - 226, - 227, - 228, - 229, - 230, - 231, - 232, - 233, - 234, - 235, - 236, - 237, - 238, - 239, - 240, - 241, - 242, - 243, - 244, - 245, - 246, - 247, - 248, - 249 - ], - "coloraxis": "coloraxis", - "symbol": "circle" - }, - "mode": "markers", - "name": "", - "scene": "scene", - "showlegend": false, - "type": "scatter3d", - "x": [ - 1, - 0, - 0.5403023058681398, - 0.8414709848078965, - -0.4161468365471424, - 0.9092974268256817, - -0.9899924966004454, - 0.1411200080598672, - -0.6536436208636119, - -0.7568024953079282, - 0.2836621854632263, - -0.9589242746631385, - 0.9601702866503661, - -0.27941549819892586, - 0.7539022543433046, - 0.6569865987187891, - -0.14550003380861354, - 0.9893582466233818, - -0.9111302618846769, - 0.4121184852417566, - -0.8390715290764524, - -0.5440211108893699, - 0.004425697988050786, - -0.9999902065507035, - 0.8438539587324921, - -0.5365729180004349, - 0.9074467814501962, - 0.4201670368266409, - 0.1367372182078336, - 0.9906073556948704, - -0.7596879128588212, - 0.6502878401571169, - -0.9576594803233847, - -0.2879033166650653, - -0.27516333805159693, - -0.9613974918795568, - 0.6603167082440802, - -0.750987246771676, - 0.9887046181866692, - 0.14987720966295234, - 0.40808206181339196, - 0.9129452507276277, - -0.5477292602242685, - 0.836655638536056, - -0.9999608263946371, - -0.008851309290403876, - -0.5328330203333975, - -0.8462204041751706, - 0.4241790073369969, - -0.9055783620066239, - 0.9912028118634736, - -0.13235175009777303, - 0.6469193223286404, - 0.7625584504796028, - -0.2921388087338362, - 0.956375928404503, - -0.9626058663135666, - 0.27090578830786904, - -0.7480575296890004, - -0.6636338842129675, - 0.15425144988758405, - -0.9880316240928618, - 0.9147423578045313, - -0.404037645323065, - 0.8342233605065102, - 0.5514266812416906, - -0.013276747223059479, - 0.9999118601072672, - -0.8485702747846052, - 0.5290826861200238, - -0.9036922050915067, - -0.428182669496151, - -0.12796368962740468, - -0.9917788534431158, - 0.7654140519453434, - -0.6435381333569994, - 0.9550736440472949, - 0.2963685787093853, - 0.2666429323599373, - 0.9637953862840878, - -0.6669380616522619, - 0.7451131604793488, - -0.9873392775238264, - -0.158622668804709, - -0.3999853149883513, - -0.9165215479156338, - 0.5551133015206257, - -0.8317747426285983, - 0.9998433086476912, - 0.017701925105413577, - 0.5253219888177297, - 0.8509035245341184, - -0.4321779448847783, - 0.9017883476488092, - -0.9923354691509287, - 0.123573122745224, - -0.6401443394691997, - -0.7682546613236668, - 0.3005925437436371, - -0.9537526527594719, - 0.9649660284921133, - -0.26237485370392877, - 0.7421541968137826, - 0.6702291758433747, - -0.16299078079570548, - 0.9866275920404853, - -0.9182827862121189, - 0.3959251501818342, - -0.8293098328631502, - -0.5587890488516162, - 0.022126756261955732, - -0.9997551733586199, - 0.853220107722584, - -0.5215510020869119, - 0.8998668269691938, - 0.43616475524782494, - 0.11918013544881928, - 0.9928726480845371, - -0.7710802229758452, - 0.6367380071391379, - -0.9524129804151563, - -0.3048106211022167, - -0.25810163593826746, - -0.9661177700083929, - 0.6735071623235862, - -0.7391806966492229, - 0.9858965815825497, - 0.16735570030280694, - 0.39185723042955, - 0.9200260381967907, - -0.562453851238172, - 0.8268286794901035, - -0.99964745596635, - -0.026551154023966794, - -0.5177697997895051, - -0.8555199789753223, - 0.4401430224960407, - -0.8979276806892913, - 0.9933903797222716, - -0.11478481378318722, - 0.6333192030862999, - 0.7738906815578891, - -0.3090227281660707, - 0.9510546532543747, - -0.9672505882738824, - 0.25382336276203626, - -0.7361927182273159, - -0.6767719568873076, - 0.17171734183077755, - -0.9851462604682474, - 0.9217512697247493, - -0.38778163540943045, - 0.8243313311075577, - 0.5661076368981803, - -0.030975031731216456, - 0.9995201585807313, - -0.8578030932449878, - 0.5139784559875352, - -0.8959709467909631, - -0.4441126687075084, - -0.11038724383904756, - -0.9938886539233752, - 0.7766859820216312, - -0.629887994274454, - 0.9496776978825432, - 0.31322878243308516, - 0.2495401179733381, - 0.9683644611001854, - -0.6800234955873388, - 0.7331903200732921, - -0.9843766433940419, - -0.1760756199485871, - -0.38369844494974187, - -0.9234584470040598, - 0.569750334265312, - -0.8218178366308225, - 0.9993732836951247, - 0.03539830273366069, - 0.5101770449416688, - 0.8600694058124533, - -0.4480736161291701, - 0.8939966636005579, - -0.9943674609282015, - 0.10598751175115685, - -0.6264444479103392, - -0.7794660696158047, - 0.31742870151970165, - -0.9482821412699473, - 0.9694593666699876, - -0.2452519854676543, - 0.7301735609948197, - 0.683261714736121, - -0.18043044929108393, - 0.9835877454343449, - -0.9251475365964139, - 0.37960773902752165, - -0.8192882452914593, - -0.5733818719904229, - 0.0398208803931389, - -0.9992068341863537, - 0.862318872287684, - -0.5063656411097588, - 0.8920048697881602, - 0.45202578717835057, - 0.10158570369662134, - 0.9948267913584063, - -0.782230889887116, - 0.6229886314423488, - -0.9468680107512126, - -0.32162240316253093, - -0.24095904923620143, - -0.9705352835374847, - 0.6864865509069841, - -0.7271425000808527, - 0.9827795820412206, - 0.18478174456066745, - 0.37550959776701204, - 0.926818505417785, - -0.577002178942952, - 0.8167426066363169, - -0.999020813314648, - -0.044242678085070965, - -0.5025443191453852, - -0.8645514486106083, - 0.4559691044442761, - -0.8899956043668333, - 0.9952666362171313, - -0.09718190589320902, - 0.6195206125592099, - 0.7849803886813105, - -0.3258098052199642, - 0.9454353340247703, - -0.9715921906288022, - 0.23666139336428602, - -0.7240971967004738, - -0.689697940935389, - 0.1891294205289584, - -0.9819521690440836, - 0.9284713207390763, - -0.3714041014380903, - 0.8141809705265618, - 0.5806111842123143, - -0.0486636092001539, - 0.9988152247235795, - -0.8667670910519801, - 0.4987131538963941, - -0.8879689066918555, - -0.45990349068959124, - -0.09277620459766088, - -0.9956869868891794 - ], - "y": [ - 1, - 0, - 0.8775825618903728, - 0.479425538604203, - 0.5403023058681398, - 0.8414709848078965, - 0.0707372016677029, - 0.9974949866040544, - -0.4161468365471424, - 0.9092974268256817, - -0.8011436155469337, - 0.5984721441039564, - -0.9899924966004454, - 0.1411200080598672, - -0.9364566872907963, - -0.35078322768961984, - -0.6536436208636119, - -0.7568024953079282, - -0.21079579943077972, - -0.977530117665097, - 0.2836621854632263, - -0.9589242746631385, - 0.70866977429126, - -0.7055403255703919, - 0.9601702866503661, - -0.27941549819892586, - 0.9765876257280235, - 0.21511998808781552, - 0.7539022543433046, - 0.6569865987187891, - 0.3466353178350258, - 0.9379999767747389, - -0.14550003380861354, - 0.9893582466233818, - -0.6020119026848236, - 0.7984871126234903, - -0.9111302618846769, - 0.4121184852417566, - -0.9971721561963784, - -0.07515112046180931, - -0.8390715290764524, - -0.5440211108893699, - -0.47553692799599256, - -0.87969575997167, - 0.004425697988050786, - -0.9999902065507035, - 0.4833047587530059, - -0.8754521746884285, - 0.8438539587324921, - -0.5365729180004349, - 0.9977982791785807, - -0.06632189735120068, - 0.9074467814501962, - 0.4201670368266409, - 0.594920663309892, - 0.803784426551621, - 0.1367372182078336, - 0.9906073556948704, - -0.354924266788705, - 0.934895055524683, - -0.7596879128588212, - 0.6502878401571169, - -0.9784534628188842, - 0.2064674819377966, - -0.9576594803233847, - -0.2879033166650653, - -0.7023970575027135, - -0.7117853423691232, - -0.27516333805159693, - -0.9613974918795568, - 0.21943996321145934, - -0.9756260054681576, - 0.6603167082440802, - -0.750987246771676, - 0.939524893748256, - -0.34248061846961253, - 0.9887046181866692, - 0.14987720966295234, - 0.7958149698139441, - 0.6055398697196009, - 0.40808206181339196, - 0.9129452507276277, - -0.07956356727854007, - 0.9968297942787993, - -0.5477292602242685, - 0.836655638536056, - -0.8817917275413242, - 0.47163900309419615, - -0.9999608263946371, - -0.008851309290403876, - -0.8733046400935156, - -0.4871745124605095, - -0.5328330203333975, - -0.8462204041751706, - -0.061905293994420546, - -0.9980820279793963, - 0.4241790073369969, - -0.9055783620066239, - 0.8064094939122546, - -0.5913575298651244, - 0.9912028118634736, - -0.13235175009777303, - 0.933315112063922, - 0.35905835402216824, - 0.6469193223286404, - 0.7625584504796028, - 0.202135120387182, - 0.979357643103917, - -0.2921388087338362, - 0.956375928404503, - -0.7148869687796651, - 0.6992400316550977, - -0.9626058663135666, - 0.27090578830786904, - -0.9746452757206577, - -0.22375564018679642, - -0.7480575296890004, - -0.6636338842129675, - -0.3383192109710552, - -0.9410314083429536, - 0.15425144988758405, - -0.9880316240928618, - 0.6090559761063562, - -0.7931272394572851, - 0.9147423578045313, - -0.404037645323065, - 0.9964679075571249, - 0.08397445569174683, - 0.8342233605065102, - 0.5514266812416906, - 0.4677318402470736, - 0.8838704235458307, - -0.013276747223059479, - 0.9999118601072672, - -0.49103472393024045, - 0.8711400001691764, - -0.8485702747846052, - 0.5290826861200238, - -0.9983462274487422, - 0.057487478104924564, - -0.9036922050915067, - -0.428182669496151, - -0.587782813560387, - -0.8090187662119064, - -0.12796368962740468, - -0.9917788534431158, - 0.3631854084160624, - -0.9317168878547055, - 0.7654140519453434, - -0.6435381333569994, - 0.9802426408101081, - -0.19779879963646227, - 0.9550736440472949, - 0.2963685787093853, - 0.6960693098638898, - 0.7179745927716441, - 0.2666429323599373, - 0.9637953862840878, - -0.22806693448309956, - 0.9736454556949781, - -0.6669380616522619, - 0.7451131604793488, - -0.9425194910508831, - 0.33415117684842055, - -0.9873392775238264, - -0.158622668804709, - -0.7904239741978156, - -0.6125601529754698, - -0.3999853149883513, - -0.9165215479156338, - 0.08838369930580556, - -0.996086503119594, - 0.5551133015206257, - -0.8317747426285983, - 0.8859318072699817, - -0.4638155159838274, - 0.9998433086476912, - 0.017701925105413577, - 0.8689582973139933, - 0.49488531755262816, - 0.5253219888177297, - 0.8509035245341184, - 0.05306853621402457, - 0.9985908724117705, - -0.4321779448847783, - 0.9017883476488092, - -0.8116121923430246, - 0.5841965844132857, - -0.9923354691509287, - 0.123573122745224, - -0.9301004142012892, - -0.36730534913419133, - -0.6401443394691997, - -0.7682546613236668, - -0.1934586046207122, - -0.981108438603097, - 0.3005925437436371, - -0.9537526527594719, - 0.7210481538680822, - -0.6928849542336957, - 0.9649660284921133, - -0.26237485370392877, - 0.9726265649744922, - 0.2323737616554845, - 0.7421541968137826, - 0.6702291758433747, - 0.32997659774057014, - 0.9439891127251193, - -0.16299078079570548, - 0.9866275920404853, - -0.616052331690985, - 0.7877052269841179, - -0.9182827862121189, - 0.3959251501818342, - -0.9956855884367365, - -0.09279121175730869, - -0.8293098328631502, - -0.5587890488516162, - -0.45989010701310373, - -0.8879758383376634, - 0.022126756261955732, - -0.9997551733586199, - 0.49872621790648564, - -0.8667595742607592, - 0.853220107722584, - -0.5215510020869119, - 0.9988159580766447, - -0.04864855487508726, - 0.8998668269691938, - 0.43616475524782494, - 0.580598912666927, - 0.8141897215084345, - 0.11918013544881928, - 0.9928726480845371, - -0.37141809547969407, - 0.9284657227653786, - -0.7710802229758452, - 0.6367380071391379, - -0.9819550195245901, - 0.18911462035089152, - -0.9524129804151563, - -0.3048106211022167, - -0.6896870271361664, - -0.7241075918674496, - -0.25810163593826746, - -0.9661177700083929, - 0.23667603734656428, - -0.9715886235161092, - 0.6735071623235862, - -0.7391806966492229 - ], - "z": [ - 1, - 0, - 0.9689124217106447, - 0.247403959254523, - 0.8775825618903726, - 0.4794255386042031, - 0.7316888688738208, - 0.6816387600233343, - 0.5403023058681395, - 0.8414709848078966, - 0.31532236239526845, - 0.9489846193555863, - 0.07073720166770246, - 0.9974949866040544, - -0.17824605564949253, - 0.9839859468739368, - -0.4161468365471428, - 0.9092974268256815, - -0.6281736227227394, - 0.778073196887921, - -0.801143615546934, - 0.5984721441039561, - -0.9243023786324638, - 0.3816609920523313, - -0.9899924966004456, - 0.14112000805986633, - -0.9941296760805461, - -0.10819513453010926, - -0.936456687290796, - -0.35078322768962067, - -0.8205593573395602, - -0.5715613187423445, - -0.6536436208636113, - -0.7568024953079288, - -0.446087489913792, - -0.8949893582285839, - -0.21079579943077884, - -0.9775301176650972, - 0.03760215288797745, - -0.9992927889753779, - 0.28366218546322713, - -0.9589242746631382, - 0.5120854772418414, - -0.8589344934265916, - 0.7086697742912607, - -0.7055403255703913, - 0.8611924171615213, - -0.5082790774992575, - 0.9601702866503665, - -0.27941549819892414, - 0.9994494182244995, - -0.03317921654755504, - 0.9765876257280232, - 0.21511998808781727, - 0.8930063446890758, - 0.4500440737806192, - 0.7539022543433034, - 0.6569865987187904, - 0.5679241732886934, - 0.8230808790115065, - 0.34663531783502416, - 0.9379999767747395, - 0.1037943572192512, - 0.9945987791111763, - -0.1455000338086153, - 0.9893582466233816, - -0.3857479374522234, - 0.9226042102393396, - -0.6020119026848251, - 0.7984871126234893, - -0.7808456836057502, - 0.624723953754191, - -0.9111302618846777, - 0.412118485241755, - -0.984765173467324, - 0.17388948538043178, - -0.9971721561963783, - -0.07515112046181109, - -0.9475798039779927, - -0.3195191936222753, - -0.8390715290764514, - -0.5440211108893713, - -0.678393850473844, - -0.7346984304047967, - -0.475536927995991, - -0.8796957599716709, - -0.24311342256102825, - -0.969997867920679, - 0.004425697988052563, - -0.9999902065507035, - 0.25168965007175614, - -0.967807997511261, - 0.48330475875300744, - -0.8754521746884276, - 0.6848703183835546, - -0.7286649758271688, - 0.843853958732494, - -0.5365729180004319, - 0.9503708470676746, - -0.311119354981124, - 0.9977982791785809, - -0.06632189735119715, - 0.9831874470475911, - 0.1825991346311375, - 0.9074467814501948, - 0.42016703682664414, - 0.7752854701292857, - 0.6316109877182414, - 0.5949206633098891, - 0.8037844265516231, - 0.37756657109729, - 0.9259824428086285, - 0.13673721820783008, - 0.9906073556948708, - -0.11259379263383901, - 0.9936411011327622, - -0.3549242667887083, - 0.9348950555246818, - -0.5751872690824057, - 0.8180217634546921, - -0.7596879128588235, - 0.6502878401571142, - -0.8969548417022905, - 0.4421221685765362, - -0.9784534628188849, - 0.2064674819377931, - -0.9991165866797339, - -0.042024352718844346, - -0.9576594803233837, - -0.2879033166650687, - -0.8566597458288405, - -0.5158818468181123, - -0.702397057502711, - -0.7117853423691256, - -0.5044627221459249, - -0.8634334728079074, - -0.2751633380515935, - -0.9613974918795578, - -0.02875563032918361, - -0.9995864713592173, - 0.2194399632114628, - -0.9756260054681568, - 0.45399184267981, - -0.8910058399248518, - 0.6603167082440828, - -0.7509872467716737, - 0.8255862790817411, - -0.5642759039618523, - 0.9395248937482572, - -0.3424806184696092, - 0.9950484010363791, - -0.09939154689884465, - 0.9887046181866688, - 0.14987720966295587, - 0.9208879708911081, - 0.38982732724638186, - 0.7958149698139418, - 0.6055398697196038, - 0.621262048380912, - 0.7836028759783575, - 0.40808206181338874, - 0.9129452507276291, - 0.16952950915565496, - 0.9855251115651202, - -0.0795635672785436, - 0.9968297942787989, - -0.323709766459238, - 0.9461564284508697, - -0.5477292602242714, - 0.8366556385360541, - -0.7376936014721196, - 0.6751356532927985, - -0.8817917275413258, - 0.471639003094193, - -0.9710643148808387, - 0.23881812402957928, - -0.9999608263946371, - -0.008851309290407429, - -0.9666846169547724, - -0.2559704110693365, - -0.8733046400935138, - -0.4871745124605126, - -0.7256268104935268, - -0.6880884622582994, - -0.5328330203333945, - -0.8462204041751725, - -0.30691025370372627, - -0.9517384599623546, - -0.061905293994417, - -0.9980820279793965, - 0.1869486370620462, - -0.9823696896284226, - 0.42417900733700337, - -0.9055783620066209, - 0.635035981413377, - -0.772482557932766, - 0.8064094939122588, - -0.5913575298651187, - 0.9276443698605873, - -0.3734647547841081, - 0.9912028118634746, - -0.13235175009776598, - 0.9931330638374922, - 0.1169902453743711, - 0.9333151120639195, - 0.3590583540221749, - 0.8154681470604958, - 0.578801953287756, - 0.6469193223286349, - 0.7625584504796074, - 0.43814818743719797, - 0.8989027566124703, - 0.20213512038717504, - 0.9793576431039185, - -0.046445729422976985, - 0.9989208147888238, - -0.29213880873384296, - 0.9563759284045009, - -0.5196681118689644, - 0.8543682189235187, - -0.7148869687796701, - 0.6992400316550927, - -0.8656576164704202, - 0.5006364859324088, - -0.9626058663135685, - 0.27090578830786216, - -0.9997039456950856, - 0.02433148087719517, - -0.9746452757206561, - -0.22375564018680333, - -0.8889878831195944, - -0.4579307192868178, - -0.7480575296889956, - -0.6636338842129729, - -0.5606165822201001, - -0.8280755084772485, - -0.33831921097104856, - -0.941031408342956, - -0.09498678980628619, - -0.9954785330494558, - 0.15425144988759107, - -0.9880316240928607, - 0.39389908153221426, - -0.9191536942035744, - 0.6090559761063619, - -0.7931272394572808, - 0.786344720000897, - -0.6177879744108904, - 0.9147423578045342, - -0.4040376453230585 - ] - } - ], - "layout": { - "coloraxis": { - "colorbar": { - "title": { - "text": "c" - } - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "legend": { - "tracegroupgap": 0 - }, - "margin": { - "t": 60 - }, - "scene": { - "domain": { - "x": [ - 0, - 1 - ], - "y": [ - 0, - 1 - ] - }, - "xaxis": { - "title": { - "text": "x" - } - }, - "yaxis": { - "title": { - "text": "y" - } - }, - "zaxis": { - "title": { - "text": "z" - } - } - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "heatmapgl": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmapgl" - } - ], - "histogram": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - } - } - }, - "text/html": [ - "
\n", - " \n", - " \n", - "
\n", - " \n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig = px.scatter_3d(pa3_df, x='x', y='y', z='z', color='c')\n", - "fig.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Py3 research env", - "language": "python", - "name": "py3_research" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/week06_bert/bert_for_text_classification.ipynb b/week06_bert/bert_for_text_classification.ipynb deleted file mode 100644 index 3412b19..0000000 --- a/week06_bert/bert_for_text_classification.ipynb +++ /dev/null @@ -1,2895 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "# Practice: A Visual Notebook to Using BERT for the First Time" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "*Credits: first part of this notebook is strongly based on Jay Alammar's [great blog post](http://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/). His blog is a great way to dive into the DL and NLP concepts.*\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "In this notebook, we will use pre-trained deep learning model to process some text. We will then use the output of that model to classify the text. The text is a list of sentences from film reviews. And we will calssify each sentence as either speaking \"positively\" about its subject of \"negatively\"." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "## Models: Sentence Sentiment Classification" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "Our goal is to create a model that takes a sentence (just like the ones in our dataset) and produces either 1 (indicating the sentence carries a positive sentiment) or a 0 (indicating the sentence carries a negative sentiment). We can think of it as looking like this:\n", - "\n", - "\n", - "\n", - "Under the hood, the model is actually made up of two model.\n", - "\n", - "* DistilBERT processes the sentence and passes along some information it extracted from it on to the next model. DistilBERT is a smaller version of BERT developed and open sourced by the team at HuggingFace. It’s a lighter and faster version of BERT that roughly matches its performance.\n", - "* The next model, a basic Logistic Regression model from scikit learn will take in the result of DistilBERT’s processing, and classify the sentence as either positive or negative (1 or 0, respectively).\n", - "\n", - "The data we pass between the two models is a vector of size 768. We can think of this of vector as an embedding for the sentence that we can use for classification.\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "## Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "The dataset we will use in this example is [SST2](https://nlp.stanford.edu/sentiment/index.html), which contains sentences from movie reviews, each labeled as either positive (has the value 1) or negative (has the value 0):\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " sentence\n", - " \n", - " label\n", - "
\n", - " a stirring , funny and finally transporting re imagining of beauty and the beast and 1930s horror films\n", - " \n", - " 1\n", - "
\n", - " apparently reassembled from the cutting room floor of any given daytime soap\n", - " \n", - " 0\n", - "
\n", - " they presume their audience won't sit still for a sociology lesson\n", - " \n", - " 0\n", - "
\n", - " this is a visually stunning rumination on love , memory , history and the war between art and commerce\n", - " \n", - " 1\n", - "
\n", - " jonathan parker 's bartleby should have been the be all end all of the modern office anomie films\n", - " \n", - " 1\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "## Installing the transformers library" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "izA3-6kffbdT" - }, - "source": [ - "Let's start by installing the huggingface transformers library so we can load our deep learning NLP model." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "To9ENLU90WGl", - "outputId": "6e5be1cb-9674-40e5-cfe7-17f562537556" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "%pip install -Uqq transformers" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zQ-42fh0hjsF" - }, - "source": [ - "## Part 1. Using BERT for text classification." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading pretrained BERT." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here we will be using the pretrained DistilBERT model. Here is an example of it:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'sequence': \"hello i'm a role model.\",\n", - " 'score': 0.052928753197193146,\n", - " 'token': 2535,\n", - " 'token_str': 'role'},\n", - " {'sequence': \"hello i'm a fashion model.\",\n", - " 'score': 0.03968575596809387,\n", - " 'token': 4827,\n", - " 'token_str': 'fashion'},\n", - " {'sequence': \"hello i'm a business model.\",\n", - " 'score': 0.03474372997879982,\n", - " 'token': 2449,\n", - " 'token_str': 'business'},\n", - " {'sequence': \"hello i'm a model model.\",\n", - " 'score': 0.03462280333042145,\n", - " 'token': 2944,\n", - " 'token_str': 'model'},\n", - " {'sequence': \"hello i'm a modeling model.\",\n", - " 'score': 0.018145091831684113,\n", - " 'token': 11643,\n", - " 'token_str': 'modeling'}]" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from transformers import pipeline\n", - "\n", - "model_name = 'distilbert-base-uncased'\n", - "unmasker = pipeline('fill-mask', model_name)\n", - "unmasker(\"Hello I'm a [MASK] model.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here is how we can use the same model to extract features from our text:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 23, 768])" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import torch\n", - "\n", - "from transformers import logging; logging.set_verbosity_error() # Ignore warning on model loading.\n", - "from transformers import DistilBertModel, DistilBertTokenizer\n", - "\n", - "tokenizer = DistilBertTokenizer.from_pretrained(model_name)\n", - "model = DistilBertModel.from_pretrained(model_name)\n", - "\n", - "text = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.'\n", - "tokenized_text = tokenizer(text, return_tensors='pt')\n", - "\n", - "with torch.no_grad():\n", - " output = model(**tokenized_text)\n", - "\n", - "output.last_hidden_state.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zQ-42fh0hjsF" - }, - "source": [ - "## Loading the dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zQ-42fh0hjsF" - }, - "source": [ - "We'll use pandas to read the dataset and load it into a dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "id": "cyoj29J24hPX" - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "dataset_url = 'https://github.com/clairett/pytorch-sentiment-classification/raw/master/data/SST2/train.tsv'\n", - "dataset = pd.read_csv(dataset_url, delimiter='\\t', header=None)\n", - "dataset.columns = ['text', 'label']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dMVE3waNhuNj" - }, - "source": [ - "For performance reasons, we'll only use 2,000 sentences from the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "gTM3hOHW4hUY" - }, - "outputs": [], - "source": [ - "dataset = dataset[:2000]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PRc2L89hh1Tf" - }, - "source": [ - "We can ask pandas how many sentences are labeled as \"positive\" (value 1) and how many are labeled \"negative\" (having the value 0)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "jGvcfcCP5xpZ", - "outputId": "2679553c-f061-4254-8b9d-b005529de44d" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "1 1041\n", - "0 959\n", - "Name: label, dtype: int64" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset['label'].value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lZDBMn3wiSX6" - }, - "source": [ - "## Preparing the Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lZDBMn3wiSX6" - }, - "source": [ - "Before we can hand our sentences to BERT, we need to so some minimal processing to put them in the format it requires." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lZDBMn3wiSX6" - }, - "source": [ - "### Tokenization" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lZDBMn3wiSX6" - }, - "source": [ - "Our first step is to tokenize the sentences -- break them up into word and subwords in the format BERT is comfortable with." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "id": "Dg82ndBA5xlN" - }, - "outputs": [], - "source": [ - "tokenized_texts = dataset['text'].apply(lambda x: tokenizer.encode(x, add_special_tokens=True)).values" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mHwjUwYgi-uL" - }, - "source": [ - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mHwjUwYgi-uL" - }, - "source": [ - "### Padding" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mHwjUwYgi-uL" - }, - "source": [ - "After tokenization, `tokenized_texts` is a list of sentences -- each sentences is represented as a list of tokens. We want BERT to process our examples all at once (as one batch). It's just faster that way. For that reason, we need to pad all lists to the same size, so we can represent the input as one 2-d array, rather than a list of lists (of different lengths)." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "URn-DWJt5xhP" - }, - "outputs": [], - "source": [ - "max_len = max(len(text) for text in tokenized_texts)\n", - "padded_texts = torch.tensor([text + [0] * (max_len - len(text)) for text in tokenized_texts])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Mdjg306wjjmL" - }, - "source": [ - "Our dataset is now in the `padded_texts` variable, we can view its dimensions below:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "jdi7uXo95xeq", - "outputId": "cc0bd2d3-921f-4dcc-d619-bcb621b2e707" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([2000, 59])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "padded_texts.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sDZBsYSDjzDV" - }, - "source": [ - "### Masking" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sDZBsYSDjzDV" - }, - "source": [ - "If we directly send `padded_texts` to BERT, that would slightly confuse it. We need to create another variable to tell it to ignore (mask) the padding we've added when it's processing its input. That's what `attention_mask` is for:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "4K_iGRNa_Ozc", - "outputId": "8802e284-ba3c-4bd3-bc6c-d9830c033b30" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([2000, 59])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "attention_mask = torch.where(padded_texts > 0, 1, 0)\n", - "attention_mask.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAD8CAYAAAC8TPVwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdl0lEQVR4nO3de7BdVYHn8e+PiEP7GtQIExMYcCbYDVQThzRQZU83SqvBsTs4NSo4I4i0EQpGrHJqeExX06OVKnpUlPEBEyEFqRFiSnxkrNARmLawawgk+OIR0fBouCZFxKCk2+rIvfnNH3tfcrg5995z7j2Pvff5fapu3bPX3mefFYr7W+usvfZesk1ERNTDIcOuQEREdC6hHRFRIwntiIgaSWhHRNRIQjsiokYS2hERNTJraEs6StLfStou6SFJl5blr5F0h6Sflb9f3fKeKyTtkPSIpHe0lJ8s6YFy3/+UpP78syIihkvSWkm7JT04zX6VObhD0o8l/ZtOzttJT3sc+Ljt3wNOAy6WdDxwOXCX7aXAXeU25b6zgROAFcCXJC0oz3UdsApYWv6s6KSSERE1dBMzZ9yZHMjCVRT5OKtZQ9v2LtvfL1/vBbYDi4GVwM3lYTcDZ5WvVwLrbe+z/TiwAzhF0iLgVbbvcXFHz7qW90RENIrtu4E9MxyyEljnwhbg8DInZ/SSbioh6RjgTcC9wJG2d5WV2yXpiPKwxcCWlreNlWXPl6+nlrf7nFUULQ8vf5lO/t1//dJuqhkxq5/++GXDrkL0wV6efcb26+b6/ne85eX+5Z6Jjo69/8f7HgL+qaVoje01XXzcYuCplu3JTNw105s6Dm1JrwBuAz5m+7kZhqPb7fAM5QcXFv/wNQDLTzrM920+utNqVtY7Xn/SsKsQLU7N1ZRGutNf+/v5vP+ZPRPcu3lJR8ceuujRf7K9fB4f13EmtuootCUdShHYX7H99bL4aUmLyl72ImB3WT4GHNXy9iXAzrJ8SZvykbB554+G9tlpMCI6ZSa8f1AfNl1WzmjW0C5neNwIbLd9TcuujcB5wNXl72+1lN8i6Rrg9RSD7PfZnpC0V9JpFMMr5wKfn+3zmypBGlE9BvbP3tntlY3AJZLWA6cCv54ccp5JJz3tNwMfAB6Q9MOy7EqKsN4g6QLgSeA9ALYfkrQBeJhi5snFticHiS6iuKL6O8Dt5U/tJYAjmmM/velpS7oVOB1YKGkMuAo4FMD29cAm4J0UkzV+A5zfyXlnDW3bf0f7sReAM6Z5z2pgdZvybcCJnVSsToY59NEUafiiCox5vkfDI7bPmWW/gYu7PW9Xs0di7hJKEdVnYGJwwyNzktAekKr1xtOIRLQ3wDHtOUloj6iqNSL9lAYqOmVgouKreSW0o/FGqYHqp1Fp/AY24W+OEtojYFT+2CLmyzhj2jF86WmOjjTQ82PD89XO7IR2dCehEM0mJqad4VwNCe3oSnrtMamJDbiB/elpR9RTE0MpZpeedkQbCcSoouLmmoR2xEEyzNIfaQznx8DzrvbSuQntqISETVSBERMVX+88oT2iEpIR7e13hkeijYRmRPVkTDumlTHd0ZUGu8rERMa0A/KHGlEHxco1Ce2gfz3rNAYRvWOL33rBsKsxo4R2RESL/XUf05a0FngXsNv2iWXZV4E3loccDvzK9jJJxwDbgUfKfVtsX1i+52QOrA+5Cbi0XG4n5iFj4/WTb0fVVVyIrP/wyE3AF4B1kwW23zf5WtJngF+3HP+o7WVtznMdsArYQhHaK2jIwr51lOCIaKcBFyJt3132oA8iScB7gbfOdA5Ji4BX2b6n3F4HnEVCe2hGvYeeRivaGYULkf8WeNr2z1rKjpX0A+A54C9sfw9YDIy1HDNWlkWXEjYR/TXR8JtrzgFubdneBRxt+5flGPY3JZ0AbUf2px3PlrSKYiiFoxfnWmmrqT3khHhE7xjxvKudOXOunaSXAP8eOHmyzPY+YF/5+n5JjwLHUfSsl7S8fQmwc7pz214DrAFYftJhuVg5g1Ef5uinNIijpykXIqfzJ8BPbL8w7CHpdcAe2xOS3gAsBR6zvUfSXkmnAfcC5wKfn0/FI/qtkwYxwd4sRvUfHpF0K3A6sFDSGHCV7RuBs3nx0AjAHwGfkDQOTAAX2t5T7ruIA1P+bqcGFyHzBxkxemp/IdL2OdOUf7BN2W3AbdMcvw04scv6DVWGHoYrjWYMmk39p/xFDEuvGs2Ef3SquBCZ29hjjhI2EYPX5AuR0WcZnqmWNKLNZ5RFEKKaEkAR7aWnHZWUXvzoSoM9PQP7cyEy5ip/XBGDpiw3FnM3Sr3hNFBRBYbMHmmahEtEc9mq/PBItWsXETFgEz6ko59OSFoh6RFJOyRd3mb/P5f0fyT9SNJDks6f7ZzpaXepDkMW+TYQMTfF87R7M6YtaQHwReBtFA/N2yppo+2HWw67GHjY9p+Wz256RNJXbP92uvMmtBuoDg1LHaTxG0U9XbnmFGCH7ccAJK0HVgKtoW3gleWCMq8A9gDjM500oR2VkICMKiim/HXc014oaVvL9prysdKTFgNPtWyPAadOOccXgI0Uj6p+JfA+2/tn+tCEdkREqctnjzxje/kM+ztZ/OUdwA8plmz8V8Adkr5n+7npTprQ7oP0GiPqq4ePZh0DjmrZbrf4y/nA1bYN7JD0OPC7wH3TnTSh3QcZU26GNL6jp3g0a89urtkKLJV0LPBzijUI3j/lmCeBM4DvSToSeCPw2EwnTWgPSQIhopp69cAo2+OSLgE2AwuAtbYfknRhuf964JPATZIeoBhOucz2MzOdN6E9JHlWdET1FE/5693tK7Y3AZumlF3f8non8PZuzpnQrrmsYxjRO8Vt7NW+5zChHRHxgurfxt7Jwr5rgXcBu22fWJb9FfBh4BflYVeWXwOQdAVwAcXCvh+1vbksP5kDC/tuAi4tr5hGn9X1wmi+IcQw9OqOyH7ppKd9E8UE8HVTyj9r+9OtBZKOp7hCegLweuBOScfZngCuA1YBWyhCewUVX5E9oRExWno8e6QvOlmN/W5Jx3R4vpXAetv7gMcl7QBOkfQE8Crb9wBIWgecRcVDu6491HixNL7RjaoPj8yndpdI+rGktZJeXZa1u21zcfkz1qa8LUmrJG2TtO0Xv5yYRxUjIjo3uUZkJz/DMtfQvo7ilstlwC7gM2X5dLdtdnI754Ed9hrby20vf91rq/1A8ohoDgPjPqSjn2GZ0+wR209Pvpb0ZeDb5eZ0t22Ola+nlseQZMggor2qD4/MKbQlLbK9q9x8N/Bg+XojcIukayguRC4F7rM9IWmvpNOAe4Fzgc/Pr+oxHxmvH5w0kDUy5KGPTnQy5e9W4HSKxxCOAVcBp0taRvFt4gngIwDlLZobKJ4XOw5cXM4cAbiIA1P+bqfiFyEjYROjp5eLIPRLJ7NHzmlTfOMMx68GVrcp3wac2FXtRlxCM2Lwat/TjuHJ80kiBqvLRRCGIqFdMwngiP4xYnx/Ay9ExvDkAuLgpIEcTbUf0476SdhEzJEzPBJDkN5499LQBWRMO2LgEr4xXwntiAGqw7eMNCzVZcRELkRGL+UPPqK/ciEyeipztyP6x7kQGcOQQI6YOye0R09CM6KuGvDAqOheHS6G1UEavxiG9LQjImrChon9Ce2IOck3luEZ5W85mT0SUQOjHFJxgMnwSERbCcmoplyIjApIQEZ0ztMuOV4NCe0RMOpjw2m0ohu1Hx6RtBZ4F7Db9oll2aeAPwV+CzwKnG/7V5KOAbYDj5Rv32L7wvI9J3NgjchNwKV21du06kkARfRPMXuk/s8euQn4ArCupewO4Arb45L+GrgCuKzc96jtZW3Ocx2wCthCEdoryOK+XRv1XvNUacSi16relexkYd+7yx50a9l3Wja3AP9hpnNIWgS8yvY95fY64CwS2jFPUxuxhHjMV+2HRzrwIeCrLdvHSvoB8BzwF7a/BywGxlqOGSvL2pK0iqJXztGLM+wehQRy9JtRs0Nb0n8DxoGvlEW7gKNt/7Icw/6mpBOg7Wz1ab+E2F4DrAFYftJhFf+yMlwJsojeqnrgzDm0JZ1HcYHyjMkLirb3AfvK1/dLehQ4jqJnvaTl7UuAnXP97KpLkEbUlME9vI1d0grgWmABcIPtq9scczrwOeBQ4BnbfzzTOecU2mVFLgP+2PZvWspfB+yxPSHpDcBS4DHbeyTtlXQacC9wLvD5uXx2HeRi4fCkwYz56tXwiKQFwBeBt1F0XLdK2mj74ZZjDge+BKyw/aSkI2Y7bydT/m4FTgcWShoDrqKYLfLPgDskwYGpfX8EfELSODABXGh7T3mqizgw5e92anoRMqEQ0Ww9nD1yCrDD9mMAktYDK4GHW455P/B1208Wn+3ds520k9kj57QpvnGaY28Dbptm3zbgxNk+r+rSi45BSOdgOLp89shCSdtatteU1+MmLQaeatkeA06dco7jgEMlfRd4JXCt7XXMIFMzYloJjhg5BjoP7WdsL59hfycTMF4CnAycQTEKcY+kLbZ/Ot1JE9oxraxHGaOoh8MjY8BRLdvtJmCMUYT/PwL/KOlu4CQgoR3DkyGlF0sjVmXq5eyRrcBSSccCPwfOphjDbvUt4AuSXgK8lGL45LMznTShHX2XkIpa6VFPu3zMxyXAZoopf2ttPyTpwnL/9ba3S/ob4MfAfoppgQ/OdN6EdvRdetovlkaswtzb29htb6J41lJr2fVTtj8FfKrTcya0ozYSdjEQFb8lMqHdAwmTiCZp8LNHojDMr/9pMCJ6bP+wKzCzhPaQJGwjKqi7edpDkdDuUsI2otlqvwhCvFgdZkKkYYmYh4R2DFodGpamSAPZQBkeiblKIEQMntLTro+EZMSIs6CHiyD0Q0K7RR2HFdLQRPRYetrRT3VsaEZdGtqKS2jHqEtIRa0ktKNJEsDRaE24uUbSWopV13fbPrEsew3wVeAY4AngvbafLfddAVxAsUbkR21vLstP5sAakZuASydXcY/BS/hGtNeE2SM3AV8AWtctuxy4y/bVki4vty+TdDzFg75PAF4P3CnpONsTwHXAKmALRWivoKaL+zZBP8fC0yBErVU8tA+Z7QDbdwN7phSvBG4uX98MnNVSvt72PtuPAzuAUyQtAl5l+56yd72u5T0REZUhd/YzLHMd0z7S9i4A27skHVGWL6boSU8aK8ueL19PLW9L0iqKXjlHL86w+3yl5xvRhbqPaXdputWHO1mV+MCOYhn6NQDLTzqs4l9Wqq+ToZAEewTlhchhV2Jmcw3tpyUtKnvZi4DdZfl0qw+Pla+nlkeXEq4RfdbQ0N4InAdcXf7+Vkv5LZKuobgQuRS4z/aEpL2STgPuBc4FPj+vmjdQAjli+FT3RRAk3QqcDiyUNAZcRRHWGyRdADwJvAegXGl4A/AwMA5cXM4cAbiIA1P+biczRw7SrxkdaQwiulD3nrbtc6bZdcY0x68GVrcp3wac2FXtoidyq/twpdGsj2HPDOlEI6dm5I8kIuZsxGaPVEJ6lqMjDXT0XHra0U7CJqKaMjwSbeXbQG+k8YuecgNmj0RUWVMavzQ+FZKedsTBElJRWQntiIM1pYc8TGn4+iNj2tFW/uAiYi4S2kPSq55mwj+ix9LTjn7KE/wieiizR6IKejl+nAYgGi897WiSYV5ATIMR/SZyITKiZ5o84yQNUoUktCOqJyEZbfX4KX+SVgDXAguAG2xfPc1xf0CxVOP7bH9tpnMmtKPvEpBRKz26EClpAfBF4G0Uq3dtlbTR9sNtjvtrYHMn501o11wCMaK3etjTPgXYYfsxAEnrgZUUi8S0+s/AbcAfdHLShHbNNXmcN3ojDXuXOg/thZK2tWyvKRcln7QYeKpleww4tfUEkhYD7wbeSkI75it/7DFyuluN/Rnby2fY3241haln/xxwWbmObkcfOufQlvRG4KstRW8A/hI4HPgw8Iuy/Erbm8r3XAFcAEwAH7Xd0RhODEd68S+WRmw09HB4ZAw4qmV7CbBzyjHLgfVlYC8E3ilp3PY3pzvpnEPb9iPAMnhhIP3nwDeA84HP2v506/GSjgfOBk6gWKn9TknHtSz8GyMkARiV1bvQ3goslXQsRT6eDbz/RR9lHzv5WtJNwLdnCmzo3fDIGcCjtv9+hi7+SmC97X3A45J2UAzU39OjOkSN1LEXn4ZmNPTqNnbb45IuoZgVsgBYa/shSReW+6+fy3l7FdpnA7e2bF8i6VxgG/Bx289SDMpvaTlmrCw7iKRVwCqAoxdn2L2JEoBRSd2Nac9+umJoeNOUsrZhbfuDnZxz3oko6aXAnwFXlEXXAZ+k+Kd/EvgM8CE6G5QvCosrsGsAlp90WMXvT2quBGuMGtE+qKqkF93YM4Hv234aYPI3gKQvA98uNzsZlI8KyRMEYyRVvJvYi9A+h5ahEUmLbO8qN98NPFi+3gjcIukaiguRS4H7evD5MQcJ24j2Gv3AKEkvo7hF8yMtxf9D0jKK9uqJyX3lAPwGiruBxoGLM3Nk/hK+ET3W5NC2/RvgtVPKPjDD8auB1fP5zHixOs7CGLQ0bNGxLIIQMXyDbNjSQDRAk3vaoyh/lBHN1ugx7TpJ2EZERxLa1ZCx32pJIxpVlZ521FaCNUaO6dkiCP2S0I5p5eaaGDVZ2DcaJyEdjZfQjl5KaEb0l1zt1E5o10wuqHYvDV10rMdP+euHhHbURsI3BiFj2hE9MtdvGQn76EZuY49aSLBFlNLTjk4lOCOGzBkeiS7kIuPwpMGMFyS0I6qvag1mGpHhyM01ET2UIItB0P5qp3ZCu0sJjogGyzztektAR4yeRk/5k/QEsBeYAMZtL5f0GuCrwDEUa0S+1/az5fFXABeUx3/U9ub5fH6/VW2cM+YmjW90ZQR62m+x/UzL9uXAXbavlnR5uX2ZpOOBs4ETKFZjv1PScU1Y3DehENEco3ghciVwevn6ZuC7wGVl+Xrb+4DHJe0ATgHu6UMdBqpqPfI0IhFzZKDhD4wy8B1JBv6X7TXAkbZ3AdjeJemI8tjFwJaW946VZQeRtApYBXD04mYMuydII+qh0WPawJtt7yyD+Q5JP5nhWLUpa9ukleG/BmD5SYdVu9nrUNV641OlUYkYgXnatneWv3dL+gbFcMfTkhaVvexFwO7y8DHgqJa3LwF2zufz6yShGFEDdnOHRyS9HDjE9t7y9duBTwAbgfOAq8vf3yrfshG4RdI1FBcilwL3zaPutVL1nnYvpYGKOmtyT/tI4BuSJs9zi+2/kbQV2CDpAuBJ4D0Ath+StAF4GBgHLm7CzJE42DAbqDQYMW9NDW3bjwEH/YXY/iVwxjTvWQ2snutnRsymlw1GGoDR1OSedsTQJVijpwxMVDu1E9ojKmEX0V562tFXCd+IHuvh7BFJK4BrgQXADbavnrL/P1LcfAjwD8BFtmcc40tot0gARkSvetqSFgBfBN5GMeV5q6SNth9uOexx4I9tPyvpTIr7U06d6bwJ7RZNmZaXxidijnr7aNZTgB3lpA0krad4nMcLoW37/7Ucv4Xi/pUZJbRrJoEc0T8C1PmFyIWStrVsrynv5p60GHiqZXuMmXvRFwC3z/ahCe2aacq3gWFL4xfTUedj2s/YXj7TqdqUtT25pLdQhPYfzvahCe3ouwRk1EZvh0c6enSHpN8HbgDOLO9zmVFCO3ouIR311dNnj2wFlko6Fvg5xXoC7289QNLRwNeBD9j+aScnTWiPqARrRHu9mj1ie1zSJcBmiil/a8vHeVxY7r8e+EvgtcCXykeCjM8y5JLQHlUZG3+xNGLxgh7O07a9Cdg0pez6ltd/Dvx5N+dMaEfQvhFLkI8gdzV7ZCgS2jEUCcSorGpndkK7ShJkEcPXxZS/oUhoV0ivxpkT/hHzkNBuvoRkREMYaPjCvkFmYkyVRizqSjjDI1FIkEXUxP5qd7Xns7DvUcA64F9QfKFYY/taSX8FfBj4RXnoleVcRSRdQXF//QTwUdub51H3WplLbzxBHzFgDR8eGQc+bvv7kl4J3C/pjnLfZ21/uvVgScdT3MZ5AsVq7HdKOi6L+05v2MMuaTRiFDV2eMT2LmBX+XqvpO0UjyKczkpgve19wOOSdlA8b/aeudYh+mvYjcZUaURiIJoa2q0kHQO8CbgXeDNwiaRzgW0UvfFnKQJ9S8vbxpg55KOCEpzRbD19YFRfzDu0Jb0CuA34mO3nJF0HfJJidOiTwGeAD9Hds2VXAasAjl6ca6VVMrX3nRCPRmn6auySDqUI7K/Y/jqA7adb9n8Z+Ha52dGzZctzrKFYK43lJx1W7f+CI66TIZQEe9RJY8e0VTxH8EZgu+1rWsoXlePdAO8GHixfbwRukXQNxYXIpcB9c/38qI+qjY23k4YlXtDU0KYYu/4A8ICkH5ZlVwLnSFpG8UXjCeAjAOVzZDdQLGo5Dlxcx5kj+eOOaDAD+xsa2rb/jvbj1JvalE2+ZzWweq6fWQVz7TUm7CPqYAQuREZn8jCoiJpIaEcv1WF8uFfSQMXAGZio9i2RCe2aS7BF9JLBCe3oo1HqebeTRit6LsMjEf1Th0YrDUuNNHn2SDRfwiZGUnraoydhF1FjCe2IiJqwYaLa9/wltPugDuOsoyTffKIr6WlHOwmSiIpKaEc76Y3PLg1bDJ4zeySaLcEajWJwbq6JKkrYRkwjt7FHFQ17eCaNRlSSDfsT2hEHybJlUVm5EBm9lHCL6C+npx29NOxhjaZKYxiFLIIQXUhwRAxZHhgV3UgvutrSqDafAec29heTtAK4FlgA3GD76kHXYRjyBx9RA+7tIgiz5Z0klfvfCfwG+KDt7890zoGGtqQFwBeBtwFjwFZJG20/PMh6DMMo9aLTQEWduUfDIx3m3ZnA0vLnVOC68ve0Bt3TPgXYYfsxAEnrgZVA40N7lIxSA1U1aTB7oHc97U7ybiWwzraBLZIOl7TI9q7pTjro0F4MPNWyPUabVkXSKmBVublvwaKfPTiAuvXSQuCZYVeiS3WsM9Sz3n2s88/6c9pCHf5b/8v5vHkvz26+019b2OHhh0na1rK9xvaalu1O8q7dMYuByoS22pQd9F2k/IevAZC0zfbyflesl1LnwaljvetYZ6hvvbthe0UPT9dJ3nWUia0OmXN15mYMOKplewmwc8B1iIgYhE7yrutMHHRobwWWSjpW0kuBs4GNA65DRMQgdJJ3G4FzVTgN+PVM49kw4OER2+OSLgE2U0yBWWv7oVnetmaW/VWUOg9OHetdxzpDfes9FNPlnaQLy/3XA5sopvvtoJjyd/5s55UrfstmREQcMOjhkYiImIeEdkREjVQ2tCWtkPSIpB2SLh92faYjaa2k3ZIebCl7jaQ7JP2s/P3qYdZxKklHSfpbSdslPSTp0rK8svWWdJik+yT9qKzzfy/LK1vnSZIWSPqBpG+X23Wo8xOSHpD0w8m5yHWo9yioZGi33P55JnA8cI6k44dbq2ndBEyd23k5cJftpcBd5XaVjAMft/17wGnAxeV/3yrXex/wVtsnAcuAFeXV9irXedKlwPaW7TrUGeAttpe1zM2uS70brZKhTcvtn7Z/C0ze/lk5tu8G9kwpXgncXL6+GThrkHWaje1dkw+lsb2XIlAWU+F6u/AP5eah5Y+pcJ0BJC0B/h1wQ0txpes8g7rWu1GqGtrT3dpZF0dOzrUsfx8x5PpMS9IxwJuAe6l4vcthhh8Cu4E7bFe+zsDngP8KtD7Qoup1hqJB/I6k+8vHSkA96t14VX2edte3dkb3JL0CuA34mO3niqdEVpftCWCZpMOBb0g6cchVmpGkdwG7bd8v6fQhV6dbb7a9U9IRwB2SfjLsCkWhqj3tut/u/rSkRQDl791Drs9BJB1KEdhfsf31srjy9Qaw/SvguxTXEqpc5zcDfybpCYohvrdK+t9Uu84A2N5Z/t4NfINiyLLy9R4FVQ3tut/uvhE4r3x9HvCtIdblIOWD128Ettu+pmVXZest6XVlDxtJvwP8CfATKlxn21fYXmL7GIr/h/+v7f9EhesMIOnlkl45+Rp4O/AgFa/3qKjsHZGS3kkxHjh5++fq4daoPUm3AqdTPLbyaeAq4JvABuBo4EngPbanXqwcGkl/CHwPeIADY61XUoxrV7Lekn6f4uLXAorOxgbbn5D0Wipa51bl8Mh/sf2uqtdZ0hsoetdQDKHeYnt11es9Kiob2hERcbCqDo9EREQbCe2IiBpJaEdE1EhCOyKiRhLaERE1ktCOiKiRhHZERI38f5y5dGU1i1MUAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.pcolormesh(attention_mask)\n", - "plt.colorbar();" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jK-CQB9-kN99" - }, - "source": [ - "## And Now, Deep Learning!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jK-CQB9-kN99" - }, - "source": [ - "Now that we have our model and inputs ready, let's run our model!\n", - "\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "id": "39UVjAV56PJz" - }, - "outputs": [], - "source": [ - "with torch.no_grad():\n", - " output = model(padded_texts, attention_mask)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FoCep_WVuB3v" - }, - "source": [ - "Let's slice only the part of the output that we need. That is the output corresponding the first token of each sentence. The way BERT does sentence classification, is that it adds a token called `[CLS]` (for classification) at the beginning of every sentence. The output corresponding to that token can be thought of as an embedding for the entire sentence.\n", - "\n", - "\n", - "\n", - "We'll save those in the `features` variable, as they'll serve as the features to our logitics regression model." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "id": "C9t60At16PVs" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(2000, 768)" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "features = output.last_hidden_state[:, 0, :].numpy()\n", - "features.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_VZVU66Gurr-" - }, - "source": [ - "The labels indicating which sentence is positive and negative now go into the `labels` variable" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "id": "JD3fX2yh6PTx" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(2000,)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "labels = dataset['label'].values\n", - "labels.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iaoEvM2evRx1" - }, - "source": [ - "## Classifier training" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iaoEvM2evRx1" - }, - "source": [ - "Let's now split our datset into a training set and testing set (even though we're using 2,000 sentences from the SST2 training set)." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "id": "ddAqbkoU6PP9" - }, - "outputs": [], - "source": [ - "from sklearn.model_selection import train_test_split\n", - "\n", - "train_features, test_features, train_labels, test_labels = train_test_split(features, labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B9bhSJpcv1Bl" - }, - "source": [ - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B9bhSJpcv1Bl" - }, - "source": [ - "## [Extra] Grid Search for Parameters" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B9bhSJpcv1Bl" - }, - "source": [ - "We can dive into Logistic regression directly with the Scikit Learn default parameters, but sometimes it's worth searching for the best value of the C parameter, which determines regularization strength." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "id": "cyEwr7yYD3Ci" - }, - "outputs": [], - "source": [ - "# from sklearn.model_selection import GridSearchCV\n", - "\n", - "# parameters = {'C': np.linspace(0.0001, 100, 20)}\n", - "# grid_search = GridSearchCV(LogisticRegression(), parameters)\n", - "# grid_search.fit(train_features, train_labels)\n", - "\n", - "# print('best parameters: ', grid_search.best_params_)\n", - "# print('best scrores: ', grid_search.best_score_)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KCT9u8vAwnID" - }, - "source": [ - "We now train the LogisticRegression model. If you've chosen to do the gridsearch, you can plug the value of C into the model declaration (e.g. `LogisticRegression(C=5.2)`)." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "gG-EVWx4CzBc", - "outputId": "9ae43345-4003-4dfe-bb22-cd9c2fa82fe0" - }, - "outputs": [], - "source": [ - "import warnings; warnings.simplefilter('ignore') # Ignore warning on model fitting.\n", - "from sklearn.linear_model import LogisticRegression\n", - "\n", - "lr_clf = LogisticRegression().fit(train_features, train_labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3rUMKuVgwzkY" - }, - "source": [ - "\n", - "\n", - "So how well does our model do in classifying sentences? One way is to check the accuracy against the testing dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "iCoyxRJ7ECTA", - "outputId": "45b90744-a478-45db-a420-2ebbaf0b9236" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "0.84" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "lr_clf.score(test_features, test_labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "75oyhr3VxHoE" - }, - "source": [ - "How good is this score? What can we compare it against? Let's first look at a dummy classifier:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "lnwgmqNG7i5l", - "outputId": "fe7730c4-446b-4a1d-f4f2-1164c45a0a31" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Dummy classifier score: 0.519 (+/- 0.00)\n" - ] - } - ], - "source": [ - "from sklearn.dummy import DummyClassifier\n", - "from sklearn.model_selection import cross_val_score\n", - "\n", - "clf = DummyClassifier()\n", - "\n", - "scores = cross_val_score(clf, train_features, train_labels)\n", - "print(\"Dummy classifier score: %0.3f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7Lg4LOpoxSOR" - }, - "source": [ - "So our model clearly does better than a dummy classifier. But how does it compare against the best models?\n", - "\n", - "For reference, the [highest accuracy score](http://nlpprogress.com/english/sentiment_analysis.html) for this dataset is currently **96.8**. DistilBERT can be trained to improve its score on this task – a process called **fine-tuning** which updates BERT’s weights to make it achieve a better performance in this sentence classification task (which we can call the downstream task). The fine-tuned DistilBERT turns out to achieve an accuracy score of **90.7**. The full size BERT model achieves **94.9**.\n", - "\n", - "And that’s it! That’s a good first contact with BERT. The next step would be to head over to the documentation and try your hand at [fine-tuning](https://huggingface.co/transformers/examples.html#glue). You can also go back and switch from distilBERT to BERT and see how that works." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 152 - }, - "id": "EJQuqV6cnWQu", - "outputId": "402d109c-01bb-485d-a510-4be8684c9c06" - }, - "source": [ - "## Part 2: Looking back." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 152 - }, - "id": "EJQuqV6cnWQu", - "outputId": "402d109c-01bb-485d-a510-4be8684c9c06" - }, - "source": [ - "__Now it is your turn to reproduce the steps above.__\n", - "\n", - "We shall revisit the first homework and see whether we could improve the results a little bit more. The average ROC-AUC on test set was around $0.9$ (using the words embeddings). \n", - "\n", - "__Let's see whether we can beat it.__" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "kz8QBEXozHJx", - "outputId": "bdf0a0d8-2ac5-4dfd-a609-72011121abda" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
should_bancomment_text
500\"Those who're in advantageous positions are th...
2501Fartsalot56 says f**k you motherclucker!!
4501Are you a fool? \\n\\nI am sorry, but you seem t...
6501I AM NOT A VANDAL!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
8500Citing sources\\n\\nCheck out the Wikipedia:Citi...
\n", - "
" - ], - "text/plain": [ - " should_ban comment_text\n", - "50 0 \"Those who're in advantageous positions are th...\n", - "250 1 Fartsalot56 says f**k you motherclucker!!\n", - "450 1 Are you a fool? \\n\\nI am sorry, but you seem t...\n", - "650 1 I AM NOT A VANDAL!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", - "850 0 Citing sources\\n\\nCheck out the Wikipedia:Citi..." - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import os; os.environ['TOKENIZERS_PARALLELISM'] = 'false' # Ignore warning on wget.\n", - "\n", - "!wget -q -nc https://raw.githubusercontent.com/neychev/made_nlp_course/master/datasets/comments_small_dataset/comments.tsv\n", - "dataset = pd.read_csv('comments.tsv', sep='\\t')\n", - "dataset[50::200]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's prepare data for our BERT model." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "tokenized_texts = dataset['comment_text'].apply(lambda x: tokenizer.encode(x, add_special_tokens=True, max_length=512, truncation=True)).values\n", - "\n", - "max_len = max(len(text) for text in tokenized_texts)\n", - "padded_texts = torch.tensor([text + [0] * (max_len - len(text)) for text in tokenized_texts])\n", - "\n", - "attention_mask = torch.where(padded_texts > 0, 1, 0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now move the model to GPU and use it for feature extraction." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "device(type='cuda', index=2)" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "gpu_num = 0\n", - "device = torch.device(f'cuda:{gpu_num}' if torch.cuda.is_available() else 'cpu')\n", - "device" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1000, 768)" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "\n", - "model.to(device)\n", - "batch_size = 16\n", - "features = []\n", - "with torch.no_grad():\n", - " for i in range(0, len(padded_texts), batch_size):\n", - " texts_batch = padded_texts[i : i + batch_size].to(device)\n", - " mask_batch = attention_mask[i : i + batch_size].to(device)\n", - " output = model(texts_batch, mask_batch)\n", - " batch_features = output.last_hidden_state[:, 0, :].cpu().numpy()\n", - " features.append(batch_features)\n", - "\n", - "features = np.concatenate(features, axis=0)\n", - "features.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now it is time to split our objects into train and test and train our classifier." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "target = dataset['should_ban'].values\n", - "train_features, test_features, y_train, y_test = train_test_split(features, target, test_size=0.5, random_state=42)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.862" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "lr_clf = LogisticRegression(C=0.1)\n", - "lr_clf.fit(train_features, y_train)\n", - "lr_clf.score(test_features, y_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's also plot the ROC curve and calculate the AUC metric." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfD0lEQVR4nO3de3BV1d3/8feXm4iAKCAiEQhyDVdLBC1IsdUK2kJRRLCDPliKjtBSWnzEzqij/lQcrFUUS30UtLUmikqhFRUvWOwgYHBQIKBQ5BJQRISqIEbg+/vj5BxPkpOcnXByOTuf10zG7L3X2Xstgp8s1t57LXN3REQk/dWr6QqIiEhqKNBFREJCgS4iEhIKdBGRkFCgi4iERIOaunCrVq28Y8eONXV5EZG0tGbNms/cvXWiYzUW6B07diQvL6+mLi8ikpbMbHtZxzTkIiISEgp0EZGQUKCLiISEAl1EJCQU6CIiIZE00M1snpl9ambryzhuZjbbzLaY2ftm9r3UV1NERJIJ0kN/AhhWzvHhQJeir0nAn46/WiIiUlFJn0N39+Vm1rGcIiOBv3hkHt6VZtbCzNq6+8epqqRIunh61Q4Wrd1VLdf60aElDPp6WbVcS1LryxY9OPeG/0v5eVPxYlE7YGfcdkHRvlKBbmaTiPTiad++fQouLXVBVYdkKoOx0+EjTAOaNa76d/Z6Fq4DYEOj3lV+LUkPqfhbZwn2JVw1w90fBR4FyM7O1soadciqBX+g6eaFlfpsVYdkKoOxWeMGtGp6Am2aNT7ucyU3GHqPpmf2hGq4lqSDVPwfUgCcGbedAexOwXklTcX3qKO934HHEZpVH5IKRgmHVAT6YmCKmeUCA4H/avw83JINgXTasYBp9VfQrHGDYr3fr7qMYuAVv6uuaorUOUkD3cxygKFAKzMrAG4DGgK4+1xgCXAJsAU4BKibk6bKCuqSY8zJhkB6NoyEOG0Ho96vSPUJ8pTLuCTHHZicshrJcavsTcRVH30OwMDMU4HvgrzkGHPyIZBIiKMQF6lWNTZ9rlSNp1ft4PcLIwEcDeagBmaeysh+7bhqYHvImw//nB050EG9bJF0oEBPY4l64tFe9t2jekeCOai8+bDuOcgn8rX935H9P3lAPW2RNKFATyMlA7zkEEn0+1gvuyzR8I4XDfAOg7/7r4ZNRNKKAj1NJBpKCRTeUDrAS4Z39HsFuEhaU6DXUmX1xis1lPLP30S+V+9bJNQU6DUgyFMoJYdTkvbGEw2jgMbCReoQBXo1CDL2XVK5AR5kDDxKvXGROkOBXkXiQ7zCve1k1j0Hn6yD0+Neo1dwi9R5CvQUiwZ5fIgfd4CX7JFHw3zCiymosYiEhQI9RRIF+XGFeFSim5qn9470xkVE4ijQU2TR2l3kf/xF6oI8Ktoz101NEUlCgV4B5T2dkv/xF2S1bc4z152X+gt3GKwwF5GkFOgVEB1SSfR0Slbb5ozs1y41F4ofMy9581NEpAwK9ICeXrUjFuZV0guPt+65yGOIHQZrvFxEAlOgBxQdaklZLzyZDoP1FIuIVIgCPYnouHn0hmfKbnaW9WYnfNc7FxGpAAV6Gcp6DPG4RYO8rDc7o/s0zCIiFaRAL0P8DdCUP4b4yTq92SkiKadAL6HkEEuV3ADVW54iUgXq1XQFaptomKf0McSovPnfDbWIiKSYeugJVPoFofJudMJ3Ya7xcRGpAuqhp1J0fLwsHQbrFX4RqTLqoR8PzYIoIrVInQ/0kvOzRMfPyxQf4iUfPdRbnSJSg+p8oMffBIUAc7LELy6hRw9FpBap04Fe6flZNKwiIrVQnb4pWuH5WfTYoYjUYnU60IGKzc8SHTvXOLmI1EJ1Zsgl0eIUSW+ARkVvhEZf2deYuYjUQnWih/70qh38fuG62ERbUYHfBo2/EareuYjUUnWihx7tmd89qnflJ9nSjVARqeUC9dDNbJiZfWBmW8xsRoLjJ5vZP8zsPTPbYGa1bkwipXOZi4jUQkkD3czqA3OA4UAWMM7MskoUmwzku3tfYCjwBzNrlOK61gw92SIiaSJID30AsMXdt7p7IZALjCxRxoFmZmZAU+Bz4EhKa1pJ0WfNK01PtohImggS6O2AnXHbBUX74j0M9AB2A+uAqe5+rOSJzGySmeWZWd7evXsrWeWKOa61QKO9cz3ZIiJpIMhNUUuwz0tsXwysBX4InAW8amZvufsXxT7k/ijwKEB2dnbJc6RUpdYCLTnZlqa7FZE0EiTQC4Az47YziPTE400AZrq7A1vM7COgO7A6JbWshAotVFHWOp+aq0VE0kiQQH8H6GJmmcAuYCxwVYkyO4AfAW+ZWRugG7A1lRWtjMALVWidTxEJgaSB7u5HzGwK8ApQH5jn7hvM7Pqi43OBO4EnzGwdkSGam9z9syqsd+rpOXMRSXOBXixy9yXAkhL75sZ9vxv4cWqrJiIiFRGqN0Xj52sJPE+LiEhIhGoul+iNUKjAPC0iIiERmh56pReriH/WXEQkjYWmh17pF4j0JqiIhERoAh0qMQGX3gQVkRAJRaBXer4W9c5FJERCEejHNV+LeuciEhKhCHTQfOciIqEJdBGRui6tH1uMn1FRLxGJSF2X1j30Cs2oKCIScmndQ4cKzKgYFT/n+SfrIpNyiYiEQFr30CslOlUuRMJcjyyKSEikfQ+9UjRVroiEUN3roYuIhFTdCvToq/4iIiFUdwI9bz788zeR7zVuLiIhlLaBXuH5W6JPtvzkAb3qLyKhlLaBXqn5WzRvi4iEWNoGOmj+FhGReGkd6CIi8h0FuohISCjQRURCIvyBnjcf5l/63ev+IiIhFd5X/6OTcEVfJOowWM+fi0iohTfQo5NwRYNcjyuKSMiFN9BBk3CJSJ0S/jF0EZE6QoEuIhISCnQRkZAIFOhmNszMPjCzLWY2o4wyQ81srZltMLN/pbaaIiKSTNKbomZWH5gDXAQUAO+Y2WJ3z48r0wJ4BBjm7jvM7LQqqq+IiJQhSA99ALDF3be6eyGQC4wsUeYq4AV33wHg7p+mtpoiIpJMkEBvB+yM2y4o2hevK3CKmb1pZmvM7OpEJzKzSWaWZ2Z5e/furVyNRUQkoSCBbgn2eYntBkB/4FLgYuAWM+ta6kPuj7p7trtnt27dusKVFRGRsgUJ9ALgzLjtDGB3gjIvu/tBd/8MWA70TU0VK0hzt4hIHRUk0N8BuphZppk1AsYCi0uUWQScb2YNzKwJMBDYmNqqBhBdN3T7vyNviWruFhGpQ5I+5eLuR8xsCvAKUB+Y5+4bzOz6ouNz3X2jmb0MvA8cAx5z9/VVWfGEtG6oiNRhgeZycfclwJIS++aW2J4FzEpd1SpJ64aKSB0VnjdF8+Z/N1WuiEgdlHazLT69ageL1u4i/+MvyGrb/LsD0eEWjZuLSB2VdoEeH+a/PXUFzP9/kQPRuc813CIidVRaDrlktW3OM9edx8Cv3vhumEVPtYhIHZd2PfRSOgzWIhYiIqRpD11EREpToIuIhIQCXUQkJBToIiIhoUAXEQkJBbqISEikb6DrVX8RkWLSN9D1qr+ISDHpG+igV/1FROKkd6CLiEiMAl1EJCQU6CIiIaFAFxEJCQW6iEhIKNBFREJCgS4iEhIKdBGRkEi7FYt+dGgJg75eBrYjsuyciIgAadhDH/T1Mjp+u1VriIqIlJB2PXSAbQ070VPriIqIFJN2PXQREUlMgS4iEhIKdBGRkFCgi4iEhAJdRCQkFOgiIiERKNDNbJiZfWBmW8xsRjnlzjGzo2amB8RFRKpZ0kA3s/rAHGA4kAWMM7OsMsrdC7yS6kqKiEhyQXroA4At7r7V3QuBXGBkgnK/Ap4HPk1h/UREJKAggd4O2Bm3XVC0L8bM2gGjgLnlncjMJplZnpnl7d27t6J1FRGRcgQJdEuwz0tsPwDc5O5HyzuRuz/q7tnunt26deuAVRQRkSCCzOVSAJwZt50B7C5RJhvINTOAVsAlZnbE3f+eikqKiEhyQQL9HaCLmWUCu4CxwFXxBdw9M/q9mT0B/FNhLiJSvZIGursfMbMpRJ5eqQ/Mc/cNZnZ90fFyx81FRKR6BJo+192XAEtK7EsY5O7+P8dfLRERqSi9KSoiEhIKdBGRkFCgi4iEhAJdRCQkFOgiIiGhQBcRCQkFuohISCjQRURCQoEuIhISCnQRkZBQoIuIhIQCXUQkJBToIiIhoUAXEQkJBbqISEgo0EVEQkKBLiISEgp0EZGQUKCLiISEAl1EJCQU6CIiIaFAFxEJCQW6iEhIKNBFREJCgS4iEhIKdBGRkFCgi4iEhAJdRCQkFOgiIiGhQBcRCYlAgW5mw8zsAzPbYmYzEhz/uZm9X/S1wsz6pr6qIiJSnqSBbmb1gTnAcCALGGdmWSWKfQT8wN37AHcCj6a6oiIiUr4gPfQBwBZ33+ruhUAuMDK+gLuvcPf9RZsrgYzUVlNERJIJEujtgJ1x2wVF+8ryC+ClRAfMbJKZ5ZlZ3t69e4PXUkREkgoS6JZgnycsaHYBkUC/KdFxd3/U3bPdPbt169bBaykiIkk1CFCmADgzbjsD2F2ykJn1AR4Dhrv7vtRUT0REggrSQ38H6GJmmWbWCBgLLI4vYGbtgReA8e7+YeqrKSIiySTtobv7ETObArwC1AfmufsGM7u+6Phc4FagJfCImQEccffsqqu2iIiUFGTIBXdfAiwpsW9u3PcTgYmprZqIiFSE3hQVEQkJBbqISEgo0EVEQkKBLiISEgp0EZGQUKCLiIREoMcWRaTmfPvttxQUFHD48OGaropUo8aNG5ORkUHDhg0Df0aBLlLLFRQU0KxZMzp27EjRi3sScu7Ovn37KCgoIDMzM/DnNOQiUssdPnyYli1bKszrEDOjZcuWFf5XmQJdJA0ozOueyvzMFegiIiGhQBeRch04cIBHHnmkUp+95JJLOHDgQIU/17dvX8aNG1ds39ChQ8nLy4ttb9u2jV69esW2V69ezZAhQ+jWrRvdu3dn4sSJHDp0KND1Xn75Zbp160bnzp2ZOXNmwjL79+9n1KhR9OnThwEDBrB+/frYsQMHDjB69Gi6d+9Ojx49ePvttwG48sor6devH/369aNjx47069cPiNzovuaaa+jduzc9evTgnnvuCVTPZHRTVETKFQ30G264odSxo0ePUr9+/TI/u2TJkjKPlWXjxo0cO3aM5cuXc/DgQU466aSkn9mzZw9XXHEFubm5nHfeebg7zz//PF9++SVNmjQp97NHjx5l8uTJvPrqq2RkZHDOOecwYsQIsrKKL5189913069fPxYuXMimTZuYPHkyr7/+OgBTp05l2LBhPPfccxQWFsZ+kTzzzDOxz//ud7/j5JNPBmDBggV88803rFu3jkOHDpGVlcW4cePo2LFjRf6oSlGgi6SR2/+xgfzdX6T0nFlnNOe2n/Ys8/iMGTP4z3/+Q79+/bjooou49NJLuf3222nbti1r164lPz+fn/3sZ+zcuZPDhw8zdepUJk2aBEDHjh3Jy8vjq6++Yvjw4QwePJgVK1bQrl07Fi1axIknnljqek8//TTjx49n48aNLF68uFRPPZE5c+ZwzTXXcN555wGR8efRo0cHav/q1avp3LkznTp1AmDs2LEsWrSoVKDn5+dz8803A9C9e3e2bdvGnj17OPHEE1m+fDlPPPEEAI0aNaJRo0bFPuvuPPvss7zxxhux+h08eJAjR47w9ddf06hRI5o3bx6ovuXRkIuIlGvmzJmcddZZrF27llmzZgGRELzrrrvIz88HYN68eaxZs4a8vDxmz57Nvn2lFy3bvHkzkydPZsOGDbRo0YLnn38+4fWeeeYZrrzySsaNG0dOTk6gOq5fv57+/fsnPLZs2bLYsEf81/e//30Adu3axZlnfrcoW0ZGBrt27Sp1nr59+/LCCy/E2r99+3YKCgrYunUrrVu3ZsKECZx99tlMnDiRgwcPFvvsW2+9RZs2bejSpQsAo0eP5qSTTqJt27a0b9+e6dOnc+qppwZqa3nUQxdJI+X1pKvTgAEDij0fPXv2bBYuXAjAzp072bx5My1btiz2mczMzNgYcv/+/dm2bVup877zzju0bt2aDh06kJGRwbXXXsv+/fs55ZRTEj71EeRJkAsuuIC1a9eWedy99BLJic47Y8YMpk6dSr9+/ejduzdnn302DRo04Ntvv+Xdd9/loYceYuDAgUydOpWZM2dy5513xj6bk5NT7F8aq1evpn79+uzevZv9+/dz/vnnc+GFF8b+lVBZCnQRqbD4ce0333yT1157jbfffpsmTZowdOjQhM9Pn3DCCbHv69evz9dff12qTE5ODps2bYqNJX/xxRc8//zzTJw4kZYtW7J///5Y2c8//5xWrVoB0LNnT9asWcPIkSNLnXPZsmVMmzat1P4mTZqwYsUKMjIy2LlzZ2x/QUEBZ5xxRqnyzZs3Z/78+UDkl0BmZiaZmZkcOnSIjIwMBg4cCER63/E3Vo8cOcILL7zAmjVrYvuefvpphg0bRsOGDTnttNMYNGgQeXl5xx3oGnIRkXI1a9aML7/8sszj//3vfznllFNo0qQJmzZtYuXKlZW6zrFjx1iwYAHvv/8+27ZtY9u2bSxatCg27DJ06FCeeuqpWI/6ySef5IILLgBgypQpPPnkk6xatSp2vqeeeopPPvkk1kMv+bVixQoAzjnnHDZv3sxHH31EYWEhubm5jBgxolT9Dhw4QGFhIQCPPfYYQ4YMoXnz5px++umceeaZfPDBBwC8/vrrxcbfX3vtNbp3705GRkZsX/v27XnjjTdwdw4ePMjKlSvp3r17pf7c4qmHLiLlatmyJYMGDaJXr14MHz6cSy+9tNjxYcOGMXfuXPr06UO3bt0499xzK3Wd5cuX065dO9q1axfbN2TIEPLz8/n444+ZNGkSmzZtom/fvpgZ2dnZscf92rRpQ25uLtOnT+fTTz+lXr16DBkyhMsuuyzpdRs0aMDDDz/MxRdfzNGjR7n22mvp2TMytDV3bmSlzeuvv56NGzdy9dVXU79+fbKysnj88cdj53jooYf4+c9/TmFhIZ06dYr15AFyc3NL3didPHkyEyZMoFevXrg7EyZMoE+fPpX6c4tnicaPqkN2drbHP1Ma1Ia7BwPQ8/f/TnWVRGqljRs30qNHj5quhtSARD97M1vj7tmJymvIRUQkJBToIiIhoUAXEQkJBbqISEgo0EVEQkKBLiISEgp0ESnX8UyfC/DAAw+UO43t3r17adiwIX/+85+L7W/atGmx7SeeeIIpU6bEtv/yl7/Qq1cvevbsSVZWFvfdd1/gOt1zzz107tyZbt268corryQs895773HeeefRu3dvfvrTn/LFF8UnRduxYwdNmzYtdt3CwkImTZpE165d6d69e2y+mmnTpsXmkOnatSstWrQIXNeKUKCLSLmqOtAXLFjAueeeG3giLoCXXnqJBx54gKVLl7Jhwwbefffd2NS0yeTn55Obm8uGDRt4+eWXueGGGzh69GipchMnTmTmzJmsW7eOUaNGxSYmi5o2bRrDhw8vtu+uu+7itNNO48MPPyQ/P58f/OAHAPzxj3+MvaH6q1/9KtALT5WhN0VF0slLM+CTdak95+m9YXjiRR2g9PS5s2bNYtasWTz77LN88803jBo1ittvv52DBw8yZswYCgoKOHr0KLfccgt79uxh9+7dXHDBBbRq1Yply5aVOn9OTg5/+MMfuOqqq9i1a1exN0XLcs8993DffffF5lxp3Lgxv/zlLwM1d9GiRYwdO5YTTjiBzMxMOnfuzOrVq2NT70Z98MEHDBkyBICLLrqIiy++ODbh1t///nc6depUaq72efPmsWnTJgDq1asXm2umZHtvv/32QHWtKAW6iJRr5syZrF+/PjZj4dKlS9m8eTOrV6/G3RkxYgTLly9n7969nHHGGbz44otAZI6Xk08+mfvvv59ly5YlDLedO3fyySefMGDAAMaMGcMzzzzDb3/726R1Km+63FmzZvG3v/2t1P4hQ4Ywe/Zsdu3aVWx6grKmy+3VqxeLFy9m5MiRLFiwIDaB18GDB7n33nt59dVXiw23RFdmuuWWW3jzzTc566yzePjhh2nTpk2szPbt2/noo4/44Q9/mLSNlaFAF0kn5fSkq8vSpUtZunQpZ599NgBfffUVmzdv5vzzz2f69OncdNNN/OQnP+H8889Peq7c3FzGjBkDRBaW+MUvflFuoAeZLvfGG2/kxhtvLPN40Oly582bx69//WvuuOMORowYEVu04rbbbmPatGmlxviPHDlCQUEBgwYN4v777+f+++9n+vTp/PWvf42Vyc3NZfTo0eWu8nQ8AgW6mQ0DHgTqA4+5+8wSx63o+CXAIeB/3P3dFNdVRGoBd+fmm2/muuuuK3VszZo1LFmyhJtvvpkf//jH3HrrreWeKycnhz179sR61Lt372bz5s106dKFE088kcLCwliQJpouN1FPN1kPPeh0ud27d2fp0qUAfPjhh7F/eaxatYrnnnuO//3f/+XAgQPUq1ePxo0bM3nyZJo0acKoUaMAuOKKK4pN4AWRQJ8zZ065fybHxd3L/SIS4v8BOgGNgPeArBJlLgFeAgw4F1iV7Lz9+/f3ylh/1yBff9egSn1WJB3l5+fX6PU/++wzb9++fWz7lVde8QEDBviXX37p7u4FBQW+Z88e37Vrl3/99dfu7r5w4UIfOXKku7v36tXLt27dWuq8mzZt8q5duxbbd+utt/odd9zh7u6XX365P/744+7ufujQIR84cKD/61//cnf3F1980fv37+8ff/yxu7sfPnzYH3zwwUDtWb9+vffp08cPHz7sW7du9czMTD9y5Eipcnv27HF396NHj/r48eNjdYl32223+axZs2LbV155pb/++uvu7j5//nwfPXp0sfZ26NDBjx07Fqie7ol/9kCel5GrQXroA4At7r4VwMxygZFAflyZkcBfii620sxamFlbd/84Bb9zRKQGlZw+d9asWWzcuDF2E7Fp06Y89dRTbNmyhRtvvJF69erRsGFD/vSnPwEwadIkhg8fTtu2bYvdFM3JyYn1ZqMuv/xyxo4dyy233MKDDz7Iddddx+zZs3F3rr766thNyksuuYQ9e/Zw4YUX4u6YGddee22g9vTs2ZMxY8aQlZVFgwYNmDNnTmwIZOLEiVx//fVkZ2eTk5MT601fdtllTJgwIem57733XsaPH89vfvMbWrduXWwa3ZycHMaOHRto2Kiykk6fa2ajgWHuPrFoezww0N2nxJX5JzDT3f9dtP06cJO755U41yRgEkD79u37b9++vcIVXvlI5E72uTf8X4U/K5KONH1u3VXR6XOD9NAT/Top+VsgSBnc/VHgUYjMhx7g2qUoyEVEEgvyYlEBcGbcdgawuxJlRESkCgUJ9HeALmaWaWaNgLHA4hJlFgNXW8S5wH81fi6SOsmGRiV8KvMzTzrk4u5HzGwK8AqRJ17mufsGM7u+6PhcYAmRJ122EHlsMfndAxEJpHHjxuzbt4+WLVtW6Q01qT3cnX379tG4ceMKfS7t1hQVqWu+/fZbCgoKOHz4cE1XRapR48aNycjIoGHDhsX2H+9NURGpQQ0bNiQzM7OmqyFpQLMtioiEhAJdRCQkFOgiIiFRYzdFzWwvUPFXRSNaAZ+lsDrpQG2uG9TmuuF42tzB3VsnOlBjgX48zCyvrLu8YaU21w1qc91QVW3WkIuISEgo0EVEQiJdA/3Rmq5ADVCb6wa1uW6okjan5Ri6iIiUlq49dBERKUGBLiISErU60M1smJl9YGZbzGxGguNmZrOLjr9vZt+riXqmUoA2/7yore+b2Qoz61sT9UylZG2OK3eOmR0tWkUrrQVps5kNNbO1ZrbBzP5V3XVMtQB/t082s3+Y2XtFbU7rWVvNbJ6ZfWpm68s4nvr8Kmux0Zr+oooWp67NXwHb/H3glKLvh9eFNseVe4PIVM2ja7re1fBzbkFk3d72Rdun1XS9q6HNvwfuLfq+NfA50Kim634cbR4CfA9YX8bxlOdXbe6hxxandvdCILo4dbzY4tTuvhJoYWZtq7uiKZS0ze6+wt33F22uJLI6VDoL8nMG+BXwPPBpdVauigRp81XAC+6+A8Dd073dQdrsQDOLTPrelEigH6neaqaOuy8n0oaypDy/anOgtwN2xm0XFO2raJl0UtH2/ILIb/h0lrTNZtYOGAXMrcZ6VaUgP+euwClm9qaZrTGzq6utdlUjSJsfBnoQWb5yHTDV3Y9VT/VqRMrzqzbPh56yxanTSOD2mNkFRAJ9cJXWqOoFafMDwE3ufjQkK/YEaXMDoD/wI+BE4G0zW+nuH1Z15apIkDZfDKwFfgicBbxqZm+5+xdVXLeakvL8qs2BXhcXpw7UHjPrAzwGDHf3fdVUt6oSpM3ZQG5RmLcCLjGzI+7+92qpYeoF/bv9mbsfBA6a2XKgL5CugR6kzROAmR4ZYN5iZh8B3YHV1VPFapfy/KrNQy51cXHqpG02s/bAC8D4NO6txUvaZnfPdPeO7t4ReA64IY3DHIL93V4EnG9mDcysCTAQ2FjN9UylIG3eQeRfJJhZG6AbsLVaa1m9Up5ftbaH7nVwceqAbb4VaAk8UtRjPeJpPFNdwDaHSpA2u/tGM3sZeB84Bjzm7gkff0sHAX/OdwJPmNk6IsMRN7l72k6ra2Y5wFCglZkVALcBDaHq8kuv/ouIhERtHnIREZEKUKCLiISEAl1EJCQU6CIiIaFAFxEJCQW6iEhIKNBFRELi/wN5Au58PlC0QAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from sklearn.metrics import roc_auc_score, roc_curve\n", - "\n", - "proba = lr_clf.predict_proba(train_features)[:, 1]\n", - "auc = roc_auc_score(y_train, proba)\n", - "plt.plot(*roc_curve(y_train, proba)[:2], label='%s AUC=%.4f' % ('train', auc))\n", - "\n", - "proba = lr_clf.predict_proba(test_features)[:, 1]\n", - "auc = roc_auc_score(y_test, proba)\n", - "plt.plot(*roc_curve(y_test, proba)[:2], label='%s AUC=%.4f' % ('test', auc))\n", - "\n", - "plt.legend();" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cc1hBVfbzHJ7" - }, - "source": [ - "So, how does it look? Did we achieve better results? \n", - "\n", - "Here come some further ideas:\n", - "\n", - "* Try using the larger BERT (e.g. BERT-base or BERT-large) and compare the results (be careful, they require more memory).\n", - "\n", - "* Using BERT output for translation? Why not ;)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "machine_shape": "hm", - "name": "A Visual Notebook to Using BERT for the First Time.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "057e5786db794af7ae7d9cb86cf271a7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ddb91ccd465f4fb586b4e65681ed421e", - "placeholder": "​", - "style": "IPY_MODEL_18877a9f1bef436da6294b8e239a3e87", - "value": " 255M/255M [00:08<00:00, 29.4MB/s]" - } - }, - "08e91434c84e461fb028d5aaafa0d2ac": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_69a2d9186a1f4f0dbe07bf0cffa9efde", - "placeholder": "​", - "style": "IPY_MODEL_d89ac674a7d042aa9640cac0d852b103", - "value": " 32/32 [00:10<00:00, 3.12it/s]" - } - }, - "095377480b704abb80aee1dfc023594e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "16598e7fb085414abedff55c7422bac3": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "18877a9f1bef436da6294b8e239a3e87": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "1902ef0b5d7e42cab7323b99d5a5e0f7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "100%", - "description_tooltip": null, - "layout": "IPY_MODEL_394aae946bd5416db64691a92fd222e8", - "max": 32, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6dabfd45bcf04128932f72f8e79fb126", - "value": 32 - } - }, - "1f288559046f4b15916737cfc6887c29": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1fce08ae597a49c6844825f9a52e2d20": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "21f5cf249e2e46dc896a537304bc6ecb": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "281af34bb99e4436b0f49c5cf46822ac": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c1b26841521d43ea9f4f21fa15f7b762", - "IPY_MODEL_d866f8b057c74513bf21b3a217718a30" - ], - "layout": "IPY_MODEL_c906b5aa2d0c4a9f81580e5dfc6c3d03" - } - }, - "285eb13a7fe34b31b74e6c56084e6c49": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3500f61b19834c6c99701d7b24a17bbb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_89374ef6e7634341b1f9d1cc2c045064", - "placeholder": "​", - "style": "IPY_MODEL_6e5f2aea17ed4f1d8bd83a3c9fccfb2e", - "value": " 49.0/49.0 [00:00<00:00, 706B/s]" - } - }, - "350fc6d45be242678a9d36d6da366f01": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "394aae946bd5416db64691a92fd222e8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4a3ec2955c694bf49cc39913b1a104dd": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4ec263486149477382f71f37759929ba": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "504cad20350e43a19dbb367cda255729": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "5b2bb84e7788484dbc9949ebb0d91ec7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_6be0571f20634e3d9a20314eee0aba64", - "IPY_MODEL_98bd16e5bf0f4604b84b17dbf3260070" - ], - "layout": "IPY_MODEL_4a3ec2955c694bf49cc39913b1a104dd" - } - }, - "5cd796c3da374bc4ba3d607bcd0f2377": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "601f514e09c54843a4ec85f69176161d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "626e3aeb683c42a7bd6d057cf78ee082": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "69a2d9186a1f4f0dbe07bf0cffa9efde": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6af0fc5df8ab44ce980d711e9cef851e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6be0571f20634e3d9a20314eee0aba64": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "Downloading: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_5cd796c3da374bc4ba3d607bcd0f2377", - "max": 465, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_985a11a5b169401f82f25f72cc81053f", - "value": 465 - } - }, - "6dabfd45bcf04128932f72f8e79fb126": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "6e5f2aea17ed4f1d8bd83a3c9fccfb2e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "84108be2c2604819a38a16a624318313": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "87b7803428d44e6c9807e64ae03c4d29": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "Downloading: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_21f5cf249e2e46dc896a537304bc6ecb", - "max": 173939, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_095377480b704abb80aee1dfc023594e", - "value": 173939 - } - }, - "87f20435e6254088b85f8397750b2ba1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1902ef0b5d7e42cab7323b99d5a5e0f7", - "IPY_MODEL_b7fb8fc65d084795a52eb4d80d549e43" - ], - "layout": "IPY_MODEL_84108be2c2604819a38a16a624318313" - } - }, - "89374ef6e7634341b1f9d1cc2c045064": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "985a11a5b169401f82f25f72cc81053f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "98bd16e5bf0f4604b84b17dbf3260070": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_16598e7fb085414abedff55c7422bac3", - "placeholder": "​", - "style": "IPY_MODEL_c861b31fa3ea4b139a0f2b940abaedd1", - "value": " 465/465 [00:00<00:00, 1.98kB/s]" - } - }, - "a23b1080e91f44b7947bb0d90b72eddf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6af0fc5df8ab44ce980d711e9cef851e", - "placeholder": "​", - "style": "IPY_MODEL_f2fea47b9f834911899bf975fa5ea737", - "value": " 174k/174k [00:01<00:00, 111kB/s]" - } - }, - "a3b23cadc2844a38857d546fdd1d1cc1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a3f0428ab70f4c9aa5e3e40518e11098": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "a72c347172ec4184b64a92678b6d6f8f": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a8f9ae6ba52d4bcea7966b1c72748bfb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_87b7803428d44e6c9807e64ae03c4d29", - "IPY_MODEL_a23b1080e91f44b7947bb0d90b72eddf" - ], - "layout": "IPY_MODEL_1f288559046f4b15916737cfc6887c29" - } - }, - "b37801f82e8f4a3fb673061e0cc49cdc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_be08b9fe75a543ea92a5cbd3943a961d", - "IPY_MODEL_057e5786db794af7ae7d9cb86cf271a7" - ], - "layout": "IPY_MODEL_285eb13a7fe34b31b74e6c56084e6c49" - } - }, - "b63953a573c948e19bb175f46acf02af": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "b7fb8fc65d084795a52eb4d80d549e43": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a72c347172ec4184b64a92678b6d6f8f", - "placeholder": "​", - "style": "IPY_MODEL_bc9cc9adc1c044629bcee301749dfc9f", - "value": " 32/32 [01:14<00:00, 2.34s/it]" - } - }, - "bc9cc9adc1c044629bcee301749dfc9f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "bd9d17c4142046c4971bb28b8dd8f82f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "Downloading: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_626e3aeb683c42a7bd6d057cf78ee082", - "max": 49, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a3f0428ab70f4c9aa5e3e40518e11098", - "value": 49 - } - }, - "be08b9fe75a543ea92a5cbd3943a961d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "Downloading: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_a3b23cadc2844a38857d546fdd1d1cc1", - "max": 255182217, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_504cad20350e43a19dbb367cda255729", - "value": 255182217 - } - }, - "bf196950f21649a3bac4154bd02c673c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c13f14f2652e48368e4ca0f9690dd430", - "IPY_MODEL_08e91434c84e461fb028d5aaafa0d2ac" - ], - "layout": "IPY_MODEL_4ec263486149477382f71f37759929ba" - } - }, - "c13f14f2652e48368e4ca0f9690dd430": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "100%", - "description_tooltip": null, - "layout": "IPY_MODEL_350fc6d45be242678a9d36d6da366f01", - "max": 32, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_601f514e09c54843a4ec85f69176161d", - "value": 32 - } - }, - "c1b26841521d43ea9f4f21fa15f7b762": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "100%", - "description_tooltip": null, - "layout": "IPY_MODEL_c4fc0e5df81e47ee8193f04118fac0d5", - "max": 32, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b63953a573c948e19bb175f46acf02af", - "value": 32 - } - }, - "c4fc0e5df81e47ee8193f04118fac0d5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c861b31fa3ea4b139a0f2b940abaedd1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c906b5aa2d0c4a9f81580e5dfc6c3d03": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d7d44fa068b048d093c4eac1f3d46497": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_bd9d17c4142046c4971bb28b8dd8f82f", - "IPY_MODEL_3500f61b19834c6c99701d7b24a17bbb" - ], - "layout": "IPY_MODEL_1fce08ae597a49c6844825f9a52e2d20" - } - }, - "d866f8b057c74513bf21b3a217718a30": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fe5fdb885d93428485a31b1d5260c09e", - "placeholder": "​", - "style": "IPY_MODEL_db1c11f8d801430ca11917ccbbdabd02", - "value": " 32/32 [01:05<00:00, 2.06s/it]" - } - }, - "d89ac674a7d042aa9640cac0d852b103": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "db1c11f8d801430ca11917ccbbdabd02": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ddb91ccd465f4fb586b4e65681ed421e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f2fea47b9f834911899bf975fa5ea737": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "fe5fdb885d93428485a31b1d5260c09e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/week06_bert/lect07_BERT.pdf b/week06_bert/lect07_BERT.pdf new file mode 100644 index 0000000..d4dcf0f Binary files /dev/null and b/week06_bert/lect07_BERT.pdf differ diff --git a/week06_bert/practice_bert_for_text_classification.ipynb b/week06_bert/practice_bert_for_text_classification.ipynb new file mode 100644 index 0000000..e5711b9 --- /dev/null +++ b/week06_bert/practice_bert_for_text_classification.ipynb @@ -0,0 +1,902 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "izA3-6kffbdT" + }, + "source": [ + "# Practice: A Visual Notebook to Using BERT for the First Time" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SEBgv15zoaX6" + }, + "source": [ + "*Credits: first part of this notebook is strongly based on Jay Alammar's [great blog post](http://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/). His blog is a great way to dive into the DL and NLP concepts.*\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dVgtANpYoaX7" + }, + "source": [ + "In this notebook, we will use pre-trained deep learning model to process some text. We will then use the output of that model to classify the text. The text is a list of sentences from film reviews. And we will calssify each sentence as either speaking \"positively\" about its subject of \"negatively\"." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0oCi6ZSnoaX7" + }, + "source": [ + "## Models: Sentence Sentiment Classification" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pyRwVEI4oaX7" + }, + "source": [ + "Our goal is to create a model that takes a sentence (just like the ones in our dataset) and produces either 1 (indicating the sentence carries a positive sentiment) or a 0 (indicating the sentence carries a negative sentiment). We can think of it as looking like this:\n", + "\n", + "\n", + "\n", + "Under the hood, the model is actually made up of two model.\n", + "\n", + "* DistilBERT processes the sentence and passes along some information it extracted from it on to the next model. DistilBERT is a smaller version of BERT developed and open sourced by the team at HuggingFace. It’s a lighter and faster version of BERT that roughly matches its performance.\n", + "* The next model, a basic Logistic Regression model from scikit learn will take in the result of DistilBERT’s processing, and classify the sentence as either positive or negative (1 or 0, respectively).\n", + "\n", + "The data we pass between the two models is a vector of size 768. We can think of this of vector as an embedding for the sentence that we can use for classification.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mQVOYe4PoaX8" + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3S7DFxaeoaX9" + }, + "source": [ + "The dataset we will use in this example is [SST2](https://nlp.stanford.edu/sentiment/index.html), which contains sentences from movie reviews, each labeled as either positive (has the value 1) or negative (has the value 0):\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " sentence\n", + " \n", + " label\n", + "
\n", + " a stirring , funny and finally transporting re imagining of beauty and the beast and 1930s horror films\n", + " \n", + " 1\n", + "
\n", + " apparently reassembled from the cutting room floor of any given daytime soap\n", + " \n", + " 0\n", + "
\n", + " they presume their audience won't sit still for a sociology lesson\n", + " \n", + " 0\n", + "
\n", + " this is a visually stunning rumination on love , memory , history and the war between art and commerce\n", + " \n", + " 1\n", + "
\n", + " jonathan parker 's bartleby should have been the be all end all of the modern office anomie films\n", + " \n", + " 1\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kPMv-7fOoaX-" + }, + "source": [ + "## Installing the transformers library" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S9zZk1UgoaX_" + }, + "source": [ + "Let's start by installing the huggingface transformers library so we can load our deep learning NLP model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "To9ENLU90WGl" + }, + "outputs": [], + "source": [ + "!pip install -Uqq transformers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zQ-42fh0hjsF" + }, + "source": [ + "## Part 1. Using BERT for text classification." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SwbK19NFoaYB" + }, + "source": [ + "## Loading pretrained BERT." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GITfp914oaYB" + }, + "source": [ + "Here we will be using the pretrained DistilBERT model from `transformers` library. The easiest way to use such model is to use a `pipeline`. This can be done as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4LyOoG8IoaYB" + }, + "outputs": [], + "source": [ + "from transformers import pipeline\n", + "\n", + "\n", + "unmasker = pipeline('fill-mask', 'distilbert-base-uncased')\n", + "unmasker(\"Hello I'm a [MASK] model.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NnA23_CpoaYC" + }, + "source": [ + "However, such approach is not very flexible and certainly doesn't allow you to fine-tune a model. For this reason we will use the model in a more manual way. For this we load the model and appropriate tokenizer and use them together. Here is how we can use them to extract features from our text:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "O8gHVTAEoaYC" + }, + "outputs": [], + "source": [ + "import torch\n", + "from transformers import DistilBertModel, DistilBertTokenizer, logging\n", + "\n", + "\n", + "logging.set_verbosity_error() # Ignore warning on model loading.\n", + "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n", + "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n", + "\n", + "text = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.'\n", + "tokenized_text = tokenizer(text, return_tensors='pt')\n", + "\n", + "with torch.no_grad():\n", + " output = model(**tokenized_text)\n", + "\n", + "output.last_hidden_state.shape" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Except for the `logging` part, everything looks very similar to the code we saw in previous practice notebooks. The first thing we do is, just like always, tokenize our text.\n", + "\n", + "> **Note:** as you can see, we used `return_tensors` keyword argument in code above. This parameters just tells tokenizer to convert result into a PyTorch tensors to use them with our model. If we don't specify this parameter, we will get exactly same results, but packed into a python `list` objects.\n", + "\n", + "Let's look at this step a little closer. What exactly does the `tokenizer.__call__` return? Let's find out:" + ], + "metadata": { + "id": "P5DAZw0tN8wi" + } + }, + { + "cell_type": "code", + "source": [ + "tokenized_text = tokenizer(text)\n", + "\n", + "for key, values in tokenized_text.items():\n", + " values_type = type(values).__name__\n", + " item_type = type(values[0]).__name__\n", + " values_sample = f\"[{', '.join(str(value) for value in values[:5])}, ...]\"\n", + " print(f\"{key}: {values_type}[{item_type}], length {len(values)}: {values_sample}\")" + ], + "metadata": { + "id": "KH77JkiNPCxA" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The contents may differ for different models, however for the `DistilBert` model tokenizer returns a `dict`-like object with two python lists under keys `\"input_ids\"` and `\"attention_mask\"`. Both lists have the same length and the attention mask seems to only have ones. We'll deal with the mask later, for now let's focus on `\"input_ids\"` which is the token ids for the tokenized sequence. Let's decode them to make sure and see what is actually going on:" + ], + "metadata": { + "id": "MzDnPSvrQdCB" + } + }, + { + "cell_type": "code", + "source": [ + "print(f\"Tokens: {tokenizer.convert_ids_to_tokens(tokenized_text['input_ids'])}\")\n", + "print(f\"Decoded sequence: '{tokenizer.decode(tokenized_text['input_ids'])}'\")" + ], + "metadata": { + "id": "SK-Yg-FDMN3I" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "We see that tokenizer actually does quite a lot of work behind the curtains: it lowercases the sequence (remember, we use `*-uncased` model, which implies that it doesn't understand the upper case), adds special tokens (`[CLS]` and `[SEP]`) and applies the BPE. That is how we get 23 tokens for such a little text." + ], + "metadata": { + "id": "syJ8naWFT87_" + } + }, + { + "cell_type": "markdown", + "source": [ + "" + ], + "metadata": { + "id": "pUmYkLeHd1m7" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SVlcZqXqoaYC" + }, + "source": [ + "## Loading the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M3YBY3DWoaYC" + }, + "source": [ + "However, working with manually edited sentence is not interesting. Let's use our model to work with a dataset for sentiment classification. We'll use pandas to read the dataset and load it into a dataframe." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cyoj29J24hPX" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "\n", + "dataset_url = (\n", + " 'https://github.com/clairett/pytorch-sentiment-classification/raw/master/data/SST2/train.tsv'\n", + ")\n", + "dataset = pd.read_csv(dataset_url, delimiter='\\t', header=None)\n", + "dataset.columns = ['text', 'label']\n", + "dataset.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dMVE3waNhuNj" + }, + "source": [ + "For performance reasons, we'll only use 2,000 sentences from the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gTM3hOHW4hUY" + }, + "outputs": [], + "source": [ + "dataset = dataset[:2000]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PRc2L89hh1Tf" + }, + "source": [ + "We can ask pandas how many sentences are labeled as \"positive\" (value 1) and how many are labeled \"negative\" (having the value 0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jGvcfcCP5xpZ" + }, + "outputs": [], + "source": [ + "dataset['label'].value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lZDBMn3wiSX6" + }, + "source": [ + "## Preparing the Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NiNzCErkoaYE" + }, + "source": [ + "Before we can hand our sentences to BERT, we need to so some processing to put them in the format it requires. First, let's split our `dataset` into separate `texts` and `labels`." + ] + }, + { + "cell_type": "code", + "source": [ + "texts = dataset['text'].tolist()\n", + "labels = dataset['label'].values" + ], + "metadata": { + "id": "m8ipGP0raY7r" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Now we need to tokenize our texts." + ], + "metadata": { + "id": "PmNUir6fcLbl" + } + }, + { + "cell_type": "code", + "source": [ + "# YOUR CODE HERE\n", + "# Tokenize the texts in dataset.\n", + "# Hint: our tokenizer can also work with lists of strings.\n", + "# tokenized_texts = ...\n", + "\n", + "for key, values in tokenized_texts.items():\n", + " values_type = type(values).__name__\n", + " item_type = type(values[0]).__name__\n", + " print(f\"{key}: {values_type}[{item_type}], length {len(values)}\")" + ], + "metadata": { + "id": "PyLowtCxajkq" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "We obtained a list of lists. However, what we want is `torch.tensor` so that we could use it with our model. It's time to remember how we used to specify the `return_tensors` option! However, if we were to just specify it blindly, we would get an error. The problem here lies in the fact that sequences tend to have different lenghts:" + ], + "metadata": { + "id": "ooqGLMl7cTe-" + } + }, + { + "cell_type": "code", + "source": [ + "for seq in tokenized_texts[\"input_ids\"][:5]:\n", + " print(len(seq))" + ], + "metadata": { + "id": "7SMycKghdkh2" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The most common solution to this problem, often used in NLP is the use of padding. Luckily for us, tokenizer from `transformers` can do the padding for us:" + ], + "metadata": { + "id": "_MjmmfMseJVf" + } + }, + { + "cell_type": "code", + "source": [ + "tokenized_texts = tokenizer(dataset['text'].tolist(), return_tensors=\"pt\", padding=True)\n", + "\n", + "for key, values in tokenized_texts.items():\n", + " values_type = type(values).__name__\n", + " print(f\"{key}: {values_type}, {values.shape}\")" + ], + "metadata": { + "id": "ONgnhkuxe9kB" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "However, we just added lot's of extra items into most of our sequences:" + ], + "metadata": { + "id": "u1AlfPcefj-m" + } + }, + { + "cell_type": "code", + "source": [ + "tokenized_texts[\"input_ids\"]" + ], + "metadata": { + "id": "XjfXDg8pgsFd" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Note how all the sequences end with zeros. We already encountered such a problem, when trained our RNN on previous lessons and we tackled it with specifying the padding index to the `CrossEntropyLoss` so that our model doesn't train to predict padding. Right now we don't want to train our model to do anything, however, we are working with a transformer, which means that it uses the self-attention operation extensively. If we were to simply add these extra items, it would be likely to affect our results. And this is exactly the place where the `\"attention_mask\"` comes into play. It is used exactly to mask the padding from getting in a way of attention!" + ], + "metadata": { + "id": "zkhDolkRgrti" + } + }, + { + "cell_type": "code", + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "plt.pcolormesh(tokenized_texts[\"attention_mask\"])\n", + "plt.axis(\"off\")\n", + "plt.colorbar()\n", + "plt.show()" + ], + "metadata": { + "id": "gSCXbXVLg_7T" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jK-CQB9-kN99" + }, + "source": [ + "## And Now, Deep Learning!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a0UsQGQ2oaYF" + }, + "source": [ + "Now that we have our model and inputs ready, let's run our model! However, running it on a cpu takes several minutes. We can speed this up via using the GPU. For this, however, we would need to split our dataset into batches of data.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Let's slice only the part of the output that we need. That is the output corresponding the first token of each sentence. The way BERT does sentence classification, is that it adds a token called `[CLS]` (for classification) at the beginning of every sentence. The output corresponding to that token can be thought of as an embedding for the entire sentence.\n", + "\n", + "\n", + "\n", + "We'll save those in the `features` variable, as they'll serve as the features to our logitics regression model. Also remember, how we created the `labels` variable to hold our labels." + ], + "metadata": { + "id": "KBqHs4d_ObTc" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "39UVjAV56PJz" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", + "model.to(device)\n", + "\n", + "batch_size = 32\n", + "features = []\n", + "with torch.no_grad():\n", + " for i in range(0, len(texts), batch_size):\n", + " texts_batch = tokenized_texts[\"input_ids\"][i : i + batch_size].to(device)\n", + " masks_batch = tokenized_texts[\"attention_mask\"][i : i + batch_size].to(device)\n", + " output = model(texts_batch, masks_batch)\n", + " batch_features = output.last_hidden_state[:, 0, :].cpu().numpy()\n", + " features.append(batch_features)\n", + "\n", + "features = np.concatenate(features, axis=0)\n", + "features.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iaoEvM2evRx1" + }, + "source": [ + "## Classifier training" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "02wlGPSLoaYG" + }, + "source": [ + "Let's now split our datset into a training set and testing set (even though we're using 2,000 sentences from the SST2 training set)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ddAqbkoU6PP9" + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "\n", + "train_features, test_features, train_labels, test_labels = train_test_split(features, labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B9bhSJpcv1Bl" + }, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uW_IiKvToaYG" + }, + "source": [ + "We can dive into Logistic regression directly with the Scikit Learn default parameters, but sometimes it's worth searching for the best value of the C parameter, which determines regularization strength." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cyEwr7yYD3Ci" + }, + "outputs": [], + "source": [ + "# YOUR CODE HERE\n", + "# [EXTRA] Grid search for parameters" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KCT9u8vAwnID" + }, + "source": [ + "We now train the LogisticRegression model. If you've chosen to do the gridsearch, you can plug the best hyperparameter values into the model declaration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gG-EVWx4CzBc" + }, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "\n", + "warnings.simplefilter('ignore') # Ignore warning on model fitting.\n", + "lr_clf = LogisticRegression().fit(train_features, train_labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3rUMKuVgwzkY" + }, + "source": [ + "\n", + "\n", + "So how well does our model do in classifying sentences? One way is to check the accuracy against the testing dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "iCoyxRJ7ECTA" + }, + "outputs": [], + "source": [ + "lr_clf.score(test_features, test_labels)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Another way to evaluate classification model is to plot the ROC curve and compute the area under it." + ], + "metadata": { + "id": "2xA5YIJPDYek" + } + }, + { + "cell_type": "code", + "source": [ + "from sklearn.metrics import roc_auc_score, roc_curve\n", + "\n", + "\n", + "plt.figure(figsize=(10, 6))\n", + "\n", + "proba = lr_clf.predict_proba(train_features)[:, 1]\n", + "auc = roc_auc_score(train_labels, proba)\n", + "plt.plot(*roc_curve(train_labels, proba)[:2], label=f'train AUC={auc:.4f}')\n", + "\n", + "proba = lr_clf.predict_proba(test_features)[:, 1]\n", + "auc = roc_auc_score(test_labels, proba)\n", + "plt.plot(*roc_curve(test_labels, proba)[:2], label=f'test AUC={auc:.4f}')\n", + "\n", + "plt.legend()\n", + "plt.show()" + ], + "metadata": { + "id": "186Bwg7UDLkU" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "75oyhr3VxHoE" + }, + "source": [ + "How good is this score? What can we compare it against? Let's first look at a dummy classifier:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lnwgmqNG7i5l" + }, + "outputs": [], + "source": [ + "from sklearn.dummy import DummyClassifier\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "clf = DummyClassifier()\n", + "scores = cross_val_score(clf, train_features, train_labels)\n", + "print(f\"Dummy classifier score: {scores.mean():.3f} (+/- {2 * scores.std():.3f})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7Lg4LOpoxSOR" + }, + "source": [ + "So our model clearly does better than a dummy classifier. But how does it compare against the best models?\n", + "\n", + "For reference, the [highest accuracy score](http://nlpprogress.com/english/sentiment_analysis.html) for this dataset is currently **96.8**. DistilBERT can be trained to improve its score on this task – a process called **fine-tuning** which updates BERT’s weights to make it achieve a better performance in this sentence classification task (which we can call the downstream task). The fine-tuned DistilBERT turns out to achieve an accuracy score of **90.7**. The full size BERT model achieves **94.9**.\n", + "\n", + "And that’s it! That’s a good first contact with BERT. The next step would be to head over to the documentation and try your hand at [fine-tuning](https://huggingface.co/transformers/examples.html#glue). You can also go back and switch from distilBERT to BERT and see how that works." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EJQuqV6cnWQu", + "outputId": "402d109c-01bb-485d-a510-4be8684c9c06" + }, + "source": [ + "## Part 2: Looking back." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "outputId": "402d109c-01bb-485d-a510-4be8684c9c06", + "id": "zIBbP_oroaYH" + }, + "source": [ + "Now it is your turn to reproduce the steps above.\n", + "\n", + "We shall revisit the first homework and see whether we could improve the results a little bit more. The average ROC-AUC on test set was around $0.9$ (using the words embeddings). \n", + "\n", + "__Let's see whether we can beat it.__" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kz8QBEXozHJx" + }, + "outputs": [], + "source": [ + "dataset_url = 'https://raw.githubusercontent.com/neychev/made_nlp_course/master/datasets/comments_small_dataset/comments.tsv'\n", + "dataset = pd.read_csv(dataset_url, sep='\\t')\n", + "dataset.head()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "One last note: this dataset contains some very long sentences, while the vast majority of sequences fall into category of 500 tokens and less:" + ], + "metadata": { + "id": "ZPyxeBCgGtfg" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XdZjuVxRoaYH" + }, + "outputs": [], + "source": [ + "texts = dataset[\"comment_text\"].tolist()\n", + "tokenized_texts = tokenizer(texts)\n", + "ids_lens = list(len(toks) for toks in tokenized_texts[\"input_ids\"])\n", + "\n", + "plt.figure(figsize=(10, 6))\n", + "plt.hist(ids_lens)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "We already know, how to tackle the problem of different sizes of sequences with padding. However, blind padding here would make pad all the sequenes to the size of the largest one, which seems to be an overkill. In such case it might be sensible to actually truncate the too-long sequences into a fixed length, say 512. And we can do this easily by specifying the `max_length` and `truncation=True` arguments to the tokenizer." + ], + "metadata": { + "id": "66wufVncHF8f" + } + }, + { + "cell_type": "code", + "source": [ + "# YOUR CODE HERE" + ], + "metadata": { + "id": "_LOYXhz_FKTr" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cc1hBVfbzHJ7" + }, + "source": [ + "So, how does it look? Did we achieve better results? \n", + "\n", + "Here come some further ideas:\n", + "\n", + "* Try using the larger BERT (e.g. BERT-base or BERT-large) and compare the results (be careful, they require more memory).\n", + "\n", + "* Using BERT output for translation? Why not ;)" + ] + } + ], + "metadata": { + "colab": { + "machine_shape": "hm", + "name": "practice_bert_for_text_classification.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "accelerator": "GPU" + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/week06_bert/practice_bert_for_text_classification_solved.ipynb b/week06_bert/practice_bert_for_text_classification_solved.ipynb new file mode 100644 index 0000000..54726e4 --- /dev/null +++ b/week06_bert/practice_bert_for_text_classification_solved.ipynb @@ -0,0 +1,1560 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "izA3-6kffbdT" + }, + "source": [ + "# Practice: A Visual Notebook to Using BERT for the First Time" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SEBgv15zoaX6" + }, + "source": [ + "*Credits: first part of this notebook is strongly based on Jay Alammar's [great blog post](http://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/). His blog is a great way to dive into the DL and NLP concepts.*\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dVgtANpYoaX7" + }, + "source": [ + "In this notebook, we will use pre-trained deep learning model to process some text. We will then use the output of that model to classify the text. The text is a list of sentences from film reviews. And we will calssify each sentence as either speaking \"positively\" about its subject of \"negatively\"." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0oCi6ZSnoaX7" + }, + "source": [ + "## Models: Sentence Sentiment Classification" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pyRwVEI4oaX7" + }, + "source": [ + "Our goal is to create a model that takes a sentence (just like the ones in our dataset) and produces either 1 (indicating the sentence carries a positive sentiment) or a 0 (indicating the sentence carries a negative sentiment). We can think of it as looking like this:\n", + "\n", + "\n", + "\n", + "Under the hood, the model is actually made up of two model.\n", + "\n", + "* DistilBERT processes the sentence and passes along some information it extracted from it on to the next model. DistilBERT is a smaller version of BERT developed and open sourced by the team at HuggingFace. It’s a lighter and faster version of BERT that roughly matches its performance.\n", + "* The next model, a basic Logistic Regression model from scikit learn will take in the result of DistilBERT’s processing, and classify the sentence as either positive or negative (1 or 0, respectively).\n", + "\n", + "The data we pass between the two models is a vector of size 768. We can think of this of vector as an embedding for the sentence that we can use for classification.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mQVOYe4PoaX8" + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3S7DFxaeoaX9" + }, + "source": [ + "The dataset we will use in this example is [SST2](https://nlp.stanford.edu/sentiment/index.html), which contains sentences from movie reviews, each labeled as either positive (has the value 1) or negative (has the value 0):\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " sentence\n", + " \n", + " label\n", + "
\n", + " a stirring , funny and finally transporting re imagining of beauty and the beast and 1930s horror films\n", + " \n", + " 1\n", + "
\n", + " apparently reassembled from the cutting room floor of any given daytime soap\n", + " \n", + " 0\n", + "
\n", + " they presume their audience won't sit still for a sociology lesson\n", + " \n", + " 0\n", + "
\n", + " this is a visually stunning rumination on love , memory , history and the war between art and commerce\n", + " \n", + " 1\n", + "
\n", + " jonathan parker 's bartleby should have been the be all end all of the modern office anomie films\n", + " \n", + " 1\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kPMv-7fOoaX-" + }, + "source": [ + "## Installing the transformers library" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S9zZk1UgoaX_" + }, + "source": [ + "Let's start by installing the huggingface transformers library so we can load our deep learning NLP model." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "To9ENLU90WGl" + }, + "outputs": [], + "source": [ + "!pip install -Uqq transformers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zQ-42fh0hjsF" + }, + "source": [ + "## Part 1. Using BERT for text classification." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SwbK19NFoaYB" + }, + "source": [ + "## Loading pretrained BERT." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GITfp914oaYB" + }, + "source": [ + "Here we will be using the pretrained DistilBERT model from `transformers` library. The easiest way to use such model is to use a `pipeline`. This can be done as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "4LyOoG8IoaYB", + "outputId": "ffc9b3d4-f5c6-43ee-c31c-947f25e5e9f7", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[{'score': 0.052928660064935684,\n", + " 'sequence': \"hello i'm a role model.\",\n", + " 'token': 2535,\n", + " 'token_str': 'role'},\n", + " {'score': 0.03968597203493118,\n", + " 'sequence': \"hello i'm a fashion model.\",\n", + " 'token': 4827,\n", + " 'token_str': 'fashion'},\n", + " {'score': 0.0347437709569931,\n", + " 'sequence': \"hello i'm a business model.\",\n", + " 'token': 2449,\n", + " 'token_str': 'business'},\n", + " {'score': 0.034622907638549805,\n", + " 'sequence': \"hello i'm a model model.\",\n", + " 'token': 2944,\n", + " 'token_str': 'model'},\n", + " {'score': 0.01814514584839344,\n", + " 'sequence': \"hello i'm a modeling model.\",\n", + " 'token': 11643,\n", + " 'token_str': 'modeling'}]" + ] + }, + "metadata": {}, + "execution_count": 2 + } + ], + "source": [ + "from transformers import pipeline\n", + "\n", + "\n", + "unmasker = pipeline('fill-mask', 'distilbert-base-uncased')\n", + "unmasker(\"Hello I'm a [MASK] model.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NnA23_CpoaYC" + }, + "source": [ + "However, such approach is not very flexible and certainly doesn't allow you to fine-tune a model. For this reason we will use the model in a more manual way. For this we load the model and appropriate tokenizer and use them together. Here is how we can use them to extract features from our text:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "O8gHVTAEoaYC", + "outputId": "42d75b3d-0ff7-4061-ef1b-efb09ca7ca49", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "torch.Size([1, 23, 768])" + ] + }, + "metadata": {}, + "execution_count": 3 + } + ], + "source": [ + "import torch\n", + "from transformers import DistilBertModel, DistilBertTokenizer, logging\n", + "\n", + "\n", + "logging.set_verbosity_error() # Ignore warning on model loading.\n", + "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n", + "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n", + "\n", + "text = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.'\n", + "tokenized_text = tokenizer(text, return_tensors='pt')\n", + "\n", + "with torch.no_grad():\n", + " output = model(**tokenized_text)\n", + "\n", + "output.last_hidden_state.shape" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Except for the `logging` part, everything looks very similar to the code we saw in previous practice notebooks. The first thing we do is, just like always, tokenize our text.\n", + "\n", + "> **Note:** as you can see, we used `return_tensors` keyword argument in code above. This parameters just tells tokenizer to convert result into a PyTorch tensors to use them with our model. If we don't specify this parameter, we will get exactly same results, but packed into a python `list` objects.\n", + "\n", + "Let's look at this step a little closer. What exactly does the `tokenizer.__call__` return? Let's find out:" + ], + "metadata": { + "id": "P5DAZw0tN8wi" + } + }, + { + "cell_type": "code", + "source": [ + "tokenized_text = tokenizer(text)\n", + "\n", + "for key, values in tokenized_text.items():\n", + " values_type = type(values).__name__\n", + " item_type = type(values[0]).__name__\n", + " values_sample = f\"[{', '.join(str(value) for value in values[:5])}, ...]\"\n", + " print(f\"{key}: {values_type}[{item_type}], length {len(values)}: {values_sample}\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KH77JkiNPCxA", + "outputId": "13ff8021-d2b2-4760-eb79-73d5d7cc5cc0" + }, + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "input_ids: list[int], length 23: [101, 19544, 2213, 12997, 17421, ...]\n", + "attention_mask: list[int], length 23: [1, 1, 1, 1, 1, ...]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "The contents may differ for different models, however for the `DistilBert` model tokenizer returns a `dict`-like object with two python lists under keys `\"input_ids\"` and `\"attention_mask\"`. Both lists have the same length and the attention mask seems to only have ones. We'll deal with the mask later, for now let's focus on `\"input_ids\"` which is the token ids for the tokenized sequence. Let's decode them to make sure and see what is actually going on:" + ], + "metadata": { + "id": "MzDnPSvrQdCB" + } + }, + { + "cell_type": "code", + "source": [ + "print(f\"Tokens: {tokenizer.convert_ids_to_tokens(tokenized_text['input_ids'])}\")\n", + "print(f\"Decoded sequence: '{tokenizer.decode(tokenized_text['input_ids'])}'\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "SK-Yg-FDMN3I", + "outputId": "3b76b053-d60a-414e-9557-8681cd77fe55" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Tokens: ['[CLS]', 'lore', '##m', 'ip', '##sum', 'do', '##lor', 'sit', 'am', '##et', ',', 'con', '##se', '##ct', '##et', '##ur', 'adi', '##pis', '##cing', 'eli', '##t', '.', '[SEP]']\n", + "Decoded sequence: '[CLS] lorem ipsum dolor sit amet, consectetur adipiscing elit. [SEP]'\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "We see that tokenizer actually does quite a lot of work behind the curtains: it lowercases the sequence (remember, we use `*-uncased` model, which implies that it doesn't understand the upper case), adds special tokens (`[CLS]` and `[SEP]`) and applies the BPE. That is how we get 23 tokens for such a little text." + ], + "metadata": { + "id": "syJ8naWFT87_" + } + }, + { + "cell_type": "markdown", + "source": [ + "" + ], + "metadata": { + "id": "pUmYkLeHd1m7" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SVlcZqXqoaYC" + }, + "source": [ + "## Loading the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M3YBY3DWoaYC" + }, + "source": [ + "However, working with manually edited sentence is not interesting. Let's use our model to work with a dataset for sentiment classification. We'll use pandas to read the dataset and load it into a dataframe." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "cyoj29J24hPX", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "outputId": "908c0b1c-1a7a-44e5-942b-3e18cd3c5e1f" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
textlabel
0a stirring , funny and finally transporting re...1
1apparently reassembled from the cutting room f...0
2they presume their audience wo n't sit still f...0
3this is a visually stunning rumination on love...1
4jonathan parker 's bartleby should have been t...1
\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + " \n", + "
\n", + "
\n", + " " + ], + "text/plain": [ + " text label\n", + "0 a stirring , funny and finally transporting re... 1\n", + "1 apparently reassembled from the cutting room f... 0\n", + "2 they presume their audience wo n't sit still f... 0\n", + "3 this is a visually stunning rumination on love... 1\n", + "4 jonathan parker 's bartleby should have been t... 1" + ] + }, + "metadata": {}, + "execution_count": 6 + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "\n", + "dataset_url = (\n", + " 'https://github.com/clairett/pytorch-sentiment-classification/raw/master/data/SST2/train.tsv'\n", + ")\n", + "dataset = pd.read_csv(dataset_url, delimiter='\\t', header=None)\n", + "dataset.columns = ['text', 'label']\n", + "dataset.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dMVE3waNhuNj" + }, + "source": [ + "For performance reasons, we'll only use 2,000 sentences from the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "gTM3hOHW4hUY" + }, + "outputs": [], + "source": [ + "dataset = dataset[:2000]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PRc2L89hh1Tf" + }, + "source": [ + "We can ask pandas how many sentences are labeled as \"positive\" (value 1) and how many are labeled \"negative\" (having the value 0)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "jGvcfcCP5xpZ", + "outputId": "a6214c63-65d0-42a4-f78d-d94fdd3bc4cc" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "1 1041\n", + "0 959\n", + "Name: label, dtype: int64" + ] + }, + "metadata": {}, + "execution_count": 8 + } + ], + "source": [ + "dataset['label'].value_counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lZDBMn3wiSX6" + }, + "source": [ + "## Preparing the Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NiNzCErkoaYE" + }, + "source": [ + "Before we can hand our sentences to BERT, we need to so some processing to put them in the format it requires. First, let's split our `dataset` into separate `texts` and `labels`." + ] + }, + { + "cell_type": "code", + "source": [ + "texts = dataset['text'].tolist()\n", + "labels = dataset['label'].values" + ], + "metadata": { + "id": "m8ipGP0raY7r" + }, + "execution_count": 9, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Now we need to tokenize our texts." + ], + "metadata": { + "id": "PmNUir6fcLbl" + } + }, + { + "cell_type": "code", + "source": [ + "# YOUR CODE HERE\n", + "# Tokenize the texts in dataset.\n", + "# Hint: our tokenizer can also work with lists of strings.\n", + "# tokenized_texts = ...\n", + "tokenized_texts = tokenizer(texts)\n", + "\n", + "for key, values in tokenized_texts.items():\n", + " values_type = type(values).__name__\n", + " item_type = type(values[0]).__name__\n", + " print(f\"{key}: {values_type}[{item_type}], length {len(values)}\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PyLowtCxajkq", + "outputId": "9d386cd9-3c37-45cd-f8fe-34f054285270" + }, + "execution_count": 10, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "input_ids: list[list], length 2000\n", + "attention_mask: list[list], length 2000\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "We obtained a list of lists. However, what we want is `torch.tensor` so that we could use it with our model. It's time to remember how we used to specify the `return_tensors` option! However, if we were to just specify it blindly, we would get an error. The problem here lies in the fact that sequences tend to have different lenghts:" + ], + "metadata": { + "id": "ooqGLMl7cTe-" + } + }, + { + "cell_type": "code", + "source": [ + "for seq in tokenized_texts[\"input_ids\"][:5]:\n", + " print(len(seq))" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7SMycKghdkh2", + "outputId": "dcb95cd1-907e-4edb-8fcb-df6cfd67d6b2" + }, + "execution_count": 11, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "20\n", + "16\n", + "45\n", + "22\n", + "25\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "The most common solution to this problem, often used in NLP is the use of padding. Luckily for us, tokenizer from `transformers` can do the padding for us:" + ], + "metadata": { + "id": "_MjmmfMseJVf" + } + }, + { + "cell_type": "code", + "source": [ + "tokenized_texts = tokenizer(dataset['text'].tolist(), return_tensors=\"pt\", padding=True)\n", + "\n", + "for key, values in tokenized_texts.items():\n", + " values_type = type(values).__name__\n", + " print(f\"{key}: {values_type}, {values.shape}\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ONgnhkuxe9kB", + "outputId": "a63fa412-5317-4788-8050-4d73c38a507b" + }, + "execution_count": 12, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "input_ids: Tensor, torch.Size([2000, 59])\n", + "attention_mask: Tensor, torch.Size([2000, 59])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "However, we just added lot's of extra items into most of our sequences:" + ], + "metadata": { + "id": "u1AlfPcefj-m" + } + }, + { + "cell_type": "code", + "source": [ + "tokenized_texts[\"input_ids\"]" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "XjfXDg8pgsFd", + "outputId": "b1635101-98de-48bb-b7cf-5b9560c96edb" + }, + "execution_count": 13, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor([[ 101, 1037, 18385, ..., 0, 0, 0],\n", + " [ 101, 4593, 2128, ..., 0, 0, 0],\n", + " [ 101, 2027, 3653, ..., 0, 0, 0],\n", + " ...,\n", + " [ 101, 2023, 2028, ..., 0, 0, 0],\n", + " [ 101, 1999, 1996, ..., 0, 0, 0],\n", + " [ 101, 1996, 3185, ..., 0, 0, 0]])" + ] + }, + "metadata": {}, + "execution_count": 13 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "Note how all the sequences end with zeros. We already encountered such a problem, when trained our RNN on previous lessons and we tackled it with specifying the padding index to the `CrossEntropyLoss` so that our model doesn't train to predict padding. Right now we don't want to train our model to do anything, however, we are working with a transformer, which means that it uses the self-attention operation extensively. If we were to simply add these extra items, it would be likely to affect our results. And this is exactly the place where the `\"attention_mask\"` comes into play. It is used exactly to mask the padding from getting in a way of attention!" + ], + "metadata": { + "id": "zkhDolkRgrti" + } + }, + { + "cell_type": "code", + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "plt.pcolormesh(tokenized_texts[\"attention_mask\"])\n", + "plt.axis(\"off\")\n", + "plt.colorbar()\n", + "plt.show()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "gSCXbXVLg_7T", + "outputId": "6cd0e97a-4127-43fe-8342-02b8117d00da" + }, + "execution_count": 14, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUwAAADxCAYAAACgTY5AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAPh0lEQVR4nO3dX4hk6VnH8e+zvepCjAk4SOLMBBecgBIR47iD5CIr7prRi8yF4s4uAaOrgjgiRgOKsi7rzURRWHGItuu4JhcOZi+kwdFJRENATOwRSXDmIg6jZnqibPaPixASd7ofL6qyW91MVZ+qeqvOOe/5fqCh/px+T13U+b3P+77nnIrMRJJ0uHva/gCS1BcGpiQ1ZGBKUkMGpiQ1ZGBKUkMGpiQ1ZGBKqlJEXIyI5yPiX6e8HxHx+xFxIyI+FxHvPKxNA1NSrZ4FTs94/4eBE+O/nwU+fFiDBqakKmXmp4CXZmxyBvhIjnwaeHNEvHVWm/fOenPvv9/uZUBq7D3f+t1tfwS16BN7H4tl23jPD7whX3xpt9G2//y5r14DvjLx0mZmbs6xu6PArYnnO+PX/mvaP8wMTK2GwSLd3Ysv7fJPV97WaNuNt/7bVzLz5Io/0j6DCEwDSuqHBPbYW9fubgPHJ54fG782VXWBaThK/ZUkr2azIXkBW8C5iLgEnAJeycypw3GoMDCvfPGzbX+EuRny0utKVZgR8efAg8CRiNgBfhP4OoDM/EPgMvAjwA3gy8BPHtZmdYFp+Ej9lSS7hW45mZmPHvJ+Aj8/T5vVBWYfK8ym7Aw0BHt09+Sc6gKzZl3vDAx0LSuBXQOzDgaCtHqDqjANFUmLSuDVDv9sTvHA7PqwURqKPhYvSTokl4agjwHVOQm73c1LA1P9YBgNw+hKn+4yMKfwAJXaEOyy9D08VsbAnGJIc7F2DuqK0aKPgdkpBoTUTaPzMA3MThlS9VgLO7nh2LPClOZjQA6TFaaKMURUuyTY7fAv5xiYHWAQSq9zSK6ZnFM9nJ3KMCTB/+VG2x9jqlYC0y+/pLsZnbjukHyfJhWVoSoNk4s+C+jDMNVQl8rKDHbTCnNtDDGp3/asMOdn8EnDM1r06WwsdTcw+zAklybZyS/PRR/pAINFs+x6Hqb0unWOHgznfvFKn57wwJK6Yc9V8u7r2pypAa4hGt18w8DUnBYNcINWfZYEr3pppNala5VyLeyI1iMTT1yXJhk+mi48cV3zM1Q0RIkVphZQamht8KpvXPRRa9qe0zSwNY8kvIGwhqvtwC7F4F+P0c/sdjeWuvvJesCDSCotvB9mrWqpnlbJTkXzSLzSR4UYPhoCK8xKGWBSWZlRrMKMiNPA08AG8Exmnj/w/tuAPwPePN7mVzPz8qw2DcwlOCTvHzu5bhst+ix/aWREbAAXgIeBHWA7IrYy8/rEZr8B/EVmfjgivhO4DHzbrHYNTK2doaXpiv2mzwPAjcy8CRARl4AzwGRgJvBN48dvAr54WKMGZmUMI/XZaNGn8RzmkYi4OvF8MzM3x4+PArcm3tsBTh34/yeBj0fELwBvAB46bIcGZmX6ME1gqGuWOa70eSEzTy6xq0eBZzPzdyPi+4GPRsQ7MnNv2j8YmB1ggEgjBa/0uQ0cn3h+bPzapMeB0wCZ+Y8RcR9wBHh+WqMGZgf0oSrsKzuj/in0I2jbwImIuJ9RUJ4FHjuwzReAHwSejYjvAO4DvjSr0eKB6RdU0qIy4dW95QMzM+9ExDngCqNThi5m5rWIeAq4mplbwC8DfxwRv8Ro+vT9mZmz2i0emOuulgxoqR6jIXmZ8zDH51RePvDaExOPrwPvmqfN3g/JHc7uZweivvNKH61Nkw7EUFVXzXla0doZmANUS1Vu8Neo3JB8FQzMwjyIpeX4mz4NGDSSRqvk/szuoWoZJkptqaHo8CcqpANqOLC1Og7JpQltjiYM625zlVyDYRipBFfJF+DBJw1PZnDHwJyfi0Bl2PGobxySS1IDzmGqVYtW6lamaouBWQlDRFotz8OsiPOqZdjxaBbPw9Q+BoZ0d5lwp8ANhFfFwGzBkCpVOwfNyyH5gBgQ0uKcwxyYIVWPOpwd6PzSwJyfXzRpmFz0WcDQKzU7DA1RpnOYvWRgSW0Idl0l7x9/TExqh3OYlXLawA5DZXkt+Yp50EoVydE8Zlf1PjCHXuWpO+y8y3CVfI380kr9lS76rNcqK07DWFo9h+SVqGX4b/Cry1wlV6fUEvxN2Dn0S6aBqR4yaNQWTyvSPoaRNJ1zmJLUQBLsuUquSaXmEK1UVaMOF5gGZp/VvHhjZzBQBRd9IuI08DSwATyTmefvss2PA0+O9sxnM/OxWW0amB1lYGiwCpSYEbEBXAAeBnaA7YjYyszrE9ucAH4NeFdmvhwR33JYuwZmR9VcPWo1aulkC1WYDwA3MvMmQERcAs4A1ye2+RngQma+PNpvPn9YowamBqWWUKlVAnt7jQPzSERcnXi+mZmb48dHgVsT7+0Apw78/9sBIuIfGA3bn8zMv5m1QwOzRzzYVb0EmleYL2TmySX2di9wAngQOAZ8KiK+KzP/Z9Y/qGUGofS6Qudh3gaOTzw/Nn5t0g7wmcx8Ffj3iPg8owDdntaogdkBtcxXGvwqokxgbgMnIuJ+RkF5Fji4Av6XwKPAn0bEEUZD9JuzGjUwZdCpQ6LIok9m3omIc8AVRvOTFzPzWkQ8BVzNzK3xez8UEdeBXeCDmfnirHYNzB4x2DQIhc5cz8zLwOUDrz0x8TiBD4z/GjEwe2SRobshq15JyOar5GvX+8A0EKTaGJgrU8uCyUF2BBqsDl9M3vvA7BqDTlqSgTk/g0caoPlOXF+7zgZmrUPtkuxUVCNvIKyVsFNZHzunNXKVXCV40GoIwgpTkhpIXPSRpGbCRR+V4Zzlejj10bKhV5h+ASU1ttf2B5huLYHpNdCSGvE8zMU4/NRQWBzs5yq5OsUDVJ1mYHaLgSFpEYMMTIf7mmQH2i0OyVWEB7aql3hppMqwMp7NDqUSVphaBQNCNXJIrn0MOmkGA1OSGjIwy7Ayk+oW6ZC8GBc95mcno95xlXy4DCxpPlaYA9akKjZUpQkGpmZxqmHY7DAnOIc5LH75pSUZmGUYRlL9osM3EL6n7Q8gSX3RqwrTuT5NcsRRKYfkUnn+9EmFXPSplweftAIGZp2cIlgfO6cBMTCl5ZTqnAzebgu6vUpuYBbmASktoeAcZkScBp4GNoBnMvP8lO1+FHgO+L7MvDqrTQOzMIfp/WMn1zEFAjMiNoALwMPADrAdEVuZef3Adm8EfhH4TJN2DUz1lkFXqTIV5gPAjcy8CRARl4AzwPUD2/0W8CHgg00aNTCX4AErlTfHkPxIREwOoTczc3P8+Chwa+K9HeDUvv1EvBM4npl/FREG5qp5JyJpBZoH5guZeXKRXUTEPcDvAe+f5/8MzBWreU7TzkDFZbFV8tvA8Ynnx8avfc0bgXcAn4wIgLcAWxHx3lkLPwamFuapPlqJMnOY28CJiLifUVCeBR57bReZrwBHvvY8Ij4J/Iqr5DqUgaUuKXFaUWbeiYhzwBVGpxVdzMxrEfEUcDUztxZptzOB6UErCSh2pU9mXgYuH3jtiSnbPtikzc4EZs1zfTWwQ9NaJF4aqfYYdOqTwLsVVcPwkVbPwKzEkKcN7Cy0Ngam+m7dnYUBPWAGpjQf76Y+UN5xXYvw4NdgGZia1yqHwIaxuswbCKtTVhXGBrFKcEguSU144roOY2UmTTAwNcuiQ2SDVrXxSp+KGVhSebHX3cQ0MJcw5Ct/7sYOREtzDlNtMsTUNw7J1ZqhV8F2GD1kYPaPB5rUDivMHupiZWaIaxAMzP4xnKQWlPvVyJUwMKewwpTWz/MwVUwXQ7wGdkQdk91NTANzCR5oUnlWmJUaesVnh6HiPHFdkppz0Ue9Y/WothiYLfPgl3oicdGnbbXONdoRqEYu+mglau0ImrLDqJSBuZ9fdEl344nrdzH0yqhtdljqrExvIFyKB7o0AN3NS4fkkrrFIfkBDsn3swORxhJwSK5ZutiBGOJqTXfz0sCchyEirV6pIXlEnAaeBjaAZzLz/IH3PwD8NHAH+BLwU5n5n7PaNDDn0MVKUMuzI+yWEqvkEbEBXAAeBnaA7YjYyszrE5v9C3AyM78cET8H/DbwyKx2DcwpPIikFpS7W9EDwI3MvAkQEZeAM8BrgZmZfz+x/aeB9x3WqIE5RRerSUNctRuduN44MY9ExNWJ55uZuTl+fBS4NfHeDnBqRluPA3992A4NzB7pYoi3yQ6kUs3vVvRCZp5cdncR8T7gJPDuw7Y1MHvEgNAQzFFhznIbOD7x/Nj4tf37ingI+HXg3Zn51cMaNTB7pNYK045Aryk3h7kNnIiI+xkF5VngsckNIuJ7gD8CTmfm800aNTC1MINO5ZW5ljwz70TEOeAKo9OKLmbmtYh4CriamVvA7wDfCHwsIgC+kJnvndWugdlRhpEGq9ANhDPzMnD5wGtPTDx+aN42DcwxA0rqgPQnKhZigEkD5U9UzK/WBY6S7FRUpe7mZXcDU4e7W6diiKrvYq+7Y3IDszJ9rMwNeb0mmefE9bUzMLUwg06lBVnqxPWVWEtgemBJamzogdnHYeI62aFIE4YemJptaB2KHYSmcg5ztTz4pLq4Sr5CQ6vOSrCTUXelQ/JaGTxSYYmBWauhV7d2GFqJ7o7IDUwtbugdRhN2KvMb/HmYQ+IBIi3JwByOWqoug1+tyITd7o7JDUxJ3WKFKUkNGZjqm1qmFhbhdESLEijwmz6rMojA9ACQ+iIhncNs1SLVkiErtSBx0aePhjwkXZSdjIpwDlNdYrCp0wxMlWDQqX7efEOFrHOawHBWKxLw9m5qi8Gn3rHCVFtcvCrDjmddvDSyc/zySx2VkJ6H2S1WXathR6QivNLncB5skgDnMJtYtOozaKWKZLpKvko1DK8NfWmCFeb6GD5SnyW5u9v2h5iqusCsoeLsKzsrLc3bu3WPB7bUYZ5W1C01V6F2BuqzBLJQhRkRp4GngQ3gmcw8f+D9bwA+Anwv8CLwSGb+x6w2BxmYfWUYqnpZ5gbCEbEBXAAeBnaA7YjYyszrE5s9Drycmd8eEWeBDwGPzGrXwCzMUJOWU2jR5wHgRmbeBIiIS8AZYDIwzwBPjh8/B/xBRETm9GX6mYF5z1s+H8t84iH6RHenX6TO+19evvK3+dyRhpvfFxFXJ55vZubm+PFR4NbEezvAqQP//9o2mXknIl4Bvhl4YdoOrTAldUZmnm77M8xyT9sfQJJW4DZwfOL5sfFrd90mIu4F3sRo8WcqA1NSjbaBExFxf0R8PXAW2DqwzRbwE+PHPwb83az5S3BILqlC4znJc8AVRqcVXczMaxHxFHA1M7eAPwE+GhE3gJcYhepMcUigSpLGHJJLUkMGpiQ1ZGBKUkMGpiQ1ZGBKUkMGpiQ1ZGBKUkP/D1kzYjRAuknzAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jK-CQB9-kN99" + }, + "source": [ + "## And Now, Deep Learning!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a0UsQGQ2oaYF" + }, + "source": [ + "Now that we have our model and inputs ready, let's run our model! However, running it on a cpu takes several minutes. We can speed this up via using the GPU. For this, however, we would need to split our dataset into batches of data.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Let's slice only the part of the output that we need. That is the output corresponding the first token of each sentence. The way BERT does sentence classification, is that it adds a token called `[CLS]` (for classification) at the beginning of every sentence. The output corresponding to that token can be thought of as an embedding for the entire sentence.\n", + "\n", + "\n", + "\n", + "We'll save those in the `features` variable, as they'll serve as the features to our logitics regression model. Also remember, how we created the `labels` variable to hold our labels." + ], + "metadata": { + "id": "KBqHs4d_ObTc" + } + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "id": "39UVjAV56PJz", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "78b7f067-325f-47f2-c218-a99331f2c7d2" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(2000, 768)" + ] + }, + "metadata": {}, + "execution_count": 15 + } + ], + "source": [ + "import numpy as np\n", + "\n", + "\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", + "model.to(device)\n", + "\n", + "batch_size = 32\n", + "features = []\n", + "with torch.no_grad():\n", + " for i in range(0, len(texts), batch_size):\n", + " texts_batch = tokenized_texts[\"input_ids\"][i : i + batch_size].to(device)\n", + " masks_batch = tokenized_texts[\"attention_mask\"][i : i + batch_size].to(device)\n", + " output = model(texts_batch, masks_batch)\n", + " batch_features = output.last_hidden_state[:, 0, :].cpu().numpy()\n", + " features.append(batch_features)\n", + "\n", + "features = np.concatenate(features, axis=0)\n", + "features.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iaoEvM2evRx1" + }, + "source": [ + "## Classifier training" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "02wlGPSLoaYG" + }, + "source": [ + "Let's now split our datset into a training set and testing set (even though we're using 2,000 sentences from the SST2 training set)." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "id": "ddAqbkoU6PP9" + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "\n", + "train_features, test_features, train_labels, test_labels = train_test_split(features, labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B9bhSJpcv1Bl" + }, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uW_IiKvToaYG" + }, + "source": [ + "We can dive into Logistic regression directly with the Scikit Learn default parameters, but sometimes it's worth searching for the best value of the C parameter, which determines regularization strength." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "id": "cyEwr7yYD3Ci" + }, + "outputs": [], + "source": [ + "# YOUR CODE HERE\n", + "# [EXTRA] Grid search for parameters" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KCT9u8vAwnID" + }, + "source": [ + "We now train the LogisticRegression model. If you've chosen to do the gridsearch, you can plug the best hyperparameter values into the model declaration." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "id": "gG-EVWx4CzBc" + }, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "\n", + "warnings.simplefilter('ignore') # Ignore warning on model fitting.\n", + "lr_clf = LogisticRegression().fit(train_features, train_labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3rUMKuVgwzkY" + }, + "source": [ + "\n", + "\n", + "So how well does our model do in classifying sentences? One way is to check the accuracy against the testing dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iCoyxRJ7ECTA", + "outputId": "3472fa11-e7bb-45aa-b5c9-d23db7392423" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0.868" + ] + }, + "metadata": {}, + "execution_count": 19 + } + ], + "source": [ + "lr_clf.score(test_features, test_labels)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Another way to evaluate classification model is to plot the ROC curve and compute the area under it." + ], + "metadata": { + "id": "2xA5YIJPDYek" + } + }, + { + "cell_type": "code", + "source": [ + "from sklearn.metrics import roc_auc_score, roc_curve\n", + "\n", + "\n", + "plt.figure(figsize=(10, 6))\n", + "\n", + "proba = lr_clf.predict_proba(train_features)[:, 1]\n", + "auc = roc_auc_score(train_labels, proba)\n", + "plt.plot(*roc_curve(train_labels, proba)[:2], label=f'train AUC={auc:.4f}')\n", + "\n", + "proba = lr_clf.predict_proba(test_features)[:, 1]\n", + "auc = roc_auc_score(test_labels, proba)\n", + "plt.plot(*roc_curve(test_labels, proba)[:2], label=f'test AUC={auc:.4f}')\n", + "\n", + "plt.legend()\n", + "plt.show()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 374 + }, + "id": "186Bwg7UDLkU", + "outputId": "26f970d5-4bad-44d4-fa09-cbe42c1c9d6c" + }, + "execution_count": 20, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAFlCAYAAADPim3FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5RV5X3/8ffDTSSiyCUqgwjWS7npoCNItahBDeIK1GgMuoxWo9gkdiVGaaW/hORnm4rFxIQGY9WAWiNgRAO/ilGrELERdNSJ4ECAAMIMKgri/RL0+f1xZqbDOMycmX1uc877tRaLc/be5+zvuBf44fs8+9khxogkSZLap1O+C5AkSerIDFOSJEkJGKYkSZISMExJkiQlYJiSJElKwDAlSZKUQJd8nbhv375x0KBB+Tq9JElS2p577rk3Yoz9mtuXtzA1aNAgKisr83V6SZKktIUQXt7bPof5JEmSEjBMSZIkJWCYkiRJSsAwJUmSlIBhSpIkKQHDlCRJUgKGKUmSpAQMU5IkSQkYpiRJkhJoNUyFEOaEELaHEFbvZX8IIcwKIWwIIbwYQjgu82VKkiQVpnQ6U3cC41vYfxZwZN2vKcAvkpclSZLUMbT6bL4Y45MhhEEtHDIJuDvGGIEVIYReIYRDYoyvZKhGSZKULZVzYdX9OT/ta+98yBvvfpSR73qn1xBO/ObtGfmu9sjEg47LgK2N3tfUbftMmAohTCHVvWLgwIEZOLUk5de9K7ewqKo232VI7TZ9xy8Z9OeNbO56eE7P+86HuwHo2T0TUSS/cvoTxBhvA24DqKioiLk8t6Tilc9As3LTTgBGD+6dl/OruI17fwknfbA0q+eoD1LX95mZ1fM0Z1J5GReO7vjNlUyEqVrg0EbvB9Rtk1QkCr37ks9AM3pw76L5H0KHkqehqZx75anU74ednMWTjGTYiPNYUDEmi+cobpkIU4uBq0II84HRwFvOl5KyL5cBp9C7LwaaApat0PNyLkJGATjsZBhxHlRcmu9K1IJWw1QIYR5wKtA3hFAD/ADoChBjvBVYAkwANgDvA15xlZx8dG5yGXAMKwLaF4yyFXoMGSog6dzNd0Er+yPwrYxVJHUQjQNUPjo3Bpx2KpXhoWxoTzAy9KgEdPwp9FIGtaXD1DhAGWwKRDpBqVSGh7LBYCQ1yzClktNSYGpLh8kAVYBW3Q+vroKDR+z9GAOBpAwzTKlDyOScpJYCkwGpjQptyKw+SF36UL4rkVRCDFMqWNmak2RgyqB0OkG5dPCIVNdJknLIMKWsSdpNck5SFmWqo2QnSJIMU8qsTHaTDFAZsLfQlKlJ2HaCJMkwpWSadp/sJmVBki7S3kKTk7AlKWMMU2qTlsJT/e8GqEYyMZyWpItkaJKkrDNMqUWGp4QyMUHbQCRJBc0wpRYtqqql+pW3GXrI/oDhCWhbt8kJ2pJU9AxTatXQQ/ZnwZU+TbxBW7pNTtCWpKJnmNJe3btyCys37czp8+YKRkvdJ7tNkqRGDFMCml8Tqn5+1KTysnyUlF8tdZ/sNkmSGjFMlZi9LaTZ3JpQJTk/qr4jZfdJkpQmw1SRa+1uvHolGZzgs8N5jZchsPskSUqDYapI1Yeokl7KIJ277pqu4eQyBJKkNjJMFYGW5jsVdXhqLSyls9il4UmSlJBhqgg0XQsKSiREtRaWDEqSpBwwTHUwzXWh6oNUyawFVT9B3LAkSSoAhqkC1trwXb2hh+xffMsXuM6TJKmDMEwVoL1NHq9/XbTDd425zpMkqYMwTBWQ5kJUSQSnvbH7JEnqAAxTBaR+InlJhai9Deel++w7SZLyzDBVYEpqIjnsfTjPoTxJUgdhmMqTlu7K63DSWRxzb5xMLknq4AxTedLc2lAd4q685oJTOotj7o0dKElSB2eYypGmnagOuzZUc8NyrvckSSphhqksahygmi5z0CG6UHvjsJwkSQ0MU1nQ3BIHHf4OvfrhPe+ykyRpD4apLCj4JQ7aM2G88bwo5zhJktTAMJVB9R2pgp8P1Z4Ok/OiJElqlmEqgxoHqYKcD9V0qM55T5IkJWaYypB7V25h5aadjB7cu2N0pByqkyQpIwxTGVJ/154dKUmSSothKqHG86RGD+5deJPNwY6UJElZZJhK4N6VW/inB1cBNNy5V7DsSEmSlBWGqXZqHKT+9ZwRhdmRcm0oSZKyrlO+C+iIOkSQAof3JEnKATtT7VA/2bxgg5QTziVJyhk7U+1UsJPNwY6UJEk5ZGeqjRqvJ1XQ7EhJkpQTdqbaoPFcqYK+c0+SJOWMYaoNOsRcqblnp4b4JElSThim0tR4eK8ggxQ4V0qSpDxwzlSaCu5xMfV37DXm3XuSJOWcYaoN8tqVahqeXn4q9fthJ//vNjtSkiTlnGEqDQVxB1/TlcwPOzkVnCouzV9NkiTJMJWOvAzxNe1EOYQnSVJBMky14N6VW1hUVUv1K2/nfoivaSfKITxJkgqSYWovGq8pNXpw7/xMPLcTJUlSwUsrTIUQxgM/AzoDd8QYZzTZPxC4C+hVd8x1McYlGa41ZzrMg4wlSVLetbrOVAihMzAbOAsYClwQQhja5LDvAffFGEcCk4FbMl1oruQ9SLnwpiRJHUo6i3aOAjbEGDfGGD8G5gOTmhwTgf3rXh8AbMtcibmV91XOXXhTkqQOJZ1hvjJga6P3NcDoJsf8EHg0hPD3wOeA0zNSXZ7kbLK5C29KktThZWoC+gXAnTHGH4cQxgD/GUIYHmP8tPFBIYQpwBSAgQNLeB5SfYhy4U1Jkjq8dMJULXBoo/cD6rY19nVgPECM8ekQQnegL7C98UExxtuA2wAqKipiO2vOmqwvztlciHLhTUmSOrR0wtSzwJEhhMGkQtRk4MImx2wBxgF3hhCGAN2B1zNZaC5kfXHO+vlQhihJkopGq2Eqxrg7hHAV8AipZQ/mxBhfCiFcD1TGGBcD1wC3hxCuJjUZ/W9jjAXXeWpJ465UVudLOR9KkqSiktacqbo1o5Y02Ta90etq4KTMlpZbeXlkjCRJ6vDSWRqhZOT8kTGSJKnDM0xJkiQlYJjif+dLSZIktZVhCudLSZKk9jNM1cnqfCmftydJUtEyTOWCz9uTJKloZepxMmqN60tJklSU7ExJkiQlYJiSJElKwDAlSZKUgGFKkiQpAcOUJElSAt7Nlw2Vc1PLIdSrXxZBkiQVHTtT2VC/rlQ915eSJKlo2ZnKlMbdqPpOlOtKSZJU9OxMZUrjbpSdKEmSSoadqUyyGyVJUskp+c7UvSu3sHLTznyXIUmSOqiSD1OLqmoBmFReludKJElSR1TyYQpg9ODeXDh6YPs+XDkX5p695917kiSpZBimkqqfeO6kc0mSSpIT0DPBieeSJJWsku5MOflckiQlVbJh6t6VW/inB1PznNo1+dy5UpIkiRIOU/V38f3rOSPaN/ncuVKSJIkSnzOV6C4+cK6UJEkq3c6UJElSJhimJEmSEjBMSZIkJWCYkiRJSsAw1VYuiSBJkhoxTLWVSyJIkqRGSnpphHZzSQRJklTHzpQkSVICdqZaUzk3NbRXr36IT5IkCTtTraufI1XPuVKSJKkRO1PpcI6UJEnaCztTkiRJCRimJEmSEjBMSZIkJWCYkiRJSsAwJUmSlEBJhql7V25h5aad+S5DkiQVgZIMU4uqagGYVF6W50okSVJHV1LrTN27cguLqmqpfuVtRg/uzYWjB+a7JEmS1MGVVGeqPkgNPWR/u1KSJCkjSqozBTD0kP1ZcOWYfJchSZKKREl1piRJkjKt5DpTaamcm3rAMaQecnzwiPzWI0mSCpZhCvYMTwAvP5X6/bCTU0FqxHn5qUuSJBU8wxSkglTjDtRhJ6cCVMWl+a1LkiQVvLTCVAhhPPAzoDNwR4xxRjPHnA/8EIjAH2KMF2awzuw7eARc+lC+q5AkSR1Mq2EqhNAZmA2cAdQAz4YQFscYqxsdcyQwDTgpxvhmCOHz2So4o+qH95wXJUmS2imdu/lGARtijBtjjB8D84FJTY65ApgdY3wTIMa4PbNlZknjIOW8KEmS1A7pDPOVAVsbva8BRjc55iiAEML/kBoK/GGM8bdNvyiEMAWYAjBwYIGsPu7wniRJSiBT60x1AY4ETgUuAG4PIfRqelCM8bYYY0WMsaJfv34ZOrUkSVL+pBOmaoFDG70fULetsRpgcYzxzzHGTcA6UuFKkiSpqKUTpp4FjgwhDA4hdAMmA4ubHPMbUl0pQgh9SQ37bcxgnZIkSQWp1TAVY9wNXAU8AqwB7osxvhRCuD6EMLHusEeAHSGEamApMDXGuCNbRUuSJBWKtNaZijEuAZY02Ta90esIfLfulyRJUskozQcdV86FuWenlkWQJElKoDTDlOtLSZKkDCndZ/O5vpQkScqA0uxMSZIkZYhhSpIkKQHDlCRJUgKGKUmSpAQMU5IkSQkYpiRJkhIomTB178otrNy0M99lSJKkIlMyYWpRVS0Ak8rL8lyJJEkqJiUTpgBGD+7NhaMH5rsMSZJUREoqTEmSJGVaSYQp50tJkqRsKYkw5XwpSZKULSURpsD5UpIkKTtKJkxJkiRlg2FKkiQpAcOUJElSAoYpSZKkBAxTkiRJCRimJEmSEjBMSZIkJWCYkiRJSsAwJUmSlIBhSpIkKQHDlCRJUgKGKUmSpAQMU5IkSQkUfZi6d+UWVm7amXpTORfmng2vrspvUZIkqWgUfZhaVFULwKTyMlh1fypIHTwCRpyX58okSVIx6JLvAnJh9ODeXDh6IFSTClKXPpTvkiRJUpEo+s6UJElSNhmmJEmSEjBMSZIkJWCYkiRJSsAwJUmSlIBhSpIkKQHDlCRJUgKGKUmSpAQMU5IkSQkYpiRJkhIwTEmSJCVgmJIkSUrAMCVJkpSAYUqSJCkBw5QkSVIChilJkqQEDFOSJEkJFHWYunflFlZu2pnvMiRJUhFLK0yFEMaHEP4YQtgQQriuhePODSHEEEJF5kpsv0VVtQBMKi/LcyWSJKlYtRqmQgidgdnAWcBQ4IIQwtBmjusJfBtYmekikxg9uDcXjh6Y7zIkSVKRSqczNQrYEGPcGGP8GJgPTGrmuH8GbgQ+zGB9kiRJBS2dMFUGbG30vqZuW4MQwnHAoTHGh1r6ohDClBBCZQih8vXXX29zsZIkSYUm8QT0EEIn4CfANa0dG2O8LcZYEWOs6NevX9JTS5Ik5V06YaoWOLTR+wF12+r1BIYDy0IIm4ETgcWFMgldkiQpm9IJU88CR4YQBocQugGTgcX1O2OMb8UY+8YYB8UYBwErgIkxxsqsVCxJklRAWg1TMcbdwFXAI8Aa4L4Y40shhOtDCBOzXaAkSVIh65LOQTHGJcCSJtum7+XYU5OXlVz9gp2jB/fOdymSJKmIFe0K6C7YKUmScqFowxS4YKckScq+og5TkiRJ2WaYkiRJSqA0wlTlXJh7Nry6Kt+VSJKkIlMaYWrV/akgdfAIGHFevquRJElFJK2lEYrCwSPg0hYfHShJktRmpdGZkiRJyhLDlCRJUgKGKUmSpAQMU5IkSQkYpiRJkhIwTEmSJCVgmJIkSUrAMCVJkpSAYUqSJCkBw5QkSVIChilJkqQEDFOSJEkJGKYkSZISMExJkiQlYJiSJElKwDAlSZKUQJd8F5At495fwkkfLIW5B8Crq+DgEfkuSZIkFaGi7Uyd9MFSBv15Y+rNwSNgxHn5LUiSJBWlou1MAWzuejjDLn0o32VIkqQiVrSdKUmSpFwwTEmSJCVgmJIkSUrAMCVJkpSAYUqSJCkBw5QkSVIChilJkqQEDFOSJEkJGKYkSZISMExJkiQlYJiSJElKwDAlSZKUgGFKkiQpAcOUJElSAoYpSZKkBAxTkiRJCRimJEmSEjBMSZIkJWCYkiRJSsAwJUmSlIBhSpIkKQHDlCRJUgKGKUmSpAQMU5IkSQmkFaZCCONDCH8MIWwIIVzXzP7vhhCqQwgvhhAeDyEclvlSJUmSCk+rYSqE0BmYDZwFDAUuCCEMbXLYC0BFjPEY4H7g3zJdqCRJUiFKpzM1CtgQY9wYY/wYmA9ManxAjHFpjPH9urcrgAGZLVOSJKkwpROmyoCtjd7X1G3bm68DDycpSpIkqaPokskvCyFcBFQAp+xl/xRgCsDAgQMzeWpJkqS8SKczVQsc2uj9gLptewghnA78H2BijPGj5r4oxnhbjLEixljRr1+/9tQrSZJUUNIJU88CR4YQBocQugGTgcWNDwghjAT+g1SQ2p75MiVJkgpTq2EqxrgbuAp4BFgD3BdjfCmEcH0IYWLdYTOB/YBfhxCqQgiL9/J1kiRJRSWtOVMxxiXAkibbpjd6fXqG65IkSeoQXAFdkiQpAcOUJElSAoYpSZKkBAxTkiRJCRimJEmSEjBMSZIkJWCYkiRJSsAwJUmSlIBhSpIkKQHDlCRJUgKGKUmSpAQMU5IkSQkYpiRJkhIwTEmSJCVgmJIkSUrAMCVJkpSAYUqSJCkBw5QkSVIChilJkqQEDFOSJEkJGKYkSZISMExJkiQlYJiSJElKwDAlSZKUgGFKkiQpAcOUJElSAoYpSZKkBAxTkiRJCRimJEmSEjBMSZIkJWCYkiRJSqBLvguQJKkQ/fnPf6ampoYPP/ww36Uoh7p3786AAQPo2rVr2p8xTEmS1Iyamhp69uzJoEGDCCHkuxzlQIyRHTt2UFNTw+DBg9P+nMN8kiQ148MPP6RPnz4GqRISQqBPnz5t7kYapiRJ2guDVOlpzzU3TEmSVIB27drFLbfc0q7PTpgwgV27drX5c+Xl5UyePHmPbaeeeiqVlZUN7zdv3szw4cMb3j/zzDOMHTuWo48+mpEjR3L55Zfz/vvvp3W+3/72txx99NEcccQRzJgxo9ljXn75ZcaNG8cxxxzDqaeeSk1NTcO+LVu2cOaZZzJkyBCGDh3K5s2bAXj88cc57rjjKC8v5+STT2bDhg0A3HnnnfTr14/y8nLKy8u544470qqzNYYpSZIKUEthavfu3S1+dsmSJfTq1atN51uzZg2ffPIJy5cv57333kvrM6+99hpf+cpXuPHGG/njH//ICy+8wPjx43nnnXda/ewnn3zCt771LR5++GGqq6uZN28e1dXVnznu2muv5eKLL+bFF19k+vTpTJs2rWHfxRdfzNSpU1mzZg3PPPMMn//85wH4xje+wa9+9Suqqqq48MIL+Zd/+ZeGz3z1q1+lqqqKqqoqLr/88rR+ztYYpiRJKkDXXXcdf/rTnygvL2fq1KksW7aMv/7rv2bixIkMHToUgL/5m7/h+OOPZ9iwYdx2220Nnx00aBBvvPEGmzdvZsiQIVxxxRUMGzaMM888kw8++KDZ882bN4+vfe1rnHnmmSxatCitGmfPns0ll1zCmDFjGradd955HHTQQa1+9plnnuGII47g8MMPp1u3bkyePLnZ81ZXV/OFL3wBgNNOO63hmOrqanbv3s0ZZ5wBwH777UePHj2A1FDd22+/DcBbb71F//790/p52su7+SRJasX//X8vUb3t7Yx+59D++/ODLw3b6/4ZM2awevVqqqqqAFi2bBnPP/88q1evbrjTbM6cOfTu3ZsPPviAE044gXPPPZc+ffrs8T3r169n3rx53H777Zx//vksXLiQiy666DPnW7BgAY899hhr167l3//937nwwgtb/RlWr17NJZdc0uy+pUuXcvXVV39me48ePfj9739PbW0thx56aMP2AQMGsHLlys8cf+yxx/LAAw/w7W9/mwcffJB33nmHHTt2sG7dOnr16sWXv/xlNm3axOmnn86MGTPo3Lkzd9xxBxMmTGDfffdl//33Z8WKFQ3ft3DhQp588kmOOuoobr755j1qaC87U5IkdRCjRo3a45b9WbNmceyxx3LiiSeydetW1q9f/5nPDB48mPLycgCOP/74hnlFjVVWVtK3b18GDhzIuHHjeOGFF9i5cyfQ/ITsdCZpn3baaQ3DaY1//f73v0/3xwXgpptu4ne/+x0jR47kd7/7HWVlZXTu3Jndu3ezfPlybrrpJp599lk2btzInXfeCcDNN9/MkiVLqKmp4dJLL+W73/0uAF/60pfYvHkzL774ImecccZeg2Bb2ZmSJKkVLXWQculzn/tcw+tly5bx3//93zz99NP06NGDU089tdlb+vfZZ5+G1507d252mG/evHmsXbuWQYMGAfD222+zcOFCrrjiCvr06cObb77ZcOzOnTvp27cvAMOGDeO5555j0qRJn/nO1jpTZWVlbN26tWF7TU0NZWVlnzm+f//+PPDAAwC8++67LFy4kF69ejFgwADKy8s5/PDDgdSQ54oVK5g4cSJ/+MMfGD16NJCaIzV+/HiAPbp2l19+Of/wD//wmfO1h50pSZIKUM+ePVucyP3WW29x4IEH0qNHD9auXbvHUFZbfPrpp9x3332sWrWKzZs3s3nzZhYtWsS8efOA1N1899xzDzFGAO666y5OO+00AK666iruuuuuPYbnHnjgAV577bVWO1MnnHAC69evZ9OmTXz88cfMnz+fiRMnfqa+N954g08//RSAG264gcsuu6zh87t27eL1118H4IknnmDo0KEceOCBvPXWW6xbtw6Axx57jCFDhgDwyiuvNHzv4sWLG7YnZWdKkqQC1KdPH0466SSGDx/OWWedxdlnn73H/vHjx3PrrbcyZMgQjj76aE488cR2nWf58uWUlZXtMUl77NixVFdX88orrzBlyhTWrl3LscceSwiBiooKbrjhBgAOOugg5s+fz7XXXsv27dvp1KkTY8eObegEtaRLly78/Oc/54tf/CKffPIJl112GcOGpTqA06dPp6KigokTJ7Js2TKmTZtGCIGxY8cye/ZsINVlu+mmmxg3bhwxRo4//niuuOIKunTpwu233865555Lp06dOPDAA5kzZw6QGhZdvHgxXbp0oXfv3g3DgkmF+qSZaxUVFbHxuhWZ9tK/ngzAsH96KmvnkCQVrzVr1mSsc6GOpblrH0J4LsZY0dzxDvNJkiQlYJiSJElKwDAlSZKUgGFKkiQpAcOUJElSAoYpSZKkBAxTkiQVoF27dnHLLbe0+/M//elPef/99/e6/4033qBr167ceuute2zfb7/99nh/5513ctVVVzW8v/vuuxk+fDgjRoxg5MiR3HTTTWnXdMMNN3DEEUdw9NFH88gjjzR7zBNPPMFxxx3H8OHDueSSS9i9ezcAixYt4phjjqG8vJyKigqeeiq19NHLL7/McccdR3l5OcOGDdvj5/n444+ZMmUKRx11FH/5l3/JwoUL0661LdIKUyGE8SGEP4YQNoQQrmtm/z4hhAV1+1eGEAZlulBJkkpJtsPUr3/9a0488cSGlc7T8fDDD/PTn/6URx99lFWrVrFixQoOOOCAtD5bXV3N/Pnzeemll/jtb3/LN7/5TT755JM9jvn000+55JJLmD9/PqtXr+awww7jrrvuAmDcuHH84Q9/oKqqijlz5nD55ZcDcMghh/D0009TVVXFypUrmTFjBtu2bQPgRz/6EZ///OdZt24d1dXVnHLKKWn/rG3RapgKIXQGZgNnAUOBC0IIQ5sc9nXgzRjjEcDNwI2ZLlSSpFJy3XXX8ac//Yny8nKmTp0KwMyZMznhhBM45phj+MEPfgDAe++9x9lnn82xxx7L8OHDWbBgAbNmzWLbtm2cdtppDY9+aWrevHn8+Mc/pra2lpqamrRquuGGG7jpppsaVkvfZ599uOKKK9L67KJFi5g8eTL77LMPgwcP5ogjjuCZZ57Z45gdO3bQrVs3jjrqKADOOOOMhm7Sfvvt1/CA5ffee6/hdbdu3RqeP/jRRx81PHoGYM6cOUybNg2ATp06NTxTMNPSeZzMKGBDjHEjQAhhPjAJqG50zCTgh3Wv7wd+HkIIMV/Lq0uSlEkPXwevrsrsdx48As6asdfdM2bMYPXq1VRVVQHw6KOPsn79ep555hlijEycOJEnn3yS119/nf79+/PQQw8BqWf2HXDAAfzkJz9h6dKlzQaIrVu38sorrzBq1CjOP/98FixYwDXXXNNqyatXr+b4449vdt/MmTP51a9+9ZntY8eOZdasWdTW1u7xyJsBAwZQW1u7x7F9+/Zl9+7dVFZWUlFRwf3337/Hw5AffPBBpk2bxvbt2xt+3vqf5+yzz2bDhg3MnDmT/v37s2vXLgC+//3vs2zZMv7iL/6Cn//85xx00EGt/pxtlc4wXxmwtdH7mrptzR4TY9wNvAX0aXIMIYQpIYTKEEJl/YMJs+WdXkN4p5ePAZAkFYdHH32URx99lJEjR3Lcccexdu1a1q9fz4gRI3jsscf4x3/8R5YvX57WsNuCBQs4//zzAZg8eXKrQ331XaCWTJ06tdkHG8+aNSu9H7DuPPPnz+fqq69m1KhR9OzZk86dOzfsP+ecc1i7di2/+c1v+P73v9+w/dBDD+XFF19kw4YN3HXXXbz22mvs3r2bmpoa/uqv/ornn3+eMWPGcO2116ZdS1vk9EHHMcbbgNsg9Wy+bJ7rxG/ens2vlySVkhY6SLkSY2TatGlceeWVn9n3/PPPs2TJEr73ve8xbtw4pk+f3uJ3zZs3j1dffbWhk7Rt2zbWr1/PkUceyb777svHH39Mt27dANi5c2dDd2vYsGE899xzfOELX/jMd7bWmSorK9ujy1RTU0NZWdPeDIwZM4bly5cDqQC5bt26Zr9z48aNvPHGG3t03vr378/w4cNZvnw55557Lj169ODLX/4yAF/5ylf45S9/2eJ/l/ZKpzNVCxza6P2Aum3NHhNC6AIcAOzIRIGSJJWinj178s477zS8/+IXv8icOXN49913AaitrWX79u1s27aNHj16cNFFFzF16lSef/75Zj9fb926dbz77rvU1tayefNmNm/ezLRp0xq6U6eccgr33HMPAB988AH33Xdfw7yradOmMXXqVF599VUgdbfcHXfcAbTemZo4cSLz58/no48+YtOmTaxfv55Ro0Z9pr7t27cDqflPN954I3/3d38HwIYNG6ifPfT888/z0Ucf0adPH2pqavjggw8AePPNN3nqqac4+uijCSHwpS99iWXLlgHw+OOPM3Ro07BxfC8AAAZBSURBVCnfmZFOZ+pZ4MgQwmBSoWkycGGTYxYDlwBPA+cBTzhfSpKk9uvTpw8nnXQSw4cP56yzzmLmzJmsWbOGMWPGAKkJ2ffccw8bNmxg6tSpdOrUia5du/KLX/wCgClTpjB+/Hj69+/P0qVLG7533rx5nHPOOXuc69xzz+WrX/0q06dP52c/+xlXXnkls2bNIsbIxRdfzNixYwGYMGECr732GqeffjoxRkIIXHbZZWn9PMOGDeP8889n6NChdOnShdmzZzcM4U2YMIE77riD/v37M3PmTP7rv/6LTz/9lG984xsNXbCFCxdy991307VrV/bdd18WLFhACIE1a9ZwzTXXEEIgxsi1117LiBEjALjxxhv52te+xne+8x369evH3LlzE1yRvQvpZJ4QwgTgp0BnYE6M8UchhOuByhjj4hBCd+A/gZHATmBy/YT1vamoqIiVlZWJfwBJkrJhzZo1DBni3NtS1Ny1DyE8F2OsaO74tOZMxRiXAEuabJve6PWHwFfaXK0kSVIH5wrokiRJCRimJEmSEjBMSZK0F95LVXrac80NU5IkNaN79+7s2LHDQFVCYozs2LGD7t27t+lzOV20U5KkjmLAgAHU1NSQ7Sd2qLB0796dAQMGtOkzhilJkprRtWtXBg8enO8y1AE4zCdJkpSAYUqSJCkBw5QkSVICaT1OJisnDuF14OUsn6Yv8EaWz6G287oUHq9JYfK6FB6vSWHKxXU5LMbYr7kdeQtTuRBCqNzbc3SUP16XwuM1KUxel8LjNSlM+b4uDvNJkiQlYJiSJElKoNjD1G35LkDN8roUHq9JYfK6FB6vSWHK63Up6jlTkiRJ2VbsnSlJkqSsKoowFUIYH0L4YwhhQwjhumb27xNCWFC3f2UIYVDuqyw9aVyX74YQqkMIL4YQHg8hHJaPOktJa9ek0XHnhhBiCMG7lrIsnWsSQji/7s/KSyGEe3NdYylK4++vgSGEpSGEF+r+DpuQjzpLSQhhTghhewhh9V72hxDCrLpr9mII4bhc1dbhw1QIoTMwGzgLGApcEEIY2uSwrwNvxhiPAG4GbsxtlaUnzevyAlARYzwGuB/4t9xWWVrSvCaEEHoC3wZW5rbC0pPONQkhHAlMA06KMQ4DvpPzQktMmn9WvgfcF2McCUwGbsltlSXpTmB8C/vPAo6s+zUF+EUOagKKIEwBo4ANMcaNMcaPgfnApCbHTALuqnt9PzAuhBByWGMpavW6xBiXxhjfr3u7AmjbY7rVVun8WQH4Z1L/4Pgwl8WVqHSuyRXA7BjjmwAxxu05rrEUpXNdIrB/3esDgG05rK8kxRifBHa2cMgk4O6YsgLoFUI4JBe1FUOYKgO2NnpfU7et2WNijLuBt4A+OamudKVzXRr7OvBwVitSq9ekri1+aIzxoVwWVsLS+XNyFHBUCOF/QggrQggt/ctcmZHOdfkhcFEIoQZYAvx9bkpTC9r6/52M6ZKLk0gtCSFcBFQAp+S7llIWQugE/AT42zyXoj11ITVscSqp7u2TIYQRMcZdea1KFwB3xhh/HEIYA/xnCGF4jPHTfBem3CuGzlQtcGij9wPqtjV7TAihC6mW7I6cVFe60rkuhBBOB/4PMDHG+FGOaitVrV2TnsBwYFkIYTNwIrDYSehZlc6fkxpgcYzxzzHGTcA6UuFK2ZPOdfk6cB9AjPFpoDup58Mpf9L6/042FEOYehY4MoQwOITQjdREwMVNjlkMXFL3+jzgiegCW9nW6nUJIYwE/oNUkHIeSPa1eE1ijG/FGPvGGAfFGAeRmsc2McZYmZ9yS0I6f3/9hlRXihBCX1LDfhtzWWQJSue6bAHGAYQQhpAKU6/ntEo1tRi4uO6uvhOBt2KMr+TixB1+mC/GuDuEcBXwCNAZmBNjfCmEcD1QGWNcDPySVAt2A6nJa5PzV3FpSPO6zAT2A35ddz/AlhjjxLwVXeTSvCbKoTSvySPAmSGEauATYGqM0c56FqV5Xa4Bbg8hXE1qMvrf+o/07AohzCP1D4u+dXPVfgB0BYgx3kpq7toEYAPwPnBpzmrz2kuSJLVfMQzzSZIk5Y1hSpIkKQHDlCRJUgKGKUmSpAQMU5IkSQkYpiRJkhIwTEmSJCVgmJIkSUrg/wP9wObXRVPAzwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "75oyhr3VxHoE" + }, + "source": [ + "How good is this score? What can we compare it against? Let's first look at a dummy classifier:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lnwgmqNG7i5l", + "outputId": "2425fa54-58a7-4224-effe-f9c7f37d3a05" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Dummy classifier score: 0.518 (+/- 0.003)\n" + ] + } + ], + "source": [ + "from sklearn.dummy import DummyClassifier\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "clf = DummyClassifier()\n", + "scores = cross_val_score(clf, train_features, train_labels)\n", + "print(f\"Dummy classifier score: {scores.mean():.3f} (+/- {2 * scores.std():.3f})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7Lg4LOpoxSOR" + }, + "source": [ + "So our model clearly does better than a dummy classifier. But how does it compare against the best models?\n", + "\n", + "For reference, the [highest accuracy score](http://nlpprogress.com/english/sentiment_analysis.html) for this dataset is currently **96.8**. DistilBERT can be trained to improve its score on this task – a process called **fine-tuning** which updates BERT’s weights to make it achieve a better performance in this sentence classification task (which we can call the downstream task). The fine-tuned DistilBERT turns out to achieve an accuracy score of **90.7**. The full size BERT model achieves **94.9**.\n", + "\n", + "And that’s it! That’s a good first contact with BERT. The next step would be to head over to the documentation and try your hand at [fine-tuning](https://huggingface.co/transformers/examples.html#glue). You can also go back and switch from distilBERT to BERT and see how that works." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EJQuqV6cnWQu", + "outputId": "402d109c-01bb-485d-a510-4be8684c9c06" + }, + "source": [ + "## Part 2: Looking back." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "outputId": "402d109c-01bb-485d-a510-4be8684c9c06", + "id": "zIBbP_oroaYH" + }, + "source": [ + "Now it is your turn to reproduce the steps above.\n", + "\n", + "We shall revisit the first homework and see whether we could improve the results a little bit more. The average ROC-AUC on test set was around $0.9$ (using the words embeddings). \n", + "\n", + "__Let's see whether we can beat it.__" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "id": "kz8QBEXozHJx", + "outputId": "d4ac5072-c505-4c8f-89a6-b09eac02856a" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
should_bancomment_text
00The picture on the article is not of the actor...
11Its madness. Shes of Chinese heritage, but JAP...
21Fuck You. Why don't you suck a turd out of my ...
31God is dead\\nI don't mean to startle anyone bu...
41THIS USER IS A PLANT FROM BRUCE PERENS AND GRO...
\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + " \n", + "
\n", + "
\n", + " " + ], + "text/plain": [ + " should_ban comment_text\n", + "0 0 The picture on the article is not of the actor...\n", + "1 1 Its madness. Shes of Chinese heritage, but JAP...\n", + "2 1 Fuck You. Why don't you suck a turd out of my ...\n", + "3 1 God is dead\\nI don't mean to startle anyone bu...\n", + "4 1 THIS USER IS A PLANT FROM BRUCE PERENS AND GRO..." + ] + }, + "metadata": {}, + "execution_count": 22 + } + ], + "source": [ + "dataset_url = 'https://raw.githubusercontent.com/neychev/made_nlp_course/master/datasets/comments_small_dataset/comments.tsv'\n", + "dataset = pd.read_csv(dataset_url, sep='\\t')\n", + "dataset.head()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "One last note: this dataset contains some very long sentences, while the vast majority of sequences fall into category of 500 tokens and less:" + ], + "metadata": { + "id": "ZPyxeBCgGtfg" + } + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "id": "XdZjuVxRoaYH", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 374 + }, + "outputId": "b3c3d368-87b5-4c37-b24c-0507f2a8ad0d" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAFlCAYAAAApo6aBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAARCElEQVR4nO3dbYyld1nH8d9ll6I8hBa6aXDbuFUaTWMiNBuswRBCDY/GrQkQjIENaVJfFAXRyMIbiL4pRkFIDEmlmJIgDwFMG6lgw0OML6hsoQJtRdZSaDeFLlAKShALly/mLgxl6852r9k5M/18ksncT2fOf/Lfc/ab+z5zTnV3AAA4eT+11QMAANgphBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMCQXVs9gCQ566yzeu/evVs9DACA47rxxhu/1t27j7VvJcJq7969OXTo0FYPAwDguKrqSw+2z6VAAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhu7Z6AKfK3oMf3OohjLn9iudv9RAAgGNwxgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGLKhsKqqP6yqm6vqc1X1rqr66ao6r6puqKrDVfWeqjp9OfaRy/rhZf/ezfwFAABWxXHDqqr2JPmDJPu6+5eTnJbkxUnekORN3f2kJPckuXS5yaVJ7lm2v2k5DgBgx9vopcBdSX6mqnYleVSSu5I8M8n7lv1XJ7lkWd6/rGfZf3FV1cxwAQBW13HDqruPJPmLJF/OWlDdm+TGJN/s7vuWw+5MsmdZ3pPkjuW29y3HP+GBP7eqLquqQ1V16OjRoyf7ewAAbLmNXAo8M2tnoc5L8rNJHp3kOSd7x919ZXfv6+59u3fvPtkfBwCw5TZyKfA3knyxu4929/8m+UCSpyU5Y7k0mCTnJDmyLB9Jcm6SLPsfl+Tro6MGAFhBGwmrLye5qKoetbxW6uIktyT5WJIXLMccSHLNsnztsp5l/0e7u+eGDACwmjbyGqsbsvYi9E8l+exymyuTvDrJq6rqcNZeQ3XVcpOrkjxh2f6qJAc3YdwAACtn1/EPSbr7dUle94DNtyV56jGO/W6SF5780AAAthfvvA4AMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwZENhVVVnVNX7qurfq+rWqvq1qnp8VV1fVV9Yvp+5HFtV9ZaqOlxVn6mqCzf3VwAAWA0bPWP15iQf6u5fSvIrSW5NcjDJR7r7/CQfWdaT5LlJzl++Lkvy1tERAwCsqOOGVVU9LsnTk1yVJN39ve7+ZpL9Sa5eDrs6ySXL8v4k7+g1n0hyRlU9cXzkAAArZiNnrM5LcjTJ31bVp6vqbVX16CRnd/ddyzFfSXL2srwnyR3rbn/nsg0AYEfbSFjtSnJhkrd291OS/Hd+dNkvSdLdnaRP5I6r6rKqOlRVh44ePXoiNwUAWEkbCas7k9zZ3Tcs6+/LWmh99f5LfMv3u5f9R5Kcu+725yzbfkx3X9nd+7p73+7dux/q+AEAVsZxw6q7v5Lkjqr6xWXTxUluSXJtkgPLtgNJrlmWr03y0uWvAy9Kcu+6S4YAADvWrg0e9/tJ3llVpye5LcnLshZl762qS5N8KcmLlmOvS/K8JIeTfGc5FgBgx9tQWHX3TUn2HWPXxcc4tpNcfpLjAgDYdrzzOgDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMAQYQUAMERYAQAMEVYAAEOEFQDAEGEFADBEWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMCQDYdVVZ1WVZ+uqn9Y1s+rqhuq6nBVvaeqTl+2P3JZP7zs37s5QwcAWC0ncsbqFUluXbf+hiRv6u4nJbknyaXL9kuT3LNsf9NyHADAjrehsKqqc5I8P8nblvVK8swk71sOuTrJJcvy/mU9y/6Ll+MBAHa0jZ6x+qskf5LkB8v6E5J8s7vvW9bvTLJnWd6T5I4kWfbfuxwPALCjHTesquo3k9zd3TdO3nFVXVZVh6rq0NGjRyd/NADAltjIGaunJfmtqro9ybuzdgnwzUnOqKpdyzHnJDmyLB9Jcm6SLPsfl+TrD/yh3X1ld+/r7n27d+8+qV8CAGAVHDesuvs13X1Od+9N8uIkH+3u303ysSQvWA47kOSaZfnaZT3L/o92d4+OGgBgBZ3M+1i9Osmrqupw1l5DddWy/aokT1i2vyrJwZMbIgDA9rDr+If8SHd/PMnHl+Xbkjz1GMd8N8kLB8YGALCteOd1AIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHHDauqOreqPlZVt1TVzVX1imX746vq+qr6wvL9zGV7VdVbqupwVX2mqi7c7F8CAGAVbOSM1X1J/qi7L0hyUZLLq+qCJAeTfKS7z0/ykWU9SZ6b5Pzl67Ikbx0fNQDACjpuWHX3Xd39qWX520luTbInyf4kVy+HXZ3kkmV5f5J39JpPJDmjqp44PnIAgBVzQq+xqqq9SZ6S5IYkZ3f3XcuuryQ5e1nek+SOdTe7c9kGALCjbTisquoxSd6f5JXd/a31+7q7k/SJ3HFVXVZVh6rq0NGjR0/kpgAAK2lDYVVVj8haVL2zuz+wbP7q/Zf4lu93L9uPJDl33c3PWbb9mO6+srv3dfe+3bt3P9TxAwCsjI38VWAluSrJrd39xnW7rk1yYFk+kOSaddtfuvx14EVJ7l13yRAAYMfatYFjnpbkJUk+W1U3Ldtem+SKJO+tqkuTfCnJi5Z91yV5XpLDSb6T5GWjIwYAWFHHDavu/pck9SC7Lz7G8Z3k8pMcFwDAtuOd1wEAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIcIKAGCIsAIAGCKsAACGCCsAgCHCCgBgiLACABgirAAAhuza6gFw4vYe/OBWD2HM7Vc8f6uHAABjnLECABgirAAAhggrAIAhwgoAYIiwAgAYIqwAAIYIKwCAIZvyPlZV9Zwkb05yWpK3dfcVm3E/sEp2yvuLeW8xgIdu/IxVVZ2W5K+TPDfJBUl+p6oumL4fAIBVsxlnrJ6a5HB335YkVfXuJPuT3LIJ98U2t1PO8uwkO2lOnH0DTrXNCKs9Se5Yt35nkl/dhPsB+H/tpEjcSQTv6tlJj5Wt/ve1ZZ8VWFWXJblsWf2vqvr8Jt7dWUm+tok/n81h3rYfc7Y9ndJ5qzecqnva8TzejuEU/fv6uQfbsRlhdSTJuevWz1m2/ZjuvjLJlZtw/z+hqg51975TcV/MMW/bjznbnszb9mTeVtNmvN3CJ5OcX1XnVdXpSV6c5NpNuB8AgJUyfsaqu++rqpcn+XDW3m7h7d198/T9AACsmk15jVV3X5fkus342Q/RKbnkyDjztv2Ys+3JvG1P5m0FVXdv9RgAAHYEH2kDADBkx4dVVT2nqj5fVYer6uBWj4cfqarbq+qzVXVTVR1atj2+qq6vqi8s389ctldVvWWZx89U1YVbO/qHj6p6e1XdXVWfW7fthOepqg4sx3+hqg5sxe/ycPIg8/b6qjqyPOZuqqrnrdv3mmXePl9Vz1633XPoKVJV51bVx6rqlqq6uapesWz3eNtOunvHfmXtxfP/meTnk5ye5N+SXLDV4/L1w/m5PclZD9j250kOLssHk7xhWX5ekn9MUkkuSnLDVo//4fKV5OlJLkzyuYc6T0ken+S25fuZy/KZW/277eSvB5m31yf542Mce8Hy/PjIJOctz5uneQ495XP2xCQXLsuPTfIfy9x4vG2jr51+xuqHH6/T3d9Lcv/H67C69ie5elm+Oskl67a/o9d8IskZVfXErRjgw013/3OSbzxg84nO07OTXN/d3+jue5Jcn+Q5mz/6h68HmbcHsz/Ju7v7f7r7i0kOZ+3503PoKdTdd3X3p5blbye5NWufZuLxto3s9LA61sfr7NmisfCTOsk/VdWNyzvxJ8nZ3X3XsvyVJGcvy+ZytZzoPJm/1fHy5bLR2++/pBTztnKqam+SpyS5IR5v28pODytW269394VJnpvk8qp6+vqdvXZO25+trjjztK28NckvJHlykruS/OXWDodjqarHJHl/kld297fW7/N4W307Paw29PE6bI3uPrJ8vzvJ32ftssNX77/Et3y/ezncXK6WE50n87cCuvur3f397v5Bkr/J2mMuMW8ro6oekbWoemd3f2DZ7PG2jez0sPLxOiuqqh5dVY+9fznJs5J8Lmvzc/9fsBxIcs2yfG2Sly5/BXNRknvXnRrn1DvRefpwkmdV1ZnL5adnLds4hR7wusTfztpjLlmbtxdX1SOr6rwk5yf513gOPaWqqpJcleTW7n7jul0eb9vIprzz+qpoH6+zys5O8vdrzyPZleTvuvtDVfXJJO+tqkuTfCnJi5bjr8vaX8AcTvKdJC879UN+eKqqdyV5RpKzqurOJK9LckVOYJ66+xtV9WdZ+486Sf60uzf6wmoeggeZt2dU1ZOzdinp9iS/lyTdfXNVvTfJLUnuS3J5d39/+TmeQ0+dpyV5SZLPVtVNy7bXxuNtW/HO6wAAQ3b6pUAAgFNGWAEADBFWAABDhBUAwBBhBQAwRFgBAAwRVgAAQ4QVAMCQ/wPygunfOAK/eQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "texts = dataset[\"comment_text\"].tolist()\n", + "tokenized_texts = tokenizer(texts)\n", + "ids_lens = list(len(toks) for toks in tokenized_texts[\"input_ids\"])\n", + "\n", + "plt.figure(figsize=(10, 6))\n", + "plt.hist(ids_lens)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "We already know, how to tackle the problem of different sizes of sequences with padding. However, blind padding here would make pad all the sequenes to the size of the largest one, which seems to be an overkill. In such case it might be sensible to actually truncate the too-long sequences into a fixed length, say 512. And we can do this easily by specifying the `max_length` and `truncation=True` arguments to the tokenizer." + ], + "metadata": { + "id": "66wufVncHF8f" + } + }, + { + "cell_type": "code", + "source": [ + "# YOUR CODE HERE\n", + "tokenized_texts = tokenizer(\n", + " texts, max_length=512, return_tensors=\"pt\", padding=True, truncation=True\n", + ")\n", + "\n", + "features = []\n", + "with torch.no_grad():\n", + " for i in range(0, len(texts), batch_size):\n", + " texts_batch = tokenized_texts[\"input_ids\"][i : i + batch_size].to(device)\n", + " masks_batch = tokenized_texts[\"attention_mask\"][i : i + batch_size].to(device)\n", + " output = model(texts_batch, masks_batch)\n", + " batch_features = output.last_hidden_state[:, 0, :].cpu().numpy()\n", + " features.append(batch_features)\n", + "\n", + "features = np.concatenate(features, axis=0)\n", + "features.shape" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "_LOYXhz_FKTr", + "outputId": "3810844f-2c91-4131-d510-bd75f1593975" + }, + "execution_count": 24, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(1000, 768)" + ] + }, + "metadata": {}, + "execution_count": 24 + } + ] + }, + { + "cell_type": "code", + "source": [ + "labels = dataset[\"should_ban\"].values\n", + "train_features, test_features, train_labels, test_labels = train_test_split(features, labels)\n", + "lr_clf = LogisticRegression(C=0.1)\n", + "lr_clf.fit(train_features, train_labels)\n", + "lr_clf.score(test_features, test_labels)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Q6dOtVJsMXGa", + "outputId": "22d7b8e5-3205-4c95-8f57-e9dcaddc2add" + }, + "execution_count": 25, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0.832" + ] + }, + "metadata": {}, + "execution_count": 25 + } + ] + }, + { + "cell_type": "code", + "source": [ + "plt.figure(figsize=(10, 6))\n", + "\n", + "proba = lr_clf.predict_proba(train_features)[:, 1]\n", + "auc = roc_auc_score(train_labels, proba)\n", + "plt.plot(*roc_curve(train_labels, proba)[:2], label=f'train AUC={auc:.4f}')\n", + "\n", + "proba = lr_clf.predict_proba(test_features)[:, 1]\n", + "auc = roc_auc_score(test_labels, proba)\n", + "plt.plot(*roc_curve(test_labels, proba)[:2], label=f'test AUC={auc:.4f}')\n", + "\n", + "plt.legend()\n", + "plt.show()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 374 + }, + "id": "spZ4yhrdMm4V", + "outputId": "02c2449b-1786-4de1-cb6b-aeff6de3c775" + }, + "execution_count": 26, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAFlCAYAAADPim3FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5RU5Znv8e8TLjIkoBE8RkAEB2S4CooI8QQ1JgrOOjCOyqDLiRMvOGM8y5NEMjjJ4BxnnaiDiYaJMcG7MXZDUANnhKiJmHgmEURDBEGFaAvdIip4jReCvOePvkx303QX7Oqu6qrvZ61eq2rvt2o/zV7oj2e/+92RUkKSJEn75xOFLkCSJKkzM0xJkiRlYJiSJEnKwDAlSZKUgWFKkiQpA8OUJElSBl0LdeC+ffumQYMGFerwkiRJOXvqqafeSCkd0tK+goWpQYMGsXr16kIdXpIkKWcR8fLe9nmZT5IkKQPDlCRJUgaGKUmSpAwMU5IkSRkYpiRJkjIwTEmSJGVgmJIkScrAMCVJkpSBYUqSJCmDNsNURNweEa9FxLq97I+ImB8RmyLimYg4Jv9lSpIkFadcOlN3AlNa2T8VGFr3Mwu4OXtZkiRJnUObz+ZLKf06Iga1MmQ6cHdKKQFPRMRBEXFYSmlrnmqUpP1278rNLFlTU+gyVCROeX8ZJ3ywotBlKM/ePWg4Ey+9pWDHz8eDjvsDWxq9r67btkeYiohZ1HavGDhwYB4OLUktqw9RK1/aAcDxgw8ucEUqBid8sIJBf3qRqm5HFroUlZB8hKmcpZQWAAsAxo8fnzry2JLaR7F2fhqHqOlj+3Pu8f4DTsAdBwLjGPnlBwtdiUpIPsJUDXB4o/cD6rZJyrNiDC7F2vkxRLWD1XfA2sWFriKbV9fCZ0YXugqVmHyEqaXAZRFRCRwPvO18KXUGxRhM2lKMwcXQUkbWLu78YeQzo2H0WYWuQiWmzTAVERXASUDfiKgGrgK6AaSUfggsA04HNgHvA19ur2Kl/dVScCrGYNIWg0uOSqGDUozqg5SXyKQmcrmb75w29ifgK3mrSNoHuXaXWgpOBpMSVgodlGJkV0dqUYdOQJfy6d6Vm/mnB9YCbXeXDE6d2P50meygSOpAhinlXUfNRarvNn37jNGGpFK2P10mOyiSOpBhSnm1L92irOw2dZBCzz+yyySpyBmmlEnzLpTdohJU6PlHdpkkFTnDlNrU2mW75hO77RaVgOadKDtDktQqw5TatGRNDeu3vsOIw3rvsc/wVIKad6LsDElSqwxTatW9Kzez8qUdHD/4YBZeMqnQ5SirXOY/2YmSpH3yiUIXoOJWf3lv+tj+Ba5EeVHfdWqNnShJ2id2ptSm4wcf7GW8UmLXSZLyyjClBi1NNN/bXCkVoX25hCdJyhsv86nBkjU1DXfn1RtxWG8v8XUWXsKTpIKwM1WG9rbUwfqt7zjRvLPzEp4kdTjDVBmpD1EtPfQX7EJJkrQ/DFMlJtcFNl0bSpKk/DBMlRgX2JQkqWMZpkqIC2yWkZbu3PNOPUkqCO/mKyEusFlGWrpzzzv1JKkg7Ex1Qm3djedlvA6Sy7pO7cVHvkhS0bAz1cncu3Iz//TA2j3WgwLvxutwuazr1F7sQklS0bAz1Qk07kTVh6hvnzHaDlSu2quDZHdIkoRhqqi1tC6Ud+Tth/oOUr4nZ9sdkiRhmCo6LXWhDFB5YAdJktRODFNFpvE6UYYoSZKKn2GqwJrfmVcfpFwnSpKkzsEwVWDNVyz3jryMXMxSktTBDFMF0HxelCuW51FLk82dKC5JakeGqQ5Wv04UNL07r+y4XIEkqUQYpjpQ4yBV9utEuVyBJKlEGKY6QPP1ojp1kMpXR8kOkiSpRBimOkD9JPOSWOogXx0lO0iSpBJhmGpn967cXHqTzO0oSZLUwAcdt6PGc6TKcpK5JEllwM5UO6pf/qDTzpFyzSZJktpkZ6qdHT/44M4ZpOC/5kc15lwnSZKasDOl1jk/SpKkVtmZkiRJysDOVJ41flRM42fuSZKk0mSYyoPmz9qD2rlSPrRYkqTSZ5jKaG/P2uu0k84lSdI+MUxlUHLP2mu+FILLIEiS1CYnoGfQ6deRaq75UggugyBJUpvsTGXUqdeRaolLIUiStE/sTEmSJGVgmJIkScrAy3z7oX4pBNeRkiRJhql9UB+iGq8l1WnWkWrpocXNefeeJEn7zDC1D+q7UZ1yLan6O/VaC0vevSdJ0j4zTO2jEYf1ZuElkwpdxv7xTj1JkvLOCeiSJEkZ2JnKgRPOJUnS3uTUmYqIKRHxfERsiog5LewfGBErIuJ3EfFMRJye/1ILp3GQ6jQTziVJUodoszMVEV2Am4AvAtXAkxGxNKW0vtGwbwGLUko3R8QIYBkwqB3qLZhOPVdKkiS1m1w6UxOATSmlF1NKO4FKYHqzMQmov/51IPBK/kosrHtXbm5YCkGSJKm5XOZM9Qe2NHpfDRzfbMy/AA9HxP8EPgl8IS/VFYH6hxl7eU+SJLUkXxPQzwHuTCl9JyImAT+OiFEppd2NB0XELGAWwMCBxbtGU/2Ec6BhXalOtaaUJEnqMLlc5qsBDm/0fkDdtsYuBBYBpJR+C/QA+jb/opTSgpTS+JTS+EMOOWT/Ku4A9RPOASedS5KkVuXSmXoSGBoRg6kNUTOBc5uN2QycAtwZEcOpDVOv57PQjuaEc0mSlIs2O1MppV3AZcBDwAZq79p7NiKujohpdcO+DlwcEb8HKoC/Syml9ipakiSpWOQ0ZyqltIza5Q4ab5vb6PV64IT8ltaxms+TcnFOSZKUCx8nU8d5UpIkaX/4OJlGnCclSZL2lWGqFK2+A9Yubrrt1bXwmdGFqUeSpBLmZb5StHZxbXhq7DOjYfRZhalHkqQSZmeqs2mp69RcfRfqyw92TE2SJJUxO1OdTUtdp+bsQkmS1GHsTHVGdp0kSSoadqYkSZIyMExJkiRlYJiSJEnKwDlT1D5KZuVLOzh+8MGFLmVPze/ec70oSZKKip0paHgmX1E+Qqb53XveqSdJUlGxM1Xn+MEHc+7xAwtbRGsrl3v3niRJRcnOVDFx5XJJkjodO1PFxi6UJEmdip0pSZKkDAxTkiRJGXiZr5Bc9kCSpE7PzlQhueyBJEmdXtl3pgq+YKcTziVJ6tTKvjNV1At2SpKkolf2YQqKZMFOSZLUKZXtZb57V25myZoa1m99hxGH9S50OZIkqZMq2zDVOEh1yCW+1h4VI0mSOq2yDVMAIw7rzcJLJnXMwerv3Gscnrx7T5KkTq+sw1TetNR1as4HFkuSVJKcgJ4PLT2guDm7UJIklaSy60y128Rzu06SJJWlsgtTeZl47mNgJElSnbILU5CHiefNJ5N7CU+SpLJVlmFqn7S2pIGX9SRJKntOQG9LS5PL7URJkqQ65d2ZckkDSZKUUXl3plzSQJIkZVTenSmw6yRJkjIp786UJElSRuXVmVp9B3O331b7+o4DXR9KkiRlVl6dqbWLGfSnF//rvfOhJElSRuXVmQKquh3J1X3msfDLGRbtlCRJqlM2nal7V27m2a1v8/7OjwtdiiRJKiFlE6aWrKnh/Z0f07N7l/1/Jp8kSVIzZXWZr2f3Low87EBGHj+w0KVIkqQSUTadKUmSpPZgmJIkScrAMCVJkpSBYUqSJCkDw5QkSVIGhilJkqQMDFOSJEkZGKYkSZIyMExJkiRlkFOYiogpEfF8RGyKiDl7GTMjItZHxLMRcW9+y5QkSSpObT5OJiK6ADcBXwSqgScjYmlKaX2jMUOBK4ETUkpvRsR/a6+CJUmSikkunakJwKaU0osppZ1AJTC92ZiLgZtSSm8CpJRey2+ZkiRJxSmXMNUf2NLofXXdtsaOAo6KiP+MiCciYkpLXxQRsyJidUSsfv311/evYkmSpCKSrwnoXYGhwEnAOcAtEXFQ80EppQUppfEppfGHHHJIng7dtntXbmblSzs67HiSJKl85BKmaoDDG70fULetsWpgaUrpTymll4AXqA1XRWHJmtpy+37qgAJXIkmSSk0uYepJYGhEDI6I7sBMYGmzMT+jtitFRPSl9rLfi3msM7PjBx/Mob16FLoMSZJUYtoMUymlXcBlwEPABmBRSunZiLg6IqbVDXsI2B4R64EVwOyU0vb2KlqSJKlYtLk0AkBKaRmwrNm2uY1eJ+BrdT+SJEllwxXQJUmSMjBMSZIkZWCYkiRJysAwJUmSlIFhSpIkKQPDlCRJUgY5LY3QKa2+A9YuBmDu9rdrt8Vm+MzoAhYlSZJKTel2ptYuhlfXNt32mdEw+qzC1CNJkkpS6XamoDY8fflBrv7RbwFY+OVJBS5IkiSVmtLtTEmSJHUAw5QkSVIGhilJkqQMDFOSJEkZGKYkSZIyMExJkiRlYJiSJEnKwDAlSZKUgWFKkiQpg5IPU/eu3MzKl3YUugxJklSiSj5MLVlTA8D0sf0LXIkkSSpFJR+mAI4ffDDnHj+w0GVIkqQSVBZhSpIkqb0YpiRJkjIo2TC17d0PeXbr26zf+k6hS5EkSSWsZMPUG+99xPs7P2bEYb2dfC5JktpN10IX0J56du/CwksmFboMSZJUwkq2MyVJktQRDFOSJEkZGKYkSZIyMExJkiRlYJiSJEnKwDAlSZKUgWFKkiQpA8OUJElSBoYpSZKkDAxTkiRJGRimJEmSMjBMSZIkZWCYkiRJysAwJUmSlIFhSpIkKQPDlCRJUgaGKUmSpAwMU5IkSRkYpiRJkjIwTEmSJGVgmJIkScrAMCVJkpSBYUqSJCkDw5QkSVIGOYWpiJgSEc9HxKaImNPKuDMjIkXE+PyVKEmSVLzaDFMR0QW4CZgKjADOiYgRLYzrBVwOrMx3kZIkScUql87UBGBTSunFlNJOoBKY3sK4fwWuAz7MY32SJElFLZcw1R/Y0uh9dd22BhFxDHB4SunB1r4oImZFxOqIWP3666/vc7GSJEnFJvME9Ij4BPBd4OttjU0pLUgpjU8pjT/kkEOyHlqSJKngcglTNcDhjd4PqNtWrxcwCngsIqqAicBSJ6FLkqRykEuYehIYGhGDI6I7MBNYWr8zpfR2SqlvSmlQSmkQ8AQwLaW0ul0qliRJKiJthqmU0i7gMuAhYAOwKKX0bERcHRHT2rtASZKkYtY1l0EppWXAsmbb5u5l7EnZy5IkSeocXAFdkiQpA8OUJElSBoYpSZKkDAxTkiRJGRimJEmSMjBMSZIkZWCYkiRJysAwJUmSlIFhSpIkKQPDlCRJUgaGKUmSpAwMU5IkSRkYpiRJkjIwTEmSJGVgmJIkScrAMCVJkpSBYUqSJCkDw5QkSVIGhilJkqQMDFOSJEkZGKYkSZIyMExJkiRlYJiSJEnKwDAlSZKUgWFKkiQpA8OUJElSBoYpSZKkDAxTkiRJGRimJEmSMjBMSZIkZWCYkiRJysAwJUmSlIFhSpIkKQPDlCRJUgaGKUmSpAwMU5IkSRkYpiRJkjIwTEmSJGVgmJIkScrAMCVJkpSBYUqSJCkDw5QkSVIGhilJkqQMDFOSJEkZGKYkSZIyMExJkiRlYJiSJEnKwDAlSZKUgWFKkiQpA8OUJElSBjmFqYiYEhHPR8SmiJjTwv6vRcT6iHgmIn4ZEUfkv1RJkqTi02aYioguwE3AVGAEcE5EjGg27HfA+JTSGGAx8G/5LlSSJKkY5dKZmgBsSim9mFLaCVQC0xsPSCmtSCm9X/f2CWBAfsuUJEkqTrmEqf7Alkbvq+u27c2FwPIsRUmSJHUWXfP5ZRFxHjAeOHEv+2cBswAGDhyYz0NLkiQVRC6dqRrg8EbvB9RtayIivgB8E5iWUvqopS9KKS1IKY1PKY0/5JBD9qdeSZKkopJLmHoSGBoRgyOiOzATWNp4QESMA35EbZB6Lf9lSpIkFac2w1RKaRdwGfAQsAFYlFJ6NiKujohpdcPmAZ8CfhoRayJi6V6+TpIkqaTkNGcqpbQMWNZs29xGr7+Q57okSZI6BVdAlyRJysAwJUmSlIFhSpIkKQPDlCRJUgaGKUmSpAwMU5IkSRkYpiRJkjIwTEmSJGVgmJIkScrAMCVJkpSBYUqSJCkDw5QkSVIGhilJkqQMDFOSJEkZGKYkSZIyMExJkiRlYJiSJEnKwDAlSZKUgWFKkiQpA8OUJElSBoYpSZKkDAxTkiRJGRimJEmSMjBMSZIkZWCYkiRJysAwJUmSlIFhSpIkKQPDlCRJUgaGKUmSpAwMU5IkSRkYpiRJkjLoWugCJEkqRn/605+orq7mww8/LHQp6kA9evRgwIABdOvWLefPGKYkSWpBdXU1vXr1YtCgQUREoctRB0gpsX37dqqrqxk8eHDOn/MynyRJLfjwww/p06ePQaqMRAR9+vTZ526kYUqSpL0wSJWf/TnnhilJkorQW2+9xQ9+8IP9+uzpp5/OW2+9tc+fGzt2LDNnzmyy7aSTTmL16tUN76uqqhg1alTD+1WrVjF58mSGDRvGuHHjuOiii3j//fdzOt7Pf/5zhg0bxpAhQ7j22mtbHPPyyy9zyimnMGbMGE466SSqq6sBWLFiBWPHjm346dGjBz/72c8A+P73v8+QIUOICN54442G73rzzTc544wzGDNmDBMmTGDdunW5/cG0wTAlSVIRai1M7dq1q9XPLlu2jIMOOmifjrdhwwY+/vhjHn/8cf74xz/m9Jlt27Zx9tlnc9111/H888/zu9/9jilTpvDuu++2+dmPP/6Yr3zlKyxfvpz169dTUVHB+vXr9xh3xRVX8KUvfYlnnnmGuXPncuWVVwJw8skns2bNGtasWcOjjz5Kz549OfXUUwE44YQT+MUvfsERRxzR5Lu+/e1vM3bsWJ555hnuvvtuLr/88px+z7YYpiRJKkJz5szhD3/4A2PHjmX27Nk89thjfO5zn2PatGmMGDECgL/6q7/i2GOPZeTIkSxYsKDhs4MGDeKNN96gqqqK4cOHc/HFFzNy5EhOPfVUPvjggxaPV1FRwd/+7d9y6qmnsmTJkpxqvOmmmzj//POZNGlSw7azzjqLQw89tM3Prlq1iiFDhnDkkUfSvXt3Zs6c2eJx169fz+c//3mgNkC1NGbx4sVMnTqVnj17AjBu3DgGDRrU6nf9xV/8BVVVVWzbti2n37U13s0nSVIb/vf/fZb1r7yT1+8c0a83V/2PkXvdf+2117Ju3TrWrFkDwGOPPcbTTz/NunXrGu40u/322zn44IP54IMPOO644zjzzDPp06dPk+/ZuHEjFRUV3HLLLcyYMYP77ruP8847b4/jLVy4kEceeYTnnnuOf//3f+fcc89t83dYt24d559/fov7VqxYwVe/+tU9tvfs2ZPf/OY31NTUcPjhhzdsHzBgACtXrtxj/NFHH83999/P5ZdfzgMPPMC7777L9u3bm/yelZWVfO1rX2uz3vrv+tznPseqVat4+eWXqa6uzin8tcYwJUlSJzFhwoQmt+zPnz+fBx54AIAtW7awcePGPcLU4MGDGTt2LADHHnssVVVVe3zv6tWr6du3LwMHDqR///5ccMEF7Nixg4MPPrjFCdm5TNKuvwyX1fXXX89ll13GnXfeyeTJk+nfvz9dunRp2L9161bWrl3Laaed1uZ3zZkzh8svv5yxY8cyevRoxo0b1+S79pdhSpKkNrTWQepIn/zkJxteP/bYY/ziF7/gt7/9LT179uSkk05q8Zb+Aw44oOF1ly5dWrzMV1FRwXPPPddwaeydd97hvvvu4+KLL6ZPnz68+eabDWN37NhB3759ARg5ciRPPfUU06dP3+M72+pM9e/fny1btjRsr66upn///nuM79evH/fffz8A7733Hvfdd1+T+WCLFi3ijDPOyGmRzd69e3PHHXcAtWtKDR48mCOPPLLNz7XFOVOSJBWhXr16tTqR++233+bTn/40PXv25LnnnuOJJ57Yr+Ps3r2bRYsWsXbtWqqqqqiqqmLJkiVUVFQAtXfz3XPPPaSUALjrrrs4+eSTAbjsssu46667mlyeu//++9m2bVuTCeKNf37zm98AcNxxx7Fx40Zeeukldu7cSWVlJdOmTdujvjfeeIPdu3cDcM0113DBBRc02V9RUcE555yT0+/61ltvsXPnTgBuvfVWJk+eTO/evfflj6tFhilJkopQnz59OOGEExg1ahSzZ8/eY/+UKVPYtWsXw4cPZ86cOUycOHG/jvP444/Tv39/+vXr17Bt8uTJrF+/nq1btzJr1ix69erF0UcfzdFHH817773HFVdcAcChhx5KZWUlV1xxBcOGDWP48OE89NBD9OrVq83jdu3ale9///ucdtppDB8+nBkzZjByZG0HcO7cuSxduhSo7cANGzaMo446im3btvHNb36z4TuqqqrYsmULJ554YpPvnj9/PgMGDKC6upoxY8Zw0UUXAbV3LI4aNYphw4axfPlyvve97+3Xn1lzUZ80O9r48eNT43Ur8u3Zb/93AEb+0/9rt2NIkkrXhg0bGD58eKHLUAG0dO4j4qmU0viWxtuZkiRJysAwJUmSlIFhSpIkKQPDlCRJUgaGKUmSpAwMU5IkSRkYpiRJKkJvvfUWP/jBD/b78zfeeCPvv//+Xve/8cYbdOvWjR/+8IdNtn/qU59q8v7OO+/ksssua3h/9913M2rUqIbHsVx//fU513TNNdcwZMgQhg0bxkMPPdTimEcffZRjjjmGUaNGcf7557Nr1y4AfvKTnzBmzBhGjx7NZz/7WX7/+98D8PzzzzN27NiGn969e3PjjTcCsGbNGiZOnMjYsWMZP348q1atyrnWfZFTmIqIKRHxfERsiog5Lew/ICIW1u1fGRGD8l2oJEnlpL3D1E9/+lMmTpzYsNJ5LpYvX86NN97Iww8/zNq1a3niiSc48MADc/rs+vXrqays5Nlnn+XnP/85l156KR9//HGTMbt37+b888+nsrKSdevWccQRR3DXXXcBtc8Y/NWvfsXatWv553/+Z2bNmgXAsGHDGlZXf+qpp+jZsydnnHEGAN/4xje46qqrWLNmDVdffTXf+MY3cv5d90WbYSoiugA3AVOBEcA5ETGi2bALgTdTSkOAG4Dr8l2oJEnlZM6cOfzhD39g7NixDSugz5s3j+OOO44xY8Zw1VVXAfDHP/6Rv/zLv+Too49m1KhRLFy4kPnz5/PKK69w8sknNzz6pbmKigq+853vUFNTQ3V1dU41XXPNNVx//fUNq6UfcMABXHzxxTl9dsmSJcycOZMDDjiAwYMHM2TIkD06Rdu3b6d79+4cddRRAHzxi1/kvvvuA+Czn/0sn/70pwGYOHFiizX/8pe/5M///M854ogjgNoHMr/zzjtA7eN3Gq/ynk+5POh4ArAppfRiXWGVwHRgfaMx04F/qXu9GPh+REQq1PLqkiTl0/I58Ora/H7nZ0bD1Gv3uvvaa69l3bp1rFmzBoCHH36YjRs3smrVKlJKTJs2jV//+te8/vrr9OvXjwcffBCoDQ0HHngg3/3ud1mxYkXDQ4kb27JlC1u3bmXChAnMmDGDhQsX8vWvf73NktetW8exxx7b4r558+bxk5/8ZI/tkydPZv78+dTU1DR55M2AAQOoqalpMrZv377s2rWL1atXM378eBYvXtzkYcj1brvtNqZOnbrH9srKyibP6bvxxhs57bTTuOKKK9i9e3fDcwHzLZfLfP2Bxr9Jdd22FseklHYBbwN9mn9RRMyKiNURsfr111/fv4pz9O5Bw3n3IB8DIEkqDQ8//DAPP/ww48aN45hjjuG5555j48aNjB49mkceeYR//Md/5PHHH8/pstvChQuZMWMGADNnzmzzUl9EtPmds2fPbvHBxvPnz8/tF6w7TmVlJV/96leZMGECvXr1okuXLk3GrFixgttuu43rrmt6EWznzp0sXbqUs88+u2HbzTffzA033MCWLVu44YYbuPDCC3OuZV/k0pnKm5TSAmAB1D6brz2PNfHSW9rz6yVJ5aSVDlJHSSlx5ZVXcskll+yx7+mnn2bZsmV861vf4pRTTmHu3LmtfldFRQWvvvpqQyfplVdeYePGjQwdOpQ/+7M/Y+fOnXTv3h2AHTt2NHS3Ro4cyVNPPcXnP//5Pb6zrc5U//79m3SZqqur6d+/eW8GJk2axOOPPw7UBsgXXnihYd8zzzzDRRddxPLly+nTp2nPZvny5RxzzDEceuihDdvuuuuuhocZn3322Q0PPM63XDpTNcDhjd4PqNvW4piI6AocCGzPR4GSJJWjXr168e677za8P+2007j99tt57733AKipqeG1117jlVdeoWfPnpx33nnMnj2bp59+usXP13vhhRd47733qKmpoaqqiqqqKq688sqG7tSJJ57IPffcA8AHH3zAokWLGuZdXXnllcyePZtXX30VqO0G3XrrrUDbnalp06ZRWVnJRx99xEsvvcTGjRuZMGHCHvW99tprAHz00Udcd911/P3f/z0Amzdv5q//+q/58Y9/3DCnqrGKiooml/gA+vXrx69+9Sug9i7BoUOHtv0Hvx9y6Uw9CQyNiMHUhqaZwLnNxiwFzgd+C5wFPOp8KUmS9l+fPn044YQTGDVqFFOnTmXevHls2LCBSZMmAbVLGNxzzz1s2rSJ2bNn84lPfIJu3bpx8803AzBr1iymTJlCv379WLFiRcP3VlRUNNztVu/MM8/kb/7mb5g7dy7f+973uOSSS5g/fz4pJb70pS8xefJkAE4//XS2bdvGF77wBVJKRAQXXHBBTr/PyJEjmTFjBiNGjKBr167cdNNNDZfwTj/9dG699Vb69evHvHnz+I//+A92797NP/zDPzR0wa6++mq2b9/OpZdeCkDXrl1ZvXo1UDsJ/5FHHuFHP/pRk2PecsstXH755ezatYsePXqwYMGCfToHuYpcMk9EnA7cCHQBbk8p/Z+IuBpYnVJaGhE9gB8D44AdwMz6Cet7M378+FT/hyBJUrHZsGEDw4c797YctXTuI+KplNL4lsbnNGcqpbQMWNZs29xGrz8Ezm7+OUmSpFLnCuiSJEkZGKYkSZIyMExJkrQX3ktVfvbnnBumJElqQY8ePdi+fbuBqoyklNi+fTs9evTYp8916KKdkh7/H5AAAAR4SURBVCR1FgMGDKC6upr2fmKHikuPHj0YMGDAPn3GMCVJUgu6devG4MGDC12GOgEv80mSJGVgmJIkScrAMCVJkpRBTo+TaZcDR7wOvNzOh+kLvNHOx9C+87wUH89JcfK8FB/PSXHqiPNyRErpkJZ2FCxMdYSIWL235+iocDwvxcdzUpw8L8XHc1KcCn1evMwnSZKUgWFKkiQpg1IPUwsKXYBa5HkpPp6T4uR5KT6ek+JU0PNS0nOmJEmS2lupd6YkSZLaVUmEqYiYEhHPR8SmiJjTwv4DImJh3f6VETGo46ssPzmcl69FxPqIeCYifhkRRxSiznLS1jlpNO7MiEgR4V1L7SyXcxIRM+r+rjwbEfd2dI3lKIf/fg2MiBUR8bu6/4adXog6y0lE3B4Rr0XEur3sj4iYX3fOnomIYzqqtk4fpiKiC3ATMBUYAZwTESOaDbsQeDOlNAS4AbiuY6ssPzmel98B41NKY4DFwL91bJXlJcdzQkT0Ai4HVnZsheUnl3MSEUOBK4ETUkojgf/V4YWWmRz/rnwLWJRSGgfMBH7QsVWWpTuBKa3snwoMrfuZBdzcATUBJRCmgAnAppTSiymlnUAlML3ZmOnAXXWvFwOnRER0YI3lqM3zklJakVJ6v+7tE8C+PaZb+yqXvysA/0rtPzg+7MjiylQu5+Ri4KaU0psAKaXXOrjGcpTLeUlA77rXBwKvdGB9ZSml9GtgRytDpgN3p1pPAAdFxGEdUVsphKn+wJZG76vrtrU4JqW0C3gb6NMh1ZWvXM5LYxcCy9u1IrV5Tura4oenlB7syMLKWC5/T44CjoqI/4yIJyKitX+ZKz9yOS//ApwXEdXAMuB/dkxpasW+/n8nb7p2xEGk1kTEecB44MRC11LOIuITwHeBvytwKWqqK7WXLU6itnv764gYnVJ6q6BV6RzgzpTSdyJiEvDjiBiVUtpd6MLU8UqhM1UDHN7o/YC6bS2OiYiu1LZkt3dIdeUrl/NCRHwB+CYwLaX0UQfVVq7aOie9gFHAYxFRBUwEljoJvV3l8vekGliaUvpTSukl4AVqw5XaTy7n5UJgEUBK6bdAD2qfD6fCyen/O+2hFMLUk8DQiBgcEd2pnQi4tNmYpcD5da/PAh5NLrDV3to8LxExDvgRtUHKeSDtr9VzklJ6O6XUN6U0KKU0iNp5bNNSSqsLU25ZyOW/Xz+jtitFRPSl9rLfix1ZZBnK5bxsBk4BiIjh1Iap1zu0SjW3FPhS3V19E4G3U0pbO+LAnf4yX0ppV0RcBjwEdAFuTyk9GxFXA6tTSkuB26htwW6idvLazMJVXB5yPC/zgE8BP627H2BzSmlawYoucTmeE3WgHM/JQ8CpEbEe+BiYnVKys96OcjwvXwduiYivUjsZ/e/8R3r7iogKav9h0bdurtpVQDeAlNIPqZ27djqwCXgf+HKH1ea5lyRJ2n+lcJlPkiSpYAxTkiRJGRimJEmSMjBMSZIkZWCYkiRJysAwJUmSlIFhSpIkKQPDlCRJUgb/H/6V0uYGDtgKAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cc1hBVfbzHJ7" + }, + "source": [ + "So, how does it look? Did we achieve better results? \n", + "\n", + "Here come some further ideas:\n", + "\n", + "* Try using the larger BERT (e.g. BERT-base or BERT-large) and compare the results (be careful, they require more memory).\n", + "\n", + "* Using BERT output for translation? Why not ;)" + ] + } + ], + "metadata": { + "colab": { + "machine_shape": "hm", + "name": "practice_bert_for_text_classification_solved.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "accelerator": "GPU" + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/week07_bert_finetuning/README.md b/week07_bert_finetuning/README.md deleted file mode 100644 index 0dd49e1..0000000 --- a/week07_bert_finetuning/README.md +++ /dev/null @@ -1,16 +0,0 @@ -How to fine-tune BERT: -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/week07_bert_finetuning/bert_finetuning.ipynb) - - -__Further readings__: -* [Blog post](http://mccormickml.com/2019/07/22/BERT-fine-tuning/) about the aforementioned notebook - -* The Illustrated BERT [blog post](http://jalammar.github.io/illustrated-bert/) - -* DistillBERT overview (distillation will be covered later in our course) [blog post](https://medium.com/huggingface/distilbert-8cf3380435b5) - -* Google AI Blog [post about open sourcing BERT](https://ai.googleblog.com/2018/11/open-sourcing-bert-state-of-art-pre.html) - -* One more [blog post explaining BERT](https://yashuseth.blog/2019/06/12/bert-explained-faqs-understand-bert-working/) - -* Great PyTorch library: [pytorch-transformers](https://github.com/huggingface/transformers) diff --git a/week07_bert_finetuning/bert_finetuning.ipynb b/week07_bert_finetuning/bert_finetuning.ipynb deleted file mode 100644 index e281de0..0000000 --- a/week07_bert_finetuning/bert_finetuning.ipynb +++ /dev/null @@ -1,4876 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "EKOTlwcmxmej" - }, - "source": [ - "# BERT Fine-Tuning Tutorial with PyTorch\n", - "\n", - "Source (by Chris McCormick and Nick Ryan):\n", - "[blog post](http://mccormickml.com/2019/07/22/BERT-fine-tuning/) and [notebook](https://colab.research.google.com/drive/1pTuQhug6Dhl9XalKB0zUGf4FIdYFlpcX)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qCgvR9INuP5q" - }, - "source": [ - "\n", - "## What is BERT?\n", - "\n", - "BERT (Bidirectional Encoder Representations from Transformers), released in late 2018, is the model we will use in this tutorial to provide readers with a better understanding of and practical guidance for using transfer learning models in NLP. BERT is a method of pretraining language representations that was used to create models that NLP practicioners can then download and use for free. You can either use these models to extract high quality language features from your text data, or you can fine-tune these models on a specific task (classification, entity recognition, question answering, etc.) with your own data to produce state of the art predictions.\n", - "\n", - "This notebook will explain how you can modify and fine-tune BERT to create a powerful NLP model that quickly gives you state of the art results. \n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DaVGdtOkuXUZ" - }, - "source": [ - "\n", - "## Advantages of Fine-Tuning\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5llwu8GBuqMb" - }, - "source": [ - "\n", - "In this tutorial, we will use BERT to train a text classifier. Specifically, we will take the pre-trained BERT model, add an untrained layer of neurons on the end, and train the new model for our classification task. Why do this rather than train a train a specific deep learning model (a CNN, BiLSTM, etc.) that is well suited for the specific NLP task you need? \n", - "\n", - "1. **Quicker Development**\n", - "\n", - " * First, the pre-trained BERT model weights already encode a lot of information about our language. As a result, it takes much less time to train our fine-tuned model - it is as if we have already trained the bottom layers of our network extensively and only need to gently tune them while using their output as features for our classification task. In fact, the authors recommend only 2-4 epochs of training for fine-tuning BERT on a specific NLP task (compared to the hundreds of GPU hours needed to train the original BERT model or a LSTM from scratch!). \n", - "\n", - "2. **Less Data**\n", - "\n", - " * In addition and perhaps just as important, because of the pre-trained weights this method allows us to fine-tune our task on a much smaller dataset than would be required in a model that is built from scratch. A major drawback of NLP models built from scratch is that we often need a prohibitively large dataset in order to train our network to reasonable accuracy, meaning a lot of time and energy had to be put into dataset creation. By fine-tuning BERT, we are now able to get away with training a model to good performance on a much smaller amount of training data.\n", - "\n", - "3. **Better Results**\n", - "\n", - " * Finally, this simple fine-tuning procedure (typically adding one fully-connected layer on top of BERT and training for a few epochs) was shown to achieve state of the art results with minimal task-specific adjustments for a wide variety of tasks: classification, language inference, semantic similarity, question answering, etc. Rather than implementing custom and sometimes-obscure architetures shown to work well on a specific task, simply fine-tuning BERT is shown to be a better (or at least equal) alternative.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZEynC5F4u7Nb" - }, - "source": [ - "\n", - "### A Shift in NLP\n", - "\n", - "This shift to transfer learning parallels the same shift that took place in computer vision a few years ago. Creating a good deep learning network for computer vision tasks can take millions of parameters and be very expensive to train. Researchers discovered that deep networks learn hierarchical feature representations (simple features like edges at the lowest layers with gradually more complex features at higher layers). Rather than training a new network from scratch each time, the lower layers of a trained network with generalized image features could be copied and transfered for use in another network with a different task. It soon became common practice to download a pre-trained deep network and quickly retrain it for the new task or add additional layers on top - vastly preferable to the expensive process of training a network from scratch. For many, the introduction of deep pre-trained language models in 2018 (ELMO, BERT, ULMFIT, Open-GPT, etc.) signals the same shift to transfer learning in NLP that computer vision saw.\n", - "\n", - "Let's get started!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RX_ZDhicpHkV" - }, - "source": [ - "# 1. Setup" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nSU7yERLP_66" - }, - "source": [ - "## 1.1. Using Colab GPU for Training\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "oYsV4H8fCpZ-", - "outputId": "a5a4a7ba-264e-4e5f-efe9-b5aade286979" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "There are 1 GPU(s) available.\n", - "We will use the GPU: Tesla K80\n" - ] - } - ], - "source": [ - "import torch\n", - "\n", - "# If there's a GPU available...\n", - "if torch.cuda.is_available(): \n", - "\n", - " # Tell PyTorch to use the GPU. \n", - " device = torch.device(\"cuda\")\n", - "\n", - " print('There are %d GPU(s) available.' % torch.cuda.device_count())\n", - "\n", - " print('We will use the GPU:', torch.cuda.get_device_name(0))\n", - "\n", - "# If not...\n", - "else:\n", - " print('No GPU available, using the CPU instead.')\n", - " device = torch.device(\"cpu\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2ElsnSNUridI" - }, - "source": [ - "## 1.2. Installing the Hugging Face Library\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0NmMdkZO8R6q", - "outputId": "8f8e7b63-7398-4b9f-e9d7-ca9898a019d4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting transformers\n", - " Downloading transformers-4.14.1-py3-none-any.whl (3.4 MB)\n", - "\u001b[K |████████████████████████████████| 3.4 MB 10.8 MB/s \n", - "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n", - "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n", - "Collecting huggingface-hub<1.0,>=0.1.0\n", - " Downloading huggingface_hub-0.2.1-py3-none-any.whl (61 kB)\n", - "\u001b[K |████████████████████████████████| 61 kB 502 kB/s \n", - "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n", - " Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n", - "\u001b[K |████████████████████████████████| 3.3 MB 23.5 MB/s \n", - "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.2)\n", - "Collecting sacremoses\n", - " Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n", - "\u001b[K |████████████████████████████████| 895 kB 38.6 MB/s \n", - "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n", - "Collecting pyyaml>=5.1\n", - " Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n", - "\u001b[K |████████████████████████████████| 596 kB 36.1 MB/s \n", - "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n", - "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n", - "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.6)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n", - "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n", - "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n", - "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n", - "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n", - "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n", - "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n", - "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n", - " Attempting uninstall: pyyaml\n", - " Found existing installation: PyYAML 3.13\n", - " Uninstalling PyYAML-3.13:\n", - " Successfully uninstalled PyYAML-3.13\n", - "Successfully installed huggingface-hub-0.2.1 pyyaml-6.0 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.14.1\n" - ] - } - ], - "source": [ - "!pip install transformers" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lxddqmruamSj" - }, - "source": [ - "The code in this notebook is actually a simplified version of the [run_glue.py](https://github.com/huggingface/transformers/blob/master/examples/run_glue.py) example script from huggingface.\n", - "\n", - "`run_glue.py` is a helpful utility which allows you to pick which GLUE benchmark task you want to run on, and which pre-trained model you want to use (you can see the list of possible models [here](https://github.com/huggingface/transformers/blob/e6cff60b4cbc1158fbd6e4a1c3afda8dc224f566/examples/run_glue.py#L69)). It also supports using either the CPU, a single GPU, or multiple GPUs. It even supports using 16-bit precision if you want further speed up." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "guw6ZNtaswKc" - }, - "source": [ - "# 2. Loading CoLA Dataset\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_9ZKxKc04Btk" - }, - "source": [ - "We'll use [The Corpus of Linguistic Acceptability (CoLA)](https://nyu-mll.github.io/CoLA/) dataset for single sentence classification. It's a set of sentences labeled as grammatically correct or incorrect. It was first published in May of 2018, and is one of the tests included in the \"GLUE Benchmark\" on which models like BERT are competing.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4JrUHXms16cn" - }, - "source": [ - "## 2.1. Download & Extract" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3ZNVW6xd0T0X" - }, - "source": [ - "We'll use the `wget` package to download the dataset to the Colab instance's file system. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5m6AnuFv0QXQ", - "outputId": "07b55b91-6eed-4fc1-dedd-63cc69449f27" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting wget\n", - " Downloading wget-3.2.zip (10 kB)\n", - "Building wheels for collected packages: wget\n", - " Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9672 sha256=e92c9692ca261f97aa7a0da639679a843eac2f0500957b6ea7b85fbff5897f5d\n", - " Stored in directory: /root/.cache/pip/wheels/a1/b6/7c/0e63e34eb06634181c63adacca38b79ff8f35c37e3c13e3c02\n", - "Successfully built wget\n", - "Installing collected packages: wget\n", - "Successfully installed wget-3.2\n" - ] - } - ], - "source": [ - "!pip install wget" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "08pO03Ff1BjI" - }, - "source": [ - "The dataset is hosted on GitHub in this repo: https://nyu-mll.github.io/CoLA/" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "pMtmPMkBzrvs", - "outputId": "3faece61-284e-4ff3-c69d-d6c0cb186e63" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading dataset...\n" - ] - } - ], - "source": [ - "import wget\n", - "import os\n", - "\n", - "print('Downloading dataset...')\n", - "\n", - "# The URL for the dataset zip file.\n", - "url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'\n", - "\n", - "# Download the file (if we haven't already)\n", - "if not os.path.exists('./cola_public_1.1.zip'):\n", - " wget.download(url, './cola_public_1.1.zip')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_mKctx-ll2FB" - }, - "source": [ - "Unzip the dataset to the file system. You can browse the file system of the Colab instance in the sidebar on the left." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0Yv-tNv20dnH", - "outputId": "483ba743-4dc9-45e0-f430-356520b3b7d8" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Archive: cola_public_1.1.zip\n", - " creating: cola_public/\n", - " inflating: cola_public/README \n", - " creating: cola_public/tokenized/\n", - " inflating: cola_public/tokenized/in_domain_dev.tsv \n", - " inflating: cola_public/tokenized/in_domain_train.tsv \n", - " inflating: cola_public/tokenized/out_of_domain_dev.tsv \n", - " creating: cola_public/raw/\n", - " inflating: cola_public/raw/in_domain_dev.tsv \n", - " inflating: cola_public/raw/in_domain_train.tsv \n", - " inflating: cola_public/raw/out_of_domain_dev.tsv \n" - ] - } - ], - "source": [ - "# Unzip the dataset (if we haven't already)\n", - "if not os.path.exists('./cola_public/'):\n", - " !unzip cola_public_1.1.zip" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oQUy9Tat2EF_" - }, - "source": [ - "## 2.2. Parse" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xeyVCXT31EZQ" - }, - "source": [ - "We can see from the file names that both `tokenized` and `raw` versions of the data are available. \n", - "\n", - "We can't use the pre-tokenized version because, in order to apply the pre-trained BERT, we *must* use the tokenizer provided by the model. This is because (1) the model has a specific, fixed vocabulary and (2) the BERT tokenizer has a particular way of handling out-of-vocabulary words." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MYWzeGSY2xh3" - }, - "source": [ - "We'll use pandas to parse the \"in-domain\" training set and look at a few of its properties and data points." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 398 - }, - "id": "_UkeC7SG2krJ", - "outputId": "c71135dd-b9f1-4f66-ebfe-1e4502524d6a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training sentences: 8,551\n", - "\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
sentence_sourcelabellabel_notessentence
1815r-671NaNI made it easy to get along with John.
6660m_021NaNThe mouse jumped out of the cheese box.
1228r-671NaNSam progressed.
4048ks081NaNTom tried to ask a question.
3815ks081NaNShe disappeared when the main party arrived.
5179kl931NaNAlmost every lawyer could answer that question.
6352d_981NaNAny woman who heard the news contributed to th...
6744m_021NaNBecause the bus drivers were on strike, the co...
4988ks081NaNThe fact that scientists have now established ...
3678ks081NaNChocolate cakes and pies are my favorite desse...
\n", - "
\n", - " \n", - " \n", - " \n", - "\n", - " \n", - "
\n", - "
\n", - " " - ], - "text/plain": [ - " sentence_source ... sentence\n", - "1815 r-67 ... I made it easy to get along with John.\n", - "6660 m_02 ... The mouse jumped out of the cheese box.\n", - "1228 r-67 ... Sam progressed.\n", - "4048 ks08 ... Tom tried to ask a question.\n", - "3815 ks08 ... She disappeared when the main party arrived.\n", - "5179 kl93 ... Almost every lawyer could answer that question.\n", - "6352 d_98 ... Any woman who heard the news contributed to th...\n", - "6744 m_02 ... Because the bus drivers were on strike, the co...\n", - "4988 ks08 ... The fact that scientists have now established ...\n", - "3678 ks08 ... Chocolate cakes and pies are my favorite desse...\n", - "\n", - "[10 rows x 4 columns]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Load the dataset into a pandas dataframe.\n", - "df = pd.read_csv(\"./cola_public/raw/in_domain_train.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n", - "\n", - "# Report the number of sentences.\n", - "print('Number of training sentences: {:,}\\n'.format(df.shape[0]))\n", - "\n", - "# Display 10 random rows from the data.\n", - "df.sample(10)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kfWzpPi92UAH" - }, - "source": [ - "The two properties we actually care about are the the `sentence` and its `label`, which is referred to as the \"acceptibility judgment\" (0=unacceptable, 1=acceptable)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "H_LpQfzCn9_o" - }, - "source": [ - "Here are five sentences which are labeled as not grammatically acceptible. Note how much more difficult this task is than something like sentiment analysis!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 195 - }, - "id": "blqIvQaQncdJ", - "outputId": "ea5e6a0b-23b0-42d6-9b05-0e136f6ee5b9" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
sentencelabel
4821I don't know that to agree with him or not.0
7669John convinced that Bill has slept.0
1731Hank plays the guitar and finds arrangements f...0
588the branch dropped bare of its apple.0
4341John is likely to appear that he will win the ...0
\n", - "
" - ], - "text/plain": [ - " sentence label\n", - "4821 I don't know that to agree with him or not. 0\n", - "7669 John convinced that Bill has slept. 0\n", - "1731 Hank plays the guitar and finds arrangements f... 0\n", - "588 the branch dropped bare of its apple. 0\n", - "4341 John is likely to appear that he will win the ... 0" - ] - }, - "execution_count": 8, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "df.loc[df.label == 0].sample(5)[['sentence', 'label']]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4SMZ5T5Imhlx" - }, - "source": [ - "\n", - "\n", - "Let's extract the sentences and labels of our training set as numpy ndarrays." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "GuE5BqICAne2" - }, - "outputs": [], - "source": [ - "# Get the lists of sentences and their labels.\n", - "sentences = df.sentence.values\n", - "labels = df.label.values" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ex5O1eV-Pfct" - }, - "source": [ - "# 3. Tokenization & Input Formatting" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-8kEDRvShcU5" - }, - "source": [ - "## 3.1. BERT Tokenizer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bWOPOyWghJp2" - }, - "source": [ - "\n", - "To feed our text to BERT, it must be split into tokens, and then these tokens must be mapped to their index in the tokenizer vocabulary.\n", - "\n", - "The tokenization must be performed by the tokenizer included with BERT--the below cell will download this for us. We'll be using the \"uncased\" version here.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 162, - "referenced_widgets": [ - "93853fa079c546b58b1c9ed8e586ae6c", - "8f66495735614898b617c27c0a89255d", - "e4306d918bbb47abae367df2c7ac2e9d", - "ac1cb16ed2ef4e6ab40e790e5e9ce47f", - "89c02af3d0f74c18a8e294f78b1f961e", - "a868dfc37b55447d8bab7350ca8cd0cf", - "ab13bda5dc204c3cbb633535494eb453", - "3eb262cbc05a4a48808623fbd11d7649", - "dbb229077bcf437aa25c5a16276e9c48", - "3dfad6bb2985460da31cc42d7e88168c", - "972be3faee024479ab0667892594d00c", - "747ba5c2117a42b28539371fe04b19ad", - "c91181d3293d457a8c6d32aca47e5c27", - "37ea42eb1ecd44eab1ea46db51fe2800", - "ef3a445c93764273b132d56d8b7545a3", - "ca26fc3ff341416f820e3ea94aa83188", - "24fd7d7781a448f6b4164bbeb64888b3", - "0d0fa745757f4c93a5e3ce9c2fadc9a3", - "da0235bd5a3642eba1b5749afe2c6f61", - "edeb0997fdb84daba10b4a9a102adee5", - "879a098d294d444c8176ce6221846d95", - "e389ecd683b64c28be15da07866f73c1", - "44e2066d588b496f97aa00cf51c9447c", - "176c3ac86a8f4dfa9ff221d67076edc5", - "93d601fafc26454b8de64f626daf3074", - "2a70c1b9bda34e6a89ca14b3b663b959", - "e9afd7edefb44fe4847ab5a9c0b24999", - "7d6e576e3c464b5693c51d35032107cb", - "fb5ff76f955942dda1bb6c98f827b893", - "f57a3f9b473543bda0c27ebcba001fad", - "b59dd15b11a64b8cace572993e8323ba", - "d0970cd03bc74b5da9f913fda325c1e0", - "9110261cd8b94147b72dec393886b308", - "521bcdb6786b4ba1a361cc8de1c97036", - "da7f2189149d4dcb8d012ffa2b9e34bb", - "04b9332a3fa9454da1a7169353b99221", - "a29e31e998974e758d09daf1a6186610", - "d2f54cdaba8e4a04bdb3b8d279afe5ef", - "ce8f59b53f894031b4ff4e0e1b63a61f", - "cf5d78f166d54aadbd8f68976f1e9ccc", - "c9f13c482ecc4448816bec89e13816ab", - "3a5ad7be77ce4127b434b749ad14b00a", - "f2dcfd7f0f9a48b08730714d44d18218", - "86fc378943aa48ee893bef7404abc96a" - ] - }, - "id": "Z474sSC6oe7A", - "outputId": "d7ac0ff8-54b1-4e2d-c3ea-a9a578fe46c6" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading BERT tokenizer...\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "93853fa079c546b58b1c9ed8e586ae6c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Downloading: 0%| | 0.00/226k [00:00 \"The first token of every sequence is always a special classification token (`[CLS]`). The final hidden state\n", - "corresponding to this token is used as the aggregate sequence representation for classification\n", - "tasks.\" (from the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf))\n", - "\n", - "You might think to try some pooling strategy over the final embeddings, but this isn't necessary. Because BERT is trained to only use this [CLS] token for classification, we know that the model has been motivated to encode everything it needs for the classification step into that single 768-value embedding vector. It's already done the pooling for us!\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "u51v0kFxeteu" - }, - "source": [ - "### Sentence Length & Attention Mask\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qPNuwqZVK3T6" - }, - "source": [ - "The sentences in our dataset obviously have varying lengths, so how does BERT handle this?\n", - "\n", - "BERT has two constraints:\n", - "1. All sentences must be padded or truncated to a single, fixed length.\n", - "2. The maximum sentence length is 512 tokens.\n", - "\n", - "Padding is done with a special `[PAD]` token, which is at index 0 in the BERT vocabulary. The below illustration demonstrates padding out to a \"MAX_LEN\" of 8 tokens.\n", - "\n", - "\n", - "\n", - "The \"Attention Mask\" is simply an array of 1s and 0s indicating which tokens are padding and which aren't (seems kind of redundant, doesn't it?!). This mask tells the \"Self-Attention\" mechanism in BERT not to incorporate these PAD tokens into its interpretation of the sentence.\n", - "\n", - "The maximum length does impact training and evaluation speed, however. \n", - "For example, with a Tesla K80:\n", - "\n", - "`MAX_LEN = 128 --> Training epochs take ~5:28 each`\n", - "\n", - "`MAX_LEN = 64 --> Training epochs take ~2:57 each`\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "l6w8elb-58GJ" - }, - "source": [ - "## 3.3. Tokenize Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "U28qy4P-NwQ9" - }, - "source": [ - "The transformers library provides a helpful `encode` function which will handle most of the parsing and data prep steps for us.\n", - "\n", - "Before we are ready to encode our text, though, we need to decide on a **maximum sentence length** for padding / truncating to.\n", - "\n", - "The below cell will perform one tokenization pass of the dataset in order to measure the maximum sentence length." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cKsH2sU0OCQA", - "outputId": "eb654716-a9b6-4cbd-ab4d-73eb7057d3ad" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Max sentence length: 47\n" - ] - } - ], - "source": [ - "max_len = 0\n", - "\n", - "# For every sentence...\n", - "for sent in sentences:\n", - "\n", - " # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n", - " input_ids = tokenizer.encode(sent, add_special_tokens=True)\n", - "\n", - " # Update the maximum sentence length.\n", - " max_len = max(max_len, len(input_ids))\n", - "\n", - "print('Max sentence length: ', max_len)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1M296yz577fV" - }, - "source": [ - "Just in case there are some longer test sentences, I'll set the maximum length to 64.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tIWAoWL2RK1p" - }, - "source": [ - "Now we're ready to perform the real tokenization.\n", - "\n", - "The `tokenizer.encode_plus` function combines multiple steps for us:\n", - "\n", - "1. Split the sentence into tokens.\n", - "2. Add the special `[CLS]` and `[SEP]` tokens.\n", - "3. Map the tokens to their IDs.\n", - "4. Pad or truncate all sentences to the same length.\n", - "5. Create the attention masks which explicitly differentiate real tokens from `[PAD]` tokens.\n", - "\n", - "The first four features are in `tokenizer.encode`, but I'm using `tokenizer.encode_plus` to get the fifth item (attention masks). Documentation is [here](https://huggingface.co/transformers/main_classes/tokenizer.html?highlight=encode_plus#transformers.PreTrainedTokenizer.encode_plus).\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2bBdb3pt8LuQ", - "outputId": "2c69be67-ed4e-4fde-b82a-1e20e2a493d0" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n", - "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2227: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n", - " FutureWarning,\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Original: Our friends won't buy this analysis, let alone the next one we propose.\n", - "Token IDs: tensor([ 101, 2256, 2814, 2180, 1005, 1056, 4965, 2023, 4106, 1010,\n", - " 2292, 2894, 1996, 2279, 2028, 2057, 16599, 1012, 102, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0])\n" - ] - } - ], - "source": [ - "# Tokenize all of the sentences and map the tokens to thier word IDs.\n", - "input_ids = []\n", - "attention_masks = []\n", - "\n", - "# For every sentence...\n", - "for sent in sentences:\n", - " # `encode_plus` will:\n", - " # (1) Tokenize the sentence.\n", - " # (2) Prepend the `[CLS]` token to the start.\n", - " # (3) Append the `[SEP]` token to the end.\n", - " # (4) Map tokens to their IDs.\n", - " # (5) Pad or truncate the sentence to `max_length`\n", - " # (6) Create attention masks for [PAD] tokens.\n", - " encoded_dict = tokenizer.encode_plus(\n", - " sent, # Sentence to encode.\n", - " add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n", - " max_length = 64, # Pad & truncate all sentences.\n", - " pad_to_max_length = True,\n", - " return_attention_mask = True, # Construct attn. masks.\n", - " return_tensors = 'pt', # Return pytorch tensors.\n", - " )\n", - " \n", - " # Add the encoded sentence to the list. \n", - " input_ids.append(encoded_dict['input_ids'])\n", - " \n", - " # And its attention mask (simply differentiates padding from non-padding).\n", - " attention_masks.append(encoded_dict['attention_mask'])\n", - "\n", - "# Convert the lists into tensors.\n", - "input_ids = torch.cat(input_ids, dim=0)\n", - "attention_masks = torch.cat(attention_masks, dim=0)\n", - "labels = torch.tensor(labels)\n", - "\n", - "# Print sentence 0, now as a list of IDs.\n", - "print('Original: ', sentences[0])\n", - "print('Token IDs:', input_ids[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aRp4O7D295d_" - }, - "source": [ - "## 3.4. Training & Validation Split\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qu0ao7p8rb06" - }, - "source": [ - "Divide up our training set to use 90% for training and 10% for validation." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "GEgLpFVlo1Z-", - "outputId": "9e68f506-acf3-4f45-f430-dc00cea3326b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "7,695 training samples\n", - " 856 validation samples\n" - ] - } - ], - "source": [ - "from torch.utils.data import TensorDataset, random_split\n", - "\n", - "# Combine the training inputs into a TensorDataset.\n", - "dataset = TensorDataset(input_ids, attention_masks, labels)\n", - "\n", - "# Create a 90-10 train-validation split.\n", - "\n", - "# Calculate the number of samples to include in each set.\n", - "train_size = int(0.9 * len(dataset))\n", - "val_size = len(dataset) - train_size\n", - "\n", - "# Divide the dataset by randomly selecting samples.\n", - "train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n", - "\n", - "print('{:>5,} training samples'.format(train_size))\n", - "print('{:>5,} validation samples'.format(val_size))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dD9i6Z2pG-sN" - }, - "source": [ - "We'll also create an iterator for our dataset using the torch DataLoader class. This helps save on memory during training because, unlike a for loop, with an iterator the entire dataset does not need to be loaded into memory." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "id": "XGUqOCtgqGhP" - }, - "outputs": [], - "source": [ - "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n", - "\n", - "# The DataLoader needs to know our batch size for training, so we specify it \n", - "# here. For fine-tuning BERT on a specific task, the authors recommend a batch \n", - "# size of 16 or 32.\n", - "batch_size = 32\n", - "\n", - "# Create the DataLoaders for our training and validation sets.\n", - "# We'll take training samples in random order. \n", - "train_dataloader = DataLoader(\n", - " train_dataset, # The training samples.\n", - " sampler = RandomSampler(train_dataset), # Select batches randomly\n", - " batch_size = batch_size # Trains with this batch size.\n", - " )\n", - "\n", - "# For validation the order doesn't matter, so we'll just read them sequentially.\n", - "validation_dataloader = DataLoader(\n", - " val_dataset, # The validation samples.\n", - " sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n", - " batch_size = batch_size # Evaluate with this batch size.\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8bwa6Rts-02-" - }, - "source": [ - "# 4. Train Our Classification Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3xYQ3iLO08SX" - }, - "source": [ - "Now that our input data is properly formatted, it's time to fine tune the BERT model. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D6TKgyUzPIQc" - }, - "source": [ - "## 4.1. BertForSequenceClassification" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1sjzRT1V0zwm" - }, - "source": [ - "For this task, we first want to modify the pre-trained BERT model to give outputs for classification, and then we want to continue training the model on our dataset until that the entire model, end-to-end, is well-suited for our task. \n", - "\n", - "Thankfully, the huggingface pytorch implementation includes a set of interfaces designed for a variety of NLP tasks. Though these interfaces are all built on top of a trained BERT model, each has different top layers and output types designed to accomodate their specific NLP task. \n", - "\n", - "Here is the current list of classes provided for fine-tuning:\n", - "* BertModel\n", - "* BertForPreTraining\n", - "* BertForMaskedLM\n", - "* BertForNextSentencePrediction\n", - "* **BertForSequenceClassification** - The one we'll use.\n", - "* BertForTokenClassification\n", - "* BertForQuestionAnswering\n", - "\n", - "The documentation for these can be found under [here](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BXYitPoE-cjH" - }, - "source": [ - "\n", - "\n", - "We'll be using [BertForSequenceClassification](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#bertforsequenceclassification). This is the normal BERT model with an added single linear layer on top for classification that we will use as a sentence classifier. As we feed input data, the entire pre-trained BERT model and the additional untrained classification layer is trained on our specific task. \n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WnQW9E-bBCRt" - }, - "source": [ - "OK, let's load BERT! There are a few different pre-trained BERT models available. \"bert-base-uncased\" means the version that has only lowercase letters (\"uncased\") and is the smaller version of the two (\"base\" vs \"large\").\n", - "\n", - "The documentation for `from_pretrained` can be found [here](https://huggingface.co/transformers/v2.2.0/main_classes/model.html#transformers.PreTrainedModel.from_pretrained), with the additional parameters defined [here](https://huggingface.co/transformers/v2.2.0/main_classes/configuration.html#transformers.PretrainedConfig)." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000, - "referenced_widgets": [ - "bf663b176284485e8db6b888bbd73d00", - "0358864aed7145e5a8b1ae70b33d8011", - "0a2a650a8e68491c85be1d84112645c3", - "6494768c95954aeaa092d1b8f494bf65", - "d4bd45ba11a94400a6b2afb62638506c", - "0a49f12eb51244f2882a979483acd105", - "801f8b39de0c488baf8c92705d3e2905", - "17b79c0e5d194a07bfff70f84ec10858", - "cca16e2b7d7446ccb05a2a09fdb1134b", - "54829969375b46d0a07900700a6e21a3", - "962dda4c15404a2c84caf50e208164b9" - ] - }, - "id": "gFsCTp_mporB", - "outputId": "2f63a147-b192-4f71-d1b6-93edc2abddcd" - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "bf663b176284485e8db6b888bbd73d00", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Downloading: 0%| | 0.00/420M [00:0012}\".format(p[0], str(tuple(p[1].size()))))\n", - "\n", - "print('\\n==== First Transformer ====\\n')\n", - "\n", - "for p in params[5:21]:\n", - " print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n", - "\n", - "print('\\n==== Output Layer ====\\n')\n", - "\n", - "for p in params[-4:]:\n", - " print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qRWT-D4U_Pvx" - }, - "source": [ - "## 4.2. Optimizer & Learning Rate Scheduler" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8o-VEBobKwHk" - }, - "source": [ - "Now that we have our model loaded we need to grab the training hyperparameters from within the stored model.\n", - "\n", - "For the purposes of fine-tuning, the authors recommend choosing from the following values (from Appendix A.3 of the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf)):\n", - "\n", - ">- **Batch size:** 16, 32 \n", - "- **Learning rate (Adam):** 5e-5, 3e-5, 2e-5 \n", - "- **Number of epochs:** 2, 3, 4 \n", - "\n", - "We chose:\n", - "* Batch size: 32 (set when creating our DataLoaders)\n", - "* Learning rate: 2e-5\n", - "* Epochs: 4 (we'll see that this is probably too many...)\n", - "\n", - "The epsilon parameter `eps = 1e-8` is \"a very small number to prevent any division by zero in the implementation\" (from [here](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)).\n", - "\n", - "You can find the creation of the AdamW optimizer in `run_glue.py` [here](https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L109)." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "id": "GLs72DuMODJO" - }, - "outputs": [], - "source": [ - "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n", - "# I believe the 'W' stands for 'Weight Decay fix\"\n", - "optimizer = AdamW(model.parameters(),\n", - " lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n", - " eps = 1e-8 # args.adam_epsilon - default is 1e-8.\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "-p0upAhhRiIx" - }, - "outputs": [], - "source": [ - "from transformers import get_linear_schedule_with_warmup\n", - "\n", - "# Number of training epochs. The BERT authors recommend between 2 and 4. \n", - "# We chose to run for 4, but we'll see later that this may be over-fitting the\n", - "# training data.\n", - "epochs = 4\n", - "\n", - "# Total number of training steps is [number of batches] x [number of epochs]. \n", - "# (Note that this is not the same as the number of training samples).\n", - "total_steps = len(train_dataloader) * epochs\n", - "\n", - "# Create the learning rate scheduler.\n", - "scheduler = get_linear_schedule_with_warmup(optimizer, \n", - " num_warmup_steps = 0, # Default value in run_glue.py\n", - " num_training_steps = total_steps)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RqfmWwUR_Sox" - }, - "source": [ - "## 4.3. Training Loop" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_QXZhFb4LnV5" - }, - "source": [ - "Below is our training loop. There's a lot going on, but fundamentally for each pass in our loop we have a training phase and a validation phase. \n", - "\n", - "> *Thank you to [Stas Bekman](https://ca.linkedin.com/in/stasbekman) for contributing the insights and code for using validation loss to detect over-fitting!*\n", - "\n", - "**Training:**\n", - "- Unpack our data inputs and labels\n", - "- Load data onto the GPU for acceleration\n", - "- Clear out the gradients calculated in the previous pass. \n", - " - In pytorch the gradients accumulate by default (useful for things like RNNs) unless you explicitly clear them out.\n", - "- Forward pass (feed input data through the network)\n", - "- Backward pass (backpropagation)\n", - "- Tell the network to update parameters with optimizer.step()\n", - "- Track variables for monitoring progress\n", - "\n", - "**Evalution:**\n", - "- Unpack our data inputs and labels\n", - "- Load data onto the GPU for acceleration\n", - "- Forward pass (feed input data through the network)\n", - "- Compute loss on our validation data and track variables for monitoring progress\n", - "\n", - "Pytorch hides all of the detailed calculations from us, but we've commented the code to point out which of the above steps are happening on each line. \n", - "\n", - "> *PyTorch also has some [beginner tutorials](https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#sphx-glr-beginner-blitz-cifar10-tutorial-py) which you may also find helpful.*" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pE5B99H5H2-W" - }, - "source": [ - "Define a helper function for calculating accuracy." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "id": "9cQNvaZ9bnyy" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "# Function to calculate the accuracy of our predictions vs labels\n", - "def flat_accuracy(preds, labels):\n", - " pred_flat = np.argmax(preds, axis=1).flatten()\n", - " labels_flat = labels.flatten()\n", - " return np.sum(pred_flat == labels_flat) / len(labels_flat)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KNhRtWPXH9C3" - }, - "source": [ - "Helper function for formatting elapsed times as `hh:mm:ss`\n" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "id": "gpt6tR83keZD" - }, - "outputs": [], - "source": [ - "import time\n", - "import datetime\n", - "\n", - "def format_time(elapsed):\n", - " '''\n", - " Takes a time in seconds and returns a string hh:mm:ss\n", - " '''\n", - " # Round to the nearest second.\n", - " elapsed_rounded = int(round((elapsed)))\n", - " \n", - " # Format as hh:mm:ss\n", - " return str(datetime.timedelta(seconds=elapsed_rounded))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cfNIhN19te3N" - }, - "source": [ - "We're ready to kick off the training!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "6J-FYdx6nFE_", - "outputId": "6847197f-8ee4-40fe-a3ff-c354aefba69a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "======== Epoch 1 / 4 ========\n", - "Training...\n", - " Batch 40 of 241. Elapsed: 0:00:26.\n", - " Batch 80 of 241. Elapsed: 0:00:52.\n", - " Batch 120 of 241. Elapsed: 0:01:17.\n", - " Batch 160 of 241. Elapsed: 0:01:43.\n", - " Batch 200 of 241. Elapsed: 0:02:09.\n", - " Batch 240 of 241. Elapsed: 0:02:35.\n", - "\n", - " Average training loss: 0.49\n", - " Training epcoh took: 0:02:36\n", - "\n", - "Running Validation...\n", - " Accuracy: 0.82\n", - " Validation Loss: 0.41\n", - " Validation took: 0:00:06\n", - "\n", - "======== Epoch 2 / 4 ========\n", - "Training...\n", - " Batch 40 of 241. Elapsed: 0:00:26.\n", - " Batch 80 of 241. Elapsed: 0:00:52.\n", - " Batch 120 of 241. Elapsed: 0:01:18.\n", - " Batch 160 of 241. Elapsed: 0:01:44.\n", - " Batch 200 of 241. Elapsed: 0:02:10.\n", - " Batch 240 of 241. Elapsed: 0:02:36.\n", - "\n", - " Average training loss: 0.30\n", - " Training epcoh took: 0:02:36\n", - "\n", - "Running Validation...\n", - " Accuracy: 0.84\n", - " Validation Loss: 0.39\n", - " Validation took: 0:00:06\n", - "\n", - "======== Epoch 3 / 4 ========\n", - "Training...\n", - " Batch 40 of 241. Elapsed: 0:00:26.\n", - " Batch 80 of 241. Elapsed: 0:00:52.\n" - ] - } - ], - "source": [ - "import random\n", - "import numpy as np\n", - "\n", - "# This training code is based on the `run_glue.py` script here:\n", - "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n", - "\n", - "# Set the seed value all over the place to make this reproducible.\n", - "seed_val = 42\n", - "\n", - "random.seed(seed_val)\n", - "np.random.seed(seed_val)\n", - "torch.manual_seed(seed_val)\n", - "torch.cuda.manual_seed_all(seed_val)\n", - "\n", - "# We'll store a number of quantities such as training and validation loss, \n", - "# validation accuracy, and timings.\n", - "training_stats = []\n", - "\n", - "# Measure the total training time for the whole run.\n", - "total_t0 = time.time()\n", - "\n", - "# For each epoch...\n", - "for epoch_i in range(0, epochs):\n", - " \n", - " # ========================================\n", - " # Training\n", - " # ========================================\n", - " \n", - " # Perform one full pass over the training set.\n", - "\n", - " print(\"\")\n", - " print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n", - " print('Training...')\n", - "\n", - " # Measure how long the training epoch takes.\n", - " t0 = time.time()\n", - "\n", - " # Reset the total loss for this epoch.\n", - " total_train_loss = 0\n", - "\n", - " # Put the model into training mode. Don't be mislead--the call to \n", - " # `train` just changes the *mode*, it doesn't *perform* the training.\n", - " # `dropout` and `batchnorm` layers behave differently during training\n", - " # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n", - " model.train()\n", - "\n", - " # For each batch of training data...\n", - " for step, batch in enumerate(train_dataloader):\n", - "\n", - " # Progress update every 40 batches.\n", - " if step % 40 == 0 and not step == 0:\n", - " # Calculate elapsed time in minutes.\n", - " elapsed = format_time(time.time() - t0)\n", - " \n", - " # Report progress.\n", - " print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n", - "\n", - " # Unpack this training batch from our dataloader. \n", - " #\n", - " # As we unpack the batch, we'll also copy each tensor to the GPU using the \n", - " # `to` method.\n", - " #\n", - " # `batch` contains three pytorch tensors:\n", - " # [0]: input ids \n", - " # [1]: attention masks\n", - " # [2]: labels \n", - " b_input_ids = batch[0].to(device)\n", - " b_input_mask = batch[1].to(device)\n", - " b_labels = batch[2].to(device)\n", - "\n", - " # Always clear any previously calculated gradients before performing a\n", - " # backward pass. PyTorch doesn't do this automatically because \n", - " # accumulating the gradients is \"convenient while training RNNs\". \n", - " # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n", - " model.zero_grad() \n", - "\n", - " # Perform a forward pass (evaluate the model on this training batch).\n", - " # In PyTorch, calling `model` will in turn call the model's `forward` \n", - " # function and pass down the arguments. The `forward` function is \n", - " # documented here: \n", - " # https://huggingface.co/transformers/model_doc/bert.html#bertforsequenceclassification\n", - " # The results are returned in a results object, documented here:\n", - " # https://huggingface.co/transformers/main_classes/output.html#transformers.modeling_outputs.SequenceClassifierOutput\n", - " # Specifically, we'll get the loss (because we provided labels) and the\n", - " # \"logits\"--the model outputs prior to activation.\n", - " result = model(b_input_ids, \n", - " token_type_ids=None, \n", - " attention_mask=b_input_mask, \n", - " labels=b_labels,\n", - " return_dict=True)\n", - "\n", - " loss = result.loss\n", - " logits = result.logits\n", - "\n", - " # Accumulate the training loss over all of the batches so that we can\n", - " # calculate the average loss at the end. `loss` is a Tensor containing a\n", - " # single value; the `.item()` function just returns the Python value \n", - " # from the tensor.\n", - " total_train_loss += loss.item()\n", - "\n", - " # Perform a backward pass to calculate the gradients.\n", - " loss.backward()\n", - "\n", - " # Clip the norm of the gradients to 1.0.\n", - " # This is to help prevent the \"exploding gradients\" problem.\n", - " torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n", - "\n", - " # Update parameters and take a step using the computed gradient.\n", - " # The optimizer dictates the \"update rule\"--how the parameters are\n", - " # modified based on their gradients, the learning rate, etc.\n", - " optimizer.step()\n", - "\n", - " # Update the learning rate.\n", - " scheduler.step()\n", - "\n", - " # Calculate the average loss over all of the batches.\n", - " avg_train_loss = total_train_loss / len(train_dataloader) \n", - " \n", - " # Measure how long this epoch took.\n", - " training_time = format_time(time.time() - t0)\n", - "\n", - " print(\"\")\n", - " print(\" Average training loss: {0:.2f}\".format(avg_train_loss))\n", - " print(\" Training epcoh took: {:}\".format(training_time))\n", - " \n", - " # ========================================\n", - " # Validation\n", - " # ========================================\n", - " # After the completion of each training epoch, measure our performance on\n", - " # our validation set.\n", - "\n", - " print(\"\")\n", - " print(\"Running Validation...\")\n", - "\n", - " t0 = time.time()\n", - "\n", - " # Put the model in evaluation mode--the dropout layers behave differently\n", - " # during evaluation.\n", - " model.eval()\n", - "\n", - " # Tracking variables \n", - " total_eval_accuracy = 0\n", - " total_eval_loss = 0\n", - " nb_eval_steps = 0\n", - "\n", - " # Evaluate data for one epoch\n", - " for batch in validation_dataloader:\n", - " \n", - " # Unpack this training batch from our dataloader. \n", - " #\n", - " # As we unpack the batch, we'll also copy each tensor to the GPU using \n", - " # the `to` method.\n", - " #\n", - " # `batch` contains three pytorch tensors:\n", - " # [0]: input ids \n", - " # [1]: attention masks\n", - " # [2]: labels \n", - " b_input_ids = batch[0].to(device)\n", - " b_input_mask = batch[1].to(device)\n", - " b_labels = batch[2].to(device)\n", - " \n", - " # Tell pytorch not to bother with constructing the compute graph during\n", - " # the forward pass, since this is only needed for backprop (training).\n", - " with torch.no_grad(): \n", - "\n", - " # Forward pass, calculate logit predictions.\n", - " # token_type_ids is the same as the \"segment ids\", which \n", - " # differentiates sentence 1 and 2 in 2-sentence tasks.\n", - " result = model(b_input_ids, \n", - " token_type_ids=None, \n", - " attention_mask=b_input_mask,\n", - " labels=b_labels,\n", - " return_dict=True)\n", - "\n", - " # Get the loss and \"logits\" output by the model. The \"logits\" are the \n", - " # output values prior to applying an activation function like the \n", - " # softmax.\n", - " loss = result.loss\n", - " logits = result.logits\n", - " \n", - " # Accumulate the validation loss.\n", - " total_eval_loss += loss.item()\n", - "\n", - " # Move logits and labels to CPU\n", - " logits = logits.detach().cpu().numpy()\n", - " label_ids = b_labels.to('cpu').numpy()\n", - "\n", - " # Calculate the accuracy for this batch of test sentences, and\n", - " # accumulate it over all batches.\n", - " total_eval_accuracy += flat_accuracy(logits, label_ids)\n", - " \n", - "\n", - " # Report the final accuracy for this validation run.\n", - " avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n", - " print(\" Accuracy: {0:.2f}\".format(avg_val_accuracy))\n", - "\n", - " # Calculate the average loss over all of the batches.\n", - " avg_val_loss = total_eval_loss / len(validation_dataloader)\n", - " \n", - " # Measure how long the validation run took.\n", - " validation_time = format_time(time.time() - t0)\n", - " \n", - " print(\" Validation Loss: {0:.2f}\".format(avg_val_loss))\n", - " print(\" Validation took: {:}\".format(validation_time))\n", - "\n", - " # Record all statistics from this epoch.\n", - " training_stats.append(\n", - " {\n", - " 'epoch': epoch_i + 1,\n", - " 'Training Loss': avg_train_loss,\n", - " 'Valid. Loss': avg_val_loss,\n", - " 'Valid. Accur.': avg_val_accuracy,\n", - " 'Training Time': training_time,\n", - " 'Validation Time': validation_time\n", - " }\n", - " )\n", - "\n", - "print(\"\")\n", - "print(\"Training complete!\")\n", - "\n", - "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VQTvJ1vRP7u4" - }, - "source": [ - "Let's view the summary of the training process." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 195 - }, - "id": "6O_NbXFGMukX", - "outputId": "a9e51eda-5eae-4800-87d5-8d016ff25bb2" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Training LossValid. LossValid. Accur.Training TimeValidation Time
epoch
10.490.440.810:00:350:00:01
20.300.450.810:00:340:00:01
30.180.500.830:00:340:00:01
40.120.570.840:00:340:00:01
\n", - "
" - ], - "text/plain": [ - " Training Loss Valid. Loss Valid. Accur. Training Time Validation Time\n", - "epoch \n", - "1 0.49 0.44 0.81 0:00:35 0:00:01\n", - "2 0.30 0.45 0.81 0:00:34 0:00:01\n", - "3 0.18 0.50 0.83 0:00:34 0:00:01\n", - "4 0.12 0.57 0.84 0:00:34 0:00:01" - ] - }, - "execution_count": 23, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Display floats with two decimal places.\n", - "pd.set_option('precision', 2)\n", - "\n", - "# Create a DataFrame from our training statistics.\n", - "df_stats = pd.DataFrame(data=training_stats)\n", - "\n", - "# Use the 'epoch' as the row index.\n", - "df_stats = df_stats.set_index('epoch')\n", - "\n", - "# A hack to force the column headers to wrap.\n", - "#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n", - "\n", - "# Display the table.\n", - "df_stats" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1-G03mmwH3aI" - }, - "source": [ - "Notice that, while the the training loss is going down with each epoch, the validation loss is increasing! This suggests that we are training our model too long, and it's over-fitting on the training data. \n", - "\n", - "(For reference, we are using 7,695 training samples and 856 validation samples).\n", - "\n", - "Validation Loss is a more precise measure than accuracy, because with accuracy we don't care about the exact output value, but just which side of a threshold it falls on. \n", - "\n", - "If we are predicting the correct answer, but with less confidence, then validation loss will catch this, while accuracy will not." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 427 - }, - "id": "68xreA9JAmG5", - "outputId": "70b8500d-7efc-4c99-de1f-05e8795e6298" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGaCAYAAACopj13AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxU5f4H8M/sMwwM+yaLKAooIu57mRviXopLmVqaZVetn91u6W33Zt2rlqWm96ZW5pr7kruolaWSS5oJ7gsomyDbsM0w5/cHMDIO6qDAYfm8X69eMWfOOfOd0SMfHr7PcySCIAggIiIiIiLRSMUugIiIiIiovmMoJyIiIiISGUM5EREREZHIGMqJiIiIiETGUE5EREREJDKGciIiIiIikTGUE1GdlZCQgODgYCxYsOCRzzF9+nQEBwdXYlV11/0+7+DgYEyfPt2mcyxYsADBwcFISEio9Po2bdqE4OBgHDt2rNLPTUT0uORiF0BE9UdFwm10dDR8fX2rsJraJzc3F//973+xc+dOpKSkwMXFBW3btsXf/vY3BAYG2nSO1157DXv27MGWLVvQrFmzcvcRBAG9evVCVlYWDh8+DLVaXZlvo0odO3YMMTExGDduHHQ6ndjlWElISECvXr0wevRovP/++2KXQ0Q1CEM5EVWb2bNnWzw+ceIEfvjhB4wcORJt27a1eM7FxeWxX8/HxwdnzpyBTCZ75HP861//wkcfffTYtVSGd999Fzt27MDAgQPRoUMHpKam4sCBAzh9+rTNoTwqKgp79uzBxo0b8e6775a7z9GjR3Hz5k2MHDmyUgL5mTNnIJVWzy9mY2JisHDhQjzzzDNWoXzIkCEYMGAAFApFtdRCRFQRDOVEVG2GDBli8bioqAg//PADWrVqZfXcvXJycmBvb1+h15NIJFCpVBWus6yaEuDy8vKwe/dudOvWDZ999pl5+5QpU1BYWGjzebp16wZvb29s374db731FpRKpdU+mzZtAlAc4CvD4/4ZVBaZTPZYP6AREVUl9pQTUY3Ts2dPjBkzBufOncOECRPQtm1bDB48GEBxOJ83bx6GDx+Ojh07okWLFujTpw/mzp2LvLw8i/OU1+NcdtvBgwcxbNgwhIWFoVu3bvjPf/4Do9FocY7yespLt2VnZ+ODDz5A586dERYWhlGjRuH06dNW7+fOnTuYMWMGOnbsiNatW2Ps2LE4d+4cxowZg549e9r0mUgkEkgkknJ/SCgvWN+PVCrFM888g4yMDBw4cMDq+ZycHOzduxdBQUFo2bJlhT7v+ymvp9xkMuF///sfevbsibCwMAwcOBDbtm0r9/jLly/jww8/xIABA9C6dWuEh4dj6NChWL9+vcV+06dPx8KFCwEAvXr1QnBwsMWf//16ytPT0/HRRx+he/fuaNGiBbp3746PPvoId+7csdiv9PgjR45g2bJl6N27N1q0aIG+ffti8+bNNn0WFREXF4fJkyejY8eOCAsLQ//+/bFkyRIUFRVZ7JeYmIgZM2agR48eaNGiBTp37oxRo0ZZ1GQymfDdd99h0KBBaN26Ndq0aYO+ffvin//8JwwGQ6XXTkQVx5FyIqqRbt26hXHjxiEyMhIRERHIzc0FACQnJ2PDhg2IiIjAwIEDIZfLERMTg6VLlyI2NhbLli2z6fw//fQTVq9ejVGjRmHYsGGIjo7GN998A0dHR0yaNMmmc0yYMAEuLi6YPHkyMjIy8O233+Lll19GdHS0eVS/sLAQL774ImJjYzF06FCEhYXh/PnzePHFF+Ho6Gjz56FWq/H0009j48aN+PHHHzFw4ECbj73X0KFDsXjxYmzatAmRkZEWz+3YsQP5+fkYNmwYgMr7vO/16aef4vvvv0f79u3xwgsvIC0tDTNnzoSfn5/VvjExMTh+/Dieeuop+Pr6mn9r8O677yI9PR2vvPIKAGDkyJHIycnBvn37MGPGDDg7OwN48FyG7OxsPPvss7h+/TqGDRuG5s2bIzY2FmvWrMHRo0exfv16q9/QzJs3D/n5+Rg5ciSUSiXWrFmD6dOnw9/f36oN61H9+eefGDNmDORyOUaPHg03NzccPHgQc+fORVxcnPm3JUajES+++CKSk5Px3HPPISAgADk5OTh//jyOHz+OZ555BgCwePFizJ8/Hz169MCoUaMgk8mQkJCAAwcOoLCwsMb8RoioXhOIiESyceNGISgoSNi4caPF9h49eghBQUHCunXrrI4pKCgQCgsLrbbPmzdPCAoKEk6fPm3eFh8fLwQFBQnz58+32hYeHi7Ex8ebt5tMJmHAgAFC165dLc779ttvC0FBQeVu++CDDyy279y5UwgKChLWrFlj3rZy5UohKChIWLRokcW+pdt79Ohh9V7Kk52dLUycOFFo0aKF0Lx5c2HHjh02HXc/Y8eOFZo1ayYkJydbbB8xYoQQGhoqpKWlCYLw+J+3IAhCUFCQ8Pbbb5sfX758WQgODhbGjh0rGI1G8/azZ88KwcHBQlBQkMWfjV6vt3r9oqIi4fnnnxfatGljUd/8+fOtji9V+vft6NGj5m2ff/65EBQUJKxcudJi39I/n3nz5lkdP2TIEKGgoMC8PSkpSQgNDRWmTZtm9Zr3Kv2MPvroowfuN3LkSKFZs2ZCbGyseZvJZBJee+01ISgoSPjtt98EQRCE2NhYISgoSPj6668feL6nn35a6Nev30PrIyLxsH2FiGokJycnDB061Gq7Uqk0j+oZjUZkZmYiPT0dXbp0AYBy20fK06tXL4vVXSQSCTp27IjU1FTo9XqbzvHCCy9YPO7UqRMA4Pr16+ZtBw8ehEwmw9ixYy32HT58OBwcHGx6HZPJhNdffx1xcXHYtWsXnnzySbz55pvYvn27xX7vvfceQkNDbeoxj4qKQlFREbZs2WLedvnyZfzxxx/o2bOneaJtZX3eZUVHR0MQBLz44osWPd6hoaHo2rWr1f52dnbmrwsKCnDnzh1kZGSga9euyMnJwZUrVypcQ6l9+/bBxcUFI0eOtNg+cuRIuLi4YP/+/VbHPPfccxYtQ56enmjUqBGuXbv2yHWUlZaWhlOnTqFnz54ICQkxb5dIJHj11VfNdQMw/x06duwY0tLS7ntOe3t7JCcn4/jx45VSIxFVPravEFGN5Ofnd99JeatWrcLatWtx6dIlmEwmi+cyMzNtPv+9nJycAAAZGRnQarUVPkdpu0RGRoZ5W0JCAjw8PKzOp1Qq4evri6ysrIe+TnR0NA4fPow5c+bA19cXX375JaZMmYK33noLRqPR3KJw/vx5hIWF2dRjHhERAZ1Oh02bNuHll18GAGzcuBEAzK0rpSrj8y4rPj4eANC4cWOr5wIDA3H48GGLbXq9HgsXLsSuXbuQmJhodYwtn+H9JCQkoEWLFpDLLb8dyuVyBAQE4Ny5c1bH3O/vzs2bNx+5jntrAoAmTZpYPde4cWNIpVLzZ+jj44NJkybh66+/Rrdu3dCsWTN06tQJkZGRaNmypfm4N954A5MnT8bo0aPh4eGBDh064KmnnkLfvn0rNCeBiKoOQzkR1Ugajabc7d9++y3+/e9/o1u3bhg7diw8PDygUCiQnJyM6dOnQxAEm87/oFU4Hvccth5vq9KJie3btwdQHOgXLlyIV199FTNmzIDRaERISAhOnz6NWbNm2XROlUqFgQMHYvXq1Th58iTCw8Oxbds2eHl54YknnjDvV1mf9+P4+9//jkOHDmHEiBFo3749nJycIJPJ8NNPP+G7776z+kGhqlXX8o62mjZtGqKionDo0CEcP34cGzZswLJly/DSSy/hH//4BwCgdevW2LdvHw4fPoxjx47h2LFj+PHHH7F48WKsXr3a/AMpEYmHoZyIapWtW7fCx8cHS5YssQhHP//8s4hV3Z+Pjw+OHDkCvV5vMVpuMBiQkJBg0w1uSt/nzZs34e3tDaA4mC9atAiTJk3Ce++9Bx8fHwQFBeHpp5+2ubaoqCisXr0amzZtQmZmJlJTUzFp0iSLz7UqPu/SkeYrV67A39/f4rnLly9bPM7KysKhQ4cwZMgQzJw50+K53377zercEomkwrVcvXoVRqPRYrTcaDTi2rVr5Y6KV7XStqpLly5ZPXflyhWYTCaruvz8/DBmzBiMGTMGBQUFmDBhApYuXYrx48fD1dUVAKDVatG3b1/07dsXQPFvQGbOnIkNGzbgpZdequJ3RUQPU7N+3CciegipVAqJRGIxQms0GrFkyRIRq7q/nj17oqioCN9//73F9nXr1iE7O9umc3Tv3h1A8aofZfvFVSoVPv/8c+h0OiQkJKBv375WbRgPEhoaimbNmmHnzp1YtWoVJBKJ1drkVfF59+zZExKJBN9++63F8n5//fWXVdAu/UHg3hH5lJQUqyURgbv957a21fTu3Rvp6elW51q3bh3S09PRu3dvm85TmVxdXdG6dWscPHgQFy5cMG8XBAFff/01AKBPnz4AilePuXdJQ5VKZW4NKv0c0tPTrV4nNDTUYh8iEhdHyomoVomMjMRnn32GiRMnok+fPsjJycGPP/5YoTBanYYPH461a9fiiy++wI0bN8xLIu7evRsNGza0Whe9PF27dkVUVBQ2bNiAAQMGYMiQIfDy8kJ8fDy2bt0KoDhgffXVVwgMDES/fv1sri8qKgr/+te/8Msvv6BDhw5WI7BV8XkHBgZi9OjRWLlyJcaNG4eIiAikpaVh1apVCAkJsejjtre3R9euXbFt2zao1WqEhYXh5s2b+OGHH+Dr62vRvw8A4eHhAIC5c+di0KBBUKlUaNq0KYKCgsqt5aWXXsLu3bsxc+ZMnDt3Ds2aNUNsbCw2bNiARo0aVdkI8tmzZ7Fo0SKr7XK5HC+//DLeeecdjBkzBqNHj8Zzzz0Hd3d3HDx4EIcPH8bAgQPRuXNnAMWtTe+99x4iIiLQqFEjaLVanD17Fhs2bEB4eLg5nPfv3x+tWrVCy5Yt4eHhgdTUVKxbtw4KhQIDBgyokvdIRBVTM7+LERHdx4QJEyAIAjZs2IBZs2bB3d0d/fr1w7Bhw9C/f3+xy7OiVCqxfPlyzJ49G9HR0di1axdatmyJ7777Du+88w7y8/NtOs+sWbPQoUMHrF27FsuWLYPBYICPjw8iIyMxfvx4KJVKjBw5Ev/4xz/g4OCAbt262XTeQYMGYfbs2SgoKLCa4AlU3ef9zjvvwM3NDevWrcPs2bMREBCA999/H9evX7eaXDlnzhx89tlnOHDgADZv3oyAgABMmzYNcrkcM2bMsNi3bdu2ePPNN7F27Vq89957MBqNmDJlyn1DuYODA9asWYP58+fjwIED2LRpE1xdXTFq1ChMnTq1wneRtdXp06fLXblGqVTi5ZdfRlhYGNauXYv58+djzZo1yM3NhZ+fH958802MHz/evH9wcDD69OmDmJgYbN++HSaTCd7e3njllVcs9hs/fjx++uknrFixAtnZ2XB1dUV4eDheeeUVixVeiEg8EqE6ZukQEZGFoqIidOrUCS1btnzkG/AQEVHdwZ5yIqIqVt5o+Nq1a5GVlVXuutxERFT/sH2FiKiKvfvuuygsLETr1q2hVCpx6tQp/Pjjj2jYsCFGjBghdnlERFQDsH2FiKiKbdmyBatWrcK1a9eQm5sLV1dXdO/eHa+//jrc3NzELo+IiGoAhnIiIiIiIpGxp5yIiIiISGQM5UREREREIuNEzxJ37uhhMlVvJ4+rqz3S0nKq9TWJaiNeK0S24bVCZBuxrhWpVAJnZ225zzGUlzCZhGoP5aWvS0QPx2uFyDa8VohsU9OuFbavEBERERGJjKGciIiIiEhkDOVERERERCJjKCciIiIiEhlDORERERGRyLj6io2MRgP0+iwUFOTBZCqqlHOmpEhhMpkq5VxUM8hkCtjbO0KjKX+5IyIiIqLyMJTbwGg0ID09GXZ2DnBx8YJMJoNEInns88rlUhiNDOV1hSAIMBgKkJFxG3K5AgqFUuySiIiIqJZg+4oN9Pos2Nk5wN7eEXK5vFICOdU9EokESqUaWq0jcnIyxC6HiIiIahGGchsUFORBrWY7AtlGrdbAYCgUuwwiIiKqRdi+YgOTqQgymUzsMqiWkEpllTbvgIiIiCpPTNJJbLu8GxkFGXBSOWFwYCQ6eLURuywADOU2Y8sK2Yp/V4iIiGqemKSTWB23EQaTAQBwpyADq+M2AkCNCOZsXyEiIiKiOi3fWIBNF380B/JSBpMB2y7vFqkqSxwppyo1ZcrLAICFC7+u1mOJiIiofioyFSFRn4zrWfG4lnUD17LikahPhgCh3P3vFNSMxRkYyuupbt3a2bTf+vXb4O3doIqrISIiIqo4QRCQUZCJq1k3cC3rBq5nxeNGVgIKS0bEtXI7NNT5Idy9BX65eQQ5Br3VOZxVTtVddrkYyuup996bafF43bo1SE5OxNSpb1hsd3JyfqzXmTfvK1GOJSIioronz5hfMgIebx4JzyrMBgDIJTL4Ovigc4MOCND5IUDnB3eNm3mul4edm0VPOQAopAoMDowU5b3ci6G8nurbt7/F40OHopGZmWG1/V75+flQq9U2v45CoXik+h73WCIiIqrdikxFuKlPLA7fmcUBPDk31dyG4mHnhmDnpghwLA7gPvYNoJDeP9qWTubk6itU60yZ8jJycnLw1lv/xIIF83D+fBxGjx6LCRNewS+/HMK2bZtx4cJ5ZGVlwt3dA/37D8KYMS9aLB95b1/4yZPH8dprkzBr1mxcvXoFW7ZsRFZWJsLCwvGPf/wTvr5+lXIsAGzcuA5r165CWtptBAYGYsqUaViyZLHFOYmIiEh8giAgLf8Orpf0gF/LuoH47JswmIwAAHuFFgE6P7TzbIWGOj801PlBq7Cr8Ot08GqDDl5t4O7ugNTU7Mp+G4+FoVwkR/5KwqafryAtMx+uOhWGdg9E51AvscuykpFxB2+9NQ0REZGIjBwAT8/iGnfu/BEajR1GjhwNOzsNTpw4jqVL/wu9Xo/Jk19/6HmXL18GqVSG554bi+zsLKxZswIfffQulixZXinHbt68AfPmzUarVm0wcuSzSExMxIwZb8LBwQHu7h6P/oEQERHRY8s15OJ6VoJ5Iua1rBvmfm+FVA4/Bx884dMZDXV+CND5w1XtXOeXHGYoF8GRv5KwfFccCo0mAEBaVgGW74oDgBoXzG/fTsX06e9h4MAhFts//PBjqFR321iefjoKc+Z8gs2b12PixFehVCofeF6j0YhvvlkOubz4r6BO54gvv5yLK1cuoXHjJo91rMFgwNKlixEaGoYvvlhk3q9Jk6aYNetDhnIiIqJqZDQZcTMnEVdLJmJey7qBlNzb5ue97DwQ6hqCAJ0/Ahz94KP1hkxa/27ayFD+GH79MxGHzyRW+LjLtzJhLLJclqfQaMK3O2Px8x+3Kny+bi290TXMu8LH2UKtViMycoDV9rKBPDdXj8JCA8LDW2Pr1k24fv0amjYNeuB5BwwYbA7LABAe3goAcOvWzYeG8ocdGxd3DpmZmfjb356x2K9Pn0jMn//5A89NREREj04QBKTmpZlXQrmWFY+E7JswCsV3unZQ2iNA54+OXu0QoPNDQ50vNHKNyFXXDAzlIrg3kD9su5jc3T0sgm2pK1cuY8mSxTh58nfo9ZbLC+n1OQ89b2kbTCkHBx0AIDv74f1dDzs2Kan4B6V7e8zlcjm8vavmhxciIqL6KKdQj+vZ8biWecO8IoremAsAUEoV8HPwRXe/rsWj4Do/OKuc6nwbyqNiKH8MXcMebYT6H4t+RVpWgdV2V50Kb4+uGTOAS5UdES+VnZ2NqVNfhp2dPSZMmAQfH18olUpcuBCHxYsXwGQyPfS80vv8WkoQHv6DyeMcS0RERI/GUGRAQs4tcw/4tax43M5LAwBIIIG31hPh7qHmPnBvrWe9bEN5VAzlIhjaPdCipxwAlHIphnYPFLEq2506dQKZmZmYNWsOWrW6+0NEYmLFW2+qgpdX8Q9KCQnxCA9vbd5uNBqRmJiIwMAHt8cQERHVdybBhNTc2yUBvDiE38xJRFFJG4qTyhEBOj90bdABATp/+Dv4QC23fclkssZQLoLSyZy1YfWV8kilUgCWI9MGgwGbN68XqyQLISHN4ejoiG3bNqNv3/7m9pt9+3YjOztL5OqIiIhqnuzCnLsroWTewPXsBOQZ8wAAKpkSDR380NPvCQQ4FrehOKkcRa647mEoF0nnUC88Ed4ARuPDWz1qmrCwlnBw0GHWrA8RFTUSEokEe/bsRE3pHlEoFBg//mXMmzcH//d/f0OPHr2QmJiIXbu2w8fHl71sRERUrxUWFeJG9k3zSijXsuKRnn8HACCVSOGt9UQbj5bmPnAvrQekEqnIVdd9DOVUYY6OTpg9ex4WLvwCS5YshoODDhER/dCuXQe88cYUscsDAAwbNhKCIGDt2lX46qsvERjYFP/+9+f44ou5UCpVYpdHRERULUyCCcm5qSUTMYtXRLmpT4JJKB4UdFY5IcDRH919uyBA5w8/Bx+oZA9e1piqhkTg7DgAQFpaDkym8j+KpKTr8PJqWOmvKZdLa+VIeW1lMpkwcGAfdO/eA2+//W6VvlZV/Z2pr2rindeIaiJeK5RZkFXmhjzxuJEVj/yi4sUl1DJ1yTKEfiX/94ejykHkisUh1rUilUrg6mpf7nMcKac6qaCgACqV5Yj47t07kJWVidat24pUFRERUeXJNxYgPjvBYjJmRkEmgOI2FF97b7T3aoOAktVQPOzc2IZSgzGUU5105swfWLx4AZ56qid0OkdcuBCHHTu2oXHjQPTo0Vvs8oiIiCrEJJiQqE8uHgXPjMf17HjcykmCgOLf8rupXRDoGFAyEdMfvvYNoJQpRK6aKoKhnOqkBg184Obmjg0bfkBWViZ0OkdERg7ApElToFDwHykiIqq5BEFARkGmefT7elY8rmcnoLCoEABgJ9egoc4PLQNCze0oDsryWyKo9mAopzrJx8cXs2fPE7sMIiKih8oz5uNGVkKZW9PfQGZhcb+zXCKDj0MDdPZuX9KG4gd3jRtXEquDGMqJiIiIqkmRqQi39EkWkzGT9SnmNhQPjRuCnJsWB3BHP/jYN4BCyrhWH/BPmYiIiKgKCIKA9Pw7Frelj8++CYPJAACwV2jRUOeHtiVrgjfU+UGrsBO5ahILQzkRERFRJcg15OF6djyuZd7tBc825AAA5FI5/Ox90M2no/mmPK5qF7ahkBlDOREREVEFGU1G3MxJxLWseHMfeHJuqvl5TzsPNHcNNgdwH3tvyKQyESummo6hnIiIiOgBBEHA7bx0i4mY8Tm3YDQZAQAOSnsE6PzRwattyWoovtDINSJXTbUNQzkRERFRGTkGPa6XrIZSGsT1hlwAgEKqgL+DL7r7dClZE9wPziontqHQY2MoJyIionrLYDIiIfuWRQBPzUsDAEgggbfWEy3dQs23pW+g9WQbClUJhnKqFDt3bscnn3yE9eu3wdu7AQAgKmoQWrdui3fe+bDCxz6ukyeP47XXJmH+/P+iTZt2lXJOIiKq3UyCCal5abiWecPcC56QcwtFQhEAwFGpQ4CjP7o06IAAnR/8HXyhlqtFrprqC4byeuqtt6bh5MnfsX37Pmg05fe9vfHGFPz115/Ytm0vVCpVNVdom/379yA9PQ0jRjwndilERFTDZBfmlOkDL/4vz5gHAFDJlPB38EVPvydK1gT3h5PKUeSKqT5jKK+n+vTpi99++wWHD/+EPn0irZ6/cycdJ078joiIfo8cyFev3gipVPq4pT5QdPReXLx4wSqUt2rVBtHRv0KhUFTp6xMRUc1QWGRAfPZNXC9zU560/HQAxW0oDey90MajZcldMf3hpfWAVFK136OIKoKhvJ564omnoNHYYf/+PeWG8gMH9qOoqAgREdbP2UqpVD5OiY9FKpXW2NF9IiJ6PCbBhOTcVPNNea5nxeNmTiJMggkA4KxyQoCjP5707YwAnT/8HHygkon3PYnIFgzl9ZRarcYTT3THwYP7kZWVBZ1OZ/H8/v174OrqCj+/hpg79984cSIGycnJUKvVaNOmHSZPfv2h/d/l9ZRfuXIZX3wxB2fP/glHR0cMGTIUbm7uVsf+8sshbNu2GRcunEdWVibc3T3Qv/8gjBnzImSy4gk2U6a8jD/+OAkA6NatuG/cy8sbGzZsv29PeXT0Xqxc+R2uX78GOzstunZ9Aq+++hqcnJzM+0yZ8jJycnLw/vsz8fnnsxEb+xccHHQYPnwURo8eV7EPmoiIHltmQbbFRMzrWQnIL8oHAKhlajTU+aKP/1PmyZiOKgeRKyaqOIZykcQkncT2K7uRnp8BZ5UTBgdGooNXm2qtoU+fSOzduwuHDkVj8OBnzNuTkhJx9uwZREWNQmzsXzh79gx69+4Ld3cPJCbewpYtGzF16itYuXI91GrbJ8Ckpd3Ga69NgslkwvPPj4NarcG2bZvLHdHeufNHaDR2GDlyNOzsNDhx4jiWLv0v9Ho9Jk9+HQAwbtx45OXlITk5EVOnvgEA0Gjuf3vi0gmloaFhePXV15CSkoyNG39AbOxfWLLke4s6srIy8fe/v4YePXqhV68IHDy4H4sXL0Djxk3QuXNXm98zERFVTEFRIW5kJZTcGbO4FeVOQQYAQCqRwsfeG+29WqOhzg+NdH7wsHNnGwrVCQzlIohJOonVcRthMBkAAHcKMrA6biMAVGswb9++I5ycnLF//x6LUL5//x4IgoA+ffoiMLAJevTobXFc165PYtKkF3HoUDQiIwfY/HqrVi1HZmYGli5dgeDgEABAv34D8eyzz1jt++GHH0Oluhv4n346CnPmfILNm9dj4sRXoVQq0b59J2zatB6ZmRno27f/A1/baDRi8eIFaNIkCAsW/M/cWhMcHIIPP3wH27dvRlTUKPP+KSnJ+OCDj82tPQMHDkFU1EDs2LGVoZyIqJKYBBMS9cnmG/Jcy4rHrZwkCBAAAK5qFzR2bIgAXTcEOPrD194HShnnClHdxFD+GI4lnsCRxN8rfNzVzBswCkaLbQaTAatiN+C3WzEVPl9n7/bo6N22wsfJ5XL07NkbW7ZsxO3bt+Hm5gYA2L9/L3x9/dC8eQuL/XqinZgAACAASURBVI1GI/T6HPj6+sHe3gEXLsRVKJQfOfIrwsLCzYEcAJydndGnTz9s3rzeYt+ygTw3V4/CQgPCw1tj69ZNuH79Gpo2DarQe42LO4c7d9LNgb5Uz5598NVXX+K33361COX29vbo3buv+bFCoUCzZqG4detmhV6XiIjuupOfUWYllBu4np2AwqJCAICdXIOGOj+0DGiOAJ0/Gur84KC0F7liourDUC6CewP5w7ZXpT59IrFp03ocOLAXI0Y8h2vXruLSpQt48cWJAICCgnysWPEddu7cjtTUFAiCYD42JyenQq+VnJyEsLBwq+3+/g2ttl25chlLlizGyZO/Q6/XWzyn11fsdYHilpzyXksqlcLX1w/JyYkW2z08PK3uzubgoMPly5cq/NpERPVRvjEfN7ITcC0zHtdKWlEyC7MAADKJDL4ODdDZu505gHto3HhXTKrXGMofQ0fvto80Qv3ur5+Y++PKclY54f/aTKqM0mwWFhYOb28f7Nu3GyNGPId9+3YDgLltY968Odi5czuGD38WLVqEwd7eHoAEH374T4uAXpmys7MxderLsLOzx4QJk+Dj4wulUokLF+KwePECmEymKnndsqT3uVtbVb1nIqLarMhUhFv65DJrgt9Akj7F3IbioXFDkHOgOYD7OjSAQsoIQlQWrwgRDA6MtOgpBwCFVIHBgY++/ODj6N07AitWfIuEhHhER+9FcHAz84hyad/41KnTzPsXFBRUeJQcADw9vZCQEG+1/caN6xaPT506gczMTMyaNQetWt3tsU9MvFXOWW0bVfHy8ja/VtlzCoKAhIR4NGoUaNN5iIjqO0EQkJ6fUWYi5g3cyL5p/p6mVdghQOePNh4t0VDnjwCdH7SK+0/CJ6JiDOUiKJ3MKfbqK6UiIvphxYpvsXDhPCQkxFsE8PJGjDdu/AFFRUUVfp3Onbti/fq1OH8+ztxXfufOHezbt8tiv9IbDpUdlTYYDFZ95wCg0Whs+gEhJKQ5nJ1dsGXLBvTrN9B8U6GDB6ORmpqC0aPHVvj9EBHVB7mGvOI2lJIlCa9lxSO7sPjfXblUDj97H3Rr0NF8V0xXtQvbUIgeAUO5SDp4tUEX33YwGqu+FeNhGjVqjCZNgnD48M+QSqXo1evuBMcuXbphz56d0GrtERDQCH/99SeOH4+Bo2PFb0X83HPjsGfPTrzxxmRERY2CSqXGtm2b4enpjZyci+b9wsJawsFBh1mzPkRU1EhIJBLs2bMT5XWOBAeHYO/eXViw4HOEhDSHRmOHbt2etNpPLpfj1Ven4pNPPsLUqa+gd+8IpKQkY8OGH9C4cSAGDbJeAYaIqL4xmoy4lZNkDt/XsuKRnJtift7TzgPNXYLNd8VsYO8FOdtQiCoFryQCAEREROLSpQto3bqteRUWAHj99TchlUqxb98uFBQUIiwsHF988RXeeGNqhV/Dzc0N8+f/D/PmzcaKFd9Z3Dzo3//+l3k/R0cnzJ49DwsXfoElSxbDwUGHiIh+aNeuA954Y4rFOYcMGYYLF+Kwc+eP+OGH1fDy8i43lANA//6DoFQqsWrVcnz11ZfQarXo0ycSkyZN5d0/iajeEQQBafnpxS0o2fG4lhmP+JybMJqKFx1wUNgjwNEPHUrWBG/o4Ac7hUbkqonqLokg4sy1wsJCfPnll9i6dSuysrIQEhKCadOmoXPnzg88bsGCBVi4cKHVdjc3N/z666+PVEtaWg5MpvI/iqSk6/Dysl4h5HHJ5dIaMVJOla+q/s7UV+7uDkhNzRa7DKIaKybpJLZd3o2Mggw43aclUm/IxbWseFwvGQW/nhWPHEPx6lYKqQL+Dj7miZgBOn+4qJ3YhkJ1lljfV6RSCVxdy1/qU9SR8unTp2Pv3r0YO3YsGjZsiM2bN2PixIlYsWIFWrdu/dDjZ86caXFHyYrcXZKIiKguuN8N6VJz06BV2JlXREnJuw0AkEACL60HwtyamwN4A60nZPdZdYqIqodoofzMmTPYsWMHZsyYgRdeeAEA8PTTT2PgwIGYO3cuVq1a9dBz9OvXDzqdroorJSIiqrm2Xt5lsZoXUHxDup3X9gEAHJU6BDj6o7N3ewQ4+sHPwRcaOQexiGoa0UL57t27oVAoMHz4cPM2lUqFqKgozJs3DykpKfDw8HjgOQRBQE5ODrRaLX/FRkREdVqeMQ+J+hQk6pOQpE9Boj4ZifpkZBRk3veYj7v8E85qp2qskogelWihPDY2Fo0aNYJWq7XY3rJlSwiCgNjY2IeG8qeeegq5ubnQarXo27cv3n77bTg58R8fIiKqvXINeUjKTUZiTjISS/6flJtiEb4VUgW8tR4Icg7En7fPIc+Yb3UeZ5UTAzlRLSJaKE9NTYWnp6fVdnd3dwBASkqK1XOldDodxowZg/DwcCgUChw9ehQ//PADzp07h/Xr10OpVFZZ3URERJUh15CLW/pkJJWMeCeVjIJnFt6dfKaUKuCl9USwcxN4az3hpfWAt9YLLmonSCXF93S4t6ccEPeGdET0aEQL5fn5+eYbuJRVujRdQUHBfY8dN26cxePIyEg0bdoUM2fOxJYtWzBixIgK13O/mbAAkJIihVwurfA5bVFV5yVxSaVSuLs7iF1GncLPk2qrnAI94rNuISEzCfFZt3AzKxHxmYnIyM8y76OSq+Cr80KrBqHwc/SGr84bvo4N4GbnbA7f9zPAvTt0Og3WnNmKtNx0uNq54NmWQ/BEww5V/daIarWa9n1FtFCuVqthMBistpeG8YquG/3ss89izpw5OHLkyCOF8gctiWgymapk6UIuiVh3mUwmLuFXibgkItUGOQZ9SatJckm/d/HId+ndLwFAJVPCS+uJEKcgeNt7wsuueOTbWe1oHb5zgbRcvU2vHWLXDB91amZxrfCaIbo/LolYhru7e7ktKqmpqQDw0H7ye0mlUnh6eiIz8/4TXh6HIAicTEo2EXHpfyKqBjmFeiTqk0pC9932k2zD3fCtlqngpfVEqGsIvLWe5v+cVVz7m4jKJ1ooDwkJwYoVK6DX6y0me54+fdr8fEUYDAYkJiaiRYsWlVonAMhkChgMBVAquYQUPZzBUAiZjDfLJartsgtzLEJ36X+lN9wBisO3t9YTLdyalfR8e6KB1hNOKkeGbyKqENGSQ2RkJL755husX7/evE55YWEhNm3ahDZt2pgngd66dQt5eXkIDAw0H5ueng4XFxeL8y1btgwFBQV44oknKr1We3tHZGTchlbrCLVaA6lUxn9syYogCDAYCpGRkQoHB2exyyEiGwiCgGxDDpL0ySWTLu8uOWgZvtXw1nqipVtzc/j2ZvgmokokWigPDw9HZGQk5s6di9TUVPj7+2Pz5s24desWPv30U/N+b7/9NmJiYnD+/Hnzth49eqB///4ICgqCUqnEsWPHsGfPHrRt2xYDBw6s9Fo1Gi3kcgVycjKg12fCZCp6rPPlFxYhr8CIIpMAmVQCjUoOtZJ3UqsLZDI5HBycodFoH74zEVUbQRCQVZhzd9TbvNRgMvSGXPN+Gnlx+A53DzUHb2+tJxyVOoZvIqpSov6Offbs2fjiiy+wdetWZGZmIjg4GF9//TXatm37wOMGDRqEkydPYvfu3TAYDPDx8cHf/vY3vPLKK5DLq+YtKRRKODtXrM+9PEf+SsLyXXEoLDPBUymXYly/EHQO9Xrs8xMR1WfF4Tvb3GpSdrlBvbFs+NbAW+uJVu5h5uDtpfVg+CYi0UgEzkoD8ODVVyrTPxb9irQs6+UeXXUqzPlb1yp/faLaiKuv0L0EQUBmYZbFnS1LQ3iuMc+8n11J+C7bcuKt9YRO6VAnwzevFSLbcPUVKjeQP2g7EVF9Vhq+7x35TtSnIK9M+NbK7eCl9UQbz3B4290N4TqlfZ0M30RU9zCUVzNXnarcAC6TSnD5ZiYCfRxFqIqISFyCICCjINMqeCflJlvcQl6rsIO31hPtPFvBS+uBBiXh20HB8E1EtRtDeTUb2j3QqqdcLpNAKZfikxUn0KOND4Z1D4RGxT8aIqp7SsO35e3liwN4ftHd8G2v0MJb64n2nq0t2k4clPe/+zIRUW3G5FfNSidzbvrpMtKzCuCiU2Fo90C0auKGzT9fQfSJBJy6eBvP9Q5C22B3kaslIno0giDgTkHGPf3eKUjSJyO/6O5vCx0U9vDSeqCDVxt4az3MbScM30RU33CiZ4nqmuhZVnmTDK7cysLy3XGIT8lB66ZuGN0nCC463rSI6jdOXqu5TIIJd/Izi9f2zk1BYk7xcoNJ+mQUFBWa93NQ2hf3ett7wsvu7si3vZLLh1YmXitEtqmJEz0ZykvUlFAOAMYiE/Ydj8fWX65CKpVgWPdA9GjtA6mU/ZJUPzFoiK84fFuPfCfmJqOwTPjWKR2sVjrx0nrAXsHwXR14rRDZpiaGcrav1EBymRT9OjZE22APrNhzHqv2XcCRv5IwLjIEfh78lS4RVR2TYEJ6/h3L4K1PRlJuikX4dlQ6wFvrhS7e7S1CuFZhJ2L1RES1F0fKS9SkkfKyBEHAsXPJWBN9Ebn5RvTt4I/BXQOgVPAOoFR/cPSv8pkEE9Ly7iCp5M6Wibl3Q7jBZDDv56jUmUe874ZvD9gxfNdIvFaIbMORcqowiUSCTqFeaNHYFesOXsLOo9fxe1wyxvYNQWgjF7HLI6IaziSYcDsv3XKZQX0SknJTLcK3k8oR3lpPdPPpeDeA23nCTqERsXoiovqDobyWsNcoML5/M3QJ9cLyPefx2Q9/oHOoJ0b2agqdnVLs8ohIZMXhOw2Jpe0mJSE8OTcFBpPRvF9p+H7COdBi5FsjZ/gmIhITQ3ktE9LQGTPHt8ePv13HzqPXceZyGkb1aoouLbx44wyiesAkmJCal1Zm5Ls0fKfCWCZ8O6uc4K31RJBzILy1XvDWesBL6wmNnKs5ERHVRAzltZBCLsMzTzZGh+aeWL47Dst2xOK3s0kY2zcYni7s8ySqC4pMRSUj33fvbHnf8G3viRDnpuaRby+tB8M3EVEtw4meJWrqRM+HMQkCfv7jFtYfugyD0YRBXQPQr6M/5DJpJVVJJL66PHmtyFSE1JLwXXb0OyU3FUahyLyfq9rZvMKJl9YTDbSe8LRzh5rhm8qoy9cKUWXiRE+qdFKJBE+19kGrpm5Ys/8iNv98BTHnkjEuMgRNfB3FLo+IShSH79v33F4+Bcm5qSiyCN8u8NZ6INQ1BF4ld7j0tPOAWq4SsXoiIqpqDOV1hJO9Cq8+3QKdL93Gyr3n8enKE3iqtQ+GdQ+EnZp/zETVxWgymke+7671nYyU3Nvm8C2BxDzyHeoaYl7txFPrAZWME7eJiOojprU6plUTN4T4O2HLL1ex73g8Tl5MxejeQWgb7M6JoESVyGgyIiX3tmXbSW4KUnJTYRJMAErCt6Z45DvMrTm87DxKbjPvASXDNxERlcFQXgeplXKM6tUUnUI98d2uOCzachatmrjh+YgguOjYf0pUEQaTESm5qRbrfCfqk5Gad9sifLtpXOCl9URLt+Z3R77t3Bm+iYjIJpzoWaK2TvR8mCKTCft+T8CWw1cgkUgw9InG6NXWF1IpR82p9qiOa6U0fJdtOSkO32kW4dtd41pmwqUHvLVeJeFbUaX1EdmCEz2JbMOJnlTtZFIpIjv6o12wO1bsvYA10Rdx9FwSxkWGwN/TQezyiKqdociA5LIj37kpSNQnITU3DQKKfzCXQAJ3O1d423mitXuYOYR72rlDwfBNRERVgCPlJerqSHlZgiDg97gUrN53ATl5RkR08MOQro2gUsqqrQaiR/Eo14qhyICkMuG77Mh3afiWSqRw17iWubNl8X8eGjeGb6qVOFJOZBuOlJOoJBIJOjTzRGgjF6w/eBm7j93A8bgUjO0bjBaNXcUuj8hKTNJJbLu8GxkFGXBSOWFwYCQ6eLWx2KewyIDk3JQybSfFI9+389LvCd9uaGDvjbae4eYQ7mHnDoWU/wwSEZH4OFJeoj6MlN/r/I07WL77PJLSc9GpuSdG9WoKnZaT0qhmiEk6idVxG2EwGczb5FI5Onm1g51Cg0R9EhL1KUi7J3x7aNysR77t3CBn+KZ6QOzvK0S1RU0cKWcoL1EfQzkAGIwm7Dx6HTuOXINKIcPwHk3wREtvLp9IVcZoMiLPmI9cYx7yjHnIM5T5umR7vjEfRxOPo7BMIC9LKpHCw869OHTbecDb3gtedh4M31Tv1YTvK0S1QU0M5fzuVc8p5FIM6dYIHZp5YPmuOHy3Kw5HziZhbGQwvF21YpdHNZDBZCwJ03nINeaXhOmyX5eEbEPx13n37GO4T9AuJZVIoZGr7xvIAeCL7rMgk3IuBBER1R0M5QQA8HbV4q3RbXD4TCLWHbiED76JwcDOAejXqSEUcqnY5VElMhQZ7hOmyx+1zisTtvOMeTCYjA88v1QihZ1cA41cDY1cAzu5Bk4qHTRyDTQKdclzxc9bfK3QQC1TQyVTQiKR4N1fP8Gdggyr8zurnBjIiYiozmEoJzOpRIInwxsgPNAVa6IvYsvhqzgWm4xxkSEI8nMSuzxC8Qo6BpOhTFjOL2fUurxgXRy484x5MJbc6v1+ZBJZcVhWFIdqjUwNZ7UT7EpCdnHQVluE6bIhWyFVVEr70+DASKuecoVUgcGBkY99biIiopqGPeUl6mtP+YOcuZyGFXvOIy0rH91bNcDwpwJhp+YycY9DEAQUmgzFI9QG61FoqxFsQ9nnir8uekiolktk0Cg05YxI3w3S5mCtuDdka6CQymvMnAJbVl8hortq+vcVopqiJvaUM5SXYCgvX0FhEbYcvoK9v8dDZ6fEs72bon2IR40JbdVNEAQUFBXef0TakI+8oruj0uXtU3p3yPtRSOXljkjfDdp3Q7X6nkBtJ1fXyfW1a8O1QlQT8Fohsk1NDOVsX6EHUillGNmzKTo198J3u+Pw361/4bezSXg+Ighujhqxy6uw4lBdYNUvnVvOiPS9ExRLg/XDQ7XCIijbK7XwsHMrd9Taok1EroZGVjdDNRERET0YR8pLcKT84YpMJkSfuInNP1+BAAFDn2iMXu18IZNW30RQk2CyHKk2lNM7XTZwl3m+dHvpmtb3o5QpywRnNcqdlCjXlLR+WLeGcEm+ylfbrhUisfBaIbINR8qpVpNJpYho74c2QW5YufcC1h64hCN/JeOFfiFo6OVg0zlMggn5xoL7jkLfOynRevWP/IeGapVMeXcUWq6Go8oRXlqvkgCtNofpsvuU/ZorexAREVF1YyinCjEJJtjZCRg9wBchl6XYFXMZH2+9hBZNdGjW2B4GoZzWkDJf5xsLHhqq1TKVxYREZ7UjGsi9rCYklo5km0ewFcXtHwzVREREVNswlNczJsFkGZYNd1f9yL/fqLW5TSQf+UX5lidsBCgBXABw4WrxJrVMXWapPDVc1M5WExQ1Je0fFhMZ5RqoZSqGaiIiIqp3GMpF8DjLvBWZipBXlF9ue0e5ExQNlkvu5RcVPPD8EkigLmnzUJeMQruqXeBrX6Z3upxl9JJTC7Hx4A0kpxoQ1swTz/ZqCkd7VWV8XERERER1Hid6lqiuiZ4xSSetbogik8jQwasNvLQeVrcuLztBMc+Yh4KiwgeeXwJJuSPSGpnavMpHuaPWcg3sFGqoZCpIJY82cdNgNGHXsev48bdrUMplGN4jEE+EN4C0ni6fSJWHk9eIbMNrhcg2nOhJ2HZ5t0UgB4AioQhHEn8HUByq7x2Rvnc5PfU9K37YlZm4qJIpHzlUPy6FXIrBXRuhfYgHvt99Hst3n8eRs0kYGxmCBm5aUWoiIiIiqg0YyqvZnYKM+z732ZMzoZKpav2NebxdtXjrudY4fCYR6w5ewgffxGBA54YY0DkACrk4PzAQERER1WQM5dXMWeVUbjB3VjlBLVeLUFHVkEgkeCK8AcKbuGHtgYvY9us1xMSmYFxkMIL9ncUuj4iIiKhG4bBlNRscGAmF1PKOjQqpAoMDI0WqqGrptEq8PCgUb4wIh7HIhP+sPoXvdsVCn294+MFERERE9QRHyqtZ6Sorj7r6Sm3VorEr/vVSR2w7fBV7YuLxx8XbeLZ3EDo086j17TpEREREj4urr5SortVXyqqvs+RvJGdj+e44XE3MRovGLhgTEQx3J43YZVENVl+vFaKK4rVCZJuauPoK21eo2vl7OuCdMe3wbO+muJiQifeWHcPuYzdQZDKJXRoRERGRKNi+QqKQSiXo084PbYPcsXLvBaw7eAlHzyVhXGQIGnnrxC6PiIiIqFpxpJxE5aJTY+qwMEx+pgUy9YX4+PvjWLP/IvILjWKXRkRERFRtOFJOopNIJGgb7IFmDV2w8efL2H88HicupOD5iGC0auImdnlEREREVY4j5VRj2KnlGBMRjBnPt4VGKcf8DWewaMtZZOQUiF0aERERUZViKKcap4mvIz54sT2GPtkYf1y8jXeWHMPBUzdh4kJBREREVEcxlFONJJdJMbBLAP41oQMCvBywYs95/HvlSdxMzRG7NCIiIqJKx1BONZqnix3eHNUKEwY0Q1J6Lj789nds+vkKDMYisUsjIiIiqjSc6Ek1nkQiQdcwb4QFuuKH6Ev48bdr+D02GWMjQ9CsobPY5RERERE9No6UU62hs1Ni4qDm+PvIVjAJAuasOYVvdsQiJ88gdmlEREREj4WhnGqd0EYumDmhI/p3aogjfyXhn18fxZG/kiBwIigRERHVUgzlVCupFDJEPRWI919oDw9nDZZsP4fP151GSkae2KURERERVZioobywsBBz5sxBt27d0LJlS4wYMQJHjhyp8HkmTpyI4OBgzJo1qwqqpJrMz8Me/3y+LUb3CcLlm5l4f+kx7Dx6HcYik9ilEREREdlM1FA+ffp0LF++HIMHD8Y777wDqVSKiRMn4tSpUzaf49ChQzh+/HgVVkk1nVQqQa+2vvj4pY5o0dgVGw5dxszvjuPKrSyxSyMiIiKyiWih/MyZM9ixYwfefPNNvPXWWxg5ciSWL18Ob29vzJ0716ZzFBYW4tNPP8WECROquFqqDVx0akwZGobJz4QhJ68Qs74/jlX7LiCvwCh2aUREREQPJFoo3717NxQKBYYPH27eplKpEBUVhRMnTiAlJeWh5/j++++Rn5/PUE4W2ga7Y9bETujZxhcHTiTg3aXHcOpCqthlEREREd2XaKE8NjYWjRo1glartdjesmVLCIKA2NjYBx6fmpqKRYsWYdq0adBoNFVZKtVCGpUcoyOC8M8xbaFVy7Fg05/4atOfuJNdIHZpRERERFZEC+Wpqanw8PCw2u7u7g4ADx0p//zzz9GoUSMMGTKkSuqjuiHQxxHvv9Aew7o3xpkraXhnyVEcOJkAE5dPJCIiohpEtDt65ufnQ6FQWG1XqVQAgIKC+49onjlzBlu2bMGKFSsgkUgqpR5XV/tKOU9Fubs7iPK69c0Lg8PQt0tjLNpwGiv3XsDv51MxZXgrBHjrxC6NbMRrhcg2vFaIbFPTrhXRQrlarYbBYH0nxtIwXhrO7yUIAmbNmoWIiAi0a9eu0upJS8uByVS9o6fu7g5ITc2u1tesz+QApg5tgaN/JWNN9EX83+eHENnRH4O6BECpkIldHj0ArxUi2/BaIbKNWNeKVCq570CwaKHc3d293BaV1NTiCXnltbYAwL59+3DmzBlMmzYNCQkJFs/l5OQgISEBbm5uUKvVlV801XoSiQSdW3ihRWMXrDtwCTuOXMfvcSkY2zcYzQNcxC6PiIiI6inRespDQkJw9epV6PV6i+2nT582P1+eW7duwWQyYdy4cejVq5f5PwDYtGkTevXqhZiYmKotnmo9BzslJgxsjjdHtQIAzF37B5b+eA7ZuYUiV0ZERET1kWgj5ZGRkfjmm2+wfv16vPDCCwCK1x3ftGkT2rRpA09PTwDFITwvLw+BgYEAgJ49e8LX19fqfJMnT0aPHj0QFRWF0NDQansfVLs1D3DBzPEd8OORa9h19AbOXE7DyJ5N0KWFV6XNVyAiIiJ6GNFCeXh4OCIjIzF37lykpqbC398fmzdvxq1bt/Dpp5+a93v77bcRExOD8+fPAwD8/f3h7+9f7jn9/PzQu3fvaqmf6g6lQoahTwaiQzNPLN8dh2U7YvHb2SSMjQyGp7Od2OURERFRPSBa+woAzJ49G2PGjMHWrVvx8ccfw2g04uuvv0bbtm3FLIvqKV93e8x4vi3GRAThWlIW3l8Wgx1HrsFYZBK7NCIiIqrjJILABZsBrr5Clu5kF2D1/gs4cT4VPu5avBAZgkAfR7HLqrd4rRDZhtcKkW1q4uoroo6UE9VUzg4qTH4mDFOHhSE334hPVpzAyr3nkVdgFLs0IiIiqoNE6yknqg1aN3VHiL8zNv98BdEnEnDyQipG9wlG22B3sUsjIiKiOoQj5UQPoVHJ8VyfILwzth0c7JT4avOfWLDxDNKz8sUujYiIiOoIhnIiGzVuoMN749pheI9A/HU1He8uPYb9x+OrfS4CERER1T0M5UQVIJdJ0a9jQ8x8qSMCfRyxev9FzFpxAvEpOWKXRkRERLUYQznRI/Bw0uCNEeF4eVBz3M7Mw0ff/o71hy6hwFAkdmlERERUC3GiJ9Ejkkgk6BTqhRaNXbHu4CXsOnoDx+NSMLZvCEIbuYhdHhEREdUiHCknekz2GgXG92+Gt55tDalUis9++ANLtv+FrNxCsUsjIiKiWoKhnKiShDR0xszx7TGoSwBiYlPwztdHcfhMInh/LiIiInoYhnKiSqSQy/DMk43x4fgO8HbT4pudsZiz5hSS0nPFLo2IiIhqMIZyoirg46bF9NFtMLZvMK4n5+D9ZTHY/utVGItMYpdGRERENVClTPQ0Go2Ijo5GZmYmevToAXd33u2QSCqR4KnWPmjV1A1r9l/E5l+u4lhsCsZFBqOpr5PY5REREVENYbwcLgAAIABJREFUUuFQPnv2bBw7dgwbN24EAAiCgBdffBHHjx+HIAhwcnLCunXr4O/vX+nFEtVGTvYqvPp0C3S+dBsr957HpytP4qnWPojq3hh2aoXY5REREVENUOH2lV9++QXt2rUzPz5w4AB+//13TJgwAZ999hkA4Ouvv668ConqiFZN3PDxSx0R0d4PP/1xE+8sPYbjcSmcCEpEREQVHylPSkpCw4YNzY8PHjwIX19fvPnmmwCAixcvYvv27ZVXIVEdolbKMapXU3QK9cR3u+KwaMtZhAe64vmIYLg6qsUuj4iIiERS4ZFyg8EAufxulj927Bi6dOlifuzn54fU1NTKqY6ojgrw0uG9ce0wokcTxN64g3eXHsO+3+NhMnHUnIiIqD6qcCj38vLCqVOnABSPisfHx6N9+/bm59PS0mBnZ1d5FRLVUTKpFJEd/fHxhI4I8nPCmuiL+Pj747ielC12aURERFTNKty+MmDAACxatAjp6em4ePEi7O3t0b17d/PzsbGxnORJVAFuThr83/CW+D0uBav3XcC/lh9HRHs/DOnWCCqlTOzyiIiIqBpUOJS/8sorSExMRHR0NOzt7fGf//wHOp0OAJCdnY0DBw7ghRdeqOw6ieo0iUSCDs08EdrIBesPXsbumBs4fj4FY/oGI6yxq9jlERERURWTCJW49IPJZIJer4darYZCUbuWektLy6n2fl53dwekprJVgaydv3EHy3efR1J6Ljo298SoXk3hqFWKXZZoeK0Q2YbXCpFtxLpWpFIJXF3ty3+uMl/IaDTCwcGh1gVyopom2N8ZH43vgCHdGuHE+RS8u+Qofj59i8snEhER1VEVDuU//fQTFixYYLFt1apVaNOmDVq1aoW///3vMBgMlVYgUX2lkEsxpFsjfDS+A3zctPhuVxxmrz6FxDS92KURERFRJatwKF+2bBmuXLlifnz58mV88skn8PDwQJcuXbBz506sWrWqUoskqs+8XbV4a3QbvNAvBPEpOfjgmxhsO3wVBqNJ7NKIiIioklQ4lF+5cgUtWrQwP965cydUKhU2bNiApUuXon///tiyZUulFklU30klEjwZ3gCzJnZEmyB3bDl8FR9+G4ML8Rlil0ZERESVoMKhPDMzE87OzubHv/32Gzr9f3t3Hh91neX7/12VVLbKnlRlT9gTSEhYZTNxAVu09WLTcu1RcHfsRmdcHj3To97p++vpnof9c+xuHW1bBW8rXrttRRTk4YKAbYJsypKNhCUgECoklUAI2be6fySUhASsYOBbSV7Pf/LgU0tO9HGow8n5nu/MmQoO7hpav+KKK1ReXj5wEQJwCwv2108XZOjRRVlqbevUb9/aqdc/LlVDMyNjAAAMZv0uyiMiIuRwOCRJ9fX1Kiws1LRp09yPt7e3q6OjY+AiBNBL5ugo/eb+Gbr+iiTlFTj01LJt2l5SyYWgAAAMUv3eUz5p0iS9/fbbGjNmjHJzc9XR0aGcnBz344cPH5bdbh/QIAH05u/no9uuHauZE2L1+ielenl1sTYXHdfiH4xTdFig0eEBAIB+6Hen/J//+Z/V2dmpRx99VKtWrdItt9yiMWPGSJJcLpfWr1+vKVOmDHigAPqWEhui/3XnVP3k2jHae6RW/2v5Nn26/Yg6OrkQFACAweKibh5UW1urnTt3KiQkRNOnT3efnzp1Sh988IFmzJihtLS0AQ30UuPmQRgKqk816f+u26eCshqlxITo7hvSlBIbYnRY3xu5AniGXAE84403DxrQO3oOZhTlGCpcLpe+3uvUW5/t0+nGVl03LUm3ZI9UgF+/p9W8BrkCeIZcATzjjUX5RX9KHzlyRBs2bNDRo0clSUlJSZo7d66Sk5Mv9i0BDACTyaTpaXalj4jQyr+Xad1XR7Vjb5WWXJ+qzNHRRocHAAD6cFGd8ueee07Lli3rtWXFbDbrwQcf1COPPDJgAV4udMoxVO07Wqs3PilVRU2jpqfZdfu8sQoL9jc6rH4hVwDPkCuAZ4ZEp3zlypV6+eWXNXnyZN1///0aO3asJGn//v167bXX9PLLLyspKUkLFy78flEDGBDjksL1/91zhT7edlhrN3+j4kMndOs1o5WTFS+zyWR0eAAAQBfRKV+4cKEsFoveeust+fr2rOnb29t1xx13qK2tTatWrRrQQC81OuUYDipqGrTik73ae7RWYxPDdNf8NMVHW40O6zuRK4BnyBXAM97YKe/3SsSysjLdeOONvQpySfL19dWNN96osrKy/kcJ4JKLi7LqX2+frHtuSJOjukH/+/9s1wd5B9XWzg2/AAAwUr/HVywWixobG8/7eENDgywWy/cKCsClYzKZlJ0Vr6wx0Xp7w36t+fIbbS+p0l3zU5WaHGF0eAAADEv97pRPnDhRf/vb31RdXd3rsZqaGr3zzjvKysoakOAAXDqhVj/94/9I1+P/M0vtHZ36//+yS3/+qET1TW1GhwYAwLDT75nyr776SnfffbesVqt+/OMfu+/meeDAAa1atUoNDQ16/fXXNW3atEsS8KXCTDmGs5a2Dq3ZdEifbj+q4EBf/WTeWM0YHyOTl1wISq4AniFXAM9440z5Ra1E3Lhxo37961+roqKix3l8fLx++ctf6uqrr76oQI1EUQ5IRypP641PSnWo4rQyRkZqyfWpsoUHGh0WuQJ4iFwBPDNkinJJ6uzsVFFRkcrLyyV13TwoPT1d77zzjlasWKGPPvro4iM2AEU50KWz06UNO8u1KvegXJ0uLcgeqR9MT5KPud/TbgOGXAE8Q64AnvHGovyi7+hpNpuVmZmpzMzMHucnT57UoUOHLvZtARjMbDbpumlJmjrOpv+7bp/e/bxM24orddcNaRoZF2p0eAAADEnGtb4AeLXI0AD9048n6qEfZehUY6t+s+Jr/WX9PjW1tBsdGgAAQ85Fd8oBDH0mk0lTU+0anxKp974o04avy7Vzn1OLr0vVpLHRRocHAMCQQaccwHcKCvDVkutT9cTiqQr089V/v1egl94vVG19i9GhAQAwJFCUA/DYmMQw/e97pmthzijtPlCjp5Zt1ee7jqnz4q4XBwAA3TwaX/nzn//s8Rvu3LnzooMB4P18fcy6afYITU+z641PSvXmp3u1pei47pqfqgRb31eUAwCAC/NoJWJaWlr/3tRkUklJyUUHZQRWIgL953K5tLnouP628YCaWtp1w8wU3Tw7RRZfnwH9PuQK4BlyBfDMoF2JuGLFigENCMDQYDKZNGdinCaOjtLfNhzQ2s3f6KuSSt05P03jUyKMDg8AgEHjom8eNNTQKQe+v+JDJ7Ti01I5a5s1Z2Ksbrt2rIIDLd/7fckVwDPkCuAZb+yUc6EngAGTPjJS/3HfDN04M0Vbiyv15KtbtaXouPi3PwAAF0ZRDmBA+Vt8dOvVo/XLu6fLHhGoZWv36Pd/262qk41GhwYAgNeiKAdwSSTZg/Xk4qm647pxKnPU6d9f266Pth5We0en0aEBAOB1DL2jZ2trq55//nmtXr1adXV1SktL02OPPaZZs2Zd8HVr1qzRypUrVVZWplOnTslut2vGjBl6+OGHlZCQcJmiB/BdzGaT5k5N1OSx0frL+v1a+fcybS2u1F03pGp0fJjR4QEA4DUMvdDz8ccf17p163TnnXcqJSVF77//voqKivTmm29q8uTJ533dM888I6fTqbS0NIWFhcnhcOidd95RR0eH1qxZI5vN1u9YuNATuPR27HXqrc/26lR9q66dmqiFOaMU6P/dvQFyBfAMuQJ4xhsv9DSsKC8oKNCiRYv0xBNP6O6775YktbS06KabbpLdbtdbb73Vr/crLi7WwoUL9a//+q+67777+h0PRTlweTS1tGvVFwe1cWe5wkP8dcd14zRl3IX/IU2uAJ4hVwDPeGNRbthM+SeffCKLxaJFixa5z/z9/XXrrbdqx44dqqqq6tf7xcfHS5Lq6uoGNE4AAyvQ31d3/GCcnlwyVdYAX724qlAvrirUydMtRocGAIBhDCvKS0pKNHLkSFmt1h7nmZmZcrlcHt0RtLa2VjU1NSosLNQTTzwhSd85jw7AO4xOCNMv756uH181SoUHa/TUsq3asKP8sv/GCgAAb2DYhZ5Op1MxMTG9zs/Mg3vSKb/++utVW1srSQoPD9cvf/lLzZw5c2ADBXDJ+PqY9cNZIzQ9za4Vn+7VW5/t09bi47prfpoS7X3/eg8AgKHIsKK8ublZFkvvO/35+/tL6pov/y4vvviiGhsbdejQIa1Zs0YNDQ0XHc/55nsuNZstxJDvC3gTmy1Evx1r1993lmv56iL96vWvtPCaMYqLtuqv6/aq+mSToiMCdecN43X11CSjwwW8Gp8rgGe8LVcMK8oDAgLU1tbW6/xMMX6mOL+Q6dOnS5KuuuoqzZ07VzfffLOCgoK0ePHifsfDhZ6A8TKSw/Xr+67QOxsP6N0N+3s85jzZpBfe2a26082alR5rUISAd+NzBfAMF3qexWaz9Tmi4nQ6JUl2u71f75eUlKT09HR9+OGHAxIfAGOEBPnpvpsmKDSo92/SWts7teqLMgOiAgDg0jKsKE9LS9OhQ4d6jZzk5+e7H++v5uZmnT5NhwAYCuoae/8mTZJq6lpU39T3YwAADFaGFeXz589XW1ub3n33XfdZa2urVq1apSlTprgvAnU4HCor69kZO3HiRK/3KyoqUmlpqdLT0y9t4AAui6jQ84+wPf7il3plTbFKvjmhTuPufwYAwIAxbKY8KytL8+fP17PPPiun06nk5GS9//77cjgcevrpp93P+8UvfqHt27dr79697rNrrrlGN9xwg8aNG6egoCAdOHBA7733nqxWq5YuXWrEjwNggC28arTe+LhUre2d7jM/X7Nump2iU/Vt2lJ8XNv2VCo6LEDZWfG6cmKcIkK++1oUAAC8kWFFuSQ988wzeu6557R69WqdOnVKqampevXVVzV16tQLvu7222/Xli1btH79ejU3N8tms2n+/PlaunSpkpLYzAAMBWcu5lz1RZlO1LUoMtRfC68a7T5fdM1o7dznVG6+Q+/nHtQHeQeVOSpKOVnxmjg6Sr4+hv0iEACAfjO5XPzuV2L7CuDNvitXqk42Kq+gQpsKK3SqvlWhVj/NmRirnMx4xUQGXcZIAWPxuQJ4xhu3r1CUd6MoB7yXp7nS0dmpwrITys13qKCsRp0ul1KTwpWdFaepqXb5W3wuQ7SAcfhcATxDUe7FKMoB73UxuVJb36IvCyuUl1+hqtomBfr7amZ6jHIy45US6103jAAGCp8rgGe8sSg3dKYcAC6V8GB//XDWCN0wM0X7jtQqt8ChTQUV+nznMSXHBCsnK14zJ8QoKKD3PnQAAC43OuXd6JQD3mugcqWhuU1biyuVm+/Q0ap6WXzNmpZqV05WnMYlhctkMg1AtIBx+FwBPEOnHAAMZA2waO7URF07JUGHK08rL79CW/cc15bi44qJCFR2VrzmZMQqLJjVigCAy4tOeTc65YD3upS50tLWoa9Lq5SX79C+8lMym0zKGtO1WjFjVKR8zKxWxODB5wrgGTrlAOBl/C0+mjMxTnMmxqmipkGbCir0ZWGFdu2vVniwn67MjNOVmfGyhwcaHSoAYAijU96NTjngvS53rrR3dCr/QI3yChwqPFgjl0sanxLRtVpxnE0WX1YrwjvxuQJ4hk45AAwCvj5mTU21aWqqTSfqmrtWKxZU6NU1e2QN8NWs9FhlZ8Uryd73X6wAAPQXRTkAXEBkaIBunjNSP5w9QiWHTyov36G/7z6m9TvKNTIuRNlZ8ZoxPkaB/vx1CgC4eHyKAIAHzCaT0kdEKn1EpOqb2rSl6LhyCxxa8clevb1hv65Ii1FOVrxGJ4SyWhEA0G8U5QDQT8GBFl03PUnzpiXqYEWd8vIrtK2kUpsKKxQXFaTszHjNnhir0CA/o0MFAAwSXOjZjQs9Ae81GHKlubVdX5VUKbfAobJjdfIxmzR5bLSys+KVPiJSZjPdc1x6gyFXAG/AhZ4AMEQF+PkqOyte2VnxOlbdoLx8hzYXHdfXe52KDPXXlRPjdGVmnKLDWK0IAOiNTnk3OuWA9xqsudLW3qndB6qVm+/QnkMnJEnpIyOVkxWvSWOj5evDjYkwsAZrrgCXG51yABhGLL5mTU+za3qaXdWnmrSpoEKbCiv00gdFCg60aHZG12rFhGir0aECAAxGp7wbnXLAew2lXOnsdKn4mxPKy3do1/5qdXS6NCYhTNmZcZo+3q4AP3oluHhDKVeAS4lOOQAMc2azSRNHRWniqCjVNbRqc9Fx5RU49OePS/WXDfs1Y3zXasWRcSGsVgSAYYSiHAAMEmr10/wZybr+iiQdOHZKufkObd1zXLn5DiXYrMrJjNesjFgFB1qMDhUAcIkxvtKN8RXAew2nXGlqade2kkrl5Tt0qOK0fH1MmjLOpuyseI1PiZCZ7jkuYDjlCvB9ML4CALigQH9fXT0pQVdPStDRqnrl5Tu0pfi4tpdUKTosQNmZcZozMU6RoQFGhwoAGEB0yrvRKQe813DPlbb2Du3Y51RefoVKDp+UySRNHBWl7Mx4ZY2JYrUi3IZ7rgCeolMOAOg3i6+PZk6I1cwJsaqqbdKmAoc2FVToj+8XKtTqpzndqxVjI4OMDhUAcJHolHejUw54L3Klt47OThUe7FqtmH+gRp0ul8Ylhik7K17T0uzyt/gYHSIMQK4AnvHGTjlFeTeKcsB7kSsXVlvfos1FXVtbqk42KdC/q7OekxWvlNgQo8PDZUSuAJ7xxqKc8RUAGOTCg/1148wU3TAjWfuO1io3v+vOoZ/vOqZke7Cys+I1Mz1G1gBWKwKAt6JT3o1OOeC9yJX+a2xu09Y9lcrNd+hIZb0svmZNS7UpJyte45LCuTHREEWuAJ6hUw4AuCyCAiy6dkqirp2SqMPHTyu3wKGtxZXaUlwpe0Sge7VieLC/0aECAESn3I1OOeC9yJWB0dLWoR17q5SbX6F9R2tlNpmUNaZrteLE0ZHyMbNacbAjVwDP0CkHABjG3+Kj2Rlxmp0Rp+MnGpVX4NCXhce1a3+1woL9dOXEOGVnxskewWpFALjc6JR3o1MOeC9y5dJp7+hUQVmNcvMdKjxYI5dLSksOV05WvKam2mTxZbXiYEKuAJ6hUw4A8Cq+PmZNGWfTlHE2nTzdok2FFcrLd+jVD/fI+pmvZqZ3rVZMsvf9IQIAGBgU5QAASVJEiL9unj1CP5yVotLDJ5VXUKEvdh/Thh3lGhkXouzMeM2YEKNAfz46AGCg8TcrAKAHs8mkCSMiNWFEpOqbxmlLcdeNiVZ8uldvb9yv6Wl25WTFa0xCGKsVAWCAUJQDAM4rONCi66Ylad7URB2qOK3cfIe2lVTqy8Ljio0MUk5WvGZnxCrU6md0qAAwqHGhZzcu9AS8F7niXZpb2/VVaZXy8it04Ngp+ZhNmjQ2WtmZ8coYGSmzme65UcgVwDNc6AkAGPQC/HyVnRmv7Mx4Oaob3KsVd+x1KiLEX9mZcbpyYpyiwwONDhUABg065d3olAPei1zxfu0dndq9v1q5+Q4VHzohSZowMlLZmXGaPNYmiy83JrocyBXAM3TKAQBDkq+PWdPS7JqWZlfNqWZtKqzQpgKHXl5drOBAi2ZnxCo7M04JNlYrAkBf6JR3o1MOeC9yZXDq7HRpzzcnlFtQoV37nOrodGl0fKiys+J1xXi7AvzoCw00cgXwDJ1yAMCwYTablDEqShmjolTX2KotRV2rFV//uFR/3bBfM8bblZ0Zr1HxoaxWBDDsUZQDAC650CA/XX9Fsn4wPUllx+qUW+DQ1j2Vys2vUEK0VdlZ8ZqVHqOQIFYrAhieGF/pxvgK4L3IlaGpqaVd20u6CvNDFXXy9TFpyjibsrPiNT4lQma65/1GrgCeYXwFAIBugf6+umpSgq6alKDyqnrlFji0pei4tpdUKTosQFd2r1aMDA0wOlQAuOTolHejUw54L3Jl+Ghr79DOfV2rFUsOn5TJJE0cFaXszDhljYmWrw+rFS+EXAE8Q6ccAIALsPj6aMaEGM2YEKOq2iZtKuharfjH92sUGmTR7Ilxys6MU1yU1ehQAWBA0SnvRqcc8F7kyvDW0dmpooMnlJvvUEFZjTo6XRqbGKacrHhNS7XL38/H6BC9BrkCeMYbO+UU5d0oygHvRa7gjFP1LdrcvVqx8mSTAv19NGNCrHKy4pQSEzLsVyuSK4BnvLEoZ3wFADBohAX764aZKZo/I1n7jtYqr6BCmwsr9Pddx5RkD1ZOVrxmpsfIGmAxOlQA6Bc65d3olAPei1zBhTQ2t2lb987zw5Wn5etj1rQ0m7Iz45WaHD6sViuSK4Bn6JQDADDAggIsumZKoq6ZkqjDx08rr8ChLcWV2lpcKXt4oLKz4jQ7I04RIf5GhwoA50WnvBudcsB7kSvor9a2Du3Y61RuvkN7j9bKbDIpc3SUsrPilDk6Sj7moblakVwBPEOnHACAy8DP4qNZGbGalRGryhONyiuo0JeFFdp9oFphwX66cmKcrsyMU0xEkNGhAoAkgzvlra2tev7557V69WrV1dUpLS1Njz32mGbNmnXB161bt04fffSRCgoKVFNTo7i4OF1zzTVaunSpQkJCLioWOuWA9yJXMBDaOzpVWFajvIIK5ZdVy+WS0pLDlZ0Vr6njbPKzDP7ViuQK4Blv7JQbWpQ//vjjWrdune68806lpKTo/fffV1FRkd58801Nnjz5vK+bMWOG7Ha75s2bp/j4eO3du1dvv/22RowYoffee0/+/v2fG6QoB7wXuYKBdvJ0i74srFBegUPO2mYF+ftqVnqssrPilBxzcc0db0CuAJ6hKD9LQUGBFi1apCeeeEJ33323JKmlpUU33XST7Ha73nrrrfO+dtu2bZoxY0aPsw8++EC/+MUv9PTTT2vhwoX9joeiHPBe5AoulU6XS3sPn1ReQYW+3utUe0enUmJDlJMVrxnjYxQUMLimPMkVwDPeWJQb9rfNJ598IovFokWLFrnP/P39deutt+oPf/iDqqqqZLfb+3ztuQW5JM2bN0+SVFZWdmkCBgAMOWaTSeNHRGr8iEjd3tSmrcVdNyZ689O9+tuG/ZqeZld2VrzGJoYN+xsTAbi0DCvKS0pKNHLkSFmt1h7nmZmZcrlcKikpOW9R3pfq6mpJUkRExIDGCQAYHoIDLZo3LUlzpybqm+OnlZfv0NY9lfqy6LhiIoOU071aMczqZ3SoAIYgw4pyp9OpmJiYXuc2m02SVFVV1a/3W7ZsmXx8fPSDH/xgQOIDAAxPJpNJI+NCNTIuVLddO1ZflVYpt8Chdz8v06ovDmrSmGhlZ8UpY2SUzGa65wAGhmFFeXNzsyyW3rdBPnORZktLi8fv9eGHH2rlypV68MEHlZycfFHxnG++51Kz2QbvBUXA5USuwCiJCeH60dxxOlp5Wp9tP6KNXx/Rjn1ORYcFaO4VybruihTFRHrPakVyBfCMt+WKYUV5QECA2traep2fKcY93aDy9ddf66mnntLVV1+tRx555KLj4UJPwHuRK/AGAWbp5pnJumF6onbvr1ZeQYXe+Wyf3vlsnyaMiFB2Vrwmj7XJ4mvcjYnIFcAzXOh5FpvN1ueIitPplCSP5slLS0v1s5/9TKmpqfrDH/4gH5/Bv2MWAODdfH3MmpZm17Q0u2pONbtXK768uljBgRb3asVEmzG/gQUwOBlWlKelpenNN99UQ0NDj4s98/Pz3Y9fyJEjR3T//fcrMjJSr7zyioKCvOdXhwCA4SEqLED/48qRumn2CO05fEK5+RXauLNcn319VKPiQ5WTFa/paXYF+g+u1YoALj/Dfsc2f/58tbW16d1333Wftba2atWqVZoyZYr7IlCHw9FrzaHT6dS9994rk8mk1157TZGRkZc1dgAAzmY2m5QxMkpLb8nQ7x6eo59cO0bNrR16/eNSPf7il/rzRyUqO3ZKBt6vD4CXM/SOno888og2bNigu+66S8nJye47er7xxhuaOnWqJGnJkiXavn279u7d637dggULVFpaqvvvv1/jxo3r8Z7JyckXvBvo+TBTDngvcgWDkcvlUpmjTnn5Dm0vqVJLW4fio63KyYzTrIxYhQQN/GpFcgXwDDPl53jmmWf03HPPafXq1Tp16pRSU1P16quvugvy8yktLZUkLV++vNdjP/rRjy6qKAcAYCCZTCaNSQjTmIQw/WRu92rFfIfe3nhA7/69TFPG2ZSdFacJIyJl5sZEwLBnaKfcm9ApB7wXuYKhpNxZr7z8Cm0uqlBDc7uiQgOUnRmnKzPjFBka8L3em1wBPOONnXKK8m4U5YD3IlcwFLW1d2rXfqdy8x3a881JmSSlj4pUTma8Jo2Nlq9P/y/7IlcAz3hjUc7l4AAAGMDia9YV42N0xfgYOWubtKmgQpsKK/TSB0UKCbJoTkacsrPiFBdl/e43AzDo0SnvRqcc8F7kCoaLzk6Xig7VKC+/QrsPVKuj06UxiWHKyexarejvd+H7cZArgGfolAMAgPMym03KHB2tzNHROtXQqs1FFcrNr9D/+ahEf1m/TzMnxCg7K14jYkNk4uJQYEihU96NTjngvcgVDGcul0v7y08pL9+hr0qr1NreqURbsHKy4jQzPVbBgRZtKT6uVV+U6URdiyJD/bXwqtGalR5rdOiA1/LGTjlFeTeKcsB7kStAl8bmdm0rqVRuvkOHj5+Wr49ZKTHBOlx5Wu0d336G+fmaddcNaRTmwHl4Y1HO+AoAAINEUICvrpmcoGsmJ+hI5Wnl5Vdo485yndtSam3v1HtflFGUA4MIRTkAAINQckyI7vhBiDbsLO/z8RN1LfrVn79Sgs2qBJtVibZgJURbFRHizzw64IUoygEAGMSiQv1VU9fS6zzAz0fBQRbt+eaENhcdd58H+ft+W6Sf9dUaYLk2bh3+AAAQ8klEQVScYQM4B0U5AACD2MKrRuuNj0vV2t7pPvPzNWvJ9anu8ZX6pjYdc9ar3NmgY9UNKnfWa+ueSjW1tLtfExHir4TonsV6XFSQ/CwXXsMIYGBQlAMAMIidKbwvtH0lONCi1OQIpSZHuM9cLpdOnm7pKtTPFOzOeq0/Uq72jq4C32SS7BFBSoy29uiq2yMC5WPu/x1HAZwf21e6sX0F8F7kCuCZgciVjs5OVZ1s0jFnV0f9zNeqk03uC0p9fcyKjw5SQnSwEu3Wrq825tUxeLB9BQAAeDUfs1lxUVbFRVk1Lc3uPm9t61BFTWOPQr30yEltKf52Xj3w7Hn1aKsSbVYl2IIVHMi8OvBdKMoBAMB38rP4KCU2RCmxIT3O65va5Kju2VXfds68eniwnxJsXd30M931uCir/JlXB9woygEAwEULDrRoXFK4xiWFu8/OzKsfO6dY37Cj9tt5dUn2iMBvi/Xur8yrY7iiKAcAAAPKZDIpMjRAkaEBmjgqyn3ea169ukHlzgbt2u/UmSvcfH3Mio8K6rW2kXl1DHUU5QAA4LLweF69ul6lR2q1pbjS/Rz3vHq0tUd3nXl1DBUU5QAAwFDnm1dvaG7TsXNWNm4vqVLjbof7OWHBfu4LS8901eOjmVfH4ENRDgAAvJI1oO959dr61h6z6secDfp81zG1tX87r26LCOx1M6SYSObV4b0oygEAwKBhMpkUEeKviBD/HvPqnZ0uVdU29eiqlzsbtPtA9Vnz6ibFRVnPWdsYrMhQ5tVhPIpyAAAw6JnNJsVGBik2MkhTU789b2vvkKO6UceqzxTrDdp7pFZbe8yr+yghOrhnsW5nXh2XF0U5AAAYsiy+fc+rNza3dRXpZ61t/Lq0Sl+cPa9u9XNfUOqeV4+yyt+PeXUMPIpyAAAw7ARdYF69xwhMdR/z6uGBSrD13AITExEoXx/m1XHxKMoBAADUc14945x5dWdtU8+LS6t7z6vHRlqVaLf2uMA0KjSAeXV4hKIcAADgAsxmk2IigxTTx7x6j/3qzgbtO9pzXj3Az6erqx7d886lIUF+Bvwk8GYU5QAAABfB4uuj5JgQJcf0nlc/c7fSY90F+469VcrNb3c/J/TMvPpZxXpCNPPqwxlFOQAAwAAKCrBobGK4xib2nFc/1dB7v/oXu4+p9ax59ejwgB671ROirYqJDGJefRigKAcAALjETCaTwoP9FR7sr4yR58yrn2pSeVXDWWsb65V/oEad3QPrPmaT4qKC3MV6gi1YidFWRYUxrz6UUJQDAAAYxGw2KSYiSDERQZqaanOfn5lXP+ZsUHl1V1d9f3mttu45Z149+qyVjdFWJdiDFcq8+qBEUQ4AAOBlzj+v3i7HWbvVy531vefVgyw9dqt3XWhqVYAfZZ834/8OAADAIBEU4KsxiWEakxjmPjszr372rHq5s165ux3ueXVJig47Z17dZlUs8+peg6IcAABgEDt7Xj19ZKT7/My8+rnFekFZz3n12DPz6mfvVw8LkJl59cuKohwAAGAIOntefcq4s+fVO3X8RGOPQv1A+SltO2te3b97Xv3ctY2hVubVLxWKcgAAgGHE4mtWkj1YSfbgHudNLe3d+9W7ivVjznrt3Fet3PwK93P6mlePj7Iq0J+S8vvivyAAAAAU6O+rMQlhGpPQc169rqFV5dUNOlZV3/XVWa/cfIda2/qeV+/aBBOs2Cjm1fuDohwAAAB9MplMCgv2V1iwv9JHnDWv7nKpuvasefXuO5j2mlePDPp2t3r312jm1ftEUQ4AAIB+MZtMskcEyR4RpMl9zKsfc357I6SyY3XaXlLlfo6/xUfxZ+bVzyrWw4b5vDpFOQAAAAbEd82rn12s79pfrbyCb+fVQ4IsPTbAJNqCFR89fObVh8dPCQAAAMOcd169sa3HFphjzgblFvSeV0+ItirR/u3axqE4r05RDgAAgMvOZDIpzOqnMGtk73n1U809Liw95mxQ0aET6ug8z7x6tFUJ9u+eV99SfFyrvijTiboWRYb6a+FVozUrPfaS/6yeoCgHAACA1zCbTLKHB8oeHthjXr29o1PHaxpVXn1mZWODDjp6zqv7WcxKiO5ZqCdGWxVq9dPWPZV64+NS911Oa+pa9MbHpZLkFYU5RTkAAAC8nq+PWYn2YCX2Ma/uqG7o2gBT1bUJJv9AtTadNa8eHGhRc2u72jtcPV7b2t6pVV+UUZQDAAAA30egv69GJ4Rp9Fnz6pK69qufNa9+9kWlZ6upa7kcYX4ninIAAAAMOaFWP02wRmpC97z6nm9O9FmAR4X6X+7Q+jS0LlsFAAAA+rDwqtHy8+1Z+vr5mrXwqtEGRdQTnXIAAAAMeWfmxtm+AgAAABhoVnqsZqXHymYLkdN52uhwemB8BQAAADAYRTkAAABgMIpyAAAAwGAU5QAAAIDBKMoBAAAAg1GUAwAAAAajKAcAAAAMRlEOAAAAGIyiHAAAADAYd/TsZjabhtX3BQYbcgXwDLkCeMaIXLnQ9zS5XC7XZYwFAAAAwDkYXwEAAAAMRlEOAAAAGIyiHAAAADAYRTkAAABgMIpyAAAAwGAU5QAAAIDBKMoBAAAAg1GUAwAAAAajKAcAAAAMRlEOAAAAGMzX6ACGm6qqKq1YsUL5+fkqKipSY2OjVqxYoRkzZhgdGuA1CgoK9P7772vbtm1yOBwKDw/X5MmT9eijjyolJcXo8ACvUVhYqJdffll79uxRTU2NQkJClJaWpoceekhTpkwxOjzAqy1btkzPPvus0tLStHr1aqPDoSi/3A4dOqRly5YpJSVFqamp2rVrl9EhAV5n+fLl2rlzp+bPn6/U1FQ5nU699dZbuuWWW7Ry5UqNHj3a6BABr3D06FF1dHRo0aJFstlsOn36tD788EMtXrxYy5Yt05w5c4wOEfBKTqdTf/rTnxQUFGR0KG4ml8vlMjqI4aS+vl5tbW2KiIjQ+vXr9dBDD9EpB86xc+dOZWRkyM/Pz332zTff6Oabb9YPf/hD/fa3vzUwOsC7NTU1ad68ecrIyNArr7xidDiAV/q3f/s3ORwOuVwu1dXVeUWnnJnyyyw4OFgRERFGhwF4tSlTpvQoyCVpxIgRGjt2rMrKygyKChgcAgMDFRkZqbq6OqNDAbxSQUGB1qxZoyeeeMLoUHqgKAcwKLhcLlVXV/OPWqAP9fX1OnHihA4ePKjf//732rdvn2bNmmV0WIDXcblc+vWvf61bbrlF48ePNzqcHpgpBzAorFmzRpWVlXrssceMDgXwOk8++aQ+/fRTSZLFYtFPfvIT/fSnPzU4KsD7fPDBBzpw4ID++Mc/Gh1KLxTlALxeWVmZ/uM//kNTp07VggULjA4H8DoPPfSQbrvtNh0/flyrV69Wa2ur2traeo2BAcNZfX29fve73+kf//EfZbfbjQ6nF8ZXAHg1p9OpBx98UGFhYXr++edlNvPXFnCu1NRUzZkzRz/+8Y/12muvqbi42OvmZQGj/elPf5LFYtE999xjdCh94tMNgNc6ffq0HnjgAZ0+fVrLly+XzWYzOiTA61ksFs2dO1fr1q1Tc3Oz0eEAXqGqqkpvvPGGbr/9dlVXV6u8vFzl5eVqaWlRW1ubysvLderUKUNjZHwFgFdqaWnRT3/6U33zzTd6/fXXNWrUKKNDAgaN5uZmuVwuNTQ0KCAgwOhwAMPV1NSora1Nzz77rJ599tlej8+dO1cPPPCAfv7znxsQXReKcgBep6OjQ48++qh2796tl156SZMmTTI6JMArnThxQpGRkT3O6uvr9emnnyouLk5RUVEGRQZ4l8TExD4v7nzuuefU2NioJ598UiNGjLj8gZ2FotwAL730kiS59y2vXr1aO3bsUGhoqBYvXmxkaIBX+O1vf6uNGzfqmmuuUW1tbY+bOlitVs2bN8/A6ADv8eijj8rf31+TJ0+WzWZTRUWFVq1apePHj+v3v/+90eEBXiMkJKTPz4433nhDPj4+XvG5wh09DZCamtrneUJCgjZu3HiZowG8z5IlS7R9+/Y+HyNPgG+tXLlSq1ev1oEDB1RXV6eQkBBNmjRJ9957r6644gqjwwO83pIlS7zmjp4U5QAAAIDB2L4CAAAAGIyiHAAAADAYRTkAAABgMIpyAAAAwGAU5QAAAIDBKMoBAAAAg1GUAwAAAAajKAcAGGbJkiW69tprjQ4DAAzna3QAAICBtW3bNt15553nfdzHx0d79uy5jBEBAL4LRTkADFE33XSTcnJyep2bzfySFAC8DUU5AAxREyZM0IIFC4wOAwDgAdolADBMlZeXKzU1VS+88ILWrl2rm2++WRMnTtTVV1+tF154Qe3t7b1eU1paqoceekgzZszQxIkTdeONN2rZsmXq6Ojo9Vyn06nf/OY3mjt3rjIyMjRr1izdc889+vLLL3s9t7KyUo8//rimT5+urKws3XfffTp06NAl+bkBwBvRKQeAIaqpqUknTpzode7n56fg4GD3nzdu3KijR4/qjjvuUHR0tDZu3KgXX3xRDodDTz/9tPt5hYWFWrJkiXx9fd3P/fzzz/Xss8+qtLRUv/vd79zPLS8v1z/8wz+opqZGCxYsUEZGhpqampSfn6/Nmzdrzpw57uc2NjZq8eLFysrK0mOPPaby8nKtWLFCS5cu1dq1a+Xj43OJ/gsBgPegKAeAIeqFF17QCy+80Ov86quv1iuvvOL+c2lpqVauXKn09HRJ0uLFi/Xwww9r1apVuu222zRp0iRJ0n/+53+qtbVVb7/9ttLS0tzPffTRR7V27VrdeuutmjVrliTpV7/6laqqqrR8+XJlZ2f3+P6dnZ09/nzy5Endd999euCBB9xnkZGR+q//+i9t3ry51+sBYCiiKAeAIeq2227T/Pnze51HRkb2+PPs2bPdBbkkmUwm3X///Vq/fr0+++wzTZo0STU1Ndq1a5euu+46d0F+5rk/+9nP9Mknn+izzz7TrFmzVFtbq7y8PGVnZ/dZUJ97oanZbO61LWbmzJmSpMOHD1OUAxgWKMoBYIhKSUnR7Nmzv/N5o0eP7nU2ZswYSdLRo0cldY2jnH1+tlGjRslsNrufe+TIEblcLk2YMMGjOO12u/z9/XuchYeHS5Jqa2s9eg8AGOy40BMAYKgLzYy7XK7LGAkAGIeiHACGubKysl5nBw4ckCQlJSVJkhITE3ucn+3gwYPq7Ox0Pzc5OVkmk0klJSWXKmQAGHIoygFgmNu8ebOKi4vdf3a5XFq+fLkkad68eZKkqKgoTZ48WZ9//rn27dvX47mvvvqqJOm6666T1DV6kpOTo9zcXG3evLnX96P7DQC9MVMOAEPUnj17tHr16j4fO1NsS1JaWpruuusu3XHHHbLZbNqwYYM2b96sBQsWaPLkye7nPfXUU1qyZInuuOMO3X777bLZbPr888+1adMm3XTTTe7NK5L07//+79qzZ48eeOAB3XLLLUpPT1dLS4vy8/OVkJCgf/mXf7l0PzgADEIU5QAwRK1du1Zr167t87F169a5Z7mvvfZajRw5Uq+88ooOHTqkqKgoLV26VEuXLu3xmokTJ+rtt9/Wf//3f+uvf/2rGhsblZSUpJ///Oe69957ezw3KSlJ7733nv74xz8qNzdXq1evVmhoqNLS0nTbbbddmh8YAAYxk4vfIwLAsFReXq65c+fq4Ycf1j/90z8ZHQ4ADGvMlAMAAAAGoygHAAAADEZRDgAAABiMmXIAAADAYHTKAQAAAINRlAMAAAAGoygHAAAADEZRDgAAABiMohwAAAAwGEU5AAAAYLD/B/F5dT/ozqE6AAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "% matplotlib inline\n", - "\n", - "import seaborn as sns\n", - "\n", - "# Use plot styling from seaborn.\n", - "sns.set(style='darkgrid')\n", - "\n", - "# Increase the plot size and font size.\n", - "sns.set(font_scale=1.5)\n", - "plt.rcParams[\"figure.figsize\"] = (12,6)\n", - "\n", - "# Plot the learning curve.\n", - "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n", - "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n", - "\n", - "# Label the plot.\n", - "plt.title(\"Training & Validation Loss\")\n", - "plt.xlabel(\"Epoch\")\n", - "plt.ylabel(\"Loss\")\n", - "plt.legend()\n", - "plt.xticks([1, 2, 3, 4])\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mkyubuJSOzg3" - }, - "source": [ - "# 5. Performance On Test Set" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DosV94BYIYxg" - }, - "source": [ - "Now we'll load the holdout dataset and prepare inputs just as we did with the training set. Then we'll evaluate predictions using [Matthew's correlation coefficient](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html) because this is the metric used by the wider NLP community to evaluate performance on CoLA. With this metric, +1 is the best score, and -1 is the worst score. This way, we can see how well we perform against the state of the art models for this specific task." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Tg42jJqqM68F" - }, - "source": [ - "### 5.1. Data Preparation\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xWe0_JW21MyV" - }, - "source": [ - "\n", - "We'll need to apply all of the same steps that we did for the training data to prepare our test data set." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "mAN0LZBOOPVh", - "outputId": "7385ca3f-72d5-45f0-bbfe-5056c2f62c4f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of test sentences: 516\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2143: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n", - " FutureWarning,\n" - ] - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Load the dataset into a pandas dataframe.\n", - "df = pd.read_csv(\"./cola_public/raw/out_of_domain_dev.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n", - "\n", - "# Report the number of sentences.\n", - "print('Number of test sentences: {:,}\\n'.format(df.shape[0]))\n", - "\n", - "# Create sentence and label lists\n", - "sentences = df.sentence.values\n", - "labels = df.label.values\n", - "\n", - "# Tokenize all of the sentences and map the tokens to thier word IDs.\n", - "input_ids = []\n", - "attention_masks = []\n", - "\n", - "# For every sentence...\n", - "for sent in sentences:\n", - " # `encode_plus` will:\n", - " # (1) Tokenize the sentence.\n", - " # (2) Prepend the `[CLS]` token to the start.\n", - " # (3) Append the `[SEP]` token to the end.\n", - " # (4) Map tokens to their IDs.\n", - " # (5) Pad or truncate the sentence to `max_length`\n", - " # (6) Create attention masks for [PAD] tokens.\n", - " encoded_dict = tokenizer.encode_plus(\n", - " sent, # Sentence to encode.\n", - " add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n", - " max_length = 64, # Pad & truncate all sentences.\n", - " pad_to_max_length = True,\n", - " return_attention_mask = True, # Construct attn. masks.\n", - " return_tensors = 'pt', # Return pytorch tensors.\n", - " )\n", - " \n", - " # Add the encoded sentence to the list. \n", - " input_ids.append(encoded_dict['input_ids'])\n", - " \n", - " # And its attention mask (simply differentiates padding from non-padding).\n", - " attention_masks.append(encoded_dict['attention_mask'])\n", - "\n", - "# Convert the lists into tensors.\n", - "input_ids = torch.cat(input_ids, dim=0)\n", - "attention_masks = torch.cat(attention_masks, dim=0)\n", - "labels = torch.tensor(labels)\n", - "\n", - "# Set the batch size. \n", - "batch_size = 32 \n", - "\n", - "# Create the DataLoader.\n", - "prediction_data = TensorDataset(input_ids, attention_masks, labels)\n", - "prediction_sampler = SequentialSampler(prediction_data)\n", - "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "16lctEOyNFik" - }, - "source": [ - "## 5.2. Evaluate on Test Set\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rhR99IISNMg9" - }, - "source": [ - "\n", - "With the test set prepared, we can apply our fine-tuned model to generate predictions on the test set." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Hba10sXR7Xi6", - "outputId": "e35f0a6e-72c5-4bd0-9c4b-dcec9ef5059d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Predicting labels for 516 test sentences...\n", - " DONE.\n" - ] - } - ], - "source": [ - "# Prediction on test set\n", - "\n", - "print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))\n", - "\n", - "# Put model in evaluation mode\n", - "model.eval()\n", - "\n", - "# Tracking variables \n", - "predictions , true_labels = [], []\n", - "\n", - "# Predict \n", - "for batch in prediction_dataloader:\n", - " # Add batch to GPU\n", - " batch = tuple(t.to(device) for t in batch)\n", - " \n", - " # Unpack the inputs from our dataloader\n", - " b_input_ids, b_input_mask, b_labels = batch\n", - " \n", - " # Telling the model not to compute or store gradients, saving memory and \n", - " # speeding up prediction\n", - " with torch.no_grad():\n", - " # Forward pass, calculate logit predictions.\n", - " result = model(b_input_ids, \n", - " token_type_ids=None, \n", - " attention_mask=b_input_mask,\n", - " return_dict=True)\n", - "\n", - " logits = result.logits\n", - " \n", - " # Move logits and labels to CPU\n", - " logits = logits.detach().cpu().numpy()\n", - " label_ids = b_labels.to('cpu').numpy()\n", - " \n", - " # Store predictions and true labels\n", - " predictions.append(logits)\n", - " true_labels.append(label_ids)\n", - "\n", - "print(' DONE.')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-5jscIM8R4Gv" - }, - "source": [ - "Accuracy on the CoLA benchmark is measured using the \"[Matthews correlation coefficient](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html)\" (MCC).\n", - "\n", - "We use MCC here because the classes are imbalanced:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "hWcy0X1hirdx", - "outputId": "ef5e6753-c244-406a-8141-5078d71b04ee" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Positive samples: 354 of 516 (68.60%)\n" - ] - } - ], - "source": [ - "print('Positive samples: %d of %d (%.2f%%)' % (df.label.sum(), len(df.label), (df.label.sum() / len(df.label) * 100.0)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cRaZQ4XC7kLs", - "outputId": "d922af70-1216-4cfb-ac37-1dde75744fd5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Calculating Matthews Corr. Coef. for each batch...\n" - ] - } - ], - "source": [ - "from sklearn.metrics import matthews_corrcoef\n", - "\n", - "matthews_set = []\n", - "\n", - "# Evaluate each test batch using Matthew's correlation coefficient\n", - "print('Calculating Matthews Corr. Coef. for each batch...')\n", - "\n", - "# For each input batch...\n", - "for i in range(len(true_labels)):\n", - " # The predictions for this batch are a 2-column ndarray (one column for \"0\" \n", - " # and one column for \"1\"). Pick the label with the highest value and turn this\n", - " # in to a list of 0s and 1s.\n", - " pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n", - " \n", - " # Calculate and store the coef for this batch. \n", - " matthews = matthews_corrcoef(true_labels[i], pred_labels_i) \n", - " matthews_set.append(matthews)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IUM0UA1qJaVB" - }, - "source": [ - "The final score will be based on the entire test set, but let's take a look at the scores on the individual batches to get a sense of the variability in the metric between batches. \n", - "\n", - "Each batch has 32 sentences in it, except the last batch which has only (516 % 32) = 4 test sentences in it.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 427 - }, - "id": "pyfY1tqxU0t9", - "outputId": "5e477de2-e6a9-466a-9b36-f3651f2996df" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvMAAAGaCAYAAACCFszYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVjU5eL+8XvAARRQ0FBLhUxF3HDXNM3cqdz3FsksbdNTdllo/eqc46nMpaRcjktqiqapgJSmmbarqZlHLNHUcouTTiIIKA7C/P7wK6cJGAadYfjU+3Vd57oOz2d57sGymw/PPGOy2Ww2AQAAADAcL08HAAAAAHB9KPMAAACAQVHmAQAAAIOizAMAAAAGRZkHAAAADIoyDwAAABgUZR4AAIMbOXKkunXr5ukYADyggqcDAICn7Nq1S9HR0ZKkBx54QC+//HKhc86dO6cuXbooNzdX7dq1U1xcXKFzDhw4oJUrV2rPnj2yWCzy8vJS7dq11aFDB40YMUL16tWzO//SpUt6//33tWXLFh09elTZ2dmqUqWKmjRporvvvlv9+vVThQqO/3rOzMxUXFycPv74Y/3yyy/Ky8tTcHCwIiIi1LVrVw0dOvQGvjP4o27duumXX34p+NpkMqlatWqqW7eu7rvvPt17773Xfe+tW7cqJSVF48ePd0VUAH8xlHkAf3m+vr7asGGDJk2aJB8fH7tjSUlJstlsxZbrOXPmaM6cOQoODlafPn1Uv3595efn6+jRo9q0aZNWrlyp3bt3KyAgQJJ04sQJjR07VsePH1fHjh01duxYBQcH69y5c9q5c6cmT56so0eP6vnnny82b1ZWloYMGaJTp06pd+/eGjx4sMxms06dOqXvvvtOy5cvp8y7Qc2aNfXss89KkvLz83XmzBklJibq2WeflcVi0ahRo67rvlu3blViYiJlHsB1ocwD+Mvr2bOnNmzYoK1bt+qee+6xO5aQkKA777xT33zzTaHr1q1bp9mzZ6t9+/aaO3euAgMD7Y4/99xzmjNnTsHXOTk5euyxx3T69GnNnj1bvXr1sjt/7NixSk5O1oEDBxzmXbNmjY4fP64XXnhBDz30UKHjFoulxNfsDllZWQU/tBiJzWbTxYsX5e/v7/C8wMBA9e/f325s+PDh6ty5sxISEq67zAPAjWDNPIC/vMaNG6thw4ZKSEiwG09OTtaRI0c0ePDgQtdYrVbFxsaqUqVKio2NLVTkJcnPz08TJ04sKLhr167Vzz//rIcffrhQkb8mMjJSDzzwgMO8x48flyR16NChyOMhISGFxk6cOKHJkyfrzjvvVNOmTdWpUyc98cQT+v777+3O27p1q0aMGKEWLVqoZcuWGjFihLZu3Vroft26ddPIkSN18OBBPfLII2rdurX69etnl/G5555Tp06d1LRpU3Xr1k3Tpk3TxYsXHb62P97/hx9+UHR0tFq2bKl27dopJiZG586dK3S+1WrV/Pnzde+996pZs2Zq06aNHn/8cR08eNDuvF27dhX8Wa9cuVL33HOPmjVrpiVLljiV64+qVKkiHx8fmc1mu/Hk5GRNmjRJvXv3VvPmzQu+l5988ondeSNHjlRiYqIkqWHDhgX/+/0/ixaLRa+88oq6d++upk2bqkOHDnr44Ye1ffv2QnnOnDmjZ599Vm3btlXz5s31yCOP6Oeff76u1wbAGHgyDwCSBg8erNdff11nzpxRjRo1JF198l6tWjXdddddhc7/7rvvZLFY1L9/f1WtWtWpOT7++GNJV5/m3ojQ0FBJV39rMHHixBLX1x84cECjRo3SlStXNGTIEDVo0EAZGRnavXu39u3bp6ZNm0qSVq5cqSlTpui2227Tk08+KUlKTEzUU089pSlTphTKnZqaqoceekhRUVHq1atXQVH//vvv9dBDD6ly5coaPny4atSooUOHDikuLk779u1TXFxcofJblF9//VWjRo1Sr1691Lt3bx08eFDx8fH6/vvvtW7dOlWsWFGSlJubq0ceeUT79u1T//799cADDygrK0tr1qzRfffdpxUrVqhZs2Z29162bJnS09M1dOhQhYSEqGbNmiXmycvLU1pamqSry2wsFouWL1+u7OxsjRgxwu7cTz75RD/99JOioqJUq1YtpaenKzExUePGjdPMmTPVt29fSdLjjz+u/Px8ffvtt5o+fXrB9a1atZIknT59Wvfdd5/OnTun/v37q2nTprp06ZL279+vHTt26I477ii45uLFi3rwwQfVvHlzTZgwQadPn9by5cv15JNPasOGDfL29i7xNQIwIBsA/EV98803tvDwcNs777xjS0tLszVp0sT273//22az2WyXLl2ytW7d2vb666/bbDabrUWLFrYHH3yw4Nrly5fbwsPDbUuWLHF6vnbt2tlatWp1w7nT09NtXbp0sYWHh9s6dOhgGz9+vG3BggW2PXv22PLy8uzOzc/Pt9177722pk2b2lJSUgrd69r56enpthYtWth69Ohhy8zMLDiemZlp6969u61Fixa2jIyMgvGuXbvawsPDbWvWrCl0z759+9p69+5tdx+bzWbbsmWLLTw83BYfH1/ia7x2/6VLl9qNL1261BYeHm5bsGBBobEvv/zS7tzMzExbly5d7P7crv2Zt23b1vbbb7+VmOOPef74v2bNmtlWr15d6Pzs7OxCYxcvXrT16tXLdvfdd9uNx8TE2MLDw4uc99FHHy3ytdlsNrs/6wcffNAWHh5uW7hwod05ixYtKvZ6AH8OLLMBAEnBwcHq1q1bwZKHLVu2KDMzs8glNtLV9eGSSrVGPCsrq8R12c6oUqWKEhISNGbMGAUGBurjjz/WG2+8oQceeEA9evTQ119/XXBuSkqKjhw5okGDBikiIqLQvby8rv5nYPv27bp48aJGjhxp95oCAgI0cuRIXbx4UTt27LC7NigoSIMGDbIbO3z4sA4fPqw+ffrIarUqLS2t4H+tW7dWpUqVilweUpSAgADdf//9dmP333+/AgIC7JarfPDBB7rtttvUpEkTu/msVqs6duyovXv3Kicnx+4+/fv3V7Vq1ZzKcU2tWrW0dOlSLV26VEuWLNHrr7+u5s2b6x//+Ifi4+Ptzq1UqVLB/7906ZLOnz+vS5cu6fbbb9exY8cK/vlxJD09XV999ZU6d+6szp07Fzp+7c/u919f253pmttvv13S1WVWAP6cWGYDAP9n8ODBGjt2rL799lvFx8crMjJS9evXL/Lca4U3Ozvb6fsHBASU6nxHqlatqokTJ2rixIk6f/68/vOf/2jTpk364IMPNG7cOCUlJSksLKxgfX3jxo0d3u/06dOSpAYNGhQ6dm3s1KlTduN16tQptHTj2LFjkqTZs2dr9uzZRc7122+/lfwC/+/+f9xdyMfHR3Xq1LHLcuzYMeXk5BT7HgJJOn/+vG6++eaCr2+99VanMvxepUqV1LFjR7uxvn37auDAgXrllVfUrVs3BQcHS7q6pWlsbKy2bdtW5Br/CxculPiD4MmTJ2Wz2Ur8s7umevXq8vX1tRsLCgqSdPUHAwB/TpR5APg/nTp1Uo0aNTR37lzt2rVL//jHP4o991rB/eMbLB1p0KCB9uzZo1OnTqlOnTo3GrdAcHCwunbtqq5du+rmm2/W/PnztXHjxoJ17+5ybc16UUaPHl3k02RJqly5sktz2Gw2hYeHa/LkycWe88f3NTjKXhoVKlTQ7bffruXLlys5OVldunSRzWbT6NGjdezYMUVHR6tp06YKDAyUt7e34uPjtWHDBuXn57tk/t9ztCbeZrO5fD4A5QNlHgD+j7e3twYMGKAFCxbIz89Pffr0KfbcVq1aKSQkRFu3btX58+cLnsg60qtXL+3Zs0dr164t2K/c1Zo3by7p6q4mklS3bl1JV5fbOHLth4sjR44UesJ99OhRu3McCQsLk3R1yccfn2KX1qlTp2S1Wu2ezlutVp06dUq33Xab3Zznz5/X7bffXmjpSVm4cuWKpP/9lubw4cM6dOiQnnrqKf3tb3+zO3ft2rWFrjeZTEXeNzQ0VCaTqcQ/OwB/bayZB4DfGTFihMaNG6d//vOfDpdB+Pj46JlnnlF2drYmTJhQ5Broy5cv68033yw4NnToUNWtW1dLliwpcrtH6epOMCtXrnSYcd++fbpw4UKRx67d99ryoIiICDVo0EDx8fE6cuRIofOvPbG94447VKlSJa1YscLutWRlZWnFihWqVKmS3c4pxWncuLHCw8O1evXqQstypKvF19klH1lZWXrvvffsxt577z1lZWWpR48eBWMDBgyQxWLR0qVLi7yPs8t6rsfly5f11VdfSfrfUqZrP1D88Wn4jz/+WGhrSul/6+v/+H0JCgrSnXfeqS+//LLQ+xWKuj+AvyaezAPA79xyyy1OfxLnkCFD9Ouvv2rOnDnq1auX3SfAHjt2TJs3b1ZaWprGjh0r6erSjgULFmjs2LF66qmn1KlTJ3Xs2FFBQUFKS0vTrl279PXXX+vRRx91OO+HH36ohIQEdenSRZGRkQoKClJ6erq++OIL7dq1S/Xr1y94467JZNJrr72mUaNGaejQoQVbU164cEF79uxR586dNXLkSFWuXFkTJ07UlClTNGzYMA0cOFDS1a0pT5w4oSlTphS5l/4fmUwmTZ8+XQ899JD69eunwYMHq379+srJydGJEyf0ySef6Nlnny30xtmihIaGau7cuTpy5IiaNGmiH374QfHx8brttts0cuTIgvOio6O1Y8cOTZ8+Xd98841uv/12BQQEKDU1Vd988418fHwUFxdX4nwlyczMVFJSkqSrRfrs2bP68MMPderUKQ0bNqxgHX69evXUoEEDvfPOO8rJyVHdunX1888/6/3331d4eLh++OEHu/s2b95cK1as0D//+U916dJFZrNZkZGRqlOnjl566SUdPHhQY8aM0YABA9SkSRNdvnxZ+/fvV61atfTcc8/d8OsCYGyUeQC4AePGjVOXLl20YsUKbd26VatWrZKXl5dCQ0N1zz336L777rN7wh8WFqb169fr/fff18cff6z58+fr4sWLqlKlipo2barXX3+9YA/y4owYMUKBgYHatWuXli5dqvT0dJnNZoWFhWncuHF6+OGH7XZTiYyM1Lp16zRv3jxt2rRJq1evVlBQkCIjIwv2M5ekBx54QNWrV9fixYs1d+5cSVef7M+dO9fuSXhJGjVqpMTERC1YsECffvqpVq9eLX9/f9WqVUsDBw50+EbV36tZs6ZiY2M1bdo0bdy4UWazWX379lVMTIzd6zObzVqwYIHee+89JSUlFbzxtnr16mrWrFnBDyY36tdff9Xzzz9f8HXFihVVr149/f3vf7fbZ97b21sLFizQtGnTlJiYqEuXLqlBgwaaNm2aDh06VKjM9+nTRykpKdq4caM2b96s/Px8TZ06VXXq1FGdOnUUHx+vuXPn6ssvv1RSUpIqV66siIiIG/68AgB/DiYbv6cDAJQz3bp1U61atVzyRB0A/sxYMw8AAAAYFGUeAAAAMCjKPAAAAGBQrJkHAAAADIon8wAAAIBBUeYBAAAAg2Kf+Rt0/ny28vNZqQQAAADX8/IyKTjYv9jjlPkblJ9vo8wDAADAI1hmAwAAABgUZR4AAAAwKMo8AAAAYFCUeQAAAMCgKPMAAACAQVHmAQAAAIOizAMAAAAGRZkHAAAADIoyDwAAABgUZR4AAAAwKMo8AAAAYFCUeQAAAMCgKng6AAAAfxWBQRXlZ/bMf3pzcq8oM/2SR+YG4D6UeQAAyoifuYIGxn/mkbkTB3dVpkdmBuBOLLMBAAAADIon8wCAUgkM8pOf2eyRuXNyc5WZnuORuQGgPKLMAwBKxc9sVp/4xR6Ze8PgR5QpyjwAXMMyGwAAAMCgDFnmrVarZsyYoU6dOikyMlLDhg3Tzp07nbp2x44dGjlypNq3b6+2bdtq+PDh+uijj9ycGAAAAHA9Q5b5SZMmadmyZerXr59efPFFeXl5acyYMdq3b5/D6z777DONHj1aV65c0fjx4/X000/Ly8tLEyZM0Nq1a8soPQAAAOAahlszn5ycrI0bN2ry5MkaNWqUJGnAgAHq06ePZs6cqZUrVxZ77cqVKxUSEqJly5bJx8dHkjRs2DB1795dSUlJGjp0aFm8BAAAUApVgvzlY/bc80drbr4y0rM9Nj/giOHK/ObNm2U2m+2Kt6+vr4YMGaJZs2bp7Nmzql69epHXZmVlqUqVKgVFXpJ8fHxUpUoV+fr6uj07AAAoPR+zlxYmnPXY/GMHFd0rgPLAcMtsUlJSVLduXfn7+9uNR0ZGymazKSUlpdhr27VrpyNHjig2NlYnT57UyZMnFRsbq+PHj2v06NHujg4AAAC4lOGezFssFtWoUaPQeEhIiCTp7Nnif3J//PHHdfLkSc2fP1///ve/JUmVKlXSvHnzdMcdd7gnMAAAAOAmhivzOTk5MhfxYSXXlslcvny52Gt9fHx06623KioqSj179lReXp7WrFmjZ555Ru+++64iIyNLnadatYBSXwMAuH4hIYGejmBYfO+uH987lFeGK/N+fn7Kzc0tNH6txDta+/6vf/1LBw4c0Lp16+TldXWF0d13360+ffrotdde0+rVq0ud59y5LOXn20p9HQAYladLjcWS6dH5bwTfu+vj6e+bZNzvHYzPy8vk8OGx4dbMh4SEFLmUxmKxSFKxb361Wq1at26d7rrrroIiL0lms1mdO3fWgQMHdOXKFfeEBgAAANzAcGU+IiJCP//8s7Kz7beI2r9/f8HxoqSnp+vKlSvKy8srdOzKlSu6cuWKbDaesAMAAMA4DFfmo6KilJuba/chT1arVQkJCWrVqlXBm2NTU1N17NixgnOqVaumypUr65NPPrFbppOdna3PPvtM4eHhRa7FBwAAAMorw62Zb968uaKiojRz5kxZLBaFhoYqMTFRqampmjp1asF5MTEx2r17tw4fPixJ8vb21ujRoxUbG6vhw4erX79+ys/P17p16/Trr78qJibGUy8JAAAAuC6GK/OSNH36dMXGxiopKUkZGRlq2LChFi5cqNatWzu87oknnlDt2rW1fPlyzZ07V1arVQ0bNtScOXPUs2fPMkoPAAAAuIYhy7yvr69iYmIcPk2Pi4srcrxv377q27evu6IBADwoMKii/Mye+09bTu4VZaZf8tj8AP56DFnmAQAoip+5gvqui/fY/B8OGSw2MARQlgz3BlgAAAAAV1HmAQAAAIOizAMAAAAGRZkHAAAADIoyDwAAABgUZR4AAAAwKMo8AAAAYFCUeQAAAMCgKPMAAACAQVHmAQAAAIOizAMAAAAGRZkHAAAADIoyDwAAABhUBU8HAAAAMLKgIH+ZzZ55Ppqbm6/09GyPzI3ygTIPAABwA8xmL3260uKRubs9EOKReVF+sMwGAAAAMCjKPAAAAGBQlHkAAADAoCjzAAAAgEFR5gEAAACDoswDAAAABsXWlADcpkqQWT5mP4/Mbc3NUUZ6rkfmBgCgrFDmAbiNj9lPr7zf2yNz/7/hH0uizAMA/txYZgMAAAAYFGUeAAAAMCjKPAAAAGBQlHkAAADAoCjzAAAAgEFR5gEAAACDYmtKAACgwKBK8jN7e2z+nNw8ZaZf9Nj8gFFR5gEAgPzM3hoe/6PH5n9/cLgyPTY7YFwsswEAAAAMijIPAAAAGBRlHgAAADAop9fM//zzz9q9e7eOHDmitLQ0mUwmBQcHKzw8XG3btlXdunXdmRMAAADAHzgs85cvX1Z8fLzef/99/fjjj7LZbEWeZzKZFB4erhEjRmjQoEHy9fV1S9hrrFar3nrrLSUlJenChQuKiIjQhAkT1KFDB6eu//DDD7Vs2TIdPXpUPj4+Cg8P1/PPP6/IyEi35gYAAABcqdgyv379esXGxurMmTNq06aNJkyYoJYtWyo0NFRBQUGy2WzKyMjQiRMn9J///EdffvmlpkyZogULFmjChAnq37+/20JPmjRJW7ZsUXR0tMLCwpSYmKgxY8YoLi5OLVu2dHjtrFmz9M4776hfv34aPny4Ll68qEOHDslisbgtLwAAAOAOxZb5f/zjHxoxYoRGjhypWrVqFXmOn5+fatSooXbt2mns2LH65ZdftGzZMv397393W5lPTk7Wxo0bNXnyZI0aNUqSNGDAAPXp00czZ87UypUri732u+++04IFCzR79mz17NnTLfkAAACAslJsmd+6datuuummUt2sVq1aeuGFFzRmzJgbDlaczZs3y2w2a+jQoQVjvr6+GjJkiGbNmqWzZ8+qevXqRV67fPlyNWvWTD179lR+fr4uXbokf39/t2UFAAAA3KnY3WxKW+R/LyQk5LqvLUlKSorq1q1bqIRHRkbKZrMpJSWl2Gt37typZs2a6c0331Tr1q3VqlUrdevWTR988IHb8gIAAADuYrhPgLVYLKpRo0ah8Ws/QJw9e7bI6zIyMpSenq6NGzfK29tbEydOVFBQkFauXKnnnntOFStWZOkNAAAADMVlZf6zzz7Tli1bNHXqVFfdskg5OTkym82Fxq/toHP58uUir7t48aIkKT09XWvWrFHz5s0lST179lTPnj01d+7c6yrz1aoFlPoaAGUjJCTQ0xHgBuX9z7U85yvP2aTynY9sKK9cVuYPHTqk9evXu73M+/n5KTc3t9D4tRJf3LaY18Zr165dUOQlycfHR71799by5cuVnZ1d6jX0585lKT+/6C07gb86T/8HxmLJ9Oj8f1bl+c/V09mk8p2vPGeTis9XnrNJns/H33V/bl5eJocPjw33CbAhISFFLqW5trVkcW9+DQoKko+PT5HvBbjppptks9mUlZXl2rAAAACAGzl8Mh8dHe30jVJTU284jDMiIiIUFxdX6Cn6/v37C44XxcvLS40aNdKZM2cKHfv111/l7e2tKlWquCc0AAAA4AYOy/zu3btVoUKFIteo/9GVK1dcFsqRqKgoLVmyRGvXri3YZ95qtSohIUGtWrUqeHNsamqqLl26pHr16tldO23aNG3fvl133HGHJCkrK0ubNm1Sy5Yt5efnVyavAQAAAHAFh2W+Ro0aatSokebPn1/ijebNm6fZs2e7LFhxmjdvrqioKM2cOVMWi0WhoaFKTExUamqq3Xr9mJgY7d69W4cPHy4Yu++++7R27VqNHz9eo0aNUuXKlRUfH6/MzEw9++yzbs8OAAAAuJLDMt+4cWMdOHDAqRuZTCaXBHLG9OnTFRsbq6SkJGVkZKhhw4ZauHChWrdu7fC6ihUravny5Zo+fbpWrFihnJwcNWnSREuXLi3xWgAAAKC8cVjmmzRpos8++0xnzpwpcm/33wsMDNTNN9/s0nDF8fX1VUxMjGJiYoo9Jy4ursjxkJAQzZgxw13RAAAAgDLjcDeb0aNHa9u2bQoODi7xRg8++KA+/fRTlwUDAAAA4JjDJ/OVKlVSpUqVyioLAAAAgFIw3D7zAAAAAK5y2SfAAoCRBAb5yM9c9CdGl4Wc3MvKTLd6bH4AwJ/DdZX58+fPq2PHjlqyZIk6dOjg6kwA4HZ+Zl/dnXSfx+bf1H+VMkWZBwDcmOteZmOz2VyZAwAAAEApsWYeAAAAMCjKPAAAAGBQTq2ZT01Ntfs6IyNDkpSWllbo2C233OKiaAAAAAAccarMd+vWTSaTqdD4xIkTC42lpKTceCoAAAAAJXKqzL/22mt2ZT47O1uvvPKKRo8erfr167stHAAAAIDiOVXmBw0aZPf1+fPn9corr6hTp05sTQkAAAB4CG+ABQAAAAyKMg8AAAAYFGUeAAAAMCin1sz/UWBgoJYvX65GjRq5Og8AAAAAJ11Xma9QoYLatWvn6iwAAAD4C6lapZK8fbw9MneeNU9pGRc9MrcrXVeZBwAAAG6Ut4+3fn3zB4/MXfPZJh6Z19VYMw8AAAAYFGUeAAAAMCjKPAAAAGBQlHkAAADAoCjzAAAAgEFd9242aWlpkqSqVau6LAwAQAoM8pOf2eyx+XNyc5WZnuOx+QEAzitVmT9z5ozefPNNbdu2TdnZ2ZKkgIAAde/eXRMmTFCNGjXcEhIA/kr8zGbdkzjNY/N/NDBGmaLMA4AROF3mU1NTNWzYMP32229q1KiR6tevL0k6duyY1q9fr+3bt2vNmjW6+eab3RYWAAAAwP84XebfeustXbhwQQsWLFCXLl3sjn3xxRcaP3683nrrLb3++usuDwkAAACgMKffALt9+3bdf//9hYq8JHXp0kX33XefvvrqK5eGAwAAAFA8p8t8RkaGwsLCij0eFhamCxcuuCQUAAAAgJI5XeZr1qyp3bt3F3v822+/Vc2aNV0SCgAAAEDJnC7zUVFR2rx5s9544w1lZmYWjGdlZenNN9/Upk2bdM8997glJAAAAIDCnH4D7JNPPqlvv/1WixYt0pIlS1S9enVJ0tmzZ5WXl6dWrVrpiSeecFtQAIVVCTLLx+znsfmtuTnKSM/12PwAAPzVOV3mK1asqLi4OCUkJGjr1q06ffq0JKlTp07q0aOHBg4cqAoVrvszqABcBx+zn5Ys6+Wx+Uc/tEUSZR4AAE8pVfuuUKGChg0bpmHDhrkrDwAAAAAnOb1mPjo6Wjt37iz2+DfffKPo6GiXhAIAAABQMqfL/O7du/Xbb78VezwtLU179uxxSSgAAAAAJXO6zJfkwoUL8vHxcdXtHLJarZoxY4Y6deqkyMhIDRs2zOFvDYozZswYNWzYUK+++qobUgIAAADu5XDN/KFDh3To0KGCr7/99lvl5eUVOi89PV2rVq1SvXr1XJ+wCJMmTdKWLVsUHR2tsLAwJSYmasyYMYqLi1PLli2dusfnn3+ub7/91s1JAQAAAPdxWOa3bt2qOXPmSJJMJpPef/99vf/++0We6+/vrxdffNH1Cf8gOTlZGzdu1OTJkzVq1ChJ0oABA9SnTx/NnDlTK1euLPEeVqtVU6dO1SOPPKLZs2e7OTEAAADgHg7L/MCBA9WuXTvZbDY99NBDeuyxx3THHXfYnWMymVSpUiXVr19fvr6+bg0rSZs3b5bZbNbQoUMLxnx9fTVkyBDNmjVLZ8+eLdgDvzjLly9XTk4OZR4AAACG5rDM16pVS7Vq1ZIkTRr+AWwAACAASURBVJ06VW3btlXt2rXLJFhxUlJSVLduXfn7+9uNR0ZGymazKSUlxWGZt1gsmjdvnl5++WVVrFjR3XEBAAAAt3F6n/mBAwe6M4fTLBaLatSoUWg8JCRE0tVPpHXkzTffVN26ddW/f3+35AMAAADKiuE+sjUnJ0dms7nQ+LUlPpcvXy722uTkZK1fv15xcXEymUwuyVOtWoBL7gMYVUhIoKcjFKs8Z5PKdz6yXb/ynK88Z5PKdz6y/Tn9Gb53hivzfn5+ys0t/PHx10p8cev2bTabXn31VfXq1Utt2rRxWZ5z57KUn29z2f2A0igPfwlZLJnFHvN0vvKcTSo+X3nOJnk+X3nOJpXvfOU5m8S/E9fLUbbyju9dyby8TA4fHhuuzIeEhBS5lMZisUhSsevlP/nkEyUnJ2vChAk6ffq03bGsrCydPn1aN910k/z8/FwfGgAAAHADw5X5iIgIxcXFKTs72+5NsPv37y84XpTU1FTl5+froYceKnQsISFBCQkJWrRoke688073BAcAAABczHBlPioqSkuWLNHatWsL9pm3Wq1KSEhQq1atCt4cm5qaqkuXLhV8kFW3bt2K3InnqaeeUteuXTVkyBA1adKkzF4HAAAAcKMMV+abN2+uqKgozZw5UxaLRaGhoUpMTFRqaqqmTp1acF5MTIx2796tw4cPS5JCQ0MVGhpa5D3r1KmjHj16lEl+AAAAwFW8XHWjpKQkRUdHu+p2Dk2fPl0jR45UUlKSXnnlFV25ckULFy5U69aty2R+AAAAoDxw2ZP51NRU7dmzx1W3c8jX11cxMTGKiYkp9py4uDin7nXtyT0AAABgNC57Mg8AAACgbDl8Mt+9e3enb5SVlXXDYQAAAAA4z2GZ/+WXX1SlSpVi927/vZycHJeFAgAAAFAyh2W+du3aCgsL0+LFi0u80bx58zR79myXBQMAAADgmMM1802aNNEPP/zg1I1MJpNLAgEAAABwjsMy37hxY6Wnp+v06dMl3uiWW25RmzZtXBYMAAAAgGMOy/xjjz2mQ4cOFfnJqX/Uv39/p7eDBAAAAHDj2JoSAAAAMKjrLvP5+flKTU2V1Wp1ZR4AAAAATrruT4BNS0tT9+7dtWTJEnXo0MGVmfAXFFzFRxV8fD0y9xXrZZ3P4IdSAABgPNdd5iXJZrO5Kgf+4ir4+Grf/L4embvl4x9KoswDAADjYc08AAAAYFCUeQAAAMCgrrvM+/n5aeDAgapevbor8wAAAABw0nWvmQ8ICNDUqVNdmQUAAABAKbDMBgAAADCoYp/M33///ZowYYLatm1bqhvu3LlTb7/9tlatWnXD4YDyIKiKj8we2jYz13pZ6WybCQAAilFsma9evbpGjhypxo0ba8CAAbrzzjt16623Fnnu0aNH9cUXXygpKUlHjhzRPffc4668QJkz+/jqo8We+Wf6nkc+EttmAgCA4hRb5mNjY7V3717NmzdPU6dO1dSpU1W5cmXVqlVLQUFBstlsysjI0MmTJ5WdnS2TyaROnTppypQpatGiRVm+BgAAAOAvyeEbYFu3bq3Fixfr5MmT2rx5s/bs2aNjx47pp59+kslkUnBwsNq0aaN27dqpV69eql27dlnlBgAAAP7ynNrNJjQ0VGPHjtXYsWPdnQcAAACAk9jNBgAAADAoyjwAAABgUJR5AAAAwKAo8wAAAIBBUeYBAAAAg6LMAwAAAAZFmQcAAAAMqlRlPi8vT+vXr9fEiRP18MMP6+DBg5KkjIwMrV+/XmfOnHFLSAAAAACFOfWhUZJ06dIljR49Wvv27VPFihWVk5OjjIwMSVJAQIBmzpypwYMHa8KECW4LCwAAAOB/nH4yP3v2bH3//feaM2eOtm3bJpvNVnDM29tbvXr10tdff+2WkAAAAAAKc7rMb968WcOHD1ePHj1kMpkKHQ8NDdUvv/zi0nAAAAAAiud0mT979qwaNmxY7PGKFSsqOzvbJaEAAAAAlMzpMh8UFOTwDa5HjhxR9erVXRIKAAAAQMmcLvMdOnRQQkKCLl26VOjYqVOnFB8fr86dO7s0HAAAAIDiOV3mx40bpwsXLmjIkCFatWqVTCaTvvrqK73xxhsaNGiQfHx89Nhjj7kzKwAAAIDfcbrMh4WF6d1335W3t7fefvtt2Ww2LVmyRIsWLVLNmjW1bNky3Xzzze7MWsBqtWrGjBnq1KmTIiMjNWzYMO3cubPE67Zs2aJnnnlG3bp1U/PmzRUVFaVp06YpMzOzDFIDAAAAruX0PvOS1LRpU33wwQf68ccfdezYMdlsNt16661q3Lixu/IVadKkSdqyZYuio6MVFhamxMREjRkzRnFxcWrZsmWx17300kuqXr26+vfvr1tuuUWHDx9WXFycvvrqK8XHx8vX17cMXwUAAABwY5wq89nZ2erfv78efPBBjRo1SuHh4QoPD3d3tiIlJydr48aNmjx5skaNGiVJGjBggPr06aOZM2dq5cqVxV779ttvq3379nZjTZs2VUxMjDZu3KhBgwa5MzoAAADgUk4ts/H391d6err8/f3dnadEmzdvltls1tChQwvGfH19NWTIEO3du1dnz54t9to/FnlJ6tGjhyTp2LFjrg8LAAAAuJHTa+abN2+uAwcOuDOLU1JSUlS3bt1CP1hERkbKZrMpJSWlVPf77bffJEnBwcEuywgAAACUBafL/MSJE7V582bFx8fLZrO5M5NDFoulyP3sQ0JCJMnhk/miLFq0SN7e3urVq5dL8gEAAABlxek3wE6dOlWVK1fW//t//08zZsxQaGio/Pz87M4xmUxatmyZy0P+Xk5Ojsxmc6Hxa29evXz5stP3+vDDD7Vu3To99thjCg0Nva481aoFXNd1KF9CQgI9HaFY5TmbVL7zledsUvnOR7brV57zledsUvnOR7Y/pz/D987pMn/69GlJKth+8trylLLm5+en3NzcQuPXSryzO9J8++23evHFF3XXXXfp6aefvu48585lKT/fc7+p+LPw9L9MFkvx25OSzbHynK88Z5OKz1ees0mez1ees0nlO195zibx78T1cpStvON7VzIvL5PDh8dOl/lPP/3UJYFuVEhISJFLaSwWiyQVuQTnjw4dOqQnnnhCDRs21KxZs+Tt7e3ynAAAAIC7Ob1mvryIiIjQzz//rOzsbLvx/fv3Fxx35OTJk3r00UdVtWpVLViwQJUqVXJbVgAAAMCdSl3ms7KytGXLFi1evFiLFy/Wli1blJWV5Y5sRYqKilJubq7Wrl1bMGa1WpWQkKBWrVqpRo0akqTU1NRC201aLBaNHj1aJpNJixcvVtWqVcssNwAAAOBqpfoE2LVr1+r111/XxYsXC3a0MZlMqlSpkiZNmmS397u7NG/eXFFRUZo5c6YsFotCQ0OVmJio1NRUTZ06teC8mJgY7d69W4cPHy4Ye/TRR3Xq1Ck9+uij2rt3r/bu3VtwLDQ01OGnxwIAAADljdNlftu2bXrppZdUp04dPf3002rQoIEk6ciRI1qxYoVefvllVatWTd26dXNb2GumT5+u2NhYJSUlKSMjQw0bNtTChQvVunVrh9cdOnRIkvTOO+8UOjZw4EDKPAAA+FMJruKvCj6eW1V9xZqv8xnZJZ+I6+Z0mX/nnXdUr149rVmzxu4Dmzp06KBBgwZp+PDhWrRoUZmUeV9fX8XExCgmJqbYc+Li4gqN/f4pPQAAwJ9dBR8vHZlzxmPzNxhXw2Nz/1U4/aPaoUOHNHDgwEKfvCpJAQEBGjBgQMGTbwAAAADu57Lfu5hMJlfdCgAAAIATnC7zDRs2VGJioi5evFjoWHZ2thITE0vcFhIAAACA6zi9Zv7RRx/VuHHjNHDgQEVHR6tevXqSpKNHjyouLk4nT57U7Nmz3RYUAAAAgD2ny3yPHj300ksvaebMmfrXv/5VsKzGZrOpYsWKeumll9SjRw+3BQUAAABgr1T7zD/wwAPq27evtm/frtOnT0uS6tSpozvuuEOBgYFuCQgAAACgaKUq85JUuXJl3X333e7IAgAAAKAUnH4D7MGDB7Vy5cpij69cuVIpKSkuCQUAAACgZE6X+Tlz5ujzzz8v9viXX36puXPnuiITAAAAACc4XeYPHDigtm3bFnu8bdu2Sk5OdkkoAAAAACVzusyfP39eQUFBxR6vXLmyzp8/75JQAAAAAErmdJmvVq2ajhw5UuzxH3/8UVWqVHFJKAAAAAAlc7rMd+zYUevWrSuy0B89elTx8fHq2LGjS8MBAAAAKJ7TW1M+8cQT2rJli4YMGaLBgwerUaNGkqSUlBTFx8fLbDbrySefdFtQAAAAAPacLvOhoaF69913NXnyZL333nt2xxo0aKDXXntNt956q6vzAQAAAChGqT40qlmzZtqwYYNSUlJ0/PhxSVLdunUVERHhjmwAAAAAHCj1J8BKUqNGjQqW2QAAAADwjOsq85J06tQpbdy4UWfOnFH9+vU1ePBg+fn5uTIbAAAAAAcclvm1a9cqLi5OS5cuVbVq1QrGt2/frnHjxiknJ0c2m00mk0mrV6/W6tWr5e/v7/bQAAAAAErYmvLzzz+Xv7+/XZG32Wx6+eWXlZOTo7Fjx+rf//63Bg4cqCNHjujdd991d14AAAAA/8fhk/lDhw7p7rvvthv77rvv9Msvv2jAgAGaMGGCJKlr16765ZdftG3bNj311FPuSwsAAACggMMn82lpaapTp47d2HfffSeTyVSo5Hfp0kUnTpxwfUIAAAAARXJY5itUqKDc3Fy7sQMHDkiSWrRoYTceFBQkq9Xq4ngAAAAAiuOwzNeqVUv79u0r+DovL0979+5VWFiYqlSpYnduenq6goOD3ZMSAAAAQCEO18z36tVL8+bNU8uWLXX77bcrPj5eaWlpGjx4cKFzk5OTVbt2bbcFBQAAAGDPYZmPjo5WUlKSXn31VUlXd7K5+eab9fDDD9udl5mZqS+++EKjRo1yW1AAAAAA9hyW+YCAAMXHx2vNmjU6ceKEQkNDNXToUFWuXNnuvGPHjmnQoEG699573RoWAAAAwP+U+AmwAQEBGj16tMNzWrRoUegNsQAAAADcy+EbYAEAAACUX5R5AAAAwKBKXGYD51Wt4idvH7NH5s6z5iotI8cjcwMAAMAzKPMu5O1jluXfKzwyd8gTD0qizAMAAPyVsMwGAAAAMCjKPAAAAGBQDst8Xl6eZs6cqVWrVjm8yXvvvac333xTNpvNpeEAAAAAFM9hmf/ggw+0ePFiNWvWzOFNIiMjtWjRIm3YsMGl4QAAAAAUz2GZ37Rpkzp27KimTZs6vEnTpk3VqVMnbdy40aXhAAAAABTPYZn/4Ycf1KFDB6du1L59e33//fcuCVUSq9WqGTNmqFOnToqMjNSwYcO0c+dOp649c+aMnn76abVp00atWrXSk08+qVOnTrk5MQAAAOB6Dst8RkaGqlWr5tSNqlatqvT0dJeEKsmkSZO0bNky9evXTy+++KK8vLw0ZswY7du3z+F12dnZio6O1t69e/X444/rb3/7mw4ePKjo6GhlZGSUSXYAAADAVRzuM+/v76/z5887daP09HT5+/u7JJQjycnJ2rhxoyZPnqxRo0ZJkgYMGKA+ffpo5syZWrlyZbHXvvfeezpx4oQSEhLUuHFjSVLnzp3Vt29fvfvuu3r66afdnh8AAABwFYdP5uvXr6/t27c7daPt27erfv36LgnlyObNm2U2mzV06NCCMV9fXw0ZMkR79+7V2bNni732448/VosWLQqKvCTVq1dPHTp00KZNm9yaGwAAAHA1h2W+Z8+e2rFjh7Zu3erwJtu2bdOOHTvUq1cvl4YrSkpKiurWrVvotwCRkZGy2WxKSUkp8rr8/HwdPny4yDfzNmvWTMePH9elS5fckhkAAABwB4dlfsSIEQoNDdUzzzyjWbNm6fTp03bHT58+rVmzZumZZ57RrbfeqhEjRrg1rCRZLBZVr1690HhISIgkFftkPj09XVarteC8P15rs9lksVhcGxYAAABwI5OthE96OnHihB577DEdP35cJpNJAQEB8vf3V3Z2trKysmSz2VS3bl0tWLBAoaGhbg/co0cP1a9fX/Pnz7cbP3XqlHr06KGXXnpJDz74YKHr/vvf/+quu+7SpEmT9PDDD9sdW7dunV588UV9+OGHCg8Pv+5stit5MlXwvu7rb0RJc9uu5MpUwVyGiUo3f/4Vq7wq+JRhIufnzrtilbeHspU095U8qyp4eyabM/N7Ml9Jc1vzrPLx4PfO0fzWvCvy8Xb4lia3Kml+T+YrOVuefLw98/ewM/N7Ml/J2fLl4+25D4Z3NP+VPJsqeJvKOJHz8+fl2eTtoXwlzZ1/xSavCp773pU0v+1KvkwVPPPPXUlzl+de93sl/m0cFhampKQkrVmzRh9//LGOHDmi3377Tf7+/mrTpo169eqloUOHys/P74aDO8PPz0+5ubmFxi9fvizp6vr5olwbt1qtxV57Pa/h3Lks5eeX/0++DQkJ1H/nveix+W9+8lVZLJklnHW5TLJc39xku/75+d6V3/kBAMUJCQnU2dnbPDJ39fHdC3qTl5dJ1aoFFHuuU49WfH19NXLkSI0cOdI1CW9ASEhIkUtpri2RKWoJjiQFBQXJx8enyKU0FotFJpOpyCU4AAAAQHlV4u81Ll68qOzsbIfnZGdn6+LFiy4L5UhERIR+/vnnQpn2799fcLwoXl5eCg8PL/KDrZKTkxUWFqaKFSu6PjAAAADgJg7L/E8//aR27dppwYIFDm+ycOFCtWvXTidPnnRpuKJERUUpNzdXa9euLRizWq1KSEhQq1atVKNGDUlSamqqjh07Zndt79699Z///EcHDx4sGPvpp5/0zTffKCoqyu3ZAQAAAFdyuMxm9erVCg4O1rhx4xze5Mknn1RiYqJWrVqlmJgYlwb8o+bNmysqKkozZ86UxWJRaGioEhMTlZqaqqlTpxacFxMTo927d+vw4cMFY/fff7/Wrl2rsWPH6uGHH5a3t7feffddhYSEFHwAFQAAAGAUDsv8zp071bt3b/n4ON7xwdfXV1FRUU5/wNSNmj59umJjY5WUlKSMjAw1bNhQCxcuVOvWrR1eFxAQoLi4OL322muaN2+e8vPz1b59e7344osKDg4uk+wAAACAqzgs86dPny5ym8ei1KtXz27pizv5+voqJibG4W8B4uLiihyvWbOm3n77bXdFAwAAAMqMwzXz+fn58vJybu9PLy8v5efnuyQUAAAAgJI5bOohISE6evSoUzc6evQoWzsCAAAAZchhmW/Tpo02bNjg1NaUGzZsUNu2bV0aDgAAAEDxHJb5Bx54QGlpaRo3bpzS09OLPCcjI0Pjxo3T+fPnnV5fDwAAAODGOXwDbLNmzfTUU09pzpw56t69u3r16qWGDRsqICBA2dnZSklJ0datW5WVlaXx48erSZMmZZUbpZRntermJ1/16PwAAABwLYdlXpLGjRunmjVrKjY2VomJiZIkk8kkm80mSbrppps0efJkDR482L1JcUPSMi5LuuzpGAAAAHChEsu8JA0ZMkT9+/fXd999pyNHjigrK0sBAQFq0KCBWrVqJbPZ7O6cAAAAAP7AqTIvSWazWe3bt1f79u3dmQcAAACAk5zbRB4AAABAuePwyXx0dHSpbmYymbRs2bIbCgQAAADAOQ7L/O7du1WhQgWn18SbTCaXhAIAAABQModlvkKFq4c7duyoQYMGqWvXrvLyYmUOAAAAUB44bOZffvmlnn32WZ08eVLjxo3TnXfeqRkzZuinn34qq3wAAAAAimGyXdswvgTJyclat26dNm3apKysLEVGRmrIkCG655575O/v7+6c5da5c1nKz3fqWwgAAACDCAkJ1NnZ2zwyd/Xx3WWxZEqSvLxMqlYtoNhznV4zExkZqSlTpujrr7/WtGnTVLFiRb388svq1KmTkpKSbjw1AAAAgFJxep/5a3x9fdWvXz/VqlVLXl5e2rFjh06dOuWObAAAAAAcKFWZP3v2rNavX6+EhASdOHFC1atX12OPPabBgwe7Kx8AAACAYpRY5nNzc7Vt2zYlJCRo+/bt8vLyUrdu3TR58mR17tyZ3W0AAAAAD3FY5l955RV9+OGHunDhgsLDwxUTE6N+/fopKCiorPIBAAAAKIbDMr9ixQr5+fnp3nvvVZMmTZSXl6fExMRizzeZTBo1apSrMwIAAAAoQonLbHJycrRhwwZt2LChxJtR5gEAAICy47DML1++vKxyAAAAACglh2W+Xbt2ZZUDAAAAQCmxFQ0AAABgUJR5AAAAwKAo8wAAAIBBUeYBAAAAg6LMAwAAAAZFmQcAAAAMijIPAAAAGBRlHgAAADAoyjwAAABgUJR5AAAAwKAo8wAAAIBBUeYBAAAAg6LMAwAAAAZVwdMBrseFCxc0Y8YMffLJJ8rJyVFkZKQmT56sRo0aObwuPz9fiYmJ+uSTT5SSkqKMjAzVrl1bffr00ejRo+Xj41NGrwAAAAC4cYZ7Mp+fn6+xY8dq48aNevDBB/Xcc8/p3LlzGjlypE6ePOnw2kuXLumFF17Q+fPnNWLECL3wwgtq1qyZ3nrrLY0dO7aMXgEAAADgGoZ7Mr9582bt27dPc+fOVY8ePSRJd999t3r37q05c+Zo+vTpxV5rNpu1atUqtWrVqmBs2LBhqlWrlmbPnq1du3apffv2bn8NAAAAgCsY7sn8xx9/rOrVq6t79+4FY1WrVtXdd9+trVu3Kjc3t9hrfXx87Ir8NT179pQkHTt2zPWBAQAAADcxXJlPSUlRkyZNZDKZ7MabNWum7OzsEpfaFOW3336TJAUHB7skIwAAAFAWDFfmLRaLqlevXmj82tjZs2dLfc933nlHgYGB6tSp0w3nAwAAAMqKR9fM5+fnO1wW83u+vr6SpJycnCJ3nbk2lpOTU6oM8+fP144dOzRlyhQFBgaW6lpJqlYtoNTXAAAAAI6EhDjXSz1a5vfs2aPo6Ginzt25c6eqVq0qPz8/Wa3WQsevjfn5+Tk9/0cffaTY2FgNHz5cw4cPd/q63zt3Lkv5+bbruhYAAADlk7Nl2l0slkxJkpeXyeHDY4+W+dtuu01Tp0516tyAgKsvIiQkpMilNNfGilqCU5Tt27fr+eefV9euXfX3v//dycQAAABA+eHRMh8SEqJBgwaV6pqIiAjt27dPNpvN7k2wycnJqlSpkkJDQ0u8x/79+zVu3Dg1a9ZMs2bNkre3d6mzAwAAAJ5muDfARkVF6ezZs9q2bVvBWFpamjZv3qzu3bvLbDYXjJ88ebLQ7jbHjh3T2LFjVatWLc2fP79Uy3IAAACA8sRwHxrVu3dvtWjRQs8//7xGjx6t4OBgrVq1Svn5+Ro/frzduaNGjZIkffrpp5KkrKwsPfLII7pw4YIeeeQRff7553bnN2zYUBEREWXxMgAAAIAbZrgy7+3trYULF2r69OmKi4vT5cuX1axZM02bNk1hYWEOr01PT9d///tfSdIbb7xR6Pi4ceMo8wAAADAMk81mYyuWG8BuNgAAAH8+ISGBOjt7W8knukH18d2d3s3GcGvmAQAAAFxFmQcAAAAMijIPAAAAGBRlHgAAADAoyjwAAABgUJR5AAAAwKAo8wAAAIBBUeYBAAAAg6LMAwAAAAZFmQcAAAAMijIPAAAAGBRlHgAAADAoyjwAAABgUJR5AAAAwKAo8wAAAIBBUeYBAAAAg6LMAwAAAAZFmQcAAAAMijIPAAAAGBRlHgAAADAoyjwAAABgUJR5AAAAwKAo8wAAAIBBUeYBAAAAg6LMAwAAAAZlstlsNk+HMLJz57KUn8+3EAAA4M+kapWK8vap4JG586xXlJZxSZLk5WVStWoBxZ7rmYQAAABAOXatTJd3LLMBAAAADIoyDwAAABgUZR4AAAAwKMo8AAAAYFCUeQAAAMCgKPMAAACAQVHmAQAAAIOizAMAAAAGRZkHAAAADIoyDwAAABgUZR4AAAAwKMo8AAAAYFAVPB3A6Ly8TJ6OAAAAgD+pkrqmyWaz2cooCwAAAAAXYpkNAAAAYFCUeQAAAMCgKPMAAACAQVHmAQAAAIOizAMAAAAGRZkHAAAADIoyDwAAABgUZR4AAAAwKMo8AAAAYFCUeQAAAMCgKng6wF+Z1WrVW2+9paSkJF24cEERERGaMGGCOnTo4OloOnv2rJYvX679+/fr+++/18WLF7V8+XK1b9/e09GUnJysxMRE7dq1S6mpqQoKClLLli31zDPPKCwszNPxdODAAc2fP18HDx7UuXPnFBgYqIiICD311FNq1aqVp+MVsmjRIs2cOVMRERFKSkryWI5du3YpOjq6yGMfffSR6tWrV8aJipacnKw5c+Zo3759unLliurUqaNRo0Zp0KBBHss0adIkJSYmFnv8yy+/VI0aNcowUWHHjx9XbGysvvvuO124cEG33PL/27v3sJzy/f/jz6RBpANhlEPMFOUQmZAus6mhzTQyDtGERrs2Y2xsh4nhyyWnPRODctoNxvksyphxiGHXlBnHHEsO27FEOt46qPX7w7f751bIfKtV4/24Ltdlfe777n61ru613vda7/VZjfHw8MDHx4d33nlH1Wxnz57lu+++Iy4ujmrVqtG5c2cCAgJo2rRpheZ4k+1uZGQkISEhJCYmUq9ePQYOHMioUaOoXr18duulzbZlyxZiY2OJi4vj3r179O/fnwULFpRLpjfJ9vjxY3bt2sWRI0e4fv06T58+pWXLlvj4+PDXv/5V9XyKojBz5kzOnDnD/fv3KSgooEmTJgwcOJChQ4diYGCgWrYX3b17lz59+pCTk8OePXto3bp1uWR7k3w9e/bk7t27xV7v5+fHpEmTVM0GkJmZybJlyzhw4AApKSnUq1cPBwcHFi1aVCZZpJhXUUBAAAcPHmT48OE0a9aMsLAw/Pz82LBhAx06dFA1240bMzRI7QAAFnJJREFUNwgNDaVZs2bY2Nhw5swZVfM87/vvv+f06dO4ublhY2NDSkoKmzZtwsPDg507d6pe9N2+fZuCggIGDRqEubk5mZmZRERE4O3tTWhoKN26dVM13/NSUlJYsWIFhoaGakfRGjFiBHZ2djpjaheiRY4dO8aYMWNwdHRk3LhxVK9enZs3b3L//n1Vc3l6ehY7CKAoCrNmzcLCwkL19ZecnMygQYMwMjLC29sbY2NjTp48ycKFC7l69Srffvutatni4uLw9vbGwsKCsWPHUlhYyObNm/Hy8mLPnj3Ur1+/wrKUdrtb9HfYpUsXZsyYQUJCAsuWLePx48fMmDFD1WyhoaFkZWXRtm1bUlJSyiXLH8l29uxZFi9eTPfu3Rk9ejTVq1fnwIEDjB8/nuvXrzNmzBhV8xUWFnLx4kWcnZ2xtLREX1+fs2fPMm/ePC5cuMA333yjWrYX/etf/6JatYpp7HiTfHZ2dowYMUJnzNraWvVsGRkZfPbZZ2RkZDBo0CAaNWpESkoKv//+e9mFUYQqzp07p1hbWytr167VjuXk5Ciurq6Kl5eXesH+V2ZmppKamqooiqIcOnRIsba2VmJjY1VO9cypU6eU3NxcnbEbN24obdq0Ub766iuVUr2aRqNRnJycFH9/f7Wj6Pjqq6+UYcOGKd7e3sonn3yiapbY2FjF2tpaOXTokKo5XiYjI0Pp2rWrEhgYqHaUUvn9998Va2trZcWKFWpHUVatWqVYW1srCQkJOuNjx45VbG1tlby8PJWSKYqvr6/i6OiopKWlaceSk5MVe3t7Zc6cORWapbTb3T59+ij9+/dXnj59qh1btGiR0qpVK+XGjRuqZrtz545SWFioKIqiODg4VMg2uTTZbt26pdy5c0dnrLCwUBk+fLjSrl075cmTJ6rme5nAwEDFxsZGefToUaXIFhsbq9jZ2SmLFi1SrK2tlUuXLpVLrjfN16NHD2X06NHlmuWPZpsxY4bSs2dP7XPLg/TMq+Tnn3/GwMCAQYMGacdq1KjBwIEDOXXqFA8ePFAxHdSpUwdTU1NVM7xMx44di52Wb968Oe+//z7Xrl1TKdWr1apVCzMzMzIyMtSOohUXF0d4eDhTp05VO0oxWVlZPH36VO0YOiIiIsjIyGDcuHHAs4yKoqic6uX27duHnp4eH3/8sdpRyM7OBqBevXo64/Xr16d69ero6+urEQuA06dP4+zsjLGxsXasQYMGODo68tNPP1VoltJsdxMTE0lMTMTT01NnvXl5eVFYWMjBgwdVywZgYWGBnp5euWR4mdJka9KkCRYWFjpjenp6uLq6kpOTU2KLRkXme5nGjRujKAqZmZllnOqZN8lWUFDA3Llz8fb2rrCW1jddd3l5eTx58qQcE/1/pcmWkZFBWFgYvr6+mJqakpubS15eXplnkWJeJZcvX8bKyoratWvrjLdr1w5FUbh8+bJKyaomRVF4+PBhpfoCkpWVRWpqKtevX2fRokUkJCRUiush4Nn6CgwMxMPDo1z7Hf+IyZMn4+DgQPv27Rk5ciTx8fFqRwIgJiaGFi1acOzYMT788EMcHBxwdHQkKCiIgoICtePpyM/P56effqJDhw5YWlqqHYcPPvgAgK+//porV65w//59wsPDta2FFXXKviR5eXnUqFGj2HjNmjVJSUlR/cDKiy5dugRAmzZtdMYbNmxIo0aNtI+L0nn48CFApdl35Ofnk5qayv379zl06BBr1qyhSZMmleJzvHXrVpKTk/niiy/UjlKi6Oho7O3tsbe3x9XVlW3btqkdiZMnT5KXl0f9+vXx8fGhffv22NvbM3LkSG7dulVm7yM98ypJSUkpsY/V3NwcoNLtQCq78PBwkpOTmTBhgtpRtKZNm8aBAwcAMDAwYMiQIYwaNUrlVM/s2bOHxMREli1bpnYULQMDA3r37k337t0xNTUlPj6eNWvW4OXlxc6dO7GyslI133//+1+SkpIICAjgb3/7G7a2thw9epTQ0FByc3P5+uuvVc33vKioKNLS0nB3d1c7CgDOzs6MGzeOVatWceTIEe34P/7xj3LtVS4NKysrzp49S2FhofZLRV5eHnFxccCzbXGDBg3UjKijqA+9aF/xPHNzc9l3vIG0tDR27NiBo6MjZmZmascBnn12n99PtGnThvnz56t69gqeraulS5cyduxY6tatq2qWklhbW9OpUyeaN2/O48eP2b59O//zP/9Deno6/v7+quUqKthnzJhBmzZtWLRoEQ8ePCAkJIQRI0YQERFBnTp1/s/vI8W8SnJyckq8Or3oCFFubm5FR6qyrl27xuzZs3FwcKBfv35qx9EaM2YMnp6eJCUlsXfvXvLy8sjPz1d95o6srCwWLlyIv79/pSpSOnbsqDPbj4uLCz179mTAgAGEhISwcOFCFdOBRqMhPT2diRMnancOvXr1QqPRsGXLFkaPHl1pCoJ9+/ZhYGBQ7rN0vAlLS0scHR356KOPMDEx4ZdffiE4OBgzMzOGDh2qWi4vLy9mzZrF9OnTGTlyJIWFhaxYsUJbNOfk5KiWrSRFeUrajtSoUaPCWgyqusLCQiZNmkRmZibTp09XO45W+/btWbt2LZmZmcTGxnL58mU0Go3asVi6dClmZmYMGTJE7SglWrlypc7yp59+ipeXF8uXL2fo0KEYGRmpkquoxdDc3JzQ0FDtAQMrKyv8/f3ZtWtXsYt2/whps1FJzZo1yc/PLzZeVMSXdNpXFJeSksLf//53jI2NWbJkiaqn619kY2NDt27dGDBgAKtXr+bixYuVoj99xYoVGBgY8Pnnn6sd5bVatWpF165diY2NVTsKNWvWBCjWg+7u7k5+fj7nz59XI1Yx2dnZREZG4uzsXGlaB3788UdmzpzJnDlzGDx4ML169WLevHn079+fb775hvT0dNWyDR06lFGjRhEeHk7fvn1xd3fn1q1b+Pr6AhRrhVRb0d9hSX23ubm52sfFqwUGBhIVFcX8+fOxsbFRO46WmZkZTk5O9O7dm5kzZ+Li4sLnn39eYTMDlSQhIYGtW7cSEBBQblOfljV9fX1GjBjBkydPVJ2Nr+jz6ObmplOffPjhhxgbG3P69OkyeZ/KU/m8ZV52OrToA1uZjphWVpmZmfj5+ZGZmcn3339f4mnnysLAwAAXFxcOHjyo6pG+Bw8esG7dOry8vHj48CF37tzhzp075Obmkp+fz507d1QtrEry7rvvVopMRX9fL05VWLRcGTICHD58mCdPnlSaFhuAzZs3Y2dnV6y1sGfPnmg0Gq5cuaJSsmcmTJhAdHQ0mzZtIjw8nF27dqEoCnp6ejRp0kTVbC8q+jssqbhLSUmRfUcphISEsHnzZiZPnlwpLhB/FTc3NzQaDZGRkaplWLRoEba2trRs2VK7z3j8+DHwbJ+i9tS8L9OoUSNA3W3zy/YbQJlOilE1vmL9CbVq1YoNGzaQnZ2tc+Tn3Llz2sfFy+Xm5jJq1Chu3rzJDz/8QIsWLdSO9Fo5OTkoikJ2drZqR88ePXpEfn4+QUFBBAUFFXvcxcWlXG+y8Ufcvn27UhxhtrOz49dffyU5OVmnwEtKSgKoNC02ERERGBoa0rNnT7WjaD18+LDE9VN0drIyXEBsbGxMp06dtMu//vor7dq1K5N+1rJUdMH6hQsXdO7HkJycTFJSUqW7oL2y2bRpE8HBwfj4+GjPvlRmRQd/yms2m9K4f/8+V65cwcXFpdhj/v7+1K9fn+joaBWSvdrt27cBdbfNRZ/R5ORknfHCwkJSUlKK3VPlj5JiXiVubm6sWbOGHTt24OPjAzw7bbp79246duyo+k1eKrOCggLGjx/P2bNnWb58Ofb29mpH0pGamlps45GVlcWBAwd49913i03PV5EsLS1LvOh18eLFaDQapk2bRvPmzSs+GCWvt5MnT3LixAk8PDxUyfQ8Nzc3QkND2blzp/ZCa0VR2LFjB4aGhpXi7zA1NZWYmBj69u1LrVq11I6jZWVlRXR0NLdu3dK5q+qPP/6Ivr5+pWpzgGd3HD5//nyZ3Z2xLL3//vu0aNGCbdu2MXDgQO2FkVu2bKFatWr06tVL5YSV1/79+5kzZw7u7u4EBASoHUdHWloaRkZGxS503bFjB1B89qKKNHXqVLKysnTGYmNj2bBhA1OnTlX9YFpaWhp169bVaWPJzc1l9erV1K5dW9Vtc8uWLbG2tiYiIoJRo0ZpW6j3799PVlZWmc1wJ8W8Stq3b4+bmxtBQUGkpKTQtGlTwsLCuHfvHvPnz1c7HgDLly8H0M7dvnfvXk6dOkXdunXx9vZWLdeCBQs4cuQIPXr0IC0tjb1792ofq127Nq6urqplAxg/fjw1atSgQ4cOmJubc//+fXbv3k1SUpLqxYGRkVGJ62fdunXo6+uruu7Gjx9PrVq16NChA6amply9epVt27ZhamrK2LFjVctVpE2bNnh4eLBq1SoePXqEra0tx44dIyoqismTJ1eKI7j79+/n6dOnlarFBsDX15fjx48zdOhQPvvsM4yNjfnll184fvw4Q4YMUfULbkxMDKtWraJbt26YmJhw9uxZwsLCcHd3p2/fvhWepzTb3SlTpjB69Gh8fX3p06cPCQkJbNq0CU9Pz3Kd9ak02Y4cOaJtm8rLyyM+Pl77un79+hWb672issXFxTFlyhRMTEzo2rUr4eHhOq/v1q1bud7t93X5jhw5wooVK/joo49o2rQpT548ISoqiqioKP7yl7+U67TGr8vWpUuXYq8pag/p3LlzuZ8NKs26W7lyJb1798bCwoK0tDTCwsK4efMms2bNKtfrXkrzmQgICMDPzw8vLy/69etHSkoK69atw9bWlk8++aRMcugplfmuJ39yubm5LF68mIiICNLT07GxseGf//wnTk5OakcDeOnRMgsLC53p5SrasGHD+O2330p8TO1sADt37mTv3r0kJiaSkZGBkZGRdl5ZR0dHVbO9zLBhw8jIyND5YlTR1q9fT0REBLdu3SIrKwszMzOcnZ0ZO3YsjRs3Vi3X8/Ly8li+fDl79uzh4cOHWFpa4uPjU2lmePD09OT27dv85z//UX0quxfFxcURHBzM5cuXSUtLw8LCggEDBuDr66tq1ps3bzJ79mwuXbpEdnY2zZs3Z9CgQXh7e6tyQX1pt7uHDx8mJCSEa9euYWZmxoABA/jiiy/K9QLF0mQLCAggLCysxOetX7+ezp07q5Jt9+7dr5yAoDyzwevzJSQksGrVKs6cOcPDhw+pVq0aVlZWuLu7M2zYsBJnv6uobCUpWp979uwp92L+dfkuXLhASEgIly5dIjU1lXfeeQc7OztGjhxJjx49VM1W5Pjx4wQHBxMfH4+hoSEuLi5MmjSpzFpIpZgXQgghhBCiipLZbIQQQgghhKiipJgXQgghhBCiipJiXgghhBBCiCpKinkhhBBCCCGqKCnmhRBCCCGEqKKkmBdCCCGEEKKKkmJeCCGEEEKIKkqKeSGEEKq6c+cONjY2BAcHqx1FCCGqHCnmhRDiT+7EiRPY2Njo/Gvbti0uLi5MnTpVeyvyPyo4OJjDhw+XUdqyc+jQIWxsbEhOTgZg//79tGrVSnsreiGE+DMov/s+CyGEqFQ+/vhjunfvDkBubi7x8fHs2LGDAwcOEBERgYWFxR/6uSEhIfTv3x9XV9eyjPt/dvr0aSwtLWnYsCEAp06d4r333qNu3boqJxNCiLIjxbwQQrwlbG1t6devn85Ys2bNmDt3LocOHcLHx0edYOXkzJkzdOzYUbt86tQpOnTooGIiIYQoe1LMCyHEW6xBgwYAGBgY6Ixv2rSJyMhIrl69yuPHjzExMaFLly6MHz8eS0tL4Fmvu4uLCwBhYWGEhYVpXx8fH6/9f2xsLGvWrOHcuXNoNBoaNGhA586dmTRpEmZmZjrve/ToUUJCQkhISMDY2Bh3d3cmTpxI9eqv313l5+eTmZkJQEFBARcvXsTFxYXU1FRycnJISEjg008/JTU1FQATExOqVZNuUyFE1aanKIqidgghhBDl58SJEwwfPpyxY8fi5eUFPGuzSUhIYN68eaSnpxMREYG5ubn2NS4uLtjb22NjY4OJiQkJCQns3LmTOnXqEBERgampKRqNhkOHDjFlyhQ6derE4MGDta8vOgOwdetWZs2aRcOGDfHw8MDCwoJ79+5x9OhRFixYQOvWrbVfCtq2bcvdu3cZMmQI5ubmREZGEhUVxYQJExg1alSpf8/SioyM1H4xEUKIqkqKeSGE+JN7VZH73nvvsXTpUlq2bKkzrtFoMDQ01BmLiYnBx8eHSZMm4efnpx23sbGhf//+LFiwQOf5SUlJuLq60rRpU7Zu3VqsV72wsJBq1appi/latWqxb98+bYGtKAru7u6kpaURFRX12t8zPT2dixcvArB9+3Z+++03goKCANi8eTMXL15k7ty52uc7ODhQo0aN1/5cIYSozKTNRggh3hKenp64ubkBz47MJyYmsnbtWvz9/Vm/fr3OBbBFhXxhYSHZ2dnk5+djY2ODkZERcXFxpXq/n3/+mfz8fL788ssSLzp9scXFxcVF50i5np4enTt3ZuPGjWRnZ1O7du1Xvp+xsTFOTk4ALFmyBCcnJ+3yt99+i7Ozs3ZZCCH+LKSYF0KIt0SzZs10itkePXrg6OjI4MGDCQoK4rvvvtM+FhMTw/Llyzl37hy5ubk6Pyc9Pb1U73fz5k0AWrduXarnN2nSpNiYiYkJAGlpaa8s5p/vl8/Ozub8+fO4u7uTmppKZmYmly9fxsvLS9sv/2KvvhBCVFVSzAshxFusffv2GBkZERsbqx2Li4vD19eXpk2bMnHiRCwtLalZsyZ6enpMmDCB8urO1NfXf+ljr3vP06dPF2slCgwMJDAwULs8ffp0pk+fDuheoCuEEFWZFPNCCPGWKygoIC8vT7u8b98+CgoKCA0N1TlartFo3uiGS82bNwfg8uXLWFlZlVnekrRq1Yq1a9cCsHHjRhISEpg9ezYAq1ev5t69e8yYMaNcMwghhBpkTi4hhHiLRUdHo9FosLOz04697Aj5qlWrKCwsLDZuaGhIWlpasXE3NzcMDAxYtmwZWVlZxR4vyyP8Rf3yTk5OPHjwgC5dumiXk5KStP9/vo9eCCH+DOTIvBBCvCUuXbrE3r17AcjLyyMxMZHt27djYGDA+PHjtc9zdXXlhx9+wM/PD09PTwwMDIiOjiY+Ph5TU9NiP9fe3p6YmBj+/e9/07hxY/T09Ojbty+NGjVi2rRpzJ49G3d3d/r164eFhQXJyclERkYyb968UvfTl1ZWVhaXLl3C29sbgNTUVK5du8aXX35Zpu8jhBCVhRTzQgjxlti3bx/79u0Dns0kY2JiQrdu3fD396ddu3ba5zk4OBAcHMzy5ctZsmQJNWrUwMnJiY0bN2qL5OfNnDmT2bNns3LlSrKzswHo27cvAF5eXjRt2pTVq1ezYcMG8vLyaNCgAV27dqVRo0Zl/juePn2agoICPvjgA+DZXV8VRdEuCyHEn43MMy+EEEIIIUQVJT3zQgghhBBCVFFSzAshhBBCCFFFSTEvhBBCCCFEFSXFvBBCCCGEEFWUFPNCCCGEEEJUUVLMCyGEEEIIUUVJMS+EEEIIIUQVJcW8EEIIIYQQVZQU80IIIYQQQlRRUswLIYQQQghRRf0/YZvF9UW2MDUAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "# Create a barplot showing the MCC score for each batch of test samples.\n", - "ax = sns.barplot(x=list(range(len(matthews_set))), y=matthews_set, ci=None)\n", - "\n", - "plt.title('MCC Score per Batch')\n", - "plt.ylabel('MCC Score (-1 to +1)')\n", - "plt.xlabel('Batch #')\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1YrjAPX2V-l4" - }, - "source": [ - "Now we'll combine the results for all of the batches and calculate our final MCC score." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "oCYZa1lQ8Jn8", - "outputId": "b4650298-0e35-4ed8-be13-83f074a617ed" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total MCC: 0.514\n" - ] - } - ], - "source": [ - "# Combine the results across all batches. \n", - "flat_predictions = np.concatenate(predictions, axis=0)\n", - "\n", - "# For each sample, pick the label (0 or 1) with the higher score.\n", - "flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n", - "\n", - "# Combine the correct labels for each batch into a single list.\n", - "flat_true_labels = np.concatenate(true_labels, axis=0)\n", - "\n", - "# Calculate the MCC\n", - "mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n", - "\n", - "print('Total MCC: %.3f' % mcc)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jXx0jPc4HUfZ" - }, - "source": [ - "Cool! In about half an hour and without doing any hyperparameter tuning (adjusting the learning rate, epochs, batch size, ADAM properties, etc.) we are able to get a good score. \n", - "\n", - "> *Note: To maximize the score, we should remove the \"validation set\" (which we used to help determine how many epochs to train for) and train on the entire training set.*\n", - "\n", - "The library documents the expected accuracy for this benchmark [here](https://huggingface.co/transformers/examples.html#glue) as `49.23`.\n", - "\n", - "You can also look at the official leaderboard [here](https://gluebenchmark.com/leaderboard/submission/zlssuBTm5XRs0aSKbFYGVIVdvbj1/-LhijX9VVmvJcvzKymxy). \n", - "\n", - "Note that (due to the small dataset size?) the accuracy can vary significantly between runs.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GfjYoa6WmkN6" - }, - "source": [ - "# Conclusion" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xlQG7qgkmf4n" - }, - "source": [ - "This notebook demonstrates that with a pre-trained BERT model you can quickly and effectively create a high quality model with minimal effort and training time using the pytorch interface, regardless of the specific NLP task you are interested in." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YUmsUOIv8EUO" - }, - "source": [ - "# Appendix\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "q2079Qyn8Mt8" - }, - "source": [ - "### Saving & Loading Fine-Tuned Model\n", - "\n", - "This first cell (taken from `run_glue.py` [here](https://github.com/huggingface/transformers/blob/35ff345fc9df9e777b27903f11fa213e4052595b/examples/run_glue.py#L495)) writes the model and tokenizer out to disk." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "6ulTWaOr8QNY", - "outputId": "1b73b37b-2598-4992-d6d7-0649f410b5c0" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving model to ./model_save/\n" - ] - }, - { - "data": { - "text/plain": [ - "('./model_save/tokenizer_config.json',\n", - " './model_save/special_tokens_map.json',\n", - " './model_save/vocab.txt',\n", - " './model_save/added_tokens.json')" - ] - }, - "execution_count": 31, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "import os\n", - "\n", - "# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n", - "\n", - "output_dir = './model_save/'\n", - "\n", - "# Create output directory if needed\n", - "if not os.path.exists(output_dir):\n", - " os.makedirs(output_dir)\n", - "\n", - "print(\"Saving model to %s\" % output_dir)\n", - "\n", - "# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n", - "# They can then be reloaded using `from_pretrained()`\n", - "model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training\n", - "model_to_save.save_pretrained(output_dir)\n", - "tokenizer.save_pretrained(output_dir)\n", - "\n", - "# Good practice: save your training arguments together with the trained model\n", - "# torch.save(args, os.path.join(output_dir, 'training_args.bin'))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Z-tjHkR7lc1I" - }, - "source": [ - "Let's check out the file sizes, out of curiosity." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "mqMzI3VTCZo5", - "outputId": "96104fe5-67d0-4310-d778-58da5194c2e1" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "total 428000K\n", - "-rw-r--r-- 1 root root 1K Feb 2 17:10 config.json\n", - "-rw-r--r-- 1 root root 427757K Feb 2 17:10 pytorch_model.bin\n", - "-rw-r--r-- 1 root root 1K Feb 2 17:10 special_tokens_map.json\n", - "-rw-r--r-- 1 root root 1K Feb 2 17:10 tokenizer_config.json\n", - "-rw-r--r-- 1 root root 227K Feb 2 17:10 vocab.txt\n" - ] - } - ], - "source": [ - "!ls -l --block-size=K ./model_save/" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fr_bt2rFlgDn" - }, - "source": [ - "The largest file is the model weights, at around 418 megabytes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-WUFUIQ8Cu8D", - "outputId": "b0c9b6c6-5fb8-4d61-d28a-be4324be5a5b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "-rw-r--r-- 1 root root 418M Feb 2 17:10 ./model_save/pytorch_model.bin\n" - ] - } - ], - "source": [ - "!ls -l --block-size=M ./model_save/pytorch_model.bin" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dzGKvOFAll_e" - }, - "source": [ - "To save your model across Colab Notebook sessions, download it to your local machine, or ideally copy it to your Google Drive." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Trr-A-POC18_" - }, - "outputs": [], - "source": [ - "# Mount Google Drive to this Notebook instance.\n", - "from google.colab import drive\n", - " drive.mount('/content/drive')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NxlZsafTC-V5" - }, - "outputs": [], - "source": [ - "# Copy the model files to a directory in your Google Drive.\n", - "!cp -r ./model_save/ \"./drive/Shared drives/ChrisMcCormick.AI/Blog Posts/BERT Fine-Tuning/\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "W0vstijw85SZ" - }, - "source": [ - "The following functions will load the model back from disk." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "nskPzUM084zL" - }, - "outputs": [], - "source": [ - "# Load a trained model and vocabulary that you have fine-tuned\n", - "model = model_class.from_pretrained(output_dir)\n", - "tokenizer = tokenizer_class.from_pretrained(output_dir)\n", - "\n", - "# Copy the model to the GPU.\n", - "model.to(device)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "BERT Fine-Tuning Sentence Classification v4.ipynb", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "0358864aed7145e5a8b1ae70b33d8011": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "04b9332a3fa9454da1a7169353b99221": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cf5d78f166d54aadbd8f68976f1e9ccc", - "placeholder": "​", - "style": "IPY_MODEL_ce8f59b53f894031b4ff4e0e1b63a61f", - "value": "Downloading: 100%" - } - }, - "0a2a650a8e68491c85be1d84112645c3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_801f8b39de0c488baf8c92705d3e2905", - "placeholder": "​", - "style": "IPY_MODEL_0a49f12eb51244f2882a979483acd105", - "value": "Downloading: 100%" - } - }, - "0a49f12eb51244f2882a979483acd105": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "0d0fa745757f4c93a5e3ce9c2fadc9a3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "176c3ac86a8f4dfa9ff221d67076edc5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "17b79c0e5d194a07bfff70f84ec10858": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "24fd7d7781a448f6b4164bbeb64888b3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2a70c1b9bda34e6a89ca14b3b663b959": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b59dd15b11a64b8cace572993e8323ba", - "max": 466062, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_f57a3f9b473543bda0c27ebcba001fad", - "value": 466062 - } - }, - "37ea42eb1ecd44eab1ea46db51fe2800": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0d0fa745757f4c93a5e3ce9c2fadc9a3", - "placeholder": "​", - "style": "IPY_MODEL_24fd7d7781a448f6b4164bbeb64888b3", - "value": "Downloading: 100%" - } - }, - "3a5ad7be77ce4127b434b749ad14b00a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3dfad6bb2985460da31cc42d7e88168c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "3eb262cbc05a4a48808623fbd11d7649": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "44e2066d588b496f97aa00cf51c9447c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_93d601fafc26454b8de64f626daf3074", - "IPY_MODEL_2a70c1b9bda34e6a89ca14b3b663b959", - "IPY_MODEL_e9afd7edefb44fe4847ab5a9c0b24999" - ], - "layout": "IPY_MODEL_176c3ac86a8f4dfa9ff221d67076edc5" - } - }, - "521bcdb6786b4ba1a361cc8de1c97036": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_04b9332a3fa9454da1a7169353b99221", - "IPY_MODEL_a29e31e998974e758d09daf1a6186610", - "IPY_MODEL_d2f54cdaba8e4a04bdb3b8d279afe5ef" - ], - "layout": "IPY_MODEL_da7f2189149d4dcb8d012ffa2b9e34bb" - } - }, - "54829969375b46d0a07900700a6e21a3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "6494768c95954aeaa092d1b8f494bf65": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cca16e2b7d7446ccb05a2a09fdb1134b", - "max": 440473133, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_17b79c0e5d194a07bfff70f84ec10858", - "value": 440473133 - } - }, - "747ba5c2117a42b28539371fe04b19ad": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_37ea42eb1ecd44eab1ea46db51fe2800", - "IPY_MODEL_ef3a445c93764273b132d56d8b7545a3", - "IPY_MODEL_ca26fc3ff341416f820e3ea94aa83188" - ], - "layout": "IPY_MODEL_c91181d3293d457a8c6d32aca47e5c27" - } - }, - "7d6e576e3c464b5693c51d35032107cb": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "801f8b39de0c488baf8c92705d3e2905": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "86fc378943aa48ee893bef7404abc96a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "879a098d294d444c8176ce6221846d95": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "89c02af3d0f74c18a8e294f78b1f961e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_972be3faee024479ab0667892594d00c", - "placeholder": "​", - "style": "IPY_MODEL_3dfad6bb2985460da31cc42d7e88168c", - "value": " 226k/226k [00:00<00:00, 568kB/s]" - } - }, - "8f66495735614898b617c27c0a89255d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9110261cd8b94147b72dec393886b308": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "93853fa079c546b58b1c9ed8e586ae6c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_e4306d918bbb47abae367df2c7ac2e9d", - "IPY_MODEL_ac1cb16ed2ef4e6ab40e790e5e9ce47f", - "IPY_MODEL_89c02af3d0f74c18a8e294f78b1f961e" - ], - "layout": "IPY_MODEL_8f66495735614898b617c27c0a89255d" - } - }, - "93d601fafc26454b8de64f626daf3074": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fb5ff76f955942dda1bb6c98f827b893", - "placeholder": "​", - "style": "IPY_MODEL_7d6e576e3c464b5693c51d35032107cb", - "value": "Downloading: 100%" - } - }, - "962dda4c15404a2c84caf50e208164b9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "972be3faee024479ab0667892594d00c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a29e31e998974e758d09daf1a6186610": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_3a5ad7be77ce4127b434b749ad14b00a", - "max": 570, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_c9f13c482ecc4448816bec89e13816ab", - "value": 570 - } - }, - "a868dfc37b55447d8bab7350ca8cd0cf": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ab13bda5dc204c3cbb633535494eb453": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ac1cb16ed2ef4e6ab40e790e5e9ce47f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_dbb229077bcf437aa25c5a16276e9c48", - "max": 231508, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_3eb262cbc05a4a48808623fbd11d7649", - "value": 231508 - } - }, - "b59dd15b11a64b8cace572993e8323ba": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "bf663b176284485e8db6b888bbd73d00": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_0a2a650a8e68491c85be1d84112645c3", - "IPY_MODEL_6494768c95954aeaa092d1b8f494bf65", - "IPY_MODEL_d4bd45ba11a94400a6b2afb62638506c" - ], - "layout": "IPY_MODEL_0358864aed7145e5a8b1ae70b33d8011" - } - }, - "c91181d3293d457a8c6d32aca47e5c27": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c9f13c482ecc4448816bec89e13816ab": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "ca26fc3ff341416f820e3ea94aa83188": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e389ecd683b64c28be15da07866f73c1", - "placeholder": "​", - "style": "IPY_MODEL_879a098d294d444c8176ce6221846d95", - "value": " 28.0/28.0 [00:00<00:00, 741B/s]" - } - }, - "cca16e2b7d7446ccb05a2a09fdb1134b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ce8f59b53f894031b4ff4e0e1b63a61f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "cf5d78f166d54aadbd8f68976f1e9ccc": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d0970cd03bc74b5da9f913fda325c1e0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "d2f54cdaba8e4a04bdb3b8d279afe5ef": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_86fc378943aa48ee893bef7404abc96a", - "placeholder": "​", - "style": "IPY_MODEL_f2dcfd7f0f9a48b08730714d44d18218", - "value": " 570/570 [00:00<00:00, 14.3kB/s]" - } - }, - "d4bd45ba11a94400a6b2afb62638506c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_962dda4c15404a2c84caf50e208164b9", - "placeholder": "​", - "style": "IPY_MODEL_54829969375b46d0a07900700a6e21a3", - "value": " 420M/420M [00:13<00:00, 35.5MB/s]" - } - }, - "da0235bd5a3642eba1b5749afe2c6f61": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "da7f2189149d4dcb8d012ffa2b9e34bb": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "dbb229077bcf437aa25c5a16276e9c48": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e389ecd683b64c28be15da07866f73c1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e4306d918bbb47abae367df2c7ac2e9d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ab13bda5dc204c3cbb633535494eb453", - "placeholder": "​", - "style": "IPY_MODEL_a868dfc37b55447d8bab7350ca8cd0cf", - "value": "Downloading: 100%" - } - }, - "e9afd7edefb44fe4847ab5a9c0b24999": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9110261cd8b94147b72dec393886b308", - "placeholder": "​", - "style": "IPY_MODEL_d0970cd03bc74b5da9f913fda325c1e0", - "value": " 455k/455k [00:00<00:00, 867kB/s]" - } - }, - "edeb0997fdb84daba10b4a9a102adee5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ef3a445c93764273b132d56d8b7545a3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_edeb0997fdb84daba10b4a9a102adee5", - "max": 28, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_da0235bd5a3642eba1b5749afe2c6f61", - "value": 28 - } - }, - "f2dcfd7f0f9a48b08730714d44d18218": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f57a3f9b473543bda0c27ebcba001fad": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "fb5ff76f955942dda1bb6c98f827b893": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/week08_question_answering/README.md b/week08_question_answering/README.md deleted file mode 100644 index f2091ab..0000000 --- a/week08_question_answering/README.md +++ /dev/null @@ -1,2 +0,0 @@ -Question Answering and TTS: -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/week08_question_answering/practice_question_answering_and_tts.ipynb) \ No newline at end of file diff --git a/week08_question_answering/lect08_Question_Answering.pdf b/week08_question_answering/lect08_Question_Answering.pdf deleted file mode 100644 index 33e1740..0000000 Binary files a/week08_question_answering/lect08_Question_Answering.pdf and /dev/null differ diff --git a/week08_question_answering/practice_question_answering_and_tts.ipynb b/week08_question_answering/practice_question_answering_and_tts.ipynb deleted file mode 100644 index 0bb69a6..0000000 --- a/week08_question_answering/practice_question_answering_and_tts.ipynb +++ /dev/null @@ -1,3261 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "DphyQXreodzp" - }, - "source": [ - "# Practice: Question Answering with a Fine-Tuned BERT (and TTS example)\n", - "\n", - "This notebook is based on great [post and corresponding notebook](https://mccormickml.com/2020/03/10/question-answering-with-a-fine-tuned-BERT/) *by Chris McCormick*. It contains some minor changes and additions (especially parts 3 and 4)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6_mAnIPKaXyw" - }, - "source": [ - "What does it mean for BERT to achieve \"human-level performance on Question Answering\"? Is BERT the greatest search engine ever, able to find the answer to any question we pose it?\n", - "\n", - "In **Part 1** of this notebook, we will discuss what it really means to apply BERT to QA, and illustrate the details.\n", - "\n", - "**Part 2** contains example code--we'll be downloading a model that's *already been fine-tuned* for question answering, and try it out on our own text! \n", - "\n", - "In **Part 3** we will apply the same approach to Russian language using the model pre-trained on SberQuAD dataset.\n", - "\n", - "And in **Part 4** and **Part 5** we will generate question and answer as audio in english and russian languages.\n", - "\n", - "**Links**\n", - "\n", - "* The [video walkthrough](https://youtu.be/l8ZYCvgGu0o) on this topic. \n", - "* The [original blog post](https://mccormickml.com/2020/03/10/question-answering-with-a-fine-tuned-BERT/) version.\n", - "* The [original Colab Notebook](https://colab.research.google.com/drive/1uSlWtJdZmLrI3FCNIlUHFxwAJiSu2J0-)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "-5uC3kgC7rRK", - "outputId": "97c24371-376c-4f56-a8fa-460d0d398bca" - }, - "outputs": [], - "source": [ - "!pip install -U transformers deeppavlov unidecode omegaconf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "d5YCFVasLbaM" - }, - "outputs": [], - "source": [ - "# This cell is optional and needed only for Russian language inference\n", - "\n", - "\n", - "# !python -m deeppavlov install squad_ru_rubert\n", - "\n", - "# # Pre-downloading the BERT for Russian language. Same result can be achieved with\n", - "# # `!python -m deeppavlov download squad_ru_rubert`\n", - "# # But it works significantly slower.\n", - "# !wget -nc https://www.dropbox.com/s/7za1o6vaffbdlcg/rubert_cased_L-12_H-768_A-12_v1.tar.gz\n", - "# !mkdir -p /root/.deeppavlov/downloads/bert_models/\n", - "# !tar -xzvf rubert_cased_L-12_H-768_A-12_v1.tar.gz -C /root/.deeppavlov/downloads/bert_models\n", - "\n", - "# !wget -nc https://www.dropbox.com/s/ns8280pd9t9n9dc/squad_model_ru_rubert.tar.gz\n", - "# !mkdir -p /root/.deeppavlov/models/\n", - "# !tar -xzvf squad_model_ru_rubert.tar.gz -C /root/.deeppavlov/models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "OfzJwB_17rRM", - "outputId": "f48ad518-2d66-4912-b720-2d772fb04b4f" - }, - "outputs": [], - "source": [ - "import torch\n", - "\n", - "assert torch.cuda.is_available(), 'Tacotron2 by NVIDIA infers only on GPU, so the Part 4 will not work on CPU-only machine'\n", - "\n", - "device = torch.device('cuda:0')\n", - "tacotron2 = torch.hub.load('nvidia/DeepLearningExamples:torchhub', 'nvidia_tacotron2', **{'map_location': device})\n", - "tacotron2.to(device)\n", - "tacotron2.eval()\n", - "\n", - "waveglow = torch.hub.load('nvidia/DeepLearningExamples:torchhub', 'nvidia_waveglow')\n", - "waveglow = waveglow.remove_weightnorm(waveglow)\n", - "waveglow.to(device)\n", - "waveglow.eval();" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "X2bUvKUffHNY" - }, - "source": [ - "## Part 1: Applying BERT to Question Answering" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Su7fixBdiUex" - }, - "source": [ - "### The SQuAD v1.1 Benchmark" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bT5ESKDxfnLf" - }, - "source": [ - "When someone mentions \"Question Answering\" as an application of BERT, what they are really referring to is applying BERT to the Stanford Question Answering Dataset (SQuAD).\n", - "\n", - "The task posed by the SQuAD benchmark is a little different than you might think. Given a question, and *a passage of text containing the answer* (often refered to as context), BERT needs to highlight the \"span\" of text corresponding to the correct answer. \n", - "\n", - "The SQuAD homepage has a fantastic tool for exploring the questions and reference text for this dataset, and even shows the predictions made by top-performing models.\n", - "\n", - "For example, here are some [interesting examples](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/Super_Bowl_50.html?model=r-net+%20(ensemble)%20(Microsoft%20Research%20Asia)&version=1.1) on the topic of Super Bowl 50.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_xN5f1bxf6K_" - }, - "source": [ - "### BERT Input Format" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ctum5SK6f9uP" - }, - "source": [ - "To feed a QA task into BERT, we pack both the question and the reference text into the input.\n", - "\n", - "![Input format for QA](https://raw.githubusercontent.com/neychev/made_nlp_course/master/week10_speech_distillation_and_perspectives/img/input_formatting_image.png)\n", - "*Image credits: [Chris McCormick](https://mccormickml.com/2020/03/10/question-answering-with-a-fine-tuned-BERT/)*\n", - "\n", - "The two pieces of text are separated by the special `[SEP]` token. \n", - "\n", - "> _Side note:_ Original BERT also uses \"Segment Embeddings\" to differentiate the question from the reference text. These are simply two embeddings (for segments \"A\" and \"B\") that BERT learned, and which it adds to the token embeddings before feeding them into the input layer. However today we will be using DistilBERT model, which relies solely on the special tokens." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xs31dcrPg5Tg" - }, - "source": [ - "### Start & End Token Classifiers" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lvOdUa9Wg-Uv" - }, - "source": [ - "BERT needs to highlight a \"span\" of text containing the answer--this is represented as simply predicting which token marks the start of the answer, and which token marks the end.\n", - "\n", - "![Start token classification](https://raw.githubusercontent.com/neychev/made_nlp_course/master/week10_speech_distillation_and_perspectives/img/start_token_classification_image.png)\n", - "*Image credits: [Chris McCormick](https://mccormickml.com/2020/03/10/question-answering-with-a-fine-tuned-BERT/)*\n", - "\n", - "For every token in the text, we feed its final embedding into the start token classifier. The start token classifier only has a single set of weights (represented by the blue \"start\" rectangle in the above illustration) which it applies to every word.\n", - "\n", - "After taking the dot product between the output embeddings and the 'start' weights, we apply the softmax activation to produce a probability distribution over all of the words. Whichever word has the highest probability of being the start token is the one that we pick.\n", - "\n", - "We repeat this process for the end token--we have a separate weight vector this.\n", - "\n", - "![End token classification](https://raw.githubusercontent.com/neychev/made_nlp_course/master/week10_speech_distillation_and_perspectives/img/end_token_classification_image.png)\n", - "*Image credits: [Chris McCormick](https://mccormickml.com/2020/03/10/question-answering-with-a-fine-tuned-BERT/)*" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "457VPa20fZzY" - }, - "source": [ - "## Part 2: Example Code" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9wpCLgVCkki5" - }, - "source": [ - "In the example code below, we'll be downloading a model that's *already been fine-tuned* for question answering, and try it out on our own text.\n", - "\n", - "If you do want to fine-tune on your own dataset, it is possible to fine-tune BERT for question answering yourself. See [run_squad.py](https://github.com/huggingface/transformers/blob/master/examples/run_squad.py) in the `transformers` library. However, you may find that the \"fine-tuned-on-squad\" model already does a good job, even if your text is from a different domain." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gVq-TuylYRDW" - }, - "source": [ - "### 1. Load Fine-Tuned BERT" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "f9nhy3PzGQ44" - }, - "source": [ - "This example uses the `transformers` [library](https://github.com/huggingface/transformers/) by huggingface. We've already installed it in the top of this notebook.\n", - "\n", - "For Question Answering we use the `DistilBertForQuestionAnswering` class from the `transformers` library.\n", - "\n", - "This class supports fine-tuning, but for this example we will keep things simpler and load a BERT model that has already been fine-tuned for the SQuAD benchmark.\n", - "\n", - "The `transformers` library has a large collection of pre-trained models which you can reference by name and load easily. The full list is in their documentation [here](https://huggingface.co/transformers/pretrained_models.html)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 231, - "referenced_widgets": [ - "f10f304cd83a424bb1acce2a79cac5a5", - "60050037056f4231ab5c6afa8ae9ec27", - "8d9aded51b004aababcda8e76798a534", - "e4c848c91a394012bbab804ff3e888a8", - "71f49045ee564ca3b485270db1e69792", - "1b981c0d4a0a4bdca32bb12960ce8a7a", - "07f8ddf6b1ae4143862d4e1888b253cb", - "32364789d1ff47fc8c4056a7b48a7634", - "7d1f7a87ed484be999ea793bf4c34dd3", - "a51916d0bd0a43fdba9e46f20a80d6a2", - "eef6db037885440f891824e5a8f5358f", - "1d6782097f9a45758fea597727c42da3", - "301ac3b9e5d04242b05d5943367bf048", - "18a7445e957446ffa372185756f6b64c", - "c6e9545c4c03427591aac4f49c774ff7", - "0ae8437438404098a5a6206b05ecf453", - "7669b34be42a4453b3e3ba0e60faee75", - "e36ef4cbcb4c466283f85f5d523d66cd", - "49107e49101e469abe491d5fb4e4466a", - "d434a8d87ce24bf7865ba75298d95729", - "2165f1cd0739407a92079f643ac74ae3", - "494fb6f653f845b3837db00241bf4798", - "3056ad61695a42ffadd7954a5a7503ed", - "2061f0fcebbd4806a5b7308ea3079a7b", - "18a3cd219c8642d3b6001f4dbed8a1a9", - "e4099cf7a2c1426d844c1febc644001a", - "fe8d0c53f5704e42b7c271fecaa48851", - "4f9a99283eec4c31b8ed463a0a5c4366", - "c30803de30894426958ad02fbc9f0002", - "3ffe97d1a93d46c5ad72625841831419", - "b12f548d82ca4586b61f701cb1abacb4", - "f7819d286ecc424e83c1d6e509ab1486", - "ed11dd778e2f49af96e269c4c37c6417", - "17f2ae98a8d1440b8beebc2f8aaff051", - "7ce0c95f422148ec92feee15262845f7", - "0bae2a0bb31b46f39a29c450e8c983c9", - "3d5da11d2072432face40e6f729f644b", - "f8318de1c5c74051adcf59d5f6261903", - "228cd2bb1c5a48088cd48fed5e8ee2e5", - "1c750fddb4f14ac397026785a004921d", - "e99dd095993f4053b5ba2a9c306582b9", - "f7d2834f66ac48719f1fd47dda33d7dc", - "3f4e292341444c128a6941020bf78ea7", - "0695270fa12047a69a2dcef569155b25", - "85e24abbc05d4937ac86793c9c36e1f5", - "3ec7e9e71b5e4feea47fe12b5f751d57", - "8ca5505516334e50b305f559633213fd", - "96eb7cdf8e9b48bfb3a619c3a6b4e3e0", - "c97f4996f8d14b9c94cfefaf1c7adfea", - "59df97cee8854be19b28d63f714816be", - "28e9a4b59d8940228874918132a4379a", - "26506f018a824fd2981562f17ada25e9", - "0e8242194958409abe8c5830767ae7b8", - "68976e9d2ac1420cb548307c96c5a8a6", - "1aaa4d2569574c89a646d202738a829e" - ] - }, - "id": "apS1yS6CdRyX", - "outputId": "077a5f84-dcf5-45ec-973c-944f48a4d101" - }, - "outputs": [], - "source": [ - "from transformers import DistilBertTokenizer, DistilBertForQuestionAnswering\n", - "\n", - "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased-distilled-squad')\n", - "model = DistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased-distilled-squad')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8imoOxoqGZ0h" - }, - "source": [ - "> _Side note:_ Apparently the vocabulary of this model is identicaly to the one in bert-base-uncased. You can load the tokenizer from `bert-base-uncased` and that works just as well." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "I__1ubvcZYow" - }, - "source": [ - "### 2. Ask a Question" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "o8MQ7b-GJIcM" - }, - "source": [ - "Now we're ready to feed in an example!\n", - "\n", - "A QA example consists of a question and a passage of text containing the answer to that question." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "kWzZP4EN-Zxg" - }, - "outputs": [], - "source": [ - "question = \"How many parameters does BERT-large have?\"\n", - "context = (\n", - " \"BERT-large is really big... it has 24-layers and an embedding size of 1,024, \"\n", - " \"for a total of 340M parameters! Altogether it is 1.34GB, so expect it to \"\n", - " \"take a couple minutes to download to your Colab instance.\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "llLvxhScKLZn" - }, - "source": [ - "We'll need to run the BERT tokenizer against both the `question` and the `context`. To feed these into BERT, we actually concatenate them together and place the special `[SEP]` token in between.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tYoX33CfKGsr", - "outputId": "033d60bf-6741-441b-d3ef-bfab2050c046" - }, - "outputs": [], - "source": [ - "# Apply the tokenizer to the input text, treating them as a text-pair.\n", - "input_ids = tokenizer.encode(question, context)\n", - "print(f'The input has a total of {len(input_ids)} tokens.')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pNRVuaKSNFG8" - }, - "source": [ - "Just to see exactly what the tokenizer is doing, let's print out the tokens with their IDs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Iow838yPNDTv", - "outputId": "9d3bfc3a-2386-4d1a-ad49-b35f4d959c98" - }, - "outputs": [], - "source": [ - "# BERT only needs the token IDs, but for the purpose of inspecting the \n", - "# tokenizer's behavior, let's also get the token strings and display them.\n", - "tokens = tokenizer.convert_ids_to_tokens(input_ids)\n", - "\n", - "# Display tokens and ids as table.\n", - "# For each token and its id...\n", - "for token, token_id in zip(tokens, input_ids):\n", - " \n", - " # If this is the [SEP] token, add some space around it to make it stand out.\n", - " if token_id == tokenizer.sep_token_id:\n", - " print()\n", - " \n", - " # Print the token string and its ID in two columns.\n", - " print('{:<12} {:>6,}'.format(token, token_id))\n", - "\n", - " if token_id == tokenizer.sep_token_id:\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CNwhEw0kQPBN" - }, - "source": [ - "We're ready to feed our example into the model!\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "HK0obn5x-1EI", - "outputId": "89898262-bfe5-4fac-ea23-f211f3703faf" - }, - "outputs": [], - "source": [ - "import torch\n", - "\n", - "inputs = tokenizer(question, context, return_tensors='pt')\n", - "with torch.no_grad():\n", - " outputs = model(**inputs)\n", - "\n", - "start_scores = outputs.start_logits\n", - "end_scores = outputs.end_logits\n", - "start_scores" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "a30sBTcqQv6X" - }, - "source": [ - ">*Side Note: Where's the padding?*\n", - ">\n", - "> The original [example code](https://huggingface.co/transformers/model_doc/bert.html?highlight=bertforquestionanswering#transformers.BertForQuestionAnswering) does not perform any padding. I suspect that this is because we are only feeding in a *single example*. If we instead fed in a batch of examples, then we would need to pad or truncate all of the samples in the batch to a single length, and supply an attention mask to tell BERT to ignore the padding tokens. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mBdS_QkIbDzh" - }, - "source": [ - "Now we can highlight the answer just by looking at the most probable start and end words. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "LeUQ44hAJmn9", - "outputId": "c9afd2de-5a24-480d-cb76-fa3726b549cd" - }, - "outputs": [], - "source": [ - "# Find the tokens with the highest `start` and `end` scores.\n", - "answer_start = torch.argmax(start_scores)\n", - "answer_end = torch.argmax(end_scores)\n", - "\n", - "# Combine the tokens in the answer and print it out.\n", - "answer = ' '.join(tokens[answer_start : answer_end + 1])\n", - "\n", - "print(f'Answer: \"{answer}\"')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "twMUWmr2brRw" - }, - "source": [ - "It got it right! Awesome :)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cERYCGKMbOXX" - }, - "source": [ - "> *Side Note: It's a little naive to pick the highest scores for start and end--what if it predicts an end word that's before the start word?! The correct implementation is to pick the highest total score for which end >= start.*" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "N6j2znkwXYsn" - }, - "source": [ - "With a little more effort, we can reconstruct any words that got broken down into subwords." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "pBrAsWMJrw7i", - "outputId": "dd9159ea-7377-4a5c-c499-e9815f8c677f" - }, - "outputs": [], - "source": [ - "answer = tokenizer.convert_tokens_to_string(tokens[answer_start : answer_end + 1])\n", - "print(f'Answer: \"{answer}\"')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-hh6nkIdXq-O" - }, - "source": [ - "### 3. Visualizing Scores" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-hG2YCHYXtg-" - }, - "source": [ - "Let's see what the scores were for all of the words. The following cells generate bar plots showing the start and end scores for every word in the input." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "gkKFa73eJkPE" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "\n", - "# Use plot styling from seaborn.\n", - "sns.set(style='darkgrid')\n", - "\n", - "# Increase the plot size and font size.\n", - "plt.rcParams['figure.figsize'] = (16, 8)\n", - "plt.rcParams['font.size'] = 16" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "W7kazkb2iEuQ" - }, - "source": [ - "Retrieve all of the start and end scores, and use all of the tokens as x-axis labels." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "C56AtMg2UBxN" - }, - "outputs": [], - "source": [ - "# Pull the scores out of PyTorch Tensors and convert them to 1D numpy arrays.\n", - "start_scores = start_scores.numpy().flatten()\n", - "end_scores = end_scores.numpy().flatten()\n", - "\n", - "# We'll use the tokens as the x-axis labels. In order to do that, they all need\n", - "# to be unique, so we'll add the token index to the end of each one.\n", - "token_labels = []\n", - "for (i, token) in enumerate(tokens):\n", - " token_labels.append('{:} - {:>2}'.format(token, i))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NIaW7RyTiLeu" - }, - "source": [ - "Create a bar plot showing the score for every input word being the \"start\" word." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 581 - }, - "id": "y6OAV1dL3-UB", - "outputId": "4b56edcf-a145-4212-ac1b-0cb5c6b187bf" - }, - "outputs": [], - "source": [ - "# Create a barplot showing the start word score for all of the tokens.\n", - "ax = sns.barplot(x=token_labels, y=start_scores, ci=None)\n", - "\n", - "# Turn the xlabels vertical.\n", - "ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha=\"center\")\n", - "\n", - "# Turn on the vertical grid to help align words to scores.\n", - "ax.grid(True)\n", - "\n", - "plt.title('Start Word Scores');" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zIwrF7y6iS1l" - }, - "source": [ - "Create a second bar plot showing the score for every input word being the \"end\" word." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 581 - }, - "id": "6tXEqIp-Tzou", - "outputId": "a2aff3b3-8767-4870-bb78-d32fae66ce29" - }, - "outputs": [], - "source": [ - "# Create a barplot showing the end word score for all of the tokens.\n", - "ax = sns.barplot(x=token_labels, y=end_scores, ci=None)\n", - "\n", - "# Turn the xlabels vertical.\n", - "ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha=\"center\")\n", - "\n", - "# Turn on the vertical grid to help align words to scores.\n", - "ax.grid(True)\n", - "\n", - "plt.title('End Word Scores');" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "awgi7Z_a9KSq" - }, - "source": [ - "**Alternate View**\n", - "\n", - "I also tried visualizing both the start and end scores on a single bar plot, but I think it may actually be more confusing then seeing them separately. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "m4VUk6R05uXS" - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "# Store the tokens and scores in a DataFrame. \n", - "# Each token will have two rows, one for its start score and one for its end\n", - "# score. The \"marker\" column will differentiate them. A little wacky, I know.\n", - "scores = []\n", - "for (i, token_label) in enumerate(token_labels):\n", - "\n", - " # Add the token's start score as one row.\n", - " scores.append({'token_label': token_label, \n", - " 'score': start_scores[i],\n", - " 'marker': 'start'})\n", - " \n", - " # Add the token's end score as another row.\n", - " scores.append({'token_label': token_label, \n", - " 'score': end_scores[i],\n", - " 'marker': 'end'})\n", - " \n", - "df = pd.DataFrame(scores)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 508 - }, - "id": "07xyo-I97Ntt", - "outputId": "0cd0ac4d-d9b2-4d0b-86b2-0ee77241e2bc" - }, - "outputs": [], - "source": [ - "# Draw a grouped barplot to show start and end scores for each word.\n", - "# The \"hue\" parameter is where we tell it which datapoints belong to which\n", - "# of the two series.\n", - "plot = sns.catplot(\n", - " x=\"token_label\", y=\"score\", hue=\"marker\",\n", - " data=df, kind=\"bar\", height=6, aspect=4\n", - ")\n", - "\n", - "# Turn the xlabels vertical.\n", - "plot.set_xticklabels(plot.ax.get_xticklabels(), rotation=90, ha=\"center\")\n", - "\n", - "# Turn on the vertical grid to help align words to scores.\n", - "plot.ax.grid(True);" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8UyBYNmeegGf" - }, - "source": [ - "### 4. More Examples" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MWtcRpPef-Ce" - }, - "source": [ - "Turn the QA process into a function so we can easily try out other examples." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rH8NbBlsfxZ_" - }, - "outputs": [], - "source": [ - "def answer_question(question, context):\n", - " # ======== Tokenize ========\n", - " # Apply the tokenizer to the input text, treating them as a text-pair.\n", - " inputs = tokenizer(question, context, return_tensors='pt')\n", - " input_ids = inputs.input_ids.numpy().flatten()\n", - "\n", - " # ======== Evaluate ========\n", - " # Run our example question through the model.\n", - " outputs = model(**inputs)\n", - " start_scores = outputs.start_logits\n", - " end_scores = outputs.end_logits\n", - "\n", - " # ======== Reconstruct Answer ========\n", - " # Find the tokens with the highest `start` and `end` scores.\n", - " answer_start = torch.argmax(start_scores)\n", - " answer_end = torch.argmax(end_scores)\n", - "\n", - " # Get the string versions of the input tokens.\n", - " token_ids = input_ids[answer_start : answer_end + 1]\n", - " tokens = tokenizer.convert_ids_to_tokens(token_ids)\n", - " answer = tokenizer.convert_tokens_to_string(tokens)\n", - "\n", - " return answer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DVlKTK-njWrX" - }, - "source": [ - "As our reference text, we've taken the Abstract of the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "mqfBGgc8AKk2" - }, - "outputs": [], - "source": [ - "bert_abstract = (\n", - " 'We introduce a new language representation model called BERT, which stands for '\n", - " 'Bidirectional Encoder Representations from Transformers. Unlike recent language '\n", - " 'representation models (Peters et al., 2018a; Radford et al., 2018), BERT is '\n", - " 'designed to pretrain deep bidirectional representations from unlabeled text by '\n", - " 'jointly conditioning on both left and right context in all layers. As a result, '\n", - " 'the pre-trained BERT model can be finetuned with just one additional output '\n", - " 'layer to create state-of-the-art models for a wide range of tasks, such as '\n", - " 'question answering and language inference, without substantial taskspecific '\n", - " 'architecture modifications. BERT is conceptually simple and empirically '\n", - " 'powerful. It obtains new state-of-the-art results on eleven natural language '\n", - " 'processing tasks, including pushing the GLUE score to 80.5% (7.7% point absolute '\n", - " 'improvement), MultiNLI accuracy to 86.7% (4.6% absolute improvement), SQuAD v1.1 '\n", - " 'question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD '\n", - " 'v2.0 Test F1 to 83.1 (5.1 point absolute improvement).'\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ay_mwbBJAP87" - }, - "source": [ - "Let's ask BERT what its name stands for (the answer is in the first sentence of the abstract)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "y4VPq6FdjxyX", - "outputId": "feaefc87-1e67-4ce9-f24f-b5c732e0c989" - }, - "outputs": [], - "source": [ - "question = \"What does the 'B' in BERT stand for?\"\n", - "answer = answer_question(question, bert_abstract)\n", - "print(f'Answer: \"{answer}\"')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B6HcijzxkTO9" - }, - "source": [ - "Let's ask BERT about example applications of itself :)\n", - "\n", - "The answer to the question comes from this passage from the abstract: \n", - "\n", - "> \"...BERT model can be finetuned with just one additional output\n", - "layer to create state-of-the-art models for **a wide range of tasks, such as\n", - "question answering and language inference,** without substantial taskspecific\n", - "architecture modifications.\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "MVNVGN5-gI06", - "outputId": "8ab4d754-6d14-4515-8772-bf08553ea7c9" - }, - "outputs": [], - "source": [ - "question = \"What are some example applications of BERT?\"\n", - "answer = answer_question(question, bert_abstract)\n", - "print(f'Answer: \"{answer}\"')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WXAJ2wkV7rRl" - }, - "source": [ - "## [Optional] Part 3. RuBERT for question answering." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TcnPsGzEbGL6" - }, - "source": [ - "Here we will use the model pre-trained on the SberQuAD dataset from the [SDSJ-2017 challenge problem B](https://github.com/sberbank-ai/data-science-journey-2017/tree/master/problem_B)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "3JslS5CG7rRl", - "outputId": "e1cca72a-e1ce-423a-a2f9-ed51b7ce4340" - }, - "outputs": [], - "source": [ - "from deeppavlov import build_model, configs\n", - "\n", - "model_ru = build_model(configs.squad.squad_ru_rubert, download=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GHVayqJ37rRl" - }, - "source": [ - "The following text is copied from [habr post on Crew Dragon flight](https://habr.com/ru/news/t/504642/)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "l5pDyTRL7rRl" - }, - "outputs": [], - "source": [ - "context = (\n", - " 'Первая многоразовая ступень ракеты-носителя Falcon 9 успешно отделилась через две с половиной '\n", - " 'минуты после старта и автоматически приземлилась на плавучую платформу Of Course I Still '\n", - " 'Love You у берегов Флориды. Через 12 минут после запуска космический корабль Crew Dragon '\n", - " 'вышел на расчетную орбиту и отделился от второй ступени ракеты.'\n", - " '\\n\\n'\n", - " 'Сближение корабля Crew Dragon с Международной космической станцией запланировано на 31 мая. '\n", - " 'К стыковочному адаптеру на узловом модуле «Гармония» американского сегмента МКС Crew Dragon '\n", - " 'должен причалить в ручном или, при необходимости, в автоматическом режиме. Эта процедура '\n", - " 'запланирована на 10:29 по времени Восточного побережья США (17:29 по московскому времени).'\n", - " '\\n\\n'\n", - " 'В испытательном полете DM2 астронавт Херли является командиром космического корабля (spacecraft '\n", - " 'commander), а его напарник Бенкен — командир по операциям стыковки и расстыковки (joint '\n", - " 'operations commander). Фактически это означает, что именно Херли управляет Crew Dragon в '\n", - " 'полете к МКС, к которой они должны пристыковаться в течение суток после старта. Херли и Бенкен '\n", - " 'также будут выполнять необходимые для сертификации НАСА проверки систем корабля в полете.'\n", - " '\\n\\n'\n", - " 'Во время полета Херли и Бенкен провели небольшую экскурсию по Crew Dragon.'\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5tVX9PJ_GPE-" - }, - "source": [ - "And here is how to use deeppavlov's model:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "05BDo1IjGFPG", - "outputId": "fba7a5d0-d835-47b3-beb1-19f9cadcaf1e" - }, - "outputs": [], - "source": [ - "question = 'Когда отделилась первая ступень?'\n", - "model_ru([context], [question])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yyRAYAc_GxAL" - }, - "source": [ - "The model returns list with answer, answer starting position in context and the answer logit.\n", - "\n", - "This yields the following `answer_question` function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "UXi5hm_AEFB4" - }, - "outputs": [], - "source": [ - "def answer_question_ru(question, context):\n", - " output = model_ru([context], [question])\n", - " return output[0][0]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0wfCi3FvHBuL" - }, - "source": [ - "Let's ask a bunch of other questions to the model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "98PIvy4g7rRm", - "outputId": "6af3638d-bf18-4a48-9832-947b37cc336c" - }, - "outputs": [], - "source": [ - "question = 'На какую дату запланирована стыковка?'\n", - "answer = answer_question_ru(question, context)\n", - "print(f'Ответ: \"{answer}\"')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "K1BK3PAm7rRn", - "outputId": "90665a68-76d3-46e0-9622-34617b142b19" - }, - "outputs": [], - "source": [ - "question = 'Кто участвует в полете?'\n", - "answer = answer_question_ru(question, context)\n", - "print(f'Ответ: \"{answer}\"')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Ugo2Wyd57rRn", - "outputId": "da924f61-81fa-446c-f1aa-0972289703d2" - }, - "outputs": [], - "source": [ - "question = 'Кто участвует в полете кроме астронавта Херли?'\n", - "answer = answer_question_ru(question, context)\n", - "print(f'Ответ: \"{answer}\"')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5B4vytlCYTvs", - "outputId": "2fdf869c-dcdd-405c-9542-f3d259191ddc" - }, - "outputs": [], - "source": [ - "question = 'Какие астронавты участвовали в полете?'\n", - "answer = answer_question_ru(question, context)\n", - "\n", - "# Notice how model finds the appropriate answer dispite slightly different context.\n", - "print(f'Ответ: \"{answer}\"')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "I-AwQsIU7rRn", - "outputId": "d72a2aed-32ad-4b70-a146-ec8febc7f7d1" - }, - "outputs": [], - "source": [ - "question = 'Какая ступень приземлилась на плавучую платформу Of Course I Still Love You?'\n", - "answer = answer_question_ru(question, context)\n", - "print(f'Ответ: \"{answer}\"')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZU4ZxB6o7rRp" - }, - "source": [ - "## Part 4. Question answering with speech using Tacotron 2." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NVstcIVxadEr" - }, - "source": [ - "### Text to speech using Tacotron 2." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aoj8MMIJbTji" - }, - "source": [ - "Tacotron 2 is a network proposed in 2017 in [Natural TTS Synthesis By Conditioning\n", - "Wavenet On Mel Spectrogram Predictions](https://arxiv.org/pdf/1712.05884.pdf) paper. This network takes an input text and maps it into the mel-frequency spectrogram. This spectrogram is then passed through a modified WaveNet (generative model for audio, original paper can be found [here](https://arxiv.org/pdf/1609.03499.pdf)) to generate the actual speech." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "btil2I1zejk3" - }, - "source": [ - "Let's look more closely at a mel spectrogram (for more info on its nature please refer to the [Tacotron 2 paper](https://arxiv.org/pdf/1712.05884.pdf))." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 298 - }, - "id": "xDAPjR_Lx-uZ", - "outputId": "a12ee1da-0bbe-4914-e572-fb0c5986e0a7" - }, - "outputs": [], - "source": [ - "assert tacotron2 is not None and waveglow is not None, 'Tacotron2 by NVIDIA infers only on GPU, so the Part 4 will not work on CPU-only machine'\n", - "utils = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_tts_utils')\n", - "\n", - "text = 'Some test text.'\n", - "sequences, lengths = utils.prepare_input_sequence([text])\n", - "with torch.no_grad():\n", - " mel, _, _ = tacotron2.infer(sequences, lengths)\n", - "\n", - "sns.reset_orig()\n", - "plt.imshow(mel[0].cpu().numpy())\n", - "plt.title('mel-frequency spectrogram');" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9odRMQS3fISF" - }, - "source": [ - "After obtaining this spectrogram, we can generate the audio with `waveglow` model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 52 - }, - "id": "BdFfCjmsUUxQ", - "outputId": "8d94bfcd-4157-4416-e6cc-b93a40edaea0" - }, - "outputs": [], - "source": [ - "from IPython.display import Audio\n", - "\n", - "sampling_rate = 22050\n", - "\n", - "with torch.no_grad():\n", - " audio = waveglow.infer(mel)\n", - "\n", - "audio_numpy = audio[0].cpu().numpy()\n", - "Audio(audio_numpy, rate=sampling_rate)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "W0mnDWpdhpdi" - }, - "source": [ - "We've generated a `.wav` format audio. We can save it using the `scipy.io.wavfile.write`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rTMNu9Krh2cW" - }, - "outputs": [], - "source": [ - "from scipy.io.wavfile import write\n", - "\n", - "write('audio.wav', sampling_rate, audio_numpy)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0pjEZy4TfT3w" - }, - "source": [ - "This yields the following `text_to_speech` function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "YEvDynkPVscj" - }, - "outputs": [], - "source": [ - "def text_to_speech(text):\n", - " # preprocessing\n", - " sequences, lengths = utils.prepare_input_sequence([text])\n", - "\n", - " # run the models\n", - " with torch.no_grad():\n", - " mel, _, _ = tacotron2.infer(sequences, lengths)\n", - " audio = waveglow.infer(mel)\n", - "\n", - " audio_numpy = audio[0].cpu().numpy()\n", - " return audio_numpy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 52 - }, - "id": "RwqwZoxsWNmq", - "outputId": "69f79ad9-2300-4a8c-bf4c-a4b952128f3c" - }, - "outputs": [], - "source": [ - "text = 'Another test text.'\n", - "audio_numpy = text_to_speech(text)\n", - "Audio(audio_numpy, rate=sampling_rate)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gYittYwlfZfU" - }, - "source": [ - "### Tying text to speech with question answering." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "83WLbVnv7rRq" - }, - "source": [ - "Let's take a look at [Mail.ru group blog post on Computer Vision on habr.com](https://habr.com/ru/company/mailru/blog/467905/)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "m2eMzsMY7rRq" - }, - "outputs": [], - "source": [ - "context = (\n", - " 'One of Mail.ru Cloud’s objectives is to provide the handiest means for accessing '\n", - " 'and searching your own photo and video archives. For this purpose, we at Mail.ru '\n", - " 'Computer Vision Team have created and implemented systems for smart image '\n", - " 'processing: search by object, by scene, by face, etc. Another spectacular '\n", - " 'technology is landmark recognition. Today, I am going to tell you how we made '\n", - " 'this a reality using Deep Learning.'\n", - " '\\n\\n'\n", - " 'Imagine the situation: you return from your vacation with a load of photos. Talking '\n", - " 'to your friends, you are asked to show a picture of a place worth seeing, like '\n", - " 'palace, castle, pyramid, temple, lake, waterfall, mountain, and so on. You rush to '\n", - " 'scroll your gallery folder trying to find one that is really good. Most likely, it '\n", - " 'is lost amongst hundreds of images, and you say you will show it later.'\n", - " '\\n\\n'\n", - " 'We solve this problem by grouping user photos in albums. This will let you find '\n", - " 'pictures you need just in few clicks. Now we have albums compiled by face, by '\n", - " 'object and by scene, and also by landmark.'\n", - " '\\n\\n'\n", - " 'Photos with landmarks are essential because they often capture highlights of our '\n", - " 'lives (journeys, for example). These can be pictures with some architecture or '\n", - " 'wilderness in the background. This is why we seek to locate such images and make '\n", - " 'them readily available to users.'\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "4tkwZk-B7rRq", - "outputId": "31b6de57-91a3-4bed-8b64-00cfa9ad0e14" - }, - "outputs": [], - "source": [ - "question = 'Why photos with landmarks are essential?'\n", - "answer = answer_question(question, context)\n", - "print(f'Answer: \"{answer}\"')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "chBm8WdIh_Bc" - }, - "source": [ - "Let's cat question and answer into one phrase and convert it to audio!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 52 - }, - "id": "j9NhT0Np7rRr", - "outputId": "c09ed603-a4cc-423c-b8b9-f8f4def9b42f" - }, - "outputs": [], - "source": [ - "text = f'{question}\\n{answer}'\n", - "audio_numpy = text_to_speech(text)\n", - "Audio(audio_numpy, rate=sampling_rate)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cFDgT4OuijIp" - }, - "source": [ - "And another one." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 69 - }, - "id": "8iqcYLwgy1nT", - "outputId": "03d7ca7d-cf6d-4b5d-b88a-b810f874f117" - }, - "outputs": [], - "source": [ - "question = \"Which places except mountain are worth seeing?\"\n", - "answer = answer_question(question, context)\n", - "print(f'Answer: \"{answer}\"')\n", - "\n", - "text = f'{question}\\n{answer}'\n", - "audio_numpy = text_to_speech(text)\n", - "Audio(audio_numpy, rate=sampling_rate)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NIUIia_G7rRs" - }, - "outputs": [], - "source": [ - "# Take your time, experiment with questions and the generated audio" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vv5jEpkw7rRs" - }, - "source": [ - "## [Optional] 5. Russian langugage speech generation" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_0uhvcBhj2k6" - }, - "source": [ - "Of course, text to speech is not specific to english language. Here is how you can do it with russian." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TD79JX0g7rRs" - }, - "outputs": [], - "source": [ - "from omegaconf import OmegaConf\n", - "\n", - "torch.hub.download_url_to_file(\n", - " 'https://raw.githubusercontent.com/snakers4/silero-models/master/models.yml',\n", - " 'latest_silero_models.yml',\n", - " progress=False\n", - ")\n", - "models = OmegaConf.load('latest_silero_models.yml')\n", - "\n", - "# see latest avaiable models\n", - "available_languages = list(models['tts_models'].keys())\n", - "print(f'Available languages {available_languages}')\n", - "\n", - "for lang in available_languages:\n", - " speakers = list(models['tts_models'][lang].keys())\n", - " print(f'Available speakers for {lang}: {speakers}')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZVaR1xG8k94K" - }, - "source": [ - "Let's choose our language and speaker and try using them!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "zKCT72on7rRt" - }, - "outputs": [], - "source": [ - "language = 'ru'\n", - "speaker = 'kseniya_16khz'\n", - "device = torch.device('cpu')\n", - "model, symbols, sample_rate, example_text, apply_tts = torch.hub.load(\n", - " 'snakers4/silero-models', 'silero_tts',\n", - " language=language, speaker=speaker\n", - ")\n", - "model = model.to(device)\n", - "\n", - "\n", - "audio = apply_tts(\n", - " texts=[example_text],\n", - " model=model,\n", - " sample_rate=sample_rate,\n", - " symbols=symbols,\n", - " device=device\n", - ")\n", - "\n", - "print(example_text)\n", - "Audio(audio[0], rate=sample_rate)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "U0O3eCX87rRt" - }, - "outputs": [], - "source": [ - "audio = apply_tts(\n", - " texts=[\"Дерзайте знать! Спасибо за внимание!\"],\n", - " model=model,\n", - " sample_rate=sample_rate,\n", - " symbols=symbols,\n", - " device=device\n", - ")\n", - "Audio(audio[0], rate=sample_rate)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "N2_1sCYvALxG" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "practice_question_answering_and_tts.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "0695270fa12047a69a2dcef569155b25": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "07f8ddf6b1ae4143862d4e1888b253cb": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0ae8437438404098a5a6206b05ecf453": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_494fb6f653f845b3837db00241bf4798", - "placeholder": "​", - "style": "IPY_MODEL_2165f1cd0739407a92079f643ac74ae3", - "value": " 28.0/28.0 [00:00<00:00, 487B/s]" - } - }, - "0bae2a0bb31b46f39a29c450e8c983c9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1c750fddb4f14ac397026785a004921d", - "placeholder": "​", - "style": "IPY_MODEL_228cd2bb1c5a48088cd48fed5e8ee2e5", - "value": "Downloading: 100%" - } - }, - "0e8242194958409abe8c5830767ae7b8": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "17f2ae98a8d1440b8beebc2f8aaff051": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_0bae2a0bb31b46f39a29c450e8c983c9", - "IPY_MODEL_3d5da11d2072432face40e6f729f644b", - "IPY_MODEL_f8318de1c5c74051adcf59d5f6261903" - ], - "layout": "IPY_MODEL_7ce0c95f422148ec92feee15262845f7" - } - }, - "18a3cd219c8642d3b6001f4dbed8a1a9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c30803de30894426958ad02fbc9f0002", - "placeholder": "​", - "style": "IPY_MODEL_4f9a99283eec4c31b8ed463a0a5c4366", - "value": "Downloading: 100%" - } - }, - "18a7445e957446ffa372185756f6b64c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e36ef4cbcb4c466283f85f5d523d66cd", - "placeholder": "​", - "style": "IPY_MODEL_7669b34be42a4453b3e3ba0e60faee75", - "value": "Downloading: 100%" - } - }, - "1aaa4d2569574c89a646d202738a829e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1b981c0d4a0a4bdca32bb12960ce8a7a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "1c750fddb4f14ac397026785a004921d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1d6782097f9a45758fea597727c42da3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_18a7445e957446ffa372185756f6b64c", - "IPY_MODEL_c6e9545c4c03427591aac4f49c774ff7", - "IPY_MODEL_0ae8437438404098a5a6206b05ecf453" - ], - "layout": "IPY_MODEL_301ac3b9e5d04242b05d5943367bf048" - } - }, - "2061f0fcebbd4806a5b7308ea3079a7b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2165f1cd0739407a92079f643ac74ae3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "228cd2bb1c5a48088cd48fed5e8ee2e5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "26506f018a824fd2981562f17ada25e9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "28e9a4b59d8940228874918132a4379a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "301ac3b9e5d04242b05d5943367bf048": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3056ad61695a42ffadd7954a5a7503ed": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_18a3cd219c8642d3b6001f4dbed8a1a9", - "IPY_MODEL_e4099cf7a2c1426d844c1febc644001a", - "IPY_MODEL_fe8d0c53f5704e42b7c271fecaa48851" - ], - "layout": "IPY_MODEL_2061f0fcebbd4806a5b7308ea3079a7b" - } - }, - "32364789d1ff47fc8c4056a7b48a7634": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "3d5da11d2072432face40e6f729f644b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_f7d2834f66ac48719f1fd47dda33d7dc", - "max": 451, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e99dd095993f4053b5ba2a9c306582b9", - "value": 451 - } - }, - "3ec7e9e71b5e4feea47fe12b5f751d57": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3f4e292341444c128a6941020bf78ea7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "3ffe97d1a93d46c5ad72625841831419": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "49107e49101e469abe491d5fb4e4466a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "494fb6f653f845b3837db00241bf4798": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4f9a99283eec4c31b8ed463a0a5c4366": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "59df97cee8854be19b28d63f714816be": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "60050037056f4231ab5c6afa8ae9ec27": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "68976e9d2ac1420cb548307c96c5a8a6": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "71f49045ee564ca3b485270db1e69792": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_eef6db037885440f891824e5a8f5358f", - "placeholder": "​", - "style": "IPY_MODEL_a51916d0bd0a43fdba9e46f20a80d6a2", - "value": " 226k/226k [00:00<00:00, 477kB/s]" - } - }, - "7669b34be42a4453b3e3ba0e60faee75": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7ce0c95f422148ec92feee15262845f7": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7d1f7a87ed484be999ea793bf4c34dd3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "85e24abbc05d4937ac86793c9c36e1f5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_8ca5505516334e50b305f559633213fd", - "IPY_MODEL_96eb7cdf8e9b48bfb3a619c3a6b4e3e0", - "IPY_MODEL_c97f4996f8d14b9c94cfefaf1c7adfea" - ], - "layout": "IPY_MODEL_3ec7e9e71b5e4feea47fe12b5f751d57" - } - }, - "8ca5505516334e50b305f559633213fd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_28e9a4b59d8940228874918132a4379a", - "placeholder": "​", - "style": "IPY_MODEL_59df97cee8854be19b28d63f714816be", - "value": "Downloading: 100%" - } - }, - "8d9aded51b004aababcda8e76798a534": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_07f8ddf6b1ae4143862d4e1888b253cb", - "placeholder": "​", - "style": "IPY_MODEL_1b981c0d4a0a4bdca32bb12960ce8a7a", - "value": "Downloading: 100%" - } - }, - "96eb7cdf8e9b48bfb3a619c3a6b4e3e0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0e8242194958409abe8c5830767ae7b8", - "max": 265481570, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_26506f018a824fd2981562f17ada25e9", - "value": 265481570 - } - }, - "a51916d0bd0a43fdba9e46f20a80d6a2": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b12f548d82ca4586b61f701cb1abacb4": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c30803de30894426958ad02fbc9f0002": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c6e9545c4c03427591aac4f49c774ff7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d434a8d87ce24bf7865ba75298d95729", - "max": 28, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_49107e49101e469abe491d5fb4e4466a", - "value": 28 - } - }, - "c97f4996f8d14b9c94cfefaf1c7adfea": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1aaa4d2569574c89a646d202738a829e", - "placeholder": "​", - "style": "IPY_MODEL_68976e9d2ac1420cb548307c96c5a8a6", - "value": " 253M/253M [00:16<00:00, 9.50MB/s]" - } - }, - "d434a8d87ce24bf7865ba75298d95729": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e36ef4cbcb4c466283f85f5d523d66cd": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e4099cf7a2c1426d844c1febc644001a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b12f548d82ca4586b61f701cb1abacb4", - "max": 466062, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_3ffe97d1a93d46c5ad72625841831419", - "value": 466062 - } - }, - "e4c848c91a394012bbab804ff3e888a8": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7d1f7a87ed484be999ea793bf4c34dd3", - "max": 231508, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_32364789d1ff47fc8c4056a7b48a7634", - "value": 231508 - } - }, - "e99dd095993f4053b5ba2a9c306582b9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "ed11dd778e2f49af96e269c4c37c6417": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "eef6db037885440f891824e5a8f5358f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f10f304cd83a424bb1acce2a79cac5a5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_8d9aded51b004aababcda8e76798a534", - "IPY_MODEL_e4c848c91a394012bbab804ff3e888a8", - "IPY_MODEL_71f49045ee564ca3b485270db1e69792" - ], - "layout": "IPY_MODEL_60050037056f4231ab5c6afa8ae9ec27" - } - }, - "f7819d286ecc424e83c1d6e509ab1486": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f7d2834f66ac48719f1fd47dda33d7dc": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f8318de1c5c74051adcf59d5f6261903": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0695270fa12047a69a2dcef569155b25", - "placeholder": "​", - "style": "IPY_MODEL_3f4e292341444c128a6941020bf78ea7", - "value": " 451/451 [00:00<00:00, 8.05kB/s]" - } - }, - "fe8d0c53f5704e42b7c271fecaa48851": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ed11dd778e2f49af96e269c4c37c6417", - "placeholder": "​", - "style": "IPY_MODEL_f7819d286ecc424e83c1d6e509ab1486", - "value": " 455k/455k [00:00<00:00, 720kB/s]" - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/week09_pagerank/README.md b/week09_pagerank/README.md deleted file mode 100644 index 649531c..0000000 --- a/week09_pagerank/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Page Rank explanation: -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/natural-language-processing/blob/master/week09_pagerank/practice_pagerank.ipynb) - -Further readings: -* https://en.wikipedia.org/wiki/PageRank \ No newline at end of file diff --git a/week09_pagerank/practice_pagerank.ipynb b/week09_pagerank/practice_pagerank.ipynb deleted file mode 100644 index 145d4c4..0000000 --- a/week09_pagerank/practice_pagerank.ipynb +++ /dev/null @@ -1,271 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# PageRank" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This page demonstrates the use of a short Python implementation of the PageRank algorithm on the link structure contained in the graph on the [PageRank Wikipedia](http://en.wikipedia.org/wiki/PageRank) page:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from IPython.display import Image\n", - "Image(url='http://upload.wikimedia.org/wikipedia/commons/f/fb/PageRanks-Example.svg')" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we will encode the links present on this graph as a count matrix `M_counts`." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[ 1. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 1. 1. 1. 1. 1. 1. 1. 0. 0.]\n", - " [ 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 0. 0. 0. 1. 1. 1. 1. 1. 1.]\n", - " [ 1. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]\n" - ] - } - ], - "source": [ - "n_pages = 11 # numbering pages A through K as 0 to 10\n", - "M_counts = np.zeros((n_pages, n_pages)) # will hold the number of link counts (assumed 0 or 1)\n", - "# columns = starting page, row = destination page, ie M_ij = whether or not there is a link from j to i\n", - "\n", - "M_counts[:,0] = 1 # page 0 (A in the graphic) is a sink because it has no outgoing links at all; \n", - "# however, M cannot contain an all-zero column, so do as if A was linking to all other pages (ie put 1's everywhere)\n", - "M_counts[2,1] = 1 # B->C\n", - "M_counts[1,2] = 1 # C->B\n", - "M_counts[0,3] = 1 # D->A\n", - "M_counts[1,3] = 1 # D->B\n", - "M_counts[1,4] = 1 # E->B\n", - "M_counts[3,4] = 1 # E->D\n", - "M_counts[5,4] = 1 # E->F\n", - "M_counts[1,5] = 1 # F->B\n", - "M_counts[4,5] = 1 # F->E\n", - "M_counts[1,6] = 1 # G,H,I->B,E\n", - "M_counts[4,6] = 1\n", - "M_counts[1,7] = 1\n", - "M_counts[4,7] = 1\n", - "M_counts[1,8] = 1\n", - "M_counts[4,8] = 1\n", - "M_counts[4,9] = 1 # J,K->E\n", - "M_counts[4,10] = 1\n", - "print(M_counts)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can make an adjacency matrix `M` out of `M_counts`, by dividing each column by its sum, ie we are making sure columns sum to 1 :" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[ 0.091 0. 0. 0.5 0. 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 1. 0.5 0.333 0.5 0.5 0.5 0.5 0. 0. ]\n", - " [ 0.091 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 0. 0. 0.333 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 0. 0. 0. 0.5 0.5 0.5 0.5 1. 1. ]\n", - " [ 0.091 0. 0. 0. 0.333 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ]\n", - " [ 0.091 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. ]]\n" - ] - } - ], - "source": [ - "M = np.empty((n_pages, n_pages))\n", - "for j in range(n_pages):\n", - " M[:,j] = M_counts[:,j] / M_counts[:,j].sum()\n", - "np.set_printoptions(precision=3)\n", - "print(M)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let us check that all the conditions on M are fulfilled." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy\n", - "def check_M(M):\n", - " \"\"\"\n", - " check that M has the right format to be used by pagerank function\n", - " \"\"\"\n", - " n_pages = M.shape[0] # n_pages is the number of rows of M\n", - " np.testing.assert_equal(M.shape[0], M.shape[1], err_msg = 'M should be square')\n", - " np.testing.assert_array_almost_equal(M.sum(axis=0), np.ones((n_pages)), \n", - " err_msg = 'assert each column sums to one (M is assumed column-stochastic)')\n", - " for j in range(n_pages):\n", - " M_column = M[:,j]\n", - " n_nonzero = np.count_nonzero(M[:,j])\n", - " np.testing.assert_array_almost_equal(M_column[M_column.nonzero()], np.ones((n_nonzero)) / n_nonzero,\n", - " err_msg = 'in column %g, all non-zero entries should be equal (and equal to 1 divided by their number)' % j)\n", - "\n", - "check_M(M) # will produce error if M does not have the right format" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And we are now ready to apply the `pagerank` function, which will iteratively apply page transitions to an randomly initialized distribution over the pages, until convergence." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "def pagerank(M, d=0.85, square_error=1e-6):\n", - " \"\"\"\n", - " M : the adjacency matrix of the pages. It is assumed to be column-stochastic (ie column sum to 1); all links have equal weight.\n", - " A page with no outgoing links (sink) is represented as a page with outgoing links to each other page (ie restart page).\n", - " d: damping factor\n", - " square_error : the algorithm iterates until the difference between two successive PageRank vectors v is less than this (in squared norm)\n", - " returns the PageRanks of all pages\n", - " \"\"\"\n", - " n_pages = M.shape[0] # n_pages is the number of rows of M\n", - " v = np.random.rand(n_pages) # initialize to random vector\n", - " v = v / v.sum() # make v sum to 1\n", - " last_v = np.ones((n_pages)) # will contain the previous v\n", - " M_hat = d * M + (1-d)/n_pages * np.ones((n_pages, n_pages)) # equation (***) in Wikipedia page\n", - " while np.square(v - last_v).sum() > square_error:\n", - " last_v = v\n", - " v = M_hat.dot(v) # at each iteration, progress one timestep\n", - " return v\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0.033, 0.384, 0.343, 0.039, 0.081, 0.039, 0.016, 0.016,\n", - " 0.016, 0.016, 0.016])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pagerank(M)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "These are the numbers (within the allowed error) displayed on the graph (the numbers on the graph are rounded exact values)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.13" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -}