Sequence_Memory_Task.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Memorizing elements of a sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The goal of this task is to memorize some specific elements of a sequence a symbols with variable length. This toy example is useful to test the capacity of RNNs to keep representations in memory over long sequences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One-hot encoding for string of symbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %load binary_code.py\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "ALPHABET = np.asarray(list(\"abcd\"), dtype=object)\n",
    "\n",
    "\n",
    "def symbols_to_binarray(s, alphabet=ALPHABET, dtype=np.float32):\n",
    "    \"\"\"One-hot encode a sequence of symbols\n",
    "    \n",
    "    This numerical representation of a string of symbols is useful\n",
    "    to feed the data and expected labels to the input and output\n",
    "    layers of recurrent networks.\n",
    "    \"\"\"\n",
    "    alphabet = np.asarray(list(alphabet), dtype=object)\n",
    "    n_samples = len(s)\n",
    "    n_features = len(alphabet)\n",
    "\n",
    "    mapping = dict(zip(alphabet, range(n_features)))\n",
    "    \n",
    "    code = np.zeros((n_samples, n_features), dtype=dtype)\n",
    "    for i, e in enumerate(s):\n",
    "        code[i, mapping[e]] = 1.0\n",
    "    return code\n",
    "\n",
    "\n",
    "def binarray_to_symbols(code, alphabet=ALPHABET):\n",
    "    \"\"\"Convert encoded data by to a string of symbols\"\"\"\n",
    "    n_samples, n_features = code.shape\n",
    "    if n_features != len(alphabet):\n",
    "        raise ValueError(\n",
    "            \"code should have %d columns (instead of %d).\"\n",
    "            % (len(alphabet), n_features)\n",
    "        )\n",
    "\n",
    "    # Make sure that the alphabet is a numpy array of symbols\n",
    "    # to make it possible to leverage numpy fancy indexing\n",
    "    if not isinstance(alphabet, np.ndarray):\n",
    "        alphabet = np.asarray(list(alphabet), dtype='object')\n",
    "\n",
    "    return \"\".join(alphabet[code.argmax(axis=1)])\n",
    "\n",
    "\n",
    "def plot_binary_tape(encoded_sequence, alphabet=ALPHABET):\n",
    "    plt.matshow(encoded_sequence, cmap=plt.cm.gray)\n",
    "    plt.xticks(np.arange(len(alphabet)), alphabet)\n",
    "    \n",
    "\n",
    "def plot_parallel_tapes(input_data, output_data,\n",
    "                        input_symbols, output_symbols):\n",
    "    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(5, 10))\n",
    "    for ax, data in zip(axes, [input_data, output_data]):\n",
    "        ax.matshow(data, cmap=plt.cm.gray)\n",
    "        ax.set_xticks(np.arange(len(ALPHABET)))\n",
    "        ax.set_xticklabels(ALPHABET, fontsize=18)\n",
    "    fig.tight_layout()\n",
    "    plt.title(\"input: %r, output: %r\" % (input_symbols, output_symbols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.,  0.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.,  0.],\n",
       "       [ 0.,  0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.,  0.],\n",
       "       [ 0.,  1.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  1.]], dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "symbol_sequence = \"abcdacbd\"\n",
    "encoded_sequence = symbols_to_binarray(symbol_sequence)\n",
    "encoded_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPYAAAHaCAYAAAAzC7QxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADNZJREFUeJzt3V2o7Xldx/HPd+Y0+ZApUWSloYEadaFZiZDRtixEFL0o\nzCDBwIsecIgKCwnPTXkjRBfVRZZYlj1ohoFoT54cSUxrxtQZpQRjFDVJUcsEzV8XZ48c58yctc7Z\na5219+e8XnCYtfde7PNds/d7//7rv/b5/2atFaDLTYceANg9YUMhYUMhYUMhYUMhYUMhYZ8SM/Oo\nmXnPoefg5Gbm/Mz8wiFnEDbs3sF/OeRMhT0zr5+Zd83Me2fmhYeeZw/OzcyrZ+bOmfnzmXngoQfa\npZl5/sy8e2bumJk/OPQ8uzQzL5mZD8zMbUked+h5zlTYSX5qrfU9Sb43yYtm5usOPdCOPS7Jb621\nviPJZ5L8zIHn2ZmZ+c4kL0ny1LXWE5LceuCRdmZmvjvJc5M8PskzcvH786Cr9lkL+9aZuSPJ25M8\nIsljDjzPrt291nr78e1XJ3nKIYfZsR9M8mdrrU8myVrrUweeZ5e+P8lfrLU+v9b6bJI3JJlDDnTu\nkH/51ZiZoyQ/lOTJa63Pz8xbknz1YafauUt/yk9OwXO1HVo58Df7Ht37sR38cZ6lFftrk3zqOOpv\nT/LkQw+0B986M/c8rp9Ictshh9mxv0/yY/c8fSp7GvXWJM+ZmQfMzEOSPDMOxbf2plw8uXRnkpfl\n4uF4k5XkA0l+9vgxPjTJ7xx2pN1Za92Z5NeS/MPx06mXH3iknVlr3Z7kT5O8O8kbk/zTYSdKxj/b\nhD5nacUGtiRsKCRsKCRsKHTi17Fnxtk3OKC11mWvm+9kxV5rXbc/L33pS6/r3wdnkUNxKCRsKHTm\nwj46Ojr0CHDqnfg3z2ZmNT8XnTn47/PDFe3t5BlwuggbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkb\nCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCm0Me2aePjPvn5l/m5kXX4+hgJO54jXPZubmXNza9WlJ\nPpLknUmet9a665L7uOYZHNC1XPPsSUn+fa31obXWF5L8SZJn72M4YHc2hf0tSe6+5O0PH78POMU2\n7d211TH2+fPnv3z76OjItb/hwDaF/ZEkj7zk7Ufm4qr9FS4NGzi8TYfi70rymJl51MzckuS5Sd6w\n/7GAk7jiir3W+uLM/FySNye5OcnvXXpGHDidbPGzgZe7OO1s8QM3CGFDIWFDIWFDIWFDIWFDIWFD\nIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDoU07gWyl+RK9zZdW\nTrq/djcyKzYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYU\nEjYUEjYUEjYUEjYUEjYUEjYUEjYU2hj2zPz+zHx8Zt5zPQYCTm6bFfuVSZ6+70GA3dkY9lrrtiSf\nug6zADviOTYUEjYUEjYUEjYU2ublrtck+cckj52Zu2fmBfsfCziJOenG7jNTvTO8je857dZal30R\nHYpDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFD\nIWFDIWFDIWFDoXOHHuC0a788b/vllZP+r+F9sWJDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFD\nIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDIWFDoY1hz8wjZ+YtM/O+mXnv\nzLzoegwGXLvZtBPEzDw8ycPXWnfMzNck+eckz1lr3XX88f6tJIrZCeTsW2td9gA3rthrrY+tte44\nvv3fSe5K8s27Hw/Ylat6jj0zj0ryXUnesY9hgN3YOuzjw/DXJrn1eOUGTqmtwp6Zr0ryuiSvXmv9\n5X5HAk5qm5Nnk+RVSf5rrfXz9/Hx/rMvxZw8O/vu6+TZNmE/Jclbk/xrknvu/CtrrTcdf7z/O6OY\nsM++awp7E2GfbcI++67p5S7g7BE2FBI2FBI2FBI2FBI2FBI2FBI2FBI2FBI2FBI2FBI2FBI2FBI2\nFBI2FBI2FBI2FBI2FBI2FBI2FBI2FBI2FDp36AE4rPZL8ybdl1i+v6+fFRsKCRsKCRsKCRsKCRsK\nCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsK\nbQx7Zh4wM++YmTtm5s6Zedn1GAy4dht3AllrfX5mnrrW+tzMnEvytpl5ylrrbddhPuAabHUovtb6\n3PHNW5LcnOSTe5sIOLGtwp6Zm2bmjiQfT/KWtdad+x0LOImtNuVba30pyRNm5qFJ3jwzR2utC3ud\nDLjMhQsXcuHChY33m6vdiXBmfjXJ/661Xn78du9WhlRo321zrXXZlpvbnBX/+pl52PHtByb54SS3\n735EYFe2ORT/piSvmpmbcvEHwR+utf5uv2MBJ3HVh+KXfQKH4pxyDsWBCsKGQsKGQsKGQsKGQsKG\nQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQsKGQltt8bNJ++Vd\nOdtuxK+hFRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsK\nCRsKCRsKCRsKCRsKCRsKCRsKbRX2zNw8M7fPzF/teyDg5LZdsW9NcmeS3p0BoMjGsGfmEUmekeQV\nSW68LRXgDNpmxf6NJL+U5Et7ngXYkSvu3TUzz0zyn2ut22fm6P7ud/78+S/fPjo6ytHR/d4VuA7m\nShvqzcyvJ/nJJF9M8oAkX5vkdWut519yn2VTPjictdZl36RXDPsr7jjzA0l+ca31rHu9X9hwQPcV\n9tW+jt1bMBTZesW+309gxYaD2sWKDZwBwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZC\nwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCV9y7a1uuvX12NV8T/h434venFRsKCRsK\nCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsKCRsK\nCRsKCRsKbXVd8Zn5UJLPJPm/JF9Yaz1pn0MBJ7PthgErydFa65P7HAbYjas5FL/xtlOAM2rbsFeS\nv52Zd83MC/c5EHBy2x6Kf99a66Mz8w1J/mZm3r/Wum2fgwHXbqsVe6310eP/fiLJ65M4eQan2Maw\nZ+ZBM/OQ49sPTvIjSd6z78GAa7fNofg3Jnn98Vak55L80Vrrr/c6FXAic9L9kWemf4PlYvbHPvvW\nWpc9QL95BoWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWE\nDYWEDYWEDYWEDYWEDYWEDYW23bvrhtV+3e32a27fqKzYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjY\nUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUGhj2DPzsJl57czcNTN3\nzsyTr8dgwLXbZsOA30zyxrXWj87MuSQP3vNMwAnNlXa6mJmHJrl9rfVtV7hP9VYZdgLhtFtrXfZF\n3HQo/ugkn5iZV87Mv8zM787Mg/YzHrArm8I+l+SJSX57rfXEJP+T5Jf3PhVwIpvC/nCSD6+13nn8\n9mtzMXTgFLti2GutjyW5e2Yee/yupyV5396nAk7kiifPkmRmHp/kFUluSfLBJC9Ya336ko9Xn11y\n8ozT7r5Onm0MexNhn23CPvuu5aw4cAYJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJ\nGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwpts43uDc11t8++5mvD39/3pxUbCgkbCgkb\nCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkb\nCgkbCm0Me2YeNzO3X/Ln0zPzousxHHBt5moupj4zNyX5SJInrbXuPn5f79XYqdC+YcBa67JdA672\nUPxpST54T9TA6XS1Yf94kj/exyDA7my9d9fM3JLkWUlevL9xgCu5cOFCLly4sPF+Wz/HnplnJ/np\ntdbT7/X+3icwVPAc+8qel+Q1uxsJ2JetVuyZeXCS/0jy6LXWZ+/1sd4fh1S4EVfsq3q5634+ce//\nNSrciGH7zTMoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwoJGwo\nJGwoJGwoJGzYoW2u+X09CBt2SNjA3ggbCrmuOJxxe9kwADh9HIpDIWFDIWFDIWFDIWFDof8HBNcj\n+oIPtZ4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8841fc410>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_binary_tape(encoded_sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPYAAAEzCAYAAAACQP57AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACLpJREFUeJzt3U2I7fddx/HPN7lqClpFXFRNpBut6KIVWykl6tQqlFC1\nGxFdFBS6UWkQW4p0kWykm4JuxI0PtBZ8RMHHutCMTbG0Krm19caiCyFWrGCDD5SA0p+LOymX3CTn\n5M4585/53NcLLpyZM8x8z73znt/5/+fc/2/WWgG63LP1AMDhCRsKCRsKCRsKCRsKCRsKCfuSmJlX\nzswnt56D85uZR2fmZ7acQdhweJu/OORKhT0zvz8zfzMzn5qZt289zxFcm5kPzsyNmfmdmXnZ1gMd\n0sy8bWY+MTPXZ+YDW89zSDPznpn59Mw8nuRVW89zpcJO8uNrrdcmeV2Sd8zMV2890IG9KskvrrW+\nJcl/JfmJjec5mJn51iTvSfLGtdZrkjy88UgHMzPfnuSHk7w6yUO5+f256ap91cJ+eGauJ/lokvuT\nfOPG8xzaU2utj57d/mCSB7cc5sC+J8lvr7U+lyRrrac3nueQvjPJ7621nllr/XeSP0gyWw50bcsv\n/lLMzEmSNyV5/VrrmZl5LMmXbTvVwd36U35yCY7VDmhl42/2I3ruY9v8cV6lFfvlSZ4+i/qbk7x+\n64GO4Btm5tnH9aNJHt9ymAP7iyQ/9OzhU9lh1IeTvHVm7puZr0jylngqvrcP5ebJpRtJ3pubT8eb\nrCSfTvKTZ4/xK5P80rYjHc5a60aSn0vyl2eHU+/beKSDWWs9keS3knwiyZ8k+fi2EyXjv21Cn6u0\nYgN7EjYUEjYUEjYUEjYUOvcLVGbGaXXY0FrrthfEWLGpt9a6sD+PPPLIhX69FyJsKCRsKCRsOKCT\nk5OtR0hygJeUOnnGZdf8sumZcfIM7hbChkLChkLChkLChkLChkLChkLChkLChkLChkLChkLChkLC\nhkLChkLChkLChkLChkI7w56ZN8/MP8zMP87Muy9iKOB8XvTSSDNzb25u7fq9ST6T5K+T/Mha68lb\nPqb3ujNUcGmk231Hkn9aa/3zWut/k/xmkh88xoDA4ewK++uTPHXL2/9y9j7gEtsVdu9zGCi2a++u\nzyR54Ja3H8jNVRvYwOnpaU5PT3d+3K6TZ9dy8+TZm5L8a5KPx8kzrpi78eTZi67Ya63/m5mfSvJn\nSe5N8iu3Rg1cTnYCod7duGJ75RkUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYU\nEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYU2rXFD+War7n9rJnbLrtdz4oNhYQNhYQNhYQN\nhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQNhYQN\nhXaGPTO/OjOfnZlPXsRAwPnts2L/WpI3H3sQ4HB2hr3WejzJ0xcwC3AgjrGhkLChkLChkLCh0D6/\n7vqNJH+V5Jtm5qmZ+bHjjwWcx5x3f+SZ6d9guZj9sa++tdZtD9BTcSgkbCgkbCgkbCgkbCgkbCgk\nbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCh07RCfpPna\n1O3XpG5/fHcrKzYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYUEjYU\nEjYUEjYUEjYUEjYUEjYUEjYUEjYU2hn2zDwwM4/NzN/PzKdm5h0XMRhw52bXxf5n5hVJXrHWuj4z\nX57kb5O8da315Nn9y4YBsJ211m3fpDtX7LXWv621rp/d/p8kTyb5usOPBxzKSzrGnplXJvm2JB87\nxjDAYey9d9fZ0/DfTfLw2cr9RY8++ugXb5+cnOTk5ORA4wF3YucxdpLMzJck+aMkf7rW+oXn3OcY\nGzb0fMfY+5w8myTvT/Ifa62ffp77hQ0butOwH0zy4SR/l+TZD/7ZtdaHzu4XNmzojsLeRdiwrTv6\ndRdw9QgbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkb\nCgkbCgkbCgkbCgkbCu29d9eLab72dvM105Puf7u7mRUbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkb\nCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCgkbCu0Me2bum5mPzcz1mbkxM++9\niMGAO7fzuuJrrWdm5o1rrc/PzLUkH5mZB9daH7mA+YA7sNdT8bXW589ufmmSe5N87mgTAee2V9gz\nc8/MXE/y2SSPrbVuHHcs4Dz2XbG/sNZ6TZL7k3zXzJwcdSrgXF7SWfG11n8m+eMkrz3OOMAh7HNW\n/Gtm5qvObr8syfcleeLYgwF3bp/dNr82yftn5p7c/EHw62utPz/uWMB5zHm3iZ2Z6n1mbaPLZbfW\nuu0f0SvPoJCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCwoZCw\noZCwoZCwoZCwoZCwoZCwoZCwodA+O4Hc1VxQ/+pr3vThhb4/rdhQSNhQSNhQSNhQSNhQSNhQSNhQ\nSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQSNhQaK+wZ+bemXliZv7w\n2AMB57fviv1wkhtJei/QDEV2hj0z9yd5KMkvJ3H1fLgC9lmxfz7Ju5J84cizAAfyomHPzFuS/Pta\n64lYreHK2LV31xuS/MDMPJTkviQvn5kPrLXedvzRgOc6PT3N6enpzo+bfTcsm5nvTvLOtdb3P+f9\nTqhxqbVvyrfWuu3Z9Ev9PXbv3xAU2XvFfsFPYMXmkrNiAxWEDYWEDYWEDYWEDYWEDYWEDYWEDYWE\nDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDYWEDQe0zzW/L4Kw4YCEDRyNsKGQDQPg\ninu+DQPOHTZw+XgqDoWEDYWEDYWEDYWEDYX+H2MgX0Yy/4upAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe856a26950>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_binary_tape(symbols_to_binarray('dcabd'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoding back to a sequence of symbols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "The `binary_to_symbols` function computes the inverse operation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abcdacbd'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarray_to_symbols(encoded_sequence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running count of symbols in a sequence\n",
    "\n",
    "Let's generate a dataset where the output tape should display the symbol that was the most frequent in the recent history of the input tape (over a fixed window size):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def running_max(encoded_input, window_size=3):\n",
    "    encoded_output = np.zeros_like(encoded_input)\n",
    "    for t in range(encoded_input.shape[0]):\n",
    "        window = encoded_input[t - window_size + 1 :t + 1]\n",
    "        frequencies = window.sum(axis=0)\n",
    "        winner = frequencies.argmax()\n",
    "        encoded_output[t, winner] = 1.\n",
    "    return encoded_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run this function on the binary represenation of an example string of symbols:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_sequence = 'abbaabcddccab'\n",
    "input_data = symbols_to_binarray(input_sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "abbaabcddccab\n",
      "aabbaaabddcca\n"
     ]
    }
   ],
   "source": [
    "output_data = running_max(input_data)\n",
    "output_sequence = binarray_to_symbols(output_data)\n",
    "print(input_sequence)\n",
    "print(output_sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaoAAAIZCAYAAAD3OXuwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHfJJREFUeJzt3Xu4bGddH/DvjxxiuGgS5Cm3BMLFqFxKgceUtlxOBS1g\nRVpvLTwIiGIFBS9cC485guAVQStaFQmgXEQUlBYLKERoRaoFQe6gQAiXQEhIEETAvP1jrU0mO/uc\nM2fv2Xt++5zP53n288zMmlnzzlq/Nd9Z71p7vTXGCAB0dY11NwAAjkRQAdCaoAKgNUEFQGuCCoDW\nBBUArQmqE0hVvb2q7rbuduxEVR2sqg8fYfpzq+opu/TeV1TVLVYwnyO2cVXvc5h5nzXP37a/IlX1\nwaq6x2GmHbFed7FNx7Se17ldLUOxnkDGGLcdY7x+t99nO0VdVVes6O3H/NdZ2zZW1aGqOncP3uf8\nqnrouuY3f5F/YEVv33Z9rtCufsZ5/d39cNMFFcejWncDltC1jXv1hbvq9zneg6KD3azZIwahoDqB\nzF0U3zjfPlRVL6mq51XV5XO34J02PffxVfWOqrqkqp5TVV8xT3twVb1h07yvqKpbVtXDktw/yWOr\n6jNV9YdLNu/LRVpVD6mqd87t+tt5nps/yxOq6pNV9YGquv+mydevqlfPrz+/qm668LpfqqoLquqy\nqvqrqrrLwrRzquqNVXVpVX20qv5bVV1z07y/ZW7TJ6vq56qqFl7//QvtfkdV3WF+/A5V9eb58Rcn\nOWXTZ3nM/H4XVtX3bpp2rap6+rw+Pl1Vb6iqU+Zpd6mqP5/be0FVPWh+/Fuq6i3zZ7zgMHtID62q\nj8zv++OHWxdHUlWnVtXzq+oTc/ueuLE85vr67YXnbnRFnVRVT01y1yS/MtfIL8/PuaKqfnir5bud\n+S1hseYeX1XvX1h391uYdsuqem1VXTy363eq6tRN8zpnq21lYR5b1uvR1lVV/V5VfWxe939WVbde\n9rWzLdfzXFfPndv7jiTfsOl9j1az31ZVfz2/9/ur6t/Nj1+vqs6b3/OSqnrZ/PjpVfU/5lq5pKpe\nUVU3OdyKuZoxhr8T5C/JB5J843z7UJJ/SHKvTL+UnpbkjQvP/WCStyW5SZLTk/zvJE+Zpz04yRs2\nzfuKJLeYb5+X5Mmbpj8rybOWbOd9ktx8vn23JJ9Ncof5/sEkX0zyC0muOU//+yRnz9Ofm+TyJHdJ\ncnKSZy62NckD5s9zjSQ/luRjSU6ep90xyTnztJsleWeSR236jH+a5LQkZyZ5T5KHztO+M8mFSe40\n379lkpvObfhQkkclOSnJtyf5wsbymZf/x5PcOsm1k7xw07J8VpLXJrnR3K47z/O82fw5v3ue7/WS\n3H5+zd2T3Ga+fbt5/t823z9rnv8LklwryW2TfCLJPbZRT89P8rIk15nb854k3ztPOzfJby88d+N9\nrzHff93Gc5dcvtuZ3yuSPHbJz/IdSW443/6uuaZusLAu75Gp3q6f5M+SPGPJbeVgjlyvh11XC9va\ndebXPiPJWxambXs9J/mZ+XOcluSMJG9PcsE87Wg1e06STy/M68ZJvna+/T+TvCjJqUkOJLnr/Pj1\nkvyHTIF33SQvSfKypWttHV+Y/tbzl6sH1asXpt06yec2PfdhC/fvneT98+0H5+hB9ZQVtvtlSR45\n397Y8K+1MP13kzxpvv3cJC9cmHadJF9KcpPDzPuSJLc7zLQfSfIHmz7jNy/c/8EkfzLfflWSH95i\nHndL8pFNj/2fhY3+OUmetjDtazaWZaZg+txW7UvyhCS/v+Tye2aSX5xvb3yBnb0w/WeTPPsY18lJ\nSf4xydctPPawJK9bqK+jBctDt6ihwy3fY57fDmvuLUnue5hp90vy5iW3lSPW65HW1RbTTps/81fu\ndD0n+dtNy/r7k3x4yZr99SRP3+L9b5Tkn5KcusTy/RdJLll2fej6O7FdtHD7c0lOqaueJbR4FtAF\nmX457bqqundV/UVVfaqqLs20h/XVC0+5dIzxDwv3P5RpI0mm7pwLNyaMMT6bKYxuPM/70TV1z316\nnvepmX4lp6rOnrsnPlZVlyV56qb3TQ6/TM7ItPFvduMkH9n02IcWbt9oi3luuH6mX6BbzfeMJH+3\nxeOpqn9ZVa+bu1k+neQHjuFzLOv6mX7lL36WCzLtVSxrqy7GnbRrqS7LrVTV98zdaJfOdXHbzMus\nqm5QVS+uqWv2siS/nWNbnlvV60Y9HnZdzd2aPzN3rV2WKRBHrqzX7aznje3kxltMy8K0I9Xs4Wr9\nzEzhc9nmCVV17ar69bmL+LJMe3OnbnTtHo2g4khuuun2R+fbn83UTZUkqaobbnrdTr4wviLJ7yf5\nuST/bIxxepJX5qoHck+vqmsv3L/ZQtsq0wazMb/rZup2+GhV3TXJY5J85xjjtHnely3M+9cydffd\naoxxapIn5urbyOZlsrFBfzjJrbb4SB/L1b+8b7Zp+uZ5brg4yecPM98PZ+qS2soLk7w8yRljjNOS\n/Pcs/zmWdXGmPYWzNs1n40fCVWokybI1crh2bXd+R1VVN0vyG0kekeR6c128PVfWxdMy7Sncdq6L\nB+boy/OjC/e3qteNz7XVutp43/snuW+mLrZTk9x8nlZHeO2y7TpS3R2tZg9X6x9Ocr0tjt8lyY8n\nOTvJOfNnufumz3JEgorDqSQPr6qbVNX1Mn1pv3ie9tYkt6mq29d0YP/QptdelKnrajtOnv8uTnJF\nVd07yTdv8byfrKprzuHzLUl+b2Hafarq31TVyUmekunY20eSfGWmbsCLq+rkqvqJJF+18LrrJvlM\nks9V1ddl6nra7NFVdVpVnZnkkZm6cZLk2fO0O9bkVjWdxPHnSb5UVY+c2/sfc9UD1y9J8uCq+vr5\ny+zcjQljjCsydQ3+YlXdaP6F/a/mz/WCJPesqu+sqgNV9dVVdfuFz3HpGOMLVXVOpi+8zV/kT5oP\nqN8mU1fu72YLNZ2wcLX/vRtj/NPc9qdW1XXnL/sfTfI781PekuRuVXXm/MX1hE2zuChbB+3hlu92\n57eM62RaPhcnuUZVPSTTHtWG62YKysvnEwAes+n1leQRh9lWNhyuXrdaV4vv+49JLqmq62QKzGya\nvt31/JIkT5iX9RlJfnjhNW/MkWv2t5I8pKq+saquMX/urx1jfCzJHyf51Xm+G593o63/kOSyeRmd\nm2Oxqj5df/3/ctVjVOcmef7CtLMy/Wq8xsJzH5fkHUkuzXTc6ZSF5//XJJ/M1CXwgPm1G8eobpXp\ni+XSzMd4Mv3a+7Ul2/nwTAeGL810wP6FubJ//GCmboqN9/9gkgcsvPa8JL+a5NWZQuf8JDebp11j\n3sguy/TL8jGZus82lsldk7xrft3rk/xkktcvzPuKJD+Uqdvj4iQ/v7G85uk/kOTd8+vflitPbrhT\nkjdnOvnhxZkONj954XWPy/Qr9sIkD9m0LE/JdBD9wkwHsM/fWA+ZThj5i/nzXJDkgfPj3z4vl8sz\nnVDwyxvremE9f1+mX/UfS/Low6yHM+d5n36Y6adl6gb7xPz+T0pSC9N/ZV6H753fb7G+7pzpZIlL\nkjxzyeV7rPN7ZZLHL1lzP5XkU5lq6ulZODkj0/Hbv5rX65sznYRzwabtasttJdOew5Hq9Ujr6jqZ\n9pgun9/jgZtqY9vrOdMJFs+b2/v2JI/e9JmOVrP3y/SD9fIk70vyTfPjp2c6TvzxeV28dH78RvMy\n/UymbeRhi+vvaH81zwSuoqZ/hnzoGOO1624L61FVD0hy6zHGE/fo/a7I1O265bE3TlwH1t0AoKcx\nxgvW3QZIHKMC+tC9w5Z0/QHQmj0qAFoTVAC0JqgAaE1QLaGmq4Vv+Y+PLG++fMrr1t2OE4na3Tl1\nu36Cir004swu9h91u2aCir3UdbBAOBJ1u2aCCoDW2gXVfIHLn6qqN9U0Iubnq+p9VfXTVXWtNTfv\nmjWNNPqhuV1vrarvXmeD5ourPram0TY/W9PwFX9ZVY9YY5vOrGn04Mvmvz+qqu1eMHTfULvHplvt\nnqh1ux90vITSGUkemuSlma7E/KVMFyJ9bJI7ZBoRdV1+NtNQA7+SqTvgIUleVFWnjDGet9eNma+i\n/apMF758VaYLuH4+yT/PNJrms9bQptMyXdD1jFw5bMbBTKPUrvvLerep3SV1q90TvG77W/cVvbe4\nivE1k5y0xeNPznR15W9YQ5sePL/3B7IwumamISI+mOmqy6esoV2Pndv1U1tMq71uz/y+T5vb9KBN\njz9jfvy162jXHn12tbt8u1rV7olct/vhr13X3xjji2Ma6ybzODunV9X1k/zp/JRz1te6/NoY4zMb\nd8YYl2cavuL0TL++9toDMl1K/8mbJ4x5K1uD+2W6xP/zNz3+s2toy55Su8ekW+2esHW7H7QLqiSp\nqodX1dsydQV8KtN4Nxv/x3D62ho2jVV0uMduvpcNmX1NknePMb6whvc+nFsked/mL5sxxsczjW10\nXFO7S+tWuyd03XbX7hhVVf1Ykl/I1G/9zEwD3H0hU9/xc9M0XEHtwu5oF1SZRrH8wBjj3osPVtU6\nD0RvuHWmkTQ3P5ZMI8Xutfck+fqqOrnRL9O/S3J2VV1jTEOpJ0mq6kZJTl1fs/aE2l1et9o9keu2\nvY6/8L6UJFX15bZV1YEkj19bi670g1X1VRt3qurUJP8l03DOf7aG9rwgU3fSkzZPqKp1/ZPiy5Pc\nIMn3bHr8cWtoy15Tu8vrVrsnct2213GP6qVJfjrJH1fVyzKdnXT/TF0o6/bJJG+qqvNy5Sm+ZyT5\nvjHG59fQnl9K8q1JnlRV35DkNZmOjdwmydlJvmkNbfq5TOvrN6vqTrnyNN87J7k4x/d/+avd5XWr\n3RO5btvrGFQ/n6koHpqpn/9jSX43Ux//O9fXrIxMv67uluQRmX59vSfJA8YYL15Lg8b4YlV9c5If\nz7SRPTXTxv7eJOetqU2frqq7JvnFXPnr9Pwk/zbT2W/H8zXT1O6yDWpWuyd43bZnhF8AWut4jAoA\nvkxQAdCaoAKgNUEFQGuCCoDWBBUArQkqAFrbtX/4rSr/oMXKjTF2/QoBapdV24u6PZ7t6h7VqgbN\nOvfcc1cyH1hWp7pVu5zodP0B0JqgAqC1fRFUBw8eXHcT4JipW1iNXbsobVVtHtV57dY3RBOrslcn\nU6hdVsnJFDuzL/aoADhxCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNa2HVRVda+qendVva+q\nHrfKRsFuUruwv2zryhRVdVKS9yS5Z5KPJPnLJP95jPGuhef4735Wbqf/4a92WQdXptiZ7e5RnZPk\n/WOMD44xvpjkxUm+bXXNgl2jdmGf2W5Q3STJhxfuXzg/Bt2pXdhnthtUvfpFYHlqF/aZ7Q5F/5Ek\nZy7cPzPTL9OrOHTo0JdvHzx40LAHdKB2YZ/Z7skUBzIdkL5Hko8m+b9xQJo9sIKTKdQue87JFDuz\nrT2qMcaXquqHkrwqyUlJfmtxQ4eu1C7sPwZOZF8xcCL7kT2qnXFlCgBaE1QAtCaoAGhNUAHQmqAC\noDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoLXtDpy4lG5XfO52RewN3ZYTQCf2qABo\nTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoA\nWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IK\ngNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYOrLsB0FFVrbsJVzHGWHcTrqbbMuL4\nZY8KgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABobVtB\nVVVnVtXrquodVfX2qnrkqhsGu0Htwv5T2xk+oKpumOSGY4y/rqrrJvl/Se43xnjXwnPajUvQcaiE\nxHAJx2KMsaOFpXZXR90ub6d1e6Lb1h7VGOPjY4y/nm//fZJ3JbnxKhsGu0Htwv6z42NUVXVWkjsk\nedNO5wV7Se3C/rCjoJq7Tl6a5FHzr1PYF9Qu7B/bDqqqumaS30/yO2OMl6+uSbC71C7sL9s9maKS\nPC/Jp8YYP3qY57Q7+tvxgHTioPSxWMHJFGp3RdTt8pxMsTPbDaq7JHl9krcl2ZjBE8YY/2vhOe22\nrI4be2KDPxYrCCq1uyLqdnmCame2FVRLzdjGvjQb/PL2YoNXu8tRt8sTVDvjyhQAtCaoAGhNUAHQ\nmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1g7s5sy7DU1gWAKA\n/cceFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0Jqg\nAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQm\nqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABo7cC6GwAdjTHW3YSrqKp1NwHWxh4V\nAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoLUd\nBVVVnVRVb6mqV6yqQbDb1C3sLzvdo3pUkncm6TV4DxyZuoV9ZNtBVVVnJLlPkmcnMaob+4K6hf1n\nJ3tUz0jymCRXrKgtsBfULewz2wqqqvr3ST4xxnhL/Cpln1C3sD8d2Obr/nWS+1bVfZKckuSrqur5\nY4zvWXzSoUOHvnz74MGDOXjw4DbfDlZiqbpN1C50UmPs7HhyVd09yaPHGN+66fGx03mvWpUf0fvd\nGGMlK/FwdTtPU7us1Krq9kS1qv+j6rVVw3LULewDO96jOuyM/SplF+zFL1O1y6rZo9oZV6YAoDVB\nBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtLbdEX6X\n0m1ogm5DN2zotpwAOrFHBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQV\nAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVB\nBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaO7DuBuylqlp3\nE7Y0xlh3E66m67ICTjz2qABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0Jqg\nAqA1QQVAa4IKgNa2HVRVdVpVvbSq3lVV76yqO6+yYbBb1C7sLzsZ5uOXkrxyjPEdVXUgyXVW1CbY\nbWoX9pHazlhIVXVqkreMMW5xhOf0G2SpKeNRLW+MsaOGLVu73dZJ1/XBcnZatye67Xb93TzJJ6vq\nvKp6c1X9ZlVde5UNg12idmGf2W5QHUhyxyS/Osa4Y5LPJnn8yloFu0ftwj6z3WNUFya5cIzxl/P9\nl8bGzv6wVO0eOnToy7cPHjyYgwcP7kXbgC1sK6jGGB+vqg9X1dljjPcmuWeSd6y2abB6y9buYlAB\n67WtkymSpKpun+TZSU5O8rdJHjLGuGxheq+j0Y11O3Cf9D14v4qD0svUbrd10nV9sBwnU+zMtoPq\nqDMWVEvr9qWY9P1i3IsNXlCxaoJqZ1yZAoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuC\nCoDWBBUArQkqAFoTVAC0JqgAaE1QAdDadkf4ZYU6DuHQbZiLZG+XU7d1cqKvD05s9qgAaE1QAdCa\noAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0\nJqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUA\nrQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWDqy7AcDRVdW6m7BvjDHW3YSrsO52zh4VAK0J\nKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQ2raDqqqeUFXv\nqKq/qaoXVtVXrLJhsFvULuwv2wqqqjoryfcnueMY43ZJTkryn1bXLNgdahf2n+2OR3V5ki8muXZV\n/VOSayf5yMpaBbtH7cI+s609qjHGJUmenuSCJB9N8ukxxp+ssmGwG9Qu7D/b2qOqqlsm+ZEkZyW5\nLMnvVdUDxhgvWGHbOMGdf/75Of/881c6T7XLbtuNuj3R1XaGba6q707yTWOM75vvPzDJnccYj1h4\nTq/xoDkm3YbzTqYhvccYOxrXW+0e/7rV7irq9kS33bP+3p3kzlV1raqqJPdM8s7VNQt2jdqFfWa7\nx6jemuT5Sf4qydvmh39jVY2C3aJ2Yf/ZVtffUjPWfbKvdes+SfauC0Xt7m/dalfX3865MgUArQkq\nAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgtW0NRc9q\ndRuWIJmGJgDowB4VAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCao\nAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0J\nKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGjtwLobAOxPY4x1N2FL\nVbXuJrBi9qgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuC\nCoDWjhhUVfWcqrqoqv5m4bHrVdVrquq9VfXqqjpt95sJx0btwvHjaHtU5yW516bHHp/kNWOMs5P8\n6XwfulG7cJw4YlCNMd6Q5NJND983yfPm289Lcr9daBfsiNqF48d2jlHdYIxx0Xz7oiQ3WGF7YDep\nXdiHdnQyxZiG+Ow5zCccgdqF/WM7QXVRVd0wSarqRkk+sdomwa5Ru7APbSeo/ijJg+bbD0ry8tU1\nB3aV2oV9qKYekMNMrHpRkrsnuX6mPv2fSPKHSV6S5KZJPpjku8YYn97itbpVlnSkdbAuVbXuJmxp\njLFUw9Tu7utYt0nP2l22btnaEYNqRzO2sS+t4wbfcWNP9maDV7vL6Vi3Sc/aFVQ748oUALQmqABo\nTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYO7ObMuw0D\n0PHy/0nfdgF0YI8KgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNU\nALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYE\nFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABo7cBuzryqdnP2\nx2yMse4mbKnbcqJfrXSskY5t4vhkjwqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA\n1gQVAK0JKgBaE1QAtHbEoKqq51TVRVX1NwuP/XxVvauq3lpVf1BVp+5+M+HYqF04fhxtj+q8JPfa\n9Nirk9xmjHH7JO9N8oTdaBjskNqF48QRg2qM8YYkl2567DVjjCvmu29KcsYutQ22Te3C8WOnx6i+\nN8krV9EQ2GNqF/aJbQdVVT0xyRfGGC9cYXtg16ld2F+2NcJvVT04yX2S3GOlrYFdtmztHjp06Mu3\nDx48mIMHD+5ms4AjqKMNuV1VZyV5xRjjdvP9eyV5epK7jzEuPsLreo3lnX7Di28wpPfyxhhLL6yd\n1G63WlEj+9ux1C1Xd8SgqqoXJbl7kusnuSjJuZnOlDo5ySXz0944xnj4Fq/ttaVHUB0Plt3gd1q7\n3WpFjexvgmpnjrpHte0ZC6ql+RJa3l5s8IKKVRNUO+PKFAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDW\nBBUArQkqAFoTVAC0JqgAaE1QAdCaoNqm888/f91NoDk1AqshqLbJlxBHo0ZgNQQVAK0JKgBaO6HG\no2L/26vxqHb7PTixGI9qZ3YtqABgFXT9AdCaoAKgNUEFQGuCCoDWBBUArf1/GloXMlKp53EAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8569859d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_parallel_tapes(input_data, output_data,\n",
    "                    input_sequence, output_sequence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This seems to work as expected. Note that in case of tie, 'a' is favored over 'b', 'b' over 'c' and so on. The function to learn is therefore fully deterministic. Let's generate a reference dataset for our RNN experiments.\n",
    "\n",
    "For let's start with random input sequences:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_sequences(n_sequences=1000, min_length=5, max_length=15,\n",
    "                       alphabet=ALPHABET, seed=None):\n",
    "    sequences = []\n",
    "    rng = np.random.RandomState(seed)\n",
    "    for i in range(n_sequences):\n",
    "        length = rng.randint(min_length, max_length)\n",
    "        sequences.append(\"\".join(rng.choice(ALPHABET, length)))\n",
    "    return sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['addddbdbca', 'caaacbcddcab', 'bbabadadbc', 'cdabd', 'dcdabbbd']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sequences(n_sequences=5, seed=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `seed` parameter makes possible to control the random number generator to get seemingly random yet reproducible deterministic outcomes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['addddbdbca', 'caaacbcddcab', 'bbabadadbc', 'cdabd', 'dcdabbbd']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sequences(n_sequences=5, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['daadbdbdaa', 'adbacb', 'acbcada', 'addbbdc', 'cbbbd']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sequences(n_sequences=5, seed=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have random input sequences, we can compute the expected ouput for each of them and generate a full dataset to train our networks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generate_running_max_task_data(n_sequences=1000, min_length=5,\n",
    "                                   max_length=15, window_size=3,\n",
    "                                   alphabet=ALPHABET, seed=None):\n",
    "    \n",
    "    # Generate the input sequences as symbols\n",
    "    input_sequences = generate_sequences(\n",
    "        n_sequences=n_sequences, min_length=min_length,\n",
    "        max_length=max_length, alphabet=alphabet, seed=seed)\n",
    "    output_sequences = []\n",
    "\n",
    "    # Allocate all the data arrays at once and pad with zeros\n",
    "    n_features = len(alphabet)\n",
    "    input_data = np.zeros((n_sequences, max_length, n_features),\n",
    "                          dtype=np.float32)\n",
    "    output_data = np.zeros((n_sequences, max_length, n_features),\n",
    "                           dtype=np.float32)\n",
    "    \n",
    "    for sequence_idx, input_sequence in enumerate(input_sequences):\n",
    "        # store the encoded sequence to the input tape\n",
    "        encoded_input = symbols_to_binarray(input_sequence)\n",
    "        input_data[sequence_idx, 0:len(input_sequence)] = encoded_input\n",
    "        encoded_output = running_max(encoded_input,\n",
    "                                     window_size=window_size)\n",
    "        output_data[sequence_idx, 0:len(encoded_output)] = encoded_output\n",
    "        output_sequences.append(binarray_to_symbols(encoded_output))\n",
    "\n",
    "    return input_sequences, output_sequences, input_data, output_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try it to generate a toy dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "(\n",
    "    input_sequences,\n",
    "    output_sequences,\n",
    "    input_data,\n",
    "    output_data,\n",
    ") = generate_running_max_task_data(n_sequences=10, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('addddbdbca', 'aadddddbba'),\n",
       " ('caaacbcddcab', 'aaaaaacbddaa'),\n",
       " ('bbabadadbc', 'aabbaaadab'),\n",
       " ('cdabd', 'aaaaa'),\n",
       " ('dcdabbbd', 'aadaabbb'),\n",
       " ('dcadd', 'aaaad'),\n",
       " ('dacaaab', 'aaaaaaa'),\n",
       " ('caabdabccd', 'aaaaaaaacc'),\n",
       " ('bbdbbdcddccda', 'aabbbbbdddcca'),\n",
       " ('babcadac', 'aabaaaaa')]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zip(input_sequences, output_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAJnCAYAAAC03nW3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHLlJREFUeJzt3XmUrHdd5/HPN7mEJDKGICMQErY4OSMwOoIiokJGxAkM\nKm64MEIQx+OoB51BEJcxERD37TiOjgsRlMUMIooHBRSvMIIRlU0SViGQREJCFhBkWPKbP+q5pOj0\nvbdv9ffpru77ep3T53TX0/XUr7q/Ve9aurpqjBEA6HLCbi8AgP1FWABoJSwAtBIWAFoJCwCthAWA\nVsLCtlTVP1TVA3d7Hd2q6qaqusdhtp1fVa88wnEPVtXjDrPtbtO+Tzja9x7m+O+qqi+bPr+wqn5n\nq8dl5+zU/Gyy/ZMzcSzf201Y2JYxxr3HGK+Y+3Sq6rer6qnHeJyb5lrPUYzpo/t7D33/Zp+3meJ1\nlzn2vXQaR7zS24n9TVesF3ScfrNjnYmNx53jez/FFMeLDrddWGDvqpn2u5Ovmu4+D8eyv+P91eFz\nzY+wsD2bPDRzcVU9s6o+MD1Mdt8N3/vkqnpTVV1XVc+oqltP227x8MB0C/TsqvqOJN+S5ElV9cGq\n+sMtLu+TVxxV9diqunRa1zumfS6f1hOr6qqquqKqvm3Dts+oqj+qqhur6pIkZ2/Y/pCqenNV3VBV\nv5zFBbambSdW1c9W1TVV9Y4k/2mTdX5WVV0y7f+FVXX60r6/taour6prq+qHNjl/J1fV86bz9XdV\n9TlLxz2rql5QVe+bjv/L0+FnV9XLp8OuqarfrarTtvgz/RRV9YCqes103v+mqr5oadu7qurBS18v\nP/Ry6F7uDdPa7z/NwF9V1S9P+7vs0Gwd4/4+WFVfuMWzMKZ9nV5Vfzz9rK6rqhdV1Z2XTmst56eq\n7l5Vfzmt66VJbr/JeXxcVV05re8JG877kebnyVX19mnbm6rqERuOe/gwjzF8+Fj5I8k7k3zZ9PmF\nSf4lyXnTBePpSV699L3vSvKGJHdOcnqS/5vkqdO285O8csO+b0pyj+nzi5I8ZcP2X0nyK1tc58OS\n3H36/IFJPpTk86avz0vy3iT3THJqkudsOO3nTR+nJLlXkiuSvGLadvskH0jytUlOTPJ9ST6W5Num\n7d+Z5LKl8/wXST6R5IRp+8Fpf4dO+/lJfmfads8kH0zyJUlOSvJz076Xf94fXTrtJyT5x+nzE5O8\nfjrOKUluneSLp+OdneTBSW41rf8vk/zCCr/72yW5PsmjsriR+k1Jrkty+sbZmL6+YOm83XX6GZ+w\ntP386fx977T+Rya5IcltV9zfXab1nbnF8/I1SU5OcpskFyf5gz0wP69O8rPT7/JLp309a9p2t2kd\nz55O+95J3pfkwUebn2n71ye54/T5I5P886Gvj/rz3O0rJh97+yO3DMtLl7bdM8mHN3zvdyx9/dAk\nb58+Pz9HD8tTG9f9B0keP33+jCRPX9r2bw6d9nSB+2iSc5a2//ihtSZ5dJJXbdj3e5auGF6+4Tw/\nJEtXgNMVxfJpf3aS/5fFFfWPJnnO0rZTp23LP+9XLW2vJFdlEaIvmq5ETtjCz+IRSf5+hZ/htyb5\n6w2HvSrJozfOxtJ6D4Xg0JXexrBcuWF/lyR51Cr72+Z8/Psk163z/GQRzo8lOWVp+7M3+Zksn/ZP\nJfnNo83PYc7za5N81VZ+fh4Ko9vVS59/OIu72stz9p6lz9+d5IydWFRVPbSq/rqq3l9V12dxC/Qz\nps132mRdh/zrJAeOsP2MLG6BLlv+3iPte7Pvf3duvidxp+V9jzE+nOT9G467vH1MX5+R5Mwkl48x\nbvEHDFV1h+nhjyuq6sYkv5ObfxbH4oxNzs/lWdy6XtWVm+xv9hmpqlOr6n9PD7fdmMW9uNOq6tBD\nUus4P2ckuX6M8S9Lh12+ydk70mVus/m5U5JU1aOr6rVVdf10nu+dLc6JsLDT7rLh86umzz+UxS3y\nJElV3XHD8bbzFyy3TvL7SX46yWeOMU5P8uLc/OTlP22yrkOuSfLxI2y/KslZS6dVy18fZd+bHXbo\nVui103GX931qbnnBXt5+QhZBuTKLK5O7VNWJm5ze07N4OOXeY4zTsrjnscp1wZVZPAS17K65OQ4f\nSvJpS9uWf6eH+31ujNJd86kzcqz726onJDknyf2mn8mDMj3Xscbz809JTp/m4pC75pY/i43HX473\nZvNzVVXdNcmvJ/nuJLebzvM/ZItP+AsLO6mSfFdV3bmqbpfkh7N47DlZPB9wr6r63Ko6OYu76cuu\nzuKhhVWcNH1cm+Smqnpokq9Y2n5xkvOr6rOnC+kn/wR1jPGJJC9IcmFVnVJV90zymNx84X3xtO6v\nqaoDSR6fT73CuzjJ46fzfHqSJ29YWyX5z0un/ZQk/2e6p/H7SR5eVV9cVSdN2zZeZu+7dNrfl+Qj\nSf46yWuyuOL5yenW+MlV9YDpOLfJ4kr6A9MT1E883A9uekL9nYfZ/OIk51TVN1fVgar6xiT/Nskf\nT9tfl+Sbpm2fn+Trln5u12TxMM3ZG/b5mVX1+Kq6VVV9w7S/F29jf1t1myyeH7xxms0Llrat5fyM\nMS5P8rdJfmz6eX1Jkodvct5+ZDrte2XxcOPvLW073Px82rTGa5OcUFWPzeIey5YIC502+0uRseHz\n5yR5aZJ3JHlbkqclyRjjrVlccf5ZkrckeeWG4/5WkntOd8tfkCRV9WtV9atHXdQYH8ziAntxFk8u\nf3OSP1za/qdJfjGLx7PfmuTPN5z292RxxfPeLB5Pf8bSca9N8g1JfjKLC+FnZfFHCYf8RpKXZBHO\nv80iFht/Js9K8ttZhOCkaa0ZY7wpi1uMz8nilu11+dSHNUaSFyb5xmnbo5J87RjjE9MV2ldO63n3\ndLxHTsf7sST3SXJjkhdtsqZlZ204Pzef+BjXZXFF9oTpvH9/kodPhyfJ/8jiiv76LG4oPHvpuB/O\n4rmGv6rFX2F94bSGS7J4juKaJE9N8nVjjOuPcX/XV9X9quoutfgLsTMPc96W/WIWT3Bfm8XzRH9y\n6Gey5vPzLUm+cFrXjyZ55obzNbJ4WO/tWVy2fmaM8WdL2w43P5dm8Ycfr57Wfe8cZg42U9OTMjC7\n6Zbv48YYL9/ttbA1VfWSLJ6kfssOnNb5WczHl859WszrwG4vAFhfY4z/uNtrYO/xUBiwLjZ7KJU9\nyENhALRyjwWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCcsWTf+M76aqeuBur2Wvmv4l+V/s9jqO\nJ+a2h9k9NsLCTvLKavYqs3sMhIWdtKX3coA1ZHaPgbAA0Gotw1JVt6mqp1XVJVV1TVV9pKreVlU/\nUVWn7PLyblVVF1bV5dO6Xj+9wdGuqKqTqupJVfW6qvpQVd1QVa+pqu/exTWdVVUXV9WN08cfVdWq\nb8C0Z5jbY2N29691/bf5ZyZ5XJLnJ/ndLN7a89wkT0ryeUnO27WVJT+VxVvo/s8s7h4/Nslzq+rk\nMcbGN9mZ1fSugi/J4m1UX5LFG0Z9JMnnJPmaJL+yk+uZ1nTbJK/I4nf4q0kuzeJ39/Is3khpPzO3\nW2R297kxxtp9JLlVkhM3OfwpWbz96BfswprOn077nUn+1dLhn57kXUnen+TkHV7Tk6Y1PW2TbbVL\nv7unT2t6zIbDf2E6/OW7NVc7cN7N7dbXZXb38cdaPhQ2xvjYWLy1aqb3tz69qm6fxVt+Jsn9dm91\n+dWxeKvSJMkY4wNJfi3J6VncutlJj8riLUWfsnHDmC4Ru+ARWbyV6bM2HP5Tu7CWHWVuj4nZ3cfW\nMixJUlXfVVVvyOLu8fuTvC/Job8jP33XFpZcdoTD7r6TC8nivcHfPMb46A6f7pHcI8nbNl45jDHe\nm8V7rO9r5nbLzO4+tpbPsVTVf0/ys1k89vqLSa5K8tEsHvv87axxEDl+mVtYWMuwJPnWJO8cYzx0\n+cCq2s0nPw+5Z5IXbXJYkvzjDq/lLUk+u6pOWqNbfv+Y5JyqOmGMcdOhA6vqTklO271l7Qhzu3Vm\ndx9b11tQH0+Sqvrk+qrqQJIn79qKbvZfq+rTD31RVacl+c4k1yf5yx1ey7OzeHjlRzZuqKrdekHX\nC5PcIcmjNxz+A7uwlp1mbrfO7O5j63qP5flJfiLJn1TVH2TxFyzfksXDCrvtmiSXVNVFufnPNs9M\n8u1jjI/s8Fp+KclXJvmRqvqCJC/L4rH9eyU5J8lDdng9SfLTWfyufqOq7pub/2Tz/kmuzf5+BbO5\n3Tqzu4+ta1h+Jotf4uOyeKz6n5L8XhaPU1+6e8vKyOLWywOTfHcWt27ekuRRY4zn7fhixvhYVX1F\nkidkcYH48SwunG9NctFOr2da0w1V9aVJfj433/I7mOQ/ZPHXUfv5/y2Z260uyOzua7V7f9kHwH60\nrs+xALBHCQsArYQFgFbCAkArYQGglbAA0EpYAGg12wskq8oLZGg3xpj91c9mlznsxOyui1nvsXS9\nacwFF1zQ9UY+sCXrNLdml73GQ2EAtBIWAFrtibCce+65u70EOGbmluPVbP+Esqp28a2rN7d7b/NA\nl5168t7s0s2T9wCwImEBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAViuHparOq6o3V9XbquoHOhcF\nczK7MK+VXnlfVScmeUuSL09yZZLXJPnmMcZlS9/j1cu02+6rl80uu8Ur74/ufknePsZ41xjjY0me\nl+Sr+5YFszG7MLNVw3LnJO9Z+vqK6TBYd2YXZrZqWNbrcQLYOrMLM1v1rYmvTHLW0tdnZXHL71Nc\neOGFn/z83HPP9W/EWQdmF2a26pP3B7J4AvTBSa5K8jfxBCg7oOHJe7PLrjienrxf6R7LGOPjVfU9\nSV6S5MQkv7V8wYR1ZXZhft7oiz3FG32xVx1P91i88h6AVsICQCthAaCVsADQSlgAaCUsALQSFgBa\nCQsArYQFgFbCAkArYQGglbAA0EpYAGi16ht9bYn/yLp3rdt/903ME+wV7rEA0EpYAGglLAC0EhYA\nWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUs\nALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBK\nWABoJSwAtBIWAFoJCwCtDuz2AmAdVdVuL4EVjTF2ewm3cLzNk3ssALQSFgBaCQsArYQFgFbCAkAr\nYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAq5XDUlVnVdVfVNWbquofqurxnQuDOZhbmF+t\n+i+mq+qOSe44xnhdVd0myd8lecQY47Jp+/r972q2bF3/9fgYY1v/f/xoczt9z/qdebZsv87uXrLy\nPZYxxnvHGK+bPv/nJJclOaNrYTAHcwvza3mOparuluTzklzSsT/YCeYW5rHtsEwPJzw/yfdOtwBh\n7ZlbmM+23pq4qm6V5PeT/O4Y44U9S4KFgwcP5uDBg+37NbfMba7Z3Su28+R9JXlmkvePMf7bJtvX\n7xk0tmy/PgF6tLmdvmf9zjxbtl9ndy/ZTli+JMkrkrwhyaGd/OAY40+n7ev322XL9uuF82hzO33P\n+p15tmy/zu5esnJYjrpjF8497Xi+cJrdve14nt114ZX3ALQSFgBaCQsArYQFgFbCAkArYQGglbAA\n0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCth\nAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBW\nwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsA\nrYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtNpW\nWKrqxKp6bVW9qGtBMDdzC/Pa7j2W701yaZLRsBbYKeYWZrRyWKrqzCQPS/KbSaptRTAjcwvz2849\nll9I8sQkNzWtBXaCuYWZrRSWqnp4kveNMV4bt/rYI8wt7IwDKx7vAUm+qqoeluTkJJ9eVc8aYzy6\nb2kc7w4ePJiDBw927tLcsiNmmN09pcbY3vOXVfWgJN8/xvjKDYd7YnQP2+5czKGqMsZouadxuLmd\ntq3fmWfL9vvs7gVdr2NZv98kHJ25hRls+x7LYXfsVt+edjzf6jO7e9vxPLvrwivvAWglLAC0EhYA\nWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQKtV30FyT1rHf6edLP6l\n9rpZxzUBe4N7LAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsIC\nQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2E\nBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQKsDu72AnVRVu72EPWOMsdtLuAW/\nP9gb3GMBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBarRyW\nqrptVT2/qi6rqkur6v6dC4O5mF2Y13b+bf4vJXnxGOPrq+pAkk9rWhPMzezCjGqV992oqtOSvHaM\ncY8jfM/6vaEHW7au78cyxtjWm7KY3f1vv87uXrLqQ2F3T3JNVV1UVX9fVb9RVad2LgxmYnZhZquG\n5UCS+yT5X2OM+yT5UJInt60K5mN2YWarPsdyRZIrxhivmb5+flw4aXbw4MEcPHiwe7dml9nNNLt7\nxkrPsSRJVb0iybePMd5aVRcmOWWM8QNL29fvgU62bD8/Tm1297f9PLt7xXbC8rlJfjPJSUnekeSx\nY4wbl7av32+XLdvPF06zu7/t59ndK1YOy1F37MK5px3PF06zu7cdz7O7LrzyHoBWwgJAK2EBoJWw\nANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALRa9R0k96R1/HfayeJfaq+b\ndVzT8czsbt06rul44x4LAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCth\nAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBW\nwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaDVgd1ewE6qqt1ewqbGGLu9hFtY\n158VsP7cYwGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFqt\nHJaq+sGqelNVvbGqnlNVt+5cGMzF7MK8VgpLVd0tyX9Jcp8xxr9LcmKSb+pbFszD7ML8Vn0/lg8k\n+ViSU6vqE0lOTXJl26pgPmYXZrbSPZYxxnVJfi7Ju5NcleSGMcafdS4M5mB2YX6rPhR2dpLvS3K3\nJGckuU1VPapxXTALswvzW/XJ+89P8qoxxvvHGB9P8oIkD+hbFszG7MLMVg3Lm5Pcv6pOqcWbo395\nkkv7lgWzMbsws1WfY3l9kmcl+dskb5gO/vWuRcFczC7Mr8YY8+y4ap4d70Nz/Q62Y3Fjfv2MMWZf\n2DrO7jrOSLK+c7KOdmJ214VX3gPQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGgl\nLAC0EhYAWgkLAK2EBYBWB+bc+br9q+91/Rff67qu45nZhdW5xwJAK2EBoJWwANBKWABoJSwAtBIW\nAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGgl\nLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQ\nSlgAaCUsALQ6MOfOq2rO3cNszC6szj0WAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBa\nCQsArYQFgFbCAkArYQGg1RHDUlXPqKqrq+qNS4fdrqpeVlVvraqXVtVt518mHBuzC7vnaPdYLkpy\n3obDnpzkZWOMc5L8+fQ1rBuzC7vkiGEZY7wyyfUbDv6qJM+cPn9mkkfMsC7YFrMLu2eV51juMMa4\nevr86iR3aFwPzMnswg7Y1pP3Y4yRZDStBXaM2YX5rBKWq6vqjklSVXdK8r7eJcFszC7sgFXC8kdJ\nHjN9/pgkL+xbDszK7MIOqMUjAofZWPXcJA9KcvssHpP+0SR/mOTiJHdJ8q4kjxxj3LDJcT3MQLsx\nRm3l+8wu62ars7sfHDEs29qxCycz2IkLp9llDsdTWLzyHoBWwgJAK2EBoJWwANBKWABoJSwAtBIW\nAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGgl\nLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQ\nSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EB\noJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFZH\nDUtVPaOqrq6qNy4d9jNVdVlVvb6qXlBVp827TDg25hZ2z1busVyU5LwNh700yb3GGJ+b5K1JfrB7\nYbBN5hZ2yVHDMsZ4ZZLrNxz2sjHGTdOXlyQ5c4a1wcrMLeyejudYvi3Jixv2AzvJ3MJMthWWqvrh\nJB8dYzynaT0wO3ML8zqw6hGr6vwkD0vy4LbVwMzMLcxvpbBU1XlJnpjkQWOMj/QuCeZhbmFn1Bjj\nyN9Q9dwkD0py+yRXJ7kgi7+mOSnJddO3vXqM8V0bjnfkHcMKxhi1le9bdW6n45pd2m11dveDo4Zl\n5R27cDKDnbhwml3mcDyFxSvvAWglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIW\nAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGgl\nLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQ\nSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EB\noJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaHXEsFTVM6rq6qp64ybbnlBV\nN1XV7eZbHqzG7MLuOdo9louSnLfxwKo6K8lDklw+x6KggdmFXXLEsIwxXpnk+k02/XySJ82yImhg\ndmH3HPNzLFX11UmuGGO8YYb1wGzMLuyMA8fyzVV1apIfyuKhhE8e3LoimIHZhZ1zrPdYzk5ytySv\nr6p3Jjkzyd9V1Wd2LwyamV3YIcd0j2WM8cYkdzj09XQBve8Y47ruhUEnsws752h/bvzcJK9Kck5V\nvaeqHrvhW8ZsK4NtMLuwe2qMeS5fVeWCS7sxxuzPi5hd5rATs7suvPIegFbCAkArYQGglbAA0EpY\nAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCtZnujLwCOT+6xANBK\nWABoJSwAtBIWAFoJCwCt/j/d6Aw5VjIoiQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8567bc450>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAJnCAYAAAA6BDzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHV5JREFUeJzt3Xu4bHdZH/DvS05CiJEkSMVAggHaPBWkFCzUK+yK+gS8\noe2jFYuC1EuhQi2KoFSOaMEr6NOitmLCRSAiAkqLCkqiabWIitzvhkDIBUhCUBAB8+sfs04y2Tln\nnzmzZ/Y7O+fzeZ7znJm1Zq15Z9a75rsus2fVGCMA0Ok23QUAgDACoJ0wAqCdMAKgnTACoJ0wAqCd\nMOJGVfWWqnpgdx37SVVtVdUHVjSv91XVg9f5PDs9B/tfVZ1TVTdU1UKf7Ufrq6p6blX9xOoqPDJh\nxI3GGF84xvjjdT/PMg1eVTesq54NMqZ/Lc8xfTBdtObnT1UdrKoXdM5vCuW7rqqGW7GV9OQivSWM\ngOORv/ZfXO3FkwgjbjRtLX7ldPtgVb2kqp5XVR+bDuF90bbHPqmq3lpV11bV+VV122ncI6vqkm3z\nvqGq7lFV35Pk4UmeWFV/U1W/vWB5N354VNUdquqCqvrg9Nwvn4afUVX/q6o+NA1/ZVXdZW66R1XV\n26bX896plkPjTj/KtId9zrnxT66qD1fVpVX18Lnht6uqn5/er49W1SVVdfI07hFVdVlVfaSqfmTb\n/G437UFeW1VvTXL/bePPrqqXTfV+pKr+2zT8HlX12mnYh6vq16vqtG3v5QMOt9xyDFvBNfOU6XVd\nPfXJ7adxtzj0c+jwYFWdl+TJSb51Wv5vmMZfXFXPqKrXVdX1VfWKqjpj2fkdi6r62qp6w/S876+q\np24b/5tVdeW0/P6oqu65AdMerq9uOzf5o6devaKqnrBtup366r5V9ZfTOnJhkpPnxi29fmWR3hpj\n+OdfxhhJcmmSr5xuH0zyd0nOy2zL6OlJ/nTuse9L8qYkd0lyRpL/k+QnpnGPTHLJtnnfkOTu0+0L\nkjxt2/hnJ3n2gnX+7yQvTnJakgNJvmIafock3zStQKcmeUmSl89N99Akd5tuPzDJx5Pcd8Fpj/Sc\nW0k+neTnkpw4zfdvk5w797pem+TMzDb+vjjJSUnumeRvknz5dP/np/kcev9/KskfJTk9yVlJ3pLk\n/dO4E5K8cZrmdklum+TLpnH3SPLgqZY7TvN41iLL7Rh75buSvDvJOUk+K8lvJXn+3HvygR1666mH\nHjs3/uIkl0/vyylJXprkBbuY35OSvHLB1/KgJPeabt87yVVJvnFu/COn13hikmclecMGTHukvjon\ns3XthVNvfGGSDyV58AJ9dVKSy5I8fuqxf53kU5nW1exi/VpoOezVB51/m/8vtwyjV8+Nu2eST2x7\n7PfM3X9IkvdMtx+Zo4fRMX8ATtOemeQfkpy2wGP/eZJrdxj/8iSPO9q0Oz1nbgqj280N+40kT5k+\nJD6R5N6Hme7Hkrxo7v4pSf5+7v1/b5KvmRv/3Zk+kJN8yfQBc5sF3oOHJfnLRZbbMS6HP0zyfXP3\nz50+uG6To4fHwUxBMzf+oiRPn7v/BdP7UcvMb5frwS8keeYRxp0+9fJnd017lL46Z3rcuXPDfjrJ\ncxboqwcm+eC2+f3fbNtwXMX6dbh/DtOxk6vnbn8iycl182/pzB86eX+SO+9BTWdntgJcv31EVZ1S\nVf9jOnRxfWZbgKdVVU3jH1JV/6+qrqmq6zLbkvucBaY94nNOrhtj/N3c/csyC7DPyWwr8r2HmebM\nzPYEkiRjjE8kuWZu/J1zy/d3/j24bIxxiy91VNWdqurCqrp8eh0vOPQa56xiuZ2Z2eucn8+BJHda\nYl5HquvQ3t1aVdW/rKqLpsNPH03yvbmpL06oqp+qqvdM7+elmR1uumPDtJmmvWOO3FeHbH8vz5xu\n79RXd07ywW3zuSzTOaPdrF+LEEbsxl233b5iuv3xzLb0kyRV9XnbptvNyeMPJLlD3fI8SJI8IbMt\n9AeMMU7L7DBIzUqo22Z2KOlnknzuGOOMJK/KTSdnjzjtUZ4zSc6oqlPm7n9+Zu/FR5J8Msk/Psw0\nV2YWKsmswFNy8xX3ytzy/Z1/D+5aVSccZr5Pz2wv7gun1/GI3HI9P9JyOxZXZLYVPj+fz2S2AbN9\n+Z+Q5B/NPfZIy397XZ/O7D1cdn6LelGSVyQ5a4xxepJfyU198fAk35DZYa7TktwtN/XFXk+bafxO\nfXXIkZbxTn11ZWaHb+d9fm56f3ezfh2VMGJZleQxVXWXqrpDkh9NcuE07o1J7lVV96nZyfqD26a9\nOsndl3nSMcaVSX43yS/V7EsHJ1bVV0yjT83sPNf1U03zJ4RPmv59JMkNVfWQJF8zN/6I0x7hObf/\nPdaPz9XytUl+c8yOVZyf5JlVdea0tfslVXVSZudEvq6qvmy6/7TcfH18SZInT893VpLvnxv3Z5l9\ncPzUtLV6clV96dzr+HiSj00nl39oW52V5LFHWG43f+DsSwVPPdy4zM6f/UDN/q7l1MxC8MJpb+1d\nme1FP7SqTszskOX8yfWrkpxzaIt6rq5/V1VfMAXz0+bew2XmdyxOzWzv9lNV9YDMgmB+3N8nubaq\nPmt6na3TTu/xkfrqkKfU7MsK98rssPlvTMN36qs/TfKZqnrc1MvfnJt/wWE369dRCSOO5HDffhnb\nbr8oyaszO1zw7iQ/mSRjjHdl9mHyB0nemeSSbdP+WpJ7VtV1VfWyJKmqX6mqX16wtkdkttX8jsyC\n7fHT8F/I7KTtR5L8SWYBMqaa/ibJ4zJbGa9N8m1J5r/Jd8Rpj/Ccj5sbd2WS6zLb+nxBku+d3oMk\n+cEkb07y+swOwz0jSY0x3pbksZm9h1dMNc0fPvnxzA6RXJrk95I8f+61/EOSr89sy/j903TfMjfd\n/ZJcn+SVmW2tbl9uL8xhltthnJXZFxwO5/zptf5xkr/O7DDu90/1XZ/kMUmek9mhyL/d9tp+c/r/\nmqr687m6XpDkuZm9nydleo+XmV9V/UhVveoItW/3mCRPq6qPJfkvuemDO5m975dldvjqLZl9YI8N\nmPawfTWNG5kdQntPZuvgz44x/mAat1NffSrJN2cWXtdk1lO/Nfecu1m/jqqmE01wTKrq0iSPHmO8\ntrsWVm/aar5wjPHle/R8F2X2JYTz9+L52DwHugsANs8Y4/LMvna+l/bkjyvZTA7TAZvCYZrjmMN0\nALSzZwRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2G0oJpdvfSGw/xAJguafnr+ou46jif6djX0\n7voJI/bS4X58FfYDvbtmwoi95LfH2K/07poJIwDabWQYVdWpVfWTVfW6qvpwVX2yqt5dVc+oqts1\nl3diVR2sqsumut5YVd/aVUxVnVRVT6yqv6qqj1fVR6vq9VX12Maazq6ql1TV9dO/36mqe3TVs1f0\n7bHRu8zb1EtInJXk0ZldDfPXM7uc8VaSJya5b5Lz2ipLfjqzSyD/98x23R+V5MVVdfIY43l7Wch0\nZcffz+zyv7+f2YWyPpnknyX5piTP3st6pppOz+yCa2cl+eUkb8ts2b02swtz3Zrp2wXpXW5hjLFx\n/5KcmOSEwwx/WpIbkty/oaZHTs99aZLPnht++yTvy+zKiCfvcU1PnGr6ycOMq6Zl9/Sppu/cNvxZ\n0/DXdvXVHrx2fbt4XXrXv5v928jDdGOMT4/ZpZVTVQeq6oyqumOSP5we8oC+6vLLY3aJ3STJGONj\nSX4lyRmZbUXtpW/P7BK/T9s+YkxrUYOHJbkqsy3deT/dUMue0rfHRO9yMxsZRklSVY+pqjdltut+\nTZIPJTn0Pf8z2gpL3r7DsLvtZSFJ/kmSd4zZtes3xd2TvHv7B8oY46ok1/eUtHf07cL0LjezkeeM\nquo/J/m5zI4l/0KSK5J8KrNjuc/NBocoxy99C8vbyDBK8ogkl44xHjI/sKo6TwAfcs8krzzMsCT5\n6z2u5Z1JvqCqTtqgLcy/TnJuVd1mjHHDoYFVdWaS0/rK2hP6dnF6l5vZ1C21zyRJVd1YX1UdSPKk\ntopu8h+q6vaH7lTVaUm+L8l1Sf5oj2t5YWaHfp6yfURVdf2R3iuS3CnJd2wb/sMNtew1fbs4vcvN\nbOqe0UuTPCPJ71bVyzP75s/DMzvk0e3DSV5XVRfkpq/InpXk348xPrnHtfxikq9P8pSqun+S12R2\nruJeSc5N8tV7XE+S/Exmy+pXq+qLctPXY784yUdy6/5Ldn27OL3LzWxqGP1sZgv+0Zkde78yyW9k\ndtz9bX1lZWS2lfTAJI/NbCvqnUm+fYxx4Z4XM8anq+prkjwhs5Xov2a2Qr8ryQV7Xc9U00er6iuS\nPDM3bWFenORfZfatslvz73vp20UL0rtsU33fogSAmU09ZwTAcUQYAdBOGAHQThgB0E4YAdBOGAHQ\nThgB0G5tf/RaVf6AiZUbY6z9r+D1LuuwF727n9kz2gCrujjVU5/61JXN63hnecDeEkYAtBNGALQT\nRrciW1tb3SUwx/KAxa3th1KdBF7cJp4T6LukzM726gsMm7ZMNnV5sDhfYNiZPSMA2gkjANoJIwDa\nCSMA2gkjANoJIwDaCSMA2gkjANotHUZVdV5VvaOq3l1VP7zKomCd9C5snqV+gaGqTkjyziRfleSD\nSV6f5NvGGG+fe8xm/Qn7Btu0v/ZPNvcv/nf7V+yL9u6mLZNNXR4szi8w7GzZPaMHJHnPGON9Y4xP\nJ7kwyTeurixYG70LG2jZMLpLkg/M3b98GgabTu/CBlo2jDbrGAYsTu/CBlr2suMfTHL23P2zM9vC\nhE23UO8ePHjwxttbW1suBwFrtuwXGA5kdhL4wUmuSPJn8QWGpW3ayfJkc0+Yr+ALDAv17qYtk01d\nHizOFxh2ttSe0RjjM1X1H5P8fpITkvza/MoMm0rvwmZycb0NsGlb4cnmbom7uB77lT2jnfkFBgDa\nCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaLXtxvYX45ePF\nbGpdAHvFnhEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA\n7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDt\nhBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2E\nEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDtlg6jqjq7qi6qqrdW1Vuq6nGr\nLAzWQd/CZqoxxnITVn1eks8bY/xVVZ2a5C+SPGyM8fZp/Fh23utSVd0lsEtjjF0txKP17fQYvcvK\n7bZ3b+2W3jMaY1w1xvir6fbfJnl7kjuvqjBYB30Lm2kl54yq6pwk903yulXMD/aCvoXNseswmg51\nvDTJ46ctTdh4+hY2y4HdTFxVJyb5rSS/PsZ4xfbxBw8evPH21tZWtra2dvN0sBJH69tE78Je280X\nGCrJ85JcM8b4gcOMdxKYlVvBFxh27NvpMXqXlfMFhp3tJoy+PMkfJ3lTkkMzefIY4/em8VZoVm4F\nYbRj306P0busnDDa2dJhdNQZW6FZg71YofUu6yCMduYXGABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgn\njABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeM\nAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wA\naCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABo\nJ4wAaCeMAGh3YJ0zr6p1zv5WY4zRXcItWHbAXrJnBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7\nYQRAO2EEQDthBEA7YQRAO2EEQDthBEC7XYVRVZ1QVW+oqleuqiBYN30Lm2e3e0aPT/K2JJt3DQQ4\nMn0LG2bpMKqqs5I8NMlzkrj4DfuCvoXNtJs9o2cl+aEkN6yoFtgL+hY20FJhVFVfl+RDY4w3xNYl\n+4S+hc217GXHvzTJN1TVQ5OcnOT2VfX8McZ3rK40WLmF+/bgwYM33t7a2srW1tZe1QjHpRpjd+dw\nq+pBSX5wjPH124Y7Obyg3S6DdajazB2HMcZKCjtS307jxqYtk01dHixuVb17a7WqvzParDUXFqNv\nYUPses/oiDO2Z7SwTdsKTzZ3S3wvti7tGbEO9ox25hcYAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeM\nAGgnjABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABot+yVXvelTbsswCEuDwAc7+wZAdBOGAHQThgB\n0E4YAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBOGAHQ\nThgB0E4YAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBO\nGAHQThgB0E4YAdBOGAHQThgB0E4YAdDuQHcBe6mquks4rDFGdwm3sKnv1V453l//ovQuq2LPCIB2\nwgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2S4dRVZ1eVS+t\nqrdX1duq6otXWRisi96FzbObS0j8YpJXjTH+TVUdSPJZK6oJ1k3vwoapZa5HUlWnJXnDGOPuOzxm\n8y50sqFcE2ZxY4xdFaZ3V0vvLm63vXtrt+xhursl+XBVXVBVf1lVv1pVp6yyMFgTvQsbaNkwOpDk\nfkl+aYxxvyQfT/KklVUF66N3YQMtG0aXJ7l8jPH66f5LM1vBYdPpXdhAS4XRGOOqJB+oqnOnQV+V\n5K0rqwrWRO/CZlrqCwxJUlX3SfKcJCcleW+SR40xrp8bv3lnNjeUk8CLW8VJYL27Onp3cb7AsLOl\nw+ioM7ZCL8wKvbi9WKH17uL07uKE0c78AgMA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2E\nEQDthBEA7YQRAO2EEQDthBEA7YQRAO0OdBfAZv7kvUsDbJZNXB7J8b1MWC17RgC0E0YAtBNGALQT\nRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNG\nALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YA\ntBNGALQTRgC0E0YAtBNGALQ70F0AcHRV1V3CvjHG6C7hFiy/o7NnBEA7YQRAO2EEQDthBEA7YQRA\nO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEC7pcOoqp5cVW+tqjdX1Yuq6rarLAzWRe/C\n5lkqjKrqnCTfneR+Y4x7Jzkhyb9dXVmwHnoXNtOy1zP6WJJPJzmlqv4hySlJPriyqmB99C5soKX2\njMYY1yb5+STvT3JFko+OMf5glYXBOuhd2EzLHqa7R5L/lOScJHdOcmpVffsK64K10LuwmZY9TPcv\nkvzJGOOaJKmqlyX50iQvXFVhcPHFF+fiiy9e9Wz1Lmu3pt69VatlrhdfVffJbOW9f5JPJnlukj8b\nYzx77jGbdyF6FrZMX6xbVWWMUbuch969lbu19u6t3bLnjN6Y5PlJ/jzJm6bB/3NVRcG66F3YTEvt\nGS00Y1uX+9rxvHWpd/e347l39zO/wABAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQDthBEA7\nYQRAO2EEQDthBEA7YQRAO2EEQLtlr/TKCm3qT94D7BV7RgC0E0YAtBNGALQTRgC0E0YAtBNGALQT\nRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNG\nALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YAtBNGALQTRgC0E0YA\ntBNGALQTRgC0O7DOmY8x1jn7Y1ZV3SUc1qbWdTzTu/uX92p/smcEQDthBEA7YQRAO2EEQDthBEA7\nYQRAO2EEQDthBEA7YQRAO2EEQDthBEA7YQRAO2EEQLsdw6iqzq+qq6vqzXPD7lBVr6mqd1XVq6vq\n9PWXCcdG78L+crQ9owuSnLdt2JOSvGaMcW6SP5zuw6bRu7CP7BhGY4xLkly3bfA3JHnedPt5SR62\nhrpgV/Qu7C/LnDO60xjj6un21UnutMJ6YJ30LmyoXV3pdYwxquqIl8Q8ePDgjbe3traytbW1m6eD\nldG7sFnqaJdXrqpzkrxyjHHv6f47kmyNMa6qqjOTXDTG+KeHmW64dDOrNsZYeCHqXTbJsfTu8WiZ\nw3S/k+Q7p9vfmeQVqysH1krvwobacc+oql6c5EFJ7pjZMfYfS/LbSV6S5K5J3pfkW8YYHz3MtLYu\nWblFty71LpvGntHOjnqYbukZW6FZg71YofUu6yCMduYXGABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgn\njABoJ4wAaCeMAGgnjABoJ4wAaCeMAGgnjABoJ4wAaLery44fzab97P2mXRbgkE17nwD2mj0jANoJ\nIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkj\nANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA\n2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2h3oLmAvVVV3CewTegX2lj0jANoJIwDaCSMA2gkjANoJ\nIwDaCSMA2gkjANoJIwDaCSMA2gkjANoJIwDaCSMA2gkjANodNYyq6vyqurqq3jw37Ger6u1V9caq\nellVnbbeMuHY6FvYXxbZM7ogyXnbhr06yb3GGPdJ8q4kT151YbBL+hb2kaOG0RjjkiTXbRv2mjHG\nDdPd1yU5aw21wdL0Lewvqzhn9F1JXrWC+cBe0rewQXYVRlX1o0k+NcZ40YrqgbXTt7B5lr7seFU9\nMslDkzx4ZdXAmulb2ExLhVFVnZfkh5I8aIzxydWWBOuhb2Fz1Rhj5wdUvTjJg5LcMcnVSZ6a2beQ\nTkpy7fSwPx1jPGbbdDvPGJYwxqhFHrds307T6l1WbtHePV4dNYyWnrEVmjXYixVa77IOwmhnfoEB\ngHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiA\ndsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2\nwgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbC\nCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2wgiAdsII\ngHbCCIB2wgiAdsIIgHbCCIB2wgiAdsIIgHbCCIB2O4ZRVZ1fVVdX1ZsPM+4JVXVDVd1hfeXBcvQu\n7C9H2zO6IMl52wdW1dlJvjrJZesoClZA78I+smMYjTEuSXLdYUY9M8kT11IRrIDehf3lmM8ZVdU3\nJrl8jPGmNdQDa6N3YXMdOJYHV9UpSX4ks8McNw5eaUWwBnoXNtux7hndI8k5Sd5YVZcmOSvJX1TV\n5666MFgxvQsb7Jj2jMYYb05yp0P3p5X6i8YY1666MFglvQub7Whf7X5xkj9Jcm5VfaCqHrXtIWNt\nlcEu6F3YX2qM9ayTVWVlZ+XGGGs/z6N3WYe96N39zC8wANBOGAHQThgB0E4YAdBOGAHQThgB0E4Y\nAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBOGAHQThgB0E4YAdBubRfXA4BF2TMCoJ0wAqCdMAKg\nnTACoJ0wAqDd/weSD/YAh1pPuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8566d6190>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAJnCAYAAAC03nW3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHa5JREFUeJzt3XuUbGdd5+HvjxwQQiQJMgIhwUAwS0FlgAFdDpdWRlcU\nQbzPgAoMjjOiSxy5iJeRIyiON9DlMDqKBFAgYkQQlxdAEJiBiYrhIkQucpEkEG6HREEGMO/8sfdJ\nik6fc/pU/6q7+pznWavXqq5dteut2m/1p2rv7q4aYwQAutxorwcAwIlFWABoJSwAtBIWAFoJCwCt\nhAWAVsLCjlTV31bVffd6HMdSVX9RVY88wrJzq+raqlrq+TBf9447G+F163pWVT15yeserKrfnk/v\n6D6xfVX18Kp6zVGWH3Hu7aZjjXPTZa+bS8sw6diRMcaXjDFeverbWeYHblVdu/DtmL/W3U7G2XL/\nNj1uK1FVG1X1vr1c3zynHtY1hqPYL3Nv0VHHW1XvqarbH2m5sMD6qb0ewEliv/2w303HmoNHfeyE\nhR2ZX7l89Xz6YFW9oKqeXVXXzLvJ7rHpsk+oqrdU1Uer6plV9Tnzshu8TZ935ZxXVd+b5CFJHl9V\n/1hVL97m8DZP/jtV1SVVdXVVvaiqzty0/JFVdUVVXVlVj1kYx72q6nVVdWhe9qtVdeNN131AVf19\nVX2oqn6+qmq+7nlV9Yqq+vC87Heq6vSFdd+tqv5mfrwuSnLThWVnVtUfVdUH58frJVV1u4Xld6iq\nV83XfWmSW23xGBzpPt2oqn6sqt45X/+vF9a97R+4VfWgeXseqqpXVtUXLSz7rF2Eh991VtWpSf4k\nyVnz9rymqm47z5+Lq+qi+bzXV9WXLbm+22zzLox5XcfaTk9YeKzeUlUPvuFDUb9aVR+rqssOPycW\nHHHuVdXvVdX75+u+qqruvLDsAVV16Xy9f6iqJ2660aNd9/Oq6g/n616S5LxN1/2VeZ1Xz9v/3pse\nl5seaVsc+1Edw5evpb+SvDvJV8+nDyb55yQXZHrF85Qkr1u47HuSvCnJ7ZKcmeR/J3nyvOzhSV6z\nad3XJrnjfPrCJE/atPzpSZ6+zXH+RZLLk9w5yalJLk7y2/Oyc+fbem6SmyX5kiQfTHL/efndk9wr\n0wuxL0jy1iSP3jTOP09yRpJzkrwtySPnZecluX+SG2f6wf+qJE+bl90kyXuTPDrJKUm+JcmnDt/P\nJLdM8k2ZYnNakhck+YOF231dkl+c132fJNckec4279Pj5m3xhfP3X5rklse57c9P8k/z/TtlXuc7\nkhzYvP02b8Mk90vyvk3rOzjf/2+e1/eYJO9KcsqS67t3kkPbvC9H3E7z8m9Ncpv59LfP9/vWC3P3\n0wvb8duTfCzJGceaewvXv/l8209LcunCsvslucvCNvpAkm/c5nUvmr9uluQu8xhevbD8oZmehzdK\n8sNJ3p/kJsfYFge29Xju9Q8mX/v7KzcMy0sXlt05ySc2XfZ7F77/uiTvnE8/PMcOy5N3MM5XJnnK\nwvdfnOT/ZQrgufNtnb+w/OeSPOMI6/qhJC/cNM6vXfj++5K8/AjXfXCSv5lP3zfJFZuW/59sCujC\nsn+d5KPz6dvPP8xutrD8ublhLLe8T5ni98Adbvv/luSihe9r/uF1383bb/M2TLKRrcPy2k3ruzLJ\nv11mfTu8b9dtpyMsvzTJgxbm7ubteEmS7zzW3NtivWfM9/Nzj3C7v5zkqUdYdt11M8XgU5u2/89k\n03Ns0/U/muRLj7Et7r2dx8+uMLpdtXD6E5neTi/Os8UDrP+Q5KxdGdXWt3341emRlp+VJFV1/rxL\n6v1VdXWmJ+jnHWPdh69763l3wuXzdX974bpnJbli03rem3n/dlWdWlX/q6ZdiFdnehV9+ryb7axM\nr8b/edN1j3WfbzufPjvJ329x+eNx23mdSZIx/QR6X6Z3pMu6fNP6Ls8uzJFjbKdU1XfPu6QOVdWh\nTO8AF+fAVtvxtgvfbzn3quqUqvrv8262qzO9+BqZ52VVffm8i/GDVfWxJP/58O0e47r/KsmBLW53\n8T4/tqreOu9GO5Tk9Hz282GrbbF4n45IWNhtt990+sr59Mcz7SZIkmyxj7zjQOvm2/50kg8fZfnh\nHxa/lmn3153GGKcn+fHc8LlzpOs+Jcm/JPmS+brftXDd9+eGP4S/INff18dk2t10r/m698sUnZqv\ne+Z8fGGr6x5pXIcf7/cluVN25sr5NpNMBxky7Qo8fN8/kYVtmumH0uHxHWl7nrOwvhtlCuDhMS+z\nvu064naqqi9I8htJvj/T7sIzk/xtPvsA91bb8cqF74809x6S5EGZdlGenuQOuX4bJ8nzkrwoydlj\njDOS/PrCsqNd90NJPrPF7Wa+T/fJtOvy28YYZ8z36epN9+lo2+KohIXdVEkeVVW3q6pbZvoBfdG8\n7I1J7lJVd62qm2Z6K77oqiQ7+VuRSvKdVfXF8w/jJyX5vfmV2GE/UVU3q6q7ZNq98bvz+acl+cck\nn5gPTn/fFut/bFWdUVXnJPnBTdf9eJJrajo4/riF67wuyWeq6ger6sZV9c1J7rmw/LRMx6yunh+v\n6w7cjjHem+Svk/zUfN17J/mGLcZ1pPv0jCRPrqo71eTL5tv47AdtOqD+yi3Wm0zHfB5QVV9d0y8z\nPCbJJ5O8dl7+hiQPnV9ZX5Bp199hVyX5vKq6xaZ13qOqvqmqDmTa5fjJJP93B+vbrqNtp5tnCteH\nk9yoqh6R6R3Los9f2I7fluSLkvzxvOxoc++0TLvFPlpVN88UuM3jOjTG+FRV3StTTBaXbXndMca/\nJHlhkoPz9r9zkofl+gB/bqbwfLiqblJVP5nkeLbFUQkLnUZu+MpxbDr9vCQvzbQb5h1JfjpJxhhv\nz/SEe3mm/f+v2XTd30py53lXxAuTpKp+vap+7TjG9pwkz8p8kDJTABaXvyrJO+cx/MIY4+Xzssdm\nekJfk+mV60Vb3M8XJ3l9pn3vf5TkmfP5P5Xp4P/VSV6S5PcPX3eMcfjg6MOTfCTTQd/fX1jnL2c6\n8PrhTD+s/2TT7T4kyZdn2jf+k0mevcV9PtJ9emqmMLx0HttvZuE30hack+mXLG5g3mbfmeRXM71C\nfkCm4zafmS/y6CQPTHJoHusfLFz375I8P8m7avqNt8PvPl6c5Dvm+/TQJN88/5A83vXdpqruU1X/\nuNXYt3C07fTWJL+U6YXABzJFZfExGZl+4H7h/Dg8Ocm3jDEOLSw/0tx7TqbdZldkehf0unz2Nn5U\nkidV1TWZjmn97sKyY133BzLF5wOZ5uMzF5b96fz19ky/VPPP+exdZSPTO6UjbYujqs9+wQarU1Xv\nzvTbUq/Y67GwPVV1aaZfzjh0zAvv/LaemGl343et+rZYrQN7PQBgfY0x7raLN+cPQ08QdoUB62Kr\nXansQ3aFAdDKOxYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSzbVNMnHF5bVfc99qXZyvzv34/0\nDw1ZAfO2h7l7fISF3eQvq9mvzN3jICzsJv8Liv3K3D0OwgJAq7UMS1WdVlU/XVWXVNWHquqTVfWO\nqvrZqrrZHg/vxvOHH713Htcbq+o79mow84f0PL6q3lBVH58/ZvSvqur793BM51TVC6rq6vnrD6vq\nvL0az24xb4+PuXviWtd/m392kkcmuTjJ72T6pLONJI9PcrckF+zZyJKfy/TxqP8j09vjRyR5flXd\ndIyx+YOWVqqqbpLkzzJ9ZO2fZfrgn08m+bIk35Tk6bs5nnlMZyR5daZtePgjfTeSvCLTh1adyMzb\nbTJ3T3BjjLX7SnLjJKdscf6Tklyb5J57MKaHz7f97iSfu3D+LTJ9AttHktx0l8f0+HlMP73Fstqj\nbfeUeUwP23T+0+bzX7FX82oX7rt5u/1xmbsn8Nda7gobY3x6zB+BWVUHqurMqrpVkj+fL3KvvRtd\nfm2Mcd3HnY4xrkny60nOzPTqZjc9NNPHhj5p84IxPyP2wIMzfRTqczad/3N7MJZdZd4eF3P3BLaW\nYUmSqnpUVb0p09vjjyT5YJLDv0d+5p4NLLnsKOfdYTcHkukztv9uTJ+dvi7umOQdm384jDE+kOnz\nxE9o5u22mbsnsLU8xlJVP5zkFzPte/3lJFcm+VSmfZ/PyhoHkZOXeQuTtQxLku9K8u4xxtctnllV\ne3nw87A7J3nJFuclybt2eSxvS/LFVXWTNXrl964k51fVjcYY1x4+s6pum+T0vRvWrjBvt8/cPYGt\n6yuozyRJVV03vqo6kOQJezai631fVd3i8DdVdXqS/5LkUJJX7fJYnptp98pPbF5QVXv1B10vSnLr\nJN+96fwf2YOx7DbzdvvM3RPYur5juTjJzyb5k6r6g0y/wfKQTLsV9tqHklxSVRfm+l/bPDvJ94wx\nPrnLY/mVJA9M8hNVdc8kL8u0b/8uSc5P8jW7PJ4k+flM2+o3q+oeuf5XNr8iyYdzYv8Fs3m7febu\nCWxdw/ILmTbiIzPtq35/kt/NtJ/6rXs3rIxMr17um+T7M726eVuSh44xLtr1wYzx6ar62iSPyfSE\n+JlMT863J7lwt8czj+ljVXWfJE/N9a/8/iLJV2X67agT+f8tmbfbHZC5e0KrvfvNPgBOROt6jAWA\nfUpYAGglLAC0EhYAWgkLAK2EBYBWwgJAq5X9gWRV+QMZ2o0xVv7Xz+Yuq7Abc3ddnFTvWDo/yOaJ\nT3xi5wcMsWbMEVjeSRUWAFZPWABoJSxL2tjY2OshsObMEU5WK/snlOt4AHRd91Xv3cdP7D+7dfB+\n3eaKObL/OXgPAEsSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGi1dFiq6oKq+ruqekdV/UjnoGCV\nzF1YraX+8r6qTknytiT/LskVSf4qyX8YY1y2cJn1+tPl+Mv7E8FO/3p5u3N33eaKObL/+cv7Y7tX\nkneOMd4zxvh0kouSfGPfsGBlzF1YsWXDcrsk71v4/vL5PFh35i6s2LJhWa/9BLB95i6s2LIfTXxF\nknMWvj8n0ys/WHfbmrsHDx687vTGxoZ/gQ/HYdmD9wcyHQC9f5Irk/xlHLxfmgOz29dw8H5bc3fd\n5oo5sv+dTAfvl3rHMsb4TFX9QJI/S3JKkt9afGLCujJ3YfV80Nca8Gp0+3zQF/vVyfSOxV/eA9BK\nWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGg\nlbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsIC\nQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2E\nBYBWwgJAK2EBoJWwANBKWABoJSwAtDqwypWPMVa5+uNWVXs9BPaJdZsr6/ZcOmzdHifWg3csALQS\nFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAq6XDUlXnVNUrq+ot\nVfW3VfWDnQODVTBvYfVq2X/HXVW3SXKbMcYbquq0JK9P8uAxxmXz8rFu/+rbv/je/8YYO9qIx5q3\n82XWa+LGv80/Eex07u4nS79jGWN8YIzxhvn0PyW5LMlZXQODVTBvYfVajrFU1blJ7pbkko71wW4w\nb2E1dhyWeXfCxUkePb8ChLVn3sLq7Oijiavqxkl+P8nvjDFetHn5wYMHrzu9sbGRjY2NndwctDjW\nvAV2ZicH7yvJs5N8ZIzxX7dY7uA97RoO3h913s6XWa+JGwfvTwQn08H7nYTl3kleneRNSQ6v5EfH\nGH86LxcW2jWE5ajzdr7Mek3cCMuJQFg6ViwsrMBuPDmFZfs8p7bvZAqLv7wHoJWwANBKWABoJSwA\ntBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArXb0CZLHsm7/Utu/HgdYPe9Y\nAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCV\nsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJA\nK2EBoJWwANBKWABoJSwAtBIWAFoJCwCtDqxy5WOMVa7+uFXVXg+BfcLcheV5xwJAK2EBoJWwANBK\nWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALTaUViq6pSqurSqXtI1IFg18xZW\na6fvWB6d5K1J1ut/jMPRmbewQkuHparOTvL1SZ6RxIdFsC+Yt7B6O3nH8rQkj0tybdNYYDeYt7Bi\nS4Wlqr4hyQfHGJfGqz72CfMWdseyH038lUkeVFVfn+SmSW5RVc8ZY3z34oUOHjx43emNjY1sbGws\neXPQYlvzNjF3YSdqp5/tXVX3S/LYMcYDN50/fG443cYYLRvxSPN2Xmbu0q5r7u4HXX/Hsl7PQtge\n8xZWYMfvWI64Yq/6WIHdeNVn7rIK3rEAwJKEBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCt\nhAWAVsICQCthAaCVsADQSlgAaLXsJ0hui3/1vX+t27+NT8wn2C+8YwGglbAA0EpYAGglLAC0EhYA\nWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUs\nALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBK\nWABoJSwAtBIWAFodWOXKxxirXP1xq6q9HsK+4bECluUdCwCthAWAVsICQCthAaCVsADQSlgAaCUs\nALQSFgBaCQsArYQFgFbCAkArYQGglbAA0GrpsFTVGVV1cVVdVlVvraqv6BwYrIq5C6u1k3+b/ytJ\n/niM8a1VdSDJzZvGBKtm7sIK1TKfmVJVpye5dIxxx6NcZvg8FrqNMXa0Ec1d9spO5+5+suyusDsk\n+VBVXVhVf1NVv1lVp3YODFbE3IUVWzYsB5LcPcn/HGPcPcnHkzyhbVSwOuYurNiyx1guT3L5GOOv\n5u8vzhZPzoMHD153emNjIxsbG0veHLQxd2HFljrGkiRV9eok3zPGeHtVHUxyszHGjywst5+adh37\nqc1d9sLJdIxlJ2G5a5JnJLlJkr9P8ogxxtULyz05adcUFnOXXScsHSv25GQFduPJae6yCidTWPzl\nPQCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGi17CdI\nbot/9b1/rdu/jU92dz6Zu/vXyT5314F3LAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwA\ntBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpY\nAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVgf2egC7aYyx\n10PYUlXt9RBuYB3HtJvWba6c7NvjeHis9p53LAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABo\nJSwAtBIWAFoJCwCthAWAVsICQKulw1JVP1pVb6mqN1fV86rqczoHBqti7sJqLRWWqjo3yX9Kcvcx\nxpcmOSXJv+8bFqyGuQurt+znsVyT5NNJTq2qf0lyapIr2kYFq2Puwoot9Y5ljPHRJL+U5B+SXJnk\nY2OMl3cODFbB3IXVW3ZX2HlJfijJuUnOSnJaVT20cVywEuYurN6yu8L+TZLXjjE+kiRV9cIkX5nk\nuV0DgxXZ1tw9ePDgdac3NjaysbGxeyOEfa6W+WzvqrprpifiPZN8MsmzkvzlGOPpC5dZrw8Nz/p9\njvlhPqN7+8YYO3qwtjt3122umCP7307n7n6y7DGWNyZ5TpK/TvKm+ezf6BoUrIq5C6u31DuWba3Y\nO5Zt82p0+3bjVZ93LKyCdywAsCRhAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA\n0EpYAGglLAC0EhYAWi37CZL70rr+6/F1+xftyfo+Vrtl3e7/Os6RZP0eJ9aDdywAtBIWAFoJCwCt\nhAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYA\nWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUs\nALQSFgBaCQsArYQFgFbCAkCrA3s9AJKq2ushsObMEfYT71gAaCUsALQSFgBaCQsArYQFgFbCAkAr\nYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWRw1LVT2zqq6qqjcvnHfLqnpZVb29ql5aVWesfphw\nfMxd2DvHesdyYZILNp33hCQvG2Ocn+TP5+9h3Zi7sEeOGpYxxmuSHNp09oOSPHs+/ewkD17BuGBH\nzF3YO8scY7n1GOOq+fRVSW7dOB5YJXMXdsGODt6PMUaS0TQW2DXmLqzOMmG5qqpukyRVddskH+wd\nEqyMuQu7YJmw/GGSh82nH5bkRX3DgZUyd2EX1LRH4AgLq56f5H5JbpVpn/RPJnlxkhckuX2S9yT5\n9jHGx7a4rt0MtBtj1HYuZ+6ybrY7d08ERw3LjlbsyckK7MaT09xlFU6msPjLewBaCQsArYQFgFbC\nAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCt\nhAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYA\nWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUs\nALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBK\nWABoJSwAtBIWAFodMyxV9cyquqqq3rxw3i9U1WVV9caqemFVnb7aYcLxMW9h72znHcuFSS7YdN5L\nk9xljHHXJG9P8qPdA4MdMm9hjxwzLGOM1yQ5tOm8l40xrp2/vSTJ2SsYGyzNvIW903GM5T8m+eOG\n9cBuMm9hRXYUlqr68SSfGmM8r2k8sHLmLazWgWWvWFUPT/L1Se7fNhpYMfMWVm+psFTVBUkel+R+\nY4xP9g4JVsO8hd1RY4yjX6Dq+Unul+RWSa5K8sRMv01zkyQfnS/2ujHGozZd7+grhiWMMWo7l1t2\n3s7XNXdpt925eyI4ZliWXrEnJyuwG09Oc5dVOJnC4i/vAWglLAC0EhYAWgkLAK2EBYBWwgJAK2EB\noJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbC\nAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCt\nhAWAVsICQCthAaCVsADQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYA\nWgkLAK2EBYBWwgJAK2EBoJWwANBKWABoJSwAtBIWAFoJCwCthAWAVsICQCthAaCVsADQSlgAaHXU\nsFTVM6vqqqp68xbLHlNV11bVLVc3PFiOuQt751jvWC5McsHmM6vqnCRfk+S9qxgUNDB3YY8cNSxj\njNckObTFoqcmefxKRgQNzF3YO8d9jKWqvjHJ5WOMN61gPLAy5i7sjgPHc+GqOjXJj2XalXDd2a0j\nghUwd2H3HO87lvOSnJvkjVX17iRnJ3l9VX1+98CgmbkLu+S43rGMMd6c5NaHv5+foPcYY3y0e2DQ\nydyF3XOsXzd+fpLXJjm/qt5XVY/YdJGxspHBDpi7sHdqjNU8v6rKE5d2Y4yVHxcxd1mF3Zi768Jf\n3gPQSlgAaCUsALQSFgBaCQsArYQFgFbCAkArYQGglbAA0EpYAGglLAC0EhYAWgkLAK2EBYBWwgJA\nK2EBoNXKPugLgJOTdywAtBIWAFoJCwCthAWAVsICQKv/D5GiYBjVCp/oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8567984d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    plot_parallel_tapes(input_data[i], output_data[i],\n",
    "                        input_sequences[i], output_sequences[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 15, 4)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's generate some reference data for this task with fixed random seeds to make it possible to compare the results of different study groups on the same data.\n",
    "\n",
    "We create 2 datasets, one with only short sequences (from 5 to 15 steps in the input sequence) and one with long sequences (from 20 to 50 steps)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!rm -rf data/running_max_task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dataset_specs = [\n",
    "    ('short', dict(min_length=5, max_length=15, window_size=3)),\n",
    "    ('long', dict(min_length=20, max_length=50, window_size=10)),\n",
    "]\n",
    "     \n",
    "fold_specs = [\n",
    "    ('train', dict(n_sequences=5000, seed=0)),\n",
    "    ('validation', dict(n_sequences=1000, seed=1)),\n",
    "    ('test', dict(n_sequences=1000, seed=2)),\n",
    "]\n",
    "\n",
    "data_folder = 'data/running_max'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving symbol sequences data/running_max/short_sequence_train.json\n",
      "Saving data data/running_max/short_input_data_train.npy\n",
      "Saving data data/running_max/short_output_data_train.npy\n",
      "Saving symbol sequences data/running_max/short_sequence_validation.json\n",
      "Saving data data/running_max/short_input_data_validation.npy\n",
      "Saving data data/running_max/short_output_data_validation.npy\n",
      "Saving symbol sequences data/running_max/short_sequence_test.json\n",
      "Saving data data/running_max/short_input_data_test.npy\n",
      "Saving data data/running_max/short_output_data_test.npy\n",
      "Saving symbol sequences data/running_max/long_sequence_train.json\n",
      "Saving data data/running_max/long_input_data_train.npy\n",
      "Saving data data/running_max/long_output_data_train.npy\n",
      "Saving symbol sequences data/running_max/long_sequence_validation.json\n",
      "Saving data data/running_max/long_input_data_validation.npy\n",
      "Saving data data/running_max/long_output_data_validation.npy\n",
      "Saving symbol sequences data/running_max/long_sequence_test.json\n",
      "Saving data data/running_max/long_input_data_test.npy\n",
      "Saving data data/running_max/long_output_data_test.npy\n"
     ]
    }
   ],
   "source": [
    "import os.path as op\n",
    "import os\n",
    "from itertools import product\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def write_dataset(data_folder, generate_data, dataset_specs, fold_specs):\n",
    "    if not op.isdir(data_folder):\n",
    "        os.makedirs(data_folder)\n",
    "\n",
    "    for combined_specs in product(dataset_specs, fold_specs):\n",
    "        ds_name, ds_params = combined_specs[0]\n",
    "        fold_name, fold_params = combined_specs[1]\n",
    "        params = {}\n",
    "        params.update(ds_params)\n",
    "        params.update(fold_params)\n",
    "\n",
    "        (input_sequences, output_sequences,\n",
    "         input_data, output_data) = generate_data(**params)\n",
    "\n",
    "        df = pd.DataFrame({\n",
    "                'input': input_sequences,\n",
    "                'output': output_sequences,\n",
    "        })\n",
    "        fname = op.join(data_folder, '%s_sequence_%s.json'\n",
    "                        % (ds_name, fold_name))\n",
    "        print('Saving symbol sequences %s' % fname)\n",
    "        df.to_json(fname, orient='records')\n",
    "\n",
    "        for name, data in (('input', input_data), ('output', output_data)):\n",
    "            fname = op.join(data_folder, '%s_%s_data_%s.npy'\n",
    "                            % (ds_name, name, fold_name))\n",
    "            print('Saving data %s' % fname)\n",
    "            np.save(fname, data)\n",
    "\n",
    "\n",
    "\n",
    "write_dataset(data_folder, generate_running_max_task_data,\n",
    "              dataset_specs, fold_specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>addddbdbca</td>\n",
       "      <td>aadddddbba</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>caaacbcddcab</td>\n",
       "      <td>aaaaaacbddaa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bbabadadbc</td>\n",
       "      <td>aabbaaadab</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cdabd</td>\n",
       "      <td>aaaaa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dcdabbbd</td>\n",
       "      <td>aadaabbb</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          input        output\n",
       "0    addddbdbca    aadddddbba\n",
       "1  caaacbcddcab  aaaaaacbddaa\n",
       "2    bbabadadbc    aabbaaadab\n",
       "3         cdabd         aaaaa\n",
       "4      dcdabbbd      aadaabbb"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "short_symbols_train = pd.read_json(\n",
    "    'data/running_max/short_sequence_train.json')\n",
    "short_symbols_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_short_train = np.load('data/running_max/short_input_data_train.npy')\n",
    "Y_short_train = np.load('data/running_max/short_output_data_train.npy')\n",
    "\n",
    "X_short_validation = np.load('data/running_max/short_input_data_validation.npy')\n",
    "Y_short_validation = np.load('data/running_max/short_output_data_validation.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAJnCAYAAACnCTVDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHW1JREFUeJzt3Xu0rHdd3/HPNzmJEBASpXILGMDGGrQusaLLcokiNuAF\n7aq1QpUAXkHBK4JSEvF+12WtVjEREIgIiNpFFZQcwUopWG5CIIDhGgjkriDlkl//mGdzJjv7nLPP\n7JnvzNnn9Vprr7P3PDPP/s3s38x7nueZM1NjjABAl5PWPQAATizCA0Ar4QGglfAA0Ep4AGglPAC0\nEh6OqKr+vqoesO5xdKqq86vqFUdYfrCqHnOYZWdV1U1VteN9q6ourKpn7ea8O1z23Kp6z9zP76yq\nB+3msmyOZc2fwyw/7Jw4lvmz/bzLJjwc0Rjj88cYL1/176mq36+qn1zi+s6vqouXtb5txvS16GU3\nYRw7mh5wLl3mOg/ze474ALoB61vl7bDK+XMs6176/Ek+Fc8rjnQe4QGglfBwRNPm+FdO319YVc+r\nqmdU1Y3Tbrgv3nbeJ1XVm6rq2qq6qKo+bVp2i91X0y6Fe1XVdyZ5eJInVtU/VtWf7GJct9glsW0X\nxs2ezVXVH1XV+6vq+qr666o6Z27ZZ1bVn1bVDVX1qiT32va7HlxVb5ku+xtJavpKVZ1cVb9UVR+q\nqnck+Zptl73H9PturKqXJLnDDlfnMVX1vqq6sqp+aO6yt562BK+tqjcl+ZIdLnvfnW7v6fIPq6rX\nTdfr7VX176bTH1VVb57G9I7p9t+y62fBNfOU6e9+1TQvbjctu8Wumq1dO1V1XpInJ/nm6e/92mn5\nwar62ap61TTmF1XVGYuubxfjv6mq7jn38/at7vn58+tV9e5pXK+pqvvNLbtvVb2yqq6b/oa/UVWn\nzC1f2fypqm+tqndV1dVV9WPblu1p/kzrePI0tiuq6uFzp39NVb12uj3eXVUXbFvvkefQGMOXr8N+\nJbkiyVdO31+Y5J+TnDfdcX4mySvnzvvOJG9IctckZyT5myQ/OS07P8krtq37piT3nL6/OMnTti3/\nzSS/eZhxnTVd/qS50y5N8ujDnP/8JLdJckqSX03y2rlll0xft05y7yTvTfLyadkdktyY5N8nOTnJ\n9yf5+NbvSfLdSS6bu86XJvnk1riSvDLJL02/9/7Tup657To8e/rdn5/kg0keNC3/uSR/neT0JGcm\n+fsk797l7X3fJNfPresuST53+v6hSe4xff+AJB9O8kULzI1HJ3nbdD1uk+QFc9ft3CTvOcJcumDr\nvHPLD063/TlJTkvy/CTP2sP6npTkz44w/k/Nv8PNwbllj5hu45OS/GCS9yc5dVp2n+n2PinJZyd5\nc5InNMyfc5L8Y5L7JTk1yS9P6966TfYyf86d1rX1ux+Q5J+SnD0tf2CSe0/ff0GSDyR52K7nzroe\n0HwdH1+5ZXheMrfsnCQf2Xbe75z7+SFJ3j59f36OHp6fPIZxnZVjCM+2y54+XfbTpweDj23doabl\nP7011iTfluRvt13+PXMPHC/bdp0fvDWuJHef7ry3nlv+7Bx6MN26DvO/++eTPH36/h1Jvnpu2Xdk\n7sH3KLf3f0/yy7u8Lf84yeMXmBt/leS7534+e7otT8rRQ3Hh1u2w7e/3M3M/f16S/5fZk5xjXt8u\nxr9TeHY1B5Ncm+QLDrPs+5O8cIXzZys8T03ynLllp02319Ztspf5c+4Ov/sPkzzlMNf515L8ym5v\ne7vaOFZXzX3/kSS3qpu/Amd+d8i7M3umvVZVdVJV/dy0u+mGzO5wI7Nno/8iyYHcctxb7pLZs/B5\n8+e981Eue90Y45/nTnvXDkPcfvk7z13+cOs+3GW3bu8zM3vguYWqekhV/e+quqaqrstsC+gzdzrv\nUdw5N78+787strzjAuvasv36nJKdd0+2qqofnnZPXj/dZrffGldVnV1V/6Nmu3JvyOyJy9btucr5\nc7N1jzE+kuSabcsXnT85zO++S5JU1ZdW1aVV9cGquj7Jd+UY5pDwsGx33/b9ldP3H87sGVmSpKru\ntO1yx/rqmg9P/542d9r2dW55RJKvz2y30+2T3COH9rN/KMkndhj3liuT3G1u3DX/c2a7XA532fcn\nOaOq5sf42bnldT3cbXakdR/usu+bvn9Pks/ZfuZpH/4LkvxCks8aY5yR5MWZjjkcoysz22qb//2f\nyOzJyfa/98mZRX7L4f7e26/Px5NcvYf1HclHcvP5c+ed1lNV90/yI0m+aYxx+nSb3ZBDt9lvZbZ7\n7XOm+fXjOfTYuor5s2X7uk/LzR/8F5k/V879vNPv3ppfz0nyoiRnjjFOT/LbOYaeCA/LVEkeW1V3\nrarPyOwOeMm07PVJ7l1VX1hVt8ps18i8q5LcM7s0xvhQZneCb50O0D46214UMOe2me2CuLaqbpPZ\nsamt9XwyyQuTXDgdjD0nySNz6AHoxdO4v7GqDiR5fG4euOclefx0nc/I7LjC1rrfleQ1SX6iqk6Z\nDkh/7Q7je8r0u++d2S7JP5xb95Or6vSqOjPJ9227XCV53Lbbe+uyv5fkUVX1ldMW312r6nMzOxZw\namYP5jdV1UOSfPVhbretA/4XHGbxc5P8QM1e6HHbzG7XS8YYNyW5PLOt4YdOB9qfkmT+wPUHkpw1\nPRDPX5//XFWfNz3gPS3JH43ZvpxF1nc0r0vyiGn+nJfZcYydfHpmQb26qk6tqqcmud3c8ttmdqzl\nI1X1r5J8z9yyVc6fFyT52qr6t1V1ama310nb1n2s8+eSbefZ+t33z+yFD380d52vG2N8rKrum9mL\ng3Ydf+HhWIzccnKNbd8/J8lLMtvN87YkP5UkY4zLM7tj/GWStyZ5xbbL/l6Sc2r2yqAXJklV/XZV\n/dYRxvMdmT0TvTqz403/6zDne2Zmuwnel9kB1ldu+93fm9kd6QNJLpq+Mo376iTflNmB2qsz24r4\nm7nL/m6Sv8gsrK/J7MFgft0PT/KlmR0TeGqSZ2wb28jsAPDbM7ttfnGM8ZfTsp+Yxn1Fkj+frsf2\n2/vZ2fn2fnWSR2X2QorrMztwf/cxxj9m9uD3vGlM35LkSK8iPHPb9Z13UZJnJXl5kn/IbAvi+6bf\nf0OSxyZ5ema7g/4pN9+ts/UAdk1VvWbu+jwrye9nOng/jXWh9VXVj1XVi49w3Z6Q5OuSXJfZ3+mP\nD3O+P5++Ls/sgPw/5+a7rX54uvyNSX4nswfvMY17ZfNnjPGmJI/L7D535XSe+dtk4fkzLXv/dNtc\nmdnf5bum+3Ey+1s8rapuTPJfcugJz67UdGAI9qxm/2nsMWOMl617LOzd9Cz5kjHG/Y565uX8vksz\ne4HARUc9M8e1A+seALCZxhjvzeylup0WOdbEccauNmCT2AVzArCrDYBWtngAaCU8ALQSHgBaCQ8A\nrYQHgFbCA0Ar4QGglfDsUs0+QfOmqjrcGwlyFDX7xMhL1z2OE4l5uxzm7nIJD512epNROB6Yu0sk\nPHTyPlwcr8zdJRIeAFptZHiq6rZV9VNV9aqq+lBVfbSq3lZVP1tVt17z8E6pqgur6l3TuF5fVd+8\nrsFMH0z1xKp6XVV9ePpo3ldX1ePWOKa7VdXzquqG6etPq+pwH9K2b5i3x8bcPXFt6scinJnkMUme\nn+QPMvv0v3OTPDHJFyU5b20jS34+s4/L/a+ZbX4/Kslzq+pWY4ztH/K1UtOnDv5FkgdO/z4zyUeT\n/Osk35jkNzvHM43p9Mw+GOzMHPpI4HOTvCzJuh98V8283SVz9wQ3xti4rySnJDl5h9OfluSmJF+y\nhjGdP/3uK5J8+tzpt8vsUwmvSXKr5jE9cRrTT+2wrNb0t/uZaUyP3Hb6r06nv2xd86rhupu3ux+X\nuXsCf23krrYxxsfHGJ9Mkqo6UFVnVNUdkvzVdJb7rm90+a0x+/jgJMkY48Ykv53kjMyeHXV6RGYf\nd/u07QvGdI9Zg2/I7COkn7nt9J9fw1hambfHxNw9gW1keJKkqh5bVW/IbPP7miQfTLL1Ovoz1jaw\n5LIjnHaPzoEk+ZdJ3jLG+Fjz7z2SeyZ52/YHjzHGB5LcsJ4h9TFvd83cPYFt5DGeqvrBJL+U2b7f\nX0tyZZKPZbbv9fezwcHkxGXewu5sZHiSfGuSK8YYD5k/sarWeXB2yzlJ/myH05LkH5rH8tYkn1dV\np27QM8d/SHJ2VZ00xrhp68SqunOS269vWC3M290zd09gm/oM7BNJUlWfGl9VHUjypLWN6JDvqarb\nbf1QVbdP8t1Jrkvy181jeXZmu2+esn1BVa3rP7y9KMkdk3zbttN/dA1j6Wbe7p65ewLb1C2e5yf5\n2ST/s6r+OLNX4Dw8s90W6/ahJK+qqotz6GWpZyb59jHGR5vH8utJvi7JU6rqS5K8NLNjC/dOcnaS\nBzePJ0l+IbO/1e9W1Rfn0EtSvyzJ1dnf/wPcvN09c/cEtqnh+cXM/siPyWxf+fuT/GFm+8nfvL5h\nZWT27OcBSR6X2bOjtyZ5xBjjkvbBjPHxqvrqJD+U2R3mpzO7816e5OLu8Uxjur6q7p/kV3LomePB\nJF+R2au79vP7XZm3ux2QuXtCq/W9chGAE9GmHuMBYJ8SHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Gpl\n/4G0qvwHIZZujLHy/z1u7rIKHXP3eLHSLZ5lfWjQBRdcsJT1wG5t0rw1d9lv7GoDoJXwANDquAjP\nueeeu+4hwDEzb2FnK3uT0Kpa40en72x9H/PBsnS9uMDcZdm8uOCQ42KLB4D9Q3gAaCU8ALQSHgBa\nCQ8ArYQHgFbCA0Ar4QGg1cLhqarzquotVfW2qvrRZQ4KVsnchfVa6J0LqurkJG9N8lVJ3pfk1Um+\nZYxx2dx5/O9vlm6v//vb3GVdvHPBIYtu8dw3ydvHGO8cY3w8ySVJHra8YcHKmLuwZouG565J3jP3\n83un02DTmbuwZouGZ7P2Q8DumbuwZot+9PX7ktxt7ue7ZfbM8WYuvPDCT31/7rnnept4NoG5C2u2\n6IsLDmR2gPZBSa5M8n/iAC0NlvDiAnOXtfDigkMW2uIZY3yiqr43yV8kOTnJ783fcWFTmbuwfj4I\njuOKD4LjeGWL5xDvXABAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngA\naCU8ALRa9IPgdsU76h6/Nu3dmRPzCfYLWzwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8\nALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBK\neABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0CrA+se\nAGyiqlr3EFjQGGPdQ7gF8+nmbPEA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABo\nJTwAtBIeAFoJDwCtFg5PVd2tqi6tqjdV1d9X1eOXOTBYBfMW1q8WfQvxqrpTkjuNMV5XVbdN8ndJ\nvmGMcdm0fPPem5xd29S3lh9j7On95Y82b6fzbN6VZ9f269zdTxbe4hljfGCM8brp+39KclmSuyxr\nYLAK5i2s31KO8VTVWUm+KMmrlrE+6GDewnrsOTzT7ornJ3nC9AwSNp55C+uzp4++rqpTkrwgyR+M\nMV60nCHBzMGDB3Pw4MGlr9e8ZdVWNXf3i728uKCSPCPJNWOMH9hh+eYd4WPX9usB2qPN2+k8m3fl\n2bX9Onf3k72E535JXp7kDUm2VvLkMcafT8s376/Pru3XO+/R5u10ns278uzafp27+8nC4Tnqit15\nj2sn8p3X3D2+nchz93jhnQsAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4A\nWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8\nALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBK\neABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGg\nlfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABotafwVNXJVfXaqvqzZQ0IVs28\nhfXa6xbPE5K8OclYwligi3kLa7RweKrqzCQPTfL0JLW0EcEKmbewfnvZ4vnVJD+S5KYljQU6mLew\nZguFp6q+NskHxxivjWeNHCfMW9gMBxa83Jcn+fqqemiSWyW5XVU9c4zxbcsbGie6gwcP5uDBg8tc\npXlLixXM3X2lxtjb8dWqemCSHx5jfN220x24PY7tdV6sQlVljLGULZXDzdtp2eZdeXZtv8/d/WBZ\n/49n8/7ScHTmLazBnrd4DrtizxqPayfys0Zz9/h2Is/d44V3LgCglfAA0Ep4AGglPAC0Eh4AWgkP\nAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCtFv0E0uPSJr5dejJ7y/RNs4ljAvYHWzwA\ntBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4\nAGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV\n8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0CrA+seQKeqWvcQjhtjjHUP4Rb8/WB/sMUDQCvhAaCV\n8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Wjg8VXV6VT2/qi6rqjdX\n1Zctc2CwKuYurNdePhbh15O8eIzxH6rqQJLbLGlMsGrmLqxRLfK5K1V1+ySvHWPc8wjn2bwPdGHX\nNvXzeMYYe/pQHnN3/9uvc3c/WXRX2z2SfKiqLq6q/1tVv1tVpy1zYLAi5i6s2aLhOZDkPkn+2xjj\nPkk+nORJSxsVrI65C2u26DGe9yZ57xjj1dPPz487L0t28ODBHDx4cNmrNXdZuRXN3X1joWM8SVJV\nL0/y7WOMy6vqwiS3HmP86NzyzdvRyq7t5/3k5u7+tp/n7n6xl/B8YZKnJzk1yTuSPGqMccPc8s37\n67Nr+/nOa+7ub/t57u4XC4fnqCt25z2unch3XnP3+HYiz93jhXcuAKCV8ADQSngAaCU8ALQSHgBa\nCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK0W/QTS49Imvl16MnvL9E2ziWM6kZm7\nu7eJY+LmbPEA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCt\nhAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4A\nWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFodWPcAOlXVuoewozHGuodwC5t6WwHHP1s8\nALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAq4XDU1VPrqo3\nVdUbq+o5VfVpyxwYrIq5C+u1UHiq6qwk35HkPmOML0hycpL/tLxhwWqYu7B+i34ez41JPp7ktKr6\nZJLTkrxvaaOC1TF3Yc0W2uIZY1yb5JeTvDvJlUmuH2P85TIHBqtg7sL6Lbqr7V5Jvj/JWUnukuS2\nVfWIJY4LVsLchfVb9MUF/ybJ344xrhljfCLJC5N8+fKGBStj7sKaLRqetyT5sqq6dVVVkq9K8ubl\nDQtWxtyFNVv0GM/rkzwzyWuSvGE6+XeWNShYFXMX1q/GGKtZcdVqVrwPrepvsBezjYHNM8ZY+cA2\nce5u4hxJNneebKKOuXu88M4FALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkP\nAK2EB4BWwgNAK+EBoNWBVa58097KfVPfwn1Tx3UiM3dhdWzxANBKeABoJTwAtBIeAFoJDwCthAeA\nVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkP\nAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQS\nHgBaCQ8ArQ6scuVVtcrVw8qYu7A6tngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGgl\nPAC0Eh4AWgkPAK2EB4BWRwxPVV1UVVdV1RvnTvuMqnppVV1eVS+pqtNXP0w4NuYubK6jbfFcnOS8\nbac9KclLxxhnJ/mr6WfYNOYubKgjhmeM8Yok1207+euTPGP6/hlJvmEF44I9MXdhcy1yjOeOY4yr\npu+vSnLHJY4HVsnchQ2wpxcXjDFGkrGksUAbcxfWZ5HwXFVVd0qSqrpzkg8ud0iwMuYubIBFwvOn\nSR45ff/IJC9a3nBgpcxd2AA12+NwmIVVz03ywCR3yGyf+FOT/EmS5yW5e5J3JvmPY4zrd7is3Rgs\n3RijdnM+c5dNs9u5eyI4Ynj2tGJ3Xlag485r7rIKwnOIdy4AoJXwANBKeABoJTwAtBIeAFoJDwCt\nhAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4A\nWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8\nALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBK\neABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGg\n1VHDU1UXVdVVVfXGudN+saouq6rXV9ULq+r2qx0mHBvzFjbXbrZ4Lk5y3rbTXpLk3mOML0xyeZIn\nL3tgsEfmLWyoo4ZnjPGKJNdtO+2lY4ybph9fleTMFYwNFmbewuZaxjGeRyd58RLWA53MW1iTPYWn\nqn48ycfGGM9Z0nhg5cxbWK8Di16wqs5P8tAkD1raaGDFzFtYv4XCU1XnJfmRJA8cY3x0uUOC1TBv\nYTPUGOPIZ6h6bpIHJrlDkquSXJDZq4FOTXLtdLZXjjEeu+1yR14xLGCMUbs536LzdrqsucvS7Xbu\nngiOGp6FV+zOywp03HnNXVZBeA7xzgUAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQS\nHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABo\nJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA\n0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvh\nAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0OmJ4quqiqrqqqt64\nw7IfqqqbquozVjc8WIy5C5vraFs8Fyc5b/uJVXW3JA9O8q5VDAqWwNyFDXXE8IwxXpHkuh0W/UqS\nJ65kRLAE5i5srmM+xlNVD0vy3jHGG1YwHlgZcxc2w4FjOXNVnZbkxzLbVfGpk5c6IlgBcxc2x7Fu\n8dwryVlJXl9VVyQ5M8nfVdVnLXtgsGTmLmyIY9riGWO8Mckdt36e7sBfPMa4dtkDg2Uyd2FzHO3l\n1M9N8rdJzq6q91TVo7adZaxsZLAH5i5srhpjNfe/qnLHZunGGCs/LmPusgodc/d44Z0LAGglPAC0\nEh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQamUf\nBAcAO7HFA0Ar4QGglfAA0Ep4AGglPAC0+v9UsU6wHxjCHwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8563c8710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaoAAAJnCAYAAADGERcBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHuBJREFUeJzt3Xm4ZHdZJ/DvmzRhJwkysgUIi1EJmAEGxsdhuYpLwBGY\ncYcHCCA6wiiorMpjWlHcRWdEZxQJoCwiIsqMM4JChBkQF/ZNQQJkgRBISBRlWPKbP+o0Xbnpe7u6\nuure93Z/Ps9zn646vzrn/Oqc99T3LNV1aowRAOjqhN3uAABsR1AB0JqgAqA1QQVAa4IKgNYEFQCt\nCarjTFW9q6ruu9v9OJZU1fOr6pkrmM5GVV24zvkcbh4c+6rqQ1V1/yN4/dVVdYct2s6pqjesrneH\nJqiOM2OMu4wxXr/u+azqw3tueudU1Xmrmt6Kjelvz89nWm+PXOc8pvl8qKq+rvH0dmQ57JKdqtcj\nUlVXb9UmqGA16hiZz059gI2s9r2sY3o0IaiOM/N7nlW1v6peVlUvqKqrptOC99j02qdV1bur6vKq\nel5VXXdqu9Yh/3SK4I5V9b1JHprkKVX1j1X1Rwv06/Rp/BPmhp1fVY+Znl5jL7Cq7l1Vb6yqK6rq\nIwf2fqvqm6vqrVV15TT83E3z+f2q+mhVfaqq/qKq7jzXdrhxN8/zEXPNN6uqV0/L8fyquu3ceGdW\n1Wuq6pNV9bGqevo0/PrTnvvlVfXuJPfcNL+7VdVbpmm+NMn1NrU/uKreNvX3A1X1TdPwR1XVe6bx\n/mFaH5uX99Or6rKquqCqHrqpeaEP6aq6VVX98fS+3l9V3zPXdo0j6vlTjlX1O0lum+RVU308aW79\nP7aqLq6qS6rqR5ad3gJ936p+509xjWn4HavqtVX1iWmZ/W5VnTw33tOm5X/VtK08ZK5tV8ad2h87\nVwfvrqp/Pdd8rzrEdj2N9+Rp+V9UVY/eNM0vmdb5lVX15iR33NT+q9O2cWVV/U1V3Xuu7V5V9aZp\n+7mkqv5rVV1n8/I+pDGGv+PoL8kFSb5uerw/yb8kOTuzvdFnJXnT3Gs/lOQdSW6d5NQk/yfJM6e2\nc5K8YdO0r05yh+nxeUl+clP7c5I8Z4t+nT6Nf8LcsNclefQhXnu7JFcl+c4kJya5aZKzprb7JTlz\nenzXJB9L8uC5cc9JcsMk10ny7CRvnWvbctzDzPP5U9u9k5yU5FcOLJskN07y0SQ/NLXdKMm9praf\nTfIXSU5JclqSdyX5yNR2UpIPJ3nCNL9vTfLZA8s0yb2SfCrJ/afnt0ry5dPjBya5/fT4vkk+neRu\n0/ONJJ9L8ovTMrhvkn9KcsYStfT6JL829fWsJB9P8rWHWv/TfC88VB1uWv8vSnL9JHeZpnf/ZaY3\nDXt7ku/aou/nZJv63TT8jknuPy2vm03r7Nlz7d+W5BbT4++YlufNd3ncb09yUZJ7zE3rtgts12dn\nVvd3TnKDJC/ONbfrl05/109y5jSP18/16WHTNE9I8sOZ1f5JU9vdM6vbEzLbnt6T5AkL1dq6PhD9\n9fzLtYPq1XNtd07yz5te+71zzx+Q5APT42039Mw+WJ55BP06PYsH1dOT/MGC0/2VJL+8Rdsp0zxv\nfLhxt5tnZkH14rnnN0zy+czC57uT/O0W4/1Dkm+ce/7YTB++mQXIxZte/39zMKj+e5JfWnAZ/GGS\nH5web2QWVNefa/+9JM84wjq6zfQebzg37FlJzjvU+s/iQXXG3LCfS/LcZaa3QP+3rd/DjPuQJG/Z\npv2tSR60S+N+y/T4T5P8wBav2267fl6SZ821fdmB5ZLZDtNnN62jn968HDfN6/Ikd92i7YlJXrHI\n+nLqj0vnHv9zkuvV3Om3JPPfEPtIZnvuu+20JB88VENV/duqel1VfbyqPpXk+5J8ydR2YlX97HS6\n5MrMNtiR2d7qtuNm9sF8yHlO07joi0/G+HRmG+ittuvr1L55+c63Xbzp9R+ee3xaZkF3LVX1gKr6\ny+mU3BWZHWF9ydxLrhhj/Mum6R7per1Vksun9zrf/1sf4XQ2a1dvVXXzqnrpdCrsyiS/k7nlWVWP\nqNkp4yum5X2XHKy5nR73ZlPzlvUx2Wo53/IQbQf8qyT7tmnPdBr3PTU7tX5FkpNzcPs6o6r+R81O\nvV+ZWcjN1+WWBBWHc9tNjy+ZHn86s1MDSZKqusWm8cYRzufAB94N5oZtnuYBF2bTufE5L07yyiSn\njTFOSfLfcvAi+0OTPCiz00knJ7n91FYLjPuRbeaZzIIsSVJVN8rs1ODFU18P+dXezE6LbF6+822b\nP/RvN/f4wiR32jzB6VrDHyT5+SRfOsY4NcmfzL2PJDm1quaX8+1y7VA8nEuS3HR6r/P9PxDY16iP\nXHtdblUfm5fHgX4tO72tHK5+5z0ryReS3GWqm4dn+uysqtsl+c0kj09y02l5vysHl/dujXvI+piz\n1XLeriYvy+wo+pDtVXWfJE9O8u1jjFOmPl0516ffyOx0352m9/NjWTCDBBXbqSSPq6pbV9VNMyus\nl05tb09yZlWdVVXXy+w04rxLs/UH9LWMMS7LbGN5+HTk8+hsHQwvSvL1VfXtVbVvusB71tR2o8yO\nGD5bVffKLJwy1/b/klxeVTfM7IMgm9q3GvfF28yzkjywqv5dVZ2U5JmZXeu7OMn/THLLqnpCVV23\nqm48TTtJXpbk6VV1SlWdluQH5ub3piSfr6ofrKrrVNV/zDW/bPHbSR5VVV9XVSdM6+jLM7tedFKS\nTyS5uqoekOQbD7EMf2Ka7n2SfHOS39/8gjr4BYfbbm4bY1yY5I1JfmZ6X1+V5NFJfnd6ydumZXLq\nFAJP3DSJS3Po9fuMmn3J5MzMTs/93lFObyuHq995N8os2K6qqltn9mF8wA0zC8lPJDmhqh6V2ZHN\nbo/73CRPqqq718yd5tZjJXn8pu36wHJ+WZJzquorp52Zcw9McIzxhSSvSLJ/Wkd3TvLIHNxJuHFm\nQfaJqjqpqn48yU02vZ9/TPLPVfUVSb4/i1r0nK6/Y+Mv17xGdW6SF861nZ7ZHtwJc699apJ3J7ki\ns+sE15t7/Y9mtpf14cwuon4hB69R3Smzc+ZXZDoPndkRym9s07ezMztNdkVmF/sPeY1qeu29k/xl\nZntsH0ny8Gn4t2Z2sfiqJK9K8l8OvMfMNu5XTm0XZLaHOt/nLcc9zDzPS/LrSV49bYjnJ7nd3Hhn\nJvmzzE4HfjTJU6bh10/ygun9vivJkzJ9mWJqv0eSt0z9eWmSl+SaXyh4SGYfuFcleX+Sb5iGPy6z\nC+JXJHlhZiF74NrWxtT3A+vuQ0ketsUyvs+0Pk7cov3W03L6ZJIP5JrXPa479fnKzELmiZve24Om\nurkis4vup2d2LeR7Mtth+WiSJy07vWnYu5J89zb1tmX9bnrdnZP8zbRu3zL1d37ePzUtg8uS/FLm\n6na3xp3avy/J+6bx35GDX/453Hb91Gn5X5TkUbnmNnKzaZ1fmdm28JOZvkyR2YHPb09tl2QWrB/M\nwc+b+yR579Sf1yf5icx9EWO7v5omANdSVRckecwY47W73Rd2XlX9WJKPjzF+awfmdXpmH2r7xhhb\n/sdPjk/7drsDQE9jjJ/e7T5A4hoV0IfTOxySU38AtOaICoDWBBUArQkqAFoTVAC0JqgAaE1QAdCa\noAKgNUG1oJrdEfTqqrrvbvdlr6rZHYNft9v9OJ6o29VQu7tLULGTRvz6AHuT2t1FgoqdVId/CbSk\ndneRoAKgtZZBVVU3qqqfqqo3V9VlVfWZqnp/Vf1MVV1/l7t3naraX1Ufnvr19qr6zt3qzHSDsqdU\n1duq6tPTLaD/uqoev4t9uk1Vvayqrpz+/riqjuSmdnuSuj0yapdFdb3Nx2lJHpPk5ZndMfTzmd3w\n7SlJ7pbZDfZ2y89ldgvrX8vsdMCjkrykqq43xnjBTnZkupvsnya53/TvC5N8JslXJfkPSZ6zk/2Z\n+nRKZjdFOy0Hbz29keS1md0o8Fimbhekdjkiu3m32W3uvHmdHOKuopndTfLqJPfchT6dM837giQ3\nnht+k8zukvrJzN0lc4f69JSpTz91iLbapXX3rKlPj9w0/NnT8NfuVl3twHtXt4v3S+36W/iv5am/\nMcbnxhhfSJKq2ldVp1bVzZL8+fSSe+1e7/IbY4x/PPBkjHFVZrdYPzWzva+d9LDMbm/+k5sbxrSF\n7YKHZHYb9BduGv5zu9CXHaVuj4jaZWEtgypJqupxVfWOzE4HfDLJx5Mc+H8Mp+5ax5L3bjPs9jvZ\nkSRfluR9Y4zP7vB8t3OHJO/f/GEzxvhYkit3p0s7R90uTO2ysJbXqKrqh5P8Ymbnrn8lySVJPpvZ\nuePnp3HAcvxSt7AeLYMqycOTXDDGeMD8wKrazYvRB9w5yasOMSxJPrjDffm7JF9ZVSc12jP9YJIz\nquqEMcbVBwZW1S2TnLx73doR6nZxapeFdd3D+3ySVNUX+1dV+5I8bdd6dND3V9VNDjypqpOT/Kck\nVyT5ix3uy4syO530jM0NVbVb/0HxlUlunuQRm4Y/dRf6stPU7eLULgvrekT18iQ/k+R/VdUfZvYN\npYdmdhplt12W5M1VdV4Ofs33tCTfM8b4zA735VeTfEuSZ1TVPZO8JrNrI2cmOSPJN+xwf5Lk5zNb\nV79VVffIwa/4fnWST+TY/h/+6nZxapeFdQ2qX8isKB6T2bn+jyb5vczO879n97qVkdne1X2TPD6z\nva+/S/KwMcZLd7wzY3yuqr4xyY9ktoH9dGYb+98nOW+n+zP16VNVdZ8kv5yDe6bnJ/nazL79diz/\nXpq6XbRDapcjULv3TVAAOLyu16gAIImgAqA5QQVAa4IKgNYEFQCtCSoAWhNUALS2tv/wW1X+gxYr\nN8ZY+68DqF3WYSdq91jliKqBVd1c7Nxzz13ZtI531gf0IagAaE1QAdCaoDqGbGxs7HYXmGN9wGqs\n7UdpXZBeXMdrELt3S6Dt7dSXKbqtk67rg8X5MsXyHFEB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNU\nALQmqABobemgqqqzq+p9VfX+qnrqKjsF66R2YW9Z6pcpqurEJH+X5OuTXJzkr5N89xjjvXOv6fVf\n+xvr9isISd9fQjja/92/aO12Wydd1weL88sUy1v2iOpeST4wxvjQGONzSV6a5MGr6xasjdqFPWbZ\noLp1kgvnnl80DYPu1C7sMcsGVa/zIrA4tQt7zLK3or84yW3mnt8msz1T6G6h2t2/f/8XH29sbLhl\nB+yiZb9MsS+zC9L3T3JJkr+KL1MsrduF+6TvxfsVfJliodrttk66rg8W58sUy1vqiGqM8fmq+s9J\n/jTJiUl+e35Dh67ULuw9bpzYQLe996TvHrwbJ7JXOaJanl+mAKA1QQVAa4IKgNYEFQCtCSoAWhNU\nALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWlv2xokL8QvUi+naL4AOHFEB0JqgAqA1QQVA\na4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB\n0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNU\nALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYE\nFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa0sHVVXdpqpeV1Xvrqp3VdUPrrJjsA7qFvaeGmMs\nN2LVLZLcYozxtqq6UZK/TfKQMcZ7p/ax7LTXpap2uwscpTHGUa3Ew9Xt9Bq1y8odbe0ez5Y+ohpj\nfGyM8bbp8T8leW+SW62qY7AO6hb2npVco6qq05PcLcmbVzE92AnqFvaGow6q6fTJy5M8YdpDhfbU\nLewd+45m5Kq6TpI/SPK7Y4xXbm7fv3//Fx9vbGxkY2PjaGYHK3G4uk3ULnRyNF+mqCQvSPLJMcYP\nHaLdBWlWbgVfpti2bqfXqF1Wzpcplnc0QXXvJK9P8o4kByby9DHG/57abeys3AqCatu6nV6jdlk5\nQbW8pYPqsBO2sbMGO7Gxq13WQVAtzy9TANCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1Q\nAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoT\nVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDW\nBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKg\nNUEFQGv71jnxqlrn5I8ZY4zd7sK1WHdAF46oAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QA\ntCaoAGhNUAHQmqACoDVBBUBrggqA1o4qqKrqxKp6a1W9alUdgnVTt7C3HO0R1ROSvCdJv/tUwNbU\nLewhSwdVVZ2W5IFJnpvEzYvYE9Qt7D1Hc0T17CRPTnL1ivoCO0Hdwh6zVFBV1b9P8vExxltjr5Q9\nQt3C3rTsrei/JsmDquqBSa6X5CZV9cIxxiNW1zVYuYXrdv/+/V98vLGxkY2NjZ3qI7BJjXF015Or\n6n5JnjTG+JZNw12oXtDRroN1qOp5wDHGWEnHtqrbqW10Wydd1weLW1XtHo9W9f+oem3VsBh1C3vA\nUR9RbTlhR1QL67b3nvTdg9+JvVJHVKyDI6rl+WUKAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDW\nBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGvL3uF3T+p264YD3MIBYGuOqABoTVAB0JqgAqA1\nQQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABo\nTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoA\nWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNb27XYHdlJV7XYXDmmMsdtduJauy2qnHO/vf1Fql53g\niAqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGht6aCq\nqlOq6uVV9d6qek9VffUqOwbronZhbzma23z8apI/GWN8W1XtS3LDFfUJ1k3twh5Sy9xPpqpOTvLW\nMcYdtnlNvxvVNOWePosbYxxVx9TuaqndxR1t7R7Plj31d/skl1XVeVX1lqr6raq6wSo7BmuidmGP\nWTao9iW5e5JfH2PcPcmnkzxtZb2C9VG7sMcsG1QXJblojPHX0/OXZ7bxQ3dqF/aYpYJqjPGxJBdW\n1RnToK9P8u6V9QrWRO3C3rPUlymSpKrOSvLcJCcl+YckjxpjXDnX3u8qa1MuSC9uFRek1e7qqN3F\n+TLF8pYOqsNO2Ma+MBv74nZiY1e7i1O7ixNUy/PLFAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUA\nrQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDW9u12B+h5WwK3b+il4/pIju91ws5xRAVAa4IK\ngNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0Jqg\nAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQm\nqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWtu32x0ADq+qdrsLe8YYY7e7cC3W39FxRAVAa4IK\ngNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALS2dFBV1dOr6t1V\n9c6qenFVXXeVHYN1UbuwtywVVFV1epLHJrn7GOOuSU5M8l2r6xash9qFvWfZ+1FdleRzSW5QVV9I\ncoMkF6+sV7A+ahf2mKWOqMYYlyf5pSQfSXJJkk+NMf5slR2DdVC7sPcse+rvjkmemOT0JLdKcqOq\netgK+wVroXZh71n21N+/SfLGMcYnk6SqXpHka5K8aFUdg/PPPz/nn3/+qierdlm7NdXucavGGEc+\nUtVZmW3Y90zymSTPT/JXY4znzL3myCdMG8vUxbpVVcYYdZTTULvHuGO1do9ny16jenuSFyb5myTv\nmAb/5qo6BeuidmHvWeqIaqEJ2yvd047nvVK1u7cdz7V7rPLLFAC0JqgAaE1QAdCaoAKgNUEFQGuC\nCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWlr3DLyvU9bYEAB04ogKgNUEFQGuC\nCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCa\noAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0\nJqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFrbt86JjzHWOfkjVlW73YVD6tqv45na3bssq2OP\nIyoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqC1bYOq\nqp5XVZdW1Tvnht20ql5TVX9fVa+uqlPW3004MmoXjh2HO6I6L8nZm4Y9LclrxhhnJPnz6Tl0o3bh\nGLFtUI0x3pDkik2DH5TkBdPjFyR5yBr6BUdF7cKxY5lrVDcfY1w6Pb40yc1X2B9YJ7ULe9BR3eF3\njDGqastboe7fv/+Ljzc2NrKxsXE0s4OVUbuwd9ThbrldVacnedUY467T8/cl2RhjfKyqbpnkdWOM\nrzjEeMPtvFm1McbCK1Ht0smR1C7XtMypvz9O8sjp8SOTvHJ13YG1UruwB217RFVVL0lyvyQ3y+yc\n/o8n+aMkL0ty2yQfSvIdY4xPHWJce6Ws3KJ7pWqXbhxRLe+wp/6WnrCNnTXYiY1d7bIOgmp5fpkC\ngNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0NpR\n3Yr+cLrdmqDbrRsO6LacADpxRAVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IK\ngNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0Jqg\nAqA1QQVAa4IKgNYEFQCtCSoAWhNUALQmqABoTVAB0JqgAqA1QQVAa4IKgNYEFQCtCSoAWtu32x3Y\nSVW1211gj1Ar0IcjKgBaE1QAtCaoAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhN\nUAHQmqACoLXDBlVVPa+qLq2qd84N+4Wqem9Vvb2qXlFVJ6+3m3Bk1C0cOxY5ojovydmbhr06yZlj\njLOS/H2Sp6+6Y3CU1C0cIw4bVGOMNyS5YtOw14wxrp6evjnJaWvoGyxN3cKxYxXXqB6d5E9WMB3Y\nSeoW9oijCqqq+rEknx1jvHhF/YG1U7ewtyx9K/qqOifJA5Pcf2W9gTVTt7D3LBVUVXV2kicnud8Y\n4zOr7RKsh7qFvanGGNu/oOolSe6X5GZJLk1ybmbfljopyeXTy940xnjcpvG2nzAsYYxRi7xu2bqd\nxlW7rNyitcu1HTaolp6wjZ012ImNXe2yDoJqeX6ZAoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKg\nNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgA\naE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkq\nAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuC\nCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCaoAKgNUEFQGuCCoDWBBUArQkqAFoTVAC0JqgAaE1QAdCa\noAKgNUEFQGuCCoDWtg2qqnpeVV1aVe88RNuPVNXVVXXT9XUPlqN24dhxuCOq85KcvXlgVd0myTck\n+fA6OgUroHbhGLFtUI0x3pDkikM0/XKSp6ylR7ACaheOHUd8jaqqHpzkojHGO9bQH1gbtQt7074j\neXFV3SDJj2Z26uSLg1faI1gDtQt715EeUd0xyelJ3l5VFyQ5LcnfVtWXrrpjsGJqF/aoIzqiGmO8\nM8nNDzyfNvh7jDEuX3XHYJXULuxdh/t6+kuSvDHJGVV1YVU9atNLxtp6BkdB7cKxo8ZYz/ZaVT4I\nWLkxxtqvK6ld1mEnavdY5ZcpAGhNUAHQmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtCaoAGhNUAHQ\nmqACoDVBBUBrggqA1gQVAK0JKgBaE1QAtLa2GycCwCo4ogKgNUEFQGuCCoDWBBUArQkqAFr7/0xB\nbtN7YnNVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8566b3210>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAJnCAYAAACnCTVDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHIZJREFUeJzt3Xu07HdZ3/HPkxwCRBRCs+SWKJfKqoCyhGotKuxCdQUU\nRGu1SrmJt2IX2FIQlMoREMUbuizV5YUIakBERF3LKiAIVBBRIyBEboZLEgiXJKAoBcm3f8zv5Aw7\nO/vMmT3zzOxzXq+19jqz57a/s+eZ/Z75zZyZGmMEALqcsekFAHB6ER4AWgkPAK2EB4BWwgNAK+EB\noJXwsK+q+puquvem13EQVXX7qrq2qvac96o6WlW/tuR5P6KqXnOwFX7G+V1bVXdc8rTvrqr7TruX\nvkyszjpnb5VOtM5dx92pqvcd5OcJD/saY9xtjPHqdf+cqvrVqnraCs/vEVV14YJHP1X+M9u4gd0L\nO8nf29LWcH1vcn4O4lSZvetM18XD9zuO8EBSm17AGpyKl+lUdCpeTyeMqfCwrz0237ywqp5bVR+b\nNsPdc9dxn1hVb6mqq6rqOVV14+mw622Smh7a36mqvivJtyV5QlX9fVX97gLrut6mgar6k6p61PTt\nyPVvAI+qqsur6oqqetzc/iPJTarqBdPl+suq+uK5831iVb1zOuwtVfXg6y+nfq6qrqmqS479vqYD\nHllVb51O+67pss6f8PHTei6rqm/fddjXVtXFVfXRqnpvVT1l1+EPrar3VNWHq+oHdq3pRJfp/Kp6\ncVV9cDr9z+3ze9tTVd24qn5m+p1eXlXPqqqzpsNO+vre5vmpqt+qqvdP1/Grquouc4ftez1NVj57\n02V/xXT9faiqfr2qbr7gac+oqp+cTveuJF+76/ez79xOx3nSdPpLq+rbdh28/wyNMXz5usGvJJcm\nue+0+2iSf0pyQWb31J6R5HVzx313kjcluV2Sc5L83yRPmw57RJLX7Drva5Pccdp9YZKn7jr82Ume\nfQPruv10+jPm9ntlkm/f57i/keSmSe6W5INJ7jd3uT6Z5BuTnJnkcUn+LsmZ0+HflOTW0+5vTvIP\nSW41d7k+leSx02m/Ock1Sc6ZDn9AkjtMu++d5ONJvmT6/oIkH0hylyRnJ7lo1+/kPknuOu3+oum4\nXz99f5ckf5/kK5OcleSnpnXc90SXafp643Samya5cZKvWGI2nprktUnOnb7+9Nh1uOT1vZXzM/fz\nPyvJjZI8K8nFc4ftdz0d+znrmL07JbnftKZzk7wqybPm1rXfab8nySVzv+tXJvn0sd9H9p/bncxm\n7Senn33v6bzvvPDsbOoPmq/D8ZXrh+elc4fdJck/7jrud819f/8k75x2L/KH42knsa6F/3DMHffO\nc/s9M8kvz12u184dVkmuSPKVN/CzL07yoLnLdfmuw1+f5D/fwGl/J8ljpt3PSfKMucO+YP53ssdp\nfybJT0+7fyjJRXOHnZ3k/+26rva8TEn+bWZ//M7Y6+ecxHXwziQXzH3/NUkuXfb63tb52eO0t5hO\n+9kLXE9rm709Dntwkr/aZ90XJ3ngtPsVu37XX73797HP3O5kFp6bzh3+m0mevOjv36Y2TtaVc7v/\nMbPNBPNzNP9ql/cmuW3Lqhaz39ouO7ZjzG5JlyW5TZJU1cOmTSlXV9XVmd1r/Rdzp7181895z9xp\n719Vf1ZVH5lO+4C5095mjzVdp6r+TVW9ctocdk2S75477W13rfkfk3xk1zr2uky3TXJekveMMa7N\nwdw2s8s6v/6DXt9bNz/TZqkfmzZbfTSzQI7MHmWc6Ho6ZuWzV1W3mjbRXTat69fmf+4NnPbc6eAT\nzd5+c5skV48x/mnu+/fkJK4r4WHVPm/X7ium3R/P7F55kqSqbr3rdAs9rzDn49O/Z8/tt/s8T7S2\n+WCcP7e2MzL743xFVX1+kl9M8r1JbjnGOCfJ3+QznxS+3a6f8/nTaW+c5LeT/HiSz51O+wdzp33/\nHmuad1GSlyQ5b4xxiyS/MHfaK3at+exc/4/dXpfp8sz+4HxeVZ2Zg7kis3v08+s/6PW9jfPzkCQP\nymzz2M2T3CGz6+HYdbHX9bT7b+s6Zu8ZmW0eu9u0roce+7kLnPYGZ2+BuU2Sc6aZO+bzc/07YDdI\neFilSvLoqrpdVd0yyQ8mecF02BuT3LWq7l5VN8lsE8O8K5Ms/P9XxhgfymzQH1pVZ9bsifk7neBk\nT66qm1bVXTPbdPObc4fds6q+oaqOJPm+JJ9I8meZbdcfST6c5IyqemRm9xznfW5VPaaqblRV/zHJ\nv8rshnrW9PXhJNdW1f0z2xx1zAuTPKKqvnC6ET9l1/neLLN7lp+sqi/L7An0Y347yddV1VfU7An9\np+b6t+cbukxvyOwPz49V1dlVdZOqutdev7CaPeH/sL0OS/L8zH6n51bVuZlt/jv2f1KWub63dX5u\nltlmzKuq6rMy+4O/+/Dd19PuEK5j9m6WWUA/VlW3S/L4ucNOdNoXJnnM9Ls+J8kT5w470dwe88PT\nzH9VZi9O+K09jrMn4eFkjFz/BjV27b4oyUuTvCvJO5I8PUnGGG/P7I/jy5O8Lclrdp32V5LcZdos\n8OIkqapfqKqf32c935nZje3DmT3f9KcnWPurMnte4uVJfmKM8fK5w16S5FuSXJXZPdxvHGN8eozx\n1syehH9dZk8a3y2zJ73nz/fPMnt+5kNJnpbkP4wxrh5j/H2Sx2R2I78qybcmue4VV2OMP8zs+YBX\nJHl7kj/e9Tt5dJKnVtXHkvzPzP2xGmO8JbN7sxdl9qjgqnzmppP9LtOnkzwwyb/MbBPL+zJ78vkz\nTEG75XT59vL0JH+R2QsC3jTtXvr6zvbOz/My25R0eWaPGl6XBa+nucu1jtn74ST3SPLRJL+f2Z2R\nkSQLnPaXkvxRZkH/i12n3XduJ+9PcnVms/drSb57uo4WUtMTQ3BgVXVpkkeNMV6x6bVwcFX1FUke\nPcZ4SNPPMz+niSObXgCwncYYf5r9H0XCUmxqA6CVTW0AtPKIB4BWwgNAK+EBoJXwANBKeABoJTwA\ntBIeAFoJz4Jq9gmI11bVvTe9lsNqesPJV256HacTc7saZne1hIdOe73JKBwGZneFhIdOdeKjwFYy\nuyskPAC02srwVNXNqurpVfX6qvpQVX2iqt5RVT9aVTfd8PJuVFVHq+o907reWFXfsqnFVNVZVfWE\nqvrrqvp4VV1TVW+oqu/d4JrOr6oXVtVHp6/fq6oTfUjboWduT47ZPX1t68cinJfkUUlelOTXk/xz\nkp0kT0jyJUku2NjKkmdm9nG5/yuzh9+PTPL8qrrJGOO5nQuZPqjrj5LcZ/r3eZl9euEXJ/mGJM/u\nXM+0plskeXVm1+HPJ3lrZtfdK5Js+o/vupnbBZnd09wYY+u+ktwoyZl77P/UJNcm+dINrOkR08++\nNMlnz+3/OUneneQjSW7SvKYnTGt6+h6H1Yauu2dMa3r4rv2fNe3/ik3NVcNlN7eLr8vsnsZfW7mp\nbYzxqTH7eN5U1ZGqOmf6TPc/no7yZZtbXX5+zD4aNkkyxvhYkl9Ick5m9446PSSzj6Z96u4DxnSL\n2YAHZ/ZRu8/btf8zN7CWVub2pJjd09hWhidJqurRVfWmzB5+fyTJB5Mcex39ORtbWHLJPvvdoXMh\nSb4gyd+OMT7Z/HP3c8ck79j9x2OM8YHMPhv+lGZuF2Z2T2Nb+RxPVf33JD+Z2bbfn0lyRZJPZrbt\n9VezxcHk9GVuYTFbGZ4kD01y6Rjj/vN7VtUmn5w95i5Jfn+P/ZLk75rX8rYkX1hVZ23RPce/S3Ln\nqjpjjHHtsT2r6jZJbr65ZbUwt4szu6exbb0H9s9JUlXXra+qjiR54sZWdNx/qarPOfZNVd08yfck\nuTrJq5rX8huZbb558u4DqmpT/+HtJUluleRhu/b//g2spZu5XZzZPY1t6yOeFyX50ST/p6p+J7NX\n4HxbZpstNu1DSV5fVRfm+MtSz0vyHWOMTzSv5WeTPDDJk6vqS5O8LLPnFu6a5M5Jvrp5PUny45ld\nV79UVffM8ZekfnmSD+fU/h/g5nZxZvc0tq3h+YnMruRHZbat/P1JfjOz7eRv3dyyMjK793PvJN+b\n2b2jtyV5yBjjBe2LGeNTVfU1SR6X2Q3mRzK78b49yYXd65nWdE1VfVWSn87xe45/kuTfZfbqrlP5\n/a7M7aILMruntdrcKxcBOB1t63M8AJyihAeAVsIDQCvhAaCV8ADQSngAaCU8ALRa238grSr/QYiV\nG2Os/X+Pm13WoWN2D4vT6hHPKj/I6ClPecrKzovtY0ZgfU6r8ACwecIDQCvhWdLOzs6ml8CWMyOw\nt7W9Seg2PkG7rdvKN/fxI4dP14sLtm1WzMjh58UFx3nEA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkP\nAK2EB4BWS4enqi6oqr+tqndU1fevclGwTmYXNmupdy6oqjOTvC3Jv09yeZI3JPnWMcYlc8fZrv/6\nHe9ccCo46P/+XnR2t21WzMjh550Ljlv2Ec+XJXnnGOPdY4xPJXlBkq9f3bJgbcwubNiy4bldkvfN\nfX/ZtB9sO7MLG7ZseLZrOwQszuzChi370deXJzl/7vvzM7vnCNtuodk9evTodbt3dnZ8xAGs0LIv\nLjiS2RO090tyRZI/jxcXLM0Tx4tbwYsLFprdbZsVM3L4eXHBcUs94hlj/HNV/dckf5TkzCS/Mn/D\nhW1ldmHzfBDcFnBvdnE+CI7DyiOe47xzAQCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8A\nrYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIe\nAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGgl\nPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQKsj6zzzMcY6\nz/6kVdWml8AhsW2zsm23pWO27ffE4eARDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8A\nrYQHgFbCA0Ar4QGglfAA0Grp8FTV+VX1yqp6S1X9TVU9ZpULg3Uwt7B5tezbrVfVrZPceozx11V1\nsyR/meTBY4xLpsPHtr2Vu7dwP/zGGAe6Ek80t9Nxtmtw42MRTgUHnd1TydKPeMYYHxhj/PW0+x+S\nXJLktqtaGKyDuYXNW8lzPFV1+yRfkuT1qzg/6GBuYTMOHJ5pc8WLkjx2ugcJW8/cwuYc6KOvq+pG\nSX47ya+PMV6y+/CjR49et3tnZyc7OzsH+XGwEieaW2C9DvLigkry3CQfGWP8tz0O9+ICVm4FLy7Y\nd26n42zX4MaLC04FXlxw3EHC85VJXp3kTUmOncmTxhh/OB0uPKzcCsKz79xOx9muwY3wnAqE57il\nw3PCMxYe1qDjxis8i3ObWpzwHOedCwBoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8\nALQSHgBaCQ8ArYQHgFbCA0CrA30C6Yls21ume2t5gM3ziAeAVsIDQCvhAaCV8ADQSngAaCU8ALQS\nHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABo\nJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA\n0OrIOs98jLHOsz9pVbXpJXBImF1YH494AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABo\nJTwAtBIeAFoJDwCthAeAVgcKT1WdWVUXV9Xvr2pBsG7mFjbroI94HpvkrUm26z3kYX/mFjZo6fBU\n1XlJHpDkl5P4sBAOBXMLm3eQRzzPSvL4JNeuaC3QwdzChi0Vnqr6uiQfHGNcHPcaOSTMLWyHZT/6\n+l5JHlRVD0hykySfU1XPG2M8bP5IR48evW73zs5OdnZ2lvxxsBILzW1idmGd6qCfLV9V90nyP8YY\nD9y1//C59azaGGMlV+INze10mNll5VY1u6eCVf0/nu26lcJizC1swIEf8dzgGbvXyBp03Gs0u6yD\nRzzHeecCAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA\n0GrZTyBdiLdyP7y27WMBEvMEpwqPeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8\nALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBK\neABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFZH1nnmY4x1\nnv1Jq6pNL+HQ8LsC1sUjHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BW\nwgNAK+EBoNXS4amqW1TVi6rqkqp6a1V9+SoXButidmGzDvKxCD+b5A/GGN9UVUeSfNaK1gTrZnZh\ng2qZz8ypqpsnuXiMccd9jjN8Hg+rNsY40JVodtmUg87uqWTZTW13SPKhqrqwqv6qqn6pqs5e5cJg\nTcwubNiy4TmS5B5J/vcY4x5JPp7kiStbFayP2YUNW/Y5nsuSXDbGeMP0/Yuyx4336NGj1+3e2dnJ\nzs7Okj8OVsbswoYt9RxPklTVq5N8xxjj7VV1NMlNxxjfP3e47eSs3Cq2k5tdNsFzPMcdJDx3T/LL\nSc5K8q4kjxxjfHTucDdeVm5F4TG7tBOe45YOzwnP2I2XNei48Zpd1kF4jvPOBQC0Eh4AWgkPAK2E\nB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaDVsp9AuhBv5X54bdvHAiS9\n82R2D6/TfXYPA494AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCt\nhAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4A\nWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCtjmx6AZ3GGJtewp6qatNLuJ5tXFOn\nbZuV0/36OBl+V9vPIx4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsID\nQCvhAaDV0uGpqidV1Vuq6s1VdVFV3XiVC4N1MbuwWUuFp6pun+Q7k9xjjPFFSc5M8p9WtyxYD7ML\nm7fs5/F8LMmnkpxdVZ9OcnaSy1e2KlgfswsbttQjnjHGVUl+Ksl7k1yR5JoxxstXuTBYB7MLm7fs\nprY7Jfm+JLdPctskN6uqh6xwXbAWZhc2b9lNbf86yWvHGB9Jkqp6cZJ7JfmNVS0M1mSh2T169Oh1\nu3d2drKzs9O3QjjF1TKfLV9Vd8/shvqlST6R5FeT/PkY49lzx9muD61Pssxl7eAz4hc3xjjQL2vR\n2d22WTEjh99BZ/dUsuxzPG9M8rwkf5HkTdPev7iqRcG6mF3YvKUe8Sx0xh7xLMy92cV13Gv0iId1\n8IjnOO9cAEAr4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIe\nAFot+wmkh9K2vrX8tr0Ff7K9v6su23b5t3FGku37PXE4eMQDQCvhAaCV8ADQSngAaCU8ALQSHgBa\nCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwA\ntBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4\nAGglPAC0OrLpBZBU1aaXwJYzI5xKPOIBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV\n8ADQSngAaCU8ALQSHgBa7RueqnpOVV1ZVW+e2++WVfWyqnp7Vb20qm6x/mXCyTG7sL1O9IjnwiQX\n7NrviUleNsa4c5I/nr6HbWN2YUvtG54xxmuSXL1r7wclee60+7lJHryGdcGBmF3YXss8x3OrMcaV\n0+4rk9xqheuBdTK7sAUO9OKCMcZIMla0FmhjdmFzlgnPlVV16ySpqtsk+eBqlwRrY3ZhCywTnt9L\n8vBp98OTvGR1y4G1MruwBWq2xeEGDqx6fpL7JDk3s23iP5Tkd5O8MMnnJXl3km8eY1yzx2ltxmDl\nxhi1yPHMLttm0dk9HewbngOdsRsva9Bx4zW7rIPwHOedCwBoJTwAtBIeAFoJDwCthAeAVsIDQCvh\nAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BW\nwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8A\nrYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIe\nAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGh1\nwvBU1XOq6sqqevPcfj9RVZdU1Rur6sVVdfP1LhNOjrmF7bXII54Lk1ywa7+XJrnrGOPuSd6e5Emr\nXhgckLmFLXXC8IwxXpPk6l37vWyMce307euTnLeGtcHSzC1sr1U8x/PtSf5gBecDncwtbMiBwlNV\nP5jkk2OMi1a0Hlg7cwubdWTZE1bVI5I8IMn9VrYaWDNzC5u3VHiq6oIkj09ynzHGJ1a7JFgPcwvb\nocYY+x+h6vlJ7pPk3CRXJnlKZq8GOivJVdPRXjfGePSu0+1/xrCEMUYtcrxl53Y6rdll5Rad3dPB\nCcOz9Bm78bIGHTdes8s6CM9x3rkAgFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNA\nK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQH\ngFbCA0Ar4QGglfAA0Ep4AGglPAC0Eh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJ\nDwCthAeAVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0\nEh4AWgkPAK2EB4BWwgNAK+EBoJXwANBKeABoJTwAtBIeAFoJDwCthAeAVvuGp6qeU1VXVtWb9zjs\ncVV1bVXdcn3Lg+WYXdheJ3rEc2GSC3bvWVXnJ/nqJO9Zx6JgBcwubKl9wzPGeE2Sq/c46KeTPGEt\nK4IVMLuwvU76OZ6q+vokl40x3rSG9cDamF3YDkdO5shVdXaSH8hsU8V1e690RbAGZhe2x8k+4rlT\nktsneWNVXZrkvCR/WVWfu+qFwYqZXdgSJ/WIZ4zx5iS3Ovb9dAO+5xjjqlUvDFbJ7ML2ONHLqZ+f\n5LVJ7lxV76uqR+46yljbyuAAzC5srxpjPbe/qnLDZuXGGGt/Xsbssg4ds3tYeOcCAFoJDwCthAeA\nVsIDQCvhAaCV8ADQSngAaCU8ALQSHgBaCQ8ArYQHgFbCA0Ar4QGglfAA0Ep4AGglPAC0WtsHwQHA\nXjziAaCV8ADQSngAaCU8ALQSHgBa/X9/R5gHcK8xugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8567867d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    plot_parallel_tapes(X_short_train[i], Y_short_train[i],\n",
    "                        short_symbols_train['input'][i],\n",
    "                        short_symbols_train['output'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training a simple RNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 15, 4)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_short_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GRID K520\n",
      "/home/ubuntu/venv/src/theano/theano/scan_module/scan_perform_ext.py:133: RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility\n",
      "  from scan_perform.scan_perform import *\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading init.py\n",
      "loaded rnn.py\n"
     ]
    }
   ],
   "source": [
    "from rnn import RnnMiniBatch\n",
    "\n",
    "n_features = X_short_train.shape[2]\n",
    "n_ouputs = Y_short_train.shape[2]\n",
    "batch_size = 256\n",
    "\n",
    "model = RnnMiniBatch(n_features, 10, n_ouputs, lr=.5,\n",
    "                     batch_size=batch_size, single_output=False,\n",
    "                     cost_function='cxe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, train loss: 0.442942, validation loss: 0.429879\n",
      "epoch 2, train loss: 0.423195, validation loss: 0.422179\n",
      "epoch 3, train loss: 0.417154, validation loss: 0.416707\n",
      "epoch 4, train loss: 0.408709, validation loss: 0.403902\n",
      "epoch 5, train loss: 0.389882, validation loss: 0.379796\n",
      "epoch 6, train loss: 0.364326, validation loss: 0.356091\n",
      "epoch 7, train loss: 0.343569, validation loss: 0.340059\n",
      "epoch 8, train loss: 0.329770, validation loss: 0.328248\n",
      "epoch 9, train loss: 0.318630, validation loss: 0.317614\n",
      "epoch 10, train loss: 0.308448, validation loss: 0.308044\n",
      "epoch 11, train loss: 0.299857, validation loss: 0.300452\n",
      "epoch 12, train loss: 0.293134, validation loss: 0.294360\n",
      "epoch 13, train loss: 0.287559, validation loss: 0.289099\n",
      "epoch 14, train loss: 0.282649, validation loss: 0.284348\n",
      "epoch 15, train loss: 0.278163, validation loss: 0.279920\n",
      "epoch 16, train loss: 0.273954, validation loss: 0.275702\n",
      "epoch 17, train loss: 0.269935, validation loss: 0.271641\n",
      "epoch 18, train loss: 0.266058, validation loss: 0.267703\n",
      "epoch 19, train loss: 0.262283, validation loss: 0.263842\n",
      "epoch 20, train loss: 0.258532, validation loss: 0.259946\n",
      "epoch 21, train loss: 0.254651, validation loss: 0.255809\n",
      "epoch 22, train loss: 0.250412, validation loss: 0.251137\n",
      "epoch 23, train loss: 0.245549, validation loss: 0.245572\n",
      "epoch 24, train loss: 0.239925, validation loss: 0.238849\n",
      "epoch 25, train loss: 0.233926, validation loss: 0.231495\n",
      "epoch 26, train loss: 0.228380, validation loss: 0.225695\n",
      "epoch 27, train loss: 0.222520, validation loss: 0.221567\n",
      "epoch 28, train loss: 0.221785, validation loss: 0.229555\n",
      "epoch 29, train loss: 0.217984, validation loss: 0.213340\n",
      "epoch 30, train loss: 0.213146, validation loss: 0.212245\n",
      "epoch 31, train loss: 0.212146, validation loss: 0.220545\n",
      "epoch 32, train loss: 0.209476, validation loss: 0.204569\n",
      "epoch 33, train loss: 0.204283, validation loss: 0.205200\n",
      "epoch 34, train loss: 0.205859, validation loss: 0.211697\n",
      "epoch 35, train loss: 0.202107, validation loss: 0.197473\n",
      "epoch 36, train loss: 0.196261, validation loss: 0.195495\n",
      "epoch 37, train loss: 0.201064, validation loss: 0.205571\n",
      "epoch 38, train loss: 0.196261, validation loss: 0.192417\n",
      "epoch 39, train loss: 0.191423, validation loss: 0.190531\n",
      "epoch 40, train loss: 0.195954, validation loss: 0.195649\n",
      "epoch 41, train loss: 0.191899, validation loss: 0.202017\n",
      "epoch 42, train loss: 0.193328, validation loss: 0.187407\n",
      "epoch 43, train loss: 0.187970, validation loss: 0.191398\n",
      "epoch 44, train loss: 0.192172, validation loss: 0.195589\n",
      "epoch 45, train loss: 0.188532, validation loss: 0.184422\n",
      "epoch 46, train loss: 0.183820, validation loss: 0.183377\n",
      "epoch 47, train loss: 0.191031, validation loss: 0.187517\n",
      "epoch 48, train loss: 0.184197, validation loss: 0.181966\n",
      "epoch 49, train loss: 0.185614, validation loss: 0.200067\n",
      "epoch 50, train loss: 0.183803, validation loss: 0.179595\n",
      "epoch 51, train loss: 0.180810, validation loss: 0.188325\n",
      "epoch 52, train loss: 0.183632, validation loss: 0.187173\n",
      "epoch 53, train loss: 0.181975, validation loss: 0.177116\n",
      "epoch 54, train loss: 0.177557, validation loss: 0.200677\n",
      "epoch 55, train loss: 0.182473, validation loss: 0.174965\n",
      "epoch 56, train loss: 0.179248, validation loss: 0.178677\n",
      "epoch 57, train loss: 0.175653, validation loss: 0.187442\n",
      "epoch 58, train loss: 0.178138, validation loss: 0.172824\n",
      "epoch 59, train loss: 0.176511, validation loss: 0.180254\n",
      "epoch 60, train loss: 0.174267, validation loss: 0.172823\n",
      "epoch 61, train loss: 0.174476, validation loss: 0.175460\n",
      "epoch 62, train loss: 0.173068, validation loss: 0.170983\n",
      "epoch 63, train loss: 0.172426, validation loss: 0.173705\n",
      "epoch 64, train loss: 0.171448, validation loss: 0.169321\n",
      "epoch 65, train loss: 0.170580, validation loss: 0.171842\n",
      "epoch 66, train loss: 0.169876, validation loss: 0.167637\n",
      "epoch 67, train loss: 0.168805, validation loss: 0.170150\n",
      "epoch 68, train loss: 0.168303, validation loss: 0.165968\n",
      "epoch 69, train loss: 0.167086, validation loss: 0.168587\n",
      "epoch 70, train loss: 0.166727, validation loss: 0.164306\n",
      "epoch 71, train loss: 0.165403, validation loss: 0.167090\n",
      "epoch 72, train loss: 0.165142, validation loss: 0.162649\n",
      "epoch 73, train loss: 0.163747, validation loss: 0.165606\n",
      "epoch 74, train loss: 0.163552, validation loss: 0.161001\n",
      "epoch 75, train loss: 0.162114, validation loss: 0.164104\n",
      "epoch 76, train loss: 0.161963, validation loss: 0.159374\n",
      "epoch 77, train loss: 0.160506, validation loss: 0.162579\n",
      "epoch 78, train loss: 0.160390, validation loss: 0.157786\n",
      "epoch 79, train loss: 0.158937, validation loss: 0.161045\n",
      "epoch 80, train loss: 0.158846, validation loss: 0.156252\n",
      "epoch 81, train loss: 0.157418, validation loss: 0.159526\n",
      "epoch 82, train loss: 0.157347, validation loss: 0.154783\n",
      "epoch 83, train loss: 0.155961, validation loss: 0.158046\n",
      "epoch 84, train loss: 0.155900, validation loss: 0.153382\n",
      "epoch 85, train loss: 0.154572, validation loss: 0.156617\n",
      "epoch 86, train loss: 0.154511, validation loss: 0.152050\n",
      "epoch 87, train loss: 0.153254, validation loss: 0.155217\n",
      "epoch 88, train loss: 0.153179, validation loss: 0.150779\n",
      "epoch 89, train loss: 0.152005, validation loss: 0.153766\n",
      "epoch 90, train loss: 0.151902, validation loss: 0.149565\n",
      "epoch 91, train loss: 0.150825, validation loss: 0.152145\n",
      "epoch 92, train loss: 0.150693, validation loss: 0.148419\n",
      "epoch 93, train loss: 0.149690, validation loss: 0.150382\n",
      "epoch 94, train loss: 0.149576, validation loss: 0.147457\n",
      "epoch 95, train loss: 0.148586, validation loss: 0.148740\n",
      "epoch 96, train loss: 0.148471, validation loss: 0.146451\n",
      "epoch 97, train loss: 0.147553, validation loss: 0.147176\n",
      "epoch 98, train loss: 0.147294, validation loss: 0.145382\n",
      "epoch 99, train loss: 0.146589, validation loss: 0.145666\n",
      "epoch 100, train loss: 0.146242, validation loss: 0.144234\n",
      "epoch 101, train loss: 0.145601, validation loss: 0.144680\n",
      "epoch 102, train loss: 0.145241, validation loss: 0.143233\n",
      "epoch 103, train loss: 0.144576, validation loss: 0.143372\n",
      "epoch 104, train loss: 0.144523, validation loss: 0.144457\n",
      "epoch 105, train loss: 0.143888, validation loss: 0.142099\n",
      "epoch 106, train loss: 0.143665, validation loss: 0.143671\n",
      "epoch 107, train loss: 0.142942, validation loss: 0.141184\n",
      "epoch 108, train loss: 0.142700, validation loss: 0.142349\n",
      "epoch 109, train loss: 0.142168, validation loss: 0.142936\n",
      "epoch 110, train loss: 0.142319, validation loss: 0.140693\n",
      "epoch 111, train loss: 0.141126, validation loss: 0.139690\n",
      "epoch 112, train loss: 0.141432, validation loss: 0.140531\n",
      "epoch 113, train loss: 0.140737, validation loss: 0.139758\n",
      "epoch 114, train loss: 0.140389, validation loss: 0.138774\n",
      "epoch 115, train loss: 0.140204, validation loss: 0.137980\n",
      "epoch 116, train loss: 0.140508, validation loss: 0.137789\n",
      "epoch 117, train loss: 0.139124, validation loss: 0.138129\n",
      "epoch 118, train loss: 0.138474, validation loss: 0.136711\n",
      "epoch 119, train loss: 0.139128, validation loss: 0.136578\n",
      "epoch 120, train loss: 0.138635, validation loss: 0.136093\n",
      "epoch 121, train loss: 0.137463, validation loss: 0.136221\n",
      "epoch 122, train loss: 0.139463, validation loss: 0.137283\n",
      "epoch 123, train loss: 0.137812, validation loss: 0.135010\n",
      "epoch 124, train loss: 0.136756, validation loss: 0.139441\n",
      "epoch 125, train loss: 0.138164, validation loss: 0.137053\n",
      "epoch 126, train loss: 0.136633, validation loss: 0.136287\n",
      "epoch 127, train loss: 0.138346, validation loss: 0.137319\n",
      "Validation score stopped improving\n"
     ]
    }
   ],
   "source": [
    "epochs = 500\n",
    "original_patience = 3\n",
    "tolerance = 1e-4\n",
    "best_val_loss = None\n",
    "patience = original_patience\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    train_loss, val_loss = 0, 0\n",
    "    num_batch_train = len(X_short_train) / batch_size\n",
    "    for i in range(num_batch_train):\n",
    "        start_idx = i * batch_size\n",
    "        end_idx = (i + 1) * batch_size\n",
    "        X_train = X_short_train[start_idx:end_idx]\n",
    "        Y_train = Y_short_train[start_idx:end_idx]\n",
    "        train_loss += model.train(X_train, Y_train) / num_batch_train\n",
    "    \n",
    "    # Monitor the validation score at each epoch\n",
    "    num_batch_validation = len(X_short_validation) / batch_size\n",
    "    for i in range(num_batch_validation):\n",
    "        start_idx = i * batch_size\n",
    "        end_idx = (i + 1) * batch_size\n",
    "        X_val = X_short_validation[start_idx:end_idx]\n",
    "        Y_val = Y_short_validation[start_idx:end_idx]\n",
    "        val_loss += model.loss(X_val, Y_val) / num_batch_validation\n",
    "\n",
    "    print(\"epoch %d, train loss: %f, validation loss: %f\"\n",
    "          % (epoch + 1, train_loss, val_loss))\n",
    "    \n",
    "    # Patience based early stopping on validation score\n",
    "    if best_val_loss is None:\n",
    "        best_val_loss = val_loss\n",
    "    elif val_loss < best_val_loss - tolerance:\n",
    "        patience = original_patience\n",
    "        best_val_loss = val_loss\n",
    "    else:\n",
    "        patience -= 1\n",
    "        if patience < 0:\n",
    "            print(\"Validation score stopped improving\")\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compute some predictions to visualize what the network predicts:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 15, 4)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_short_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(256, 15, 4)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_validation_batch = X_short_validation[0:256]\n",
    "X_validation_batch.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predicted = model.predictions(X_validation_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(256, 15, 4)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Input symbols for the first sequence of the first batch:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'daadbdbdaaaaaaa'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarray_to_symbols(X_validation_batch[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicted symbols:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'aaaaadbdaaacbdd'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarray_to_symbols(predicted[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Expected / correct output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'aaaaadbdaaaaaaa'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarray_to_symbols(Y_short_validation[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAJnCAYAAADsoeauAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm4ZHdZJ/DvmzQICYEE8CEskU0hsshjGCI4BnpEfHAB\n0XF0gFHWcRdcAEEcaVFwBxlFXJBVFjEiLuMoIBGYYQ/ILiBrEkhIQjaDDND9mz/qdKdyu9N9+96q\nW/fm/Xyep5+uOqfOOe+tes/51jl1qk6NMQIA13bHrLoAANgKAg+AFgQeAC0IPABaEHgAtCDwAGhB\n4O0QVfW+qrr3qus4WlW1p6pevKR5f6Kqvnkjy6mqF1TVL0+3d1fVOUcx7cOr6o1HXzFsTlXdpqr2\nVdUx0/1/qqpHHcX0n6iq+y6vwiMuf8Pr6TWM31dVt1vv/ATeDjHGuMsY4w3LXs6RGuwaptl3mNHL\n/KLnuIbb653Wl1APYdooPWULlnNUG+tFz28Kj48vavkrcrR9vOq+X9p6Om27Hna4xwg8ri1qi6bp\nYKs2iItejjcw298y19Mjvv4Cb4c4xOG7V1TVC6vq8ulw593XPPaJVfX+qvpcVT2vqr5iGnfQ4bjp\nsMDtq+qHkjwkyROq6oqq+qt1lneg0arqtlX1+qmuVye56Zpl/XlVfaaqLp0ed6e5cd9RVe+qqsuq\n6lNr9zKq6geq6pNVdVFV/fwharheVb18WvbZVfV1c9N+fVW9cxr38iTXO8Rz/KSqurCqPl5VD5kb\nfpOq+uuprrcmuf2a6Z411XtZVb2jqr5pbtzpVfXmqrqkqj5dVb9bVdfZztPOPZ9HVFU3qqoXVdVn\np757clXVNO5qh6/qqsNxx1bV05KckeT3pl77n9Nj9lXVT1bVR6fX4jc2M791mO/dJ1bVv0498v6q\netDcuNtX1eum3ruwqv60qm60BdMeU1W/NU330STfcYi/4aur6q3Ta/mqqjppbvprXGeW3F93rqrX\nVNXFVXV+VT1p7vne8HpaVY+flnduVT3ycK/nIY0x/NsB/5J8PMk3T7f3JPn3JPfP7N3P05O8ee6x\nn0jyniS3THJSkv+T5JencQ9P8sY1896X5HbT7ecneeqa8c9O8ux11vnmJL+V5DqZbYAuT/KiufEP\nT3L8NP6ZSd41N+4+Se483b5rkvOTfNd0/05JrkjyTUmum+S3k3xpzXPyxSTfk+TYJD+b5GPT7esm\n+WSSx073//P02KdO0+6e5rW/7nsn+bckXzONf/n07/pJ7pzk3CRvmKv7odPzfEySn0nymSTXncad\nluT0adytk3wgyWO387RH2ZcvSvKX02t66yQfSvLIadxTkrx47rG3yazXjpnun7X/sWt68R+TnJjk\nlGl+j9rE/P4myRPW+bd8b5KTp9vfN/XAzab7t09y36k/bprk9UmeuQXT/kiSD+aqdfmsJHvn/uZ/\nyqwf75TkuCRn7n+OcuR1Zin9leSE6bE/PS33BklOX8B6ev/Mtgn7/9aXZm7bta7XeNkbav8W8y8H\nB96r58bdKcnn1zz2h+buf1uSf51uPzxHDrxf3mCNXzWtUNefG/aSzG2k1jz+xGnZJ1zD+N9J8ozp\n9i8meencuOOS/L81z8mb5sZXkk9PK/u9k5y3Zt7/NwcH3nzdf5bkF6YV74tJ7jA37mlrn8M18/5c\nkrtew7ifSvLKnTTtYaY5dnoNTp0b9kNJzpp7TY4UUI86RC9+69z9H03y2o3ObzP/krwryQOvYdyD\nkrxzidM+YLr9ulx9Xb7fIf7mp8+N/9rpNTnmSOvMsvoryYOTnH0Nj9uTja+nz1vzt35NjjLwHNLc\nuS6Yu/35zA4TzL+e82cdfirJLbagplskuWSM8e9zwz65/8Z06OnXpsM3l2UWzCPTYc+q+oaqOms6\nPHZpkh9OcpO5eZ+7f15jjM8nuXjN8ufHj+n+LZLcPMl5ax77yTX3D1X3zafaduXg5/OAqnpcVX2g\nZodpL0lyo7m/6Q5V9bc1O4x7WWZheZPtPO1RuGlmey3zz+WnMtsbWa9xiGGb6d1DzW9dquoHa3ZI\n/ZLpOblLpuesqm42HYY7d3o+X5yrP5+Lnnb/a3HzHKb3JmvH79+TvHkOs84ssb9OyWyv7ZpsdD1d\nz3NxWALv2uur1tz+9HT7ysze6SVJqurkNdNteIOR2WGMk6rquLlht56b50OSPDDJfccYN0py28ze\n4e3/UPqlSV6V5FZjjBOT/MHcuE9ntiLtr/u4zK2ck/nxxyS5VWYr0Gdy8Eb41mvuH6ruTye5MMmX\nc/DzuX85ZyR5fJL/MsY4cYxxUpLL5up+TmaHe756+pufnGm928bTrtdFme0Z32bNc7N/g3a1Xkuy\n3l5b+1zv3whudH5HVFW3TvJHSX48yY2n5+R9ueo5eXpmhxLvMj2fP5Crns9lTvuZXEPvXcOw/UdZ\nLpqmPdw6s5T+yiyIDvdVgY2up+t5Lg5L4F07VZIfq6pbVtWNM2vGl0/j3p3kzlV1t6q6XmaHGOZd\nkMM36zUaY3wyyTuS/FJVXWf6kPs75x5yg8wOqXyuqo7PbEOQNeMvGWN8sapOzywg9/uLJN9ZVf+x\nqq6b5Kk5uH/vXlXfXVW7MjvE8oUkb5n+fbmqHjPV9T1J7nGIP2F/3WdkdnLAn48x9iV5ZZI9VXX9\nmp1k87BctXE9IbNAvKiqrltVv5jkhmv+piuSfL6qTs3sEN12n/ZqanZiyEHfAR1j7E3yiiRPq6ob\nTBvvn07yp9ND3pXk3lV1Ss1O0njSmllckDUnAE0eV1UnVtUpSR6T2eHlzcxvPY7P7Pm5KMkxVfWI\nzPa09rtBZoF7eVXdMrMw2IppX5HkMdO6fFKSJ66pu5L8t6r62inQnpqr+vZI68yi+2u//5Xk5lX1\n2Kr6iqo6YVqf99voevqKJA+f+1ufkqMk8HamkYPfzY41t1+a5NVJPprkI0l+JUnGGB/OrPFfm9kJ\nAW9cM+2fJLnTdHjllUlSVX9QVc9ZZ20PSfINmR3v/8UkL5wb96LMDlGcl9m72DevWfaPJXlqVV2e\n5H/kqg1dxhjvz+xd8Esz2/P6XK5+eGNktnf4/dO4hyb5njHG3jHG/g/JH57ZIZ3vy2xjMO8zSS6Z\n5v3iJD88PVdJ8hOZreDnZ/Y5wvPmpvv76d+HMztZ6N9z9UMtj5uek8szeyf/8h0w7QFT6FyR5L2H\nGp/kJzPbmH8ss156SWafA2eM8drMXsP3JHl7ZieQzL/ez0ryvTU7k/h35ob/VZKzMwu4v830fG9k\nflX1d1W1NiQOMsb4QGYndbw5s9f5Lpmd7LXfL2V2osZl03L/Yv+ylzztHyf5h8zeqL5jftr9pWe2\nXr0g00klmb1JWM86s+j+2v83XZHZZ40PmGr6cGafk++vd0Pr6Rjj7zP7XP910zz/MQdvBw+rpg//\nuBap2ZdpHzXGeN2qa2Fnq6qHJrnTGOPJW7S8fZkdJjvcZ0CwIbtWXQCwfY0xXrLqGmBRHNIEthOH\nnFgahzQBaMEeHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwJvnWp2pfBD/ogu61OzK2Kfteo6\nOtG3i6F3rx0EHlvpUD96DTuB3r0WEHhspaO93hpsF3r3WkDgAdDCtgy86WKSv1JVb62qC6vqC1X1\nkar61aq6/orLu05V7amqT051vbuqvn9VxUwXZ3xCVf1zVV1ZVZdW1dur6sdXWNMpVfWKqrps+vfX\nVbXRC3PuGPr26Ohdttp2vTzQrZI8KsmZmV09+cuZXUDwCUm+Psn9V1ZZ8utJjkvye5kd5nhEkpdV\n1fXGGC887JQLNl3F+B+S3Gf6/0WZXT3465J8d5Jnb2U9U00nJnlDZq/hc5J8ILPX7nVJVr3RXzZ9\nu056l5UYY2y7f0muk+TYQwx/apJ9Se6xgpoePi3740lOmBt+w8yuCHxxkuttcU1PmGr6lUOMqxW9\ndk+fanrYmuHPnIa/blV9tQV/u75df116178t/7ctD2mOMb40xtibJFW1q6pOqqqbZnZJ9yQ5fXXV\n5Tljdgn7JMkY4/Ikf5DkpFx1Gfut8tAkn8tsg3o1Y1pTV+BBSc7P7B37vF9fQS1bSt8eFb3LltuW\ngZckVfVjVfWezA5zXJzks0n2fw/mpJUVlnzwMMNuu5WFJPmaJP8yxvjiFi/3cG6X5CNrN1pjjPOT\nXLaakraOvl03vcuW25af4VXVzyT5rcyO7f9Okk8n+WJmx9ZfkG0c1PSlb2F725aBl+QHknx8jPFt\n8wOrapUf+u93pyR/c4hhSfKxLa7lQ0m+tqquu43eKX8syR2q6pgxxr79A6vq5klutLqytoS+XT+9\ny5bbru84v5wkVXWgvqraleSJK6voKj9aVTfcf6eqbpTkR5JckuT1W1zLSzI7TPYLa0dU1aq+KPuq\nJDdL8oNrhv/cCmrZavp2/fQuW2677uGdmeRXk/zvqvrLzM4oe0hmh4dW7cIkb62q5+eq07tvleTR\nY4wvbHEtz0rygCS/UFX3SPKazD47unOSOyS53xbXkyS/kdlr9cdVdfdcdWr3PZNclGv3L1bo2/XT\nu2y57Rp4v5lZcz0qs89CPpPkzzL7HOQDqysrI7N3e/dO8uOZvRv8UJKHjjFevuXFjPGlqvrWJD+b\n2Yr6tMw2Gh9O8vytrmeq6dKqOiPJM3LVO+V/SvKfMjtb8dr8e4T6dr0F6V1WoFZ3BjAAbJ3t+hke\nACyUwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0sLQvnleVL/ixcGOMpf/ahd5lGbaidzm8pe7hLeqi\nfU95ylMWMh9Yr+3Ut3oXFsMhTQBaEHgAtLAjAm/37t2rLgGOmr6F7WVpPx5dVWO7ffawustssShb\nddKK3mXRnLSyejtiDw8ANkvgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0sOHAq6r7V9W/VNVH\nqurnFlkULJPehZ429EsrVXVskg8l+ZYk5yV5e5IHjzE+OPcYv1bBwm321yr0Lqvil1ZWb6N7eKcn\n+dcxxifGGF9K8vIk37W4smBp9C40tdHAu2WSc+bunzsNg+1O70JTGw287XW8B9ZP70JTuzY43XlJ\nTpm7f0pm75SvZs+ePQdu79692+VS2A70LjS10ZNWdmX2wf99k3w6ydvig3+2wAJOWtG7rISTVlZv\nQ3t4Y4wvV9VPJPmHJMcm+ZP5DQZsV3oX+nIBWHYUF4Blp7KHt3p+aQWAFgQeAC0IPABaEHgAtCDw\nAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0IPABaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0I\nPABaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0IPABaEHgAtCDwAGhB4AHQgsADoAWBB0AL\nAg+AFgQeAC0IPABaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0IPABaEHgAtCDwAGhB4AHQ\ngsADoAWBB0ALAg+AFgQeAC0IPABaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0IPABaEHgA\ntCDwAGhB4AHQgsADoAWBB0ALAg+AFjYceFV1SlWdVVXvr6r3VdVjFlkYLIO+hb5qjLGxCatOTnLy\nGOOfq+oGSc5O8qAxxgen8WOj816Wqlp1CWzSGGNTL+KR+nZ6jN5l4Tbbu2zehvfwxhjnjzH+ebr9\nb0k+mOQWiyoMlkHfQl8L+Qyvqm6T5OuTvHUR84OtoG+hl00H3nRY6Mwkj53eMcO2p2+hn12bmbiq\nrpPkL5L86RjjVWvH79mz58Dt3bt3Z/fu3ZtZHCzEkfo20btwbbSZk1YqyQuTXDzG+OlDjPfBPwu3\ngJNWDtu302P0LgvnpJXV20zgfVOSNyR5T5L9M3nSGOPvp/E2GizcAgLvsH07PUbvsnACb/U2HHhH\nnLGNBkuwFRsNvcsyCLzV80srALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABo\nQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwA\nWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIP\ngBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABoQeAB0MKu\nVRcA29G555676hKu5sQTT1x1CYd06aWXrrqEgxx//PGrLuEgV1555apLIPbwAGhC4AHQgsADoAWB\nB0ALAg+AFgQeAC0IPABaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFjYVeFV1bFW9q6r+ZlEFwbLp\nW+hps3t4j03ygSRjAbXAVtG30NCGA6+qbpXk25M8N0ktrCJYIn0LfW1mD++ZSR6fZN+CaoGtoG+h\nqQ0FXlV9Z5LPjjHeFe+S2SH0LfS2a4PTfWOSB1bVtye5XpIbVtWLxhg/OP+gPXv2HLi9e/fu7N69\ne4OLg4VYV98myTOe8YwDt+91r3vlXve619ZVyY63d+/e7N27d9VlsEaNsbnP7avqPkkeN8Z4wJrh\nY7PzXrQqb+p3ujHGQl7Ea+rbadw455xzFrGYhbnrXe+66hIO6dJLL111CQc5/vjjV13CQa688sqF\n9S4bt6jv4W2vZIP10bfQyEYPaR4wxnh9ktcvoBbYMvoW+vFLKwC0IPAAaEHgAdCCwAOgBYEHQAsC\nD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWNn0B2GuccZVrje1g2+3ivcnsAr5b\ncRHNqhq7dm36ylkLdeqpp666hEN63/vet+oSDnL22WevuoSD3P3ud3cB2G3AHh4ALQg8AFoQeAC0\nIPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4A\nLQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEH\nQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdBCjTGWM+Oq5cx4E5b1t25WVa26hB1jjLH0J2s79u6+\nfftWXcIhHXOM98zrtRW9y+HpVgBaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0IPABaEHgA\ntCDwAGhB4AHQgsADoIUNB15VnVhVZ1bVB6vqA1V1z0UWBsuid6GnXZuY9llJ/m6M8b1VtSvJ8Quq\nCZZN70JDG7oeXlXdKMm7xhi3O8xjtt01xVwPb+fb7DXFdmrvuh7ezud6eKu30W69bZILq+r5VfXO\nqvrjqjpukYXBkuhdaGqjgbcryWlJfn+McVqSK5M8cWFVwfLoXWhqo4F3bpJzxxhvn+6fmdlGBLY7\nvQtNbSjwxhjnJzmnqu4wDfqWJO9fWFWwJHoX+trQSStJUlV3S/LcJNdN8tEkjxhjXDY3ftt98O+k\nlZ1vER/878TeddLKzuekldXbcOAdccbbcKMh8Ha+rdhobMfeFXg7n8BbPd0KQAsCD4AWBB4ALQg8\nAFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaGHXqgvYSi7Ds37b8VJK\nW/n6HXvssVu2rPU47TQXZV+vt73tbasu4SCnn376qksg9vAAaELgAdCCwAOgBYEHQAsCD4AWBB4A\nLQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEH\nQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHg\nAdCCwAOgBYEHQAs1xljOjKvGsua9UVW16hLYpDHG0l/EqhpXXHHFshdzVE444YRVl7BjHH/88asu\n4SBXXnnllvQuh2cPD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQ\neAC0IPAAaGHDgVdVT6qq91fVe6vqpVX1FYssDJZF70JPGwq8qrpNkv+e5LQxxl2THJvkvy6uLFgO\nvQt97drgdJcn+VKS46pqb5Ljkpy3sKpgefQuNLWhPbwxxueS/HaSTyX5dJJLxxivXWRhsAx6F/ra\n6CHN2yf5qSS3SXKLJDeoqocusC5YCr0LfW30kOZ/SPKmMcbFSVJVr0zyjUleMv+gPXv2HLi9e/fu\n7N69e4OLg4VZV+8+/elPP3D7jDPOyBlnnLGVNbLD7d27N3v37l11GaxRY4yjn6jqbpltIO6R5AtJ\nXpDkbWOMZ889Zmxk3stUVasugU0aY2zqRVxv715xxRWbWczCnXDCCasuYcc4/vjjV13CQa688spN\n9y6bt9HP8N6d5EVJ3pHkPdPgP1pUUbAsehf62tAe3rpmbA+PJdiKd8n28HY2e3hcE7+0AkALAg+A\nFgQeAC0IPABaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0IPABaEHgAtCDwAGjB5YHYUbbq\n8kB6l0VzeaDVs4cHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8\nAFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsC\nD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0sGuZM6+qZc4e\nlubNb37zqkvYEe55z3uuuoSDvOUtb1l1CWxT9vAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQ\neAC0IPAAaEHgAdCCwAOgBYEHQAsCD4AWDht4VfW8qrqgqt47N+zGVfWaqvpwVb26qk5cfplwdPQu\nsNaR9vCen+T+a4Y9Mclrxhh3SPKP033YbvQucDWHDbwxxhuTXLJm8AOTvHC6/cIkD1pCXbApehdY\nayOf4d1sjHHBdPuCJDdbYD2wTHoXGtvUFc/HGKOqxqKKga1ypN597nOfe+D2aaedltNOO21L6gKW\nZyOBd0FVnTzGOL+qbp7ks4suCpZk3b376Ec/egvLArbCRg5p/nWSh023H5bkVYsrB5ZK70JjR/pa\nwsuSvCnJHavqnKp6RJJfS3K/qvpwkm+e7sO2oneBtQ57SHOM8eBrGPUtS6gFFkbvAmv5pRUAWhB4\nALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACzXGWM6M\nq5YzY1obY9Syl1FV4yu/8iuXvZijcvnll6+6hEN63OMet+oSDnLRRRetuoSD/OEf/uGW9C6HZw8P\ngBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILA\nA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg\n8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALSwa9UFwHZ061vfetUlXM1d7nKXVZdwSGee\neeaqSzjILW5xi1WXwDZlDw+AFgQeAC0IPABaEHgAtCDwAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0I\nPABaEHgAtCDwAGjhiIFXVc+rqguq6r1zw36zqj5YVe+uqldW1Y2WWyYcHX0LrLWePbznJ7n/mmGv\nTnLnMcbdknw4yZMWXRhskr4FruaIgTfGeGOSS9YMe80YY990961JbrWE2mDD9C2w1iI+w3tkkr9b\nwHxgK+lbaGZTgVdVT07yxTHGSxdUDyydvoWedm10wqp6eJJvT3LfhVUDS7bevj3vvPMO3D7hhBNy\nwxvecLmFca1yySWX5NJLL111GayxocCrqvsneXyS+4wxvrDYkmA5jqZvb3nLW25NUVwrnXTSSTnp\npJMO3P/EJz6xumI4YD1fS3hZkjcluWNVnVNVj0zyu0lukOQ1VfWuqvr9JdcJR0XfAmsdcQ9vjPHg\nQwx+3hJqgYXRt8BafmkFgBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwA\nWhB4ALQg8ABoQeAB0ILAA6CFDV3xHK7t3vnOd666hKu54x3vuOoSDmn+qt7bxamnnrrqEg5y1lln\nrboEYg8PgCYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABo\nQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwA\nWhB4ALQg8ABoQeAB0ILAA6AFgQdACwIPgBYEHgAtCDwAWhB4ALSwa9UFAEf2+c9/ftUlHNJNbnKT\nVZdwkO36XLF69vAAaEHgAdCCwAOgBYEHQAsCD4AWBB4ALQg8AFoQeAC0IPAAaEHgAdCCwAOgBYEH\nQAuHDbyqel5VXVBV7z3EuJ+tqn1VdePllQcbo3eBtY60h/f8JPdfO7CqTklyvySfXEZRsAB6F7ia\nwwbeGOONSS45xKhnJHnCUiqCBdC7wFpH/RleVX1XknPHGO9ZQj2wNHoXejuqC8BW1XFJfj6zQ0IH\nBi+0IlgCvQsc7RXPb5/kNkneXVVJcqskZ1fV6WOMzy64Nliko+rdffv2HbhdVZmmgXU5//zzc/75\n56+6DNY4qsAbY7w3yc3236+qjye5+xjjc4suDBbpaHv3mGN8Y4eNO/nkk3PyyScfuP/ud797hdWw\n35G+lvCyJG9KcoeqOqeqHrHmIWNplcEm6F1grcPu4Y0xHnyE8bdbbDmwGHoXWMtxGwBaEHgAtCDw\nAGhB4AHQgsADoAWBB0ALAg+AFgQeAC0IPABaEHgAtCDwAGhB4MGSjLE9f5/6wgsvXHUJB7n44otX\nXcJBXN5HtpOyAAAAV0lEQVTn2kfgwZJs18C76KKLVl3CQQQeW0HgAdCCwAOghVrWYZeq2p7Hc9jR\nxhi17GXoXZZhK3qXw1ta4AHAduKQJgAtCDwAWhB4ALQg8ABoQeAB0ML/B4QCFDQkt8M8AAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe833316d90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_parallel_tapes(Y_short_validation[0], predicted[0],\n",
    "                    binarray_to_symbols(X_validation_batch[0]),\n",
    "                    binarray_to_symbols(predicted[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating data for a simple memory task\n",
    "\n",
    "The goal of this task is to output the 2nd and 5th element of random input sequences.\n",
    "\n",
    "To solve it, the RNN has to rember the current position (count the element of the sequence) and store some representation of the 2nd and 5th element in its recurrent layer activations.  This task can be made artificially hard by increasing the average lentgh of the sequences as it is likely to be harder to memorize old events."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['addddbdbca', 'caaacbcddcab', 'bbabadadbc', 'cdabd', 'dcdabbbd']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sequences(n_sequences=5, min_length=5, max_length=15, seed=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's generate the sequences as a data array padded with zeros."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_memory_task_data(n_sequences=1000, min_length=5,\n",
    "                              max_length=15, alphabet=ALPHABET,\n",
    "                              seed=None):\n",
    "    \n",
    "    # Generate the input sequences as symbols\n",
    "    input_sequences = generate_sequences(\n",
    "        n_sequences=n_sequences, min_length=min_length,\n",
    "        max_length=max_length, alphabet=alphabet, seed=seed)\n",
    "    output_sequences = []\n",
    "    \n",
    "    # 2 steps are used for the expected output\n",
    "    n_steps_per_sequence = (max_length + 2)\n",
    "\n",
    "    # Allocate all the data arrays at once and pad with zeros\n",
    "    n_features = len(alphabet)\n",
    "    input_data = np.zeros((n_sequences, n_steps_per_sequence, n_features),\n",
    "                          dtype=np.float32)\n",
    "    output_data = np.zeros((n_sequences, n_steps_per_sequence, n_features),\n",
    "                           dtype=np.float32)\n",
    "    \n",
    "    for sequence_idx, input_sequence in enumerate(input_sequences):\n",
    "        # store the encoded sequence to the input tape\n",
    "        encoded_input = symbols_to_binarray(input_sequence)\n",
    "        input_data[sequence_idx, 0:len(input_sequence)] = encoded_input\n",
    "        \n",
    "        # store the encoded output to the output tape by taking\n",
    "        # the 2nd and 5th symbols of the input sequence\n",
    "        output_sequence = input_sequence[1] + input_sequence[4]\n",
    "        output_sequences.append(output_sequence)\n",
    "        encoded_output = symbols_to_binarray(output_sequence)\n",
    "        start = len(input_sequence)\n",
    "        stop = start + 2\n",
    "        output_data[sequence_idx, start:stop] = encoded_output\n",
    "        \n",
    "    return input_sequences, output_sequences, input_data, output_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's generate some reference data for this task with fixed random seeds to make it possible to compare the results of different study groups on the same data.\n",
    "\n",
    "Again, we create 2 datasets, one with only short sequences (from 5 to 15 steps in the input sequence) and one with long sequences (from 20 to 50 steps)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!rm -rf data/memory_task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving symbol sequences data/memory_task/short_sequence_train.json\n",
      "Saving data data/memory_task/short_input_data_train.npy\n",
      "Saving data data/memory_task/short_output_data_train.npy\n",
      "Saving symbol sequences data/memory_task/short_sequence_validation.json\n",
      "Saving data data/memory_task/short_input_data_validation.npy\n",
      "Saving data data/memory_task/short_output_data_validation.npy\n",
      "Saving symbol sequences data/memory_task/short_sequence_test.json\n",
      "Saving data data/memory_task/short_input_data_test.npy\n",
      "Saving data data/memory_task/short_output_data_test.npy\n",
      "Saving symbol sequences data/memory_task/long_sequence_train.json\n",
      "Saving data data/memory_task/long_input_data_train.npy\n",
      "Saving data data/memory_task/long_output_data_train.npy\n",
      "Saving symbol sequences data/memory_task/long_sequence_validation.json\n",
      "Saving data data/memory_task/long_input_data_validation.npy\n",
      "Saving data data/memory_task/long_output_data_validation.npy\n",
      "Saving symbol sequences data/memory_task/long_sequence_test.json\n",
      "Saving data data/memory_task/long_input_data_test.npy\n",
      "Saving data data/memory_task/long_output_data_test.npy\n"
     ]
    }
   ],
   "source": [
    "dataset_specs = [\n",
    "    ('short', dict(min_length=5, max_length=15)),\n",
    "    ('long', dict(min_length=20, max_length=50)),\n",
    "]\n",
    "     \n",
    "fold_specs = [\n",
    "    ('train', dict(n_sequences=5000, seed=0)),\n",
    "    ('validation', dict(n_sequences=1000, seed=1)),\n",
    "    ('test', dict(n_sequences=1000, seed=2)),\n",
    "]\n",
    "\n",
    "data_folder = 'data/memory_task'\n",
    "write_dataset(data_folder, generate_memory_task_data,\n",
    "              dataset_specs, fold_specs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's possible to reload the save data using pandas for loading the symbolic data in the json files and numpy for the binary data in the `.npy` files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "short_symbols_train = pd.read_json(\n",
    "    'data/memory_task/short_sequence_train.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>addddbdbca</td>\n",
       "      <td>dd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>caaacbcddcab</td>\n",
       "      <td>ac</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bbabadadbc</td>\n",
       "      <td>ba</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cdabd</td>\n",
       "      <td>dd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dcdabbbd</td>\n",
       "      <td>cb</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          input output\n",
       "0    addddbdbca     dd\n",
       "1  caaacbcddcab     ac\n",
       "2    bbabadadbc     ba\n",
       "3         cdabd     dd\n",
       "4      dcdabbbd     cb"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "short_symbols_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_short_train = np.load('data/memory_task/short_input_data_train.npy')\n",
    "Y_short_train = np.load('data/memory_task/short_output_data_train.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAK1CAYAAAAg3+4oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGRpJREFUeJzt3XuwrXdd3/HPNznhfkk0NkYCJWixBltkLGlnauUMVBto\nUelMUaDWQGpLQextitFGEhAhUCw6lcEZTbhJoQFBZEqRayDt0BRmAAnXAAmE3Ai50gA1l1//WM8m\nKyv77LPPvq11zvf1mtmz116XZ/3W2r/93s96nnWpMUYA6OWoZQ8AgL0n/gANiT9AQ+IP0JD4AzQk\n/gANiT8rqaouqaqfXPY49lJVnV5VF21w+oVVdcYBTnt4Vd1ZVev+TVfVOVX1hs2cd53L7q+qK+Z+\nvryqnrCZy7KztjMHFok/K2mM8aNjjA/v9vVU1Wur6rd2cHmnV9Vrdmp5C8b0tdXLrsI41jX9g/ng\nTi7zANfz3X+CK7q8g90Pm77vp38UjzvQ6eIPcGTa8B+F+LOSpk0Lj58On1NVF1TV66rqlmmT0I8v\nnPfMqvp0Vd1QVedX1b2n0+6xKWV6aPyDVfUvkjw9yfOr6ptV9Y5NjOseD60XHorf7Q+uqt5SVVdX\n1U1V9aGqOmXutO+tqj+rqpur6uIkP7hwXT9VVZ+bLvtfktT0lao6uqpeUVXXVdWXkvzDhcuePF3f\nLVX1niTHr3NzzqiqK6vqqqr693OXve/0iOiGqvp0kseuc9lT17u/p8v/bFV9YrpdX6yqfzAd/8yq\n+sw0pi9N9/+aQ1mjrao6a/q9XzvNiwdNp91tE9V03OVV9YSqOi3Jryf5+en3/fHp9Aur6qVVdfE0\n5j+tquO2urxNjP/OqnrE3M+Ljz7n58+W58BBjTF8+Vq5rySXJXn8dPicJN9Octo08V+S5CNz5708\nyV8keUiS45L8zyS/NZ12epKLFpZ9Z5JHTIdfk+RFC6e/KsmrDjCuh0+XP2ruuA8medYBzn96kvsn\nOSbJK5N8fO60N09f903yqCRfS/Lh6bTjk9yS5B8nOTrJv0ly29r1JHl2ks/O3eYPJrljbVxJPpLk\nFdP1/r1pWa9fuA1vnK77R5N8PckTptPPTfKhJMcmOSnJJUm+usn7+9QkN80t6weS/PB0+ElJTp4O\n/2SSW5M8Zgtz41lJLp1ux/2T/Mncbduf5IoN5tLZa+edO/3C6b4/Jcn9krw1yRu2sbwzk7xzg/F/\nd/4daA7uxBw42Jc1fw4XF40x3j1ms/6Pkzx67rSR5PfHGFeOMW5M8ttJnnYIy675H8YYzx1jPHfb\nI54t67VjjFvHGLcleWGSR1fVA6vq6Mz+qF8wxvj2GOPTSV43N5YnJblkjPG2McYdY4zfTXLN3KKf\nmuSVc7f5JblrjfBhSf5Wkt8cY9w2xrgoyTsXb2eSF07XfUlmAVq7z/5Jkt8eY9w0xvhakt9buOxG\n9/cZSc4bY7x/uv1XjTE+Px1+1xjjsunwh5O8J7N/TIfqGUl+Z4xx+Rjj1szWvn+hNrej87trzgu3\n5/VjjM+MMb6V5DeTPLWqFs+3qeWNMc4dYzx5E5ddXM6iLc+BzRB/DhfXzh3+VpL7LPyxzz80/2pm\na5xLVVVHVdW506aPmzNbYxyZrdF9X5J9uee41/xAZmuj8+bPe+JBLnvjGOPbc8d9ZZ0hLl7+xLnL\nH2jZB7rs2v19UpIvrXP+VNUTq+p/V9X1VXVjZnH73vXOexAn5u6356uZ3ZcnbGFZaxZvzzFZf1PZ\nXtrOHDgo8edI8bCFw1dNh2/N7KF8kqSqvn/hcof6rJVbp+/3mztucZlrnpHkZzLbBPLgJCfnrjXF\n65Lcvs6411yV5KFz4675n5NcvcFlr05yXFXNj/Gv5p639UD32UbLPtBlr5wOX5HkhxbPPO0T+JMk\nL0/yV8YYxyV5Vw5hTXXOVZlt8pm//tszW0FY/H0fndk/2jUH+n0v3p7bknxjG8vbyLdy9/lz4gGW\ns505cFDiz5Ggkjynqh5SVd+T5D9mti09ST6Z5FFV9eiquk9m+w/mXZvkEdmkMcZ1mYXuF6cdbs/K\nwo7aOQ9I8v+S3FBV98/sYfnacu5I8rYk50w7WE9J8ku5KwLvmsb9lKral+RXc/d/Mhck+dXpNh+X\n2XbmtWV/JcnHkrywqo6pqp9I8o/WGd9Z03U/KrN9E/9tbtm/XlXHVtVJSZ63cLlK8tyF+3vtsucl\neWZVPX565POQqvrhJPeavr6R5M6qemKSnz7A/ba2E/bsA5z8piT/tmY73x+Q2f365jHGnUm+kNmj\nwidV1TFJzkpy77nLXpPk4QubdCrJP62qH5n+Yb4oyVumTYxbWd7BfCLJM6b5c1pm+z/W89+zxTmw\nGeLP4WC9Z4KMhcP/NbNtyF/KbGfgi5NkjPGFzP6Y35fk80kuWrjseUlOqaobq+ptSVJVf1BVr95g\nPL+c5D9kFrJTkvyvA5zv9Zltnrgys52mH1m47l/J7B/ENUnOn74yjfsbmW17P3e6nh/KbMfqmj9M\n8ueZ/XP7WGZr1fPLfnqSv53khiQvyGx/wryR2U7dL2Z23/ynMcb7ptNeOI37siTvnm7H4v39xqx/\nf380yTMz27l9U2Y7Ux82xvhmZvG6YBrT05Js9OyqkxZu77zzk7whyYeTfDmzNennTdd/c5LnJPmj\nzDaZ/N/cfdPIW6bv11fVx+ZuzxuSvDaztel7TWPd0vKq6jeq6l0b3LZ/neTJSW7M7Pf09vXONMa4\nPtubAxuqaa8xHLaq6rIkZ4wxPrDssbB906ONN48xfmKPru+DmT275/yDnvkIsm/ZAwCYNz3DaE/C\nP2cr+x4Oazb7AOzw21UcDmz2AWjImj9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP\n0JD4AzQk/ptQVadX1Z1VdaAPXWATqury6e1z2SPm7vYdqfNW/NlL630oC6y6I3Leij97qd17pnNE\nOCLnrfgDNLRy8a+qB1TVi6vq4qq6rqq+U1WXVtVLq+q+Sx7eMVV1TlV9ZRrXJ6vq55c5oKq6V1U9\nv6o+UVW3VtVNVfXRqnruEsf00Kq6oKpunr7+rKoO9CHnRwxz99Cs2tztNm9X8WMcT0pyRpK3Jvnj\nJLcn2Z/k+Ukek+S0pY0seVmS+yX5/cweCj4zyZuq6j5jjMUPyN51VXWvzD7A+XHT99cn+U6Sv5nk\nKUletYQxHZvZB2uflOTVST6T2e/vA0mWHcDdZu5u0qrN3ZbzdoyxUl9Jjkly9DrHvyjJnUkeu4Qx\nnT5d92VJHjh3/IOSXJ7k+iT3WcK4nj+N68XrnFZL+v29ZBrTLy0c/8rp+A8sY1x7dNvN3c2Pa6Xm\nbsd5u3KbfcYYt40x7kiSqtpXVcdV1fFJ3j+d5dTljS6vHmN8c+2HMcYtSf4gyXGZrSXstWckuSGz\nuNzNmGbuEvxckmsyW5Ob97IljGVPmbuHZNXmbrt5u3LxT5Kqek5V/UVmDwOvT/L1JGvPsz1uaQNL\nPrvBcSfv5UAmfy3J58YYf7mE6z6QRyS5dPEPeIxxTZKblzOkvWPubtqqzd1283bltvlX1b9L8orM\ntgP+bpKrkvxlZtviXpsV/YcF5i6Hk5WLf5JfTHLZGOOJ80dW1TJ3lq05Jck71zkuSb68x2NJks8n\n+ZGqutcKrUF9Ockjq+qoMcada0dW1YlJHry8Ye0Jc3fzVm3utpu3q7gmcnuSVNV3x1ZV+5KcubQR\n3eVfVdWD1n6oqgcneXaSG5N8aAnjeWNmmxLOWjyhqpb1wpQ/TXJCkn+2cPyvLWEse83c3bxVm7vt\n5u0qrvm/NclLk/yPqnp7Zs9KeHpmD5+X7bokF1fVa3LX0+VOSvLPxxjfWcJ4fi/Jk5OcVVWPTfLe\nzLY1PyrJI5P81BLG9PLMfl9/WFU/nrueMvd3knwjR+irJSfm7uat2tztN2+X/XSjdZ5ydVRma0qX\nZjYZLktybpK/ntlTrl6whDGdnuSOJI9Pck6Sr0xj+2SSX1jy/XXvJL+R5JIk385sTe7iJM9e4pge\nmuQtme0ouznJOzLboXZZjsCnzM3dbnP30Ma2UnO327yt6UYD0MgqbvMHYJeJP0BD4g/QkPgDNCT+\nAA2JP0BD4g/Q0K69wreqvICAHTfG2PVXWpq77LS9mLeHalfX/HfqlWhnn332Tr2CD4DY7APQkvgD\nNHRYxH///v3LHgLAEWXX3titqsaqbWdf3lvcs1Ps8OVw1G6HLwCrSfwBGhJ/gIbEH6Ah8QdoSPwB\nGhJ/gIa2HP+qOq2qPldVl1bVr+3koGA3mbuwxRd5VdXRST6f5O8nuTLJR5M8bYzx2bnzeJEXO267\nL5bZ7Nzd1iBhwZH0Iq9Tk3xxjHH5GOO2JG9O8rM7NyzYNeYuZOvxf0iSK+Z+/tp0HKw6cxey9fh7\nWMzhytyFbP2TvK5M8tC5nx+a2RrU3ZxzzjnfPbx//37vzskq2NTchSPdVnf47stsp9kTklyV5P/E\nDl/2wA7s8N3U3N3WIGHBKu7w3dKa/xjj9qr6lSR/nuToJOfN//HAqjJ3Ycb7+XNY8X7+HI5Wcc3f\nK3wBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIa2+kle\nm+ItlA9fq/Z23In5BDvJmj9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk\n/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4\nAzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP\n0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9A\nQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gAN\niT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk\n/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4\nAzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP\n0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9A\nQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gAN\n7Vv2APbSGGPZQ1hXVS17CPewimMCdo41f4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CG\nxB+gIfEHaEj8ARoSf4CGtvyunlV1eZJbktyR5LYxxqk7NSjYTeYubO8tnUeS/WOMG3ZqMLBHzF3a\n2+5mH2/6zuHK3KW17cR/JHlfVX2sqn55pwYEe8Dcpb3tbPb5u2OMq6vq+5K8t6o+N8a4aKcGBrvI\n3KW9La/5jzGunr5fl+TtSew047Bg7sIW419V96uqB06H75/kp5N8aicHBrvB3IWZrW72OSHJ26cP\n+d6X5I1jjPfs2Khg95i7kKTGGLuz4KrdWfA27NZt3a4pRGzCGGPX76xVnLsc3vZi3h4qr/AFaEj8\nARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARrazsc4Hna8dfLm\nreLbX/v9wc6x5g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/Q\nkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD\n4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/Q0L5lD2AvjTGWPYR1VdWyh3APqzgmYOdY\n8wdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbE\nH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/\ngIbEH6Ah8QdoSPwBGhJ/gIbEH6ChfcsewF6qqmUPYV1jjGUP4R5W9b4CdoY1f4CGxB+gIfEHaEj8\nARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEH\naEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+g\nIfEHaEj8ARrat5sLH2Ps5uIPWVUtewjrWtVxAUcua/4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ\n+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPi\nD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0tG83F15V\nu7l4gHsYYyx7CHezqh205g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2J\nP0BD4g/Q0Ibxr6rzq+raqvrU3HHfU1XvraovVNV7qurY3R8mHBpzFzZ2sDX/1yQ5beG4M5O8d4zx\nyCTvn36GVWPuwgY2jP8Y46IkNy4c/TNJXjcdfl2Sn9uFccG2mLuwsa1s8z9hjHHtdPjaJCfs4Hhg\nN5m7MNnWDt8x+8ic1frYHNgEc5futvIxjtdW1fePMa6pqhOTfH2nBwW7xNxl11144YW58MILlz2M\ng6qDfd5lVT08yTvHGH9j+vnlSa4fY7ysqs5McuwY4x47zqrKWhU7boyx6Q9ENXd7WsXP8D2UebtX\nNox/Vb0pyeOSHJ/ZNtIXJHlHkguSPCzJ5UmeOsa4aZ3LrtZvgCPCZv+IzN2+xH9zDrrmv+UF+wNi\nF+zFH5G5e3gT/83xCl+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEa\nEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI\n/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHx\nB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEa2rfsAQDspKpa9hAOC9b8ARoSf4CG\nxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoS\nf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8\nARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEH\naEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+g\nIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CG\nxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoS\nf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8\nARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEH\naGjD+FfV+VV1bVV9au64c6rqa1X18enrtN0fJhwacxc2drA1/9ckWfwDGUn+8xjjMdPXu3dnaLAt\n5i5sYMP4jzEuSnLjOifV7gwHdoa5Cxvb6jb/51XVJ6vqvKo6dkdHBLvL3IVsLf6vTnJykh9LcnWS\n39nREcHuMXdhcsjxH2N8fUyS/FGSU3d+WLDzzF24yyHHv6pOnPvxKUk+daDzwioxd+Eu+zY6sare\nlORxSY6vqiuSnJ1kf1X9WGbPnLgsyb/c9VHCITJ3YWM1ewS8Cwuu2p0F09oYY9efrWPustP2Yt4e\nKq/wBWhI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+A\nhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEa\nEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI\n/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2ioxhjLHgMAe8yaP0BD4g/QkPgDNCT+AA2JP0BD\n/x/QAOtdUEbzxQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8568102d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAK1CAYAAAAnsfJ4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGOVJREFUeJzt3XuUrXdd3/HPNzmEcJMEaBFIwkXFVaAidZFeUDiCUsCl\noF2VgkUCiLRYsZUWYUlNuMhF5eKqLOhSCMgtoALCaquA3FKlFisgitwkgRBCAiQhFKVc8usfz3O+\n2WcyM2fOnJnZ+5x5vdaadfbsZ+/n+c3ev9nv/Tx7z9k1xggAJMlJyx4AAKtDFABoogBAEwUAmigA\n0EQBgCYKLF1V/WVV3XvZ4ziRVNXLq+oZO7Ceg1V1yW5u50jbYG+JAks3xrjbGOM9u72dnXqgXFjf\nOVV1/k6tb4eN+eu43858vz1yN7cxb+fiqrrvCq/vmG+HrcxZUYATV50g29mrv7Ad2dmfZTfWt+tE\ngaVbfEZVVedV1eur6hVVdc18aOl71lz2yVX1V1V1ZVW9rKpuOC87p6ouXLPua6vq26rqp5M8PMmT\nqurLVfX7WxjXHebrn7Rw3ruq6jHzt4c9S66q762qP6mqq6rq04ee1VXVD1XV+6vqS/P5567Zzu9U\n1WVVdXVVvbuq7rKw7EjXXbvNn1xYfKuqeut8O76rqs5auN5dq+ptVfXFqvpcVT1lPv9G8zPSK6vq\nr5Lcc8327lFVfz6v84Ikp65Z/uCq+sA83k9U1T+fz39UVX14vt7fzPfH2tv7KVX1+aq6qKoevmbx\nlh4Qq+q2VfXm+ef6eFX91MKyw/YUFw9bVdUrk5yV5C3z/PiPC/f/Y6vq0qr6bFU9cbvr28LYN5q/\nd1p7O8xz+h1V9YX5NntVVd184XpnVtUbquqK+TL/ZeH6m9+WYwxfvpb6leSiJPedT5+X5O+SPCDT\ns6xnJXnvwmUvTvIXSW6X5PQk/zPJM+Zl5yS5cM26r01yp/n0+Umevmb5i5K8aINx3WG+/kkL570z\nyaPXueztk1yT5KFJTk5yiyR3n5fdJ8ld59P/MMnnkjx44brnJLlJkhskeUGS9y8s2/C6R9jmy+dl\n35vklCQvPHTbJLlZksuS/Id52U2TnD0ve06Sdyc5LckZSf4yyafnZack+VSSn5u39y+SfO3QbZrk\n7CRXJ7nf/P1tk3znfPpBSe44n753kq8kucf8/cEkX0/ya/NtcO8k/zfJnbcxl96T5Dfmsd49yRVJ\nvn+9+3/e7iXrzcM19/+rk9woyd3m9d1vO+ubz/tgkn+1wdjPySbzd83535bkfvPtdav5PnvBvOzk\neTvPm8d9wyT32vJtuOwHBF++cv0ovHVh2V2S/O2ay/70wvcPTPKJ+fSmv1TzL/EzjmJchx4UthKF\npyT5vS2u94VJnr/BstPmbd7sSNfdbJuZovCahe9vkuQbmR7oH5bk/2xwvb9Jcv+F7x976IEu04P1\npWsu/8e5Lgr/NcnztngbvDHJE+bTBzNF4UYLy1+X5KlHOY/OnH/Gmyyc96wk5693/x/pQXzh/r/z\nwnnPTfJb21nfFsa/6fw9wnUfkuTP59P/NFO8Ttrqthe/HD5iFV2+cPpvk5y6eAgnyeI7VT6d6Rnp\nsp2R5JPrLaiqf1xV75x35a9O8rgkt5yXnVxVz5kPtXwp0wPJyPTsb9PrZnoQXHeb8zo+09+M8ZUk\nV2a6rTYc67x87e27uOzSNZf/1MLpMzJF5Xqq6oFV9b/mwzpXZdpzuOXCRa4aY/zdmvUe7f162yRX\nzj/r4vhvd5TrWWvl5ltV3bqqLqiqz8zz5pU5fF58aoxx7XbWLQocj85ac/qz8+mvJLnxoQVV9a1r\nrne0L9QdenC58cJ5a9d5yCWZdunX85okb0pyxhjjtCQvyXUvQD48yY9kOiRx8yR3nJfVFq776U22\nmUwPDkmSqrpppsNLl85jvdMG17ks1799F5etfYC9/cLpS5J8+9oV1vSaz+8l+ZUkf3+McXqS/77w\ncyTJ6VW1eDvfPtcP0JF8Nskt5p91cfyH4njY/Mj178uN5sfa2+PQuLa7vo0caf4uelaSbya52zxv\nHpHrHs8vSXJWVZ18lNtPIgocfyrJ46vqdlV1iyS/mOSCedkHk9y1qu5eVadmOhS16PJs/GB4PWOM\nz2d6AHjE/Iz+0dn4QfjVSX6gqv5lVR2oqltW1d3nZTfN9Ez4a1V1dqYQZGHZ/0tyZVXdJNMve9Ys\n3+i6r9lkm5XkQVV1r6o6JckzMr02c2mS/5bkNlX1c1V1w6q62bzuJHl9kqdU1WlVdUaSn13Y3nuT\nfKOqnlBVN6iqH8vhL0S/NMmjquq+VXXSfB99Z6bj+6ck+UKSa6vqgUnuv85t+LR5vd+X5IeS/M7a\nCyy8+HvW2mVjjEuS/EmSZ88/13cleXSSV80X+cB8m5w+P+D++zWruDzr379PrekF+LtmOsTzumNc\n30aONH8X3TRTRK6pqtsl+U8Ly/53poA/p6puXFWnVtU/2+ogRIFVs967I8aa069J8tZMhyo+nuSZ\nSTLG+FiSpyd5e5KPJrlwzXVfmuQuNb1T5w1JUlUvqaoXbzKex2b6hftCptc3/njdQU8PSA9K8sQk\nX0zy/iTfNS9+fJKnV9U1Sf5zrntQSZLfznSo5NJML+q+d82YN7zuGOPTm2xzZArVufOyeyT51/P1\nvpzkB5P8cKYHj49lOh6eJE+bx3NRkj+Yxzfm630tyY9lemD8YpIfz7QHcGg870vyqEwvll+d5F1J\nzpq394RMwbky02saa9/9dVmSqzI9239lksfN9+daZ2Z6s8FGexEPy/RawGeTvCHJL40x3jEve2Wm\nB96L55/tghx+Wz87UwCuqqqfXzj/3Uk+kWle/eoY4+3bXV9N76Z72HoD38L8XfS0JP8oyZeSvCXT\n/XDofvpmpvv22zPtTV6S6b7akppfmIDjQlVdlOQxC7/o7CNV9YtJrhhj/OYebOsOmV57ObDd4/PH\nowPLHgDAVo0xfnnZYzjROXwEsLF9dyjF4SMAmj0FAJooANBEAYAmCgA0UQCgiQIATRQAaKIAQBMF\nAJooANBEAYAmCltQVefMH+xx72WP5XhWVRdX1TuXPY79xNw9dvtt3ooCe2m9D9CBVbev5q0osJfq\nyBeBlbOv5q0oANBWLgpVddOqemZV/WlVfb6qvlpVH6+qZ1fVjZY8vBtU1XlV9al5XB+sqocuc0BV\ndUpVPamqPlBVX6mqq6vqfVX1M0sc05lV9fqq+tL89eaqOpoPMD8umbtHZ9Xm7n6dt2ut4sdxnpHk\nMUl+N8mrknwj04eKPynTh48/YGkjS56b5MZJfiPTLuWjkry2qk4dY7xirwdTVack+cMk95n//e0k\nX8304e0/muRFSxjTaUnek+l+fHGSD2e6/96RZNkPjLvN3N2iVZu7+3zeHm6MsVJfSW6Q5OR1zn96\nkmuT3HMJYzpn3vZFSW62cP63JLk4yReTnLqEcT1pHtcz11lWS7r/njWP6ZFrzn/BfP47ljGuPfrZ\nzd2tj2ul5u5+nrdrv1bu8NEY4+tjjG8mSVUdqKrTq+pWSf5ovsjZyxtdXjzG+PKhb8YY1yR5SZLT\nMz2r2Gs/keTKTA86hxnzjF6ChyT5XKZnfoueu4Sx7Clz96is2tzdt/N2rZWLQpJU1eOr6i8y7U5+\nMckVSQ69T/j0pQ0s+etNzrvjXg5k9h1JPjLG+NoStr2ROyX5+Npf7DHG55J8aTlD2jvm7pat2tzd\n1/N20cq9plBVP5/k1zIdZ3xhks8m+VqmY30vz4qGDMxdTgQrF4Ukj0hy0RjjgYtnVtUyX6Q75C5J\n3rLOeUnyyT0eS5J8NMk/qKpTVugZ1yeT3LmqThpjXHvozKq6TZKbL29Ye8Lc3bpVm7v7ed4eZhWf\nuXwjSaqqx1ZVB5I8eWkjus6/rapvOfRNVd08yb9JclWSdy9hPK/OdEjiqWsXVNWy/uDmTUluneQn\n15z/C0sYy14zd7du1ebufp63h1nFPYXfTfLsJP+jqt6Y6V0SD8+0G75sn0/yp1V1fq57W98ZSX5q\njPHVJYzn15P8cJKnVtU9k7wt07Hsuya5c5IfXMKYfiXT/fWbVfU9ue6tff8kyRdyYv91qLm7das2\nd/fzvD3MKkbhVzPdAY/JdFz2siSvy3RM9sPLG1ZGpmcN907yM5meVXw0yU+MMS5YyoDG+HpV3T/J\nEzNN6F/O9Iv1sSTnL2lMV1fV9yV5fq571vWuJN+f6V04J/L/IWPubnVAKzZ39/m8PUwt752LAKya\nVXxNAYAlEQUAmigA0EQBgCYKADRRAKCJAgBt1/54rar8AQQ7boyx639Zau6y0/Zi3u4UeworYKc+\nHOPcc8/dyQ8dAfYhUQCgiQIATRROIAcPHlz2EIDj3K79h3herNu6VTyGv7yPY9icF5o5HnmhGYDj\nkigA0EQBgCYKADRRAKCJAgBNFABo245CVT2gqj5SVR+vql/YyUHBbjJ3YWPb+uO1qjo5yUeT/ECS\nS5O8L8nDxhh/vXAZfwC0Rf54beuO9Y+AzF2WYT/88drZST4xxrh4jPH1JBckefDODQt2jbkLm9hu\nFG6X5JKF7z8znwerztyFTWw3CnavOV6Zu7CJ7Ubh0iRnLnx/ZqZnXLDqzF3YxHaj8GdJvqOq7lBV\npyR5aJI379ywYNeYu7CJbX1G8xjjG1X175L8YZKTk7x08d0bsKrMXdicz1NYAd6SunU+T4Hj0X54\nSyoAJyBRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUA2rY+T2GrVu2/\nhF7V/w56VccF7D/2FABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYK\nADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRR\nAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJ\nAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBN\nFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABo\nogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBA\nEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKAd2M2VV9Vurv6EMcZY\n9hCux30H+5M9BQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0U\nAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGii\nAEATBQCaKADQRAGAJgoAtAPLHsBeGmMsewjrqqplDwEgiT0FABaIAgBNFABoogBAEwUAmigA0EQB\ngCYKADRRAKCJAgBNFABoogBAEwUA2rb/l9SqujjJNUm+meTrY4yzd2pQsJvMXdjYsfzX2SPJwTHG\nlTs1GNgj5i5s4FgPH/kgAI5X5i6s41iiMJK8var+rKoeu1MDgj1g7sIGjuXw0b3GGJdV1d9L8raq\n+sgY48KdGhjsInMXNrDtPYUxxmXzv59P8sYkXqzjuGDuwsa2FYWqunFV3Ww+fZMk90/yoZ0cGOwG\ncxc2t93DR7dO8sb5A+cPJHn1GOOtOzYq2D3mLmyixhi7s+Kq3VnxMditn/VYzQ9QbMEYY9dvrFWc\nuxzf9mLe7hR/0QxAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKAd\ny8dxHndW9b+oXsX/0ntVbytgd9lTAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABo\nogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBA\nEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUA2oFlD4CkqpY9hOsZYyx7CNezircTnGjsKQDQ\nRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGA\nJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoA\nNFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEA\noIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkC\nAE0UAGiiAEATBQCaKADQDix7ACRjjGUP4XqqatlDAJbAngIATRQAaKIAQBMFAJooANBEAYAmCgA0\nUQCgiQIATRQAaKIAQBMFAJooANBEAYAmCgA0UQCgiQIATRQAaKIAQBMFAJooANBEAYAmCgA0UQCg\niQIATRQAaKIAQBMFAJooANBEAYAmCgA0UQCgiQIATRQAaKIAQBMFANqB3Vz5GGM3V3/UqmrZQ1jX\nqo4L2H/sKQDQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAG3TKFTVy6rq\n8qr60MJ5t6iqt1XVx6rqrVV12u4PE46OuQvbc6Q9hfOTPGDNeU9O8rYxxp2T/NH8Pawacxe2YdMo\njDEuTHLVmrN/JMkr5tOvSPKQXRgXHBNzF7ZnO68p3HqMcfl8+vIkt97B8cBuMnfhCI7pheYxfbTa\nan28GmyBuQvr287HcV5eVd86xvhcVd0myRUbXfC8887r0wcPHszBgwe3sTnYMVueu7BfbScKb07y\nyCTPnf9900YXXIwCrIAtz13Yr2rai95gYdVrk9wnya0yHYP9pSS/n+T1Sc5KcnGSHx9jXL3Odcdm\n616Gqlr2EDhGY4wt3YnHOnd3aryQbH3eroJNo3BMKxYFdsFe/HKJAjvteIqCv2gGoIkCAE0UAGii\nAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAtp1PXtuyVfv8glX7fIdDVu12\nAvYvewoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGA\nJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoA\nNFEAoIkCAE0UAGgHlj2AvVRVyx4Cx4kxxrKHcBhzl71iTwGAJgoANFEAoIkCAE0UAGiiAEATBQCa\nKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQ\nRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAO3AsgcAq6iqlj2ElTfGWPYQ\n1uW+Ozb2FABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJ\nAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBNFABoogBAEwUAmigA0EQBgCYKADRRAKCJAgBN\nFABoogBAEwUAmigA0A4sewDA8amqlj0EdoE9BQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCa\nKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQ\nRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGAJgoANFEAoIkCAE0UAGiiAEATBQCaKADQRAGA\nJgoANFEAoIkCAE0UAGiiAEATBQCaKADQNo1CVb2sqi6vqg8tnHdeVX2mqt4/fz1g94cJR8fche05\n0p7C+UnW/uKMJM8fY9xj/vqD3RkaHBNzF7Zh0yiMMS5MctU6i2p3hgM7w9yF7dnuawo/W1UfrKqX\nVtVpOzoi2F3mLmxiO1F4cZI7JvnuJJcled6Ojgh2j7kLR3DUURhjXDFmSX4rydk7PyzYeeYuHNlR\nR6GqbrPw7Y8m+dBGl4VVYu7CkR3YbGFVvTbJfZLcqqouSXJukoNV9d2Z3slxUZLH7foo4SiZu7A9\nNe1J78KKq3ZnxexrY4xdf/eQuctO24t5u1P8RTMATRQAaKIAQBMFAJooANBEAYAmCgA0UQCgiQIA\nTRQAaKIAQBMFAJooANBEAYAmCgA0UQCgiQIATRQAaKIAQBMFAJooANBEAYAmCgA0UQCgiQIATRQA\naKIAQBMFAJooANBEAYAmCgA0UQCgiQIATRQAaKIAQBMFAJooANBEAYAmCgA0UQCgiQIATRQAaKIA\nQBMFAJooANBEAYAmCgA0UQCgiQIATRQAaKIAQBMFAJooANBqjLHsMQCwIuwpANBEAYAmCgA0UQCg\niQIA7f8Dm69MpRqwCpAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe8330a65d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAK1CAYAAAAg3+4oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGGxJREFUeJzt3Xuw/Hdd3/HXO/mRhEiFYGoIEG4Kg4GqlDG2pYUMtp2g\ntYJtocIAQWpVaLVTCzKWknCpt1bFQSZMW4i0AjFFbnYscg1ECxRaGy7hEiSJ5EIu5FouDSSf/rHf\nH9mc38n5ndue3fN7Px4zZ357ds/u7727n/M83/Pd3bM1xggAvRy17AEA2HviD9CQ+AM0JP4ADYk/\nQEPiD9CQ+LOyquqTVfX4Zc+xE1X1kKq6o6rW/V6rqrOr6r9s87LPrKoLdzbhXS7vjqp62DbPe1lV\nPXE6vO3rxF0dbv3shPizssYYjx5jfHDR/09V/W5VvXwXL+/Mqjp3k19+pLzQZtzN4U3b4u22bQu4\nv5e5fg53WXfc3WniD8tVyx5gAY7E63TEEX9W1jq7Es6vqtdX1S3TLqHHrvnaF1XVp6rqhqp6XVUd\nO512yO6R6Vfp76qqf5rk6UleWFW3VtXbNzHXIb+KV9UFVfXc6dORQ7d+n1tVV1bVVVX1C3PHjyTH\nVdV50/X6X1X1vXOX+6Kq+vx02qeq6smHjlOvqqqbqurTB2+v6YTnVNXF03n/fLqu82d8wTTPFVX1\nk2tO+5Gq+rOqurmq/qKqzlpz+jOr6vKqur6qfmnNTIe7TqdU1Vuq6trp/K/a4HZbV1UdW1WvnG7T\nK6vqt6rqmOm0Ld/f+3X9VNVpVfWhqrpxOu1VVXWPufPd/e05xvDhYyU/klya5InT4bOTfC3JGZlt\nWf5ykg/Nfe1lST6e5AFJTkjyJ0lePp12ZpIL11z2HUkeNh0+N8nL1pz+6iSvvpu5HjKd/6i5496f\n5Cc3+No3JLlnkkcnuTbJD81dr9uS/HiSo5P8QpIvJDl6Ov0fJrnfdPipSf5vkpPmrtc3kvz8dN6n\nJrkpyQnT6T+c5KHT4ccn+UqSx0yfn5HkS0lOTXJ8kjeuuU2ekORR0+G/Mn3tj02fn5rk1iR/M8kx\nSX5jmuOJh7tO08dF03numeTYJI/bxtp4WZL/keTE6eNPD96H27y/9+v6+atJTstsQ/7BSS5O8vOb\nuQ1t+bOfXDjGeOeYrfrfS/J9c6eNJL8zxrhyjHFjkn+b5Ce2cNl32VUxxnj+GOP5O574Ti8dY3xt\njPHJzGIxP9vHxhhvGWPcnuQ3kxyX5K9Pc7x5jPGl6fD5SS5J8oNz5712jPHbY4zbp9M/m+RHpq//\nozHGpdPhDyZ5V5K/NZ3vqUleN8a4eIzx1SR32bIfY3xgjPGp6fAnkpyX2Q+EZPYD6Q/HGH8yxrgt\nyb/JLFDz7u46nZbk5CQvmG6P/zfG+NMt3ZIzT88suNePMa5P8tIkz9zC+dfumtqX62eM8b/HGP9z\njHHHGOPyJP8hd95PGxJ/9pNr5g5/NbNdC/Nr+Itzh/8iyf33ZKrN2Wi2Kw4emH6wXZFZIFNVz5p2\nv9xYVTdmtuX3HXPnvXLN/3P53HmfVFUfrqovT+f94bnznrzOTN9SVT9YVe+fds3clOSn5857/zUz\nfzXJl9fMsd51un+SBya5fIxxtw9EbtL9M7uu8/Pv9P7ed+unqh5RVf+tqq6uqpsz+6H1HetdwFri\nz5HkQWsOXzUd/kpmuzaSJFV1vzXn2+qzU74y/Xv83HFrL/Nws81H+5S52Y7KLJBXVdWDM9uSe36S\n+44xTkjyydx1K/MBa/6fB0/nPTbJHyT59STfOZ33j+bOe/U6M817Y5K3JXngGOM+SV4zd96r1sx8\nfA4NznrX6crMIvagqjo6O3NVZrtE5uff6f29H9fPOZnt6vnuMca9k/zrbLLr4s+RopI8r6oeUFX3\nzeyb4LzptIuSPKqqvq+qjstsn/S8a5Js+vntY4zrMvvme2ZVHT09WPpdhznbi6vqnlX1qMz2If/+\n3GmPraqnVNWBJP8iydeTfDjJt2UWluuTHFVVz8lsy3/ed1bVz1XVParqHyV5ZGaRP2b6uD7JHVX1\npCR/d+585yc5s6q+Z4r3WWsu915Jbhxj3FZVp2W2m+WgP0jy96rqcdODrC/LoS25u+v00cx+8Pxq\nVR1fVcdV1d9Y7wabHoR91nqnJXlTZrfpiVV1YpKXJDn42oLt3N/7df3cK7PHX75aVY9M8rObnUP8\n2S/WewbE2ueWvzGz/dp/ntm+8VckyRjjc5kF6j2Z7RO/cM15X5vk1GnXyluSpKpeU1XnbDDPTyV5\nQWZxPTWzBxw3mv0DST4/zfDvxhjvmTvtbUmeluSGJM9I8uPTPvyLM3tg9EOZPeD66MweiJy/3A8n\neXiS65K8PMk/GGPcOMa4NcnPZRb5GzLbR/ytZ6KMMd6Z5JVJ3pfkc0neu+Y2eV6Sl1XVLZnt0//9\nufN+KrPfRt6Y2dbxDbnrbomNrtPtSX40yXdntvvii5k9/nAX0w+V+07Xbz2vSPKxzB6k/fh0eNv3\nd/bv+vlXmf1gviWz3xLPyyZ/E6npEWPY16rq0iTPHWO8b9mzsHNV9bgkzxtjPGOP/r926+fAsgcA\nWGt6BtB2ngXEJtntA9CQ3T4ADdnyB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEa\nEn+AhsR/E6rqzKq6o6oev+xZ9rPpzTnev+w5OrF2d+5IXbfiz15a7w1ZYNUdketW/NlLdfgvgZVz\nRK5b8QdoaOXiX1X3qqpXVNVHquq6qvp6VV1SVb9SVfdc8nj3qKqzq+ryaa6Lquppyxyoqo6pqhdW\n1f+pqq9U1U1V9dGqev4SZzqlqs6vqpunj3dU1eHeoHrfs3a3ZtXWbrd1u4pv4/jAJM9N8uYkv5fk\nm0lOT/LCJI9JcsbSJkt+LcnxSX4ns18Fn5PkTVV13Bjj9Xs9zPQm13+c5AnTv/85ydeTfG+SpyR5\n9RJmuk+SD2Z2P56T5OLM7r/3JVl2ABfN2t2kVVu7LdftGGOlPpLcI8nR6xz/siR3JPmBJcx05vR/\nX5rkL80d/+1JLkvy5STHLWGuF05zvWKd02pJ998vTzM9e83xvzUd/75lzLVH193a3fxcK7V2O67b\nldvtM8b4xhjj9iSpqgNVdUJVnZjkvdOXnLa86XLOGOPWg5+MMW5J8pokJ2S2lbDXnpHkhszichdj\nWrlL8OQkX8psS27ery1hlj1l7W7Jqq3ddut25eKfJFX1vKr6eGa/Bn45ybVJDj7P9oSlDZZ8eoPj\nHrqXg0wenuQzY4zblvB/352HJblk7TfwGONLSW5ezkh7x9rdtFVbu+3W7crt86+qf5nk32e2H/CV\nSa5Kcltm++J+Nyv6AwusXfaTlYt/kmcmuXSM8aT5I6tqmQ+WHXRqkj9c57gk+cIez5Ikn03yPVV1\nzAptQX0hySOq6qgxxh0Hj6yqk5Pce3lj7Qlrd/NWbe22W7eruCXyzSSpqm/NVlUHkrxoaRPd6Wer\n6tsPflJV907yM0luTPKBJczzhsx2Jbx47QlVtawXprwtyUlJnrXm+F9cwix7zdrdvFVbu+3W7Spu\n+b85ya8k+e9V9dbMnpXw9Mx+fV6265J8pKrOzZ1Pl3tgkn8yxvj6Eub57SQ/muTFVfUDSd6d2b7m\nRyV5RJK/s4SZfj2z++s/VtVjc+dT5v5akutzhL5acmLtbt6qrd1+63bZTzda5ylXR2W2pXRJZovh\n0iS/muSRmT3l6iVLmOnMJLcneWKSs5NcPs12UZJ/vOTb69gkv5Tkk0m+ltmW3EeS/MwSZzolyX/N\n7IGym5O8PbMH1C7NEfiUubnrbe1ubbaVWrvd1m1NVxqARlZxnz8ACyb+AA2JP0BD4g/QkPgDNCT+\nAA2JP0BDC3uFb1V5AQG7boyx8FdaWrvstr1Yt1vVast/N18dd9ZZZ+3mKwsB9lSr+AMwI/4ADYn/\nNp1++unLHgFg2xb2h91W8UGzVd2/vrw/vb//eMCX/cgDvgCsBPEHaEj8ARoSf4CGxB+gIfEHaEj8\nARradvyr6oyq+kxVXVJVv7ibQ8EiWbuwzRd5VdXRST6b5G8nuTLJR5P8xBjj03Nfs3IvlPEir/1v\npy+W2a9rl/3tSHqR12lJPj/GuGyM8Y0k5yX5sd0bCxbG2oVsP/4PSPLFuc+vmI6DVWftQrYff78W\ns19Zu5Dtx//KJKfMfX5KZltQsOqsXcj24/+xJA+vqodU1TFJnpbkHbs3FiyMtQvZ5nv4jjG+WVX/\nLMkfJzk6yWvnny0Bq8rahRl/z38FeKrn5vl7/uxHR9JTPQHYx8QfoCHxB2hI/AEaEn+AhsQfoCHx\nB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQf\noCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+A\nhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEa\nOrDICx9jLPLit6yqlj0CwEqw5Q/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+\nAA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgD\nNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BDBxZ54VW1yIvfsjHGskdY16rd\nTsCRz5Y/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ\n+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPi\nD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADR1Y5IWPMRZ58VtWVcseAWAl2PIHaEj8ARoSf4CGxB+g\nIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CG\nxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoS\nf4CGxB+goQOLvPCqWuTFs0BjjGWPcAjrCXaPLX+AhsQfoCHxB2hI/AEaEn+AhsQfoCHxB2hI/AEa\nEn+AhsQfoCHxB2hI/AEaEn+Ahrb9Vz2r6rIktyS5Pck3xhin7dZQsEjWLuzsTzqPJKePMW7YrWFg\nj1i7tLfT3T7+wDr7lbVLazuJ/0jynqr6WFX91G4NBHvA2qW9nez2edwY4+qq+stJ3l1VnxljXLhb\ng8ECWbu0t+34jzGunv69rqremuS0JL6B2DUXXHBBLrjggl2/XGsXktrOe7VW1fFJjh5j3FpV35bk\nXUleOsZ419zXrN6bwLJpq/oevmOMHe2rt3ZZhp2u20XY7pb/SUneOr2h9oEkb5j/5oEVZu1Ctrnl\nv6kLtvW0rx2pW/6b/H9W78qzr63ilr9X+AI0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0\nJP4ADYk/QEPiD9CQ+AM0JP4ADe3kbRwPa9X+LPD0N9zZBLcVHNls+QM0JP4ADYk/QEPiD9CQ+AM0\nJP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ\n+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPi\nD9CQ+AM0dGCRF15Vi7x4FmiMsewRDmE9we6x5Q/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD\n4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2J\nP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BDB5Y9wF4aYyx7\nhHVV1bJHOMQqzgTsHlv+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgD\nNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/Q\nkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNHRg2QPspapa9gjrGmMse4RDrOptBewO\nW/4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ\n+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPi\nD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0dGDZA5BU1bJHYMWNMZY9wiGs2/3Nlj9AQ+IP0JD4AzQk/gAN\niT9AQ+IP0JD4AzQk/gANiT9AQ+IP0JD4AzQk/gANiT9AQxvGv6peV1XXVNUn5o67b1W9u6o+V1Xv\nqqr7LH5M2BprFzZ2uC3/c5Ocsea4FyV59xjjEUneO30Oq8bahQ1sGP8xxoVJblxz9N9P8vrp8OuT\nPHkBc8GOWLuwse3s8z9pjHHNdPiaJCft4jywSNYuTHb0gO+Yvb3Q6r3FEByGtUt324n/NVV1vySp\nqpOTXLu7I8HCWLsw2U7835Hk2dPhZyd52+6NAwtl7cKkNnpj6Kp6U5InJDkxs32kL0ny9iTnJ3lQ\nksuSPHWMcdM65/UrNbtujLGpdw0/0tauN3Df3za7bvfShvHf0QWv4DcQ+99efBOt4toV//1tFePv\nFb4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9CQ+AM0JP4ADYk/QEPiD9DQ\ngWUPAKto1f5+vr+dz26z5Q/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2J\nP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+\nAA2JP0BD4g/QkPgDNCT+AA2JP0BD4g/QkPgDNCT+AA2JP0BDB5Y9AKyiqlr2CLBQtvwBGhJ/gIbE\nH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/\ngIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwB\nGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8Qdo\nSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah\n8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbE\nH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/\ngIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwB\nGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8QdoSPwBGhJ/gIbEH6Ah8Qdo\naMP4V9XrquqaqvrE3HFnV9UVVfVn08cZix8TtsbahY0dbsv/3CRrv0FGkt8cYzxm+njnYkaDHbF2\nYQMbxn+McWGSG9c5qRYzDuwOaxc2tt19/v+8qi6qqtdW1X12dSJYLGsXsr34n5PkoUm+P8nVSX5j\nVyeCxbF2YbLl+I8xrh2TJP8pyWm7PxbsPmsX7rTl+FfVyXOfPiXJJ+7ua2GVWLtwpwMbnVhVb0ry\nhCQnVtUXk5yV5PSq+v7MnjlxaZKfXviUsEXWLmysZr8BL+CCqxZzwbQ2xlj4s3WsXXbbXqzbrfIK\nX4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8\nARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEH\naEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGxB+g\nIfEHaEj8ARoSf4CGxB+gIfEHaEj8ARoSf4CGaoyx7BkA2GO2/AEaEn+AhsQfoCHxB2hI/AEa+v/p\nIOlDI1AtdQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe832e045d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    plot_parallel_tapes(X_short_train[i], Y_short_train[i],\n",
    "                        short_symbols_train['input'][i],\n",
    "                        short_symbols_train['output'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}