graph_backprop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ nan, 0.76])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M = np.array([0, 1])\n",
    "V = np.array([np.NaN, 0.76])\n",
    "Y = np.array([0, 0])\n",
    "# Y = M * V\n",
    "# Y[M == 1] = V[M == 1]\n",
    "M * np.nan_to_num(V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GraphNet:\n",
    "    # Make an empty graph on n nodes with linear activation\n",
    "    def __init__(self, n):\n",
    "        self.size = n\n",
    "        self.adj = np.zeros((n, n))\n",
    "        self.weights = np.zeros((n, n))\n",
    "        self.act = np.asarray(['linear' for i in range(self.size)])\n",
    "        self.input_nodes = np.asarray([0]) # Default\n",
    "        self.output_nodes = np.asarray([n-1]) # Default\n",
    "        self.loss_function = 'SE' # Default squared error -- Other option will be 'CE', cross-entropy\n",
    "        #self.tempstates = [np.zeros(self.size)] # Needs to be dynamically resized\n",
    "        #self.states = [np.zeros(self.size)] # Needs to be dynamically resized\n",
    "        #self.mode = 'overwrite'\n",
    "        #self.time = 0\n",
    "    \n",
    "    # Options are 'SE' and 'CE'\n",
    "    # 'CE' should only be used for binary outputs\n",
    "    def set_loss_function(self, lf):\n",
    "        self.loss_function = lf\n",
    "    \n",
    "    # Sets the mode for the step method\n",
    "    # Options: overwrite, add\n",
    "    def mode(self, m):\n",
    "        self.mode = m\n",
    "    \n",
    "    # Connect an edge from node i to node j\n",
    "    def connect(self, i, j):\n",
    "        self.adj[i,j] = 1\n",
    "    \n",
    "    # Connect a list of nodes in a sequential loop\n",
    "    def connect_loop(self, v):\n",
    "        for i in range(len(v)):\n",
    "            self.adj[v[i], v[(i+1)%len(v)]] = 1\n",
    "    \n",
    "    # Connect in a clique, other than self-connections\n",
    "    def connect_complete(self, v):\n",
    "        for i in range(len(v)):\n",
    "            for j in range(len(v)):\n",
    "                if i == j:\n",
    "                    continue\n",
    "                self.adj[v[i], v[j]] = 1\n",
    "    \n",
    "    # Change activation on node i\n",
    "    # Options: linear, relu, sigmoid/logistic, tanh\n",
    "    def activation(self, i, f):\n",
    "        self.act[i] = f\n",
    "    \n",
    "    # Change activations on a list of nodes\n",
    "    def activations(self, v, f):\n",
    "        for i in range(len(v)):\n",
    "            self.act[v[i]] = f\n",
    "    \n",
    "    # Get the list of node indices for inputs to node i\n",
    "    def inputs(self, i):\n",
    "        return np.nonzero(self.adj[:,i])[0] # Returns a tuple even on a 1D array\n",
    "    \n",
    "    # Get the list of node indices for outputs from node i\n",
    "    def outputs(self, i):\n",
    "        return np.nonzero(self.adj[i,:])[0] # Returns a tuple even on a 1D array\n",
    "    \n",
    "    # Reset weights\n",
    "    def reset_weights(self):\n",
    "        self.weights = np.copy(self.adj)\n",
    "        self.reset_state() # No reason to save the state if the weights were reset.\n",
    "    \n",
    "    # Initialize random edge weights, assuming edge weights are all 0 or 1\n",
    "    def init_random(self):\n",
    "        self.weights = np.random.normal(size=(self.size, self.size)) * self.adj\n",
    "    \n",
    "    # Normalize rows to satisfy Markov state transition rules\n",
    "    def init_markov(self):\n",
    "        self.weights = np.random.random((self.size, self.size)) * self.adj\n",
    "        rowsums = np.sum(self.weights, axis=1, keepdims=True)\n",
    "        rowsums[rowsums == 0] = 1 # Prevent division by zero when normalizing the columns\n",
    "        self.weights = self.weights/rowsums\n",
    "    \n",
    "    def init_uniform(self):\n",
    "        self.weights = np.copy(self.adj)\n",
    "    \n",
    "    # Make the vector of indices be the input nodes\n",
    "    def define_input(self, v):\n",
    "        self.input_nodes = v\n",
    "    \n",
    "    # Make the vector of indices be the output nodes\n",
    "    def define_output(self, v):\n",
    "        self.output_nodes = v\n",
    "        \n",
    "    # numpy activation functions\n",
    "    # could be static, but meh\n",
    "    def relu(self, x):\n",
    "        return x * (x > 0)\n",
    "    \n",
    "    def sigmoid(self, x):\n",
    "        return 1.0/(1.0+np.exp(-x))\n",
    "    \n",
    "    def tanh(self, x):\n",
    "        return (np.exp(2*x) - 1)/(np.exp(2*x) + 1)\n",
    "    \n",
    "    # TODO: Redesign the steps so that multiple steps are done and the array of states is optionally returned,\n",
    "    # so that it doesn't modify the state variable every time it runs the network.\n",
    "    # This will allow for greater thread concurrency.\n",
    "    \n",
    "    # Combine all the old step functions\n",
    "    # Length of the output is n+1 because of the initial state of 0\n",
    "    def step(self, input_vals, n, weights = None):\n",
    "        temp_weights = self.weights\n",
    "        if weights is not None:\n",
    "            temp_weights = weights\n",
    "        x_states = [np.zeros(self.size)]\n",
    "        z_states = [np.zeros(self.size)]\n",
    "        input_vals = np.array(input_vals)\n",
    "        \n",
    "        for t in range(n):\n",
    "            input_state = np.zeros(self.size)\n",
    "            if input_vals.ndim == 1 and t == 0:\n",
    "                # Single value input\n",
    "                input_state[self.input_nodes] = input_vals\n",
    "            elif input_vals.ndim == 2:\n",
    "                # Sequence input\n",
    "                if t < input_vals.shape[0]:\n",
    "                    input_state[self.input_nodes] = input_vals[t]\n",
    "\n",
    "            # Bias neurons need to be activated before the step\n",
    "            input_state[self.act == 'bias'] = 1\n",
    "            \n",
    "            # This feels like it could be really bad.\n",
    "            # This makes it *absolutely mandatory* that there are no connections feeding into the input neurons.\n",
    "            z_states[-1] += input_state\n",
    "\n",
    "            # Activation functions are not run on the input nodes prior to entering the network.\n",
    "            # Preprocessing should take care of that anyway.\n",
    "            # So the inputs are considered the z state for their respective neurons.\n",
    "            # The x states on those neurons are all zero, but it doesn't really matter, \n",
    "            # since they can't come up in the backprop anyway.\n",
    "            \n",
    "            # Weighted sum of inputs (always adding the input state in)\n",
    "            new_x_state = z_states[-1] @ temp_weights\n",
    "            \n",
    "            # Apply activation functions to update the state (temporary states are stored for backprop)\n",
    "            new_z_state = np.copy(new_x_state)\n",
    "\n",
    "            # Applying activation functions in separate sets for each neuron that has a given one\n",
    "            new_z_state[self.act == 'relu'] = self.relu(new_z_state[self.act == 'relu'])\n",
    "            # This is kind of dumb, but initializing with 'linear' means numpy will cut off the string 'sigmoid'\n",
    "            new_z_state[self.act == 'sigma'] = self.sigmoid(new_z_state[self.act == 'sigma']) \n",
    "            new_z_state[self.act == 'tanh'] = self.tanh(new_z_state[self.act == 'tanh'])\n",
    "            \n",
    "            x_states.append(new_x_state)\n",
    "            z_states.append(new_z_state)\n",
    "        return np.array(x_states), np.array(z_states)\n",
    "    \n",
    "    # Simple function to get cleaner output\n",
    "    def get_output(self, input_vals, n_steps, seq_len=1):\n",
    "        x, z = self.step(input_vals, n_steps)\n",
    "        return z[-seq_len:][:, self.output_nodes]\n",
    "    \n",
    "    # Combine all the old error functions (for MSE)\n",
    "    # Error calculation must be separate for backprop, because we don't want to run through the steps multiple times.\n",
    "    # So this is just for model evaluation (for a particular training example)\n",
    "    def get_loss(self, input_sequence, label_sequence, delay, weights=None):\n",
    "        using_weights = False\n",
    "        temp_weights = None\n",
    "        if weights is not None:\n",
    "            using_weights = True\n",
    "            temp_weights = weights\n",
    "        seq_len = 1\n",
    "        steps = delay\n",
    "        label_sequence = np.array(label_sequence)\n",
    "        if label_sequence.ndim == 2:\n",
    "            seq_len = label_sequence.shape[0]\n",
    "            steps = delay + seq_len - 1\n",
    "        x_states, z_states = self.step(input_sequence, steps, temp_weights if using_weights else None)\n",
    "        outputs = z_states[-seq_len:, self.output_nodes]\n",
    "        if self.loss_function == 'SE':\n",
    "            # Squared error\n",
    "            error = np.sum(np.subtract(outputs, label_sequence)**2)\n",
    "            return 0.5*error # 1/2 sum (z-y)^2\n",
    "        else:\n",
    "            # Binary cross-entropy\n",
    "            crossentropy = -np.sum(label_sequence * np.log(outputs) + (1 - label_sequence) * np.log(1 - outputs))\n",
    "            return crossentropy\n",
    "    \n",
    "    def derivative_relu(self, x):\n",
    "        return 1 * (x > 0)\n",
    "    \n",
    "    # Note that this uses Z instead of X\n",
    "    def derivative_sigmoid(self, z):\n",
    "        return z * (1 - z)\n",
    "    \n",
    "    def derivative_tanh(self, x):\n",
    "        return 4/(np.exp(x) + np.exp(-x))**2\n",
    "    \n",
    "    def derivative_activation(self, X, Z, t):\n",
    "        V = np.ones(self.size)\n",
    "        #V[self.act == 'linear'] = 1\n",
    "        V[self.act == 'bias'] = 0\n",
    "        V[self.act == 'relu'] = self.derivative_relu(X[t][self.act == 'relu'])\n",
    "        V[self.act == 'sigma'] = self.derivative_sigmoid(Z[t][self.act == 'sigma']) \n",
    "        V[self.act == 'tanh'] = self.derivative_tanh(X[t][self.act == 'tanh'])\n",
    "        return V\n",
    "    \n",
    "    def backprop(self, input_sequence, label_sequence, delay, debug=False, weights=None):\n",
    "        # Calculate the required number of steps using the delay and the length of the label sequence\n",
    "        # If the label sequence is not two-dimensional, then it's just a single output value\n",
    "        using_weights = False\n",
    "        if weights is not None:\n",
    "            using_weights = True\n",
    "        seq_len = 1\n",
    "        steps = delay\n",
    "        label_sequence = np.array(label_sequence)\n",
    "        if label_sequence.ndim == 2:\n",
    "            seq_len = label_sequence.shape[0]\n",
    "            steps = delay + seq_len - 1\n",
    "        # Outputs at each time step before and after applying activation functions\n",
    "        X, Z = self.step(input_sequence, steps)\n",
    "        # Output labels\n",
    "        Y = np.zeros((seq_len, self.size))\n",
    "        Y[:, self.output_nodes] = label_sequence\n",
    "        # X and Z are steps x size\n",
    "        # Y is seq_len x size\n",
    "        # Square error\n",
    "        E = np.sum((Z[-seq_len:, self.output_nodes] - label_sequence)**2)\n",
    "        # Masking values\n",
    "        M = np.zeros(self.size)\n",
    "        M[self.output_nodes] = 1\n",
    "        # Alias to make things easier to keep track of...\n",
    "        W = self.weights if not using_weights else weights\n",
    "        #if debug:\n",
    "        #    print('W', '\\n'+str(W))\n",
    "        # Going to be dividing by weights, but throw them out entirely if they are zero\n",
    "        # Turns out the bug was that the values are uninitialized when the where argument is used.\n",
    "        # Easy enough to fix. I still think it's weird that the result depended on whether the array was printed or not.\n",
    "        W_recip = np.zeros_like(W)\n",
    "        np.reciprocal(W, out=W_recip, where=(W != 0))\n",
    "        if debug:\n",
    "            print('W recip', '\\n'+str(W_recip))\n",
    "        # Time delay\n",
    "        T = delay\n",
    "        # Column of 1s to perform broadcasting before a dot product\n",
    "        C = np.ones((self.size, 1))\n",
    "        # The gradient for each weight in the adjacency matrix\n",
    "        D = np.zeros((self.size, self.size))\n",
    "        for S in range(0, seq_len):\n",
    "            # Path concatenation matrix: Weights times activations (applied to X, at various time steps)\n",
    "            B_final = W * self.derivative_activation(X, Z, T+S)\n",
    "            if debug:\n",
    "                print('B at', T+S, '\\n'+str(B_final))\n",
    "            # First order term is relatively simple\n",
    "            # Output factor needs to only be calculated for places it should be, otherwise it can end up as nan\n",
    "            output_factor = np.zeros((1, self.size))\n",
    "            output_factor[:, self.output_nodes] = Z[(T+S):(T+S+1),self.output_nodes] - Y[S:(S+1),self.output_nodes] if self.loss_function == 'SE' else -Y[S:(S+1),self.output_nodes]/Z[(T+S):(T+S+1),self.output_nodes] + (1 - Y[S:(S+1),self.output_nodes])/(1 - Z[(T+S):(T+S+1),self.output_nodes])\n",
    "            if debug:\n",
    "                print('output factor','\\n'+str(output_factor)) \n",
    "            # output_factor now contains M anyway, so the factor of M was taken out of here\n",
    "            first_order = Z[(T+S-1):(T+S)].T * W_recip * output_factor * B_final\n",
    "            if debug:\n",
    "                print('first order', '\\n'+str(first_order))\n",
    "            D += first_order\n",
    "            B_prod = B_final.copy()\n",
    "            for N in range(1, T+S):\n",
    "                # Pre-multiply by the previous time's path concatenation matrix\n",
    "                B_prev = W * self.derivative_activation(X, Z, T+S-N)\n",
    "                if debug:\n",
    "                    print('B at', T+S-N, '\\n'+str(B_prev))\n",
    "                # Calculate the next order, lining up the matrix product to get a sum over outputs\n",
    "                # Again, factor of M was taken out because output_factor already has it\n",
    "                next_order = Z[(T+S-N-1):(T+S-N)].T * W_recip * B_prev * ((C @ output_factor) @ B_prod.T)\n",
    "                if debug:\n",
    "                    print('next order', '\\n'+str(next_order))\n",
    "                D += next_order\n",
    "                B_prod = B_prev @ B_prod\n",
    "                if debug:\n",
    "                    print('B_prod', '\\n'+str(B_prod))\n",
    "        # This gradient is calculated for one training example over enough time steps to reach time zero\n",
    "        if debug:\n",
    "            print('D', '\\n'+str(D))\n",
    "        return D, E\n",
    "    \n",
    "    # Example to make sure this makes sense\n",
    "    # T = 1, seq_len = 1\n",
    "    # S = 0, B_1 first\n",
    "    # Z[0:1].T / W * (Z[1:2] - Y[0:1]) * M * B_1\n",
    "    # N loop doesn't activate\n",
    "    #\n",
    "    # T = 2, seq_len = 1\n",
    "    # S = 0, B_2 first\n",
    "    # Z[1:2].T / W * (Z[2:3] - Y[0:1]) * M * B_2\n",
    "    # Z[0:1].T / W * (C @ ((Z[2:3] - Y[0:1]) * M) @ (B_1 @ B2).T)\n",
    "    \n",
    "    # Manually check to see if the gradients line up with what the backprop function gives\n",
    "    def gradient_check(self, input_sequence, label_sequence, delay, epsilon):\n",
    "        backprop_gradient = self.backprop(input_sequence, label_sequence, delay)[0]\n",
    "        manual_gradient = np.zeros((self.size, self.size))\n",
    "        base_err = self.get_loss(input_sequence, label_sequence, delay)\n",
    "        for i in range(self.size):\n",
    "            for j in range(self.size):\n",
    "                if self.weights[i, j] == 0:\n",
    "                    continue\n",
    "                dw = np.zeros((self.size, self.size))\n",
    "                dw[i, j] = epsilon\n",
    "                grad = (self.get_loss(input_sequence, label_sequence, delay, self.weights + dw) - base_err)/epsilon\n",
    "                manual_gradient[i, j] = grad\n",
    "        print('Backprop-calculated gradient:')\n",
    "        print(backprop_gradient)\n",
    "        print('Manually-calculated gradient:')\n",
    "        print(manual_gradient)\n",
    "        print('Difference:')\n",
    "        print(backprop_gradient-manual_gradient)\n",
    "    \n",
    "    # Uses the whole dataset as the batches\n",
    "    def train(self, train_x, train_y, delay, epochs, learning_rate, accelerated=False, momentum=0.9):\n",
    "        errors = []\n",
    "        prev_delta_weights = np.zeros((self.size, self.size))\n",
    "        for ep in range(epochs):\n",
    "            avg_D = np.zeros((self.size, self.size))\n",
    "            avg_E = 0\n",
    "            for k in range(len(train_x)):\n",
    "                temp_weights = None\n",
    "                if accelerated:\n",
    "                    temp_weights = self.weights - momentum * prev_delta_weights\n",
    "                D, E = self.backprop(train_x[k], train_y[k], delay, weights=temp_weights)\n",
    "                avg_D += D\n",
    "                avg_E += E\n",
    "            avg_D /= len(train_x)\n",
    "            avg_E /= len(train_x)\n",
    "            #print(avg_D)\n",
    "            #print(avg_E)\n",
    "            if accelerated:\n",
    "                delta_weights = momentum * prev_delta_weights + learning_rate * avg_D\n",
    "                self.weights += -delta_weights\n",
    "                prev_delta_weights = delta_weights\n",
    "            else:\n",
    "                self.weights += - learning_rate * avg_D\n",
    "            # Evaluate error to see how we're doing.\n",
    "            errors.append(avg_E)\n",
    "        print(self.weights)\n",
    "        return errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1.],\n",
       "       [0., 0.]])"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bintest = GraphNet(2)\n",
    "bintest.connect(0,1)\n",
    "bintest.set_loss_function('CE')\n",
    "bintest.activation(1, 'sigma')\n",
    "bintest.init_markov()\n",
    "bintest.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.73105858]])"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bintest.get_output([1], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backprop-calculated gradient:\n",
      "[[ 0.         -0.26894142]\n",
      " [ 0.          0.        ]]\n",
      "Manually-calculated gradient:\n",
      "[[ 0.         -0.26894044]\n",
      " [ 0.          0.        ]]\n",
      "Difference:\n",
      "[[ 0.0000000e+00 -9.8305617e-07]\n",
      " [ 0.0000000e+00  0.0000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "bintest.gradient_check([[1]], [[1]], 1, epsilon=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "logtest = GraphNet(3)\n",
    "logtest.connect(0,2)\n",
    "logtest.connect(1,2)\n",
    "logtest.activation(2, 'sigma')\n",
    "logtest.activation(1, 'bias')\n",
    "logtest.set_loss_function('CE')\n",
    "logtest.init_random()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_0 = np.random.normal(loc=0, scale=0.6, size=(1000, 1))\n",
    "class_1 = np.random.normal(loc=2, scale=1.4, size=(1000, 1))\n",
    "data = np.concatenate((class_0, class_1))\n",
    "labels = np.concatenate((np.zeros((1000, 1)), np.ones((1000, 1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.         0.         3.2956112]\n",
      " [-0.         0.        -2.3610941]\n",
      " [-0.         0.        -0.       ]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.282552521295103,\n",
       " 0.2637960409312155,\n",
       " 0.23030393521088524,\n",
       " 0.19047245181430902,\n",
       " 0.1563106977139622,\n",
       " 0.13579271770316814,\n",
       " 0.1271986341902479,\n",
       " 0.12395111975840686,\n",
       " 0.12160129948930759,\n",
       " 0.11903965736576233,\n",
       " 0.11662702226477746,\n",
       " 0.11483368947572505,\n",
       " 0.11385367657729771,\n",
       " 0.11363741643054119,\n",
       " 0.11400380434249513,\n",
       " 0.1147300142706393,\n",
       " 0.11560500016787828,\n",
       " 0.11645504985028081,\n",
       " 0.11715247748625915,\n",
       " 0.11761568347330544]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# At least a little faster than the version without accelerated gradient descent, but still slow computationally\n",
    "logtest.train(data, labels, delay=1, epochs=20, learning_rate=0.5, accelerated=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 0., 0.]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = GraphNet(3)\n",
    "test.connect(0,1)\n",
    "test.connect(1,2)\n",
    "test.activations([1,2], 'sigma')\n",
    "test.init_markov()\n",
    "test.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.weights *= 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, z = test.step([1], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.         0.         0.        ]\n",
      " [0.         0.9        0.        ]\n",
      " [0.         0.         0.63985455]]\n",
      "[[1.         0.         0.        ]\n",
      " [0.         0.7109495  0.5       ]\n",
      " [0.         0.5        0.65472058]]\n"
     ]
    }
   ],
   "source": [
    "print(x)\n",
    "print(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.        , -0.01443618,  0.        ],\n",
       "        [ 0.        ,  0.        , -0.05549274],\n",
       "        [ 0.        ,  0.        ,  0.        ]]), 0.11921787702676306)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.backprop([[1]], [[1]], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backprop-calculated gradient:\n",
      "[[ 0.         -0.01443618  0.        ]\n",
      " [ 0.          0.         -0.05549274]\n",
      " [ 0.          0.          0.        ]]\n",
      "Manually-calculated gradient:\n",
      "[[ 0.         -0.01443618  0.        ]\n",
      " [ 0.          0.         -0.05549272]\n",
      " [ 0.          0.          0.        ]]\n",
      "Difference:\n",
      "[[ 0.00000000e+00 -4.35924800e-09  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00 -1.90231157e-08]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "test.gradient_check([[1]], [[1]], 2, epsilon=0.000001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear = GraphNet(3)\n",
    "linear.connect(0,2)\n",
    "linear.connect(1,2)\n",
    "linear.activation(1, 'bias')\n",
    "linear.define_input([0])\n",
    "linear.define_output([2])\n",
    "linear.init_markov() # Makes it really be slope and intercept 1 to begin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x140223a3400>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXFElEQVR4nO3db4xcZ3XH8d/ZyQSPy59JlJUaT+IYqZXTBNdeZZVG8osSQ+NAglnxp2kKqFJf+E2R4rQY2aUiiUSFJauEFyBVUUFtlYi6EHdLKFVKFSNUS0asWYfUjVNFgE0mVCxKNvzxQsa7py92rz07e587d/7cuXdmvh8JaXdmdvOskH56fO55zmPuLgBAcU3kvQAAQDKCGgAKjqAGgIIjqAGg4AhqACi4q7L4pdddd51v27Yti18NACPp9OnTP3X3ybj3Mgnqbdu2aW5uLotfDQAjyczOh96j9AEABUdQA0DBEdQAUHAENQAUHEENAAVHUANAwWXSngcA42J2vq6jT7+glxeXtKVa0cG92zUzVevrf4OgBoAuzc7Xdfj4c1pqLEuS6otLOnz8OUnqa1hT+gCALh19+oXLIR1Zaizr6NMv9PW/w44aABLElTak1ZCuLy7F/szLgde7RVADQEBcaePgl5+VTGosh2/Hqm4u93UdlD4AICCutNFY8cSQlqRf/OqSZufrfVsHQQ0AAd2WMBor3tc6NUENAAFbqpWuf7afdWqCGgACogeH3egl5FsR1AAQMDNVU7US/2CwZCaTVK2UVS7Zuvcq5VJPId+KoAaABA/vu1WVcmnda5VySX/zhzv1gyP36MxDd+noB3aqVq3IJNWqFX36fTv6euCF9jwASBAFbtIx8ZmpWt+PjTcjqAGMvaRDLVnO8EiLoAYwcjoZlJTmUEtWMzzSMvfkxu1uTE9PO5fbAshDa/BKkklyrdaPW0N795FngkfBW9WqFZ08tKfPK15bo9lpd5+Oe4+HiQBGStxpwmg7Gu2Mm08NdtLv3O8ZHmkR1ABGSrswbZ1u18lcjn72RncidVCbWcnM5s3sa1kuCAB6kSZMozCfna/rF7+6tOH9CVPmvdGd6GRH/YCk57NaCAD0w8G92zf0PbeKwvzo0y+osbLxOd2bN5Uz743uRKquDzO7QdI9kv5a0p9nuiIASCHU2dHc9xz3kLB5Zxwqk7y21Mi8N7oTaXfUn5X0cUkroQ+Y2X4zmzOzuYWFhb4sDgDiRJ0d9cUluTY+JJyZqsXurE3S+2+7EsChMkleteiQtkFtZvdK+om7n076nLs/5u7T7j49OTnZtwUCQKs0V2CFuj9OnLuykYwL8zxr0SFpSh+7Je0zs3dL2iTpzWb2uLt/ONulAUC8UMmi+fU0n0lzPLwI2ga1ux+WdFiSzOztkj5GSAPI05ZqJbb+3FyySPMZKfs5Hf1AHzWAoZOmZDEsZY00Opr14e7flPTNTFYCACmlnWjX7jPDgqFMAIZO2qFLw1DWSIOhTACGSqdDl4ZF0lAmdtQActe6Q77z5kmdOLcQu2NOM3RJymccaVZ4mAggV3GHVx4/dSF4mKXToUujgKAGkKu4HXKr5vDtZOjSqCCoAeQqbajWF5c0O1/vaOjSqCCoAeSqk1CN6s+fft8O1dZ+zlo+M6y90kkIagC5SrNDjkQlkJmpmk4e2qMfHrlHj963qzDjSLNC1weAXMUdTLnz5kk9fupC7OdbSyWj0iudhKAGkLu4sD1xbiHVrI5xQOkDQCGN0qyOXrGjBpCL6JBLfXFJJTMtu687WThKszp6RVADyETSPI7WY+DLa6MsWk8WjkP9OQ2CGkDP4o6AP3m6fjmIWwM46ZBLc2cHVlGjBtCTuCPgT5y6kHhVVrtDLqN2srBXBDWAniQNSWoVBXC7zo1x7OxIQlAD6Eknu9/q5rKk5EMu49rZkYSgBtCTTna/0fj7manaumPgJVs9CD6qJwt7xcNEAD05uHf7hkH+Ia8tNS5/TUdHegQ1gJ7E9TtffP2SXr3Y2PBZas/dIagB9Kx1dxx3XRa15+4R1ADaiju8IoVPDXKqsL+43BZAorjdcXnCJJMay1fyo1Iu8SCwB0mX29L1ASDRI0+d3fCgsLHi60JaGs27CouCoAYQNDtfj30oGMKJwmxQowbGXNLwpE53yHR1ZIOgBsZYa/05Gp40d/6V4OD+SLlkG2rUdHVkg9IHMMbi5nQsNZb1+KkLiSFdrZR19AM7R/6uwqJgRw2MsW5qypVySQ/vu5WThQPEjhoYY53WlNk554OgBsZY0hS7OCcP7SGkc0BQA2OseYpdVGveXI6PhWqlPNjF4bK2NWoz2yTpW5LesPb5r7j7Q1kvDED/hVrxWud0HPzys2qsXOnoKE+YHt53ax5LhtLtqH8taY+775S0S9LdZnZHtssC0G9xV2Y9eOyM/mr2uXWfm5mq6egH13d0HP3gTkoeOWq7o/bVYSC/WPu2vPa//g8IAdAXoV1z6MqsJ05d0PRN164LYjo6iiVVe56ZlSSdlvRbkj7v7t+O+cx+SfslaevWrf1cI4AWoTAOHWCRwq14LnHrd8Glepjo7svuvkvSDZJuN7O3xXzmMXefdvfpycnJfq8TwJq4Esbh489dDu/Q7d9JrXjM6Ci2jro+3H1R0jcl3Z3JagC0lRTGocCtLy7p4N7tssDvZEZHsbUNajObNLPq2tcVSe+UdC7rhQGIFwrjqAwSJwroD92xdUNYM6Oj+NLsqK+XdMLMvifpO5K+4e5fy3ZZAEJCYRzVquN2zVEd+lMzO/TofbuY0TFk0nR9fE/S1ADWAoy9pJGjkbhbv6Nd8cxUTQeOnYn93dFOnI6O4cNQJqAgkjo2WlvnpPB9hLVqJXbyHXXo4UVQAwWR9JAwar1Lc1ls0o4bw4mgBgoi6SFh2t128/fcAD46CGqgILYklCza7bZbUYceLUzPAwoibuRopVzSnTdPBm9b4aDKeGBHDeSsufb8lkpZm8oTWrzY0JZqRXfePKknT9eDP8sDwvFAUAM5aq09Ly41VCmX9Oh9uzQzVdPuI89sKHlEeEA4PghqYIBaOzcuvn4psfacVNrgoMr4IKiBAYnr3AiJ3gs9YKxVK4T0GOFhIjAgcZ0bIaa1m1YCDxgpeYwXghoYkE46NJpnRLfeaUjJY/xQ+gAy1FyTnjDTsqe/HInZHIgQ1EBGWmvScSFdKZe0qTyhVy82NrxH6x0iBDWQkVBNumSmFffLR7slMZsDiQhqoE9aD64sLm3cJUvSirt+cOSeDa8zmwMhBDXQB3EHV0LiShrUoZGErg+gD9K23lHSQDfYUQM9iModSYdXmtFah24Q1ECXWssd7XCaEN2i9AF0qZOThpQ80At21EAKcddgpTlpaBJdHOgZQQ20ETdM6cFjZ7T56pJ++Xp4R12rVnTy0J5BLRMjjNIH0EZcicMl/fL1ZZVLFvszlDrQTwQ10EZSieM3rr5KtbW+6JKthjaDk9BvlD6ANkIzoSXptaWGzjx014BXhHFDUGOsxD0UnJmqBV+XVi+dffDYGcXNvWNwEgaBoMbYiHsoePj4c5o7/4qePF3f8Lp05Wj33PlX9MSpC+vCmjo0BsW8g/m4aU1PT/vc3Fzffy/Qi91HnoktYZQS5kTXUu66gV6Z2Wl3n457jx01xkbooWDSMP+43TUwaHR9YGyE6slRt0ZIdCs4kBeCGmMjdFHs/b9344bXW3Vy3yHQb22D2sxuNLMTZva8mZ01swcGsTCg30IXxX5qZsfl10Po7kCe2j5MNLPrJV3v7t81szdJOi1pxt3/J/QzPEzEsIqbiFcplzjAgsz19DDR3X8s6cdrX//czJ6XVJMUDGqgSJpnRkcdHrVA10b0Pd0dKJKO2vPMbJukb0l6m7v/rOW9/ZL2S9LWrVtvO3/+fP9WCXQpaWY0O2UUSdKOOvXDRDN7o6QnJR1oDWlJcvfH3H3a3acnJye7Xy3QB7Pzde0+8owOHDsTnBlNNweGRao+ajMrazWkn3D349kuCehNJzev0M2BYZCm68MkfUHS8+7+meyXBPSmk5tX6ObAMEhT+tgt6SOS9pjZmbX/vTvjdQFdS7tLZlYHhkWaro//0uqNQsBQeEulrMWlRux77bo+gCJi1gdGTuhE+DWby5r/JLOjMXw4Qo6Rs3gxfjcdeh0oOnbUGEpJI0dDN7Lw4BDDih01hk7UfldfXJLryijS2fm6pPDwJR4cYlgR1Bg6ce13zYdXQsOXeHCIYUXpA0MndNFsc1seQ/4xSghqFEpz7fktlbLMVh8CRnVoabVXlItmMU4IahRG69Hv5l7oqA69qTwRG9ImUYPGyCKokbvmMaRJlhrLwaPhLlHqwMgiqJGrTgYoJUm6nQUYdgQ1Biau97mTAUqSVK2U9etLKxtuYKHsgVFGUGMgWnfOUc25k5A2SQ/vu1USN7BgvBDUGIhQ73M0JCmN5jo0wYxxQlCjr0JHu0OjR5fdVSmX1oV4qP2OOjTGFScT0Tez83Ud/Mqz6452H/zKs5qdrwd7nKNTg82nCD90x1aOgANN2FGjbx556qway+v3wo1l1yNPndVD77l1Q006Ct+4U4TTN11LHRpYQ1Cjb14NjBF99WLjcsg+8tTZy597w1Xhf9BxBBy4gqBGz6K6dBq/aqxc/npxqaHDx5+TxMNBIAk1avSkeeRoSLVSltR+6h2AeAQ1etLuwEp5wi73Poc6P9JeRguMK4IaPUkK2Vq1oqMf3Lnu5pU4TL0DklGjRqKkK6+k8LVXtWpFJw/tWffawb3bg50fAMLYUSOo3ZVXUmfXXnHzCtAddtQIavfwr3nA/6byxLoB/6Hwpe0O6BxBjaBQ/bl1oNLiUkOVckmP3reLEAYyQOkDQaGHfCUz2uyAASKoERRXfzYpOO2ONjsgGwQ1gmamanr/bTVZ02surfu+GW12QDYIaiQ6cW5hw8jRuLCmzQ7IDg8TIanzOdKu1fY6ptsB2SOoEbwmS+rsQAuAbLQtfZjZF83sJ2b234NYEAZndr6u3Uee0YFjZ4JdHJ0caAGQjTQ76r+X9DlJ/5jtUtAv7Y59R59pd7lsfXHp8s8xxB/IT9ugdvdvmdm27JeCfkgqYzSHa7upd9Jqv3T0cwQzkB+6PkZM2pnPaXqe094ODiBbfQtqM9tvZnNmNrewsNCvX4sOpZ35nKbnmVu/gWLoW1C7+2PuPu3u05OTk/36tehQ2pnPcQ8Jm/HAECgOSh8jppMujebLZTeXJ3TN5jLjR4ECavsw0cy+JOntkq4zs5ckPeTuX8h6YejeG66auFynvmZzWQ+959Z1oRvX8eGyDZ8DUAxpuj7uH8RC0Lu4AG6+9TuS9MCRoAaKh9LHCOm144Ppd0AxEdQjpNeOD6bfAcVEUI+QXjo+6PIAiouhTEMqOiZeX1xSyUzL7qpWyiqXTI3lKwdV4gKYY+HAcCGoh1DrQ8PoBOHiUkPlCdM1m8ttL5rlWDgwPAjqAgsNV3rkqbPBOR2NFdfmq6/S/CfvGvBqAWSFoC6ouOFKDx47owPHzrT9Wbo3gNHCw8SCimu1Szsiie4NYLQQ1AXV7a6Y7g1g9BDUBdXNrpgZHcBookads9ADw4N7t7e9gSVSKZcIaGCEEdQ5SnMbS9QrbVpfo46+r9EDDYw8gjpH7YYjNfc6p7kHEcBoIqhzFHpgWI95nQMqwPjiYWKOQg8MTas7aACQCOpcHdy7XRbzuksbRpMCGF8EdY5mpmrBQyycLgQQoUadg+YHg9Hku1acLgQQIagHpHksaXOrXVxIc7oQQDOCus/i2ugkreuXjit3lMy04k7rHYANCOo+Ch1g2VSeaHvCcMVdPzhyzyCWCWDIENR9FDrAkuYYODVpACF0ffTJ7Hw99qBKGtSkASRhR90HUckjpFop69eXVtbtrJnVASAtgjqgk9kacSWPiEm6d+f1mr7pWmZ1AOgKQR0jzVS7ZkmHU1zSk6frmr7pWp08tCeT9QIYbQR1jKSpdtH7zTvjLdVKYn26eSIeAHSKh4kxkqbaHT7+nOqLS/Km7++8eVKVcqmr3wkA7RDUa2bn69p95Bm99dC/acLiRiWtHkqJ22mfOLegT79vh2oJLXa03wHoFkGtKzXpaKccOtYd97q0uluemarp5KE9+ux9uzbsrmm/A9ALglrhro2SmUxXLo29ZnM59uerTa/PTNUu766bf5b6NIBu8TBR4fpx67Huh796NvZzrRttbmMB0E+pdtRmdreZvWBmL5rZoawXNWih+nHr668tNWI/F3odAPqhbVCbWUnS5yW9S9Itku43s1uyXtggHdy7fUNd2STdefPkutfSBjoA9FOaHfXtkl509++7++uS/knSe7Nd1mDNTNX0/ttq667Fig6qNN9dGBfoPCgEkLU0QV2T9KOm719ae20dM9tvZnNmNrewsNCv9Q3MiXMLG+ZENx9ykXhQCCAfaR4mhu5fXf+C+2OSHpOk6enp0FWAmelkNkfc50MnC1sfNPKgEMCgpQnqlyTd2PT9DZJezmY53el0Nkfc55uvx2pG/RlA3tKUPr4j6bfN7K1mdrWkP5L01WyX1Zl2sznSfN618Z8O1J8BFEHbHbW7XzKzj0p6WlJJ0hfdPb6hOCehPujo9bRljmg+NKNIARRJqgMv7v51SV/PeC1dC4XvlmqlozJHrVphFCmAwhmJI+RJbXOUOQAMu5EI6qS2uXZlDtrsABTd0M36CLXhhdrmSmaxU+9KZpQ5AAyFoQrqTtvwpPiRpUmvA0DRFDaoW3fOd948qS99+0cbArbdNVe1wIPGpCH/AFAkhaxRtw7yry8u6fFTF4K74PriknYfeWbdXI4I8zkADLtC7qhDg/yThMog0dedHC8HgCIpXFDPztcTb/ROEiqDMJ8DwDArTFDPztf18FfParHHIfzc9g1g1BQiqFu7OdJgiBKAcVGIh4lpatIfvmPrugMqH7pjKw8JAYyFQuyo25UratWKPjWzY8Pr0zddy0NCACOvEEGdNNEuaZfMQ0IA46AQpY+4XmdJumZzmRkcAMZeIXbU9DoDQFghglqijAEAIYUofQAAwghqACg4ghoACo6gBoCCI6gBoOAIagAoOPMMrqQyswVJ5/v+i7N1naSf5r2IHPB3jxf+7uK6yd0n497IJKiHkZnNuft03usYNP7u8cLfPZwofQBAwRHUAFBwBPUVj+W9gJzwd48X/u4hRI0aAAqOHTUAFBxBDQAFR1C3MLOPmZmb2XV5r2UQzOyomZ0zs++Z2b+YWTXvNWXJzO42sxfM7EUzO5T3egbBzG40sxNm9ryZnTWzB/Je0yCZWcnM5s3sa3mvpVsEdRMzu1HSH0i6kPdaBugbkt7m7r8r6X8lHc55PZkxs5Kkz0t6l6RbJN1vZrfku6qBuCTpL9z9dyTdIenPxuTvjjwg6fm8F9ELgnq9RyV9XNLYPGF19/9w90tr356SdEOe68nY7ZJedPfvu/vrkv5J0ntzXlPm3P3H7v7dta9/rtXQGotbOszsBkn3SPq7vNfSC4J6jZntk1R392fzXkuO/lTSv+e9iAzVJP2o6fuXNCaBFTGzbZKmJH0735UMzGe1uvlayXshvSjMVVyDYGb/Kek3Y976hKS/lHTXYFc0GEl/t7v/69pnPqHVfyI/Mci1DZjFvDY2/3oyszdKelLSAXf/Wd7ryZqZ3SvpJ+5+2szenvd6ejFWQe3u74x73cx2SHqrpGfNTFr95/93zex2d/+/AS4xE6G/O2JmfyLpXknv8NFurH9J0o1N398g6eWc1jJQZlbWakg/4e7H817PgOyWtM/M3i1pk6Q3m9nj7v7hnNfVMQ68xDCzH0qadveiT9vqmZndLekzkn7f3RfyXk+WzOwqrT4wfYekuqTvSPpjdz+b68IyZqu7j3+Q9Iq7H8h7PXlY21F/zN3vzXst3aBGjc9JepOkb5jZGTP727wXlJW1h6YflfS0Vh+o/fOoh/Sa3ZI+ImnP2v/HZ9Z2mRgS7KgBoODYUQNAwRHUAFBwBDUAFBxBDQAFR1ADQMER1ABQcAQ1ABTc/wMJB66Xh1p6ewAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scatter = 0.1\n",
    "slope = 0.5\n",
    "intercept = 2\n",
    "# Uniformly scattered points off a line\n",
    "train_lin_x = [[-5 + 0.1*i + scatter*(-1+2*random.random())] for i in range(100)]\n",
    "train_lin_y = [[intercept + slope*(-5+0.1*i) + scatter*(-1+2*random.random())] for i in range(100)]\n",
    "plt.scatter(train_lin_x, train_lin_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear.get_output([0], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.         0.         0.5001095 ]\n",
      " [0.         0.         1.98903566]\n",
      " [0.         0.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "# Estimated slope and intercept are present in the weights of the trained network\n",
    "errs = linear.train(train_lin_x, train_lin_y, delay=1, epochs=100, learning_rate=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x140226950b8>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAYEUlEQVR4nO3deZCc9X3n8fenjxkd6NZwWCdX2ACxOcYy2E5MfKwF6zI5YC2SMjZLSq6svca7rtr1ZrfIxrtbiSsbxwdeKIWbcjnYQLyKI6/jBWxIJQZGsjgFtkwwGhBohG6ENNPT3/3jeXrU0zOjaUk9aj1Pf15VXf308zzT/X30SJ/56de/3/MoIjAzs+wrtLsAMzNrDQe6mVlOONDNzHLCgW5mlhMOdDOznCi164MXLlwYy5cvb9fHm5ll0vr167dHRM9429oW6MuXL6evr69dH29mlkmSfjnRNne5mJnlhAPdzCwnHOhmZjnhQDczywkHuplZTjjQzcxywoFuZpYTmQv0F17by1/8/Qu8se9gu0sxMzuhZC7QfzGwj68/tJnt+wbbXYqZ2Qklc4FeKgiAoeFqmysxMzuxZC7Qy6WkZAe6mdlo2Qv0Qi3Qfes8M7N6kwa6pGmSHpf0pKRnJf3JOPt0S7pX0mZJj0laPhXFApSLSZdLxS10M7NRmmmhHwTeHxHvAC4AVkq6pGGf64GdEXEW8JfAl1pb5iGlYlLyoAPdzGyUSQM9EvvSl+X00djfcSVwV7p8H/ABSWpZlXW60kCvuMvFzGyUpvrQJRUlbQS2AT+MiMcadlkEbAGIiAqwG1gwzvusltQnqW9gYOCoCi4VPcrFzGw8TQV6RAxHxAXAYmCFpPMbdhmvNT6mCR0RayKiNyJ6e3rGveHGpGp96ENVt9DNzOod0SiXiNgF/AhY2bCpH1gCIKkEzAF2tKC+Mcppl8tQxS10M7N6zYxy6ZE0N12eDnwQeL5ht7XAJ9Llq4CHImJKmtC1L0UrVQe6mVm9Zu4pehpwl6QiyS+Ab0fE9yR9EeiLiLXAbcA9kjaTtMxXTVXBtS6XQX8pamY2yqSBHhFPAReOs/7GuuUDwNWtLW18tYlFHoduZjZa9maKeuq/mdm4Mhfohy7O5S4XM7N6mQv0kVEubqGbmY2SuUAvFkRBnilqZtYoc4EOSSvdLXQzs9EyHOhuoZuZ1ctooMstdDOzBpkM9FKx4JmiZmYNMhnoXcUCgxV3uZiZ1ctkoJeKcgvdzKxBJgO9XCx42KKZWYNMBnqpIN+CzsysQSYDvatU8MW5zMwaZDLQSwV5HLqZWYNMBrpnipqZjeVANzPLiYwGuqj4JtFmZqNkMtBLxQKDvkm0mdkomQz0rmLBLXQzswaZDPSSL85lZjZGJgPdM0XNzMbKaKB7pqiZWaNMBnqp4JmiZmaNJg10SUskPSxpk6RnJd0wzj6XSdotaWP6uHFqyk34jkVmZmOVmtinAnw+IjZImgWsl/TDiHiuYb9HI+IjrS9xLN+xyMxsrElb6BGxNSI2pMt7gU3Aoqku7HA8U9TMbKwj6kOXtBy4EHhsnM2XSnpS0vclnTfBz6+W1Cepb2Bg4IiLrSkVRTVg2GPRzcxGNB3okk4C7gc+FxF7GjZvAJZFxDuArwPfHe89ImJNRPRGRG9PT8/R1ky5mJTtVrqZ2SFNBbqkMkmYfzMiHmjcHhF7ImJfurwOKEta2NJK65SLAvBsUTOzOs2MchFwG7ApIr48wT6npvshaUX6vm+0stB6Iy10X8/FzGxEM6Nc3gN8HHha0sZ03R8BSwEi4hbgKuAPJVWAt4BVETFlzedSLdB9o2gzsxGTBnpE/AOgSfa5CbipVUVNpivtcvFYdDOzQzI7UxTwbFEzszqZDPRyyaNczMwaZTPQC+5yMTNrlM1A9zh0M7MxMhnoJX8pamY2RiYDvcstdDOzMTIZ6LVx6L5rkZnZIZkM9NrUf08sMjM7JKOB7qn/ZmaNMh3ovjiXmdkhmQz0Q6Nc3EI3M6vJZKAfGuXiFrqZWU0mA90tdDOzsTIZ6CN96A50M7MR2Qz09GqLg+5yMTMbkclAr3W5uIVuZnZIJgPdF+cyMxsro4Hui3OZmTXKZKBLolSQW+hmZnUyGeiQ9KN7pqiZ2SGZDfRyscCgr+ViZjYi04Fe8dUWzcxGTBrokpZIeljSJknPSrphnH0k6WuSNkt6StJFU1PuIeWiGKq4y8XMrKbUxD4V4PMRsUHSLGC9pB9GxHN1+1wOnJ0+3gXcnD5PmVKh4Ouhm5nVmbSFHhFbI2JDurwX2AQsatjtSuDuSPwEmCvptJZXW6erVPCwRTOzOkfUhy5pOXAh8FjDpkXAlrrX/YwN/ZYqFeSZomZmdZoOdEknAfcDn4uIPY2bx/mRMc1nSasl9UnqGxgYOLJKG5SLBY9DNzOr01SgSyqThPk3I+KBcXbpB5bUvV4MvNq4U0SsiYjeiOjt6ek5mnpHlItyl4uZWZ1mRrkIuA3YFBFfnmC3tcC16WiXS4DdEbG1hXWO4Ra6mdlozYxyeQ/wceBpSRvTdX8ELAWIiFuAdcAVwGZgP3Bd60sdrVQUFbfQzcxGTBroEfEPjN9HXr9PAJ9uVVHNKBcL7B2qHM+PNDM7oXmmqJlZTmQ40D1T1MysXmYDvVT0TFEzs3qZDfQuj3IxMxsls4GezBR1l4uZWU1mA71ccgvdzKxedgO94JmiZmb1shvoxYIvzmVmViezgV4q+vK5Zmb1Mhvo5aIYqlZJJqmamVmGA71ABAxXHehmZpDhQC8Vk8vLVBzoZmZAhgO9q5iUPugvRs3MgAwHeqmQttD9xaiZGZDhQC+XktI9ucjMLJHdQC840M3M6mU30EtJl4vHopuZJTIb6KW0he7ZomZmicwGetmjXMzMRslwoHuUi5lZvQwHur8UNTOrl9lAr80U9ZeiZmaJzAZ6l1voZmajTBrokm6XtE3SMxNsv0zSbkkb08eNrS9zrFIa6BXfKNrMDIBSE/vcCdwE3H2YfR6NiI+0pKIm1b4UHay4y8XMDJpooUfEI8CO41DLESm7hW5mNkqr+tAvlfSkpO9LOq9F73lYHuViZjZaM10uk9kALIuIfZKuAL4LnD3ejpJWA6sBli5dekwfWrvaoke5mJkljrmFHhF7ImJfurwOKEtaOMG+ayKiNyJ6e3p6julzu3y1RTOzUY450CWdKknp8or0Pd841vedjK+HbmY22qRdLpK+BVwGLJTUD/wxUAaIiFuAq4A/lFQB3gJWxXG4c7Ovh25mNtqkgR4R10yy/SaSYY3H1aHrobuFbmYGGZ4pWh6Z+u8WupkZZDjQiyN96A50MzPIcKBLoqtYYNBdLmZmQIYDHZIrLrqFbmaWyHagF0Sl6ha6mRlkPNC7SgXfgs7MLJXpQC8VCu5yMTNLZTrQyyV5HLqZWSrbgV4oeBy6mVkq24FedKCbmdVkOtCTYYvucjEzg4wHernoUS5mZjUZD3S30M3MajIe6O5DNzOryXSgl4oFhjxT1MwMyHigdxXFUMUtdDMzyHiglwoFKlUHupkZZDzQy6WCZ4qamaWyHegF+UtRM7NUtgPdo1zMzEZkOtA9U9TM7JBMB7pnipqZHZLxQHcL3cysZtJAl3S7pG2SnplguyR9TdJmSU9Juqj1ZY7PfehmZoc000K/E1h5mO2XA2enj9XAzcdeVnNKxQKVahDhVrqZ2aSBHhGPADsOs8uVwN2R+AkwV9JprSrwcLqKAvBYdDMzWtOHvgjYUve6P103hqTVkvok9Q0MDBzzB5eKSfmeLWpm1ppA1zjrxm0yR8SaiOiNiN6enp5j/uByGuhDFbfQzcxaEej9wJK614uBV1vwvpMq17pc3EI3M2tJoK8Frk1Hu1wC7I6IrS1430mVCmkL3SNdzMwoTbaDpG8BlwELJfUDfwyUASLiFmAdcAWwGdgPXDdVxTaqtdA9Ft3MrIlAj4hrJtkewKdbVtERqPWhe7aomVnmZ4qmo1zcQjczy3agl0bGobuFbmaW6UDvKvpLUTOzmkwHeq3L5aDvK2pmlu1AX7ZgBgA/e31vmysxM2u/TAf64nnTOWV2N30v7Wx3KWZmbZfpQJdE77L59L10uGuHmZl1hkwHOkDv8nm8uvsAr+56q92lmJm1VfYDfdl8APp+6W4XM+tsmQ/0Xz1tFjO6iqx3t4uZdbjMB3qpWOCCJXPdQjezjpf5QAfoXT6fTVv3sO9gpd2lmJm1TT4Cfdk8qgE/fdmtdDPrXLkI9AuXzqUgPB7dzDpaLgJ91rQy55w6m/XuRzezDpaLQAd45/J5bHh5JxVfqMvMOlRuAv3iZfPYPzjM86/5ui5m1plyE+iXnrmAgmDd08fldqZmZiec3AT6ybOm8ZvnnMx31ve728XMOlJuAh1g1YqlDOw9yEPPb2t3KWZmx12uAv03z+nh5Fnd3PvElnaXYmZ23OUq0EvFAlf3LubhF7axdbevvmhmnSVXgQ7wr3uXUA24r6+/3aWYmR1XTQW6pJWSXpC0WdIXxtn+SUkDkjamjz9ofanNWbZgJu8+cwH39m2hWo12lWFmdtxNGuiSisA3gMuBc4FrJJ07zq73RsQF6ePWFtd5RFatWEr/zrd4dPP2dpZhZnZcNdNCXwFsjogXI2IQ+Gvgyqkt69h8+LxTOHlWNzf/aHO7SzEzO26aCfRFQP2wkf50XaPflfSUpPskLRnvjSStltQnqW9gYOAoym1Od6nIp953Jj95cQeP/7NvfGFmnaGZQNc46xo7p/8WWB4Rbwf+H3DXeG8UEWsiojcient6eo6s0iP0eyuWsvCkLr7+0M+n9HPMzE4UzQR6P1Df4l4MvFq/Q0S8EREH05d/BVzcmvKO3vSuIqt/4wwe/fl2Nvg66WbWAZoJ9CeAsyWdLqkLWAWsrd9B0ml1Lz8KbGpdiUfv99+1jPkzu/j6g26lm1n+TRroEVEBPgP8gCSovx0Rz0r6oqSPprt9VtKzkp4EPgt8cqoKPhIzu0tc/97TefiFAZ7csqvd5ZiZTSlFtGesdm9vb/T19U355+w9MMT7/vxHnNkzk29/6lKk8b4SMDPLBknrI6J3vG25mynaaNa0Mv9p5Tk88dJO7t/wSrvLMTObMrkPdICrL17CRUvn8qfrNrF7/1C7yzEzmxIdEeiFgvjvv3U+O/cP8ud//3y7yzEzmxIdEegA571tDtdeupxvPvYyT/X7C1Izy5+OCXSA//Avf4Wek7r53L0befNgpd3lmJm1VEcF+uxpZb6y6gJe2v4m//W7z9CuET5mZlOhowId4N1nLuSGD/wKf/PTV/iOr5luZjnScYEO8Jn3n8V7zlrAjWuf4YXX9ra7HDOzlujIQC8WxFc+diGzppX5N3c+4dvVmVkudGSgA/TM6uaOT76T3W8N8fHbHmfnm4PtLsnM7Jh0bKADnL9oDrd+opeXd+znk3c+wT6PfDGzDOvoQAe45IwFfOP3LuKZV3Zz3R2Ps2u/W+pmlk0dH+gAHzr3FL666gKe3LKb37n5H9myY3+7SzIzO2IO9NRH3v427rl+Bdv3HuS3//c/+nK7ZpY5DvQ67zpjAQ/823fTXSpw9S3/xK2Pvki16slHZpYNDvQGZ508i7/9d+/lfef08D/+bhPX3v44r+850O6yzMwm5UAfx/yZXaz5+MX86e/8Gut/uZMP/sWP+atHXmSwUm13aWZmE3KgT0AS16xYyrobfp3e5fP4n+s28eGvPMIPnn3N3TBmdkJyoE/i9IUzueO6Fdxx3TspCD51z3pWfvUR7l/fz9CwW+xmduLI/T1FW2louMrfPbWVW378C55/bS+nzO7mdy9azFUXL+aMnpPaXZ6ZdYDD3VPUgX4UIoIfvTDA3f/0Ej/+2QDVgIuWzuXD553Kh849xeFuZlPGgT6Ftu05wAM/fYW1G1/lua17gKSb5tIzF3DJGQu45PT5nDx7WpurNLO8cKAfJ6/seosHN73Ow89vo++lnexNrw1z6uxpnL9oDr+2aA7nnHoSZ508i2ULZlAu+isMMzsyxxzoklYCXwWKwK0R8WcN27uBu4GLgTeAj0XES4d7zzwGer3KcJXntu7hiZd28nT/Lp5+ZTcvbn+T2h93qSAWz5vOkvkzWDJ/Bm+bM41TZk/j1DnT6JnVzcKTupk3o4tiQe09EDM7oRwu0EtN/HAR+AbwIaAfeELS2oh4rm6364GdEXGWpFXAl4CPHXvp2VUqFnj74rm8ffHckXVvHqzwi4F9bN6WPF7esZ8tO/bz/ae3snP/0Jj3kGDO9DJzp5eZM6OL2dNKzJ5WZta0EjO7S8zsKjK9q8SMriLTy0W6ywWmlYt0lwp0l4p0lQp0FQuUS6JcLFAuFCgVlTwKBYoFUSqIYu0hUfAvELPMmjTQgRXA5oh4EUDSXwNXAvWBfiXw39Ll+4CbJCl8085RZnaXxoR8zYGhYV7fc4Ctuw/wxr5B3njzINv3HmTn/iF2vzXEzv2D7D1Q4dVdb7H3QIX9g8O8OVhhKv6EC0puAiKlIS8oSJA+F5SM00+yX6Sb0uf614d+OUjJI/2JUetHlutqGPWz9cVN8PummV9D9e95ojjxKrLj4WPvXMIf/PoZLX/fZgJ9EbCl7nU/8K6J9omIiqTdwAJge/1OklYDqwGWLl16lCXn07RykWULZrJswcymfyYiODBUZf9ghQOVKm8NDnNgaJjB4SoHh6oMDlcZqqTPw1WGhoPKcJWhajA8XKVSDYarwXAEw8PJczWgWq0tBxEwXE2ek9dBkCxXg/QXSrI9AqK2DCO/bIJ0BSNPI/WPLI9azwTrx//t1dTvtBOwaREnYlF2XCw8qXtK3reZQB+vEdH4N7GZfYiINcAaSPrQm/hsOwxJTO8qMr2r2O5SzOwE0Mwwi35gSd3rxcCrE+0jqQTMAXa0okAzM2tOM4H+BHC2pNMldQGrgLUN+6wFPpEuXwU85P5zM7Pja9Iul7RP/DPAD0iGLd4eEc9K+iLQFxFrgduAeyRtJmmZr5rKos3MbKxm+tCJiHXAuoZ1N9YtHwCubm1pZmZ2JDxV0cwsJxzoZmY54UA3M8sJB7qZWU607WqLkgaAXx7ljy+kYRZqh+jE4+7EY4bOPO5OPGY48uNeFhE9421oW6AfC0l9E11tLM868bg78ZihM4+7E48ZWnvc7nIxM8sJB7qZWU5kNdDXtLuANunE4+7EY4bOPO5OPGZo4XFnsg/dzMzGymoL3czMGjjQzcxyInOBLmmlpBckbZb0hXbXMxUkLZH0sKRNkp6VdEO6fr6kH0r6efo8r921TgVJRUk/lfS99PXpkh5Lj/ve9DLOuSFprqT7JD2fnvNLO+FcS/r36d/vZyR9S9K0PJ5rSbdL2ibpmbp1455fJb6W5ttTki46ks/KVKDX3bD6cuBc4BpJ57a3qilRAT4fEb8KXAJ8Oj3OLwAPRsTZwIPp6zy6AdhU9/pLwF+mx72T5KbkefJV4P9GxL8A3kFy7Lk+15IWAZ8FeiPifJJLc9duMJ+3c30nsLJh3UTn93Lg7PSxGrj5SD4oU4FO3Q2rI2IQqN2wOlciYmtEbEiX95L8A19Ecqx3pbvdBfxWeyqcOpIWA/8KuDV9LeD9JDcfh5wdt6TZwG+Q3FOAiBiMiF10wLkmuXz39PQuZzOAreTwXEfEI4y9g9tE5/dK4O5I/ASYK+m0Zj8ra4E+3g2rF7WpluNC0nLgQuAx4JSI2ApJ6AMnt6+yKfMV4D8C1fT1AmBXRFTS13k752cAA8AdaTfTrZJmkvNzHRGvAP8LeJkkyHcD68n3ua430fk9pozLWqA3dTPqvJB0EnA/8LmI2NPueqaapI8A2yJiff3qcXbN0zkvARcBN0fEhcCb5Kx7ZTxpn/GVwOnA24CZJN0NjfJ0rptxTH/fsxbozdywOhcklUnC/JsR8UC6+vXaf7/S523tqm+KvAf4qKSXSLrT3k/SYp+b/rcc8nfO+4H+iHgsfX0fScDn/Vx/EPjniBiIiCHgAeDd5Ptc15vo/B5TxmUt0Ju5YXXmpf3GtwGbIuLLdZvqb8b9CeD/HO/aplJE/OeIWBwRy0nO7UMR8fvAwyQ3H4ecHXdEvAZskXROuuoDwHPk/FyTdLVcImlG+ve9dty5PdcNJjq/a4Fr09EulwC7a10zTYmITD2AK4CfAb8A/ku765miY3wvyX+zngI2po8rSPqTHwR+nj7Pb3etU/hncBnwvXT5DOBxYDPwHaC73fW1+FgvAPrS8/1dYF4nnGvgT4DngWeAe4DuPJ5r4Fsk3xMMkbTAr5/o/JJ0uXwjzbenSUYBNf1ZnvpvZpYTWetyMTOzCTjQzcxywoFuZpYTDnQzs5xwoJuZ5YQD3cwsJxzoZmY58f8BFJXL1rkPMy4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Seems to work decently well\n",
    "plt.plot(errs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backprop-calculated gradient:\n",
      "[[ 0.          0.          0.271837  ]\n",
      " [ 0.          0.         -0.05490077]\n",
      " [ 0.          0.          0.        ]]\n",
      "Manually-calculated gradient:\n",
      "[[ 0.          0.          0.27184926]\n",
      " [ 0.          0.         -0.05490027]\n",
      " [ 0.          0.          0.        ]]\n",
      "Difference:\n",
      "[[ 0.00000000e+00  0.00000000e+00 -1.22583089e-05]\n",
      " [ 0.00000000e+00  0.00000000e+00 -5.00004597e-07]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "linear.gradient_check(train_lin_x[0], train_lin_y[0], delay=1, epsilon=0.000001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing a regular NN as a graph\n",
    "xor_net = GraphNet(6)\n",
    "xor_net.define_input([0,1])\n",
    "xor_net.connect(0,3)\n",
    "xor_net.connect(0,4)\n",
    "xor_net.connect(1,3)\n",
    "xor_net.connect(1,4)\n",
    "xor_net.connect(2,3)\n",
    "xor_net.connect(2,4)\n",
    "xor_net.connect(2,5)\n",
    "xor_net.connect(3,5)\n",
    "xor_net.connect(4,5)\n",
    "xor_net.activation(2, 'bias')\n",
    "xor_net.activations([3,4,5], 'sigma')\n",
    "xor_net.init_random()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.          0.         -0.          5.06013291 -3.86322932  0.        ]\n",
      " [-0.         -0.         -0.         -5.16692561  3.59482771 -0.        ]\n",
      " [-0.         -0.         -0.         -3.14425786 -1.98745604 -2.68042021]\n",
      " [-0.          0.          0.          0.         -0.          5.51608361]\n",
      " [-0.          0.          0.         -0.         -0.          5.42085205]\n",
      " [-0.         -0.          0.          0.          0.          0.        ]]\n"
     ]
    }
   ],
   "source": [
    "train_x = [[0,0], [0,1], [1,0], [1,1]]\n",
    "train_y = [[0], [1], [1], [0]]\n",
    "errs_xor = xor_net.train(train_x, train_y, delay=2, epochs=5000, learning_rate=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x14022712f60>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU9b3/8ddnJisEQgJhSwIJgmhAWRxAAcUFFa1i64ptFaut11auVntvf3rbe9trf4/fr7Wt1/bqtWLV2qpF3NG2onUXZAmrgCIxbAHZd7In3/vHHOwQAwyQ5MycvJ+Pxzxmzvd8z8znm8fwnsP3nDljzjlERCS4Qn4XICIirUtBLyIScAp6EZGAU9CLiAScgl5EJOBS/C6gqW7durmioiK/yxARSSoLFizY5pzLa25dXEFvZhOA3wBh4PfOuZ83WX8LcCvQAOwDbnbOrfDW3Q3c5K27zTk383CvVVRURGlpaTxliYiIx8zWHmrdEaduzCwMPAhcBJQA15pZSZNuTzvnTnHODQXuBe7zti0BJgGDgAnA/3jPJyIibSSeOfqRQJlzrtw5VwtMAy6L7eCc2xOz2BE48C2sy4Bpzrka59xqoMx7PhERaSPxTN3kA+tjliuAUU07mdmtwJ1AGnBuzLZzmmyb38y2NwM3A/Tp0yeeukVEJE7x7NFbM21fum6Cc+5B59wJwP8BfnyU2051zkWcc5G8vGaPJYiIyDGKJ+grgMKY5QJg42H6TwO+eozbiohIC4sn6OcDA8ys2MzSiB5cnRHbwcwGxCx+BVjlPZ4BTDKzdDMrBgYA846/bBERidcR5+idc/VmNgWYSfT0ysecc8vN7B6g1Dk3A5hiZuOBOmAnMNnbdrmZTQdWAPXArc65hlYai4iINMMS7TLFkUjEHct59Ptq6pn6XjnnDMxjWJ+cVqhMRCRxmdkC51ykuXWBuQRCXX0jv31zFYvX7/K7FBGRhBKYoM9IjX4Pq7qu0edKREQSS2CCPj0lOpSqOh0CEBGJFZigD4WMjNQQ1Qp6EZGDBCboITp9o6AXETlYoII+MzVMVa2CXkQkVuCCvrpeB2NFRGIFKugztEcvIvIlAQt6HYwVEWkqUEGfmaaDsSIiTQUr6FPDOo9eRKSJQAV9uk6vFBH5kkAFfWZqWJdAEBFpInBBr6kbEZGDBSroM1JDOr1SRKSJQAV9VnoqVXUN1Ddo+kZE5IBABX12ZvQHs/ZU1/tciYhI4ghU0HfOTAVgT1Wdz5WIiCSOQAV99oGgr1bQi4gcEKigP7BHv1t79CIiXwhU0H+xR1+lOXoRkQMCFfSdM7RHLyLSVKCCvkuHaNDvrKz1uRIRkcQRqKDPSA2T0yGVjbuq/C5FRCRhBCroAXp3yVTQi4jECFzQ98rOZIOCXkTkC4EL+hPyOrJmWyU19brmjYgIxBn0ZjbBzFaaWZmZ3dXM+jvNbIWZLTWzN82sb8y6BjNb7N1mtGTxzRlS2IXahkaWb9zT2i8lIpIUjhj0ZhYGHgQuAkqAa82spEm3RUDEOXcq8Bxwb8y6KufcUO82sYXqPqRRxbmkho2XFm1o7ZcSEUkKKXH0GQmUOefKAcxsGnAZsOJAB+fc2zH95wDfbMkij0bXrHSuPK2QP364lnU7KundJZOMlDAhAzMws+g9B+45eNnsy218eduQQchbDpm3HDLM/rEu5G0Tsub6x6732kJH1z8cMlLDIVLCRpp3nxoOkRoKkZpipIRCpIajNYlI+xVP0OcD62OWK4BRh+l/E/C3mOUMMysF6oGfO+dearqBmd0M3AzQp0+fOEo6vJ9cWkLnzBTe+ngLyzbsprqukUbncA4c3n3sY8A5590f98snnJSQ/eNDIBwN/9RwiIzUMJmpYTLTovcd0r68nJEWpmNaCp0zU8jOTPVuaV88TksJ3GEekcCJJ+ib2x1sNg7N7JtABBgX09zHObfRzPoBb5nZR865zw56MuemAlMBIpHIcUdtRmqYuy86mbsvOvmYtneu+Q+Aph8SjY7oB0hj9D56i25zYN2BD5jGmLaD1jfSpE/8z1HX4KhvcNQ3NlLX4KhraKS+oZHaBkd9QyN1DTHtjY7a+kbqGxupb4g+rq5voKq2gcraBnZV1rJxV/RxdV30Pp5f68pMDZOdmUrXrDS6d0qne6cMundOp3undPI6ZdCjczr5OZnkZaXrfxYiPokn6CuAwpjlAmBj005mNh74ETDOOVdzoN05t9G7Lzezd4BhwGdNt08kB6ZovCU/S/GVc47qukb219azu6rui9ueA48ro/e7qurYvq+GLXtrWLZxD9v31dDY5OM6IzVEn9wO9Mnt6N1nckL3LAb27KQPAZFWFk/QzwcGmFkxsAGYBHw9toOZDQMeBiY457bEtOcAlc65GjPrBozh4AO1ksDMLDqNkxamW1Z63NvVNzSyfX8tW/bUsGVvNRU7q1i3ozJ6217JrLJtB/1vIbdjGgN7dGJgz06U9O7M8D5d6Ncti1BI4S/SEo4Y9M65ejObAswEwsBjzrnlZnYPUOqcmwH8EsgCnvX2zNZ5Z9icDDxsZo1Ez/D5uXNuRbMvJIGREg7Ro3MGPTpnANlfWu+cY+u+Gso27+OTTXtZuWkvn2zeyzPz13/xAdA5I4WhfXIY3qcLI4tyOa0oh/SUcBuPRCQYzCXY0cdIJOJKS0v9LkN80NjoKN+2n4XrdrJo3U4WrdvFys17cS56LGBUv1zG9u/GuBPz6N89S9M9IjHMbIFzLtLsOgW9JLI91XXMK9/BB2XbeG/VVsq37gegX7eOTBjck4tP6cWg3p0V+tLuKeglMDbsquLtT7bw2rJNfFi+nYZGR2FuJl8bms9VkUIKczv4XaKILxT0Ekg79tfy9xWbeWXpRj4o24ZzMKZ/V66OFDJhcE/N6Uu7oqCXwNuwq4rnF1Tw7IL1rN9RRbesdK4/oy/fGNWHrkdxxpBIslLQS7vR2OiY9dk2HvtgNW+v3Ep6SojLh+dz81knUNyto9/libSawwV9POfRiySNUMg4c0AeZw7Io2zLXh6btYbnF1QwvbSCrw7N57bz+tO3qwJf2hft0UvgbdlbzcPvlvPknLXUNzouH5bPbecN0IFbCRRN3YgAW/ZU89C7n/HU3HUA3DimmO+dcwKdM1J9rkzk+CnoRWJ8vruKX85cyQsLN9C1Yxp3nH8ik0YUkhLWlTgleR0u6PXOlnanV3Ym9109lFemjKV/9yx+/NIyLv7t+8wp3+53aSKtQkEv7dYpBdlMu/l0Hr7uNCprG5g0dQ53Tl/Mtn01R95YJIko6KVdMzMuHNSTN+4Yx5Rz+vPKko2c+6t3eHLOWhqaXmtZJEkp6EWAzLQw/3LhQP52+1kM6p3Nj19axuUPzWaFfmReAkBBLxKjf/csnv7OKH4zaSgbdlYy8YEP+NXMlVTH8WtbIolKQS/ShJlx2dB83rhjHJcNzeeBt8v4ym/fZ/6aHX6XJnJMFPQih5DTMY1fXz2EJ24cSXVdI1f97kP+4+Vl7Kup97s0kaOioBc5gnEn5vH6HWdxw+gi/jRnLRfc9y5vf7LlyBuKJAgFvUgcOqan8NOJg3jultF0SE/hW3+Yz/enLWLH/lq/SxM5IgW9yFE4rW8Of7ltLLedN4BXl37O+Pve5eXFG0i0b5iLxFLQixyl9JQwd55/Iq/eNpbC3A7cPm0xkx+fz7rtlX6XJtIsBb3IMTqpZ2de+O5ofnppCQvW7OD8/3qX/3mnjLqGRr9LEzmIgl7kOIRDxg1jivn7D8ZxzsDu3PvaSi757QcsWKtTMSVxKOhFWkCv7Ex+d91pPHJ9hL3VdVzx0If86MWP2F1V53dpIgp6kZZ0fkkP3rhzHDeNLebP89Zx3q/f5dnS9TTqujniIwW9SAvrmJ7Cv19SwowpYynMzeRfn1vKV/9nFgvW7vS7NGmnFPQirWRwfjbP3zKa/7pmCJv3VHPFQ7P5/rRFfL67yu/SpJ1R0Iu0olDI+NqwAt76wdlMOac/f122iXN/9S73//1TXUpB2kxcQW9mE8xspZmVmdldzay/08xWmNlSM3vTzPrGrJtsZqu82+SWLF4kWXRMT+FfLhzIm3eO45yT8rj/76s46963eeyD1dTU68qY0rqO+JuxZhYGPgXOByqA+cC1zrkVMX3OAeY65yrN7LvA2c65a8wsFygFIoADFgCnOecOOVmp34yV9mDx+l38cuYnzCrbTn6XTG4fP4DLh+Xrd2vlmB3vb8aOBMqcc+XOuVpgGnBZbAfn3NvOuQNfC5wDFHiPLwTecM7t8ML9DWDCsQxCJEiGFnbhqW+fzpM3jaJrVho/fG4pF97/Hi8srKBeX7iSFhZP0OcD62OWK7y2Q7kJ+NvRbGtmN5tZqZmVbt26NY6SRIJh7IBuvHzrGB76xnBSQiHunL6Ec3/9Ln+et05TOtJi4gl6a6at2fkeM/sm0WmaXx7Nts65qc65iHMukpeXF0dJIsFhZlx0Si/+dvuZTL3uNLp0SOXuFz7i7F++w+OzVlNVq8CX4xNP0FcAhTHLBcDGpp3MbDzwI2Cic67maLYVkegZOhcM6snLt47hjzeOpCAnk/98ZQVn/PxN7n3tEzbvqfa7RElS8RyMTSF6MPY8YAPRg7Ffd84tj+kzDHgOmOCcWxXTnkv0AOxwr2kh0YOxh7wQiA7GivzD/DU7+P375by+YjMpIeOSU3tz09hiBudn+12aJJjDHYxNOdLGzrl6M5sCzATCwGPOueVmdg9Q6pybQXSqJgt41swA1jnnJjrndpjZz4h+OADcc7iQF5GDjSjKZURRLuu2V/L47NVMn7+eFxdtYGRxLjeNLWb8yT0Ih5qbIRX5hyPu0bc17dGLHNqe6jqembeeP8xew4ZdVRTmZjL5jCKuHlFI54xUv8sTHx1uj15BL5KE6hsaeX3FZh6ftZr5a3bSIS3MlacVcMPoIvrlZfldnvhAQS8SYMs27OaxWat5dcnn1DY0cs7APL41ppgzB3TDm0qVdkBBL9IObN1bw1Nz1/LknHVs21dD/+5Z3DC6iMuH59Mh7YiH4yTJKehF2pGa+gb+svRzHp+1ho827KZzRgrXjuzD9aOLyO+S6Xd50koU9CLtkHOOBWt38visNby2fBPOOSYM7sm3xhQT6ZujaZ2AOa7TK0UkOZkZkaJcIkW5bNhVxZ8+XMuf563jrx9tYnB+Z741uphLhvQiPSXsd6nSyrRHL9KOVNU28OKiDTw+azWrtuyjd3YGt48fwBXDC3TlzCSnqRsROYhzjvdXbeO+Nz5l8fpdFHfryB3nn8glp/QipC9gJaXjvUyxiASMmXHWiXm8+L3RPHJ9hLRwiNv+vIjLH5rNsg27/S5PWpiCXqQdMzPOL+nB324/k19dNYSKnVVc+sAH/PtLy9hdWed3edJCFPQiQihkXHlaAW/+YByTzyjiqblrueD+d3l/lX4fIggU9CLyhezMVH46cRAzpoylU0Yq1z06j5/OWE51na6Jn8wU9CLyJYPzs3n1n8dy45hi/jB7DVf97kM27qryuyw5Rgp6EWlWRmqY/7i0hN9fH2H1tv1c+t8fMLd8u99lyTFQ0IvIYY0v6cFLt44hu0N0Kmfm8k1+lyRHSUEvIkfUv3sWL3x3NIPyO/PdJxcwvXS93yXJUVDQi0hcunRI48mbRjGmfzd++NxSXlhY4XdJEicFvYjErWN6Co9cH2H0CV351+eW8rqmcZKCgl5EjkpGapip10cYnJ/NlD8vYsHanX6XJEegoBeRo5aVnsIfbhhBr+wMbnlyAZt2V/tdkhyGgl5EjklOxzQeuT5CZU09//TkAn2pKoEp6EXkmJ3YoxP3XTOUJet3ce9rK/0uRw5BQS8ix+XCQT2ZfEZfHpu1mvc+1bVxEpGCXkSO290Xn8yA7ln8y7NL2Lm/1u9ypAkFvYgct4zUMPdPGsqO/bX8v79+7Hc50oSCXkRaxKDe2Xz7zH48u6BC18RJMAp6EWkxt583gIKcTP7txY+oqddZOIkirqA3swlmttLMyszsrmbWn2VmC82s3syubLKuwcwWe7cZLVW4iCSezLQwP7tsMJ9t3c8fZ6/1uxzxHDHozSwMPAhcBJQA15pZSZNu64AbgKebeYoq59xQ7zbxOOsVkQR3zkndOevEPP77rVXsqtSB2UQQzx79SKDMOVfunKsFpgGXxXZwzq1xzi0FGluhRhFJMndfdBJ7a+p54K0yv0sR4gv6fCD2mqQVXlu8Msys1MzmmNlXm+tgZjd7fUq3btV5uCLJ7uRenblyeAF//HAt63dU+l1OuxdP0Fszbe4oXqOPcy4CfB2438xO+NKTOTfVORdxzkXy8vKO4qlFJFH94IKBYGivPgHEE/QVQGHMcgGwMd4XcM5t9O7LgXeAYUdRn4gkqZ7ZGVw7opDnF1ZQsVN79X6KJ+jnAwPMrNjM0oBJQFxnz5hZjpmle4+7AWOAFcdarIgkl1vOPoGQGQ+985nfpbRrRwx651w9MAWYCXwMTHfOLTeze8xsIoCZjTCzCuAq4GEzW+5tfjJQamZLgLeBnzvnFPQi7USv7EyuihQwvXQ9G3dV+V1Ou2XOHc10e+uLRCKutLTU7zJEpIVU7Kzk7F++wzdP78tPJw7yu5zAMrMF3vHQL9E3Y0WkVRXkdODSIb15tnQ9e6rr/C6nXVLQi0iru2lsMftrG3hm3vojd5YWp6AXkVY3OD+bUcW5/GH2Guob9L3KtqagF5E28e0z+7FhVxWvLd/kdyntjoJeRNrEeSd1p6hrB37//mq/S2l3FPQi0iZCIWPy6CIWr9/Fsg27/S6nXVHQi0ibuXx4ARmpIZ6au87vUtoVBb2ItJnszFQmDunNy4s3sFenWrYZBb2ItKlvjOpLZW0DLy3a4Hcp7YaCXkTa1KkF2QzO78xTc9eRaN/MDyoFvYi0KTPjG6P68smmvSxct9PvctoFBb2ItLmJQ3rTKT2Fp+booGxbUNCLSJvrmJ7CxKG9+euyz3X9mzagoBcRX1wdKaS6rpFXl3zudymBp6AXEV+cWpDNwB6dmF6qC521NgW9iPjCzLgqUsDi9btYtXmv3+UEmoJeRHzz1WH5pISMZxdU+F1KoCnoRcQ33bLSOe/k7rywsII6Xb641SjoRcRXV0cK2bavlrc/2eJ3KYGloBcRX407MY+8TumavmlFCnoR8VVKOMTlw/N565MtbNlb7Xc5gaSgFxHfXXVaIQ2NThc6ayUKehHxXf/uWZzWN4fppRW60FkrUNCLSEK4OlJA2ZZ9LFq/y+9SAkdBLyIJ4Sun9qZDWpjp8/VN2ZamoBeRhJCVnsJXTunFK0s2Ullb73c5gaKgF5GEcfWIQvbXNvCXpbrQWUuKK+jNbIKZrTSzMjO7q5n1Z5nZQjOrN7Mrm6ybbGarvNvklipcRIIn0jeHfnkddaGzFnbEoDezMPAgcBFQAlxrZiVNuq0DbgCebrJtLvATYBQwEviJmeUcf9kiEkRmxtWRQuav2Un51n1+lxMY8ezRjwTKnHPlzrlaYBpwWWwH59wa59xSoOnFKi4E3nDO7XDO7QTeACa0QN0iElCXD88nHDKml+qbsi0lnqDPB2L/H1XhtcUjrm3N7GYzKzWz0q1bt8b51CISRN07ZXDOwO48v7CCel3orEXEE/TWTFu832iIa1vn3FTnXMQ5F8nLy4vzqUUkqK6OFLB1bw3vrNSOX0uIJ+grgMKY5QJgY5zPfzzbikg7dc5J3emWlc4zOijbIuIJ+vnAADMrNrM0YBIwI87nnwlcYGY53kHYC7w2EZFDSg2HuEIXOmsxRwx651w9MIVoQH8MTHfOLTeze8xsIoCZjTCzCuAq4GEzW+5tuwP4GdEPi/nAPV6biMhhXRWJXujsxYW60NnxskS7gFAkEnGlpaV+lyEiCeCKh2azq7KWv985DrPmDvnJAWa2wDkXaW6dvhkrIgnrmkghn23dz/w1O/0uJakp6EUkYV06pDedM1L444dr/C4lqSnoRSRhZaaFuTpSyGvLNrFljw7KHisFvYgktG+e3pf6RsfT89b5XUrSUtCLSEIr6taRcSfm8fTcddTpm7LHREEvIgnv+jP6smVvDa8v3+x3KUlJQS8iCe/sgd0pyMnkiQ/X+F1KUlLQi0jCC4eM607vy7zVO1i+cbff5SQdBb2IJIVJI/rQIS3MI++V+11K0lHQi0hSyO6QyrUj+/DK0s+p2FnpdzlJRUEvIknjxrHFADz2wRp/C0kyCnoRSRr5XTKZOKQ30+avY3dlnd/lJA0FvYgkle+c2Y/K2gaenLvW71KShoJeRJJKSe/OjDsxj0c/WM3+mnq/y0kKCnoRSTq3jx/Ajv21Oq8+Tgp6EUk6w/vkcM7APKa+V87eas3VH4mCXkSS0h3nn8iuyjoen7XG71ISnoJeRJLSqQVdGH9yDx55v1xn4ByBgl5EktYd5w9gX009D75T5ncpCU1BLyJJa1DvbK4YXsDjs1azdvt+v8tJWAp6EUlq/3rhQFLDIf7/Xz/xu5SEpaAXkaTWo3MGt4w7gdeWb2JO+Xa/y0lICnoRSXrfObMfvbMz+MnLy6mt169QNaWgF5Gkl5kW5p7LBrNy814eeV+XMW5KQS8igTC+pAcXn9KT37y5itXbdGA2loJeRALjp5cOIj0lxL+98BGNjc7vchKGgl5EAqN75wx+dPHJfFi+ncdmrfa7nIQRV9Cb2QQzW2lmZWZ2VzPr083sGW/9XDMr8tqLzKzKzBZ7t9+1bPkiIge7ZkQh55f04N7XVrJi4x6/y0kIRwx6MwsDDwIXASXAtWZW0qTbTcBO51x/4L+AX8Ss+8w5N9S73dJCdYuINMvM+MUVp9KlQyq3T1tEVW2D3yX5Lp49+pFAmXOu3DlXC0wDLmvS5zLgCe/xc8B5ZmYtV6aISPxyO6bx66uHsGrLPn780jKca9/z9fEEfT6wPma5wmtrto9zrh7YDXT11hWb2SIze9fMzmzuBczsZjMrNbPSrVu3HtUARESac+aAPL4/fgDPL6zgD7PX+F2Or+IJ+ub2zJt+PB6qz+dAH+fcMOBO4Gkz6/yljs5Ndc5FnHORvLy8OEoSETmy284dwPklPfi/f/mY2Z9t87sc38QT9BVAYcxyAbDxUH3MLAXIBnY452qcc9sBnHMLgM+AE4+3aBGReIRCxn1XD6G4W0du+dMCPt281++SfBFP0M8HBphZsZmlAZOAGU36zAAme4+vBN5yzjkzy/MO5mJm/YABgL62JiJtplNGKo/fMIKM1DDXPzqPDbuq/C6pzR0x6L059ynATOBjYLpzbrmZ3WNmE71ujwJdzayM6BTNgVMwzwKWmtkSogdpb3HO7WjpQYiIHE5hbgeeuHEk+2vruf7RuWzbV+N3SW3KEu1odCQScaWlpX6XISIBNLd8O5Mfn0d+l0ye/s7p9Oic4XdJLcbMFjjnIs2t0zdjRaTdGNWvK098aySbdldzzcMfsrGdTOMo6EWkXRnVryt/+vYotu+v5YqHZvPx58H/9qyCXkTaneF9cph28+k4B1c+NJu3P9nid0mtSkEvIu3SoN7ZvHTrGIq6deSmJ+bz0DufBfaKlwp6EWm3emZnMP2fzuCiwb34xWufcOMT89mxv9bvslqcgl5E2rWO6Sk88PVh/Oyrg5ldtp2Lf/M+76wM1lSOgl5E2j0z47rT+/LC90bTKSOFGx6fzw+mL2FXZTD27hX0IiKewfnZvHrbWP753P68tHgD4+97j+ml65N+7l5BLyISIz0lzA8uGMiMKWPok5vJD59byqUPfMCc8u1+l3bMFPQiIs0Y1Dub5787mt9eO4yd+2uZNHUO1z06l9I1yXcVF10CQUTkCKrrGnhi9hqmvlfO9v21jD6hK987uz9j+nclUX5j6XCXQFDQi4jEqaq2gafmruXh98rZureG/t2zmHxGX742vICs9BRfa1PQi4i0oOq6Bv6y9HOe+HANSyt20yk9hUuG9OJrwwqI9M0hFGr7vXwFvYhIK1m0bid/mrOW15ZtorK2gYKcTL42LJ+LT+nFST07tdnUjoJeRKSVVdbWM3P5Jl5YuIFZZdtodFCQk8n5JT24oKQnI4pySAm33vkvCnoRkTa0dW8Nb368mddXbOaDsm3U1jfSKSOFUcVdGdu/K2P6d6N/96wW3dtX0IuI+GR/TT3vr9rKu59uZVbZdtbtqASge6d0RvXryml9unBa31xO6tWJ1OPY4z9c0Pt7mFhEJOA6pqcwYXAvJgzuBcD6HZXM/mwbs8q2M2/1Dl5ZshGAzNQw553cnQe+PrzFa1DQi4i0ocLcDlyT24drRvQBYOOuKhas3cnCdTvpkBZulddU0IuI+Kh3l0x6d8nk0iG9W+01dAkEEZGAU9CLiAScgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAJd60bM9sKrD2Op+gGbGuhcpJFextzexsvaMztxfGMua9zLq+5FQkX9MfLzEoPdWGfoGpvY25v4wWNub1orTFr6kZEJOAU9CIiARfEoJ/qdwE+aG9jbm/jBY25vWiVMQdujl5ERA4WxD16ERGJoaAXEQm4wAS9mU0ws5VmVmZmd/ldz/Ews8fMbIuZLYtpyzWzN8xslXef47Wbmf3WG/dSMxses81kr/8qM5vsx1jiZWaFZva2mX1sZsvN7HavPbDjNrMMM5tnZku8Mf+n115sZnO9+p8xszSvPd1bLvPWF8U8191e+0ozu9CfEcXHzMJmtsjMXvWWgz7eNWb2kZktNrNSr61t39fOuaS/AWHgM6AfkAYsAUr8rus4xnMWMBxYFtN2L3CX9/gu4Bfe44uBvwEGnA7M9dpzgXLvPsd7nOP32A4z5l7AcO9xJ+BToCTI4/Zqz/IepwJzvbFMByZ57b8Dvus9/h7wO+/xJOAZ73GJ955PB4q9fwthv8d3mHHfCTwNvOotB328a4BuTdra9H3t+x+hhf6QZwAzY5bvBu72u67jHFNRk6BfCfTyHvcCVnqPHwaubdoPuBZ4OKb9oH6JfgNeBs5vL+MGOgALgVFEvxmZ4rV/8d4GZgJneI9TvH7W9P0e2y/RbkAB8CZwLk++olkAAAKASURBVPCqV39gx+vV11zQt+n7OihTN/nA+pjlCq8tSHo45z4H8O67e+2HGnvS/k28/6IPI7qHG+hxe9MYi4EtwBtE9053OefqvS6x9X8xNm/9bqAryTXm+4EfAo3ecleCPV4AB7xuZgvM7GavrU3f10H5cXBrpq29nDd6qLEn5d/EzLKA54HvO+f2mDU3jGjXZtqSbtzOuQZgqJl1AV4ETm6um3ef1GM2s0uALc65BWZ29oHmZroGYrwxxjjnNppZd+ANM/vkMH1bZcxB2aOvAApjlguAjT7V0lo2m1kvAO9+i9d+qLEn3d/EzFKJhvxTzrkXvObAjxvAObcLeIfovGwXMzuwExZb/xdj89ZnAztInjGPASaa2RpgGtHpm/sJ7ngBcM5t9O63EP0wH0kbv6+DEvTzgQHe0fs0ogduZvhcU0ubARw40j6Z6Bz2gfbrvaP1pwO7vf8KzgQuMLMc74j+BV5bQrLorvujwMfOuftiVgV23GaW5+3JY2aZwHjgY+Bt4EqvW9MxH/hbXAm85aITtjOASd5ZKsXAAGBe24wifs65u51zBc65IqL/Rt9yzn2DgI4XwMw6mlmnA4+Jvh+X0dbva78PVLTgAY+LiZ6p8RnwI7/rOc6x/Bn4HKgj+kl+E9G5yTeBVd59rtfXgAe9cX8ERGKe50agzLt9y+9xHWHMY4n+V3QpsNi7XRzkcQOnAou8MS8D/sNr70c0uMqAZ4F0rz3DWy7z1veLea4feX+LlcBFfo8tjrGfzT/OugnseL2xLfFuyw9kU1u/r3UJBBGRgAvK1I2IiByCgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnD/CzLLvvXa/aCaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(errs_xor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.          0.         -0.         -6.16814857  4.84210956  0.        ]\n",
      " [-0.          0.         -0.          6.19967277 -5.12386351 -0.        ]\n",
      " [ 0.          0.          0.         -3.54654842 -2.70140589 -3.73907307]\n",
      " [ 0.         -0.         -0.         -0.         -0.          7.45318735]\n",
      " [-0.          0.         -0.         -0.         -0.          7.71306033]\n",
      " [-0.          0.         -0.         -0.         -0.         -0.        ]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2fd3965e940>]"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de3hU9b3v8fd3JjcgISEhXEyAcJMSRKGGq7e2WovdFt27WtFqsdXSdtdTe9x9emz305vtPme33W1td90iXtqjtSK63buc1tbijUoFJIggF4EQEEK4JATCJeQymd/5YxZ0CAEGmGTNrHxezzPPWuu3fmvyXRg/s/Jba9Yy5xwiIhJcIb8LEBGRrqWgFxEJOAW9iEjAKehFRAJOQS8iEnAZfhfQUf/+/V1ZWZnfZYiIpJWVK1fWO+eKO1uXckFfVlZGZWWl32WIiKQVM3v/VOs0dCMiEnAKehGRgFPQi4gEnIJeRCTgFPQiIgGnoBcRCTgFvYhIwAUm6Bub2vj5y5tZU3PA71JERFJKyn1h6lyFQvCzlzeRETYuLi3wuxwRkZQRmCP6vJxMSgp6sWnPIb9LERFJKYEJeoDRA3PZtOew32WIiKSUQAX9mIF5bNl7mEh71O9SRERSRqCCfkhhb1rbo+w70up3KSIiKSNQQT+wbw4Auxubfa5ERCR1BCzoswHYc1BBLyJyTKCCfpB3RL/nUIvPlYiIpI5ABX1RbjYhg706ohcROS5QQR8OGcV52Rq6ERGJE6igh9gJ2d0HNXQjInJMIINeQzciIn8TwKDX0I2ISLzgBX1eDvub2miJtPtdiohISghe0HuXWO7VOL2ICBDEoM/3rqWPG76p2nuY5jYd4YtIzxS8oD/+7djYEf1jb1RzzU8X892F6/wsS0TEN8EL+rwTj+ifX1kDwILKHboaR0R6pMAFfUHvTLIyQuw51EzdoRbe232I6y8eTNTBXzbX+12eiEi3SyjozWyGmW00syozu7+T9feZ2XozW2Nmr5jZsLh17Wb2jvdamMziT1Fr7BLLxmbe3BIL9ruvGEH/3CyWbK7r6h8vIpJyzvjMWDMLAw8BHwVqgBVmttA5tz6u2yqgwjnXZGZfAn4E3OKtO+qcm5Dkuk9rYF4Oew62sGRzPfm9Mhlfks9lo/qzpKqeaNQRCll3liMi4qtEjugnA1XOuWrnXCswH7ghvoNz7jXnXJO3uAwoTW6ZZ2dgfg47Dxzlr1X1TB9ZRDhkXDG6mPrDrby3W8+UFZGeJZGgLwF2xC3XeG2nchfwx7jlHDOrNLNlZnZjZxuY2RyvT2Vd3fkPr4wvyWd7QxO1jc1cPro/AJePik2XVMXe3zlH/eEWnHPn/fNERFJZIkHf2ThHp+loZrcDFcCP45qHOucqgNuAB81s5Elv5tw851yFc66iuLg4gZJOb+qIouPzxwJ+UH4Oowfk8sbmetrao3zhqZVU/OBl7nj8LV1jLyKBlkjQ1wBD4pZLgdqOnczsGuCfgZnOueNfS3XO1XrTauB1YOJ51JuQiy7oy4xxg/juJ8oZVtTnePuHxhSzdMs+bn9sOX9ev4ePjRvIkqp6fv7K5q4uSUTEN4kE/QpgtJkNN7MsYBZwwtUzZjYReIRYyO+Na+9nZtnefH/gMiD+JG6XyAiHmHvHpdx52fAT2u+6fASZ4RDLtzYw58oRPHJHBTdfWsq8v1SzvvZgV5clIuILS2SM2sw+DjwIhIEnnHP/YmYPAJXOuYVm9jIwHtjlbbLdOTfTzKYT+wCIEvtQedA59/jpflZFRYWrrKw89z06g+37mqg50MS0EUWYGQeaWrnmp4sZnN+L+XOm0jsrjHPoyhwRSStmttIbJj95XaqdjOzqoO/Mn9bu5h+fXklGKITDETLjlklD+MZ1Y+mVFe7WWkREzsXpgv6M19H3BDMuGsSCL0zjj2t3kxkOUX+4haeWvc/qmkYen11B/9xsv0sUETlnCnpPRVkhFWWFx5evLR/IV+av4u//4688MXsSowfm+VidiMi5C9y9bpLl2nGDmD9nGk0t7cz4+Rt8df4q1u5s9LssEZGzpqA/jQlDCnjx3iu4c3oZi9bv4fp/X8I3/+tdjrRE/C5NRCRhCvozGNg3h29dX87Sb17NnCtH8Mxb2/nkw2+yfV/TmTcWEUkBCvoE9c3J5JsfH8uTn5vMrsZmZj60hL9W6bbHIpL6FPRn6YrRxSy85zIG5GXzmSfe4oklW3W/HBFJaQr6czCsqA8v/ONlXP2BATzw+/X804LVul+OiKQsBf05ys3OYO7tl/I/r7mQF1bt5FOPLKXhSKvfZYmInERBfx5CIePea0bz6Gcq2Lj7ELc8svT4s2pFRFKFgj4JPlo+kF9/djK1B45y89yl7GjQFTkikjoU9EkybWQRv7l7CgeaWrl57lKq9h72uyQREUBBn1QTh/bj2S9MIxKNMmveMjbt0WMLRcR/CvokGzu4L/PnTMUMbp23jA27dJ97EfGXgr4LjBqQx7NzppIZDnHbo8tYV6t75IiIfxT0XWREcS7PfmEqvTLD3P7Yct7brSN7EfGHgr4LDSvqwzNzppKdEebTjy5ns8bsRcQHCvouNqyoD7/9/BRCIePWR5ezpU5X44hI91LQd4MRxbk88/kpgOO2R5exrf6I3yWJSA+ioO8mowbk8fTdU2mNRLnt0WX6UpWIdBsFfTcaMyiP39w9hSOt7dz66DJ2Hjjqd0ki0gMo6LvZuAvy+c1dU2g82sZtjy5jd6PujSMiXUtB74Pxpfk8+bnJ7Dvcym2PLmOvboQmIl1IQe+TiUP78evPTmL3wWZue2w59Ydb/C5JRAJKQe+jirJCnrhzEjX7m7j9seXsU9iLSBdQ0Pts6ogiHp89ia31R/jUI0up1QlaEUmyhILezGaY2UYzqzKz+ztZf5+ZrTezNWb2ipkNi1s328w2e6/ZySw+KC4b1Z8nPzeZvQdbuHnuUqr1pSoRSaIzBr2ZhYGHgOuAcuBWMyvv0G0VUOGcuxh4HviRt20h8B1gCjAZ+I6Z9Ute+cExZUQRz8yZSnNbOzfPXcranboRmogkRyJH9JOBKudctXOuFZgP3BDfwTn3mnPu2DeAlgGl3vzHgEXOuQbn3H5gETAjOaUHz0Ul+Tz3xWlkZ4S4dd4yllfv87skEQmARIK+BNgRt1zjtZ3KXcAfz2ZbM5tjZpVmVllXV5dAScE1ojiX5780nQF9s7n98eUsqNxx5o1ERE4jkaC3Ttpcpx3NbgcqgB+fzbbOuXnOuQrnXEVxcXECJQXbBQW9eOFLlzFleBFff34N//vFDbRHO/0nFxE5o0SCvgYYErdcCtR27GRm1wD/DMx0zrWczbZysvzemfzqs5P4zLRhzPtLNXOerORgc5vfZYlIGkok6FcAo81suJllAbOAhfEdzGwi8AixkN8bt+ol4Foz6+edhL3Wa5MEZIZDPHDDRXz/hnG8vqmO63+xhNU7DvhdloikmTMGvXMuAtxDLKA3AAucc+vM7AEzm+l1+zGQCzxnZu+Y2UJv2wbg+8Q+LFYAD3htchbumFbGgi9MJdIe5aa5b/L4kq04p6EcEUmMpVpgVFRUuMrKSr/LSEkHmlr52nNreHnDHq7+wAD+zz+MZ0DfHL/LEpEUYGYrnXMVna3TN2PTSEHvLB79zKV8+/pyllTVc81PF/P8yhod3YvIaSno04yZ8bnLh/PHe69gzKA8vvbcau781Qrd215ETklBn6ZGFOfy7JxpfG/mOFZsa+Dqn7zOgy9vormt3e/SRCTFKOjTWChkzJ5exqL7ruLqsQN58OXNXP2TxfxhzS4N54jIcQr6ACgp6MVDt32Q+XOm0rdXJl/+7dvc+B9vsmRzvQJfRBT0QTJ1RBG//x+X88NPjqfuYDO3P76cWfOWUblNV7SK9GS6vDKgWiLtPLN8O798bQv1h1u4YnR/vnTVSKaNLMKssztTiEg6O93llQr6gGtqjfDU0vd5bMlW6g61cMmQAr501UiuLR9IKKTAFwkKBb3Q3NbOC2/v5JG/bOH9fU2MKO7DF68ayY0TSsjK0AieSLpT0MtxkfYof1y7m4df38L6XQcZ1DeHu68Yzq2Th9InO8Pv8kTkHCno5STOORZvqmPu4i0sq24gv1cms6eXcef0Mgr7ZPldnoicJQW9nNbb2/cz9/Ut/Hn9HnIyQ8yaNJS7rxhOab/efpcmIglS0EtCqvYeYu7iav571U4ccOOEEr56zWiGFCrwRVKdgl7OSu2Bozz2xlaeXv4+zsEd04Zxz4dH0U9DOiIpS0Ev52RX41F+tmgTz6+soU92BvdePZo7p5eREdZVOiKpRrcplnMyOL8XP7rpEv5475VcOqwfP/jDBv7h4TdZX3vQ79JE5Cwo6OWMxgzK41d3TuLfb51I7YGjzPzlEh57o1r30RFJEwp6SYiZ8YlLLuDl+67i6rED+MEfNjDnqZU0tUb8Lk1EzkBBL2eloHcWc2+/lG9dX84rG/Ywa94y6g61+F2WiJyGgl7Omplx1+XDmXdHBZv2HOKWeUvZd1hhL5KqFPRyzq4pH8iTn5tC7YGjzP7VWxxqbvO7JBHphIJezsvk4YU8/OlLeW/XIe757SraozpBK5JqFPRy3j78gQF8d+Y4Fm+q45evVvldjoh0oKCXpPj0lKH8/cQSfvHqZtbubPS7HBGJo6CXpDAzvvuJcRT2yeLrz68h0h71uyQR8SjoJWnye2fyvZnjWL/rIP/5do3f5YiIR0EvSXXdRYP44NACfvLnTfoylUiKSCjozWyGmW00syozu7+T9Vea2dtmFjGzmzqsazezd7zXwmQVLqnJzPhfMz7A3kMtLFixw+9yRIQEgt7MwsBDwHVAOXCrmZV36LYduBP4bSdvcdQ5N8F7zTzPeiUNTBlRxAeHFvDEX7fpckuRFJDIEf1koMo5V+2cawXmAzfEd3DObXPOrQF0Bk4A+PwVI9je0MSi9bv9LkWkx0sk6EuA+L/Ba7y2ROWYWaWZLTOzGzvrYGZzvD6VdXV1Z/HWkqquHTeIwfk5LKjUSVkRvyUS9NZJ29n8PT7Uuxn+bcCDZjbypDdzbp5zrsI5V1FcXHwWby2pKhwybpxYwuJNdbrpmYjPEgn6GmBI3HIpUJvoD3DO1XrTauB1YOJZ1Cdp7JMfLKE96vjdOzv9LkWkR0sk6FcAo81suJllAbOAhK6eMbN+ZpbtzfcHLgPWn2uxkl5GDchjfEk+v1+zy+9SRHq0Mwa9cy4C3AO8BGwAFjjn1pnZA2Y2E8DMJplZDXAz8IiZrfM2HwtUmtlq4DXgX51zCvoe5KPlA1ldc0DDNyI+ykikk3PuReDFDm3fjptfQWxIp+N2bwLjz7NGSWPXjB3ITxdt4rX39vKpSUPOvIGIJJ2+GStdauzgPC7Iz2HRhj1+lyLSYynopUuZGR8ZO4C/VtXTphudifhCQS9dbvrI/jS1trOmRrcvFvGDgl663JThhQAsq97ncyUiPZOCXrpcUW42YwbmKehFfKKgl24xbWQRldv20xrROL1Id1PQS7eYPLyQo23trN910O9SRHocBb10iwlDCgBYveOAz5WI9DwKeukWg/NzGJCXzTsKepFup6CXbmFmXDKkQEf0Ij5Q0Eu3mTCkgOr6IzQ2tfldikiPoqCXbnNsnP6dGh3Vi3QnBb10m/Gl+ZjphKxId1PQS7fpm5PJyOJcnZAV6WYKeulWE7wTss6dzdMoReR8KOilW11Sms++I63UNjb7XYpIj6Ggl241vjR2QvZdnZAV6TYKeulWHxiUR0bIeHenblks0l0U9NKtcjLDXDgwT/emF+lGCnrpdheX5vPuzkadkBXpJgp66XYXleRzoKmNmv1H/S5FpEdQ0Eu3u7g0H0Dj9CLdREEv3W7MoDwyw6ZxepFuoqCXbpedEWbMoDzW6ohepFso6MUX40sKWFOjb8iKdAcFvfji4tJ8DjZH2N7Q5HcpIoGXUNCb2Qwz22hmVWZ2fyfrrzSzt80sYmY3dVg328w2e6/ZySpc0tv4Ep2QFekuZwx6MwsDDwHXAeXArWZW3qHbduBO4Lcdti0EvgNMASYD3zGzfudftqS7CwfmkRUO8a5OyIp0uUSO6CcDVc65audcKzAfuCG+g3Num3NuDRDtsO3HgEXOuQbn3H5gETAjCXVLmsvKCDF2sL4hK9IdEgn6EmBH3HKN15aIhLY1szlmVmlmlXV1dQm+taS7i0ryWVvbSDSqE7IiXSmRoLdO2hL9PzOhbZ1z85xzFc65iuLi4gTfWtLdxaX5HGqO8L5OyIp0qUSCvgYYErdcCtQm+P7ns60E3PiS2C2L1+iWxSJdKpGgXwGMNrPhZpYFzAIWJvj+LwHXmlk/7yTstV6bCKMH5pKVEdIXp0S62BmD3jkXAe4hFtAbgAXOuXVm9oCZzQQws0lmVgPcDDxiZuu8bRuA7xP7sFgBPOC1iZAZDlE+uK9OyIp0sYxEOjnnXgRe7ND27bj5FcSGZTrb9gngifOoUQLs4tJ8Xnh7J9GoIxTq7JSOiJwvfTNWfHVRST6HWyJU1x/xuxSRwFLQi68+ODR2Qvbt7ft9rkQkuBT04quRxbn0651J5TaduhHpKgp68ZWZcemwflRu0xG9SFdR0IvvKsoKqa4/wr7DLX6XIhJICnrx3aSy2H3uKt/XUb1IV1DQi+8uKsknKyPESgW9SJdQ0IvvsjPCXFKazwqdkBXpEgp6SQkVZYWs3dlIc1u736WIBI6CXlJCxbB+tLU7Vu/QDc5Ekk1BLynh0mE6ISvSVRT0khIKemdx4cBclm/VOL1IsinoJWVMH9mfFVsbaI10fCKliJwPBb2kjGkjizja1s47GqcXSSoFvaSMqSOKCBn8tare71JEAkVBLykjv1cmF5Xks3TLPr9LEQkUBb2klOkj+7Nqx36aWiN+lyISGAp6SSnTRxbR1u5YobtZiiSNgl5SSkVZPzLDxpsapxdJGgW9pJTeWRlMHNqPNzVOL5I0CnpJOdNHFrG2tpH9R1r9LkUkEBT0knKuvLAY5+ANDd+IJIWCXlLOJaUF9Oudyevv7fW7FJFAUNBLygmHjKsuLOb1TXVEo87vckTSnoJeUtKHxgyg4Ugra3Y2+l2KSNpT0EtKuvLCYszg9Y0avhE5Xwp6SUmFfbKYMKSA1zbW+V2KSNpLKOjNbIaZbTSzKjO7v5P12Wb2rLd+uZmVee1lZnbUzN7xXnOTW74E2YfHDGBNzQH2HW7xuxSRtHbGoDezMPAQcB1QDtxqZuUdut0F7HfOjQJ+Bvwwbt0W59wE7/XFJNUtPcCHxwzAOVi8SUf1IucjkSP6yUCVc67aOdcKzAdu6NDnBuD/evPPA1ebmSWvTOmJxl3QlwF52Sxav8fvUkTSWiJBXwLsiFuu8do67eOciwCNQJG3briZrTKzxWZ2RWc/wMzmmFmlmVXW1enoTWJCIWPGRYN49b29HGnR3SxFzlUiQd/ZkXnHi5tP1WcXMNQ5NxG4D/itmfU9qaNz85xzFc65iuLi4gRKkp7i78YPpiUS5VV9eUrknCUS9DXAkLjlUqD2VH3MLAPIBxqccy3OuX0AzrmVwBbgwvMtWnqOirJCivOyefHdXX6XIpK2Egn6FcBoMxtuZlnALGBhhz4Lgdne/E3Aq845Z2bF3slczGwEMBqoTk7p0hOEQ8Z1Gr4ROS9nDHpvzP0e4CVgA7DAObfOzB4ws5let8eBIjOrIjZEc+wSzCuBNWa2mthJ2i865xqSvRMSbB/3hm9e3qCTsiLnwpxLrXuJVFRUuMrKSr/LkBQSjTqu+rfXKC3ozTNzpvpdjkhKMrOVzrmKztbpm7GS8kIhY9akoSyt3kd13WG/yxFJOwp6SQs3V5SSETLmr9hx5s4icgIFvaSFAXk5XDN2IM+vrKEl0u53OSJpRUEvaeP2qcNoONLK71Z1vLpXRE5HQS9p47JRRVxU0peHF2+hXQ8kEUmYgl7Shpnx5Q+NYmv9Ef7fah3ViyRKQS9p5WPjBlE+uC8/+tN7NLdprF4kEQp6SSuhkPGt68upbWzm0b/oS9YiiVDQS9qZNrKIGeMG8fDiLdQeOOp3OSIpT0EvaembHx8LwNeeW01UJ2ZFTktBL2lpaFFvvn19OW9u2cfjS7b6XY5ISlPQS9q6ZdIQPjZuID966T2WVe/zuxyRlKWgl7RlZvzok5cwrKgPc56spGrvIb9LEklJCnpJa/m9M/nVnZPIyghzx+Nv6aZnIp1Q0EvaG1LYmyc/N5nWSJSb5y5l7c5Gv0sSSSkKegmE8gv68twXp5GdEeKmuW/ywts1fpckkjIU9BIYI4pz+e97LuOS0gLuW7Ca+559h/1HWv0uS8R3CnoJlAF5OTx99xS+8pFRLFxdy9U/XcxTS7fRGon6XZqIbxT0EjgZ4RD3XTuGP3zlCkYV5/Kt363jw//2Ok8vf18PGJceSc+MlUBzzvHG5np+8ueNrK5pJC87gxsnljBzwgVcOrQfoZD5XaJIUpzumbEKeukRnHO8vX0/v1m2nT+8u4vWSJTivGyuLR/IFaOLmTK8kH59svwuU+ScKehF4hxqbuO1jXX8ae0uXnuvjqPe7Y4/MCiPSWWFjLugL+MuyOfCQblkZ4R9rlYkMQp6kVNojUR5d+cBllU3sKx6H6u2H+CwN46fETJGDchlRHEfhhX1YXhRH8r696Gsf2+Kc7Mx07CPpA4FvUiColHH9oYm1tUeZP2uRjbsOsTW+iPsaGgiEneXzOyMEIPycxjUN+ekaVFuNoV9MunXO4uC3lmEdR5AusHpgj6ju4sRSWWhkHlH7X34u4sHH2+PtEfZeeAoW+uPsK3+CLWNzexqbGZPYzOrth9gd2Mzre0nX8JpBgW9MunXJ4vC3ln065NFQa9M8nIyyc3JoG9OBrnZGceX83IyyItb7pMV1l8Oct4U9CIJyAiHGFYUG8JhzMnrnXPsb2pjd2MzDUdaaWhqZf+R1th83PKOhibebWrjcEvk+BDRmfTKDNMrK3ziNDNMTlaY3l5bTuax9SF6Z2WQnREiOyNE1rFXOBw3H5tmd1iOX5+dEdIHTIAkFPRmNgP4ORAGHnPO/WuH9dnAk8ClwD7gFufcNm/dN4C7gHbgK865l5JWvUiKMDMK+2RReBZX7rRHHUdaIxxqjnC4OcKh5jYOtZy4fLglQnNbO0fb2jnaGuVoW4SjrbHlxqNt7GlspqktwtHWKM1t7TS1RkjWc1gyw3b8QyAcCpEZNjLCRkYoREbICIeMzHDIa/Paj82HQydOT2j723JmyAh724VDRtiMUMgIG4RDx+Zj05AZ4RDeNL6vdejL8TY7oS9n3C5kcetDRshibeZNQ2YYsb/U0umD8IxBb2Zh4CHgo0ANsMLMFjrn1sd1uwvY75wbZWazgB8Ct5hZOTALGAdcALxsZhc65/RUZ+nxwiGjb04mfXMyk/aezjna2h1H29ppjURpbY/Gpsde7e20nLB88vyx9cembe1RIlFHpD1Ke9TR5s1HTpg6ItEozZFj83HrotET247NR6O0tafWOcKz8bfwj4V+KP7DIG5dbPlYv9P1gbGD+/LL2z6Y9FoTOaKfDFQ556pjO2fzgRuA+KC/AfiuN/888EuLfdzdAMx3zrUAW82synu/pckpX0TimRlZGUZWRvp86b096mhrjxJ1jvaoIxqF9mPz3rQ96nCu8/boCW0cnz9Ve+fbc8I2x35erC02dXHzUQd0WHbOdej/t/c8vj4Kjs76xKZDC3t3yb9xIkFfAuyIW64Bppyqj3MuYmaNQJHXvqzDtiUdf4CZzQHmAAwdOjTR2kUkAMIhIxzS9xW6UiIf+50NRHX8e+tUfRLZFufcPOdchXOuori4OIGSREQkUYkEfQ0wJG65FKg9VR8zywDygYYEtxURkS6USNCvAEab2XAzyyJ2cnVhhz4Lgdne/E3Aqy72TayFwCwzyzaz4cBo4K3klC4iIok44xi9N+Z+D/ASscsrn3DOrTOzB4BK59xC4HHgKe9kawOxDwO8fguInbiNAF/WFTciIt1Lt0AQEQmA090CIX2uwRIRkXOioBcRCTgFvYhIwKXcGL2Z1QHvn8db9Afqk1ROutA+9wza557hXPd5mHOu0y8ipVzQny8zqzzVCYmg0j73DNrnnqEr9llDNyIiAaegFxEJuCAG/Ty/C/CB9rln0D73DEnf58CN0YuIyImCeEQvIiJxFPQiIgEXmKA3sxlmttHMqszsfr/rSRYze8LM9prZ2ri2QjNbZGabvWk/r93M7Bfev8EaM0v+M8m6gZkNMbPXzGyDma0zs3u99sDut5nlmNlbZrba2+fvee3DzWy5t8/PeneQxbsj7LPePi83szI/6z8fZhY2s1Vm9ntvOdD7bGbbzOxdM3vHzCq9ti793Q5E0Mc91/Y6oBy41XtebRD8GpjRoe1+4BXn3GjgFW8ZYvs/2nvNAR7uphqTLQL8k3NuLDAV+LL33zPI+90CfMQ5dwkwAZhhZlOJPX/5Z94+7yf2fGaIe04z8DOvX7q6F9gQt9wT9vnDzrkJcdfLd+3vtvOehZjOL2Aa8FLc8jeAb/hdVxL3rwxYG7e8ERjszQ8GNnrzjwC3dtYvnV/A74g9nL5H7DfQG3ib2CM764EMr/347zmx24ZP8+YzvH7md+3nsK+lXrB9BPg9safSBX2ftwH9O7R16e92II7o6fy5tic9mzZABjrndgF40wFee+D+Hbw/zycCywn4fntDGO8Ae4FFwBbggHMu4nWJ368TntMMHHtOc7p5EPg6EPWWiwj+Pjvgz2a20nteNnTx73YiDwdPBwk9m7YHCNS/g5nlAv8JfNU5d9Css92Lde2kLe3228UeyjPBzAqA/wLGdtbNm6b9PpvZ9cBe59xKM/vQseZOugZmnz2XOedqzWwAsMjM3jtN36Tsc1CO6Hvas2n3mNlgAG+612sPzL+DmWUSC/mnnXMveM2B328A59wB4HVi5ycKvOcww4n7darnNKeTy4CZZrYNmE9s+OZBgr3POOdqveleYh/ok+ni3+2gBH0iz7UNkvhn9M4mNoZ9rP0z3pn6qUDjsT8H04nFDt0fBzY45yRcwxMAAADxSURBVH4atyqw+21mxd6RPGbWC7iG2AnK14g9hxlO3ufOntOcNpxz33DOlTrnyoj9P/uqc+7TBHifzayPmeUdmweuBdbS1b/bfp+YSOIJjo8Dm4iNa/6z3/Ukcb+eAXYBbcQ+3e8iNi75CrDZmxZ6fY3Y1UdbgHeBCr/rP8d9vpzYn6drgHe818eDvN/AxcAqb5/XAt/22kcAbwFVwHNAttee4y1XeetH+L0P57n/HwJ+H/R99vZttfdadyyruvp3W7dAEBEJuKAM3YiIyCko6EVEAk5BLyIScAp6EZGAU9CLiAScgl5EJOAU9CIiAff/AT+7ju4WaYDqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Since this takes like 5000 epochs, I wanted to try it with accelerated gradient descent to compare\n",
    "train_x = [[0,0], [0,1], [1,0], [1,1]]\n",
    "train_y = [[0], [1], [1], [0]]\n",
    "errs_xor = xor_net.train(train_x, train_y, delay=2, epochs=500, learning_rate=1, accelerated=True)\n",
    "plt.plot(errs_xor)\n",
    "# It trains in 1/10 the number of epochs, so great!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.04542586]]\n",
      "[[0.96183042]]\n",
      "[[0.95941788]]\n",
      "[[0.04101035]]\n"
     ]
    }
   ],
   "source": [
    "print(xor_net.get_output([0,0], 2))\n",
    "print(xor_net.get_output([0,1], 2))\n",
    "print(xor_net.get_output([1,0], 2))\n",
    "print(xor_net.get_output([1,1], 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backprop-calculated gradient:\n",
      "[[ 0.          0.          0.         -0.00343948  0.0060595   0.        ]\n",
      " [ 0.          0.          0.          0.00262133 -0.00607109  0.        ]\n",
      " [ 0.          0.          0.          0.00031637  0.00371128  0.00448841]\n",
      " [ 0.          0.          0.          0.          0.         -0.00738545]\n",
      " [ 0.          0.          0.          0.          0.         -0.01025068]\n",
      " [ 0.          0.          0.          0.          0.          0.        ]]\n",
      "Manually-calculated gradient:\n",
      "[[ 0.          0.          0.         -0.00343948  0.00605951  0.        ]\n",
      " [ 0.          0.          0.          0.00262133 -0.00607107  0.        ]\n",
      " [ 0.          0.          0.          0.00031638  0.0037113   0.00448846]\n",
      " [ 0.          0.          0.          0.          0.         -0.00738545]\n",
      " [ 0.          0.          0.          0.          0.         -0.01025067]\n",
      " [ 0.          0.          0.          0.          0.          0.        ]]\n",
      "Difference:\n",
      "[[ 0.00000000e+00  0.00000000e+00  0.00000000e+00 -7.06915511e-09\n",
      "  -4.80730830e-09  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00 -1.63876312e-09\n",
      "  -1.63176157e-08  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00 -9.44279003e-09\n",
      "  -2.45146741e-08 -4.57195418e-08]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00 -6.35515791e-09]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00 -9.27005170e-09]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "xor_net.gradient_check(train_x, train_y, delay=2, epsilon=0.000001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "bpcheck = GraphNet(8)\n",
    "bpcheck.define_input([0,1])\n",
    "bpcheck.define_output([6,7])\n",
    "bpcheck.connect(0,2)\n",
    "bpcheck.connect(1,2)\n",
    "bpcheck.connect_loop([2,3,4,5])\n",
    "bpcheck.connect(4,6)\n",
    "bpcheck.connect(4,7)\n",
    "#bpcheck.init_random()\n",
    "bpcheck.init_uniform()\n",
    "# Square loop with two inputs and outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1.]\n",
      " [0. 0.]\n",
      " [0. 0.]\n",
      " [0. 0.]\n",
      " [1. 1.]]\n",
      "[[1. 1.]\n",
      " [0. 0.]\n",
      " [0. 0.]\n",
      " [0. 0.]\n",
      " [1. 1.]]\n"
     ]
    }
   ],
   "source": [
    "# Make sure regular stepping works\n",
    "#print(bpcheck.step([0,1]))\n",
    "#print(bpcheck.step([0,0]))\n",
    "#print(bpcheck.step([0,0]))\n",
    "#print(bpcheck.step([0,0]))\n",
    "# Make sure multiple stepping works\n",
    "#bpcheck.reset_state()\n",
    "print(bpcheck.get_output([0,1], 12, 5))\n",
    "print(bpcheck.get_output([1,0], 12, 5))\n",
    "# These are the same, as expected. Bug fixed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.         0.         1.         0.         0.         0.\n",
      "  0.         0.        ]\n",
      " [0.         0.         1.11742165 0.         0.         0.\n",
      "  0.         0.        ]\n",
      " [0.         0.         0.         0.91886753 0.         0.\n",
      "  0.         0.        ]\n",
      " [0.         0.         0.         0.         0.91886753 0.\n",
      "  0.         0.        ]\n",
      " [0.         0.         0.         0.         0.         0.76954337\n",
      "  1.0599282  1.0599282 ]\n",
      " [0.         0.         0.76954337 0.         0.         0.\n",
      "  0.         0.        ]\n",
      " [0.         0.         0.         0.         0.         0.\n",
      "  0.         0.        ]\n",
      " [0.         0.         0.         0.         0.         0.\n",
      "  0.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "train_x = [[[0,1]]]\n",
    "train_y = [[[1,1], [0,0], [0,0], [0,0], [0.5,0.5]]]\n",
    "errs_bpc = bpcheck.train(train_x, train_y, delay=4, epochs=200, learning_rate=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1402277a470>]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXhUlEQVR4nO3de3Rc5X3u8e8zI8lXbIMtDPUFOcbQuoEEokMuTahXAjkmaezmbpqzQtqcw8qhPmkWJ+uUHFpWSlbahKw2bXrcNqRlNe1K6lxJ1NYpaRp6SU9ILcBcHOMgHIOFAQsbbMA3XX79Y7bkuUkay9KM3tHzWUtLe7+zNfPznvGjV+/e796KCMzMLH25RhdgZmaTw4FuZtYkHOhmZk3CgW5m1iQc6GZmTaKlUS+8ZMmS6OjoaNTLm5kl6d577302ItqrPdawQO/o6KC7u7tRL29mliRJj4/2mIdczMyahAPdzKxJONDNzJqEA93MrEk40M3MmkRNgS5pvaTdknok3VTl8Q9I6pO0I/v675NfqpmZjWXc0xYl5YEtwNVAL7BdUldE/Lhs069ExOYpqNHMzGpQSw/9CqAnIvZExElgK7Bxassa3fa9h/j97+6mf3CoUSWYmU1LtQT6MmBf0Xpv1lbunZIelPR1SSuqPZGk6yV1S+ru6+ubQLlw3+PP8cff7+HkgAPdzKxYLYGuKm3ld8X4W6AjIi4Fvgd8sdoTRcTtEdEZEZ3t7VVnro4rnyuUM+Qbc5iZlagl0HuB4h73cmB/8QYRcTAiTmSrXwBeNTnlVZKyQHcH3cysRC2Bvh1YI2mVpDZgE9BVvIGk84tWNwC7Jq/EUvns74VB99DNzEqMe5ZLRAxI2gzcBeSBOyJip6Rbge6I6AI+LGkDMAAcAj4wVQV7yMXMrLqarrYYEduAbWVttxQtfwz42OSWVt2pIRcHuplZseRmig730D3kYmZWKr1AH+6hO8/NzEokF+hZnnvIxcysTHKBPjLk4kA3MyuRbKD7LBczs1LJBfrIWS4OdDOzEskF+vBBUV+by8ysVHqBnlXsHrqZWankAl3yQVEzs2qSC/S8x9DNzKpKL9BznlhkZlZNcoE+PLHIQy5mZqWSC3Sfh25mVl16ge6rLZqZVZVcoI+c5eIeuplZieQCfWTIxROLzMxKJBjohe8eQzczK5VcoHvIxcysuuQC3QdFzcyqSy/QPbHIzKyq5ALdE4vMzKpLLtA9scjMrLr0At0X5zIzqyq5QPflc83Mqksu0D3kYmZWXXqBLs8UNTOrJrlAHznLxT10M7MSyQX6qWu5ONDNzIqlG+jOczOzEskFuodczMyqSy7QfS0XM7Pqagp0Sesl7ZbUI+mmMbZ7l6SQ1Dl5JZbyaYtmZtWNG+iS8sAW4BpgLXCtpLVVtjsL+DDwo8kuslgu54lFZmbV1NJDvwLoiYg9EXES2ApsrLLdJ4DbgOOTWF+FnKf+m5lVVUugLwP2Fa33Zm0jJF0GrIiIvxvriSRdL6lbUndfX99pFwvF13KZ0I+bmTWtWgJdVdpG4lRSDvgs8L/He6KIuD0iOiOis729vfYqi+Syij3kYmZWqpZA7wVWFK0vB/YXrZ8FvBz4Z0l7gdcAXVN1YDTns1zMzKqqJdC3A2skrZLUBmwCuoYfjIjDEbEkIjoiogO4B9gQEd1TUbCHXMzMqhs30CNiANgM3AXsAr4aETsl3Sppw1QXWG7kLBcfFDUzK9FSy0YRsQ3YVtZ2yyjbrjvzssaWk4dczMzKJTdTFAqTi3zaoplZqSQDPSd5yMXMrEyyge4hFzOzUkkGemHIpdFVmJlNL0kGek6eWGRmVi7NQPdBUTOzCkkGel4OdDOzckkGei4nBocaXYWZ2fSSZqB7YpGZWYUkA91DLmZmlZIM9FzOE4vMzMqlGeieWGRmViHJQPfEIjOzSkkGek6+fK6ZWblEA91DLmZm5ZIMdF8+18ysUpKBnpMnFpmZlUsz0HO4h25mVibJQPfEIjOzSkkGeuFaLg50M7NiaQa6e+hmZhWSDPS8xJAPipqZlUgy0HM5TywyMyuXZqB7YpGZWYUkA90Ti8zMKiUZ6DmJQee5mVmJRAPddywyMyuXZKB7yMXMrFKSgV64losD3cysWJKB7h66mVmlmgJd0npJuyX1SLqpyuMfkvSQpB2SfiBp7eSXekphpuhUvoKZWXrGDXRJeWALcA2wFri2SmB/OSIuiYhXArcBfzDplRbJ5XweuplZuVp66FcAPRGxJyJOAluBjcUbRMSRotV5wJSmbd63oDMzq9BSwzbLgH1F673Aq8s3kvTrwI1AG/DGak8k6XrgeoCVK1eebq0jfHEuM7NKtfTQVaWtIk0jYktErAZ+E/itak8UEbdHRGdEdLa3t59epUUKQy4T/nEzs6ZUS6D3AiuK1pcD+8fYfivwy2dS1HjyPm3RzKxCLYG+HVgjaZWkNmAT0FW8gaQ1RatvBR6dvBIr+RZ0ZmaVxh1Dj4gBSZuBu4A8cEdE7JR0K9AdEV3AZklXAf3Ac8B1U1m0x9DNzCrVclCUiNgGbCtru6Vo+Tcmua4x5X0LOjOzCknOFPXEIjOzSukGuhPdzKxEkoGe9y3ozMwqJBnoPihqZlYpzUD3xCIzswpJBnpe8pCLmVmZJAM9J08sMjMrl2ag50QEhEPdzGxEkoGeV+F6YZ5cZGZ2SpKBnstlge4eupnZiDQDPeuhO8/NzE5JMtDzWdUecjEzOyXJQB/uoXvIxczslKQDPTy5yMxsRJKBnvdBUTOzCkkGepbnHkM3MyuSZqDnhs9ycaCbmQ1LMtDzPihqZlYhyUDPeaaomVmFNAM954lFZmblkgx0TywyM6uUZKB7YpGZWaWkA91nuZiZnZJkoI9MLPJMUTOzEUkGuicWmZlVSjTQC4nu29CZmZ2SZKAPD7k40M3MTkky0EfuWOQhFzOzEWkG+siQS4MLMTObRpIM9LzH0M3MKiQZ6DnPFDUzq1BToEtaL2m3pB5JN1V5/EZJP5b0oKR/knTB5Jd6is9yMTOrNG6gS8oDW4BrgLXAtZLWlm12P9AZEZcCXwdum+xCi42c5eKJRWZmI2rpoV8B9ETEnog4CWwFNhZvEBF3R8TRbPUeYPnkllnK13IxM6tUS6AvA/YVrfdmbaP5IPCdag9Iul5St6Tuvr6+2qssMzxT1EMuZman1BLoqtJWNUkl/TegE/hMtccj4vaI6IyIzvb29tqrLHNqyMWBbmY2rKWGbXqBFUXry4H95RtJugq4GfjFiDgxOeVV5zsWmZlVqqWHvh1YI2mVpDZgE9BVvIGky4DPAxsi4sDkl1nKE4vMzCqNG+gRMQBsBu4CdgFfjYidkm6VtCHb7DPAfOBrknZI6hrl6SaFr+ViZlapliEXImIbsK2s7Zai5asmua4x+RZ0ZmaVkpwpKk8sMjOrkGSg+1ouZmaV0gx034LOzKxCkoEuTywyM6uQZKB7YpGZWaU0A93XcjEzq5BkoMsTi8zMKiQZ6B5yMTOrlGag+1ouZmYVkgx0ZVX7LBczs1OSDHRPLDIzq5RmoHtikZlZhSQD3ROLzMwqJRnoI0MuPihqZjYizUDPeWKRmVm5JAPdE4vMzColGehQ6KV7yMXM7JR0A13ykIuZWZFkA13yWS5mZsWSDXQPuZiZlUo30CVPLDIzK5JsoOdyYmDIiW5mNizZQF+6YBZPHT7e6DLMzKaNZAO9Y/E8Hj/4UqPLMDObNtIN9CXzePzgUR8YNTPLpBvoi+dxYmCIp4542MXMDFIO9CVzAdj7rIddzMwg5UBfPA+AvR5HNzMDEg708xbMZlZLzj10M7NMsoGey4mOxfP46bNHG12Kmdm0UFOgS1ovabekHkk3VXn8Skn3SRqQ9K7JL7O6CxbP9amLZmaZcQNdUh7YAlwDrAWulbS2bLMngA8AX57sAseyask8Hj/kUxfNzKC2HvoVQE9E7ImIk8BWYGPxBhGxNyIeBOo6F//Cc+dzcmCIXU8fqefLmplNS7UE+jJgX9F6b9Z22iRdL6lbUndfX99EnqLEm35uKa158a37nzzj5zIzS10tga4qbRMa44iI2yOiMyI629vbJ/IUJc6Z18a6i8/l2zv2M+hhFzOb4WoJ9F5gRdH6cmD/1JRz+t5x2TIOvHCCf+95ttGlmJk1VC2Bvh1YI2mVpDZgE9A1tWXV7o0/dy4LZrewdfsTjS7FzKyhxg30iBgANgN3AbuAr0bETkm3StoAIOm/SOoF3g18XtLOqSy62KyWPL/y6gv4zsNPs6fvxXq9rJnZtFPTeegRsS0iLoqI1RHxyaztlojoypa3R8TyiJgXEYsj4uensuhyH3z9KtryOT7/L3vq+bJmZtNKsjNFi7WfNYv3dK7gm/f38uTzxxpdjplZQzRFoAN8aN1qJPHZf/xJo0sxM2uIpgn0ZYvm8IHXdfCN+3p5xBONzGwGappAB7hh3Wrmz2rhtn/Y3ehSzMzqrqkCfdHcNm5YdyHff+QA9+w52OhyzMzqqqkCHeBXf6GD8xbM5lPfeYQIzx41s5mj6QJ9dmueG6++iB37nudbO3yNFzObOZou0AHe+arlXLZyEZ/4u10ceulko8sxM6uLpgz0fE586h2XcuRYP5/8+12NLsfMrC6aMtABLj7vLD70i6v5xn29/OBRX7jLzJpf0wY6wOY3XsiqJfP4v3c+xLGTg40ux8xsSjV1oM9uzfO7b7+EJw4d5Xf+tm7XCzMza4imDnSA165ezA3rVrN1+z7uvL+30eWYmU2Zpg90gBuvvogrOs7h5jsfpueAL7FrZs1pRgR6Sz7H5669jDmteW740r0cPTnQ6JLMzCbdjAh0gPMWzuaz730ljx54kQ//zQ4GBocaXZKZ2aSaMYEOcOVF7Xz8bT/P93Y9w29/+2FfGsDMmkpLowuot+te18GBF46z5e7HWLpgNh+56qJGl2RmNilmXKADfPTNF/PMkRP84fceZV5bC//jypc1uiQzszM2IwNdEr/3jks4dnKQT27bxfPHTvLRN1+MpEaXZmY2YTMy0AFaszNfFsxpZcvdj/Hc0X4+sfHl5HMOdTNL04wNdChcxOt33/5yzp7byp/882M8+dwx/vC9r+TseW2NLs3M7LTNqLNcqpHE/1n/s3zy7S/nh48d5K2f+zfuf+K5RpdlZnbaZnygD3vfqy/gG//zdeRy4j2f/yFb7u6h3+eqm1lCHOhFLlm+kL//X2/g6rVL+cxdu3nbH/+A+9xbN7NEONDLLJzbyp+871V84f2dHD7Wzzv/9P/z0a89wL5DRxtdmpnZmGb0QdGxXL12Ka9dvZg/+t5P+OIPH+db9z/JuztXcMO61aw4Z26jyzMzq6BGTX/v7OyM7u7uhrz26Xr68HG23N3D1u1PMDgUvPFnl/L+117A6y9cQs6nOZpZHUm6NyI6qz7mQK/d/ueP8aUfPc7W/9jHwZdOsmzRHN566fn80qXnc8myhZ6YZGZTzoE+yU4MDPIPDz/Nt3fs598e7aN/MPiZhbN5w5p2Xr9mCa9dvZgl82c1ukwza0IO9Cl0+Gg/d+18mrt3H+Dfe57lyPHCtdaXLZrDK1Ys5BXLF3Hp8kVctHQ+58xrcy/ezM7IGQe6pPXAHwF54M8j4lNlj88C/gp4FXAQeG9E7B3rOZsl0IsNDA7x4JOH6d57iAd6D/Ng7/PsO3Rs5PEFs1tY1T6f1UvmsXLxXM5bMJulC2Zz7oJZLF0wm3PmtnlM3szGNFagj3uWi6Q8sAW4GugFtkvqiogfF232QeC5iLhQ0ibg08B7z7z0tLTkc1y+8mwuX3n2SNvBF0/w0JOH2dP3Ej999iX2PPsi9+w5yDfvf7Ly53NiyfxZLJzTysI5rSyY08KCbPms2a3Mac0zpzXH7NZ89lW8XFhvzedoyYl8TrTmc+RzqrruvxTMmk8tpy1eAfRExB4ASVuBjUBxoG8EPp4tfx34f5IUvoMEi+fPYt3F57Lu4tL2kwND9L14gmeOHOfAkeM8c6Sw3PfCCQ4f6+fI8X72P3+cXU+9wJHj/bxwfHJvm5dT4RdQTiCEBAJy2YIoXBahuL3wO+BUm5S1Z9tS3JZtMxET/WUz4V9RE/jBZP5tNi19+E1reNsrfmbSn7eWQF8G7Cta7wVePdo2ETEg6TCwGHi2eCNJ1wPXA6xcuXKCJTeHtpYcyxbNYdmiOTVtPzQUnBgY4nj/IMf6BzneP8jx/iGO9Q9yon+Q4wODHDs5xMDQEAODweBQMDAUDA4N0T/Kev/QEBEQEYXvQAQMZb+HI2KkLQiGorAM2fZl7UHhSYJTz3G6JtoFmGjPYSJ9jgn3Uib8b5vx/aKms3BO65Q8by2BXq1zUP4Jq2UbIuJ24HYojKHX8NqWyeXEnLY8c9rynD3+5mY2A9Uy9b8XWFG0vhzYP9o2klqAhcChySjQzMxqU0ugbwfWSFolqQ3YBHSVbdMFXJctvwv4vsfPzczqa9whl2xMfDNwF4XTFu+IiJ2SbgW6I6IL+AvgryX1UOiZb5rKos3MrFJNF+eKiG3AtrK2W4qWjwPvntzSzMzsdPjyuWZmTcKBbmbWJBzoZmZNwoFuZtYkGna1RUl9wOMT/PEllM1CnUama22u6/S4rtM3XWtrtrouiIj2ag80LNDPhKTu0a421mjTtTbXdXpc1+mbrrXNpLo85GJm1iQc6GZmTSLVQL+90QWMYbrW5rpOj+s6fdO1thlTV5Jj6GZmVinVHrqZmZVxoJuZNYnkAl3Sekm7JfVIuqmBdayQdLekXZJ2SvqNrP3jkp6UtCP7eksDatsr6aHs9buztnMk/aOkR7Pvdb1PhqSLi/bJDklHJH2kUftL0h2SDkh6uKit6j5Sweeyz9yDki6vc12fkfRI9tp3SlqUtXdIOla07/6sznWN+t5J+li2v3ZL+q9TVdcYtX2lqK69knZk7XXZZ2Pkw9R+xgq3H0vji8Llex8DXga0AQ8AaxtUy/nA5dnyWcBPgLUU7q360Qbvp73AkrK224CbsuWbgE83+H18GrigUfsLuBK4HHh4vH0EvAX4DoU7c70G+FGd63oz0JItf7qoro7i7Rqwv6q+d9n/gweAWcCq7P9svp61lT3++8At9dxnY+TDlH7GUuuhj9ywOiJOAsM3rK67iHgqIu7Lll8AdlG4t+p0tRH4Yrb8ReCXG1jLm4DHImKiM4XPWET8K5V31RptH20E/ioK7gEWSTq/XnVFxHcjYvgu4fdQuGtYXY2yv0azEdgaESci4qdAD4X/u3WvTYW7cr8H+Jupev1RahotH6b0M5ZaoFe7YXXDQ1RSB3AZ8KOsaXP2Z9Md9R7ayATwXUn3qnBjboClEfEUFD5swLkNqGvYJkr/gzV6fw0bbR9Np8/dr1HoyQ1bJel+Sf8i6Q0NqKfaezed9tcbgGci4tGitrrus7J8mNLPWGqBXtPNqOtJ0nzgG8BHIuII8KfAauCVwFMU/tyrt1+IiMuBa4Bfl3RlA2qoSoXbGG4AvpY1TYf9NZ5p8bmTdDMwAHwpa3oKWBkRlwE3Al+WtKCOJY323k2L/ZW5ltLOQ133WZV8GHXTKm2nvc9SC/RablhdN5JaKbxZX4qIbwJExDMRMRgRQ8AXmMI/NUcTEfuz7weAO7Manhn+Ey77fqDedWWuAe6LiGeyGhu+v4qMto8a/rmTdB3wS8D7Iht0zYY0DmbL91IYq76oXjWN8d41fH/ByA3r3wF8ZbitnvusWj4wxZ+x1AK9lhtW10U2NvcXwK6I+IOi9uJxr7cDD5f/7BTXNU/SWcPLFA6oPUzpjbyvA75dz7qKlPSYGr2/yoy2j7qA92dnIrwGODz8Z3M9SFoP/CawISKOFrW3S8pnyy8D1gB76ljXaO9dF7BJ0ixJq7K6/qNedRW5CngkInqHG+q1z0bLB6b6MzbVR3un4OjxWygcMX4MuLmBdbyewp9EDwI7sq+3AH8NPJS1dwHn17mul1E4w+ABYOfwPgIWA/8EPJp9P6cB+2wucBBYWNTWkP1F4ZfKU0A/hd7RB0fbRxT+HN6SfeYeAjrrXFcPhfHV4c/Zn2XbvjN7jx8A7gPeVue6Rn3vgJuz/bUbuKbe72XW/pfAh8q2rcs+GyMfpvQz5qn/ZmZNIrUhFzMzG4UD3cysSTjQzcyahAPdzKxJONDNzJqEA93MrEk40M3MmsR/Ahsl9VtdWO3LAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(errs_bpc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.50000155, 0.50000155],\n",
       "       [0.        , 0.        ],\n",
       "       [0.        , 0.        ],\n",
       "       [0.        , 0.        ],\n",
       "       [0.25000193, 0.25000193]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Trained perfectly\n",
    "bpcheck.get_output([0,1], 12, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "fc = GraphNet(7)\n",
    "fc.connect(0,1)\n",
    "fc.connect_complete([1,2,3,4,5])\n",
    "fc.connect(5,6)\n",
    "#fc.activation(6, 'sigma')\n",
    "fc.init_markov()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x140227d9898>]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD5CAYAAAAp8/5SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXyU5b338c8vO1nIzk4gCZuIoBAJIgJWq9Aq1Kqt1mNt66n6nENftrWLnnPa4/H4PEe7aD2VVm1taxfr1lapuxXESmWXRcSEkAgESEgISxYgCbmeP2aCMU7IJJnJLPm+X6+8mJn7mplfbibf3Lnu674uc84hIiLRKybUBYiISHAp6EVEopyCXkQkyinoRUSinIJeRCTKKehFRKJcnD+NzGwB8AAQC/zSOXdPp+1zgZ8AU4FrnHPPdNh2A/Af3rt3O+ceO9175eTkuLFjx/r9DYiICGzYsKHWOZfra1u3QW9mscBS4JNAJbDOzJY5597r0Gw38CXgW52emwX8J1AEOGCD97mHunq/sWPHsn79+u7KEhGRDsxsV1fb/Om6mQmUOefKnXPNwBPA4o4NnHMfOOe2AG2dnnsp8Jpzrs4b7q8BC3pUvYiI9Ik/QT8S2NPhfqX3MX/49Vwzu8nM1pvZ+pqaGj9fWkRE/OFP0JuPx/ydN8Gv5zrnHnHOFTnninJzfXYxiYhIL/kT9JXA6A73RwH7/Hz9vjxXREQCwJ+gXweMN7N8M0sArgGW+fn6rwCXmFmmmWUCl3gfExGRftJt0DvnWoEleAJ6O/CUc26bmd1lZosAzOxcM6sErgYeNrNt3ufWAf+N55fFOuAu72MiItJPLNymKS4qKnIaXiki0jNmtsE5V+Rrm66MDRPOOZ5at4fGE62hLkVEooyCPkxs23eU7/xpC0+s29N9YxGRHlDQh4myAw0ArCk/GOJKRCTaKOjDRHmNJ+jXflBHW1t4nTcRkcimoA8TO2sbATjc1EJJdX2IqxGRaKKgDxPlNY1MGpYGwGp134hIACnow0Bbm6OitoE543IYlTlIQS8iAaWgDwP7jhzjeEsbBbmpzCrIZm2F+ulFJHAU9GGgvMbTP1+Qm8KsgmwONbVQekD99CISGAr6MNA+4qYwN5Xi/CwAVu9U942IBIaCPgzsrGkkLSmOnNQERmclMzJjEGsqNCWQiASGgj4MlNc2UJCbipln+v7igizWqJ9eRAJEQR8GymsaKcxJOXV/VkE2dY3N7PBeLSsi0hcK+hBrPNHK/iPHKRySeuqx8wqyAVhToX56Eek7BX2IVXiviC3ocEQ/KnMQI9KTNJ5eRAJCQR9iO70jbgpyPzyiNzNmFWSzpryOcFsvQEQij4I+xMprGjGDMdnJH3l8VkE2BxubT81qKSLSWwr6ECuvbWR0ZjJJ8bEfeby4wDueXt03ItJHCvoQ23mggYLclI89npeVzPD0JFZrPL2I9JGCPoQ8k5k1UpCT+rFtZkZxfhZryg+qn15E+kRBH0JVR49zrOWkzyN68PTT1zY0nzphKyLSGwr6EGqfzKww9+NH9OAJeoDV5eq+EZHeU9CH0M5Tk5n5PqIfk53M0MGJOiErIn2ioA+h8poGUhPjyE1L9Lm9fTz9ao2nF5E+UNCHUHltIwW5KacmM/PF009/gnLvFbQiIj2loA+h8prGj0x94Mup+enVfSMivaSgD5Gm5lb2Hj7W5YnYdvk5KQxJS2SNTsiKSC8p6EPk1GRm3QT9h/30Gk8vIr2joA+RjuvEdqe4IIsD9SdO/XIQEekJBX2ItE9mlt9NHz18OJ5eywuKSG8o6ENkZ00DIzMGfWwyM18KclLISdV4ehHpHQV9iLSvE+sPTz99lvrpRaRX/Ap6M1tgZiVmVmZmt/vYnmhmT3q3rzGzsd7H483sMTPbambbzeyOwJYfmZxzVPgxtLKjWQXZVB89wa6DTUGsTESiUbdBb2axwFJgITAZuNbMJndqdiNwyDk3DrgfuNf7+NVAonPuLGAGcHP7L4GBrProCRqbT3Y59YEvszQ/vYj0kj9H9DOBMudcuXOuGXgCWNypzWLgMe/tZ4CLzHO5pwNSzCwOGAQ0A0cDUnkE+3COG/+6btrb5qQmKOhFpMf8CfqRwJ4O9yu9j/ls45xrBY4A2XhCvxHYD+wGfuSc+9jQETO7yczWm9n6mpqaHn8Tkabcxzqx3TEziguyWVOheW9EpGf8CXpfE7F0Tpqu2swETgIjgHzgNjMr+FhD5x5xzhU554pyc3P9KCmy7axpJCUhlqGDfU9m1pVZ+VnsP3Kc3XXqpxcR//kT9JXA6A73RwH7umrj7aZJB+qALwAvO+danHMHgFVAUV+LjnTltY3kdzOZmS+nxtNrOgQR6QF/gn4dMN7M8s0sAbgGWNapzTLgBu/tq4DlztO/sBv4hHmkALOA9wNTeuTaeaChR/3z7cYNSSU7Rf30ItIz3Qa9t899CfAKsB14yjm3zczuMrNF3maPAtlmVgZ8E2gfgrkUSAXexfML49fOuS0B/h4iyvGWk+w7csznOrHd8fTTazy9iPRMnD+NnHMvAi92euz7HW4fxzOUsvPzGnw9PpBV1DbinH9z3PgyqyCbF7dWUXnoGKOzkgNcnYhEI10Z2896MpmZL8X5nn76t9V9IyJ+UtD3s/Yx9L3pugEYPySVLPXTi0gPKOj7Wbl3MrNBCd1PZuZLTIxRnJ+lkTci4jcFfT9rXye2L4rzs9h7+Bh7NJ5eRPygoO9Hzjm/1ontzqxCzU8vIv5T0PejA/UnaDjRSuGQ3vXPt5swJI2M5Hj104uIXxT0/aivJ2LbtffTK+hFxB8K+n7U16GVHc0qyKby0DEqD6mfXkROT0Hfj8prGhkUH8uwwUl9fq328fQafSMi3VHQ96OdNQ0U5KYQE9Ozycx8mTQsjfRB6qcXke4p6PtRT9aJ7c6p8fQaeSMi3VDQ95PjLSepPHSsz0MrOyouyGZ3XRN7Dx8L2GuKSPRR0PeTXQeb+jSZmS/t68iuUfeNiJyGgr6f9Gad2O6cMWww6YPidUJWRE5LQd9PPlwnNnBH9DExxrljs1hdoSN6Eemagr6flNc0Mjw9ieQEv5YA8Nusgix2HWxi/xH104uIbwr6frIzAJOZ+aJ1ZEWkOwr6fuCco7yX68R254zhg0lLitN4ehHpkoK+H9Q0nKD+RGtAh1a2i9V4ehHphoK+H3w4x03gj+jBMx1CRW0jVUeOB+X1RSSyKej7QSAnM/PlVD+9Rt+IiA8K+n6ws6aBpPgYRqQPCsrrTx4xmLTEOFbrhKyI+BDYsX7iU3lNA/k5qQGZzMyX2Bjj3PysHl8h23qyjcPHWjjU2MzBxmaONZ9kVkF2r9ezFZHwpKDvB+W1jUwZmR7U95hVkMXy9w+wac9hAOoaT1DX+GGIH2pspq6pmboOt48ca8G5j77O5OGDeeSLMxiVmRzUekWk/yjog+xE60n21DWxeNqIoL5Pez/9Z5au+ti2+FgjMzmBrBTP1xkjBpOdkvCRx7JSEqhtOMF/PPsuix5cxc+vm06x9zVFJLIp6INs18Em2lzwRty0O2tkOj/5/Nm0nGz7SHhnpSSQmhiHmX/dRlNGpvPV367nul+u4T8Xncn1s8YEtW4RCT4FfZCVB2EyM1/MjM+cM7LPr1OYm8qz/3o+X39iE9979l227z/KnZefSUKcztuLRCr99AbZTu/QyvwgDa0MhsFJ8fzii0X8y/xCHl+zm+t+uZqa+hOhLktEeklBH2TlNY0MHZxIamJk/fEUG2N8Z8EkfnrtOWzde4TFD77Fu3uPhLosEekFBX2Q7axpoCAnuN02wXT5tBE8c8tszIwrf/4Pntu0N9QliUgPKeiDyDlHeU0DhUMip9vGlykj03luyflMG5XBrU9s4n9e2s7JNtf9E0UkLCjog+hgYzNHj7dG9BF9u5zURH7/z8X806w8Hl5Zzld+s44jx1pCXZaI+MGvoDezBWZWYmZlZna7j+2JZvakd/saMxvbYdtUM3vbzLaZ2VYzSwpc+eEt2HPc9LeEuBju/sxZ/N8rprCqrJbPLF1F2YGGUJclIt3oNujNLBZYCiwEJgPXmtnkTs1uBA4558YB9wP3ep8bB/weuMU5dyYwHxgwh4HBWCc2HFxXPIbHvzqLo8dauGLpKl7fXh3qkkTkNPw5op8JlDnnyp1zzcATwOJObRYDj3lvPwNcZJ4rdC4BtjjnNgM45w46504GpvTwV17TQGJcDCMygjOZWSjNzM9i2dfmMCYnmX/+7XqWrijDdZ5PQUTCgj9BPxLY0+F+pfcxn22cc63AESAbmAA4M3vFzDaa2Xd8vYGZ3WRm681sfU1NTU+/h7BVXtNIfk4KsUGazCzURmYM4umbZ3P51BH88JUSvvbHd2g80RrqskSkE3+C3ldKdT5066pNHDAHuM777xVmdtHHGjr3iHOuyDlXlJub60dJkaE8SOvEhpNBCbE8cM3Z3L5wEi9s3c/F963k5Xf36+heJIz4E/SVwOgO90cB+7pq4+2XTwfqvI+vdM7VOueagBeB6X0tOhI0t7axu64pKkbcdMfMuGVeIc/cMpuM5ARu+f1Gvvybdew62Bjq0kQE/4J+HTDezPLNLAG4BljWqc0y4Abv7auA5c5zSPcKMNXMkr2/AOYB7wWm9PC2u66Rk20u4sfQ98SMMZn8dcn5fO+yyayrqOOT97/JA3/bwfGWAXNaRiQsdRv03j73JXhCezvwlHNum5ndZWaLvM0eBbLNrAz4JnC797mHgPvw/LLYBGx0zr0Q+G8j/LTPcTMQjug7iouN4cY5+bx+23wumTyU+/9WysIH/s6bpdFz7kUk0li49aUWFRW59evXh7qMPvv5Gzu59+X32XrnJaQlxYe6nJD5+44avv/cNipqG/n01OF879OTGZY+YC6lEOk3ZrbBOVfka5uujA2SnTUN5KYlDuiQB7hgfC4vf/0CbvvkBP72XjUX/fgNfvn3clpPtoW6NJEBQ0EfJOU1DRRG+YgbfyXGxfK1i8bz2jfmMTM/i7tf2M5lP32L9R9oMXOR/qCgDxLP0MqB1T/fnbzsZH71pXN56J9mcPRYC1c99DbfeWYzdY3NoS5NJKop6IOgrrGZw00tFOToiL4zM2PBlGG89s153DyvgD9v3MsnfvwGT6zdTZtmxBQJCgV9EETrHDeBlJIYxx0Lz+DFWy9gwtA0bv/zVq586B9srdTiJiKBpqAPgv5aJzYaTBiaxpM3zeK+z01jT10Ti5a+xbef3syBo8dDXZpI1FDQB0F5TSMJcTGMzIy+ycyCwcz47PRRLP/WfG66oIBnN+1l/o/eYOmKMl1sJRIACvog2FnTyNjs5KidzCxYBifFc8enzuC1b8zjgvE5/PCVEi768Uqe37JPc+eI9IGCPgjKI3yd2FAbm5PCw9cX8fhXixk8KJ4lj7/D5x5+W/33Ir2koA+wlpOeycwG0hw3wTK7MIfnvzaH//nsWZTXNLJo6Vt86+nNVKv/XqRHFPQBtruuidY2pyP6AImNMa6dmceKb8/nprkFLNu0jwvVfy/SIwr6AIu2dWLDxeCkeO5YeAavfXMuc8fnqv9epAcU9AHWPoZeV8UGx5jsFB66fsZH+u+vfuhttlQeDnVpImFLQR9g5TUN5KQmkj5oYE9mFmzt/ff3fPYsPjjYyKIHV3HbU5vZf+RYqEsTCTtxoS4g2pTXRP/ygeEiNsa4ZmYen546nAdXlPHrtz7gr1v28cVZY/g/8wvJTk0MdYkiYUFH9AFWXtuoWSv7WZq3//712+axaNoIfrWqgrk/WMF9r5Vy9HhLqMsTCTkFfQAdamymrrFZI25CZHRWMj+6ehqvfmMu8ycO4X9f38HcH6zgoZU7OdasEToycCnoA6i81jvHjcbQh9S4IWksvW46z39tDmePzuCel95n7g9X8Lu3P6C5VQueyMCjoA+ggbpObLiaMjKd33x5Jk/fch752Sl877ltfOLHb/CnDZWc1JTIMoAo6AOovKaR+FhjlCYzCyvnjs3iyZtn8dhXZpKRHM9tT2/m0p+8yUtb92sMvgwICvoA2lnTwJjsFOJitVvDjZkxb0Iuf10yh59fNx3nHP/nDxtZ9OAqVpbWKPAlqimRAkjrxIY/M2PhWcN59Rvz+NHV0zjU1MwNv1rL5x9ZzTqtYStRSkEfIK3eycx0RWxkiI0xrpoxiuW3zee/F59JRW0jVz/0Ntc/uobV5Qd1hC9RRUEfIHsOHaPlpNM6sREmIS6G688by5vfvpB/+9Qktu+v55pHVnP1Q2/zRskBBb5EBQV9gJRW1wOepfEk8gxKiOWmuYW89d0L+a9FZ7Lv8DG+9Ot1LHpwFS+/W6WFyyWiKegDpLTKE/Tjh6rrJpIlxcdyw+yxvPHtC7n3yrOoP97CLb/fwIIH3uS5TXtpPalx+BJ5FPQBUlJdT15WMskJmj4oGiTExfD5c/P42zfn8cA1ZwNw6xObuOi+lTyxdrcuvJKIoqAPkNLqenXbRKG42BgWnz2Sl2+dy8PXz2BwUjy3/3kr83+4gt+sqtDiJxIRFPQB0NzaRnlNIxOHqdsmWsXEGJeeOYxlS87nsa/MZGTmIO7863vMuXc5D63cScOJ1lCXKNIl9TMEQEVtI61tTkf0A0D7hVfzJuSypvwgD64o456X3ufnb+zkS7PH8qXZY8lMSQh1mSIfoaAPgBLviJuJwxT0A0lxQTbFBdls3nOYB1eU8cDrO3j4zZ18dvoovnJ+PuOG6C88CQ8K+gAoraonLsY0mdkANW10Br/4YhElVfX86q0KntlQyeNrdjN/Yi43zslnzrgczCzUZcoA5lcfvZktMLMSMyszs9t9bE80sye929eY2dhO2/PMrMHMvhWYssNLSXU9+TkpJMTplMdANnFYGvdeNZV/3P4JvnHxBN7de5TrH13Lgp/8nSfX7daJWwmZbpPJzGKBpcBCYDJwrZlN7tTsRuCQc24ccD9wb6ft9wMv9b3c8FRaXc8EdduIV05qIrdePJ5Vt1/ID6+aihl8909bOf+e5dz3Wik19SdCXaIMMP4cgs4Eypxz5c65ZuAJYHGnNouBx7y3nwEuMu/fqmb2GaAc2BaYksNLU3Mru+uamKgTsdJJYlwsVxeN5qVbL+DxrxZzTl4GP12+g/PvWc63nt7Me/uOhrpEGSD86aMfCezpcL8SKO6qjXOu1cyOANlmdgz4LvBJoMtuGzO7CbgJIC8vz+/iw0HZgQac09QH0jUzY3ZhDrMLc6iobeTXqyp4en0lz2yoZHZhNjfOyefCiUOIiVE/vgSHP0f0vj59nSf+6KrNfwH3O+caTvcGzrlHnHNFzrmi3NxcP0oKHyVVGnEj/svPSeGuxVNYfcdF3L5wEhW1jdz42Houum8lv337Axo1Hl+CwJ8j+kpgdIf7o4B9XbSpNLM4IB2ow3Pkf5WZ/QDIANrM7Lhz7sE+Vx4mSqvrSYyLIS8rOdSlSARJT47nlnmF3Dgnn5fereLRtyr4/nPb+MHLJVxxzki+UJzHGcMHh7pMiRL+BP06YLyZ5QN7gWuAL3Rqswy4AXgbuApY7jzzu17Q3sDM7gQaoinkAUqqGxg/NJVY/dktvRAfG8OiaSO4fOpwNu4+zB/W7OKp9Xv43epdTM/L4LriMXx66nCS4mNDXapEsG67bpxzrcAS4BVgO/CUc26bmd1lZou8zR7F0ydfBnwT+NgQzGhVWqU5bqTvzIwZYzK573Nns+bfLuJ7l03m8LEWbnt6M8X/73X++/n32Flz2h5QkS5ZuC2sUFRU5NavXx/qMvxypKmFaXe9yh0LJ3HzvMJQlyNRxjnH6vI6/rBmF69sq6LlpGNWQRbXFY/h0jOH6boN+Qgz2+CcK/K1TVfG9kHpAe9iIzoRK0FgZpxXmM15hdnU1J/g6Q17eHzNbr72x3fISU3g6qLRXHtuHnnZOj8kp6eg74P320fcqOtGgiw3LZF/mT+OW+YW8veyWv6wehcPr9zJQyt3csH4XK4rzuOiSUOIi9VRvnycgr4PSqvqSUuMY3h6UqhLkQEiJubD2TP3HznGk+v28MTaPdz8uw0MG5zEVTNGceWMUeRr7WLpQEHfByXeqQ80YZWEwvD0QXz94gksuXAcy98/wONrd/OzN8p4cEUZ0/MyuGrGaD49dTjpg+JDXaqEmIK+l5xzlFbXs3DK8FCXIgNcXGwMl5w5jEvOHEb10eP85Z29/GlDJf/2l63c+ddtXDJ5KFfNGMUF43M1DHiAUtD3Uk39CQ43tTBRi4FLGBk6OIlb5hVy89wCtu49wjMbKlm2eR/Pb9nPkLRErjhnJFfOGKUhwQOMgr6X2hcb0YgbCUdmxtRRGUwdlcG/f/oMVrx/gGc2VPLLtyp4+M1ypo5K58rpo1g0bYRWxBoAFPS9VKIRNxIhEuNiWTBlOAumDKe24QTPbdrHMxsq+c9l27j7hfe4aNJQrpwxivkTc4nXqJ2opKDvpdLqenJSE8hOTQx1KSJ+y0lN5MY5+dw4J5/39h3lTxsrefadvby8rYrslAQumzqcy6eNYHpepmbTjCIK+l4qqW5QP6dEtMkjBjN5xGRuXziJlSU1/GljJX9ct4fH3t7FiPQkPu0N/bNGpmtkWYRT0PdCW5tjR3U9nysa3X1jkTAXHxvDxZOHcvHkodQfb+Fv26t5fvN+fvOPD/jF3yvIy0rm8mnDuWzqCCZpOHFEUtD3wt7Dx2hqPqk56CXqpCXFc8U5o7jinFEcaWrhlW1V/HXLPh5aWc7SFTsZNyT1VPdOYa5GnEUKBX0vtJ+IVdeNRLP05Hg+d+5oPnfuaGobTvDSu1U8v3kfD7y+g5/8bQeThw/msmnDuXzqCEZrPYawpqDvhVNDKzWGXgaInNRErp81hutnjaHqyHFe2Lqf57fs4wcvl/CDl0uYNjqDy6cOZ+FZwxmZMSjU5UonCvpeKK2uZ2TGINKSdGm5DDzD0pNOjdzZU9fEC1v389fN+7j7he3c/cJ2powczKWTPVfqThiaqj79MKCg74WSqnodzYsAo7OSuWVeIbfMK6SitpFXtlXx6rYqfvxaKT9+rZSx2cme6RkmD+WcvExNwRAiCvoeajnZRnlNI/MmRtYi5iLBlp+Tcir0Dxw9zmvbq3l1WzW/XlXBI2+Wk5OayCcnD+GSycOYPS6bxDgtj9hfFPQ9tOtgI80n23RFrMhpDBmcxHXFY7iueAxHj7fwRkkNr2yrYtmmffxx7R5SEmKZP2kIl545jPkTcxmsbtCgUtD3UEmVZ91OjbgR8c/gpHgWTRvBomkjONF6kn+UHeTV96p47b1qXtiyn/hY47zCHC6ZPJSLzxjKMK3vEHAK+h4qqa4nxmDcEPXRi/RUYlwsF04awoWThnD3Zxzv7D7Eq+9V88q2Kv7j2Xf5j2ff5Yzhg7lwYi7zJw5hel6GVs0KAAV9D5VW1TM2O4WkePUvivRFbIxRNDaLorFZ3LFwEqXVDawoOcCK9w/w8Jvl/OyNnQxOiuOCCblcOHEI8ybkkpumuaV6Q0HfQ6XV9eq2EQkwM2PisDQmDkvjlnmFHD3ewqodtZ7gL6nhhS37AThrZLrnaH/SEKaNytAoHj8p6HvgeMtJPjjYyGXTRoS6FJGoNjgpnoVneS7AamtzvLf/KG+UHOCNkhoeXFHG/y4vIzM5nnkTPF08cyfkkqV59bukoO+BsgMNtDnNQS/Sn2JijCkj05kyMp0lnxjP4aZm3txRyxvvH2BlaQ3PbtqHGZw9OoO543OZMz6Hs0dnaG79DhT0PVDqnfpg4jCdiBUJlYzkhFOjeNraHFv3HjnVxfPT5Tt44PUdJCfEUpyfxfnjcpgzPoeJQwf2rJsK+h4oqa4nITaGMdkpoS5FRPAc7U8bncG00Rl8/eIJHGlq4e3yg6wqq2VVWS0rSrYDkJOawOzCHOaMy2H2uGxGZQ6sSdgU9D1QWlVPQW6K/iQUCVPpyfEsmDKMBVOGAbDv8LFTof9W2UGWbd4HwNjsZM/R/rgczivMJiM5uvv3FfQ9UFrdQNHYzFCXISJ+GpExiKuLRnN10Wicc+w40MBbOzzB/9ymffxhzW7MYMqIdM73hv6MMZmkJkZXNEbXdxNE9cdb2Hv4GF8YmhfqUkSkF8yMCUPTmDA0ja/MyaflZBtbKg/z1g5PV8+jb5Xz0MqdxMYYU0YMZmZ+FjPzszl3bGbEH/Er6P1UWu2Z+kAjbkSiQ3xsDDPGZDFjTBa3XjyexhOtbNx9iLUVdaypqOOxt3fxi79XYOb5uS9uD/78TIakRdY0DQp6P3044kZBLxKNUhLjuGB8LheM98xMe7zlJJv3HGZtRR1rP6jj6Q2VPPb2LgAKclIoLsg6ddQf7outKOj9VFJVT3JCbNj/h4pIYCTFx1JckE1xQTbgmaL83b1HPMFfUcfzW/bzx7V7ABiZMYjigiyKxmQxY0wm44ekEhNGV+36FfRmtgB4AIgFfumcu6fT9kTgt8AM4CDweefcB2b2SeAeIAFoBr7tnFsewPr7TWl1PeOHpoXVf56I9J/42BjOycvknLxMbp5XyMk2R0lVPWsqDrK2oo6VJTX8eeNeANIS4zg7L4Nz8jKZnpfBOaMzSU8O3VTM3Qa9mcUCS4FPApXAOjNb5px7r0OzG4FDzrlxZnYNcC/weaAWuNw5t8/MpgCvACMD/U30h9Lqej4xaUioyxCRMBEbY0weMZjJIwbz5fPzcc7xwcEmNu46xMbdh9i4+zAPLt9Bm/O0Hzcklel5GUzPy2T6mEzG5fbfUb8/R/QzgTLnXDmAmT0BLAY6Bv1i4E7v7WeAB83MnHPvdGizDUgys0Tn3Ik+V96PahtOUNvQrMnMRKRLZkZ+Tgr5OSlcOWMUAA0nWtmy5/Cp4H/1vWqeWl8JQFpS3Kkj/ul5mZydlxG0BVj8CfqRwJ4O9yuB4q7aOOdazewIkI3niL7dlcA7vkLezG4CbgLIywu/4Ys6ESsivZGaGMfscTnMHpcDgHOO8tpG71H/Yd7ZfYgHXt+Bc2AGV00fxQ+vnhbwOvwJel9/W7ietDGzM/F051zi6w2cc48AjwAUFRV1fu2QK63yBhKlAJcAAAl0SURBVL2O6EWkD8yMwtxUCnNTubpoNOC5RmfzniNs3H0oaIM9/An6SmB0h/ujgH1dtKk0szggHagDMLNRwF+ALzrndva54hAoqW4gIzleix6ISMClJcUzZ7xn8rVg8WfSlnXAeDPLN7ME4BpgWac2y4AbvLevApY755yZZQAvAHc451YFquj+1r7YyECe/U5EIle3Qe+cawWW4Bkxsx14yjm3zczuMrNF3maPAtlmVgZ8E7jd+/gSYBzwPTPb5P2KqKErzjlKq+rVbSMiEcuvcfTOuReBFzs99v0Ot48DV/t43t3A3X2sMaT2HzlO/YlWJuhErIhEKM23242Sap2IFZHIpqDvRvuImwlDtaqUiEQmBX03SqrrGTo4MeKnKRWRgUtB3432ETciIpFKQX8aJ9scO6ob1D8vIhFNQX8au+uaONHaphE3IhLRFPSnUaKpD0QkCijoT6N9MrPxGnEjIhFMQX8aJdX15GUlk5yghbhEJHIp6E+jtEojbkQk8inou3Ci9SQVtY1MHKZuGxGJbAr6LlTUNtLa5nRELyIRT0HfhVMjbjS0UkQinIK+C6XV9cTFGAU56roRkcimoO9CSVUD+TkpJMRpF4lIZFOKdaG0ul5XxIpIVFDQ+9DU3MruuiZdESsiUUFB78OO6gYAjbgRkaigoPfh1KpS6roRkSigoPehtKqexLgY8rKSQ12KiEifKeh9KKmuZ/zQVGJjLNSliIj0mYLeB60qJSLRREHfyeGmZqqPntCIGxGJGgr6TkrbR9zoRKyIRAkFfSftI24mKehFJEoo6DspraonLSmOYYOTQl2KiEhAKOg7KamuZ+LQNMw04kZEooOCvgPnnOa4EZGoo6DvoKb+BIebWjTiRkSiioK+g/YTsRpDLyLRREHfQfuqUhOGarEREYkeCvoOSqvryUlNJDs1MdSliIgEjF9Bb2YLzKzEzMrM7HYf2xPN7Env9jVmNrbDtju8j5eY2aWBKz3wSqobmDhMR/MiEl26DXoziwWWAguBycC1Zja5U7MbgUPOuXHA/cC93udOBq4BzgQWAD/zvl7YaWtz7NAcNyISheL8aDMTKHPOlQOY2RPAYuC9Dm0WA3d6bz8DPGiegeiLgSeccyeACjMr877e24Ep/0PvVx3la4+/0+vnn3SOpuaTGnEjIlHHn6AfCezpcL8SKO6qjXOu1cyOANnex1d3eu7Izm9gZjcBNwHk5eX5W/tHJMXFMr6PJ1HPHpXBJ84Y0qfXEBEJN/4Eva9LRJ2fbfx5Ls65R4BHAIqKij623R9jc1L42XUzevNUEZGo5s/J2EpgdIf7o4B9XbUxszggHajz87kiIhJE/gT9OmC8meWbWQKek6vLOrVZBtzgvX0VsNw557yPX+MdlZMPjAfWBqZ0ERHxR7ddN94+9yXAK0As8Cvn3DYzuwtY75xbBjwK/M57srUOzy8DvO2ewnPithX4V+fcySB9LyIi4oN5DrzDR1FRkVu/fn2oyxARiShmtsE5V+Rrm66MFRGJcgp6EZEop6AXEYlyCnoRkSgXdidjzawG2NWHl8gBagNUTjCovr5RfX2j+vomnOsb45zL9bUh7IK+r8xsfVdnnsOB6usb1dc3qq9vwr2+rqjrRkQkyinoRUSiXDQG/SOhLqAbqq9vVF/fqL6+Cff6fIq6PnoREfmoaDyiFxGRDhT0IiJRLiKDvi+LlfdDbaPNbIWZbTezbWZ2q482883siJlt8n59v7/q61DDB2a21fv+H5tFzjz+17sPt5jZ9H6qa2KH/bLJzI6a2dc7ten3/WdmvzKzA2b2bofHsszsNTPb4f03s4vn3uBts8PMbvDVJkj1/dDM3vf+//3FzDK6eO5pPwtBrO9OM9vb4f/xU10897Q/70Gs78kOtX1gZpu6eG7Q91+fOeci6gvPVMk7gQIgAdgMTO7U5l+Ah7y3rwGe7Mf6hgPTvbfTgFIf9c0Hng/xfvwAyDnN9k8BL+FZJWwWsCZE/9dVeC4ECen+A+YC04F3Ozz2A+B27+3bgXt9PC8LKPf+m+m9ndlP9V0CxHlv3+urPn8+C0Gs707gW358Bk778x6s+jpt/zHw/VDtv75+ReIR/anFyp1zzUD7YuUdLQYe895+BrjIu1h50Dnn9jvnNnpv1wPb8bFObgRYDPzWeawGMsxseD/XcBGw0znXlyulA8I59yaetRY66vg5ewz4jI+nXgq85pyrc84dAl4DFvRHfc65V51zrd67q/Gs8BYSXew/f/jz895np6vPmx2fA/4Y6PftL5EY9L4WK+8cpB9ZrBxoX6y8X3m7jM4B1vjYfJ6ZbTazl8zszH4tzMMBr5rZBu/i7J35s5+D7Rq6/uEK9f4DGOqc2w+eX/CAr5Xlw2E/AnwFz19ovnT3WQimJd6upV910fUVDvvvAqDaObeji+2h3H9+icSg78ti5f3GzFKBPwFfd84d7bR5I57uiGnAT4Fn+7M2r/Odc9OBhcC/mtncTttDug/Ns2zlIuBpH5vDYf/5Kxw+i/+OZ4W3P3TRpLvPQrD8HCgEzgb24+ke6Szk+w+4ltMfzYdq//ktEoO+L4uV9wszi8cT8n9wzv2583bn3FHnXIP39otAvJnl9Fd93vfd5/33APAXPH8idxTqhd0XAhudc9WdN4TD/vOqbu/O8v57wEebkO5H78nfy4DrnLdDuTM/PgtB4Zyrds6ddM61Ab/o4n1Dvf/igM8CT3bVJlT7ryciMej7slh50Hn78x4Ftjvn7uuizbD2cwZmNhPP/8PB/qjP+54pZpbWfhvPSbt3OzVbBnzRO/pmFnCkvZuin3R5FBXq/ddBx8/ZDcBzPtq8AlxiZpnerolLvI8FnZktAL4LLHLONXXRxp/PQrDq63jO54ou3tefn/dguhh43zlX6WtjKPdfj4T6bHBvvvCMCCnFczb+372P3YXnAw2QhOdP/jJgLVDQj7XNwfOn5RZgk/frU8AtwC3eNkuAbXhGEKwGZvfz/ivwvvdmbx3t+7BjjQYs9e7jrUBRP9aXjCe40zs8FtL9h+eXzn6gBc9R5o14zvu8Duzw/pvlbVsE/LLDc7/i/SyWAV/ux/rK8PRvt38O20eijQBePN1noZ/q+533s7UFT3gP71yf9/7Hft77oz7v479p/9x1aNvv+6+vX5oCQUQkykVi142IiPSAgl5EJMop6EVEopyCXkQkyinoRUSinIJeRCTKKehFRKLc/wenTN/cGOrBpQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fc.get_output([1], 20, 20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.        , -0.04393084,  0.        ,  0.        ,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        , -0.09128415, -0.09127294, -0.09122264,\n",
       "         -0.09219175,  0.        ],\n",
       "        [ 0.        , -0.05071242,  0.        , -0.05351786, -0.05291075,\n",
       "         -0.06516243,  0.        ],\n",
       "        [ 0.        , -0.06939477, -0.07258681,  0.        , -0.0717497 ,\n",
       "         -0.08580668,  0.        ],\n",
       "        [ 0.        , -0.07501129, -0.07868004, -0.0782708 ,  0.        ,\n",
       "         -0.09337181,  0.        ],\n",
       "        [ 0.        , -0.06726503, -0.07135794, -0.0711031 , -0.07034976,\n",
       "          0.        , -0.12378711],\n",
       "        [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "          0.        ,  0.        ]]), 0.9100175704677823)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fc.backprop([[1]], [[1]], delay=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backprop-calculated gradient:\n",
      "[[ 0.         -0.04393084  0.          0.          0.          0.\n",
      "   0.        ]\n",
      " [ 0.          0.         -0.09128415 -0.09127294 -0.09122264 -0.09219175\n",
      "   0.        ]\n",
      " [ 0.         -0.05071242  0.         -0.05351786 -0.05291075 -0.06516243\n",
      "   0.        ]\n",
      " [ 0.         -0.06939477 -0.07258681  0.         -0.0717497  -0.08580668\n",
      "   0.        ]\n",
      " [ 0.         -0.07501129 -0.07868004 -0.0782708   0.         -0.09337181\n",
      "   0.        ]\n",
      " [ 0.         -0.06726503 -0.07135794 -0.0711031  -0.07034976  0.\n",
      "  -0.12378711]\n",
      " [ 0.          0.          0.          0.          0.          0.\n",
      "   0.        ]]\n",
      "Manually-calculated gradient:\n",
      "[[ 0.         -0.04393083  0.          0.          0.          0.\n",
      "   0.        ]\n",
      " [ 0.          0.         -0.09128469 -0.09127351 -0.09122317 -0.09219242\n",
      "   0.        ]\n",
      " [ 0.         -0.05071256  0.         -0.05351803 -0.05291097 -0.06516267\n",
      "   0.        ]\n",
      " [ 0.         -0.06939511 -0.07258719  0.         -0.07175003 -0.08580714\n",
      "   0.        ]\n",
      " [ 0.         -0.07501167 -0.07868041 -0.07827121  0.         -0.09337244\n",
      "   0.        ]\n",
      " [ 0.         -0.06726523 -0.07135829 -0.07110344 -0.07035007  0.\n",
      "  -0.12378703]\n",
      " [ 0.          0.          0.          0.          0.          0.\n",
      "   0.        ]]\n",
      "Difference:\n",
      "[[ 0.00000000e+00 -1.06026519e-08  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  5.42111832e-07  5.70974072e-07\n",
      "   5.30032959e-07  6.77389178e-07  0.00000000e+00]\n",
      " [ 0.00000000e+00  1.41461429e-07  0.00000000e+00  1.76985218e-07\n",
      "   2.19000829e-07  2.45300262e-07  0.00000000e+00]\n",
      " [ 0.00000000e+00  3.42242524e-07  3.78055447e-07  0.00000000e+00\n",
      "   3.30202666e-07  4.58883523e-07  0.00000000e+00]\n",
      " [ 0.00000000e+00  3.85220038e-07  3.69439179e-07  4.07252211e-07\n",
      "   0.00000000e+00  6.36522240e-07  0.00000000e+00]\n",
      " [ 0.00000000e+00  2.04866916e-07  3.47989881e-07  3.39725891e-07\n",
      "   3.08583149e-07  0.00000000e+00 -8.41926094e-08]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "fc.gradient_check([[1]], [[1]], delay=10, epsilon=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}