Print Things.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================\n",
      "Sigma: 1\n",
      "N: 70\n",
      "Rho: 1e-05\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.001075\n",
      "         Iterations: 5000\n",
      "         Function evaluations: 8371\n",
      "         Gradient evaluations: 8371\n",
      "N\n",
      "70\n",
      "\n",
      "sigma\n",
      "1\n",
      "\n",
      "rho\n",
      "1e-05\n",
      "\n",
      "W\n",
      "(array([[ 0.12499954,  0.24585235,  0.22118693,  0.74194332, -0.09482024,\n",
      "         0.21466581,  1.00631939,  0.39847833, -0.15230157,  0.40491002,\n",
      "        -0.84144332, -0.56935407,  0.0051245 , -1.16500378, -1.36063927,\n",
      "        -0.04643047, -0.61831262,  0.02902469, -1.42003251, -1.03031277,\n",
      "         0.76500671, -0.10400655, -0.02458852, -1.15920794, -1.08348087,\n",
      "         0.3317156 , -1.37170094,  0.98208925, -0.51705019,  0.05845514,\n",
      "        -0.29731198,  1.39011614,  0.18801272, -0.74983759,  1.22051389,\n",
      "        -0.937588  , -0.10540283, -1.65058822, -1.77238822,  0.17214847,\n",
      "         0.7059292 ,  0.07561034,  0.08839814, -0.31467994, -0.53822852,\n",
      "        -0.65418666, -0.65015316,  0.53474061,  0.06240729, -1.68101923,\n",
      "         0.22058677, -0.14155731, -0.33286267,  0.38213244,  0.77625546,\n",
      "         0.46767072, -0.45201478, -0.10268411,  0.22171532,  0.71730737,\n",
      "        -0.27113969, -0.08018227, -0.86794954, -0.84297552,  0.49135292,\n",
      "         0.84993239, -0.04945418,  1.08117523,  0.60896852, -0.59579113],\n",
      "       [ 0.27593663,  1.41400171,  0.10296647,  1.47674824, -1.76830087,\n",
      "         0.95794645,  0.29913446, -0.38267253, -0.07762103, -1.3673077 ,\n",
      "        -0.1735078 ,  0.42315053,  1.11782976, -0.36350032, -0.783623  ,\n",
      "        -0.88689652,  0.92088218,  0.68316911,  0.08501113,  0.56271699,\n",
      "         0.11999723,  0.74600987, -0.74476845, -0.27370742, -0.02246797,\n",
      "        -0.97985807,  0.15307835,  0.22508303,  0.20363275,  0.04720033,\n",
      "        -1.2075773 , -0.26029158, -0.48716319, -0.60760053,  0.3107951 ,\n",
      "         0.20820074,  1.28013424, -0.19886233, -0.09204441, -0.06716159,\n",
      "        -1.4185388 ,  0.05104357,  0.05145404,  1.61563706,  0.20914778,\n",
      "         0.14484477, -0.01699959, -0.85851891,  0.80238471,  0.3845806 ,\n",
      "         0.98095867, -0.6974407 ,  0.86068947, -1.38354502,  0.33421678,\n",
      "         1.61636978, -0.89774905, -0.5522063 ,  0.02264304, -0.33128098,\n",
      "        -1.36126827, -0.06126321, -0.83562612,  0.31272192, -0.35053736,\n",
      "         1.13102176, -0.54266739, -0.53231966,  0.58713611, -0.70082564]]),)\n",
      "\n",
      "b\n",
      "(array([ 0.41777225,  1.33159602, -1.22369965,  0.20176686, -0.08768939,\n",
      "        0.44973855, -1.28445575, -1.13142668,  0.31375907,  0.21304948,\n",
      "        0.93952208, -0.57242143, -0.23096088,  0.57351559,  0.45303943,\n",
      "       -1.45509063,  1.46063453,  0.81790943, -2.31574049, -0.22990613,\n",
      "       -0.72126336,  0.51255573,  1.06114902, -1.76877078,  1.61961545,\n",
      "        0.32434686,  1.2395335 ,  1.49880902, -0.58365245, -0.65924635,\n",
      "       -0.79913176, -0.44660641, -0.0214165 ,  0.64043467,  0.81390211,\n",
      "        0.57274908, -0.01225928,  1.37124344, -1.73510098,  1.88108913,\n",
      "        0.11302588, -0.68828546, -0.88993455,  0.56874692, -0.58799108,\n",
      "        0.38655926,  0.70025257, -0.14657988, -1.16003003, -1.28905175,\n",
      "       -0.29690887,  0.8202541 , -0.26076873, -1.23170785,  0.10711298,\n",
      "       -0.13498554, -0.18315785,  0.42348638, -0.00384295, -0.77874299,\n",
      "        0.8528825 ,  0.46366268,  0.42016405,  0.72574788, -0.96551683,\n",
      "       -1.05994143,  0.35715636,  0.35559663,  0.45485892,  2.65566392]),)\n",
      "\n",
      "v\n",
      "[ 0.66764764  1.19551082  0.8768322   0.53782673 -0.24233965  0.52226792\n",
      " -0.80680715 -0.61917614 -0.42160921  0.20447247  1.7105683  -1.5149043\n",
      "  0.45446687 -1.09809989 -0.43393989  1.20792549 -0.4438415  -0.97757442\n",
      " -1.81687658  0.58161033 -0.38086328 -0.02534793 -0.605021   -1.33085136\n",
      "  1.84580054  0.56977852 -1.64367198  0.82253677 -0.82076849  0.7164832\n",
      " -0.91456021 -0.17173229  0.38679644  0.86880136 -1.0316713  -0.3192794\n",
      " -0.22439573 -0.88322822  1.82339284  0.00477905 -0.72758652  0.86448318\n",
      "  1.68924693  0.64429076 -1.12988343 -0.29363689  0.9590286  -0.34352543\n",
      "  0.86927607  0.83155851 -0.58060631 -0.50281241 -2.37372347 -0.96178787\n",
      " -0.19761853 -0.68277778  0.86415835 -1.20824302 -0.13212319  0.00293111\n",
      "  1.18964936 -1.17672933  1.0310647  -0.02038023 -0.74312745  0.37125807\n",
      " -0.08378524 -0.51165196  0.41651763 -0.1148972 ]\n",
      "\n",
      "Convergence?\n",
      "False\n",
      "\n",
      "Best Method?\n",
      "CG\n",
      "\n",
      "Objective Function Start\n",
      "12.167999144399474\n",
      "\n",
      "Objective Function End\n",
      "0.001074901904179415\n",
      "\n",
      "Gradient Norm Start\n",
      "28.75940398996043\n",
      "\n",
      "Gradient Norm End\n",
      "0.00023641191497727958\n",
      "\n",
      "Computation time\n",
      "30.05\n",
      "\n",
      "Final Train Error\n",
      "7.627246067418733e-05\n",
      "\n",
      "Final Test Error\n",
      "0.0008121836270188385\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.model_selection import train_test_split\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "df = pd.read_csv('DATA.csv')\n",
    "\n",
    "train, test = train_test_split(df, test_size=0.255, random_state=1939671)\n",
    "\n",
    "X = np.array(train[['x1', 'x2']])\n",
    "y = np.array(train['y'])\n",
    "\n",
    "X_test = np.array(test[['x1', 'x2']])\n",
    "y_test = np.array(test['y'])\n",
    "\n",
    "\n",
    "def tanh(s, sigma):\n",
    "    prod = 2 * sigma * s\n",
    "    return (np.exp(prod) - 1) / (np.exp(prod) + 1)\n",
    "\n",
    "\n",
    "def feedforward(X, W, b, v, sigma):\n",
    "    linear_layer = (np.dot(X, W) + b)\n",
    "    activation = tanh(linear_layer, sigma)\n",
    "    pred = np.dot(activation, v)\n",
    "\n",
    "    return pred\n",
    "\n",
    "def backpropagation(x0, funcArgs):\n",
    "    \n",
    "    X = funcArgs[0]\n",
    "    y = funcArgs[1]\n",
    "    sigma = funcArgs[2]\n",
    "    N = funcArgs[3]\n",
    "    rho = funcArgs[4]\n",
    "    P = len(y)\n",
    "    \n",
    "    W = x0[:int(X.shape[1] * N)].reshape((X.shape[1], N))\n",
    "    b = x0[int(X.shape[1] * N):int(X.shape[1] * N + N)]\n",
    "    v = x0[int(X.shape[1] * N + N):]\n",
    "\n",
    "    linear_layer = (np.dot(X, W) + b)\n",
    "    a_2 = tanh(linear_layer, sigma)\n",
    "    dJdf = (1 / P) * (np.dot(a_2, v) - y)\n",
    "    dtanh = 1 - tanh(linear_layer, sigma) ** 2\n",
    "\n",
    "    dW1_1 = np.tensordot(dJdf, np.transpose(v), axes=0)\n",
    "    dW1_2 = dW1_1 * dtanh\n",
    "\n",
    "    dv = np.dot(dJdf, a_2) + rho * v\n",
    "    db = np.sum(dW1_2, axis=0) + rho * b\n",
    "    dW = np.tensordot(np.transpose(X), dW1_2, axes=1) + rho * W\n",
    "\n",
    "    return np.concatenate((dW, db, dv), axis=None)\n",
    "\n",
    "def loss(x0, funcArgs, test=False):\n",
    "    X = funcArgs[0]\n",
    "    y = funcArgs[1]\n",
    "    sigma = funcArgs[2]\n",
    "    N = funcArgs[3]\n",
    "    rho = funcArgs[4]\n",
    "\n",
    "    W = x0[:int(X.shape[1] * N)].reshape((X.shape[1], N))\n",
    "    b = x0[int(X.shape[1] * N):int(X.shape[1] * N + N)]\n",
    "    v = x0[int(X.shape[1] * N + N):]\n",
    "\n",
    "    P = len(y)\n",
    "    norm = np.linalg.norm(x0)\n",
    "    pred = feedforward(X, W, b, v, sigma)\n",
    "    if test:\n",
    "        res = ((np.sum((pred - y) ** 2)) * P ** (-1)) * 0.5\n",
    "    else:\n",
    "        res = ((np.sum((pred - y) ** 2)) * P ** (-1) + rho * norm**2) * 0.5\n",
    "\n",
    "    return res\n",
    "\n",
    "\n",
    "def loss_test(X, y, sigma, W, b, v):\n",
    "    P = len(y)\n",
    "    pred = feedforward(X, W, b, v, sigma)\n",
    "    res = ((np.sum((pred - y) ** 2)) * P ** (-1)) * 0.5\n",
    "\n",
    "    return res\n",
    "\n",
    "\n",
    "def feedforwardplot(x1, x2, W, b, v, sigma):\n",
    "    X = np.array([x1, x2])\n",
    "    linear_layer = (np.dot(X, W) + b)\n",
    "    activation = tanh(linear_layer, sigma)\n",
    "    pred = np.dot(activation, v)\n",
    "\n",
    "    return pred\n",
    "\n",
    "\n",
    "def train(X, y, sigma, N, rho, W, b, v, max_iter=1000,\n",
    "          tol=1e-5, method='CG', func=loss, disp=False):\n",
    "          \n",
    "    x0 = np.concatenate((W, b, v), axis=None)\n",
    "    funcArgs = [X, y, sigma, N, rho]\n",
    "    \n",
    "    res = minimize(func,\n",
    "                   x0,\n",
    "                   args=funcArgs, \n",
    "                   method=method, \n",
    "                   tol=tol,\n",
    "                   jac=backpropagation,\n",
    "                   options={'maxiter':max_iter, \n",
    "                            'disp': disp})  \n",
    "    \n",
    "    return res\n",
    "    \n",
    "\n",
    "def plotting(W, b, v, sigma):\n",
    "    fig = plt.figure(figsize=(12, 8))\n",
    "    ax = plt.axes(projection='3d')\n",
    "    # create the grid\n",
    "    x = np.linspace(-3, 3, 50)\n",
    "    y = np.linspace(-2, 2, 50)\n",
    "    X_plot, Y_plot = np.meshgrid(x, y)\n",
    "\n",
    "    Z = []\n",
    "    for x1 in x:\n",
    "        z = []\n",
    "        for x2 in y:\n",
    "            z.append(feedforwardplot(x1, x2, W, b, v, sigma))\n",
    "        Z.append(z)\n",
    "    Z = np.array(Z)\n",
    "\n",
    "    ax.plot_surface(X_plot, Y_plot, Z, rstride=1, cstride=1, cmap='viridis', edgecolor='none')\n",
    "\n",
    "    ax.set_xlabel('x')\n",
    "    ax.set_ylabel('y')\n",
    "    ax.set_zlabel('z')\n",
    "    ax.set_title('F(x) learnt from MLP BackPropagation')\n",
    "    plt.show()\n",
    "\n",
    "sigma = 1\n",
    "N = 70\n",
    "rho = 1e-5\n",
    "method = 'CG'\n",
    "\n",
    "W = np.random.normal(size=(X.shape[1], N))\n",
    "b = np.random.normal(size=N)\n",
    "v = np.random.normal(size=N)\n",
    "\n",
    "x0 = np.concatenate((W, b, v), axis=None)\n",
    "funcArgs = [X, y, sigma, N, rho]\n",
    "\n",
    "print('===================')\n",
    "print('Sigma:', sigma)\n",
    "print('N:', N)\n",
    "print('Rho:', rho)\n",
    "\n",
    "loss_start = loss(x0, funcArgs)\n",
    "grad_norm_start = np.linalg.norm(backpropagation(x0, \n",
    "                            funcArgs=funcArgs))\n",
    "\n",
    "start = time.time()\n",
    "res = train(X, y, sigma=sigma, \n",
    "            N=N, rho=rho, \n",
    "            W=W, b=b, v=v,\n",
    "            max_iter=5000, tol=1e-6, \n",
    "            method=method, func=loss,\n",
    "            disp=True)\n",
    "stop = time.time()\n",
    "\n",
    "loss_end = res.fun\n",
    "\n",
    "W=res.x[:int(X.shape[1] * N)].reshape((X.shape[1], N)),\n",
    "b=res.x[int(X.shape[1] * N):int(X.shape[1] * N + N)],\n",
    "v=res.x[int(X.shape[1] * N + N):]\n",
    "\n",
    "funcArgs_test = [X_test, y_test, sigma, N, rho]\n",
    "\n",
    "loss(x0, funcArgs)\n",
    "\n",
    "val_loss = loss(res.x, funcArgs_test, test=True)\n",
    "\n",
    "grad_norm_end = np.linalg.norm(backpropagation(np.concatenate((W, b, v), axis=None), \n",
    "                            funcArgs=funcArgs))\n",
    "\n",
    "train_loss = loss(res.x, funcArgs, test=True)\n",
    "\n",
    "best_loss_start = loss_start\n",
    "best_loss_end = loss_end\n",
    "N_best = N\n",
    "sigma_best = sigma\n",
    "rho_best = rho\n",
    "convergence = res.success\n",
    "method_best = method\n",
    "\n",
    "print('N')\n",
    "print(N_best)\n",
    "print('')\n",
    "print('sigma')\n",
    "print(sigma_best)\n",
    "print('')\n",
    "print('rho')\n",
    "print(rho_best)\n",
    "print('')\n",
    "print('W')\n",
    "print(W)\n",
    "print('')\n",
    "print('b')\n",
    "print(b)\n",
    "print('')\n",
    "print('v')\n",
    "print(v)\n",
    "print('')\n",
    "print('Convergence?')\n",
    "print(convergence)\n",
    "print('')\n",
    "print('Best Method?')\n",
    "print(method_best)\n",
    "print('')\n",
    "print('Objective Function Start')\n",
    "print(best_loss_start)\n",
    "print('')\n",
    "print('Objective Function End')\n",
    "print(best_loss_end)\n",
    "print('')\n",
    "print('Gradient Norm Start')\n",
    "print(grad_norm_start)\n",
    "print('')\n",
    "print('Gradient Norm End')\n",
    "print(grad_norm_end)\n",
    "print('')\n",
    "print('Computation time')\n",
    "print(round(stop-start, 2))\n",
    "print('')\n",
    "print('Final Train Error')\n",
    "print(train_loss)\n",
    "print('')\n",
    "print('Final Test Error')\n",
    "print(val_loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================\n",
      "Sigma: 1\n",
      "N: 70\n",
      "Rho: 1e-05\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.001372\n",
      "         Iterations: 5000\n",
      "         Function evaluations: 8286\n",
      "         Gradient evaluations: 8286\n",
      "N\n",
      "70\n",
      "\n",
      "sigma\n",
      "1\n",
      "\n",
      "rho\n",
      "1e-05\n",
      "\n",
      "Convergence?\n",
      "False\n",
      "\n",
      "Best Method?\n",
      "CG\n",
      "\n",
      "Objective Function Start\n",
      "3.9203789914884295\n",
      "\n",
      "Objective Function End\n",
      "0.0013722166771667965\n",
      "\n",
      "Gradient Norm Start\n",
      "3.65625481076387\n",
      "\n",
      "Gradient Norm End\n",
      "2.123438703032642e-05\n",
      "\n",
      "Computation time\n",
      "36.22\n",
      "\n",
      "Final Train Error\n",
      "0.00035705536688818216\n",
      "\n",
      "Final Test Error\n",
      "0.0020141031986431537\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.model_selection import train_test_split\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "\n",
    "df = pd.read_csv('DATA.csv')\n",
    "\n",
    "train, test = train_test_split(df, test_size=0.255, random_state=1939671)\n",
    "\n",
    "X = np.array(train[['x1', 'x2']])\n",
    "y = np.array(train['y'])\n",
    "\n",
    "X_test = np.array(test[['x1', 'x2']])\n",
    "y_test = np.array(test['y'])\n",
    "\n",
    "\n",
    "def rbf(X, c, sigma):\n",
    "    \"\"\"\n",
    "    This function is only applied for a single observation\n",
    "    x belongs to R^2\n",
    "    c belongs to R^{2, 10}\n",
    "    return R^10, 186\n",
    "    \"\"\"\n",
    "    minus_matrix = []\n",
    "    for i in range(len(c.T)):\n",
    "        minus_matrix.append(X - c.T[i])\n",
    "    minus_matrix = np.array(minus_matrix)\n",
    "\n",
    "    return np.exp(-(np.linalg.norm(minus_matrix, axis=2)/sigma)**2)\n",
    "\n",
    "\n",
    "def feedforward(X, c, v, sigma):\n",
    "    \"\"\"\n",
    "    This function is only applied for a single observation\n",
    "    x belongs to R^2\n",
    "    c belongs to R^{2, 10}\n",
    "    v belongs to R^N\n",
    "    return float\n",
    "    \"\"\"\n",
    "    \n",
    "    pred = np.dot(rbf(X, c, sigma).T, v)\n",
    "    return pred\n",
    "\n",
    "\n",
    "def backpropagation(x0, funcArgs):\n",
    "\n",
    "    X = funcArgs[0]\n",
    "    y = funcArgs[1]\n",
    "    sigma = funcArgs[2]\n",
    "    N = funcArgs[3]\n",
    "    rho = funcArgs[4]\n",
    "    P = len(y)\n",
    "    \n",
    "    c = x0[:int(X.shape[1]*N)].reshape((X.shape[1],N))\n",
    "    v = x0[int(X.shape[1]*N):]\n",
    "    \n",
    "    z_1 = rbf(X, c, sigma).T\n",
    "    dJdf = (1/P)*(np.dot(z_1, v) - y)\n",
    "\n",
    "    minus_matrix = []\n",
    "    for i in range(len(c.T)):\n",
    "        minus_matrix.append(X - c.T[i])\n",
    "    minus_matrix = np.array(minus_matrix)\n",
    "\n",
    "    dW1_1 = np.dot(dJdf.reshape((P, 1)), v.reshape((1,N)))\n",
    "    dzdc = ((2*z_1)/(sigma**2))*minus_matrix.T\n",
    "\n",
    "    dv = np.dot(dJdf, z_1) + rho*v\n",
    "    dc = np.sum(dzdc*dW1_1, axis=1) + rho*c\n",
    "\n",
    "    return np.concatenate((dc, dv), axis=None)\n",
    "\n",
    "\n",
    "def loss(x0, funcArgs, test=False):\n",
    "    \n",
    "    X = funcArgs[0]\n",
    "    y = funcArgs[1]\n",
    "    sigma = funcArgs[2]\n",
    "    N = funcArgs[3]\n",
    "    rho = funcArgs[4]\n",
    "    \n",
    "    c = x0[:int(X.shape[1]*N)].reshape((X.shape[1],N))\n",
    "    v = x0[int(X.shape[1]*N):]\n",
    "\n",
    "    P = len(y)\n",
    "    pred = feedforward(X, c, v, sigma)\n",
    "    norm = np.linalg.norm(x0)\n",
    "    if test:\n",
    "        res = ((np.sum((pred - y) ** 2)) * P ** (-1)) * 0.5\n",
    "    else:\n",
    "        res = ((np.sum((pred - y) ** 2)) * P ** (-1) + rho * norm ** 2) * 0.5\n",
    "    \n",
    "    return res\n",
    "\n",
    "\n",
    "def feedforwardplot(x_i_1, x_i_2, c, v, sigma):\n",
    "    x_i = np.array([x_i_1, x_i_2])\n",
    "    pred = np.dot(np.exp(-(np.linalg.norm((x_i - c.T), axis=1)/sigma)**2), v)\n",
    "    return pred\n",
    "\n",
    "\n",
    "def train(X, y, sigma, N, rho, c_init, \n",
    "          v_init, max_iter=1000, tol=1e-5, method='CG', func=loss, disp=False):\n",
    "    \n",
    "    x0 = np.concatenate((c_init, v_init), axis=None)\n",
    "    funcArgs = [X, y, sigma, N, rho]\n",
    "\n",
    "    res = minimize(func,\n",
    "                   x0,\n",
    "                   args=funcArgs, \n",
    "                   method=method, \n",
    "                   tol=tol,\n",
    "                   jac=backpropagation,\n",
    "                   options={'maxiter':max_iter, \n",
    "                            'disp': disp})    \n",
    "    \n",
    "    return res\n",
    "\n",
    "sigma = 1\n",
    "N = 70\n",
    "rho = 1e-5\n",
    "method = 'CG'\n",
    "\n",
    "c = np.random.normal(size=(X.shape[1], N))\n",
    "v = np.random.normal(size=N)\n",
    "\n",
    "x0 = np.concatenate((c, v), axis=None)\n",
    "funcArgs = [X, y, sigma, N, rho]\n",
    "\n",
    "print('===================')\n",
    "print('Sigma:', sigma)\n",
    "print('N:', N)\n",
    "print('Rho:', rho)\n",
    "\n",
    "loss_start = loss(x0, funcArgs)\n",
    "grad_norm_start = np.linalg.norm(backpropagation(x0, \n",
    "                                 funcArgs=funcArgs))\n",
    "\n",
    "start = time.time()\n",
    "res = train(X, y, sigma=sigma, \n",
    "            N=N, rho=rho, \n",
    "            c_init=c, v_init=v,\n",
    "            max_iter=5000, tol=1e-6, \n",
    "            method=method, func=loss,\n",
    "            disp=True)\n",
    "stop = time.time()\n",
    "\n",
    "loss_end = res.fun\n",
    "\n",
    "funcArgs_test = [X_test, y_test, sigma, N, rho]\n",
    "\n",
    "loss(x0, funcArgs)\n",
    "\n",
    "val_loss = loss(res.x, funcArgs_test, test=True)\n",
    "\n",
    "grad_norm_end = np.linalg.norm(backpropagation(res.x, \n",
    "                               funcArgs=funcArgs))\n",
    "\n",
    "train_loss = loss(res.x, funcArgs, test=True)\n",
    "\n",
    "best_loss_start = loss_start\n",
    "best_loss_end = loss_end\n",
    "N_best = N\n",
    "sigma_best = sigma\n",
    "rho_best = rho\n",
    "convergence = res.success\n",
    "method_best = method\n",
    "\n",
    "print('N')\n",
    "print(N_best)\n",
    "print('')\n",
    "print('sigma')\n",
    "print(sigma_best)\n",
    "print('')\n",
    "print('rho')\n",
    "print(rho_best)\n",
    "print('')\n",
    "print('Convergence?')\n",
    "print(convergence)\n",
    "print('')\n",
    "print('Best Method?')\n",
    "print(method_best)\n",
    "print('')\n",
    "print('Objective Function Start')\n",
    "print(best_loss_start)\n",
    "print('')\n",
    "print('Objective Function End')\n",
    "print(best_loss_end)\n",
    "print('')\n",
    "print('Gradient Norm Start')\n",
    "print(grad_norm_start)\n",
    "print('')\n",
    "print('Gradient Norm End')\n",
    "print(grad_norm_end)\n",
    "print('')\n",
    "print('Computation time')\n",
    "print(round(stop-start, 2))\n",
    "print('')\n",
    "print('Final Train Error')\n",
    "print(train_loss)\n",
    "print('')\n",
    "print('Final Test Error')\n",
    "print(val_loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}