SVM-one_against_one.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aggregate-lightning",
   "metadata": {},
   "source": [
    "#  One against one SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "future-constitution",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bottom-astronomy",
   "metadata": {},
   "outputs": [],
   "source": [
    "def SVM(SV, y):\n",
    "    matrix = []\n",
    "    for i, sv1 in enumerate(SV):\n",
    "        temp = []\n",
    "        for j, sv2 in enumerate(SV):\n",
    "            temp.append(y[j] * np.dot(sv2.T,  sv1))\n",
    "        #temp.append(y[i])\n",
    "        temp.append(1)\n",
    "        matrix.append(np.array(temp))\n",
    "    \n",
    "    y_new = list(y)\n",
    "    y_new.append(0)\n",
    "    y_new = np.array(y_new)\n",
    "    matrix.append(y_new)  \n",
    "    matrix = np.matrix(np.array(matrix))\n",
    "    \n",
    "    matrix_inv = np.linalg.inv(matrix)\n",
    "    \n",
    "    print(matrix)\n",
    "    print(matrix_inv)\n",
    "    print(y_new)\n",
    "    \n",
    "    lambdas = np.dot(np.array(matrix_inv), y_new)\n",
    "    lambdas = np.array(lambdas)\n",
    "    \n",
    "    print(lambdas)\n",
    "    w = []\n",
    "    for i in range(len(SV)):\n",
    "        w.append(y[i] * np.dot(SV[i], lambdas[i]))\n",
    "    w = np.array(w)\n",
    "    W = w.sum(axis=0)\n",
    "    print('w =', W, 'w0 =', lambdas[-1])\n",
    "    return (W, lambdas[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "promotional-berkeley",
   "metadata": {},
   "source": [
    "- \\# of classifiers = (R (R - 1)/2); R = # of classes\n",
    "- a binary classifier is trained to discriminate each pair of classes\n",
    "- decision through majority voting"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe52337e",
   "metadata": {},
   "source": [
    "## Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "understanding-waterproof",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([\n",
    "    # class_1\n",
    "    [0, 10],\n",
    "    [1, 8],\n",
    "    [2, 9],\n",
    "    [2, 8],\n",
    "    [3, 10],\n",
    "    [3, 9],\n",
    "    [4, 10],\n",
    "    [4, 9],\n",
    "    [4, 8],\n",
    "    [5, 9],\n",
    "    # class_2\n",
    "    [2, 6],\n",
    "    [0, 5],\n",
    "    [4, 5],\n",
    "    [2, 4],\n",
    "    [5, 3],\n",
    "    [0, 2],\n",
    "    [2, 2],\n",
    "    [5, 2],\n",
    "    [2, 1],\n",
    "    [3, 0],\n",
    "    # class_3\n",
    "    [9, 6],\n",
    "    [8, 5],\n",
    "    [7, 3],\n",
    "    [8, 3],\n",
    "    [9, 3],\n",
    "    [10, 3],\n",
    "    [7, 1],\n",
    "    [8, 1],\n",
    "    [8, 0],\n",
    "    [9, 0]  \n",
    "])\n",
    "y = np.array([\n",
    "    1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
    "    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, \n",
    "    3, 3, 3, 3, 3, 3, 3, 3, 3, 3\n",
    "])\n",
    "# SVs\n",
    "SV12_X = np.array([\n",
    "    [1, 8], [2, 8], [4, 8], [2, 6], [9, 6]\n",
    "])\n",
    "SV12_y = np.array([1, 1, 1, -1, -1])\n",
    "\n",
    "SV23_X = np.array([\n",
    "    [7, 3], [7, 1], [5, 9], [5, 3], [5, 2]\n",
    "])\n",
    "SV23_y = np.array([1, 1, -1, -1, -1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bright-wilderness",
   "metadata": {},
   "source": [
    "We classify a point and based on the output of all the classifiers we determine the class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "complicated-german",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  65   66   68  -50  -57    1]\n",
      " [  66   68   72  -52  -66    1]\n",
      " [  68   72   80  -56  -84    1]\n",
      " [  50   52   56  -40  -54    1]\n",
      " [  57   66   84  -54 -117    1]\n",
      " [   1    1    1   -1   -1    0]]\n",
      "[[ 2.49116863e+13 -1.87108426e+14  1.62196740e+14  4.27831133e+13\n",
      "  -4.27831133e+13 -3.81625782e+00]\n",
      " [ 7.06733708e+13  6.88456112e+13 -1.39518982e+14 -4.99587621e+13\n",
      "   4.99587621e+13  1.08225108e-01]\n",
      " [-9.55850570e+13  1.18262815e+14 -2.26777579e+13  7.17564876e+12\n",
      "  -7.17564876e+12  7.08032708e-01]\n",
      " [ 3.08688286e+13 -6.05191508e+13  2.96503222e+13  4.06168798e+12\n",
      "  -4.06168798e+12 -5.03318903e+00]\n",
      " [-3.08688286e+13  6.05191508e+13 -2.96503222e+13 -4.06168798e+12\n",
      "   4.06168798e+12  1.03318903e+00]\n",
      " [-4.38672439e-01 -1.13997114e+00 -1.42135642e+00  3.94227994e+00\n",
      "   5.77200577e-02 -4.21609196e-14]]\n",
      "[ 1  1  1 -1 -1  0]\n",
      "[ 0.6953125  -0.0390625  -0.13769531  0.63134766 -0.12744141 -7.        ]\n",
      "w = [-0.04931641  1.125     ] w0 = -6.999999999999993\n",
      "[[  58   52  -62  -44  -41    1]\n",
      " [  52   50  -44  -38  -37    1]\n",
      " [  62   44 -106  -52  -43    1]\n",
      " [  44   38  -52  -34  -31    1]\n",
      " [  41   37  -43  -31  -29    1]\n",
      " [   1    1   -1   -1   -1    0]]\n",
      "[[ 2.52431753e+13 -2.52431753e+13 -4.37005509e+13  2.55417505e+14\n",
      "  -2.11716955e+14  1.92333655e+00]\n",
      " [-2.52431753e+13  2.52431753e+13  4.37005509e+13 -2.55417505e+14\n",
      "   2.11716955e+14 -4.42333655e+00]\n",
      " [ 4.09862309e+13 -4.09862309e+13 -2.25288554e+13  7.57295260e+13\n",
      "  -5.32006706e+13  1.48842816e+00]\n",
      " [-2.36417266e+14  2.36417266e+14  7.03008862e+13 -1.92716715e+13\n",
      "  -5.10292147e+13 -2.07232401e+00]\n",
      " [ 1.95431035e+14 -1.95431035e+14 -4.77720308e+13 -5.64578545e+13\n",
      "   1.04229885e+14 -2.91610415e+00]\n",
      " [-1.80376085e+00 -6.96239151e-01  7.42526519e-02 -1.41224687e+00\n",
      "   4.83799421e+00 -4.52054552e-14]]\n",
      "[ 1  1 -1 -1 -1  0]\n",
      "[-0.09375    0.625     -0.15625    0.3203125  0.359375  -6.       ]\n",
      "w = [1.1015625 0.0703125] w0 = -5.999999999999993\n"
     ]
    }
   ],
   "source": [
    "W_12, W0_12 = SVM(SV12_X, SV12_y)\n",
    "W_23, W0_23 = SVM(SV23_X, SV23_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "revised-phoenix",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = np.array([\n",
    "    [3.7, 7.3],\n",
    "    [3. , 6.],\n",
    "    [4.3, 5.7]\n",
    "])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "substantial-nigeria",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------\n",
      "Element 0 : [3.7 7.3]\n",
      "C_1: 1.030029296875007\n",
      "C_2: -1.4109374999999922\n",
      "--------------------------\n",
      "Element 1 : [3. 6.]\n",
      "C_1: -0.3979492187499929\n",
      "C_2: -2.273437499999993\n",
      "--------------------------\n",
      "Element 2 : [4.3 5.7]\n",
      "C_1: -0.799560546874992\n",
      "C_2: -0.8624999999999936\n"
     ]
    }
   ],
   "source": [
    "for i, el in enumerate(X_test):\n",
    "    print('--------------------------')\n",
    "    print('Element', i, ':', el)\n",
    "    print('C_1:', np.dot(el, W_12) + W0_12)\n",
    "    print('C_2:', np.dot(el, W_23) + W0_23)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "supported-buffalo",
   "metadata": {},
   "source": [
    "According to the classifiers' outputs, we determine the class. \n",
    "\n",
    "Note: this method leaves blank non-categorisabel spaces"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}