widrow_hoff.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "characteristic-camera",
   "metadata": {},
   "source": [
    "# Widrow-Off Discrimination Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cheap-proxy",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd4240df",
   "metadata": {},
   "source": [
    "## Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "grand-jason",
   "metadata": {},
   "outputs": [],
   "source": [
    "class sequential_widrow_hoff(object):\n",
    "    \"\"\" Widrow Hoff learning algorithm (sequential)\n",
    "        Parameters\n",
    "        ----------\n",
    "        X: np.array\n",
    "            Feature vectors\n",
    "        Y: np.array\n",
    "            Class vector\n",
    "        a: np.array\n",
    "            Initialised weights\n",
    "        b: np.array\n",
    "            Margin vector\n",
    "        epochs: int, optional\n",
    "            Number of training epochs. -1 is until convergence (default is -1)\n",
    "        max_epochs: int, optional\n",
    "            Maximum number of epoches\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, X, Y, a, b, alpha=0.1, epochs=-1, max_epochs=500):\n",
    "        self.X = X\n",
    "        self.Y = Y\n",
    "        self.a = a\n",
    "        self.b = b\n",
    "        self.alpha = alpha\n",
    "        self.epochs = epochs\n",
    "        self.max_epochs = max_epochs\n",
    "    \n",
    "    def train(self):\n",
    "        \"\"\"\n",
    "        Performs the training\n",
    "        \"\"\"\n",
    "        if self.epochs == -1:\n",
    "            self.train_until_convergence()\n",
    "        else: \n",
    "            self.train_until_epoch()\n",
    "            \n",
    "\n",
    "    def classify(self, x, a):\n",
    "        g_x = np.dot(a, x)\n",
    "        if g_x > 0:\n",
    "            return 1\n",
    "        else: \n",
    "            return -1\n",
    "        \n",
    "    def train_until_convergence(self):\n",
    "        e = 0\n",
    "        i = 0\n",
    "        while i < self.max_epochs:\n",
    "            print('==========================================')\n",
    "            print('EPOCH ', i+1)\n",
    "            e = 0\n",
    "            #iteration\n",
    "            for j, datum in enumerate(self.X):\n",
    "                print('iteration ', i+1, '.', j+1)\n",
    "                print('------------------')\n",
    "                datum = np.insert(datum, 0, 1)\n",
    "                temp = datum\n",
    "                #sample normalisation\n",
    "                if self.Y[j] == -1:\n",
    "                    datum = datum * -1\n",
    "                b = self.b\n",
    "                if len(self.b) > 1:\n",
    "                    b = self.b[j]\n",
    "                g_x = np.dot(self.a, datum)\n",
    "                \n",
    "                y_hat = self.classify(temp, self.a)\n",
    "                if self.Y[j] != y_hat:\n",
    "                    e +=1\n",
    "                self.a = self.a + self.alpha * (b - g_x) * datum\n",
    "            \n",
    "                \n",
    "                print('g(x): ', g_x)\n",
    "                print('Y: ', self.Y[j])\n",
    "                print('y_hat: ', y_hat)\n",
    "                print('boundary: ', self.a)\n",
    "                \n",
    "                i += 1 \n",
    "                \n",
    "            if e == 0:\n",
    "                return\n",
    "                \n",
    "    def train_until_epoch(self):\n",
    "        for i in range(self.epochs):\n",
    "            print('==========================================')\n",
    "            print('EPOCH ', i+1)\n",
    "            #iteration\n",
    "            for j, datum in enumerate(self.X):\n",
    "                print('iteration ', i+1, '.', j+1)\n",
    "                print('------------------')\n",
    "                datum = np.insert(datum, 0, 1)\n",
    "                temp = datum\n",
    "                #sample normalisation\n",
    "                if self.Y[j] == -1:\n",
    "                    datum = datum * -1\n",
    "                b = self.b\n",
    "                if len(self.b) > 1:\n",
    "                    b = self.b[j]\n",
    "                g_x = np.dot(self.a, datum)\n",
    "                self.a = self.a + self.alpha * (b - g_x) * datum\n",
    "                print('g(x): ', g_x)\n",
    "                print('Y: ', self.Y[j])\n",
    "                print('y_hat: ', self.classify(temp, self.a))\n",
    "                print('boundary: ', self.a)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "incident-tobacco",
   "metadata": {},
   "source": [
    "## Batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "hundred-criterion",
   "metadata": {},
   "outputs": [],
   "source": [
    "class batch_widrow_hoff(object):\n",
    "    \"\"\" Widrow Hoff learning algorithm (batch)\n",
    "        Parameters\n",
    "        ----------\n",
    "        X: np.array\n",
    "            Feature vectors\n",
    "        Y: np.array\n",
    "            Class vector\n",
    "        a: np.array\n",
    "            Initialised weights\n",
    "        b: np.array\n",
    "            Margin vector\n",
    "        epochs: int, optional\n",
    "            Number of training epochs. -1 is until convergence (default is -1)\n",
    "        max_epochs: int, optional\n",
    "            Maximum number of epoches\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, X, Y, a, b, alpha=0.1, epochs=-1, max_epochs=500):\n",
    "        self.X = X\n",
    "        self.Y = Y\n",
    "        self.a = a\n",
    "        self.b = b\n",
    "        self.alpha = alpha\n",
    "        self.epochs = epochs\n",
    "        self.max_epochs = max_epochs\n",
    "    \n",
    "    def train(self):\n",
    "        \"\"\"\n",
    "        Performs the training\n",
    "        \"\"\"\n",
    "        if self.epochs == -1:\n",
    "            self.train_until_convergence()\n",
    "        else: \n",
    "            self.train_until_epoch()\n",
    "        return\n",
    "            \n",
    "\n",
    "    def classify(self, x, a):\n",
    "        g_x = np.dot(a, x)\n",
    "        if g_x > 0:\n",
    "            return 1\n",
    "        else: \n",
    "            return -1\n",
    "        \n",
    "    def train_until_convergence(self):\n",
    "        e = 0\n",
    "        i = 0\n",
    "        while i < self.max_epochs:\n",
    "            print('==========================================')\n",
    "            print('EPOCH ', i+1)\n",
    "            delta = 0\n",
    "            e = 0\n",
    "            #iteration\n",
    "            for j, datum in enumerate(self.X):\n",
    "                print('iteration ', i+1, '.', j+1)\n",
    "                print('------------------')\n",
    "                datum = np.insert(datum, 0, 1)\n",
    "                temp = datum\n",
    "                #sample normalisation\n",
    "                if self.Y[j] == -1:\n",
    "                    datum = datum * -1\n",
    "                b = self.b\n",
    "                if len(self.b) > 1:\n",
    "                    b = self.b[j]\n",
    "                g_x = np.dot(self.a, datum)\n",
    "                \n",
    "                y_hat = self.classify(temp, self.a)\n",
    "                if self.Y[j] != y_hat:\n",
    "                    e +=1\n",
    "                delta += (b - g_x) * datum\n",
    "                \n",
    "                print('g(x): ', g_x)\n",
    "                print('Y: ', self.Y[j])\n",
    "                print('y_hat: ', y_hat)\n",
    "                \n",
    "                i += 1 \n",
    "                \n",
    "            self.a = self.a + self.alpha * delta\n",
    "            print('boundary: ', self.a)   \n",
    "            if e == 0:\n",
    "                break\n",
    "                \n",
    "    def train_until_epoch(self):\n",
    "        for i in range(self.epochs):\n",
    "            print('==========================================')\n",
    "            print('EPOCH ', i+1)\n",
    "            \n",
    "            delta = 0\n",
    "\n",
    "            #iteration\n",
    "            for j, datum in enumerate(self.X):\n",
    "                print('iteration ', i+1, '.', j+1)\n",
    "                print('------------------')\n",
    "                datum = np.insert(datum, 0, 1)\n",
    "                temp = datum\n",
    "                #sample normalisation\n",
    "                if self.Y[j] == -1:\n",
    "                    datum = datum * -1\n",
    "                b = self.b\n",
    "                if len(self.b) > 1:\n",
    "                    b = self.b[j]\n",
    "                g_x = np.dot(self.a, datum)\n",
    "                \n",
    "                delta += (b - g_x) * datum\n",
    "                \n",
    "                print('g(x): ', g_x)\n",
    "                print('Y: ', self.Y[j])\n",
    "                print('y_hat: ', self.classify(temp, self.a))\n",
    "                \n",
    "            self.a = self.a + self.alpha * delta\n",
    "            print('boundary: ', self.a)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d188772",
   "metadata": {},
   "source": [
    "## Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "imperial-saskatchewan",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================================\n",
      "EPOCH  1\n",
      "iteration  1 . 1\n",
      "------------------\n",
      "g(x):  1.0\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "boundary:  [1. 0. 0.]\n",
      "iteration  2 . 2\n",
      "------------------\n",
      "g(x):  1.0\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "boundary:  [1.15 0.15 0.3 ]\n",
      "iteration  3 . 3\n",
      "------------------\n",
      "g(x):  1.75\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "boundary:  [1.125 0.1   0.275]\n",
      "iteration  4 . 4\n",
      "------------------\n",
      "g(x):  -1.1\n",
      "Y:  -1\n",
      "y_hat:  1\n",
      "boundary:  [0.965 0.58  0.115]\n",
      "iteration  5 . 5\n",
      "------------------\n",
      "g(x):  0.31000000000000016\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "boundary:  [0.946 0.618 0.134]\n",
      "iteration  6 . 6\n",
      "------------------\n",
      "g(x):  1.1760000000000002\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "boundary:  [0.9636 0.5652 0.0988]\n",
      "==========================================\n",
      "EPOCH  7\n",
      "iteration  7 . 1\n",
      "------------------\n",
      "g(x):  1.1612\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "boundary:  [0.94748 0.5652  0.06656]\n",
      "iteration  8 . 2\n",
      "------------------\n",
      "g(x):  1.6458\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "boundary:  [1.0329  0.65062 0.2374 ]\n",
      "iteration  9 . 3\n",
      "------------------\n",
      "g(x):  2.5715399999999997\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "boundary:  [0.925746 0.436312 0.130246]\n",
      "iteration  10 . 4\n",
      "------------------\n",
      "g(x):  0.25294400000000017\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "boundary:  [0.9010404 0.5104288 0.1055404]\n",
      "iteration  11 . 5\n",
      "------------------\n",
      "g(x):  0.22535760000000005\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "boundary:  [0.87357616 0.56535728 0.13300464]\n",
      "iteration  12 . 6\n",
      "------------------\n",
      "g(x):  1.0885049599999999\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "boundary:  [0.88242666 0.53880579 0.11530365]\n"
     ]
    }
   ],
   "source": [
    "x = np.array([[0.0, 2.0],\n",
    "               [1.0, 2.0],\n",
    "               [2.0, 1.0],\n",
    "               [-3.0, 1.0],\n",
    "               [-2.0, -1.0], \n",
    "               [-3.0, -2.0]\n",
    "              ])\n",
    "y = np.array([1, 1, 1, -1, -1, -1])\n",
    "a = np.array([1.0, 0.0, 0.0])\n",
    "b = [1.0, 2.5, 1.5, 0.5, 0.5, 1.0]\n",
    "alpha = 0.1\n",
    "\n",
    "trainer = sequential_widrow_hoff(x, y, a, b, alpha=0.1)\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "grand-textbook",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================================\n",
      "EPOCH  1\n",
      "iteration  1 . 1\n",
      "------------------\n",
      "g(x):  1.0\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "iteration  2 . 2\n",
      "------------------\n",
      "g(x):  1.0\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "iteration  3 . 3\n",
      "------------------\n",
      "g(x):  1.0\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "iteration  4 . 4\n",
      "------------------\n",
      "g(x):  -1.0\n",
      "Y:  -1\n",
      "y_hat:  1\n",
      "iteration  5 . 5\n",
      "------------------\n",
      "g(x):  -1.0\n",
      "Y:  -1\n",
      "y_hat:  1\n",
      "iteration  6 . 6\n",
      "------------------\n",
      "g(x):  -1.0\n",
      "Y:  -1\n",
      "y_hat:  1\n",
      "boundary:  [0.4 1.6 0.4]\n",
      "==========================================\n",
      "EPOCH  7\n",
      "iteration  7 . 1\n",
      "------------------\n",
      "g(x):  1.2\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "iteration  8 . 2\n",
      "------------------\n",
      "g(x):  2.8\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "iteration  9 . 3\n",
      "------------------\n",
      "g(x):  4.0\n",
      "Y:  1\n",
      "y_hat:  1\n",
      "iteration  10 . 4\n",
      "------------------\n",
      "g(x):  4.0\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "iteration  11 . 5\n",
      "------------------\n",
      "g(x):  3.2\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "iteration  12 . 6\n",
      "------------------\n",
      "g(x):  5.2\n",
      "Y:  -1\n",
      "y_hat:  -1\n",
      "boundary:  [ 0.84 -1.78 -1.06]\n"
     ]
    }
   ],
   "source": [
    "x = np.array([[0.0, 2.0],\n",
    "               [1.0, 2.0],\n",
    "               [2.0, 1.0],\n",
    "               [-3.0, 1.0],\n",
    "               [-2.0, -1.0],\n",
    "               [-3.0, -2.0],\n",
    "              ])\n",
    "y = np.array([1, 1, 1, -1, -1, -1])\n",
    "a = np.array([1, 0, 0])\n",
    "b = np.array([1])\n",
    "alpha = 0.1\n",
    "\n",
    "trainer = batch_widrow_hoff(x, y, a, b, alpha=alpha)\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "gothic-correlation",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}