HW3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feedforward ANN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1/(1+math.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate, momentum = 0.3, 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_ca, w_cb, w_c0, w_dc, w_d0 = 0.1, 0.1, 0.1, 0.1, 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input Layer\n",
    "a, b = 1, 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2 0.549833997312478\n"
     ]
    }
   ],
   "source": [
    "# Forward Pass\n",
    "net_c = w_ca*a + w_cb*b + w_c0\n",
    "c = sigmoid(net_c)\n",
    "print(net_c, c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1549833997312478 0.5386684799635422\n"
     ]
    }
   ],
   "source": [
    "net_d = w_dc*c + w_d0\n",
    "d = sigmoid(net_d)\n",
    "print(net_d, d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.10641338568957434\n"
     ]
    }
   ],
   "source": [
    "# Error\n",
    "t = 1\n",
    "E = 0.5*math.pow(t-d,2)\n",
    "print(E)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.11464307343435433 0.002837606062162546\n"
     ]
    }
   ],
   "source": [
    "# Backpropagation\n",
    "delta_d = (t-d)*d*(1-d)\n",
    "delta_c = (delta_d*w_dc)*c*(1-c)\n",
    "print(delta_d, delta_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.018910397799179697 0.1189103977991797\n"
     ]
    }
   ],
   "source": [
    "# Update Weights\n",
    "delta_w_dc = learning_rate*delta_d*c\n",
    "w_dc += delta_w_dc\n",
    "print(delta_w_dc, w_dc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0343929220303063 0.1343929220303063\n"
     ]
    }
   ],
   "source": [
    "delta_w_d0 = learning_rate*delta_d\n",
    "w_d0 += delta_w_d0\n",
    "print(delta_w_d0, w_d0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0008512818186487638 0.10085128181864877\n"
     ]
    }
   ],
   "source": [
    "delta_w_ca = learning_rate*delta_c*a\n",
    "w_ca += delta_w_ca\n",
    "print(delta_w_ca, w_ca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0008512818186487638 0.10085128181864877\n"
     ]
    }
   ],
   "source": [
    "delta_w_c0 = learning_rate*delta_c\n",
    "w_c0 += delta_w_c0\n",
    "print(delta_w_c0, w_c0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 0.1\n"
     ]
    }
   ],
   "source": [
    "delta_w_cb = learning_rate*delta_c*b\n",
    "w_cb += delta_w_cb\n",
    "print(delta_w_cb, w_cb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.10085128181864877 0.1 0.10085128181864877 0.1189103977991797 0.1343929220303063\n"
     ]
    }
   ],
   "source": [
    "print(w_ca, w_cb, w_c0, w_dc, w_d0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2nd iteration using 2nd training example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward Pass\n",
    "a, b = 0, 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.20085128181864878 0.5500446947195489\n"
     ]
    }
   ],
   "source": [
    "net_c = w_ca*a + w_cb*b + w_c0\n",
    "c = sigmoid(net_c)\n",
    "print(net_c, c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1997989554867362 0.5497842349651983\n"
     ]
    }
   ],
   "source": [
    "net_d = w_dc*c + w_d0\n",
    "d = sigmoid(net_d)\n",
    "print(net_d, d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.15113135250813423\n"
     ]
    }
   ],
   "source": [
    "# Error\n",
    "t = 0\n",
    "E = 0.5*math.pow(t-d,2)\n",
    "print(E)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.13608343498038789 -0.00400490715224327\n"
     ]
    }
   ],
   "source": [
    "# Backpropagation\n",
    "delta_d = (t-d)*d*(1-d)\n",
    "delta_c = (delta_d*w_dc)*c*(1-c)\n",
    "print(delta_d, delta_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.005436233415790779 0.11347416438338892\n"
     ]
    }
   ],
   "source": [
    "delta_w_dc = learning_rate*delta_d*c + momentum*delta_w_dc\n",
    "w_dc += delta_w_dc\n",
    "print(delta_w_dc, w_dc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.009871400666840696 0.12452152136346561\n"
     ]
    }
   ],
   "source": [
    "delta_w_d0 = learning_rate*delta_d + momentum*delta_w_d0\n",
    "w_d0 += delta_w_d0\n",
    "print(delta_w_d0, w_d0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0007661536367838874 0.10161743545543266\n"
     ]
    }
   ],
   "source": [
    "delta_w_ca = learning_rate*delta_c*a + momentum*delta_w_ca\n",
    "w_ca += delta_w_ca\n",
    "print(delta_w_ca, w_ca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.0012014721456729811 0.09879852785432702\n"
     ]
    }
   ],
   "source": [
    "delta_w_cb = learning_rate*delta_c*b + momentum*delta_w_cb\n",
    "w_cb += delta_w_cb\n",
    "print(delta_w_cb, w_cb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.0004353185088890937 0.10041596330975969\n"
     ]
    }
   ],
   "source": [
    "delta_w_c0 = learning_rate*delta_c + momentum*delta_w_c0\n",
    "w_c0 += delta_w_c0\n",
    "print(delta_w_c0, w_c0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.10161743545543266 0.09879852785432702 0.10041596330975969 0.11347416438338892 0.12452152136346561\n"
     ]
    }
   ],
   "source": [
    "print(w_ca, w_cb, w_c0, w_dc, w_d0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|Iteration|w_ca|w_cb|w_c0|w_dc|w_d0|\n",
    "| --      | -- | -- | -- | -- | -- |\n",
    "| 0       | 0.1| 0.1| 0.1| 0.1| 0.1|\n",
    "| 1       | 0.10085128181864877 | 0.1 | 0.10085128181864877 | 0.1189103977991797 |0.1343929220303063|\n",
    "| 2       | 0.10161743545543266 | 0.09879852785432702 | 0.10041596330975969 | 0.11347416438338892 | 0.12452152136346561|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.10278262873279762\n"
     ]
    }
   ],
   "source": [
    "# Retest error with latest weights\n",
    "a, b, t = 1, 0, 1\n",
    "net_c = w_ca*a + w_cb*b + w_c0\n",
    "c = sigmoid(net_c)\n",
    "net_d = w_dc*c + w_d0\n",
    "d =  sigmoid(net_d)\n",
    "E = 0.5*math.pow(t-d,2)\n",
    "print(E)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.14937885466499826\n"
     ]
    }
   ],
   "source": [
    "# Input Layer\n",
    "a, b, t = 0, 1, 0\n",
    "net_c = w_ca*a + w_cb*b + w_c0\n",
    "c = sigmoid(net_c)\n",
    "net_d = w_dc*c + w_d0\n",
    "d =  sigmoid(net_d)\n",
    "E = 0.5*math.pow(t-d,2)\n",
    "print(E)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}