micrograd/micrograd-part2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:47.858467400Z",
     "start_time": "2023-09-18T10:02:47.855057Z"
    }
   },
   "id": "d6c672d0051c9e4d"
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [],
   "source": [
    "class Value:\n",
    "  \n",
    "  def __init__(self, data, _children=(), _op='', label=''):\n",
    "    self.data = data\n",
    "    self.grad = 0.0\n",
    "    self._backward = lambda: None\n",
    "    self._prev = set(_children)\n",
    "    self._op = _op\n",
    "    self.label = label\n",
    "\n",
    "  def __repr__(self):\n",
    "    return f\"Value(data={self.data}, grad={self.grad})\"\n",
    "  \n",
    "  def __add__(self, other):\n",
    "    other = other if isinstance(other, Value) else Value(other)\n",
    "    out = Value(self.data + other.data, (self, other), '+')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += 1.0 * out.grad\n",
    "      other.grad += 1.0 * out.grad\n",
    "    out._backward = _backward\n",
    "    \n",
    "    return out\n",
    "\n",
    "  def __mul__(self, other):\n",
    "    other = other if isinstance(other, Value) else Value(other)\n",
    "    out = Value(self.data * other.data, (self, other), '*')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += other.data * out.grad\n",
    "      other.grad += self.data * out.grad\n",
    "    out._backward = _backward\n",
    "      \n",
    "    return out\n",
    "  \n",
    "  def __pow__(self, other):\n",
    "    assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
    "    out = Value(self.data**other, (self,), f'**{other}')\n",
    "\n",
    "    def _backward():\n",
    "        self.grad += other * (self.data ** (other - 1)) * out.grad\n",
    "    out._backward = _backward\n",
    "\n",
    "    return out\n",
    "  \n",
    "  def __rmul__(self, other): # other * self\n",
    "    return self * other\n",
    "\n",
    "  def __truediv__(self, other): # self / other\n",
    "    return self * other**-1\n",
    "\n",
    "  def __neg__(self): # -self\n",
    "    return self * -1\n",
    "\n",
    "  def __sub__(self, other): # self - other\n",
    "    return self + (-other)\n",
    "\n",
    "  def __radd__(self, other): # other + self\n",
    "    return self + other\n",
    "\n",
    "  def tanh(self):\n",
    "    x = self.data\n",
    "    t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "    out = Value(t, (self, ), 'tanh')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += (1 - t**2) * out.grad\n",
    "    out._backward = _backward\n",
    "    \n",
    "    return out\n",
    "  \n",
    "  def exp(self):\n",
    "    x = self.data\n",
    "    out = Value(math.exp(x), (self, ), 'exp')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += out.data * out.grad # NOTE: in the video I incorrectly used = instead of +=. Fixed here.\n",
    "    out._backward = _backward\n",
    "    \n",
    "    return out\n",
    "  \n",
    "  \n",
    "  def backward(self):\n",
    "    \n",
    "    topo = []\n",
    "    visited = set()\n",
    "    def build_topo(v):\n",
    "      if v not in visited:\n",
    "        visited.add(v)\n",
    "        for child in v._prev:\n",
    "          build_topo(child)\n",
    "        topo.append(v)\n",
    "    build_topo(self)\n",
    "    \n",
    "    self.grad = 1.0\n",
    "    for node in reversed(topo):\n",
    "      node._backward()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:47.868594Z",
     "start_time": "2023-09-18T10:02:47.858467400Z"
    }
   },
   "id": "128f74785be519ef"
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "def trace(root):\n",
    "  # builds a set of all nodes and edges in a graph\n",
    "  nodes, edges = set(), set()\n",
    "  def build(v):\n",
    "    if v not in nodes:\n",
    "      nodes.add(v)\n",
    "      for child in v._prev:\n",
    "        edges.add((child, v))\n",
    "        build(child)\n",
    "  build(root)\n",
    "  return nodes, edges\n",
    "\n",
    "def draw_dot(root):\n",
    "  dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n",
    "  \n",
    "  nodes, edges = trace(root)\n",
    "  for n in nodes:\n",
    "    uid = str(id(n))\n",
    "    # for any value in the graph, create a rectangular ('record') node for it\n",
    "    dot.node(name = uid, label = \"{ %s | data %.4f | grad %.4f }\" % (n.label, n.data, n.grad), shape='record')\n",
    "    if n._op:\n",
    "      # if this value is a result of some operation, create an op node for it\n",
    "      dot.node(name = uid + n._op, label = n._op)\n",
    "      # and connect this node to it\n",
    "      dot.edge(uid + n._op, uid)\n",
    "\n",
    "  for n1, n2 in edges:\n",
    "    # connect n1 to the op node of n2\n",
    "    dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "\n",
    "  return dot.render(directory='graphs')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:47.874196900Z",
     "start_time": "2023-09-18T10:02:47.868594Z"
    }
   },
   "id": "3884677d8135e12e"
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [],
   "source": [
    "class Neuron:\n",
    "  \n",
    "  def __init__(self, nin):\n",
    "    self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "    self.b = Value(random.uniform(-1,1))\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    # w * x + b\n",
    "    act = sum((wi*xi for wi, xi in zip(self.w, x)), self.b)\n",
    "    out = act.tanh()\n",
    "    return out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return self.w + [self.b]\n",
    "\n",
    "class Layer:\n",
    "  \n",
    "  def __init__(self, nin, nout):\n",
    "    self.neurons = [Neuron(nin) for _ in range(nout)]\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    outs = [n(x) for n in self.neurons]\n",
    "    return outs[0] if len(outs) == 1 else outs\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [p for neuron in self.neurons for p in neuron.parameters()]\n",
    "\n",
    "class MLP:\n",
    "  \n",
    "  def __init__(self, nin, nouts):\n",
    "    sz = [nin] + nouts\n",
    "    self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
    "  \n",
    "  def __call__(self, x):\n",
    "    for layer in self.layers:\n",
    "      x = layer(x)\n",
    "    return x\n",
    "  \n",
    "  def parameters(self):\n",
    "    return [p for layer in self.layers for p in layer.parameters()]\n",
    "  \n",
    "  def zero_grad(self):\n",
    "    for p in self.parameters():\n",
    "      p.grad = 0.0\n",
    "  \n",
    "  def update(self, lr=0.1):\n",
    "    for p in self.parameters():\n",
    "      p.data = -lr * p.grad\n",
    "          "
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:47.898618200Z",
     "start_time": "2023-09-18T10:02:47.879728700Z"
    }
   },
   "id": "7d07dc384bf1d714"
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Value(data=-0.6866307737236155, grad=0.0), Value(data=-0.7452110592515695, grad=0.0), Value(data=-0.1461124180793787, grad=0.0), Value(data=-0.42758917010076747, grad=0.0), Value(data=-0.1539533657714578, grad=0.0), Value(data=-0.7915388137601387, grad=0.0), Value(data=-0.1692676218233169, grad=0.0), Value(data=0.3396536630220943, grad=0.0), Value(data=0.037571696625607665, grad=0.0), Value(data=-0.9700774499092877, grad=0.0), Value(data=-0.4575134233000955, grad=0.0), Value(data=0.8623572275882625, grad=0.0), Value(data=0.37768544754096034, grad=0.0), Value(data=0.20628314682541982, grad=0.0), Value(data=-0.15415933182689723, grad=0.0), Value(data=0.796658837651059, grad=0.0), Value(data=0.536116832176617, grad=0.0), Value(data=0.5611505407356372, grad=0.0), Value(data=-0.49266988194042693, grad=0.0), Value(data=0.5956162422132556, grad=0.0), Value(data=0.9122929103430713, grad=0.0), Value(data=-0.37292695389273356, grad=0.0), Value(data=-0.1348546810901885, grad=0.0), Value(data=0.2669924969926436, grad=0.0), Value(data=-0.42596602142890516, grad=0.0), Value(data=-0.5256885793548143, grad=0.0), Value(data=-0.923748915635622, grad=0.0), Value(data=0.6164772165275871, grad=0.0), Value(data=-0.015348690211872684, grad=0.0), Value(data=0.3146406289935444, grad=0.0), Value(data=0.9235556243426022, grad=0.0), Value(data=0.6728504018112305, grad=0.0), Value(data=-0.266457833260479, grad=0.0), Value(data=0.7587982437182756, grad=0.0), Value(data=-0.8239190609448042, grad=0.0), Value(data=-0.019948736901535202, grad=0.0), Value(data=-0.8150910089486247, grad=0.0), Value(data=0.13163336782138635, grad=0.0), Value(data=0.4733746580786613, grad=0.0), Value(data=-0.7596050444130196, grad=0.0), Value(data=0.2559001778868075, grad=0.0)]\n"
     ]
    },
    {
     "data": {
      "text/plain": "Value(data=0.6480805947186898, grad=0.0)"
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = [2.0, 3.0, -1.0]\n",
    "n = MLP(3, [4, 4, 1])\n",
    "print(n.parameters())\n",
    "n(x)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:47.898618200Z",
     "start_time": "2023-09-18T10:02:47.882372500Z"
    }
   },
   "id": "295136950ce40f86"
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [],
   "source": [
    "xs = [\n",
    "  [2.0, 3.0, -1.0],\n",
    "  [3.0, -1.0, 0.5],\n",
    "  [0.5, 1.0, 1.0],\n",
    "  [1.0, 1.0, -1.0],\n",
    "]\n",
    "ys = [1.0, -1.0, -1.0, 1.0] # desired target"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:47.898618200Z",
     "start_time": "2023-09-18T10:02:47.888554800Z"
    }
   },
   "id": "306510a4a76e8a3e"
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 5.458779523225546\n",
      "1 4.664210224292212\n",
      "2 5.183198218430247\n",
      "3 4.386930984951293\n",
      "4 4.898069926226806\n",
      "5 3.299859704484054\n",
      "6 4.086224244781954\n",
      "7 1.1619808027662033\n",
      "8 1.0065499774422133\n",
      "9 1.3093880089024814\n",
      "10 0.17153060440176224\n",
      "11 0.06617245484342965\n",
      "12 0.05628162056683992\n",
      "13 0.049838841407583344\n",
      "14 0.044995680713417786\n",
      "15 0.04111198558813155\n",
      "16 0.03788550764546951\n",
      "17 0.035144291645663996\n",
      "18 0.03277824625849876\n",
      "19 0.03071127514098696\n"
     ]
    }
   ],
   "source": [
    "for k in range(20):\n",
    "  \n",
    "  # forward pass\n",
    "  ypred = [n(x) for x in xs]\n",
    "  loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
    "  \n",
    "  # backward pass\n",
    "  n.zero_grad()\n",
    "  loss.backward()\n",
    "  \n",
    "  # update\n",
    "  for p in n.parameters():\n",
    "    p.data += -0.1 * p.grad\n",
    "  \n",
    "  print(k, loss.data)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:47.918461200Z",
     "start_time": "2023-09-18T10:02:47.895610900Z"
    }
   },
   "id": "51a8799416f19b7d"
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "data": {
      "text/plain": "'graphs\\\\Digraph.gv.svg'"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(loss)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-09-18T10:02:48.073615800Z",
     "start_time": "2023-09-18T10:02:47.918461200Z"
    }
   },
   "id": "64f5f2fe2a7a1b8d"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}