feat(nnetwork.ipynb): Initialize 4 Step text

- change 45_000 epochs to 5_000

main.py

@@ -32,7 +32,7 @@ def main():
     network = NeuralNetwork([8, 16, 1])
 
     print("Start training...")
-    train_network(network, verbose=True, size_data=size, epochs=45_000)
+    train_network(network, verbose=True, size_data=size, epochs=5_000)
     print("End training...")
 
     while True:

nnetwork.ipynb

@@ -131,11 +131,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "id": "7ca39a42",
    "metadata": {},
    "outputs": [],
    "source": [
+    "import random\n",
+    "\n",
     "# neuron class 2\n",
     "class Neuron:\n",
     "    \"\"\"\n",
@@ -215,15 +217,9 @@
    "id": "5593a84a",
    "metadata": {},
    "source": [
-    "The test result is a bit random due to the randomly initialized weights and bias in each Neuron. None of the neurons has been trained for this input."
+    "The test result is a bit random due to the randomly initialized weights and bias in each Neuron. None of the neurons has been trained for this input.\n",
-   ]
+    "\n",
-  },
+    "## Step 4 - Backward Pass Function"
-  {
-   "cell_type": "markdown",
-   "id": "aa57ae8e",
-   "metadata": {},
-   "source": [
-    "# 3"
    ]
   },
   {
@@ -233,37 +229,79 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import math\n",
     "import random\n",
     "\n",
-    "# Neuron 3\n",
+    "# neuron class 3\n",
     "class Neuron:\n",
-    "    def __init__(self, isize: int) -> None:\n",
+    "    \"\"\"\n",
+    "    z                   : linear combination of inputs and weights plus bias (pre-activation)\n",
+    "    y                   : output of the activation function (sigmoid(z))\n",
+    "    w                   : list of weights, one for each input\n",
+    "    \"\"\"\n",
+    "    def __init__(self, isize):\n",
+    "        # number of inputs to this neuron\n",
     "        self.isize = isize\n",
-    "        self.weight = [random.uniform(0, 1) for _ in range(self.isize)]\n",
+    "        # importance to each input\n",
-    "        self.bias = random.uniform(0, 1)\n",
+    "        self.weight = [random.uniform(-1, 1) for _ in range(self.isize)]\n",
+    "        # importance of the neuron\n",
+    "        self.bias = random.uniform(-1, 1)\n",
     "\n",
-    "    def forward(self, inputs: list) -> float:\n",
+    "    def forward(self, x, activate=True):\n",
-    "        assert len(inputs) == self.isize, \"error: incorrect inputs number\"\n",
+    "        \"\"\"\n",
-    "        total = sum(self.weight[i] * inputs[i] for i in range(self.isize)) + self.bias\n",
+    "        x               : list of input values to the neuron\n",
-    "        return self.sigmoid(total)\n",
+    "        \"\"\"\n",
+    "        # computes the weighted sum of inputs and add the bias\n",
+    "        self.z = sum(w * xi for w, xi in zip(self.weight, x)) + self.bias\n",
+    "        # normalize the output between 0 and 1 if activate\n",
+    "        last_output = sigmoid(self.z) if activate else self.z\n",
+    "\n",
+    "        return last_output\n",
     "    \n",
-    "    def sigmoid(x: float) -> float:\n",
+    "    # adjust weight and bias of neuron\n",
-    "        return 1/(1 + math.exp(-x))\n",
+    "    def backward(self, x, dcost_dy, learning_rate):\n",
+    "        \"\"\"\n",
+    "        x               : list of input values to the neuron  \n",
+    "        dcost_dy        : derivate of the cost function `(2 * (output - target))`\n",
+    "        learning_rate   : learning factor (adjust the speed of weight/bias change during training)\n",
     "\n",
-    "    # target needs to be between 0 and 1\n",
+    "        weight -= learning_rate * dC/dy * dy/dz * dz/dw\n",
-    "    def train(self, inputs: list, target: float, learning_rate: float = 0.1):\n",
+    "        bias   -= learning_rate * dC/dy * dy/dz * dz/db\n",
-    "        z = sum(self.weight[i] * inputs[i] for i in range(self.isize)) + self.bias\n",
+    "        \"\"\"\n",
-    "        output = self.sigmoid(z)\n",
+    "        # dy/dz: derivate of the sigmoid activation\n",
+    "        dy_dz = sigmoid_deriv(self.z)\n",
+    "        # dz/dw = x\n",
+    "        dz_dw = x\n",
     "\n",
-    "        error = output - target\n",
+    "        assert len(dz_dw) >= self.isize, \"too many value for input size\"\n",
-    "        d_sigmoid = output * (1 - output)\n",
+    "\n",
-    "        dz = error * d_sigmoid\n",
+    "        # dz/db = 1\n",
+    "        dz_db = 1\n",
     "\n",
     "        for i in range(self.isize):\n",
-    "            self.weight[i] -= learning_rate * dz * inputs[i]\n",
+    "            # update each weight `weight -= learning_rate * dC/dy * dy/dz * x_i`\n",
+    "            self.weight[i] -= learning_rate * dcost_dy * dy_dz * dz_dw[i]\n",
     "\n",
-    "        self.bias -= learning_rate * dz\n"
+    "        # update bias: bias -= learning_rate * dC/dy * dy/dz * dz/db\n",
+    "        self.bias -= learning_rate * dcost_dy * dy_dz * dz_db\n",
+    "\n",
+    "        # return gradient vector len(input) dimension\n",
+    "        return [dcost_dy * dy_dz * w for w in self.weight]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c9baabf",
+   "metadata": {},
+   "source": [
+    "The `backward()` method train the neuron by adjusting its weights and bias using **the gradient descent**. This is based on erros and gradient of the activation function:\n",
+    "\n",
+    "1. **derivates sigmoid, inputs, and lineear combination**:\n",
+    "    \n",
+    "2. **adjust each input weight**:\n",
+    "\n",
+    "3. **adjust neuron bias**:\n",
+    "\n",
+    "4. **return gradient vector**:"
    ]
   }
  ],