From 39a88325c549e499057479aba3dad779631e7389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20GUEZO?= Date: Thu, 5 Jun 2025 23:16:45 +0200 Subject: [PATCH] feat(nnetwork.ipynb): Initialize 4 Step text - change 45_000 epochs to 5_000 --- main.py | 2 +- nnetwork.ipynb | 98 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/main.py b/main.py index bb5c4f7..e9018f4 100644 --- a/main.py +++ b/main.py @@ -32,7 +32,7 @@ def main(): network = NeuralNetwork([8, 16, 1]) print("Start training...") - train_network(network, verbose=True, size_data=size, epochs=45_000) + train_network(network, verbose=True, size_data=size, epochs=5_000) print("End training...") while True: diff --git a/nnetwork.ipynb b/nnetwork.ipynb index 97fedeb..be23278 100644 --- a/nnetwork.ipynb +++ b/nnetwork.ipynb @@ -131,11 +131,13 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "7ca39a42", "metadata": {}, "outputs": [], "source": [ + "import random\n", + "\n", "# neuron class 2\n", "class Neuron:\n", " \"\"\"\n", @@ -215,15 +217,9 @@ "id": "5593a84a", "metadata": {}, "source": [ - "The test result is a bit random due to the randomly initialized weights and bias in each Neuron. None of the neurons has been trained for this input." - ] - }, - { - "cell_type": "markdown", - "id": "aa57ae8e", - "metadata": {}, - "source": [ - "# 3" + "The test result is a bit random due to the randomly initialized weights and bias in each Neuron. None of the neurons has been trained for this input.\n", + "\n", + "## Step 4 - Backward Pass Function" ] }, { @@ -233,37 +229,79 @@ "metadata": {}, "outputs": [], "source": [ - "import math\n", "import random\n", "\n", - "# Neuron 3\n", + "# neuron class 3\n", "class Neuron:\n", - " def __init__(self, isize: int) -> None:\n", + " \"\"\"\n", + " z : linear combination of inputs and weights plus bias (pre-activation)\n", + " y : output of the activation function (sigmoid(z))\n", + " w : list of weights, one for each input\n", + " \"\"\"\n", + " def __init__(self, isize):\n", + " # number of inputs to this neuron\n", " self.isize = isize\n", - " self.weight = [random.uniform(0, 1) for _ in range(self.isize)]\n", - " self.bias = random.uniform(0, 1)\n", + " # importance to each input\n", + " self.weight = [random.uniform(-1, 1) for _ in range(self.isize)]\n", + " # importance of the neuron\n", + " self.bias = random.uniform(-1, 1)\n", "\n", - " def forward(self, inputs: list) -> float:\n", - " assert len(inputs) == self.isize, \"error: incorrect inputs number\"\n", - " total = sum(self.weight[i] * inputs[i] for i in range(self.isize)) + self.bias\n", - " return self.sigmoid(total)\n", + " def forward(self, x, activate=True):\n", + " \"\"\"\n", + " x : list of input values to the neuron\n", + " \"\"\"\n", + " # computes the weighted sum of inputs and add the bias\n", + " self.z = sum(w * xi for w, xi in zip(self.weight, x)) + self.bias\n", + " # normalize the output between 0 and 1 if activate\n", + " last_output = sigmoid(self.z) if activate else self.z\n", + "\n", + " return last_output\n", " \n", - " def sigmoid(x: float) -> float:\n", - " return 1/(1 + math.exp(-x))\n", + " # adjust weight and bias of neuron\n", + " def backward(self, x, dcost_dy, learning_rate):\n", + " \"\"\"\n", + " x : list of input values to the neuron \n", + " dcost_dy : derivate of the cost function `(2 * (output - target))`\n", + " learning_rate : learning factor (adjust the speed of weight/bias change during training)\n", "\n", - " # target needs to be between 0 and 1\n", - " def train(self, inputs: list, target: float, learning_rate: float = 0.1):\n", - " z = sum(self.weight[i] * inputs[i] for i in range(self.isize)) + self.bias\n", - " output = self.sigmoid(z)\n", + " weight -= learning_rate * dC/dy * dy/dz * dz/dw\n", + " bias -= learning_rate * dC/dy * dy/dz * dz/db\n", + " \"\"\"\n", + " # dy/dz: derivate of the sigmoid activation\n", + " dy_dz = sigmoid_deriv(self.z)\n", + " # dz/dw = x\n", + " dz_dw = x\n", "\n", - " error = output - target\n", - " d_sigmoid = output * (1 - output)\n", - " dz = error * d_sigmoid\n", + " assert len(dz_dw) >= self.isize, \"too many value for input size\"\n", + "\n", + " # dz/db = 1\n", + " dz_db = 1\n", "\n", " for i in range(self.isize):\n", - " self.weight[i] -= learning_rate * dz * inputs[i]\n", + " # update each weight `weight -= learning_rate * dC/dy * dy/dz * x_i`\n", + " self.weight[i] -= learning_rate * dcost_dy * dy_dz * dz_dw[i]\n", "\n", - " self.bias -= learning_rate * dz\n" + " # update bias: bias -= learning_rate * dC/dy * dy/dz * dz/db\n", + " self.bias -= learning_rate * dcost_dy * dy_dz * dz_db\n", + "\n", + " # return gradient vector len(input) dimension\n", + " return [dcost_dy * dy_dz * w for w in self.weight]" + ] + }, + { + "cell_type": "markdown", + "id": "0c9baabf", + "metadata": {}, + "source": [ + "The `backward()` method train the neuron by adjusting its weights and bias using **the gradient descent**. This is based on erros and gradient of the activation function:\n", + "\n", + "1. **derivates sigmoid, inputs, and lineear combination**:\n", + " \n", + "2. **adjust each input weight**:\n", + "\n", + "3. **adjust neuron bias**:\n", + "\n", + "4. **return gradient vector**:" ] } ],