mirror of
https://github.com/guezoloic/neural-network.git
synced 2026-01-25 09:34:23 +00:00
209 lines
7.5 KiB
Python
209 lines
7.5 KiB
Python
import math
|
|
import random
|
|
from typing import Optional, Callable
|
|
|
|
|
|
def sigmoid(x: float) -> float:
|
|
return 1 / (1 + math.exp(-x))
|
|
|
|
|
|
def sigmoid_deriv(x: float) -> float:
|
|
y: float = sigmoid(x)
|
|
return y * (1 - y)
|
|
|
|
|
|
class Neuron:
|
|
def __init__(self, input_size: int) -> None:
|
|
"""
|
|
Set up the neuron's parameters (weights, and bias)
|
|
|
|
:param input_size: Number of incomming inputs (must be > 0)
|
|
"""
|
|
# Store input dimensions for structural consistency
|
|
self._input_size: int = input_size
|
|
|
|
# Scale each input influence using random weights.
|
|
self._weight: list[float] = [
|
|
random.uniform(-1., 1.) for _ in range(input_size)
|
|
]
|
|
|
|
# Initialize a shift to the activation threshold with a random bias
|
|
self._bias: float = random.uniform(-1., 1.)
|
|
|
|
def forward(self, x: list[int], fn: Optional[Callable[[float], float]] = None) -> float:
|
|
"""
|
|
Execute the neuron's forward pass.
|
|
|
|
:param x: Description
|
|
:param activate: Description
|
|
"""
|
|
|
|
if len(x) != self._input_size:
|
|
raise ValueError(
|
|
f"Input vertex dimension {len(x)} mismatches the "
|
|
f"stored size {self._input_size}")
|
|
|
|
self._z: float = sum(welement * xelement for welement,
|
|
xelement in zip(self._weight, x)) + self._bias
|
|
|
|
return fn(self._z) if fn is not None else self._z
|
|
|
|
def backward(self, dz_dw: list[float], dcost_dy: float, learning_rate: float,
|
|
fn_deriv: Optional[Callable[[float], float]] = None) -> list[float]:
|
|
# Check dimension consistency
|
|
if len(dz_dw) != self._input_size:
|
|
raise ValueError(
|
|
f"Input vertex dimension {len(dz_dw)} mismatches "
|
|
f"stored size {self._input_size}")
|
|
|
|
# Local gradient: dy/dz (defaults to 1.0 for linear identity)
|
|
dy_dz: float = fn_deriv(self._z) if fn_deriv is not None else 1.
|
|
|
|
# Compute common error term (delta)
|
|
# dC/dz = dC/dy * dy/dz
|
|
delta: float = dcost_dy * dy_dz
|
|
|
|
# Update weights: weight -= learning_rate * dC/dz * dz/dw
|
|
for i in range(self._input_size):
|
|
self._weight[i] -= learning_rate * delta * dz_dw[i]
|
|
|
|
# Update bias: bias -= learning_rate * dC/dz * dz/db (where dz/db = 1)
|
|
self._bias -= learning_rate * delta * 1.0
|
|
|
|
# Return input gradient (dC/dx): dC/dz * dz/dx (where dz/dx = weights)
|
|
# This vector allows the error to flow back to previous layers.
|
|
return [delta * w for w in self._weight]
|
|
|
|
def __repr__(self) -> str:
|
|
jmp: int = int(math.sqrt(self._input_size))
|
|
text: list[str] = []
|
|
|
|
for i in range(0, self._input_size, jmp):
|
|
line: str = str.join("", str(self._weight[i: (i + jmp)]))
|
|
text.append(line)
|
|
|
|
return f"weight:\n{str.join("\n", text)}\nbias: {self._bias}"
|
|
|
|
# # neuron class
|
|
# class Neuron:
|
|
# """
|
|
# z : linear combination of inputs and weights plus bias (pre-activation)
|
|
# y : output of the activation function (sigmoid(z))
|
|
# w : list of weights, one for each input
|
|
# """
|
|
# def __init__(self, input_size):
|
|
# # number of inputs to this neuron
|
|
# self._input_size = input_size
|
|
# # importance to each input
|
|
# self._weight = [random.uniform(-1, 1) for _ in range(self._input_size)]
|
|
# # importance of the neuron
|
|
# self._bias = random.uniform(-1, 1)
|
|
|
|
# def forward(self, x, activate=True):
|
|
# """
|
|
# x : list of input values to the neuron
|
|
# """
|
|
# # computes the weighted sum of inputs and add the bias
|
|
# self._z = sum(w * xi for w, xi in zip(self.weight, x)) + self.bias
|
|
# # normalize the output between 0 and 1 if activate
|
|
# last_output = sigmoid(self.z) if activate else self.z
|
|
|
|
# return last_output
|
|
|
|
# # adjust weight and bias of neuron
|
|
# def backward(self, x, dcost_dy, learning_rate):
|
|
# """
|
|
# x : list of input values to the neuron
|
|
# dcost_dy : derivate of the cost function `(2 * (output - target))`
|
|
# learning_rate : learning factor (adjust the speed of weight/bias change during training)
|
|
|
|
# weight -= learning_rate * dC/dy * dy/dz * dz/dw
|
|
# bias -= learning_rate * dC/dy * dy/dz * dz/db
|
|
# """
|
|
# # dy/dz: derivate of the sigmoid activation
|
|
# dy_dz = sigmoid_deriv(self.z)
|
|
# # dz/dw = x
|
|
# dz_dw = x
|
|
|
|
# assert len(dz_dw) >= self.isize, "too many value for input size"
|
|
|
|
# # dz/db = 1
|
|
# dz_db = 1
|
|
|
|
# for i in range(self.isize):
|
|
# # update each weight `weight -= learning_rate * dC/dy * dy/dz * x_i`
|
|
# self.weight[i] -= learning_rate * dcost_dy * dy_dz * dz_dw[i]
|
|
|
|
# # update bias: bias -= learning_rate * dC/dy * dy/dz * dz/db
|
|
# self.bias -= learning_rate * dcost_dy * dy_dz * dz_db
|
|
|
|
# # return gradient vector len(weight) dimension
|
|
# return [dcost_dy * dy_dz * w for w in self.weight]
|
|
|
|
# def __repr__(self):
|
|
# pass
|
|
|
|
# class Layer:
|
|
# def __init__(self, input_size, output_size):
|
|
# """
|
|
# input_size : size of each neuron input
|
|
# output_size : size of neurons
|
|
# """
|
|
# self.size = output_size
|
|
# # list of neurons
|
|
# self.neurons = [Neuron(input_size) for _ in range(output_size)]
|
|
|
|
# def forward(self, inputs, activate=True):
|
|
# self.inputs = inputs
|
|
# # give the same inputs to each neuron in the layer
|
|
# return [neuron.forward(inputs, activate) for neuron in self.neurons]
|
|
|
|
# # adjust weight and bias of the layer (all neurons)
|
|
# def backward(self, dcost_dy_list, learning_rate=0.1):
|
|
# # init layer gradient vector len(input) dimention
|
|
# input_gradients = [0.0] * len(self.inputs)
|
|
|
|
# for i, neuron in enumerate(self.neurons):
|
|
# dcost_dy = dcost_dy_list[i]
|
|
# grad_to_input = neuron.backward(self.inputs, dcost_dy, learning_rate)
|
|
|
|
# # accumulate the input gradients from all neurons
|
|
# for j in range(len(grad_to_input)):
|
|
# input_gradients[j] += grad_to_input[j]
|
|
|
|
# # return layer gradient
|
|
# return input_gradients
|
|
|
|
# class NeuralNetwork:
|
|
# def __init__(self, layer_size):
|
|
# self.layers = [Layer(layer_size[i], layer_size[i+1]) for i in range(len(layer_size) - 1)]
|
|
|
|
# def forward(self, inputs):
|
|
# output = inputs
|
|
# for i, layer in enumerate(self.layers):
|
|
# activate = (i != len(self.layers) - 1) # deactivate sigmoid latest neuron
|
|
# output = layer.forward(output, activate=activate)
|
|
# return output
|
|
|
|
# def backward(self, inputs, targets, learning_rate=0.1):
|
|
# """
|
|
# target must be a list with the same length that the final layer
|
|
# input
|
|
# """
|
|
# output = self.forward(inputs)
|
|
|
|
# # computes the initial gradient of the cost function for each neuron
|
|
# # by using Mean Squared Error's derivate: dC/dy = 2 * (output - target)
|
|
# dcost_dy_list = [2 * (o - t) for o, t in zip(output, targets)]
|
|
|
|
# grad = dcost_dy_list
|
|
# for layer in reversed(self.layers):
|
|
# # backpropagate the gradient of the layer to update weights and biases
|
|
# grad = layer.backward(grad, learning_rate)
|
|
|
|
# # return final gradient
|
|
# return grad
|
|
|
|
# if __name__ == "__main__":
|
|
# print("you might want to run main.py instead of network.py")
|