From 7e37b7a987c95df535ac578968cd461e6b12d265 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Thu, 26 Oct 2023 07:48:17 +0530 Subject: [PATCH 01/10] Added automatic differentiation algorithm --- machine_learning/auto-diff.py | 314 ++++++++++++++++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 machine_learning/auto-diff.py diff --git a/machine_learning/auto-diff.py b/machine_learning/auto-diff.py new file mode 100644 index 000000000000..6a8a35e0e3c5 --- /dev/null +++ b/machine_learning/auto-diff.py @@ -0,0 +1,314 @@ +""" +Demonstration of the Automatic Differentiation (Reverse mode). + +Reference: https://en.wikipedia.org/wiki/Automatic_differentiation + +Author: Poojan Smart +Email: smrtpoojan@gmail.com + +Examples: + +>>> with GradientTracker() as tracker: +... a = Variable([2.0, 5.0]) +... b = Variable([1.0, 2.0]) +... m = Variable([1.0, 2.0]) +... c = a + b +... d = a * b +... e = c / d +>>> print(tracker.gradient(e, a)) +[-0.25 -0.04] +>>> print(tracker.gradient(e, b)) +[-1. -0.25] +>>> print(tracker.gradient(e, m)) +None + +>>> with GradientTracker() as tracker: +... a = Variable([[2.0, 5.0]]) +... b = Variable([[1.0], [2.0]]) +... c = a @ b +>>> print(tracker.gradient(c, a)) +[[1. 2.]] +>>> print(tracker.gradient(c, b)) +[[2.] + [5.]] + +>>> with GradientTracker() as tracker: +... a = Variable([[2.0, 5.0]]) +... b = a ** 3 +>>> print(tracker.gradient(b, a)) +[[12. 75.]] +""" +from __future__ import annotations + +from enum import Enum + +import numpy as np + + +class Variable: + """ + Class represents n-dimensional object which is used to wrap + numpy array on which operations will be performed and gradient + will be calculated. + """ + + def __init__(self, value): + self.value = np.array(value) + + # pointers to the operations to which the Variable is input + self.param_to = [] + # pointer to the operation of which the Variable is output of + self.result_of = None + + def __str__(self) -> str: + return f"Variable({self.value})" + + def numpy(self) -> np.ndarray: + return self.value + + def __add__(self, other: Variable) -> Variable: + result = Variable(self.value + other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.ADD, params=[self, other], output=result) + return result + + def __sub__(self, other: Variable) -> Variable: + result = Variable(self.value - other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.SUB, params=[self, other], output=result) + return result + + def __mul__(self, other: Variable) -> Variable: + result = Variable(self.value * other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.MUL, params=[self, other], output=result) + return result + + def __truediv__(self, other: Variable) -> Variable: + result = Variable(self.value / other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.DIV, params=[self, other], output=result) + return result + + def __matmul__(self, other: Variable) -> Variable: + result = Variable(self.value @ other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation( + OpType.MATMUL, params=[self, other], output=result + ) + return result + + def __pow__(self, power: int) -> Variable: + result = Variable(self.value**power) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation( + OpType.POWER, + params=[self], + output=result, + other_params={"power": power}, + ) + return result + + def add_param_to(self, param_to: Operation): + self.param_to.append(param_to) + + def add_result_of(self, result_of: Operation): + self.result_of = result_of + + +class OpType(Enum): + """ + Class represents list of supported operations on Variable for + gradient calculation. + """ + + ADD = 0 + SUB = 1 + MUL = 2 + DIV = 3 + MATMUL = 4 + POWER = 5 + + +class Operation: + """ + Class represents operation between single or two Variable objects. + Operation objects contains type of operation, pointers to input Variable + objects and pointer to resulting Variable from the operation. + """ + + def __init__( + self, + op_type: OpType, + other_params: dict | None = None, + ): + self.op_type = op_type + self.params: list[Variable] = [] + self.output: Variable | None = None + self.other_params = {} if other_params is None else other_params + + def add_params(self, params: list[Variable]): + self.params = params + + def add_output(self, output: Variable): + self.output = output + + def __eq__(self, value) -> bool: + if isinstance(value, OpType): + return self.op_type == value + return False + + +class GradientTracker: + """ + Class contains methods to compute partial derivatives of Variable + based on the computation graph. + """ + + def __new__(cls): + """ + Executes at the creation of class object and returns if + object is already created. This class follows singleton + design pattern. + """ + if not hasattr(cls, "instance"): + cls.instance = super().__new__(cls) + return cls.instance + + def __init__(self): + self.enabled = False + + def __enter__(self): + self.enabled = True + return self + + def __exit__(self, exc_type, exc_value, tb): + self.enabled = False + + def add_operation( + self, + op_type: OpType, + params: list[Variable], + output: Variable, + other_params: dict | None = None, + ): + """ + Adds Operation object to the related Variable objects for + creating computational graph for calculating gradients. + + Args: + op_type: Operation type + params: Input parameters to the operation + output: Output variable of the operation + """ + operation = Operation(op_type, other_params=other_params) + param_nodes = [] + for param in params: + param.add_param_to(operation) + param_nodes.append(param) + output.add_result_of(operation) + + operation.add_params(param_nodes) + operation.add_output(output) + + def gradient(self, target: Variable, source: Variable) -> np.ndarray | None: + """ + Reverse accumulation of partial derivatives to calculate gradients + of target variable with respect to source variable. + + Args: + target: target variable for which gradients are calculated. + source: source variable with respect to which the gradients are + calculated. + + Returns: + Gradient of the source variable with respect to the target variable + """ + # partial derivatives with respect to target + partial_deriv = {target: np.ones_like(target.numpy())} + + # iterating through each operations in the computation graph + operation_queue = [target.result_of] + while len(operation_queue) > 0: + operation = operation_queue.pop() + for param in operation.params: + # as per the chain rule, multiplying partial derivatives + # of variables with respect to the target + dparam_doutput = self.derivative(param, operation) + dparam_dtarget = dparam_doutput * partial_deriv[operation.output] + if param in partial_deriv: + partial_deriv[param] += dparam_dtarget + else: + partial_deriv[param] = dparam_dtarget + + if param.result_of: + operation_queue.append(param.result_of) + + if source in partial_deriv: + return partial_deriv[source] + return None + + def derivative(self, param: Variable, operation: Operation) -> np.ndarray: + """ + Compute the derivative of given operation/function + + Args: + param: variable to be differentiated + operation: function performed on the input variable + + Returns: + Derivative of input variable with respect to the output of + the operation + """ + params = operation.params + + derivative = None + + if operation == OpType.ADD: + derivative = np.ones_like(params[0].numpy(), dtype=np.float64) + elif operation == OpType.SUB: + if params[0] == param: + derivative = np.ones_like(params[0].numpy(), dtype=np.float64) + else: + derivative = -np.ones_like(params[1].numpy(), dtype=np.float64) + elif operation == OpType.MUL: + derivative = ( + params[1].numpy().T if params[0] == param else params[0].numpy().T + ) + elif operation == OpType.DIV: + if params[0] == param: + derivative = 1 / params[1].numpy() + else: + derivative = -params[0].numpy() / (params[1].numpy() ** 2) + elif operation == OpType.MATMUL: + derivative = ( + params[1].numpy().T if params[0] == param else params[0].numpy().T + ) + elif operation == OpType.POWER: + power = operation.other_params["power"] + derivative = power * (params[0].numpy() ** (power - 1)) + return derivative + + +if __name__ == "__main__": + import doctest + + doctest.testmod() From 1fbe703961a90584d2c3ac9f50ba041a2d2a5272 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Thu, 26 Oct 2023 07:53:56 +0530 Subject: [PATCH 02/10] file name changed --- machine_learning/automatic_differentiation.py | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 machine_learning/automatic_differentiation.py diff --git a/machine_learning/automatic_differentiation.py b/machine_learning/automatic_differentiation.py new file mode 100644 index 000000000000..6a8a35e0e3c5 --- /dev/null +++ b/machine_learning/automatic_differentiation.py @@ -0,0 +1,314 @@ +""" +Demonstration of the Automatic Differentiation (Reverse mode). + +Reference: https://en.wikipedia.org/wiki/Automatic_differentiation + +Author: Poojan Smart +Email: smrtpoojan@gmail.com + +Examples: + +>>> with GradientTracker() as tracker: +... a = Variable([2.0, 5.0]) +... b = Variable([1.0, 2.0]) +... m = Variable([1.0, 2.0]) +... c = a + b +... d = a * b +... e = c / d +>>> print(tracker.gradient(e, a)) +[-0.25 -0.04] +>>> print(tracker.gradient(e, b)) +[-1. -0.25] +>>> print(tracker.gradient(e, m)) +None + +>>> with GradientTracker() as tracker: +... a = Variable([[2.0, 5.0]]) +... b = Variable([[1.0], [2.0]]) +... c = a @ b +>>> print(tracker.gradient(c, a)) +[[1. 2.]] +>>> print(tracker.gradient(c, b)) +[[2.] + [5.]] + +>>> with GradientTracker() as tracker: +... a = Variable([[2.0, 5.0]]) +... b = a ** 3 +>>> print(tracker.gradient(b, a)) +[[12. 75.]] +""" +from __future__ import annotations + +from enum import Enum + +import numpy as np + + +class Variable: + """ + Class represents n-dimensional object which is used to wrap + numpy array on which operations will be performed and gradient + will be calculated. + """ + + def __init__(self, value): + self.value = np.array(value) + + # pointers to the operations to which the Variable is input + self.param_to = [] + # pointer to the operation of which the Variable is output of + self.result_of = None + + def __str__(self) -> str: + return f"Variable({self.value})" + + def numpy(self) -> np.ndarray: + return self.value + + def __add__(self, other: Variable) -> Variable: + result = Variable(self.value + other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.ADD, params=[self, other], output=result) + return result + + def __sub__(self, other: Variable) -> Variable: + result = Variable(self.value - other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.SUB, params=[self, other], output=result) + return result + + def __mul__(self, other: Variable) -> Variable: + result = Variable(self.value * other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.MUL, params=[self, other], output=result) + return result + + def __truediv__(self, other: Variable) -> Variable: + result = Variable(self.value / other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation(OpType.DIV, params=[self, other], output=result) + return result + + def __matmul__(self, other: Variable) -> Variable: + result = Variable(self.value @ other.value) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation( + OpType.MATMUL, params=[self, other], output=result + ) + return result + + def __pow__(self, power: int) -> Variable: + result = Variable(self.value**power) + + with GradientTracker() as tracker: + # if tracker is enabled, computation graph will be updated + if tracker.enabled: + tracker.add_operation( + OpType.POWER, + params=[self], + output=result, + other_params={"power": power}, + ) + return result + + def add_param_to(self, param_to: Operation): + self.param_to.append(param_to) + + def add_result_of(self, result_of: Operation): + self.result_of = result_of + + +class OpType(Enum): + """ + Class represents list of supported operations on Variable for + gradient calculation. + """ + + ADD = 0 + SUB = 1 + MUL = 2 + DIV = 3 + MATMUL = 4 + POWER = 5 + + +class Operation: + """ + Class represents operation between single or two Variable objects. + Operation objects contains type of operation, pointers to input Variable + objects and pointer to resulting Variable from the operation. + """ + + def __init__( + self, + op_type: OpType, + other_params: dict | None = None, + ): + self.op_type = op_type + self.params: list[Variable] = [] + self.output: Variable | None = None + self.other_params = {} if other_params is None else other_params + + def add_params(self, params: list[Variable]): + self.params = params + + def add_output(self, output: Variable): + self.output = output + + def __eq__(self, value) -> bool: + if isinstance(value, OpType): + return self.op_type == value + return False + + +class GradientTracker: + """ + Class contains methods to compute partial derivatives of Variable + based on the computation graph. + """ + + def __new__(cls): + """ + Executes at the creation of class object and returns if + object is already created. This class follows singleton + design pattern. + """ + if not hasattr(cls, "instance"): + cls.instance = super().__new__(cls) + return cls.instance + + def __init__(self): + self.enabled = False + + def __enter__(self): + self.enabled = True + return self + + def __exit__(self, exc_type, exc_value, tb): + self.enabled = False + + def add_operation( + self, + op_type: OpType, + params: list[Variable], + output: Variable, + other_params: dict | None = None, + ): + """ + Adds Operation object to the related Variable objects for + creating computational graph for calculating gradients. + + Args: + op_type: Operation type + params: Input parameters to the operation + output: Output variable of the operation + """ + operation = Operation(op_type, other_params=other_params) + param_nodes = [] + for param in params: + param.add_param_to(operation) + param_nodes.append(param) + output.add_result_of(operation) + + operation.add_params(param_nodes) + operation.add_output(output) + + def gradient(self, target: Variable, source: Variable) -> np.ndarray | None: + """ + Reverse accumulation of partial derivatives to calculate gradients + of target variable with respect to source variable. + + Args: + target: target variable for which gradients are calculated. + source: source variable with respect to which the gradients are + calculated. + + Returns: + Gradient of the source variable with respect to the target variable + """ + # partial derivatives with respect to target + partial_deriv = {target: np.ones_like(target.numpy())} + + # iterating through each operations in the computation graph + operation_queue = [target.result_of] + while len(operation_queue) > 0: + operation = operation_queue.pop() + for param in operation.params: + # as per the chain rule, multiplying partial derivatives + # of variables with respect to the target + dparam_doutput = self.derivative(param, operation) + dparam_dtarget = dparam_doutput * partial_deriv[operation.output] + if param in partial_deriv: + partial_deriv[param] += dparam_dtarget + else: + partial_deriv[param] = dparam_dtarget + + if param.result_of: + operation_queue.append(param.result_of) + + if source in partial_deriv: + return partial_deriv[source] + return None + + def derivative(self, param: Variable, operation: Operation) -> np.ndarray: + """ + Compute the derivative of given operation/function + + Args: + param: variable to be differentiated + operation: function performed on the input variable + + Returns: + Derivative of input variable with respect to the output of + the operation + """ + params = operation.params + + derivative = None + + if operation == OpType.ADD: + derivative = np.ones_like(params[0].numpy(), dtype=np.float64) + elif operation == OpType.SUB: + if params[0] == param: + derivative = np.ones_like(params[0].numpy(), dtype=np.float64) + else: + derivative = -np.ones_like(params[1].numpy(), dtype=np.float64) + elif operation == OpType.MUL: + derivative = ( + params[1].numpy().T if params[0] == param else params[0].numpy().T + ) + elif operation == OpType.DIV: + if params[0] == param: + derivative = 1 / params[1].numpy() + else: + derivative = -params[0].numpy() / (params[1].numpy() ** 2) + elif operation == OpType.MATMUL: + derivative = ( + params[1].numpy().T if params[0] == param else params[0].numpy().T + ) + elif operation == OpType.POWER: + power = operation.other_params["power"] + derivative = power * (params[0].numpy() ** (power - 1)) + return derivative + + +if __name__ == "__main__": + import doctest + + doctest.testmod() From 86b3397d668e215e3b84ba075d061978e53ecd76 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:22:28 +0530 Subject: [PATCH 03/10] Resolved pre commit errors --- machine_learning/auto-diff.py | 314 ------------------ machine_learning/automatic_differentiation.py | 40 ++- 2 files changed, 24 insertions(+), 330 deletions(-) delete mode 100644 machine_learning/auto-diff.py diff --git a/machine_learning/auto-diff.py b/machine_learning/auto-diff.py deleted file mode 100644 index 6a8a35e0e3c5..000000000000 --- a/machine_learning/auto-diff.py +++ /dev/null @@ -1,314 +0,0 @@ -""" -Demonstration of the Automatic Differentiation (Reverse mode). - -Reference: https://en.wikipedia.org/wiki/Automatic_differentiation - -Author: Poojan Smart -Email: smrtpoojan@gmail.com - -Examples: - ->>> with GradientTracker() as tracker: -... a = Variable([2.0, 5.0]) -... b = Variable([1.0, 2.0]) -... m = Variable([1.0, 2.0]) -... c = a + b -... d = a * b -... e = c / d ->>> print(tracker.gradient(e, a)) -[-0.25 -0.04] ->>> print(tracker.gradient(e, b)) -[-1. -0.25] ->>> print(tracker.gradient(e, m)) -None - ->>> with GradientTracker() as tracker: -... a = Variable([[2.0, 5.0]]) -... b = Variable([[1.0], [2.0]]) -... c = a @ b ->>> print(tracker.gradient(c, a)) -[[1. 2.]] ->>> print(tracker.gradient(c, b)) -[[2.] - [5.]] - ->>> with GradientTracker() as tracker: -... a = Variable([[2.0, 5.0]]) -... b = a ** 3 ->>> print(tracker.gradient(b, a)) -[[12. 75.]] -""" -from __future__ import annotations - -from enum import Enum - -import numpy as np - - -class Variable: - """ - Class represents n-dimensional object which is used to wrap - numpy array on which operations will be performed and gradient - will be calculated. - """ - - def __init__(self, value): - self.value = np.array(value) - - # pointers to the operations to which the Variable is input - self.param_to = [] - # pointer to the operation of which the Variable is output of - self.result_of = None - - def __str__(self) -> str: - return f"Variable({self.value})" - - def numpy(self) -> np.ndarray: - return self.value - - def __add__(self, other: Variable) -> Variable: - result = Variable(self.value + other.value) - - with GradientTracker() as tracker: - # if tracker is enabled, computation graph will be updated - if tracker.enabled: - tracker.add_operation(OpType.ADD, params=[self, other], output=result) - return result - - def __sub__(self, other: Variable) -> Variable: - result = Variable(self.value - other.value) - - with GradientTracker() as tracker: - # if tracker is enabled, computation graph will be updated - if tracker.enabled: - tracker.add_operation(OpType.SUB, params=[self, other], output=result) - return result - - def __mul__(self, other: Variable) -> Variable: - result = Variable(self.value * other.value) - - with GradientTracker() as tracker: - # if tracker is enabled, computation graph will be updated - if tracker.enabled: - tracker.add_operation(OpType.MUL, params=[self, other], output=result) - return result - - def __truediv__(self, other: Variable) -> Variable: - result = Variable(self.value / other.value) - - with GradientTracker() as tracker: - # if tracker is enabled, computation graph will be updated - if tracker.enabled: - tracker.add_operation(OpType.DIV, params=[self, other], output=result) - return result - - def __matmul__(self, other: Variable) -> Variable: - result = Variable(self.value @ other.value) - - with GradientTracker() as tracker: - # if tracker is enabled, computation graph will be updated - if tracker.enabled: - tracker.add_operation( - OpType.MATMUL, params=[self, other], output=result - ) - return result - - def __pow__(self, power: int) -> Variable: - result = Variable(self.value**power) - - with GradientTracker() as tracker: - # if tracker is enabled, computation graph will be updated - if tracker.enabled: - tracker.add_operation( - OpType.POWER, - params=[self], - output=result, - other_params={"power": power}, - ) - return result - - def add_param_to(self, param_to: Operation): - self.param_to.append(param_to) - - def add_result_of(self, result_of: Operation): - self.result_of = result_of - - -class OpType(Enum): - """ - Class represents list of supported operations on Variable for - gradient calculation. - """ - - ADD = 0 - SUB = 1 - MUL = 2 - DIV = 3 - MATMUL = 4 - POWER = 5 - - -class Operation: - """ - Class represents operation between single or two Variable objects. - Operation objects contains type of operation, pointers to input Variable - objects and pointer to resulting Variable from the operation. - """ - - def __init__( - self, - op_type: OpType, - other_params: dict | None = None, - ): - self.op_type = op_type - self.params: list[Variable] = [] - self.output: Variable | None = None - self.other_params = {} if other_params is None else other_params - - def add_params(self, params: list[Variable]): - self.params = params - - def add_output(self, output: Variable): - self.output = output - - def __eq__(self, value) -> bool: - if isinstance(value, OpType): - return self.op_type == value - return False - - -class GradientTracker: - """ - Class contains methods to compute partial derivatives of Variable - based on the computation graph. - """ - - def __new__(cls): - """ - Executes at the creation of class object and returns if - object is already created. This class follows singleton - design pattern. - """ - if not hasattr(cls, "instance"): - cls.instance = super().__new__(cls) - return cls.instance - - def __init__(self): - self.enabled = False - - def __enter__(self): - self.enabled = True - return self - - def __exit__(self, exc_type, exc_value, tb): - self.enabled = False - - def add_operation( - self, - op_type: OpType, - params: list[Variable], - output: Variable, - other_params: dict | None = None, - ): - """ - Adds Operation object to the related Variable objects for - creating computational graph for calculating gradients. - - Args: - op_type: Operation type - params: Input parameters to the operation - output: Output variable of the operation - """ - operation = Operation(op_type, other_params=other_params) - param_nodes = [] - for param in params: - param.add_param_to(operation) - param_nodes.append(param) - output.add_result_of(operation) - - operation.add_params(param_nodes) - operation.add_output(output) - - def gradient(self, target: Variable, source: Variable) -> np.ndarray | None: - """ - Reverse accumulation of partial derivatives to calculate gradients - of target variable with respect to source variable. - - Args: - target: target variable for which gradients are calculated. - source: source variable with respect to which the gradients are - calculated. - - Returns: - Gradient of the source variable with respect to the target variable - """ - # partial derivatives with respect to target - partial_deriv = {target: np.ones_like(target.numpy())} - - # iterating through each operations in the computation graph - operation_queue = [target.result_of] - while len(operation_queue) > 0: - operation = operation_queue.pop() - for param in operation.params: - # as per the chain rule, multiplying partial derivatives - # of variables with respect to the target - dparam_doutput = self.derivative(param, operation) - dparam_dtarget = dparam_doutput * partial_deriv[operation.output] - if param in partial_deriv: - partial_deriv[param] += dparam_dtarget - else: - partial_deriv[param] = dparam_dtarget - - if param.result_of: - operation_queue.append(param.result_of) - - if source in partial_deriv: - return partial_deriv[source] - return None - - def derivative(self, param: Variable, operation: Operation) -> np.ndarray: - """ - Compute the derivative of given operation/function - - Args: - param: variable to be differentiated - operation: function performed on the input variable - - Returns: - Derivative of input variable with respect to the output of - the operation - """ - params = operation.params - - derivative = None - - if operation == OpType.ADD: - derivative = np.ones_like(params[0].numpy(), dtype=np.float64) - elif operation == OpType.SUB: - if params[0] == param: - derivative = np.ones_like(params[0].numpy(), dtype=np.float64) - else: - derivative = -np.ones_like(params[1].numpy(), dtype=np.float64) - elif operation == OpType.MUL: - derivative = ( - params[1].numpy().T if params[0] == param else params[0].numpy().T - ) - elif operation == OpType.DIV: - if params[0] == param: - derivative = 1 / params[1].numpy() - else: - derivative = -params[0].numpy() / (params[1].numpy() ** 2) - elif operation == OpType.MATMUL: - derivative = ( - params[1].numpy().T if params[0] == param else params[0].numpy().T - ) - elif operation == OpType.POWER: - power = operation.other_params["power"] - derivative = power * (params[0].numpy() ** (power - 1)) - return derivative - - -if __name__ == "__main__": - import doctest - - doctest.testmod() diff --git a/machine_learning/automatic_differentiation.py b/machine_learning/automatic_differentiation.py index 6a8a35e0e3c5..e456bd5937f1 100644 --- a/machine_learning/automatic_differentiation.py +++ b/machine_learning/automatic_differentiation.py @@ -41,6 +41,8 @@ from __future__ import annotations from enum import Enum +from types import TracebackType +from typing import Self import numpy as np @@ -52,13 +54,13 @@ class Variable: will be calculated. """ - def __init__(self, value): + def __init__(self, value) -> None: self.value = np.array(value) # pointers to the operations to which the Variable is input - self.param_to = [] + self.param_to: list[Operation] = [] # pointer to the operation of which the Variable is output of - self.result_of = None + self.result_of: Operation = Operation(OpType.NOOP) def __str__(self) -> str: return f"Variable({self.value})" @@ -127,10 +129,10 @@ def __pow__(self, power: int) -> Variable: ) return result - def add_param_to(self, param_to: Operation): + def add_param_to(self, param_to: Operation) -> None: self.param_to.append(param_to) - def add_result_of(self, result_of: Operation): + def add_result_of(self, result_of: Operation) -> None: self.result_of = result_of @@ -146,6 +148,7 @@ class OpType(Enum): DIV = 3 MATMUL = 4 POWER = 5 + NOOP = 6 class Operation: @@ -159,16 +162,14 @@ def __init__( self, op_type: OpType, other_params: dict | None = None, - ): + ) -> None: self.op_type = op_type - self.params: list[Variable] = [] - self.output: Variable | None = None self.other_params = {} if other_params is None else other_params - def add_params(self, params: list[Variable]): + def add_params(self, params: list[Variable]) -> None: self.params = params - def add_output(self, output: Variable): + def add_output(self, output: Variable) -> None: self.output = output def __eq__(self, value) -> bool: @@ -183,7 +184,7 @@ class GradientTracker: based on the computation graph. """ - def __new__(cls): + def __new__(cls) -> Self: """ Executes at the creation of class object and returns if object is already created. This class follows singleton @@ -193,14 +194,19 @@ def __new__(cls): cls.instance = super().__new__(cls) return cls.instance - def __init__(self): + def __init__(self) -> None: self.enabled = False - def __enter__(self): + def __enter__(self) -> Self: self.enabled = True return self - def __exit__(self, exc_type, exc_value, tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: self.enabled = False def add_operation( @@ -209,7 +215,7 @@ def add_operation( params: list[Variable], output: Variable, other_params: dict | None = None, - ): + ) -> None: """ Adds Operation object to the related Variable objects for creating computational graph for calculating gradients. @@ -242,6 +248,7 @@ def gradient(self, target: Variable, source: Variable) -> np.ndarray | None: Returns: Gradient of the source variable with respect to the target variable """ + # partial derivatives with respect to target partial_deriv = {target: np.ones_like(target.numpy())} @@ -260,7 +267,8 @@ def gradient(self, target: Variable, source: Variable) -> np.ndarray | None: partial_deriv[param] = dparam_dtarget if param.result_of: - operation_queue.append(param.result_of) + if param.result_of != OpType.NOOP: + operation_queue.append(param.result_of) if source in partial_deriv: return partial_deriv[source] From d1e7f731c081dd89f6fb99f1224ff1c9cf048888 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:21:28 +0530 Subject: [PATCH 04/10] updated dependency --- machine_learning/automatic_differentiation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/machine_learning/automatic_differentiation.py b/machine_learning/automatic_differentiation.py index e456bd5937f1..0bca29c0dc15 100644 --- a/machine_learning/automatic_differentiation.py +++ b/machine_learning/automatic_differentiation.py @@ -3,7 +3,7 @@ Reference: https://en.wikipedia.org/wiki/Automatic_differentiation -Author: Poojan Smart +Author: Poojan smart Email: smrtpoojan@gmail.com Examples: @@ -42,9 +42,9 @@ from enum import Enum from types import TracebackType -from typing import Self import numpy as np +from typing_extensions import Self class Variable: @@ -184,13 +184,15 @@ class GradientTracker: based on the computation graph. """ + instance = None + def __new__(cls) -> Self: """ Executes at the creation of class object and returns if object is already created. This class follows singleton design pattern. """ - if not hasattr(cls, "instance"): + if cls.instance == None: cls.instance = super().__new__(cls) return cls.instance From 8645334b74e6d7288d364e3821d6ee49e5d428d7 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:27:58 +0530 Subject: [PATCH 05/10] added noqa for ignoring check --- machine_learning/automatic_differentiation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machine_learning/automatic_differentiation.py b/machine_learning/automatic_differentiation.py index 0bca29c0dc15..967edd844ad5 100644 --- a/machine_learning/automatic_differentiation.py +++ b/machine_learning/automatic_differentiation.py @@ -44,7 +44,7 @@ from types import TracebackType import numpy as np -from typing_extensions import Self +from typing_extensions import Self # noqa: UP035 class Variable: @@ -192,7 +192,7 @@ def __new__(cls) -> Self: object is already created. This class follows singleton design pattern. """ - if cls.instance == None: + if cls.instance is None: cls.instance = super().__new__(cls) return cls.instance From ad5b56d98d862035cd54b1a948ff0256a02bdeb2 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:35:15 +0530 Subject: [PATCH 06/10] adding typing_extension for adding Self type in __new__ --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 05d9f1e8c545..5901fc0dc67e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ tensorflow ; python_version < '3.12' tweepy xgboost # yulewalker # uncomment once audio_filters/equal_loudness_filter.py is fixed +typing_extensions \ No newline at end of file From 1997de6367dcb5078cf696b535697e2d4a2546c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 06:05:59 +0000 Subject: [PATCH 07/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5901fc0dc67e..8937f6bb0dae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,6 @@ statsmodels sympy tensorflow ; python_version < '3.12' tweepy -xgboost # yulewalker # uncomment once audio_filters/equal_loudness_filter.py is fixed -typing_extensions \ No newline at end of file +typing_extensions +xgboost From 71234dd6db9c087ec1a065b3d8fa86166bebf5a1 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:39:44 +0530 Subject: [PATCH 08/10] sorted requirement.text dependency --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5901fc0dc67e..d6e333296f55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,6 @@ statsmodels sympy tensorflow ; python_version < '3.12' tweepy -xgboost # yulewalker # uncomment once audio_filters/equal_loudness_filter.py is fixed -typing_extensions \ No newline at end of file +typing_extensions # +xgboost From a3d7418990fab489c6a5856f14ff8588c26dde49 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Oct 2023 02:41:03 +0000 Subject: [PATCH 09/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/automatic_differentiation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine_learning/automatic_differentiation.py b/machine_learning/automatic_differentiation.py index 08756660c1ae..7c02f806a75d 100644 --- a/machine_learning/automatic_differentiation.py +++ b/machine_learning/automatic_differentiation.py @@ -35,7 +35,7 @@ class Variable: """ Class represents n-dimensional object which is used to wrap numpy array on which operations will be performed and the gradient will be calculated. - + Examples: >>> Variable(5.0) Variable(5.0) From 4bea57d94a63f1106bdd599b8284a8f29a43d7e4 Mon Sep 17 00:00:00 2001 From: Poojan Smart <44301271+PoojanSmart@users.noreply.github.com> Date: Fri, 27 Oct 2023 08:18:32 +0530 Subject: [PATCH 10/10] resolved ruff --- machine_learning/automatic_differentiation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/machine_learning/automatic_differentiation.py b/machine_learning/automatic_differentiation.py index 08756660c1ae..cd2e5cdaa782 100644 --- a/machine_learning/automatic_differentiation.py +++ b/machine_learning/automatic_differentiation.py @@ -35,7 +35,7 @@ class Variable: """ Class represents n-dimensional object which is used to wrap numpy array on which operations will be performed and the gradient will be calculated. - + Examples: >>> Variable(5.0) Variable(5.0) @@ -317,7 +317,8 @@ def derivative(self, param: Variable, operation: Operation) -> np.ndarray: power = operation.other_params["power"] return power * (params[0].to_ndarray() ** (power - 1)) - raise ValueError(f"invalid operation type: {operation.op_type}") + err_msg = f"invalid operation type: {operation.op_type}" + raise ValueError(err_msg) if __name__ == "__main__":