From 60235f49defb07cf43de929f5cf4b45218c3b69f Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Sun, 23 Apr 2023 17:01:55 +0530 Subject: [PATCH 01/17] tanh function been added --- maths/tanh.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 maths/tanh.py diff --git a/maths/tanh.py b/maths/tanh.py new file mode 100644 index 000000000000..9a42ae31f235 --- /dev/null +++ b/maths/tanh.py @@ -0,0 +1,32 @@ +""" +This script demonstrates the implementation of the tangent hyperbolic or tanh function. + +The function takes a vector of K real numbers as input and then (e^x - e^(-x))/(e^x + e^(-x)). +After through tanh, the element of the vector mostly -1 between 1. + +Script inspired from its corresponding Wikipedia article +https://en.wikipedia.org/wiki/Activation_function +""" +import numpy +import numpy as np + + +def tanh(vector): + ''' + Implements the tanh function + + Parameters: + vector: np.array, list, tuple consisting real values + + Returns: + tanh (np.array): The input numpy array after applying + tanh. + + mathematically (e^x - e^(-x))/(e^x + e^(-x)) can be written as (2/(1+e^(-2x))-1 + ''' + exp_vector = np.exp(-2 * vector) + return (2 / (1 + exp_vector)) - 1 + + +if __name__ == '__main__': + print(tanh(np.array([1, 5, 6, 113, 13, 16, -5.23]))) From dbea59dfbc7033a675956246c7389c81c705b195 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Sun, 23 Apr 2023 17:24:14 +0530 Subject: [PATCH 02/17] tanh function been added --- maths/tanh.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/maths/tanh.py b/maths/tanh.py index 9a42ae31f235..70306c5ee047 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -11,8 +11,8 @@ import numpy as np -def tanh(vector): - ''' +def tanh_func(vector): + """ Implements the tanh function Parameters: @@ -23,10 +23,15 @@ def tanh(vector): tanh. mathematically (e^x - e^(-x))/(e^x + e^(-x)) can be written as (2/(1+e^(-2x))-1 - ''' + + Examples: + >>> tanh_func(np.array([1, 5, 6, 113, 13, 16, -5.23])) + [ 0.76159416 0.9999092 0.99998771 1. 1. 1. -0.99994268] + """ + exp_vector = np.exp(-2 * vector) return (2 / (1 + exp_vector)) - 1 if __name__ == '__main__': - print(tanh(np.array([1, 5, 6, 113, 13, 16, -5.23]))) + print(tanh_func(np.array([0.23]))) From 6e3f28d9d9b5378c6f3aed45f4c95bfc00e1bdcc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 11:58:40 +0000 Subject: [PATCH 03/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- maths/tanh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maths/tanh.py b/maths/tanh.py index 70306c5ee047..93ef6ec462de 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -33,5 +33,5 @@ def tanh_func(vector): return (2 / (1 + exp_vector)) - 1 -if __name__ == '__main__': +if __name__ == "__main__": print(tanh_func(np.array([0.23]))) From d42af7b70f5315589c687219cb0b781db5b51375 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Sun, 23 Apr 2023 17:34:24 +0530 Subject: [PATCH 04/17] tanh function is added --- maths/tanh.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/maths/tanh.py b/maths/tanh.py index 70306c5ee047..a8c1d7cf3dfe 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -9,14 +9,15 @@ """ import numpy import numpy as np +import doctest -def tanh_func(vector): +def tanh_func(vector: np.array) -> np.array: """ Implements the tanh function Parameters: - vector: np.array, list, tuple consisting real values + vector: np.array Returns: tanh (np.array): The input numpy array after applying @@ -26,7 +27,9 @@ def tanh_func(vector): Examples: >>> tanh_func(np.array([1, 5, 6, 113, 13, 16, -5.23])) - [ 0.76159416 0.9999092 0.99998771 1. 1. 1. -0.99994268] + array([ 0.76159416, 0.9999092 , 0.99998771, 1. , 1. , + 1. , -0.99994268]) + """ exp_vector = np.exp(-2 * vector) @@ -34,4 +37,5 @@ def tanh_func(vector): if __name__ == '__main__': - print(tanh_func(np.array([0.23]))) + doctest.testmod() + From fa675a47fda6af87e6f2e03a18b8d84df12e27a0 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Sun, 23 Apr 2023 17:42:22 +0530 Subject: [PATCH 05/17] tanh function is added --- maths/tanh.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maths/tanh.py b/maths/tanh.py index a8c1d7cf3dfe..f07cf2d402e0 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -38,4 +38,3 @@ def tanh_func(vector: np.array) -> np.array: if __name__ == '__main__': doctest.testmod() - From a6b5d0547e4be8337f1ddabc090a60a2c213fa4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 12:20:11 +0000 Subject: [PATCH 06/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- maths/tanh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maths/tanh.py b/maths/tanh.py index f07cf2d402e0..2ce383e53642 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -36,5 +36,5 @@ def tanh_func(vector: np.array) -> np.array: return (2 / (1 + exp_vector)) - 1 -if __name__ == '__main__': +if __name__ == "__main__": doctest.testmod() From 2827b159d166306a68823ec7fadfea77b0f11e36 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Sun, 23 Apr 2023 18:32:08 +0530 Subject: [PATCH 07/17] tanh function added --- maths/tanh.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/maths/tanh.py b/maths/tanh.py index f07cf2d402e0..cd28a59903fd 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -1,18 +1,18 @@ """ -This script demonstrates the implementation of the tangent hyperbolic or tanh function. +This script demonstrates the implementation of the tangent hyperbolic +or tanh function. -The function takes a vector of K real numbers as input and then (e^x - e^(-x))/(e^x + e^(-x)). -After through tanh, the element of the vector mostly -1 between 1. +The function takes a vector of K real numbers as input and +then (e^x - e^(-x))/(e^x + e^(-x)). After through tanh, the +element of the vector mostly -1 between 1. Script inspired from its corresponding Wikipedia article https://en.wikipedia.org/wiki/Activation_function """ -import numpy import numpy as np -import doctest -def tanh_func(vector: np.array) -> np.array: +def tangent_hyperbolic(vector: np.array) -> np.array: """ Implements the tanh function @@ -26,7 +26,7 @@ def tanh_func(vector: np.array) -> np.array: mathematically (e^x - e^(-x))/(e^x + e^(-x)) can be written as (2/(1+e^(-2x))-1 Examples: - >>> tanh_func(np.array([1, 5, 6, 113, 13, 16, -5.23])) + >>> tangent_hyperbolic(np.array([1,5,6,113,13,16,-5.23])) array([ 0.76159416, 0.9999092 , 0.99998771, 1. , 1. , 1. , -0.99994268]) @@ -36,5 +36,7 @@ def tanh_func(vector: np.array) -> np.array: return (2 / (1 + exp_vector)) - 1 -if __name__ == '__main__': - doctest.testmod() +if __name__ == "__main__": + import doctest + + doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE) From 9114b527a1b530323a80cc33106c15970b968d5e Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Sun, 23 Apr 2023 19:17:44 +0530 Subject: [PATCH 08/17] tanh function added --- maths/tanh.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/maths/tanh.py b/maths/tanh.py index cd28a59903fd..7330f378373e 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -26,9 +26,8 @@ def tangent_hyperbolic(vector: np.array) -> np.array: mathematically (e^x - e^(-x))/(e^x + e^(-x)) can be written as (2/(1+e^(-2x))-1 Examples: - >>> tangent_hyperbolic(np.array([1,5,6,113,13,16,-5.23])) - array([ 0.76159416, 0.9999092 , 0.99998771, 1. , 1. , - 1. , -0.99994268]) + >>> tangent_hyperbolic(np.array([1,5,6,-0.67])) + array([ 0.76159416, 0.9999092 , 0.99998771, -0.58497988]) """ @@ -39,4 +38,4 @@ def tangent_hyperbolic(vector: np.array) -> np.array: if __name__ == "__main__": import doctest - doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE) + doctest.testmod() From a4ca89775c25680d5e802aa8c3803e583cde4791 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Mon, 24 Apr 2023 20:30:04 +0530 Subject: [PATCH 09/17] tanh function is added --- maths/tanh.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/maths/tanh.py b/maths/tanh.py index 7330f378373e..d6233da639ec 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -29,6 +29,9 @@ def tangent_hyperbolic(vector: np.array) -> np.array: >>> tangent_hyperbolic(np.array([1,5,6,-0.67])) array([ 0.76159416, 0.9999092 , 0.99998771, -0.58497988]) + >>> tangent_hyperbolic(np.array([8,10,2,-0.98,13])) + array([ 0.99999977, 1. , 0.96402758, -0.7530659 , 1. ]) + """ exp_vector = np.exp(-2 * vector) From e0495023d63bef348bb2774afbf3b54418d55da8 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 25 Apr 2023 18:01:07 +0200 Subject: [PATCH 10/17] Apply suggestions from code review --- maths/tanh.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/maths/tanh.py b/maths/tanh.py index d6233da639ec..ddab3e1ab717 100644 --- a/maths/tanh.py +++ b/maths/tanh.py @@ -20,8 +20,7 @@ def tangent_hyperbolic(vector: np.array) -> np.array: vector: np.array Returns: - tanh (np.array): The input numpy array after applying - tanh. + tanh (np.array): The input numpy array after applying tanh. mathematically (e^x - e^(-x))/(e^x + e^(-x)) can be written as (2/(1+e^(-2x))-1 @@ -34,8 +33,7 @@ def tangent_hyperbolic(vector: np.array) -> np.array: """ - exp_vector = np.exp(-2 * vector) - return (2 / (1 + exp_vector)) - 1 + return (2 / (1 + np.exp(-2 * vector))) - 1 if __name__ == "__main__": From 66cc81a30592a5734d9d3d767320d014df933926 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Thu, 27 Apr 2023 12:00:39 +0530 Subject: [PATCH 11/17] ELU activation function is added --- maths/elu_activation.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 maths/elu_activation.py diff --git a/maths/elu_activation.py b/maths/elu_activation.py new file mode 100644 index 000000000000..441c3cf4ee43 --- /dev/null +++ b/maths/elu_activation.py @@ -0,0 +1,39 @@ +""" +Implements the Exponential Linear Unit or ELU function. + +The function takes a vector of K real numbers and a real number alpha as +input and then applies the ELU function to each element of the vector. + +Script inspired from its corresponding Wikipedia article +https://en.wikipedia.org/wiki/Rectifier_(neural_networks) +""" + +import numpy as np + + +def elu_activation(vector: np.array, alpha: float) -> np.array: + """ + Implements the ELU activation function. + Parameters: + vector: np.array + alpha: float + return: + elu (np.array): The input numpy array after applying elu. + + Mathematically, f(x) = x, x>0 else (alpha * (e^x -1)), x<=0, alpha >=0 + + Examples: + >>> elu_activation(vector=np.array([2.3,0.6,-2,-3.8,9]), alpha=0.3) + array([ 2.3 , 0.6 , -0.25939942, -0.29328877, 9. ]) + + >>> elu_activation(vector=np.array([-9.2,-0.3,-2.45,0.45]), alpha=0.067) + array([-0.06699323, -0.01736518, -0.06121833, 0.45 ]) + + """ + return np.where(vector > 0, vector, (alpha * (np.exp(vector) - 1))) + + +if __name__ == '__main__': + import doctest + + doctest.testmod() From d40c74b379098b9d3d280b03c6af2338abbc079b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 27 Apr 2023 06:35:23 +0000 Subject: [PATCH 12/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- maths/elu_activation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maths/elu_activation.py b/maths/elu_activation.py index 441c3cf4ee43..cc1631551b54 100644 --- a/maths/elu_activation.py +++ b/maths/elu_activation.py @@ -33,7 +33,7 @@ def elu_activation(vector: np.array, alpha: float) -> np.array: return np.where(vector > 0, vector, (alpha * (np.exp(vector) - 1))) -if __name__ == '__main__': +if __name__ == "__main__": import doctest doctest.testmod() From 5453434e967e03ba54a1537d06099683ddddb35f Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Fri, 28 Apr 2023 09:57:32 +0530 Subject: [PATCH 13/17] elu activation is added --- maths/elu_activation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maths/elu_activation.py b/maths/elu_activation.py index cc1631551b54..2f5ded173f0a 100644 --- a/maths/elu_activation.py +++ b/maths/elu_activation.py @@ -9,9 +9,10 @@ """ import numpy as np +from typing import TYPE_CHECKING -def elu_activation(vector: np.array, alpha: float) -> np.array: +def elu_activation(vector: np.ndarray, alpha: float) -> np.ndarray: """ Implements the ELU activation function. Parameters: From 0d1546eb757d438ab88b386eb64779ff7e897ad2 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Fri, 28 Apr 2023 10:03:31 +0530 Subject: [PATCH 14/17] ELU activation is added --- maths/elu_activation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maths/elu_activation.py b/maths/elu_activation.py index 2f5ded173f0a..d4b5e71b9368 100644 --- a/maths/elu_activation.py +++ b/maths/elu_activation.py @@ -9,7 +9,6 @@ """ import numpy as np -from typing import TYPE_CHECKING def elu_activation(vector: np.ndarray, alpha: float) -> np.ndarray: From 112611aad41ca676bfabfc0019e21ecd3769ccdd Mon Sep 17 00:00:00 2001 From: Dipankar Mitra <50228537+Mitra-babu@users.noreply.github.com> Date: Fri, 28 Apr 2023 19:37:26 +0530 Subject: [PATCH 15/17] Update maths/elu_activation.py Co-authored-by: Christian Clauss --- maths/elu_activation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maths/elu_activation.py b/maths/elu_activation.py index d4b5e71b9368..4e9b96f59bec 100644 --- a/maths/elu_activation.py +++ b/maths/elu_activation.py @@ -11,7 +11,7 @@ import numpy as np -def elu_activation(vector: np.ndarray, alpha: float) -> np.ndarray: +def exponential_linear_unit(vector: np.ndarray, alpha: float) -> np.ndarray: """ Implements the ELU activation function. Parameters: From dad8cf18b661d7b7d0e84b3dc623960c75b6628b Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Fri, 28 Apr 2023 19:50:26 +0530 Subject: [PATCH 16/17] Exponential_linear_unit activation is added --- .../activation_functions/exponential_linear_unit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename maths/elu_activation.py => neural_network/activation_functions/exponential_linear_unit.py (77%) diff --git a/maths/elu_activation.py b/neural_network/activation_functions/exponential_linear_unit.py similarity index 77% rename from maths/elu_activation.py rename to neural_network/activation_functions/exponential_linear_unit.py index 4e9b96f59bec..71ba07cfe773 100644 --- a/maths/elu_activation.py +++ b/neural_network/activation_functions/exponential_linear_unit.py @@ -15,18 +15,18 @@ def exponential_linear_unit(vector: np.ndarray, alpha: float) -> np.ndarray: """ Implements the ELU activation function. Parameters: - vector: np.array - alpha: float + vector: the array containing input of elu activation + alpha: hyper-parameter return: elu (np.array): The input numpy array after applying elu. Mathematically, f(x) = x, x>0 else (alpha * (e^x -1)), x<=0, alpha >=0 Examples: - >>> elu_activation(vector=np.array([2.3,0.6,-2,-3.8,9]), alpha=0.3) + >>> exponential_linear_unit(vector=np.array([2.3,0.6,-2,-3.8,9]), alpha=0.3) array([ 2.3 , 0.6 , -0.25939942, -0.29328877, 9. ]) - >>> elu_activation(vector=np.array([-9.2,-0.3,-2.45,0.45]), alpha=0.067) + >>> exponential_linear_unit(vector=np.array([-9.2,-0.3,-2.45,0.45]), alpha=0.067) array([-0.06699323, -0.01736518, -0.06121833, 0.45 ]) """ From 34be18cadf2193302c2d3bf4c7440f8fa3dbcbd6 Mon Sep 17 00:00:00 2001 From: Mitra-babu Date: Fri, 28 Apr 2023 19:59:28 +0530 Subject: [PATCH 17/17] Exponential_linear_unit activation is added --- .../activation_functions/exponential_linear_unit.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/neural_network/activation_functions/exponential_linear_unit.py b/neural_network/activation_functions/exponential_linear_unit.py index 71ba07cfe773..7a3cf1d84e71 100644 --- a/neural_network/activation_functions/exponential_linear_unit.py +++ b/neural_network/activation_functions/exponential_linear_unit.py @@ -23,11 +23,12 @@ def exponential_linear_unit(vector: np.ndarray, alpha: float) -> np.ndarray: Mathematically, f(x) = x, x>0 else (alpha * (e^x -1)), x<=0, alpha >=0 Examples: - >>> exponential_linear_unit(vector=np.array([2.3,0.6,-2,-3.8,9]), alpha=0.3) - array([ 2.3 , 0.6 , -0.25939942, -0.29328877, 9. ]) + >>> exponential_linear_unit(vector=np.array([2.3,0.6,-2,-3.8]), alpha=0.3) + array([ 2.3 , 0.6 , -0.25939942, -0.29328877]) + + >>> exponential_linear_unit(vector=np.array([-9.2,-0.3,0.45,-4.56]), alpha=0.067) + array([-0.06699323, -0.01736518, 0.45 , -0.06629904]) - >>> exponential_linear_unit(vector=np.array([-9.2,-0.3,-2.45,0.45]), alpha=0.067) - array([-0.06699323, -0.01736518, -0.06121833, 0.45 ]) """ return np.where(vector > 0, vector, (alpha * (np.exp(vector) - 1)))