From 4270b7c12af042559cb6ddc51e1dc1ab4c4b2684 Mon Sep 17 00:00:00 2001 From: kausthub-kannan Date: Sat, 7 Oct 2023 02:37:33 +0530 Subject: [PATCH 1/4] Added Squareplus Activation Function --- .../activation_functions/squareplus.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 neural_network/activation_functions/squareplus.py diff --git a/neural_network/activation_functions/squareplus.py b/neural_network/activation_functions/squareplus.py new file mode 100644 index 000000000000..c5e8e77e10a7 --- /dev/null +++ b/neural_network/activation_functions/squareplus.py @@ -0,0 +1,37 @@ +""" +Squareplus Activation Function + +Use Case: Squareplus designed to enhance positive values and suppress negative values. +For more detailed information, you can refer to the following link: +https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Squareplus +""" + +import numpy as np + + +def squareplus(vector: np.ndarray) -> np.ndarray: + """ + Implements the SquarePlus activation function. + + Parameters: + vector (np.ndarray): The input array for the SquarePlus activation. + + Returns: + np.ndarray: The input array after applying the SquarePlus activation. + + Formula: f(x) = x^2 if x > 0 else f(x) = 0 + + Examples: + >>> squareplus(np.array([2.3, 0.6, -2, -3.8])) + array([5.29, 0.36, 0. , 0. ]) + + >>> squareplus(np.array([-9.2, -0.3, 0.45, -4.56])) + array([0. , 0. , 0.2025, 0. ]) + """ + return np.where(vector > 0, vector**2, 0) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() From d1b9f6d750ce3f624c179a0f4b0cbcf04e6ca42e Mon Sep 17 00:00:00 2001 From: kausthub-kannan Date: Sat, 7 Oct 2023 02:46:31 +0530 Subject: [PATCH 2/4] Added parameter beta to the function --- neural_network/activation_functions/squareplus.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/neural_network/activation_functions/squareplus.py b/neural_network/activation_functions/squareplus.py index c5e8e77e10a7..ffbd6c85bc6b 100644 --- a/neural_network/activation_functions/squareplus.py +++ b/neural_network/activation_functions/squareplus.py @@ -9,12 +9,13 @@ import numpy as np -def squareplus(vector: np.ndarray) -> np.ndarray: +def squareplus(vector: np.ndarray, beta: float) -> np.ndarray: """ Implements the SquarePlus activation function. Parameters: vector (np.ndarray): The input array for the SquarePlus activation. + beta (float): size of the curved region Returns: np.ndarray: The input array after applying the SquarePlus activation. @@ -22,13 +23,13 @@ def squareplus(vector: np.ndarray) -> np.ndarray: Formula: f(x) = x^2 if x > 0 else f(x) = 0 Examples: - >>> squareplus(np.array([2.3, 0.6, -2, -3.8])) + >>> squareplus(np.array([2.3, 0.6, -2, -3.8]), beta=2) array([5.29, 0.36, 0. , 0. ]) - >>> squareplus(np.array([-9.2, -0.3, 0.45, -4.56])) - array([0. , 0. , 0.2025, 0. ]) + >>> squareplus(np.array([-9.2, -0.3, 0.45, -4.56]), beta=3) + array([0. , 0. , 0.091125, 0. ]) """ - return np.where(vector > 0, vector**2, 0) + return np.where(vector > 0, vector**beta, 0) if __name__ == "__main__": From d9397c353e82e6c60f6aede512dffdae20c0c036 Mon Sep 17 00:00:00 2001 From: kausthub-kannan Date: Sun, 8 Oct 2023 19:13:11 +0530 Subject: [PATCH 3/4] Fixed Squareplus Function --- neural_network/activation_functions/squareplus.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/neural_network/activation_functions/squareplus.py b/neural_network/activation_functions/squareplus.py index ffbd6c85bc6b..b40373166400 100644 --- a/neural_network/activation_functions/squareplus.py +++ b/neural_network/activation_functions/squareplus.py @@ -20,16 +20,16 @@ def squareplus(vector: np.ndarray, beta: float) -> np.ndarray: Returns: np.ndarray: The input array after applying the SquarePlus activation. - Formula: f(x) = x^2 if x > 0 else f(x) = 0 + Formula: f(x) = ( x + (x^2 + b)**0.5 ) / 2 Examples: >>> squareplus(np.array([2.3, 0.6, -2, -3.8]), beta=2) - array([5.29, 0.36, 0. , 0. ]) + array([2.5 , 1.06811457, 0.22474487, 0.12731349]) >>> squareplus(np.array([-9.2, -0.3, 0.45, -4.56]), beta=3) - array([0. , 0. , 0.091125, 0. ]) + array([0.0808119 , 0.72891979, 1.11977651, 0.15893419]) """ - return np.where(vector > 0, vector**beta, 0) + return (vector + np.sqrt(vector**2 + beta)) / 2 if __name__ == "__main__": From e644192e78187b0d3f809b3911442412c1dd354e Mon Sep 17 00:00:00 2001 From: Tianyi Zheng Date: Sun, 8 Oct 2023 12:06:09 -0400 Subject: [PATCH 4/4] Update neural_network/activation_functions/squareplus.py --- neural_network/activation_functions/squareplus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neural_network/activation_functions/squareplus.py b/neural_network/activation_functions/squareplus.py index b40373166400..40fa800d6b4a 100644 --- a/neural_network/activation_functions/squareplus.py +++ b/neural_network/activation_functions/squareplus.py @@ -20,7 +20,7 @@ def squareplus(vector: np.ndarray, beta: float) -> np.ndarray: Returns: np.ndarray: The input array after applying the SquarePlus activation. - Formula: f(x) = ( x + (x^2 + b)**0.5 ) / 2 + Formula: f(x) = ( x + sqrt(x^2 + b) ) / 2 Examples: >>> squareplus(np.array([2.3, 0.6, -2, -3.8]), beta=2)