From f523758f8285d1af8d2bda975c98cd9fa01cbfca Mon Sep 17 00:00:00 2001 From: Shivansh Bhatnagar Date: Sat, 14 Oct 2023 11:53:15 +0530 Subject: [PATCH 1/3] Added A General Swish Activation Function inNeural Networks --- .../swish_activation_function.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 neural_network/activation_functions/swish_activation_function.py diff --git a/neural_network/activation_functions/swish_activation_function.py b/neural_network/activation_functions/swish_activation_function.py new file mode 100644 index 000000000000..56b578209140 --- /dev/null +++ b/neural_network/activation_functions/swish_activation_function.py @@ -0,0 +1,52 @@ +""" +This script demonstrates the implementation of the General Swish Functions +* https://en.wikipedia.org/wiki/Swish_function + +The function takes a vector x of K real numbers as input and returns x * sigmoid(bx). +Swish is a smooth, non-monotonic function defined as f(x) = x * sigmoid(bx). +Extensive experiments shows that Swish consistently matches or outperforms ReLU +on deep networks applied to a variety of challenging domains such as +image classification and machine translation. + +This script is inspired by a corresponding research paper. +* https://blog.paperspace.com/swish-activation-function/ +""" + +import numpy as np + + +def sigmoid(vector: np.ndarray) -> np.ndarray: + """ + Mathematical function sigmoid takes a vector x of K real numbers as input and + returns 1/ (1 + e^-x). + https://en.wikipedia.org/wiki/Sigmoid_function + + >>> sigmoid(np.array([-1.0, 1.0, 2.0])) + array([0.26894142, 0.73105858, 0.88079708]) + """ + return 1 / (1 + np.exp(-vector)) + + +def general_swish(vector: np.ndarray, trainable_parameter: int) -> np.ndarray: + """ + Parameters: + vector (np.ndarray): A numpy array consisting of real values + trainable_parameter: Use to implement various Swish Activation Functions + + Returns: + swish_vec (np.ndarray): The input numpy array, after applying swish + + Examples: + >>> general_swish(np.array([-1.0, 1.0, 2.0]), 2) + array([-0.11920292, 0.88079708, 1.96402758]) + + >>> general_swish(np.array([-2]), 1) + array([-0.23840584]) + """ + return vector * sigmoid(trainable_parameter * vector) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() From 71fb132e023784f3ee1a9b7f9ce2bebd513bdd98 Mon Sep 17 00:00:00 2001 From: Shivansh Bhatnagar Date: Tue, 17 Oct 2023 22:58:41 +0530 Subject: [PATCH 2/3] Added the general swish function in the SiLU function and renamed it as swish.py --- .../{sigmoid_linear_unit.py => swish.py} | 21 ++++++++ .../swish_activation_function.py | 52 ------------------- 2 files changed, 21 insertions(+), 52 deletions(-) rename neural_network/activation_functions/{sigmoid_linear_unit.py => swish.py} (72%) delete mode 100644 neural_network/activation_functions/swish_activation_function.py diff --git a/neural_network/activation_functions/sigmoid_linear_unit.py b/neural_network/activation_functions/swish.py similarity index 72% rename from neural_network/activation_functions/sigmoid_linear_unit.py rename to neural_network/activation_functions/swish.py index 0ee09bf82d38..1a15abf8a86a 100644 --- a/neural_network/activation_functions/sigmoid_linear_unit.py +++ b/neural_network/activation_functions/swish.py @@ -12,6 +12,7 @@ This script is inspired by a corresponding research paper. * https://arxiv.org/abs/1710.05941 +* https://blog.paperspace.com/swish-activation-function/ """ import numpy as np @@ -49,6 +50,26 @@ def sigmoid_linear_unit(vector: np.ndarray) -> np.ndarray: return vector * sigmoid(vector) + +def swish(vector: np.ndarray, trainable_parameter: int) -> np.ndarray: + """ + Parameters: + vector (np.ndarray): A numpy array consisting of real values + trainable_parameter: Use to implement various Swish Activation Functions + + Returns: + swish_vec (np.ndarray): The input numpy array, after applying swish + + Examples: + >>> swish(np.array([-1.0, 1.0, 2.0]), 2) + array([-0.11920292, 0.88079708, 1.96402758]) + + >>> swish(np.array([-2]), 1) + array([-0.23840584]) + """ + return vector * sigmoid(trainable_parameter * vector) + + if __name__ == "__main__": import doctest diff --git a/neural_network/activation_functions/swish_activation_function.py b/neural_network/activation_functions/swish_activation_function.py deleted file mode 100644 index 56b578209140..000000000000 --- a/neural_network/activation_functions/swish_activation_function.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -This script demonstrates the implementation of the General Swish Functions -* https://en.wikipedia.org/wiki/Swish_function - -The function takes a vector x of K real numbers as input and returns x * sigmoid(bx). -Swish is a smooth, non-monotonic function defined as f(x) = x * sigmoid(bx). -Extensive experiments shows that Swish consistently matches or outperforms ReLU -on deep networks applied to a variety of challenging domains such as -image classification and machine translation. - -This script is inspired by a corresponding research paper. -* https://blog.paperspace.com/swish-activation-function/ -""" - -import numpy as np - - -def sigmoid(vector: np.ndarray) -> np.ndarray: - """ - Mathematical function sigmoid takes a vector x of K real numbers as input and - returns 1/ (1 + e^-x). - https://en.wikipedia.org/wiki/Sigmoid_function - - >>> sigmoid(np.array([-1.0, 1.0, 2.0])) - array([0.26894142, 0.73105858, 0.88079708]) - """ - return 1 / (1 + np.exp(-vector)) - - -def general_swish(vector: np.ndarray, trainable_parameter: int) -> np.ndarray: - """ - Parameters: - vector (np.ndarray): A numpy array consisting of real values - trainable_parameter: Use to implement various Swish Activation Functions - - Returns: - swish_vec (np.ndarray): The input numpy array, after applying swish - - Examples: - >>> general_swish(np.array([-1.0, 1.0, 2.0]), 2) - array([-0.11920292, 0.88079708, 1.96402758]) - - >>> general_swish(np.array([-2]), 1) - array([-0.23840584]) - """ - return vector * sigmoid(trainable_parameter * vector) - - -if __name__ == "__main__": - import doctest - - doctest.testmod() From 5a5f2792f807ebd92aaf529922936130c352cc9b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 17:30:09 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- neural_network/activation_functions/swish.py | 1 - 1 file changed, 1 deletion(-) diff --git a/neural_network/activation_functions/swish.py b/neural_network/activation_functions/swish.py index 1a15abf8a86a..ab3d8fa1203b 100644 --- a/neural_network/activation_functions/swish.py +++ b/neural_network/activation_functions/swish.py @@ -50,7 +50,6 @@ def sigmoid_linear_unit(vector: np.ndarray) -> np.ndarray: return vector * sigmoid(vector) - def swish(vector: np.ndarray, trainable_parameter: int) -> np.ndarray: """ Parameters: