From e77c9515d7027fba7ee8eaade03e61a5801c0e65 Mon Sep 17 00:00:00 2001 From: paramthakkar123 Date: Sat, 9 Dec 2023 14:44:30 +0530 Subject: [PATCH 1/7] added GELU activation functions file --- .../gaussian_error_linear_unit.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 neural_network/activation_functions/gaussian_error_linear_unit.py diff --git a/neural_network/activation_functions/gaussian_error_linear_unit.py b/neural_network/activation_functions/gaussian_error_linear_unit.py new file mode 100644 index 000000000000..d7257ca2e315 --- /dev/null +++ b/neural_network/activation_functions/gaussian_error_linear_unit.py @@ -0,0 +1,34 @@ +""" +Gaussian Error Linear Unit (GELU) + +Use Case: GELU allows small negative values when the input is less than zero, providing a richer gradient for backpropagation +For more detailed information, you can refer to the following link: +https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Gaussian-error_linear_unit_(GELU) +""" + +from scipy.special import erf +import numpy as np + +def gaussian_error_linear_unit(vector: np.ndarray)->np.ndarray: + """ + Implements the GELU activation function. + Parameters: + vector: the array containing input of gelu activation + return: + gelu (np.array): The input numpy array after applying gelu. + + Examples: + >>> gaussian_error_linear_unit(vector=np.array([-1.1, 2.3, -3.4, 4.5])) + array([-1.49232667e-01, 2.30000000e+00, -1.14555950e-03, 4.50000000e+00]) + + >>> gaussian_error_linear_unit(vector=np.array([-9.2,-0.3,0.45,-4.56])) + array([-0.00000000e+00, -1.14626573e-01, 4.50000000e-01, -1.16630255e-05]) + + + """ + cdf = 0.5 * (1.0 + erf(vector / np.sqrt(2.0))) + return np.where(vector > 0, vector, (vector * cdf)) + +if __name__ == "__main__": + import doctest + doctest.testmod() \ No newline at end of file From 1864989723923939193eae228ab771c8a6a2e294 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 09:17:00 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../activation_functions/gaussian_error_linear_unit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/neural_network/activation_functions/gaussian_error_linear_unit.py b/neural_network/activation_functions/gaussian_error_linear_unit.py index d7257ca2e315..11998cc0bec9 100644 --- a/neural_network/activation_functions/gaussian_error_linear_unit.py +++ b/neural_network/activation_functions/gaussian_error_linear_unit.py @@ -9,7 +9,8 @@ from scipy.special import erf import numpy as np -def gaussian_error_linear_unit(vector: np.ndarray)->np.ndarray: + +def gaussian_error_linear_unit(vector: np.ndarray) -> np.ndarray: """ Implements the GELU activation function. Parameters: @@ -29,6 +30,8 @@ def gaussian_error_linear_unit(vector: np.ndarray)->np.ndarray: cdf = 0.5 * (1.0 + erf(vector / np.sqrt(2.0))) return np.where(vector > 0, vector, (vector * cdf)) + if __name__ == "__main__": import doctest - doctest.testmod() \ No newline at end of file + + doctest.testmod() From 1d1e480ba3f5ce8aa6d85c4c4c796946acb880ac Mon Sep 17 00:00:00 2001 From: Param Thakkar <128291516+ParamThakkar123@users.noreply.github.com> Date: Sat, 9 Dec 2023 14:56:09 +0530 Subject: [PATCH 3/7] Update gaussian_error_linear_unit.py --- .../activation_functions/gaussian_error_linear_unit.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/neural_network/activation_functions/gaussian_error_linear_unit.py b/neural_network/activation_functions/gaussian_error_linear_unit.py index 11998cc0bec9..64c6883cf725 100644 --- a/neural_network/activation_functions/gaussian_error_linear_unit.py +++ b/neural_network/activation_functions/gaussian_error_linear_unit.py @@ -6,9 +6,8 @@ https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Gaussian-error_linear_unit_(GELU) """ -from scipy.special import erf import numpy as np - +from scipy.special import erf def gaussian_error_linear_unit(vector: np.ndarray) -> np.ndarray: """ @@ -27,8 +26,7 @@ def gaussian_error_linear_unit(vector: np.ndarray) -> np.ndarray: """ - cdf = 0.5 * (1.0 + erf(vector / np.sqrt(2.0))) - return np.where(vector > 0, vector, (vector * cdf)) + return np.where(vector > 0, vector, (vector * 0.5 * (1.0 + erf(vector / np.sqrt(2.0))))) if __name__ == "__main__": From 78c25c41a1cd0ee620a235c0d749432a18ffc22e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 09:26:39 +0000 Subject: [PATCH 4/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../activation_functions/gaussian_error_linear_unit.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/neural_network/activation_functions/gaussian_error_linear_unit.py b/neural_network/activation_functions/gaussian_error_linear_unit.py index 64c6883cf725..0f2661fe142e 100644 --- a/neural_network/activation_functions/gaussian_error_linear_unit.py +++ b/neural_network/activation_functions/gaussian_error_linear_unit.py @@ -9,6 +9,7 @@ import numpy as np from scipy.special import erf + def gaussian_error_linear_unit(vector: np.ndarray) -> np.ndarray: """ Implements the GELU activation function. @@ -26,7 +27,9 @@ def gaussian_error_linear_unit(vector: np.ndarray) -> np.ndarray: """ - return np.where(vector > 0, vector, (vector * 0.5 * (1.0 + erf(vector / np.sqrt(2.0))))) + return np.where( + vector > 0, vector, (vector * 0.5 * (1.0 + erf(vector / np.sqrt(2.0)))) + ) if __name__ == "__main__": From 4e680ff904c0b189a10a62f40c6d8a99e825f284 Mon Sep 17 00:00:00 2001 From: Param Thakkar <128291516+ParamThakkar123@users.noreply.github.com> Date: Sat, 9 Dec 2023 15:01:31 +0530 Subject: [PATCH 5/7] Update gaussian_error_linear_unit.py --- .../activation_functions/gaussian_error_linear_unit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neural_network/activation_functions/gaussian_error_linear_unit.py b/neural_network/activation_functions/gaussian_error_linear_unit.py index 0f2661fe142e..ced582972990 100644 --- a/neural_network/activation_functions/gaussian_error_linear_unit.py +++ b/neural_network/activation_functions/gaussian_error_linear_unit.py @@ -1,7 +1,7 @@ """ Gaussian Error Linear Unit (GELU) -Use Case: GELU allows small negative values when the input is less than zero, providing a richer gradient for backpropagation +Use Case: GELU permits negative inputs for enhanced backprop gradients. For more detailed information, you can refer to the following link: https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Gaussian-error_linear_unit_(GELU) """ From 0bd63dcb1be45975a001715beb8d388c467e4abd Mon Sep 17 00:00:00 2001 From: Param Thakkar <128291516+ParamThakkar123@users.noreply.github.com> Date: Wed, 20 Dec 2023 09:34:26 +0530 Subject: [PATCH 6/7] Delete neural_network/activation_functions/gaussian_error_linear_unit.py --- .../gaussian_error_linear_unit.py | 38 ------------------- 1 file changed, 38 deletions(-) delete mode 100644 neural_network/activation_functions/gaussian_error_linear_unit.py diff --git a/neural_network/activation_functions/gaussian_error_linear_unit.py b/neural_network/activation_functions/gaussian_error_linear_unit.py deleted file mode 100644 index ced582972990..000000000000 --- a/neural_network/activation_functions/gaussian_error_linear_unit.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Gaussian Error Linear Unit (GELU) - -Use Case: GELU permits negative inputs for enhanced backprop gradients. -For more detailed information, you can refer to the following link: -https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Gaussian-error_linear_unit_(GELU) -""" - -import numpy as np -from scipy.special import erf - - -def gaussian_error_linear_unit(vector: np.ndarray) -> np.ndarray: - """ - Implements the GELU activation function. - Parameters: - vector: the array containing input of gelu activation - return: - gelu (np.array): The input numpy array after applying gelu. - - Examples: - >>> gaussian_error_linear_unit(vector=np.array([-1.1, 2.3, -3.4, 4.5])) - array([-1.49232667e-01, 2.30000000e+00, -1.14555950e-03, 4.50000000e+00]) - - >>> gaussian_error_linear_unit(vector=np.array([-9.2,-0.3,0.45,-4.56])) - array([-0.00000000e+00, -1.14626573e-01, 4.50000000e-01, -1.16630255e-05]) - - - """ - return np.where( - vector > 0, vector, (vector * 0.5 * (1.0 + erf(vector / np.sqrt(2.0)))) - ) - - -if __name__ == "__main__": - import doctest - - doctest.testmod() From a327e628164bfa52864353103e9151827c187882 Mon Sep 17 00:00:00 2001 From: Param Thakkar <128291516+ParamThakkar123@users.noreply.github.com> Date: Wed, 20 Dec 2023 09:35:21 +0530 Subject: [PATCH 7/7] Rename maths/gaussian_error_linear_unit.py to neural_network/activation_functions/gaussian_error_linear_unit.py --- .../activation_functions}/gaussian_error_linear_unit.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {maths => neural_network/activation_functions}/gaussian_error_linear_unit.py (100%) diff --git a/maths/gaussian_error_linear_unit.py b/neural_network/activation_functions/gaussian_error_linear_unit.py similarity index 100% rename from maths/gaussian_error_linear_unit.py rename to neural_network/activation_functions/gaussian_error_linear_unit.py