From 5365253dd071ab1eb2fcf96c2d4de54b7f1e28ca Mon Sep 17 00:00:00 2001 From: Kuldeep Borkar Jr Date: Mon, 17 Oct 2022 21:15:53 +0530 Subject: [PATCH 1/4] Implemented Gelu Function --- maths/gelu.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 maths/gelu.py diff --git a/maths/gelu.py b/maths/gelu.py new file mode 100644 index 000000000000..f1792ccb96a9 --- /dev/null +++ b/maths/gelu.py @@ -0,0 +1,49 @@ +""" +This script demonstrates the implementation of the Gelu function. + +The function takes a vector x of K real numbers as input and +resukts x * sigmoid(1.702*x). +The Gaussian Error Linear Unit (GELU), is a high-performing neural +network activation function. + +Script inspired from its corresponding research paper +https://arxiv.org/abs/1606.08415 +""" + +import numpy as np + + +def sigmoid(vector: np.array) -> np.array: + """ + Gelu function can be implemented easily with the help of + sigmoid function + """ + return 1 / (1 + np.exp(-vector)) + + +def gelu(vector: np.array) -> np.array: + """ + Implements the Gelu function + + Parameters: + vector (np.array): A numpy array of shape (1,n) + consisting of real values + + Returns: + gelu_vec (np.array): The input numpy array, after applying + gelu. + + Examples: + >>> gelu(np.array([-1.0, 1.0, 2.0])) + array([-0.15420423, 0.84579577, 1.93565862]) + + >>> gelu(np.array([-3])) + array([-0.01807131]) + """ + return vector * sigmoid(1.702 * vector) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() From ee8fb3280e9f20159a02d6158a5ea65cea575529 Mon Sep 17 00:00:00 2001 From: Kuldeep Borkar Jr Date: Wed, 19 Oct 2022 22:13:04 +0530 Subject: [PATCH 2/4] Renamed file and added more description to function --- ...{gelu.py => gaussian_error_linear_unit.py} | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) rename maths/{gelu.py => gaussian_error_linear_unit.py} (56%) diff --git a/maths/gelu.py b/maths/gaussian_error_linear_unit.py similarity index 56% rename from maths/gelu.py rename to maths/gaussian_error_linear_unit.py index f1792ccb96a9..d47bbb7aef06 100644 --- a/maths/gelu.py +++ b/maths/gaussian_error_linear_unit.py @@ -1,8 +1,10 @@ """ -This script demonstrates the implementation of the Gelu function. +This script demonstrates an implementation of the +Gaussian Error Linear Unit function. +https://en.wikipedia.org/wiki/Activation_function#Comparison_of_activation_functions The function takes a vector x of K real numbers as input and -resukts x * sigmoid(1.702*x). +returns x * sigmoid(1.702*x). The Gaussian Error Linear Unit (GELU), is a high-performing neural network activation function. @@ -15,13 +17,17 @@ def sigmoid(vector: np.array) -> np.array: """ - Gelu function can be implemented easily with the help of - sigmoid function + Mathematical function sigmoid takes a vector x of K real numbers + as input and returns 1/ (1 + e^-x). + https://en.wikipedia.org/wiki/Sigmoid_function + + >>> sigmoid(np.array([-1.0, 1.0, 2.0])) + array([0.26894142, 0.73105858, 0.88079708]) """ return 1 / (1 + np.exp(-vector)) -def gelu(vector: np.array) -> np.array: +def gaussian_error_linear_unit(vector: np.array) -> np.array: """ Implements the Gelu function @@ -34,10 +40,10 @@ def gelu(vector: np.array) -> np.array: gelu. Examples: - >>> gelu(np.array([-1.0, 1.0, 2.0])) + >>> gaussian_error_linear_unit(np.array([-1.0, 1.0, 2.0])) array([-0.15420423, 0.84579577, 1.93565862]) - >>> gelu(np.array([-3])) + >>> gaussian_error_linear_unit(np.array([-3])) array([-0.01807131]) """ return vector * sigmoid(1.702 * vector) From f4223bf0f7b352697fb83f625122ed6efe009b4e Mon Sep 17 00:00:00 2001 From: Kuldeep Borkar Jr Date: Wed, 19 Oct 2022 22:19:57 +0530 Subject: [PATCH 3/4] Extended the name GELU --- maths/gaussian_error_linear_unit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maths/gaussian_error_linear_unit.py b/maths/gaussian_error_linear_unit.py index d47bbb7aef06..8807206a9d1b 100644 --- a/maths/gaussian_error_linear_unit.py +++ b/maths/gaussian_error_linear_unit.py @@ -29,7 +29,7 @@ def sigmoid(vector: np.array) -> np.array: def gaussian_error_linear_unit(vector: np.array) -> np.array: """ - Implements the Gelu function + Implements the Gaussian Error Linear Unit (GELU) function Parameters: vector (np.array): A numpy array of shape (1,n) From 05065ccac2c694fcd74675202796090b8b9f5a98 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 19 Oct 2022 19:15:26 +0200 Subject: [PATCH 4/4] Update gaussian_error_linear_unit.py --- maths/gaussian_error_linear_unit.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/maths/gaussian_error_linear_unit.py b/maths/gaussian_error_linear_unit.py index 8807206a9d1b..7b5f875143b9 100644 --- a/maths/gaussian_error_linear_unit.py +++ b/maths/gaussian_error_linear_unit.py @@ -1,15 +1,13 @@ """ -This script demonstrates an implementation of the -Gaussian Error Linear Unit function. -https://en.wikipedia.org/wiki/Activation_function#Comparison_of_activation_functions +This script demonstrates an implementation of the Gaussian Error Linear Unit function. +* https://en.wikipedia.org/wiki/Activation_function#Comparison_of_activation_functions -The function takes a vector x of K real numbers as input and -returns x * sigmoid(1.702*x). -The Gaussian Error Linear Unit (GELU), is a high-performing neural -network activation function. +The function takes a vector of K real numbers as input and returns x * sigmoid(1.702*x). +Gaussian Error Linear Unit (GELU) is a high-performing neural network activation +function. -Script inspired from its corresponding research paper -https://arxiv.org/abs/1606.08415 +This script is inspired by a corresponding research paper. +* https://arxiv.org/abs/1606.08415 """ import numpy as np @@ -17,8 +15,8 @@ def sigmoid(vector: np.array) -> np.array: """ - Mathematical function sigmoid takes a vector x of K real numbers - as input and returns 1/ (1 + e^-x). + Mathematical function sigmoid takes a vector x of K real numbers as input and + returns 1/ (1 + e^-x). https://en.wikipedia.org/wiki/Sigmoid_function >>> sigmoid(np.array([-1.0, 1.0, 2.0]))