From 77a18e34e506fe6b96b7bc88b0f633195a55778d Mon Sep 17 00:00:00 2001 From: kausthub-kannan Date: Sat, 7 Oct 2023 00:26:54 +0530 Subject: [PATCH 1/2] Added Mish Activation Function --- neural_network/activation_functions/mish.py | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 neural_network/activation_functions/mish.py diff --git a/neural_network/activation_functions/mish.py b/neural_network/activation_functions/mish.py new file mode 100644 index 000000000000..418345936590 --- /dev/null +++ b/neural_network/activation_functions/mish.py @@ -0,0 +1,39 @@ +""" +Mish Activation Function + +Use Case: Improved version of the ReLU activation function used in Computer Vision. +For more detailed information, you can refer to the following link: +https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Mish +""" + +import numpy as np + + +def mish(vector: np.ndarray) -> np.ndarray: + """ + Implements the GELU activation function. + + Parameters: + vector (np.ndarray): The input array for Mish activation. + + Returns: + np.ndarray: The input array after applying the Mish activation. + + Formula: + f(x) = x * np.tanh(np.softplus(x)) = x * np.tanh(np.log(1 + np.exp(x))) + + Examples: + >>> mish(vector=np.array([2.3,0.6,-2,-3.8])) + array([ 2.26211893, 0.46613649, -0.25250148, -0.08405831]) + + >>> mish(np.array([-9.2, -0.3, 0.45, -4.56])) + array([-0.00092952, -0.15113318, 0.33152014, -0.04745745]) + + """ + return vector * np.tanh(np.log(1 + np.exp(vector))) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() From 22261ca9d648a694b4ded415514dfeed40bfc189 Mon Sep 17 00:00:00 2001 From: Tianyi Zheng Date: Fri, 6 Oct 2023 15:13:15 -0400 Subject: [PATCH 2/2] Apply suggestions from code review --- neural_network/activation_functions/mish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neural_network/activation_functions/mish.py b/neural_network/activation_functions/mish.py index 418345936590..e4f98307f2ba 100644 --- a/neural_network/activation_functions/mish.py +++ b/neural_network/activation_functions/mish.py @@ -11,7 +11,7 @@ def mish(vector: np.ndarray) -> np.ndarray: """ - Implements the GELU activation function. + Implements the Mish activation function. Parameters: vector (np.ndarray): The input array for Mish activation. @@ -20,7 +20,7 @@ def mish(vector: np.ndarray) -> np.ndarray: np.ndarray: The input array after applying the Mish activation. Formula: - f(x) = x * np.tanh(np.softplus(x)) = x * np.tanh(np.log(1 + np.exp(x))) + f(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + e^x)) Examples: >>> mish(vector=np.array([2.3,0.6,-2,-3.8]))