diff --git a/neural_network/activation_functions/parametric_relu.py b/neural_network/activation_functions/parametric_relu.py new file mode 100644 index 000000000000..5f343f9fc5f9 --- /dev/null +++ b/neural_network/activation_functions/parametric_relu.py @@ -0,0 +1,88 @@ +""" +Parametric Rectified Linear Unit (PReLU) + +Use Case: PReLU addresses the problem of dying ReLU by allowing a +small, learnable slope for negative values, which can improve model +performance. + +For more detailed information, you can refer to the following link: +https://en.wikipedia.org/wiki/Rectifier_(neural_networks)#Parametric_ReLU +""" + +import numpy as np + + +def parametric_rectified_linear_unit( + vector: np.ndarray, alpha: np.ndarray +) -> np.ndarray: + """ + Implements the Parametric ReLU (PReLU) activation function. + + Parameters: + vector (np.ndarray): The input array for PReLU activation. + alpha (np.ndarray): The learnable slope for negative values, + must be the same shape as vector. + + Returns: + np.ndarray: The input array after applying the PReLU activation. + + Formula: + f(x) = x if x > 0 else f(x) = alpha * x + + Examples: + >>> parametric_rectified_linear_unit( + ... vector=np.array([2.3, 0.6, -2, -3.8]), + ... alpha=np.array([0.3]) + ... ) + array([ 2.3 , 0.6 , -0.6 , -1.14]) + + >>> parametric_rectified_linear_unit( + ... vector=np.array([-9.2, -0.3, 0.45, -4.56]), + ... alpha=np.array([0.067]) + ... ) + array([-0.6164 , -0.0201 , 0.45 , -0.30552]) + + >>> parametric_rectified_linear_unit( + ... vector=np.array([0, 0, 0]), + ... alpha=np.array([0.1, 0.1, 0.1]) + ... ) + array([0., 0., 0.]) + + >>> parametric_rectified_linear_unit( + ... vector=np.array([-1, -2, -3]), + ... alpha=np.array([0.5, 1, 1.5]) + ... ) + array([-0.5, -2. , -4.5]) + + >>> parametric_rectified_linear_unit( + ... vector=np.array([-1, 2, -3]), + ... alpha=np.array([1, 0.5, 2]) + ... ) + array([-1., 2., -6.]) + + >>> parametric_rectified_linear_unit( + ... vector=np.array([-5, -10]), + ... alpha=np.array([2, 3]) + ... ) + array([-10, -30]) + + >>> parametric_rectified_linear_unit( + ... vector=np.array([-1, -2]), + ... alpha=np.array([1, 0]) + ... ) + array([-1, 0]) + + >>> parametric_rectified_linear_unit( + ... vector=np.array([1, -1]), + ... alpha=np.array([0.5, 2]) + ... ) + array([ 1., -2.]) + """ + + return np.where(vector > 0, vector, alpha * vector) + + +if __name__ == "__main__": + import doctest + + doctest.testmod()