From 4893c31ddba159eb7434ec65220efd77602493e2 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Mon, 9 Oct 2023 11:59:16 +0530 Subject: [PATCH 01/13] Create categorical_cross_entropy.py --- .../categorical_cross_entropy.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 machine_learning/loss_functions/categorical_cross_entropy.py diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py new file mode 100644 index 000000000000..04fee989d76a --- /dev/null +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -0,0 +1,55 @@ +""" +Categorical Cross-Entropy Loss + +This function calculates the Categorical Cross-Entropy Loss between true class +labels and predicted class probabilities. + +Formula: +Categorical Cross-Entropy Loss = -Σ(y_true * log(y_pred)) + +Resources: +- [Wikipedia - Cross entropy](https://en.wikipedia.org/wiki/Cross_entropy) +""" + +import numpy as np + +def categorical_crossentropy( + y_true: np.ndarray, y_pred: np.ndarray, epsilon: float = 1e-15 +) -> float: + """ + Calculate Categorical Cross-Entropy Loss between true class labels and + predicted class probabilities. + + Parameters: + - y_true: True class labels (one-hot encoded) as a NumPy array. + - y_pred: Predicted class probabilities as a NumPy array. + - epsilon: Small constant to avoid numerical instability. + + Returns: + - ce_loss: Categorical Cross-Entropy Loss as a floating-point number. + + Example: + >>> true_labels = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) + >>> pred_probs = np.array([[0.9, 0.1, 0.0], [0.2, 0.7, 0.1], [0.0, 0.1, 0.9]]) + >>> categorical_crossentropy(true_labels, pred_probs) + 0.18913199175146167 + + >>> y_true = np.array([[1, 0], [0, 1]]) + >>> y_pred = np.array([[0.9, 0.1, 0.0], [0.2, 0.7, 0.1]]) + >>> categorical_crossentropy(y_true, y_pred) + Traceback (most recent call last): + ... + ValueError: Input arrays must have the same length. + """ + if y_true.shape != y_pred.shape: + raise ValueError("Input arrays must have the same length.") + + # Clip predicted probabilities to avoid log(0) + y_pred = np.clip(y_pred, epsilon, 1 - epsilon) + + # Calculate categorical cross-entropy loss + return -np.sum(y_true * np.log(y_pred)) / len(y_true) + +if __name__ == "__main__": + import doctest + doctest.testmod() From 35fa3f3e1858a1b974527b79de5d58d26f383e23 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 06:34:48 +0000 Subject: [PATCH 02/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/loss_functions/categorical_cross_entropy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index 04fee989d76a..f0ef40f39091 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -13,6 +13,7 @@ import numpy as np + def categorical_crossentropy( y_true: np.ndarray, y_pred: np.ndarray, epsilon: float = 1e-15 ) -> float: @@ -43,13 +44,15 @@ def categorical_crossentropy( """ if y_true.shape != y_pred.shape: raise ValueError("Input arrays must have the same length.") - + # Clip predicted probabilities to avoid log(0) y_pred = np.clip(y_pred, epsilon, 1 - epsilon) # Calculate categorical cross-entropy loss return -np.sum(y_true * np.log(y_pred)) / len(y_true) + if __name__ == "__main__": import doctest + doctest.testmod() From 341df1882f9b1c280fd31a804a5f3cab4c4d7144 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:42:53 +0530 Subject: [PATCH 03/13] Update machine_learning/loss_functions/categorical_cross_entropy.py Co-authored-by: Tianyi Zheng --- machine_learning/loss_functions/categorical_cross_entropy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index f0ef40f39091..c85c80d4392d 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -14,7 +14,7 @@ import numpy as np -def categorical_crossentropy( +def categorical_cross_entropy( y_true: np.ndarray, y_pred: np.ndarray, epsilon: float = 1e-15 ) -> float: """ From b673376910cea6479987180cb123282d8bdd76c4 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:08:06 +0530 Subject: [PATCH 04/13] Update categorical_cross_entropy.py --- .../categorical_cross_entropy.py | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index c85c80d4392d..79fdf9fdc6a2 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -32,27 +32,47 @@ def categorical_cross_entropy( Example: >>> true_labels = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) >>> pred_probs = np.array([[0.9, 0.1, 0.0], [0.2, 0.7, 0.1], [0.0, 0.1, 0.9]]) - >>> categorical_crossentropy(true_labels, pred_probs) - 0.18913199175146167 + >>> categorical_cross_entropy(true_labels, pred_probs) + 0.567395975254385 >>> y_true = np.array([[1, 0], [0, 1]]) >>> y_pred = np.array([[0.9, 0.1, 0.0], [0.2, 0.7, 0.1]]) - >>> categorical_crossentropy(y_true, y_pred) + >>> categorical_cross_entropy(y_true, y_pred) Traceback (most recent call last): ... - ValueError: Input arrays must have the same length. + ValueError: Input arrays must have the same shape. + + >>> y_true = np.array([[2, 0, 1], [1, 0, 0]]) + >>> y_pred = np.array([[0.9, 0.1, 0.0], [0.2, 0.7, 0.1]]) + >>> categorical_cross_entropy(y_true, y_pred) + Traceback (most recent call last): + ... + ValueError: y_true must be one-hot encoded. + + >>> y_true = np.array([[1, 0, 0], [0, 1, 0]]) + >>> y_pred = np.array([[0.9, 0.1, 0.1], [0.2, 0.7, 0.1]]) + >>> categorical_cross_entropy(y_true, y_pred) + Traceback (most recent call last): + ... + ValueError: Predicted probabilities must sum to approximately 1. """ if y_true.shape != y_pred.shape: - raise ValueError("Input arrays must have the same length.") + raise ValueError("Input arrays must have the same shape.") + if not np.all((y_true == 0) | (y_true == 1)): + raise ValueError("y_true must be one-hot encoded.") + + if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, + rtol=epsilon, atol=epsilon)): + raise ValueError("Predicted probabilities must sum to approximately 1.") + # Clip predicted probabilities to avoid log(0) y_pred = np.clip(y_pred, epsilon, 1 - epsilon) # Calculate categorical cross-entropy loss - return -np.sum(y_true * np.log(y_pred)) / len(y_true) + return -np.sum(y_true * np.log(y_pred)) if __name__ == "__main__": import doctest - doctest.testmod() From 06110d95dec1d958578573aa09ec65bb3358ece5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 07:41:11 +0000 Subject: [PATCH 05/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../loss_functions/categorical_cross_entropy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index 79fdf9fdc6a2..2dce5a2d7e77 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -62,10 +62,9 @@ def categorical_cross_entropy( if not np.all((y_true == 0) | (y_true == 1)): raise ValueError("y_true must be one-hot encoded.") - if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, - rtol=epsilon, atol=epsilon)): + if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, rtol=epsilon, atol=epsilon)): raise ValueError("Predicted probabilities must sum to approximately 1.") - + # Clip predicted probabilities to avoid log(0) y_pred = np.clip(y_pred, epsilon, 1 - epsilon) @@ -75,4 +74,5 @@ def categorical_cross_entropy( if __name__ == "__main__": import doctest + doctest.testmod() From 1cff490a1ce595c3058bf1316c8ab32bbfbdc987 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:13:48 +0530 Subject: [PATCH 06/13] Update categorical_cross_entropy.py --- machine_learning/loss_functions/categorical_cross_entropy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index 2dce5a2d7e77..68f97f966ccd 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -62,7 +62,8 @@ def categorical_cross_entropy( if not np.all((y_true == 0) | (y_true == 1)): raise ValueError("y_true must be one-hot encoded.") - if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, rtol=epsilon, atol=epsilon)): + if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, + rtol=epsilon, atol=epsilon)): raise ValueError("Predicted probabilities must sum to approximately 1.") # Clip predicted probabilities to avoid log(0) @@ -74,5 +75,4 @@ def categorical_cross_entropy( if __name__ == "__main__": import doctest - doctest.testmod() From 65c470296fa5a1aa0aaf53f58d6d4c2f0a8b00a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 07:44:23 +0000 Subject: [PATCH 07/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/loss_functions/categorical_cross_entropy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index 68f97f966ccd..2dce5a2d7e77 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -62,8 +62,7 @@ def categorical_cross_entropy( if not np.all((y_true == 0) | (y_true == 1)): raise ValueError("y_true must be one-hot encoded.") - if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, - rtol=epsilon, atol=epsilon)): + if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, rtol=epsilon, atol=epsilon)): raise ValueError("Predicted probabilities must sum to approximately 1.") # Clip predicted probabilities to avoid log(0) @@ -75,4 +74,5 @@ def categorical_cross_entropy( if __name__ == "__main__": import doctest + doctest.testmod() From eeb0bc617a2e9c4fd7785a7f7bafaca0f228e293 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:13:39 +0530 Subject: [PATCH 08/13] Update machine_learning/loss_functions/categorical_cross_entropy.py Co-authored-by: Tianyi Zheng --- machine_learning/loss_functions/categorical_cross_entropy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index 2dce5a2d7e77..5c3b6cb1cb5b 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -59,7 +59,7 @@ def categorical_cross_entropy( if y_true.shape != y_pred.shape: raise ValueError("Input arrays must have the same shape.") - if not np.all((y_true == 0) | (y_true == 1)): + if not np.array_equal(y_true[y_true != 0], [1]): raise ValueError("y_true must be one-hot encoded.") if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, rtol=epsilon, atol=epsilon)): From c60aad47462d978acb5d234d0d0371718a77ccad Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:14:09 +0530 Subject: [PATCH 09/13] Update machine_learning/loss_functions/categorical_cross_entropy.py Co-authored-by: Tianyi Zheng --- .../loss_functions/categorical_cross_entropy.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index 5c3b6cb1cb5b..353131b65472 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -49,6 +49,13 @@ def categorical_cross_entropy( ... ValueError: y_true must be one-hot encoded. + >>> y_true = np.array([[1, 0, 1], [1, 0, 0]]) + >>> y_pred = np.array([[0.9, 0.1, 0.0], [0.2, 0.7, 0.1]]) + >>> categorical_cross_entropy(y_true, y_pred) + Traceback (most recent call last): + ... + ValueError: y_true must be one-hot encoded. + >>> y_true = np.array([[1, 0, 0], [0, 1, 0]]) >>> y_pred = np.array([[0.9, 0.1, 0.1], [0.2, 0.7, 0.1]]) >>> categorical_cross_entropy(y_true, y_pred) From e194822415fdec02ebeb3d143526ec8c583ab589 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Tue, 10 Oct 2023 14:25:37 +0530 Subject: [PATCH 10/13] Update machine_learning/loss_functions/categorical_cross_entropy.py Co-authored-by: Tianyi Zheng --- machine_learning/loss_functions/categorical_cross_entropy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index 353131b65472..b8c10e38e6fa 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -66,7 +66,7 @@ def categorical_cross_entropy( if y_true.shape != y_pred.shape: raise ValueError("Input arrays must have the same shape.") - if not np.array_equal(y_true[y_true != 0], [1]): + if np.any((y_test != 0) & (y_test != 1)) or np.any(y_test.sum(axis=1) != 1): raise ValueError("y_true must be one-hot encoded.") if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, rtol=epsilon, atol=epsilon)): From f163f474b00808dfec30994b492ba93287469476 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Tue, 10 Oct 2023 14:25:50 +0530 Subject: [PATCH 11/13] Update machine_learning/loss_functions/categorical_cross_entropy.py Co-authored-by: Tianyi Zheng --- machine_learning/loss_functions/categorical_cross_entropy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index b8c10e38e6fa..a35a860e94ab 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -5,7 +5,7 @@ labels and predicted class probabilities. Formula: -Categorical Cross-Entropy Loss = -Σ(y_true * log(y_pred)) +Categorical Cross-Entropy Loss = -Σ(y_true * ln(y_pred)) Resources: - [Wikipedia - Cross entropy](https://en.wikipedia.org/wiki/Cross_entropy) From cef2b2795a919d13688912ed14f05c1dd231a130 Mon Sep 17 00:00:00 2001 From: Arnav Kohli <95236897+THEGAMECHANGER416@users.noreply.github.com> Date: Tue, 10 Oct 2023 15:28:01 +0530 Subject: [PATCH 12/13] Update categorical_cross_entropy.py --- machine_learning/loss_functions/categorical_cross_entropy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index a35a860e94ab..b1507cde7b67 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -66,14 +66,14 @@ def categorical_cross_entropy( if y_true.shape != y_pred.shape: raise ValueError("Input arrays must have the same shape.") - if np.any((y_test != 0) & (y_test != 1)) or np.any(y_test.sum(axis=1) != 1): + if np.any((y_true != 0) & (y_true != 1)) or np.any(y_true.sum(axis=1) != 1): raise ValueError("y_true must be one-hot encoded.") if not np.all(np.isclose(np.sum(y_pred, axis=1), 1, rtol=epsilon, atol=epsilon)): raise ValueError("Predicted probabilities must sum to approximately 1.") # Clip predicted probabilities to avoid log(0) - y_pred = np.clip(y_pred, epsilon, 1 - epsilon) + y_pred = np.clip(y_pred, epsilon, 1) # Calculate categorical cross-entropy loss return -np.sum(y_true * np.log(y_pred)) @@ -81,5 +81,4 @@ def categorical_cross_entropy( if __name__ == "__main__": import doctest - doctest.testmod() From 94bb4cee17f339ffe16de1a043e3830c5a968f0c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 09:59:28 +0000 Subject: [PATCH 13/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/loss_functions/categorical_cross_entropy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/machine_learning/loss_functions/categorical_cross_entropy.py b/machine_learning/loss_functions/categorical_cross_entropy.py index b1507cde7b67..68f98902b473 100644 --- a/machine_learning/loss_functions/categorical_cross_entropy.py +++ b/machine_learning/loss_functions/categorical_cross_entropy.py @@ -81,4 +81,5 @@ def categorical_cross_entropy( if __name__ == "__main__": import doctest + doctest.testmod()