From ec79a8337a14c4f8e060a11299f1e4c06141a34a Mon Sep 17 00:00:00 2001 From: AtomicVar Date: Sat, 13 Jan 2024 15:58:13 +0800 Subject: [PATCH 1/2] Add KL divergence loss algorithm --- machine_learning/loss_functions.py | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/machine_learning/loss_functions.py b/machine_learning/loss_functions.py index 36a760326f3d..68c88210baf8 100644 --- a/machine_learning/loss_functions.py +++ b/machine_learning/loss_functions.py @@ -471,6 +471,41 @@ def perplexity_loss( return np.mean(perp_losses) +# Kullback-Leibler divergence loss +def kl_divergence_loss(y_true: np.ndarray, y_pred: np.ndarray) -> float: + """ + Calculate the Kullback-Leibler divergence (KL divergence) loss between true labels + and predicted probabilities. + + KL divergence loss quantifies dissimilarity between true labels and predicted + probabilities. It's often used in training generative models. + + KL = Σ(y_true * ln(y_true / y_pred)) + + Reference: https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence + + Parameters: + - y_true: True class probabilities + - y_pred: Predicted class probabilities + + >>> true_labels = np.array([0.2, 0.3, 0.5]) + >>> predicted_probs = np.array([0.3, 0.3, 0.4]) + >>> kl_divergence_loss(true_labels, predicted_probs) + 0.030478754035472025 + >>> true_labels = np.array([0.2, 0.3, 0.5]) + >>> predicted_probs = np.array([0.3, 0.3, 0.4, 0.5]) + >>> kl_divergence_loss(true_labels, predicted_probs) + Traceback (most recent call last): + ... + ValueError: Input arrays must have the same length. + """ + if len(y_true) != len(y_pred): + raise ValueError("Input arrays must have the same length.") + + kl_loss = y_true * np.log(y_true / y_pred) + return np.sum(kl_loss) + + if __name__ == "__main__": import doctest From 1997d704cbcbe83bbc570e2125f660c3c7f89f32 Mon Sep 17 00:00:00 2001 From: Tianyi Zheng Date: Sun, 2 Jun 2024 20:10:00 -0700 Subject: [PATCH 2/2] Apply suggestions from code review --- machine_learning/loss_functions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/machine_learning/loss_functions.py b/machine_learning/loss_functions.py index 68c88210baf8..30679c504bf0 100644 --- a/machine_learning/loss_functions.py +++ b/machine_learning/loss_functions.py @@ -471,8 +471,7 @@ def perplexity_loss( return np.mean(perp_losses) -# Kullback-Leibler divergence loss -def kl_divergence_loss(y_true: np.ndarray, y_pred: np.ndarray) -> float: +def kullback_leibler_divergence(y_true: np.ndarray, y_pred: np.ndarray) -> float: """ Calculate the Kullback-Leibler divergence (KL divergence) loss between true labels and predicted probabilities. @@ -490,11 +489,11 @@ def kl_divergence_loss(y_true: np.ndarray, y_pred: np.ndarray) -> float: >>> true_labels = np.array([0.2, 0.3, 0.5]) >>> predicted_probs = np.array([0.3, 0.3, 0.4]) - >>> kl_divergence_loss(true_labels, predicted_probs) + >>> kullback_leibler_divergence(true_labels, predicted_probs) 0.030478754035472025 >>> true_labels = np.array([0.2, 0.3, 0.5]) >>> predicted_probs = np.array([0.3, 0.3, 0.4, 0.5]) - >>> kl_divergence_loss(true_labels, predicted_probs) + >>> kullback_leibler_divergence(true_labels, predicted_probs) Traceback (most recent call last): ... ValueError: Input arrays must have the same length.