From bdde1f9183ea60eca6afeb05c10c9a9da60a0765 Mon Sep 17 00:00:00 2001 From: AnuragSingh0000 <143340413+AnuragSingh0000@users.noreply.github.com> Date: Tue, 8 Oct 2024 02:00:51 +0530 Subject: [PATCH 1/9] Added gradient descent with momentum --- machine_learning/gradient_descent_momentum.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 machine_learning/gradient_descent_momentum.py diff --git a/machine_learning/gradient_descent_momentum.py b/machine_learning/gradient_descent_momentum.py new file mode 100644 index 000000000000..399c7d4299ba --- /dev/null +++ b/machine_learning/gradient_descent_momentum.py @@ -0,0 +1,82 @@ +import numpy as np + +# List of input, output pairs +train_data = ( + ((5, 2, 3), 15), + ((6, 5, 9), 25), + ((11, 12, 13), 41), + ((1, 1, 1), 8), + ((11, 12, 13), 41), +) +test_data = (((515, 22, 13), 555), ((61, 35, 49), 150)) +parameter_vector = [2, 4, 1, 5] +m = len(train_data) +LEARNING_RATE = 0.009 +MOMENTUM = 0.9 + +# Initialize velocity (for momentum) +velocity = [0] * len(parameter_vector) + +def _error(example_no, data_set="train"): + return calculate_hypothesis_value(example_no, data_set) - output(example_no, data_set) + +def _hypothesis_value(data_input_tuple): + hyp_val = 0 + for i in range(len(parameter_vector) - 1): + hyp_val += data_input_tuple[i] * parameter_vector[i + 1] + hyp_val += parameter_vector[0] + return hyp_val + +def output(example_no, data_set): + if data_set == "train": + return train_data[example_no][1] + elif data_set == "test": + return test_data[example_no][1] + return None + +def calculate_hypothesis_value(example_no, data_set): + if data_set == "train": + return _hypothesis_value(train_data[example_no][0]) + elif data_set == "test": + return _hypothesis_value(test_data[example_no][0]) + return None + +def summation_of_cost_derivative(index, end=m): + summation_value = 0 + for i in range(end): + if index == -1: + summation_value += _error(i) + else: + summation_value += _error(i) * train_data[i][0][index] + return summation_value + +def get_cost_derivative(index): + return summation_of_cost_derivative(index, m) / m + +def run_gradient_descent_with_momentum(): + global parameter_vector, velocity + absolute_error_limit = 0.000002 + relative_error_limit = 0 + j = 0 + while True: + j += 1 + temp_parameter_vector = [0] * len(parameter_vector) + for i in range(len(parameter_vector)): + cost_derivative = get_cost_derivative(i - 1) + velocity[i] = MOMENTUM * velocity[i] + cost_derivative + temp_parameter_vector[i] = parameter_vector[i] - LEARNING_RATE * velocity[i] + + if np.allclose(parameter_vector, temp_parameter_vector, atol=absolute_error_limit, rtol=relative_error_limit): + break + parameter_vector = temp_parameter_vector + print(("Number of iterations:", j)) + +def test_gradient_descent(): + for i in range(len(test_data)): + print(("Actual output value:", output(i, "test"))) + print(("Hypothesis output:", calculate_hypothesis_value(i, "test"))) + +if __name__ == "__main__": + run_gradient_descent_with_momentum() + print("\nTesting gradient descent with momentum for a linear hypothesis function.\n") + test_gradient_descent() From 8a38a5fc274ea3bc713c4818e61aea164a4ac629 Mon Sep 17 00:00:00 2001 From: AnuragSingh0000 <143340413+AnuragSingh0000@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:49:33 +0530 Subject: [PATCH 2/9] Gradient Descent with Momentum completed --- machine_learning/gradient_descent_momentum.py | 74 +++++++++++++++++-- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/machine_learning/gradient_descent_momentum.py b/machine_learning/gradient_descent_momentum.py index 399c7d4299ba..9f412a045218 100644 --- a/machine_learning/gradient_descent_momentum.py +++ b/machine_learning/gradient_descent_momentum.py @@ -1,3 +1,7 @@ +""" +Implementation of gradient descent algorithm using momentum for minimizing cost of a linear hypothesis +function. +""" import numpy as np # List of input, output pairs @@ -18,30 +22,73 @@ velocity = [0] * len(parameter_vector) def _error(example_no, data_set="train"): + """ + Calculate the error (difference between predicted and actual output) for a given example. + Args: + example_no (int): Index of the example in the dataset. + data_set (str): The dataset to use, either "train" or "test". + Returns: + float: The difference between the predicted output and the actual output. + """ return calculate_hypothesis_value(example_no, data_set) - output(example_no, data_set) + def _hypothesis_value(data_input_tuple): + """ + Compute the hypothesis value (predicted output) for a given input tuple. + Args: + data_input_tuple (tuple): The input tuple (features) for the example. + Returns: + float: The hypothesis value for the given input. + """ hyp_val = 0 for i in range(len(parameter_vector) - 1): hyp_val += data_input_tuple[i] * parameter_vector[i + 1] hyp_val += parameter_vector[0] return hyp_val + def output(example_no, data_set): + """ + Retrieve the actual output (label) for a given example from the specified dataset. + Args: + example_no (int): Index of the example in the dataset. + data_set (str): The dataset to use, either "train" or "test". + Returns: + int: The actual output value for the specified example. + """ if data_set == "train": return train_data[example_no][1] elif data_set == "test": return test_data[example_no][1] return None + def calculate_hypothesis_value(example_no, data_set): + """ + Calculate the hypothesis value (predicted output) for a given example. + Args: + example_no (int): Index of the example in the dataset. + data_set (str): The dataset to use, either "train" or "test". + Returns: + float: The hypothesis value for the specified example. + """ if data_set == "train": return _hypothesis_value(train_data[example_no][0]) elif data_set == "test": return _hypothesis_value(test_data[example_no][0]) return None + def summation_of_cost_derivative(index, end=m): + """ + Calculate the summation of the cost derivative for a given index. + Args: + index (int): The index of the parameter for which the derivative is calculated. + end (int): The number of examples to consider (defaults to the size of the training set). + Returns: + float: The summation of the cost derivatives for the given parameter. + """ summation_value = 0 for i in range(end): if index == -1: @@ -50,16 +97,29 @@ def summation_of_cost_derivative(index, end=m): summation_value += _error(i) * train_data[i][0][index] return summation_value + def get_cost_derivative(index): + """ + Compute the cost derivative with respect to a parameter. + Args: + index (int): The index of the parameter. + Returns: + float: The cost derivative for the specified parameter. + """ return summation_of_cost_derivative(index, m) / m + def run_gradient_descent_with_momentum(): + """ + Run gradient descent with momentum to minimize the cost function. + This function updates the parameter vector using velocity and the learning rate. + """ global parameter_vector, velocity absolute_error_limit = 0.000002 relative_error_limit = 0 - j = 0 + iteration = 0 while True: - j += 1 + iteration += 1 temp_parameter_vector = [0] * len(parameter_vector) for i in range(len(parameter_vector)): cost_derivative = get_cost_derivative(i - 1) @@ -69,12 +129,16 @@ def run_gradient_descent_with_momentum(): if np.allclose(parameter_vector, temp_parameter_vector, atol=absolute_error_limit, rtol=relative_error_limit): break parameter_vector = temp_parameter_vector - print(("Number of iterations:", j)) + print(f"Number of iterations: {iteration}") + def test_gradient_descent(): + """ + Test the trained model on the test dataset and print actual vs predicted outputs. + """ for i in range(len(test_data)): - print(("Actual output value:", output(i, "test"))) - print(("Hypothesis output:", calculate_hypothesis_value(i, "test"))) + print(f"Actual output value: {output(i, 'test')}") + print(f"Hypothesis output: {calculate_hypothesis_value(i, 'test')}") if __name__ == "__main__": run_gradient_descent_with_momentum() From 7d710c812d65eb3985444576ad5f47ca283e9d4b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 06:29:23 +0000 Subject: [PATCH 3/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/gradient_descent_momentum.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/machine_learning/gradient_descent_momentum.py b/machine_learning/gradient_descent_momentum.py index 9f412a045218..c8df37a36189 100644 --- a/machine_learning/gradient_descent_momentum.py +++ b/machine_learning/gradient_descent_momentum.py @@ -2,6 +2,7 @@ Implementation of gradient descent algorithm using momentum for minimizing cost of a linear hypothesis function. """ + import numpy as np # List of input, output pairs @@ -21,16 +22,19 @@ # Initialize velocity (for momentum) velocity = [0] * len(parameter_vector) + def _error(example_no, data_set="train"): """ Calculate the error (difference between predicted and actual output) for a given example. Args: example_no (int): Index of the example in the dataset. - data_set (str): The dataset to use, either "train" or "test". + data_set (str): The dataset to use, either "train" or "test". Returns: float: The difference between the predicted output and the actual output. """ - return calculate_hypothesis_value(example_no, data_set) - output(example_no, data_set) + return calculate_hypothesis_value(example_no, data_set) - output( + example_no, data_set + ) def _hypothesis_value(data_input_tuple): @@ -125,8 +129,13 @@ def run_gradient_descent_with_momentum(): cost_derivative = get_cost_derivative(i - 1) velocity[i] = MOMENTUM * velocity[i] + cost_derivative temp_parameter_vector[i] = parameter_vector[i] - LEARNING_RATE * velocity[i] - - if np.allclose(parameter_vector, temp_parameter_vector, atol=absolute_error_limit, rtol=relative_error_limit): + + if np.allclose( + parameter_vector, + temp_parameter_vector, + atol=absolute_error_limit, + rtol=relative_error_limit, + ): break parameter_vector = temp_parameter_vector print(f"Number of iterations: {iteration}") @@ -140,7 +149,10 @@ def test_gradient_descent(): print(f"Actual output value: {output(i, 'test')}") print(f"Hypothesis output: {calculate_hypothesis_value(i, 'test')}") + if __name__ == "__main__": run_gradient_descent_with_momentum() - print("\nTesting gradient descent with momentum for a linear hypothesis function.\n") + print( + "\nTesting gradient descent with momentum for a linear hypothesis function.\n" + ) test_gradient_descent() From 6cbd4ff0c3c5305833dcd02cd54ebd3ac5199f05 Mon Sep 17 00:00:00 2001 From: AnuragSingh0000 <143340413+AnuragSingh0000@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:26:54 +0530 Subject: [PATCH 4/9] Resolved Formatting issues --- machine_learning/frequent_pattern_growth.py | 4 ++- machine_learning/gradient_descent_momentum.py | 26 +++++++++---------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/machine_learning/frequent_pattern_growth.py b/machine_learning/frequent_pattern_growth.py index 947f8692f298..08bf6c04deac 100644 --- a/machine_learning/frequent_pattern_growth.py +++ b/machine_learning/frequent_pattern_growth.py @@ -240,7 +240,9 @@ def ascend_tree(leaf_node: TreeNode, prefix_path: list[str]) -> None: ascend_tree(leaf_node.parent, prefix_path) -def find_prefix_path(base_pat: frozenset, tree_node: TreeNode | None) -> dict: # noqa: ARG001 +def find_prefix_path( + base_pat: frozenset, tree_node: TreeNode | None +) -> dict: # noqa: ARG001 """ Find the conditional pattern base for a given base pattern. diff --git a/machine_learning/gradient_descent_momentum.py b/machine_learning/gradient_descent_momentum.py index c8df37a36189..7f28ab9b4fd9 100644 --- a/machine_learning/gradient_descent_momentum.py +++ b/machine_learning/gradient_descent_momentum.py @@ -1,6 +1,6 @@ """ -Implementation of gradient descent algorithm using momentum for minimizing cost of a linear hypothesis -function. +Implementation of gradient descent algorithm using momentum +for minimizing cost of a linear hypothesis function. """ import numpy as np @@ -25,23 +25,23 @@ def _error(example_no, data_set="train"): """ - Calculate the error (difference between predicted and actual output) for a given example. + Calculate the error for a given example. Args: example_no (int): Index of the example in the dataset. data_set (str): The dataset to use, either "train" or "test". Returns: - float: The difference between the predicted output and the actual output. + float: The difference between predicted output and actual output. """ - return calculate_hypothesis_value(example_no, data_set) - output( - example_no, data_set - ) + hypo_value = calculate_hypothesis_value(example_no, data_set) + output_value = output(example_no, data_set) + return hypo_value - output_value def _hypothesis_value(data_input_tuple): """ Compute the hypothesis value (predicted output) for a given input tuple. Args: - data_input_tuple (tuple): The input tuple (features) for the example. + data_input_tuple: The input tuple (features) for the example. Returns: float: The hypothesis value for the given input. """ @@ -54,7 +54,8 @@ def _hypothesis_value(data_input_tuple): def output(example_no, data_set): """ - Retrieve the actual output (label) for a given example from the specified dataset. + Retrieve the actual output (label) for a given example + from the specified dataset. Args: example_no (int): Index of the example in the dataset. data_set (str): The dataset to use, either "train" or "test". @@ -89,7 +90,8 @@ def summation_of_cost_derivative(index, end=m): Calculate the summation of the cost derivative for a given index. Args: index (int): The index of the parameter for which the derivative is calculated. - end (int): The number of examples to consider (defaults to the size of the training set). + end (int): The number of examples to consider + (defaults to the size of the training set). Returns: float: The summation of the cost derivatives for the given parameter. """ @@ -152,7 +154,5 @@ def test_gradient_descent(): if __name__ == "__main__": run_gradient_descent_with_momentum() - print( - "\nTesting gradient descent with momentum for a linear hypothesis function.\n" - ) + print("\nTesting gradient descent momentum for a linear hypothesis function.\n") test_gradient_descent() From a095eb0fd9b178619faf4b7c4b6ea066679d1710 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 06:57:47 +0000 Subject: [PATCH 5/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/frequent_pattern_growth.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/machine_learning/frequent_pattern_growth.py b/machine_learning/frequent_pattern_growth.py index 08bf6c04deac..947f8692f298 100644 --- a/machine_learning/frequent_pattern_growth.py +++ b/machine_learning/frequent_pattern_growth.py @@ -240,9 +240,7 @@ def ascend_tree(leaf_node: TreeNode, prefix_path: list[str]) -> None: ascend_tree(leaf_node.parent, prefix_path) -def find_prefix_path( - base_pat: frozenset, tree_node: TreeNode | None -) -> dict: # noqa: ARG001 +def find_prefix_path(base_pat: frozenset, tree_node: TreeNode | None) -> dict: # noqa: ARG001 """ Find the conditional pattern base for a given base pattern. From 785622c0a5de15d7ee599a13364e5fa06ecefcb9 Mon Sep 17 00:00:00 2001 From: AnuragSingh0000 <143340413+AnuragSingh0000@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:07:01 +0530 Subject: [PATCH 6/9] Added return type --- machine_learning/gradient_descent_momentum.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/machine_learning/gradient_descent_momentum.py b/machine_learning/gradient_descent_momentum.py index 7f28ab9b4fd9..2b01ec93d86d 100644 --- a/machine_learning/gradient_descent_momentum.py +++ b/machine_learning/gradient_descent_momentum.py @@ -23,7 +23,7 @@ velocity = [0] * len(parameter_vector) -def _error(example_no, data_set="train"): +def _error(example_no, data_set="train") -> float: """ Calculate the error for a given example. Args: @@ -37,7 +37,7 @@ def _error(example_no, data_set="train"): return hypo_value - output_value -def _hypothesis_value(data_input_tuple): +def _hypothesis_value(data_input_tuple) -> float: """ Compute the hypothesis value (predicted output) for a given input tuple. Args: @@ -52,7 +52,7 @@ def _hypothesis_value(data_input_tuple): return hyp_val -def output(example_no, data_set): +def output(example_no, data_set) -> int: """ Retrieve the actual output (label) for a given example from the specified dataset. @@ -69,7 +69,7 @@ def output(example_no, data_set): return None -def calculate_hypothesis_value(example_no, data_set): +def calculate_hypothesis_value(example_no, data_set) -> float: """ Calculate the hypothesis value (predicted output) for a given example. Args: @@ -85,7 +85,7 @@ def calculate_hypothesis_value(example_no, data_set): return None -def summation_of_cost_derivative(index, end=m): +def summation_of_cost_derivative(index, end=m) -> float: """ Calculate the summation of the cost derivative for a given index. Args: @@ -104,7 +104,7 @@ def summation_of_cost_derivative(index, end=m): return summation_value -def get_cost_derivative(index): +def get_cost_derivative(index) -> float: """ Compute the cost derivative with respect to a parameter. Args: @@ -115,7 +115,7 @@ def get_cost_derivative(index): return summation_of_cost_derivative(index, m) / m -def run_gradient_descent_with_momentum(): +def run_gradient_descent_with_momentum() -> None: """ Run gradient descent with momentum to minimize the cost function. This function updates the parameter vector using velocity and the learning rate. @@ -143,7 +143,7 @@ def run_gradient_descent_with_momentum(): print(f"Number of iterations: {iteration}") -def test_gradient_descent(): +def test_gradient_descent() -> None: """ Test the trained model on the test dataset and print actual vs predicted outputs. """ From dc755229733aa18bf7bd0093838f8da8187151a3 Mon Sep 17 00:00:00 2001 From: AnuragSingh0000 <143340413+AnuragSingh0000@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:28:36 +0530 Subject: [PATCH 7/9] Fixed more return type issues --- machine_learning/frequent_pattern_growth.py | 4 +++- machine_learning/gradient_descent_momentum.py | 17 ++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/machine_learning/frequent_pattern_growth.py b/machine_learning/frequent_pattern_growth.py index 947f8692f298..08bf6c04deac 100644 --- a/machine_learning/frequent_pattern_growth.py +++ b/machine_learning/frequent_pattern_growth.py @@ -240,7 +240,9 @@ def ascend_tree(leaf_node: TreeNode, prefix_path: list[str]) -> None: ascend_tree(leaf_node.parent, prefix_path) -def find_prefix_path(base_pat: frozenset, tree_node: TreeNode | None) -> dict: # noqa: ARG001 +def find_prefix_path( + base_pat: frozenset, tree_node: TreeNode | None +) -> dict: # noqa: ARG001 """ Find the conditional pattern base for a given base pattern. diff --git a/machine_learning/gradient_descent_momentum.py b/machine_learning/gradient_descent_momentum.py index 2b01ec93d86d..883c5da0841a 100644 --- a/machine_learning/gradient_descent_momentum.py +++ b/machine_learning/gradient_descent_momentum.py @@ -14,14 +14,12 @@ ((11, 12, 13), 41), ) test_data = (((515, 22, 13), 555), ((61, 35, 49), 150)) -parameter_vector = [2, 4, 1, 5] +parameter_vector = [0.0, 0.0, 0.0, 0.0] +velocity = [0.0] * len(parameter_vector) m = len(train_data) LEARNING_RATE = 0.009 MOMENTUM = 0.9 -# Initialize velocity (for momentum) -velocity = [0] * len(parameter_vector) - def _error(example_no, data_set="train") -> float: """ @@ -45,7 +43,7 @@ def _hypothesis_value(data_input_tuple) -> float: Returns: float: The hypothesis value for the given input. """ - hyp_val = 0 + hyp_val = 0.0 for i in range(len(parameter_vector) - 1): hyp_val += data_input_tuple[i] * parameter_vector[i + 1] hyp_val += parameter_vector[0] @@ -66,7 +64,7 @@ def output(example_no, data_set) -> int: return train_data[example_no][1] elif data_set == "test": return test_data[example_no][1] - return None + return -1 def calculate_hypothesis_value(example_no, data_set) -> float: @@ -82,7 +80,7 @@ def calculate_hypothesis_value(example_no, data_set) -> float: return _hypothesis_value(train_data[example_no][0]) elif data_set == "test": return _hypothesis_value(test_data[example_no][0]) - return None + return -1 def summation_of_cost_derivative(index, end=m) -> float: @@ -95,7 +93,7 @@ def summation_of_cost_derivative(index, end=m) -> float: Returns: float: The summation of the cost derivatives for the given parameter. """ - summation_value = 0 + summation_value = 0.0 for i in range(end): if index == -1: summation_value += _error(i) @@ -124,9 +122,10 @@ def run_gradient_descent_with_momentum() -> None: absolute_error_limit = 0.000002 relative_error_limit = 0 iteration = 0 + while True: iteration += 1 - temp_parameter_vector = [0] * len(parameter_vector) + temp_parameter_vector = [0.0] * len(parameter_vector) for i in range(len(parameter_vector)): cost_derivative = get_cost_derivative(i - 1) velocity[i] = MOMENTUM * velocity[i] + cost_derivative From e64627d1829d333734b18aa77e0e196b219b54db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:59:08 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/frequent_pattern_growth.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/machine_learning/frequent_pattern_growth.py b/machine_learning/frequent_pattern_growth.py index 08bf6c04deac..947f8692f298 100644 --- a/machine_learning/frequent_pattern_growth.py +++ b/machine_learning/frequent_pattern_growth.py @@ -240,9 +240,7 @@ def ascend_tree(leaf_node: TreeNode, prefix_path: list[str]) -> None: ascend_tree(leaf_node.parent, prefix_path) -def find_prefix_path( - base_pat: frozenset, tree_node: TreeNode | None -) -> dict: # noqa: ARG001 +def find_prefix_path(base_pat: frozenset, tree_node: TreeNode | None) -> dict: # noqa: ARG001 """ Find the conditional pattern base for a given base pattern. From 60e7323618dcab9d767e9d5d54219141f4d014f2 Mon Sep 17 00:00:00 2001 From: AnuragSingh0000 Date: Wed, 9 Oct 2024 21:47:33 +0000 Subject: [PATCH 9/9] updating DIRECTORY.md --- DIRECTORY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DIRECTORY.md b/DIRECTORY.md index f0a34a553946..d6598578e9a5 100644 --- a/DIRECTORY.md +++ b/DIRECTORY.md @@ -589,6 +589,7 @@ * [Frequent Pattern Growth](machine_learning/frequent_pattern_growth.py) * [Gradient Boosting Classifier](machine_learning/gradient_boosting_classifier.py) * [Gradient Descent](machine_learning/gradient_descent.py) + * [Gradient Descent Momentum](machine_learning/gradient_descent_momentum.py) * [K Means Clust](machine_learning/k_means_clust.py) * [K Nearest Neighbours](machine_learning/k_nearest_neighbours.py) * [Linear Discriminant Analysis](machine_learning/linear_discriminant_analysis.py)