diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst
index 8deeb3cfae1d3..0b962465982d6 100644
--- a/doc/source/whatsnew/v1.3.0.rst
+++ b/doc/source/whatsnew/v1.3.0.rst
@@ -425,7 +425,9 @@ Plotting
 
 - Bug in :func:`scatter_matrix` raising when 2d ``ax`` argument passed (:issue:`16253`)
 - Prevent warnings when matplotlib's ``constrained_layout`` is enabled (:issue:`25261`)
--
+- Bug in :func:`DataFrame.plot` was showing the wrong colors in the legend if the function was called repeatedly and some calls used ``yerr`` while others didn't (:issue:`39522`)
+- Bug in :func:`DataFrame.plot` was showing the wrong colors in the legend if the function was called repeatedly and some calls used ``secondary_y`` and others use ``legend=False`` (:issue:`40044`)
+
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/pandas/plotting/_matplotlib/core.py b/pandas/plotting/_matplotlib/core.py
index 3b0d59501ba05..0caef6db022d4 100644
--- a/pandas/plotting/_matplotlib/core.py
+++ b/pandas/plotting/_matplotlib/core.py
@@ -598,7 +598,7 @@ def _make_legend(self):
             if leg is not None:
                 title = leg.get_title().get_text()
                 # Replace leg.LegendHandles because it misses marker info
-                handles.extend(handle)
+                handles.extend([i for i in handle if i not in self.legend_handles])
                 labels = [x.get_text() for x in leg.get_texts()]
 
             if self.legend:
@@ -641,6 +641,8 @@ def _get_ax_legend_handle(self, ax: Axes):
         other_leg = None
         if other_ax is not None:
             other_leg = other_ax.get_legend()
+            other_handle, _ = other_ax.get_legend_handles_labels()
+            handle.extend(other_handle)
         if leg is None and other_leg is not None:
             leg = other_leg
             ax = other_ax
diff --git a/pandas/tests/plotting/frame/test_frame_legend.py b/pandas/tests/plotting/frame/test_frame_legend.py
new file mode 100644
index 0000000000000..f86df1f67959f
--- /dev/null
+++ b/pandas/tests/plotting/frame/test_frame_legend.py
@@ -0,0 +1,73 @@
+from matplotlib.container import ErrorbarContainer
+from matplotlib.lines import Line2D
+
+from pandas import DataFrame
+
+
+def test_mixed_yerr():
+    # https://github.com/pandas-dev/pandas/issues/39522
+
+    df = DataFrame([{"x": 1, "a": 1, "b": 1}, {"x": 2, "a": 2, "b": 3}])
+
+    ax = df.plot("x", "a", c="orange", yerr=0.1, label="orange")
+    df.plot("x", "b", c="blue", yerr=None, ax=ax, label="blue")
+
+    result_handles, result_labels = ax.get_legend_handles_labels()
+
+    assert isinstance(result_handles[0], Line2D)
+    assert isinstance(result_handles[1], ErrorbarContainer)
+
+    expected_labels = ["blue", "orange"]
+    assert result_labels == expected_labels
+
+
+def test_all_have_yerr():
+    # https://github.com/pandas-dev/pandas/issues/39522
+
+    df = DataFrame([{"x": 1, "a": 1, "b": 1}, {"x": 2, "a": 2, "b": 3}])
+
+    ax = df.plot("x", "a", c="orange", yerr=0.1, label="orange")
+    df.plot("x", "b", c="blue", yerr=0.1, ax=ax, label="blue")
+
+    result_handles, result_labels = ax.get_legend_handles_labels()
+
+    assert isinstance(result_handles[0], ErrorbarContainer)
+    assert isinstance(result_handles[1], ErrorbarContainer)
+
+    expected_labels = ["orange", "blue"]
+    assert result_labels == expected_labels
+
+
+def test_none_have_yerr():
+    # https://github.com/pandas-dev/pandas/issues/39522
+
+    df = DataFrame([{"x": 1, "a": 1, "b": 1}, {"x": 2, "a": 2, "b": 3}])
+
+    ax = df.plot("x", "a", c="orange", label="orange")
+    df.plot("x", "b", c="blue", ax=ax, label="blue")
+
+    result_handles, result_labels = ax.get_legend_handles_labels()
+
+    assert isinstance(result_handles[0], Line2D)
+    assert isinstance(result_handles[1], Line2D)
+
+    expected_labels = ["orange", "blue"]
+    assert result_labels == expected_labels
+
+
+def test_legend_false():
+    # https://github.com/pandas-dev/pandas/issues/40044
+
+    df = DataFrame({"a": [1, 1], "b": [2, 2]})
+    df2 = DataFrame({"d": [2.5, 2.5]})
+
+    ax = df.plot(legend=True, color={"a": "blue", "b": "green"}, secondary_y="b")
+    df2.plot(legend=False, color="red", ax=ax)
+
+    handles, result_labels = ax.get_legend_handles_labels()
+    result_colors = [i.get_color() for i in handles]
+    expected_labels = ["a", "d"]
+    expected_colors = ["blue", "red"]
+
+    assert result_labels == expected_labels
+    assert result_colors == expected_colors