diff --git a/pandas/indexes/base.py b/pandas/indexes/base.py
index 5082fc84982c6..c9b7be5ead649 100644
--- a/pandas/indexes/base.py
+++ b/pandas/indexes/base.py
@@ -1973,7 +1973,7 @@ def difference(self, other):
         except TypeError:
             pass
 
-        return this._shallow_copy(the_diff, name=result_name)
+        return this._shallow_copy(the_diff, name=result_name, freq=None)
 
     def symmetric_difference(self, other, result_name=None):
         """
diff --git a/pandas/tests/indexes/test_datetimelike.py b/pandas/tests/indexes/test_datetimelike.py
index 7502a4ce26b04..b04e840ffc849 100644
--- a/pandas/tests/indexes/test_datetimelike.py
+++ b/pandas/tests/indexes/test_datetimelike.py
@@ -732,6 +732,31 @@ def test_fillna_datetime64(self):
                            dtype=object)
             self.assert_index_equal(idx.fillna('x'), exp)
 
+    def test_difference_of_union(self):
+        # GH14323: Test taking the union of differences of an Index.
+        # Difference of DatetimeIndex does not preserve frequency,
+        # so a differencing operation should not retain the freq field of the
+        # original index.
+        i = pd.date_range("20160920", "20160925", freq="D")
+
+        a = pd.date_range("20160921", "20160924", freq="D")
+        expected = pd.DatetimeIndex(["20160920", "20160925"], freq=None)
+        a_diff = i.difference(a)
+        tm.assert_index_equal(a_diff, expected)
+        tm.assert_attr_equal('freq', a_diff, expected)
+
+        b = pd.date_range("20160922", "20160925", freq="D")
+        b_diff = i.difference(b)
+        expected = pd.DatetimeIndex(["20160920", "20160921"], freq=None)
+        tm.assert_index_equal(b_diff, expected)
+        tm.assert_attr_equal('freq', b_diff, expected)
+
+        union_of_diff = a_diff.union(b_diff)
+        expected = pd.DatetimeIndex(["20160920", "20160921", "20160925"],
+                                    freq=None)
+        tm.assert_index_equal(union_of_diff, expected)
+        tm.assert_attr_equal('freq', union_of_diff, expected)
+
 
 class TestPeriodIndex(DatetimeLike, tm.TestCase):
     _holder = PeriodIndex
@@ -938,6 +963,30 @@ def test_no_millisecond_field(self):
         with self.assertRaises(AttributeError):
             DatetimeIndex([]).millisecond
 
+    def test_difference_of_union(self):
+        # GH14323: Test taking the union of differences of an Index.
+        # Difference of Period MUST preserve frequency, but the ability
+        # to union results must be preserved
+        i = pd.period_range("20160920", "20160925", freq="D")
+
+        a = pd.period_range("20160921", "20160924", freq="D")
+        expected = pd.PeriodIndex(["20160920", "20160925"], freq='D')
+        a_diff = i.difference(a)
+        tm.assert_index_equal(a_diff, expected)
+        tm.assert_attr_equal('freq', a_diff, expected)
+
+        b = pd.period_range("20160922", "20160925", freq="D")
+        b_diff = i.difference(b)
+        expected = pd.PeriodIndex(["20160920", "20160921"], freq='D')
+        tm.assert_index_equal(b_diff, expected)
+        tm.assert_attr_equal('freq', b_diff, expected)
+
+        union_of_diff = a_diff.union(b_diff)
+        expected = pd.PeriodIndex(["20160920", "20160921", "20160925"],
+                                  freq='D')
+        tm.assert_index_equal(union_of_diff, expected)
+        tm.assert_attr_equal('freq', union_of_diff, expected)
+
 
 class TestTimedeltaIndex(DatetimeLike, tm.TestCase):
     _holder = TimedeltaIndex
@@ -1149,3 +1198,28 @@ def test_fillna_timedelta(self):
         exp = pd.Index(
             [pd.Timedelta('1 day'), 'x', pd.Timedelta('3 day')], dtype=object)
         self.assert_index_equal(idx.fillna('x'), exp)
+
+    def test_difference_of_union(self):
+        # GH14323: Test taking the union of differences of an Index.
+        # Difference of TimedeltaIndex does not preserve frequency,
+        # so a differencing operation should not retain the freq field of the
+        # original index.
+        i = pd.timedelta_range("0 days", "5 days", freq="D")
+
+        a = pd.timedelta_range("1 days", "4 days", freq="D")
+        expected = pd.TimedeltaIndex(["0 days", "5 days"], freq=None)
+        a_diff = i.difference(a)
+        tm.assert_index_equal(a_diff, expected)
+        tm.assert_attr_equal('freq', a_diff, expected)
+
+        b = pd.timedelta_range("2 days", "5 days", freq="D")
+        b_diff = i.difference(b)
+        expected = pd.TimedeltaIndex(["0 days", "1 days"], freq=None)
+        tm.assert_index_equal(b_diff, expected)
+        tm.assert_attr_equal('freq', b_diff, expected)
+
+        union_of_difference = a_diff.union(b_diff)
+        expected = pd.TimedeltaIndex(["0 days", "1 days", "5 days"],
+                                     freq=None)
+        tm.assert_index_equal(union_of_difference, expected)
+        tm.assert_attr_equal('freq', union_of_difference, expected)