From c54e3feffb9b4bad863fd0afd264a0eb786ec74f Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Fri, 8 Feb 2019 01:21:27 -0500 Subject: [PATCH 01/14] Fix concat not respecting order of OrderedDict --- pandas/core/common.py | 4 ++-- pandas/core/reshape/concat.py | 3 ++- pandas/tests/frame/test_constructors.py | 4 ++-- pandas/tests/reshape/test_concat.py | 10 +++++++++- pandas/tests/sparse/frame/test_frame.py | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index b4de0daa13b16..2bf7457505956 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -13,7 +13,7 @@ from pandas._libs import lib, tslibs import pandas.compat as compat -from pandas.compat import PY36, OrderedDict, iteritems +from pandas.compat import PY36, PY37, OrderedDict, iteritems from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike from pandas.core.dtypes.common import ( @@ -212,7 +212,7 @@ def try_sort(iterable): def dict_keys_to_ordered_list(mapping): # when pandas drops support for Python < 3.6, this function # can be replaced by a simple list(mapping.keys()) - if PY36 or isinstance(mapping, OrderedDict): + if PY37 or isinstance(mapping, OrderedDict): keys = list(mapping.keys()) else: keys = try_sort(mapping) diff --git a/pandas/core/reshape/concat.py b/pandas/core/reshape/concat.py index 53671e00e88b4..2b9bb9a1e7829 100644 --- a/pandas/core/reshape/concat.py +++ b/pandas/core/reshape/concat.py @@ -6,6 +6,7 @@ import pandas.core.dtypes.concat as _concat +from collections import OrderedDict from pandas import DataFrame, Index, MultiIndex, Series, compat from pandas.core import common as com from pandas.core.arrays.categorical import ( @@ -253,7 +254,7 @@ def __init__(self, objs, axis=0, join='outer', join_axes=None, if isinstance(objs, dict): if keys is None: - keys = sorted(objs) + keys = com.dict_keys_to_ordered_list(objs) objs = [objs[k] for k in keys] else: objs = list(objs) diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index 90ad48cac3a5f..71dddacdee846 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -11,7 +11,7 @@ import pytest from pandas.compat import ( - PY3, PY36, OrderedDict, is_platform_little_endian, lmap, long, lrange, + PY3, PY36, PY37, OrderedDict, is_platform_little_endian, lmap, long, lrange, lzip, range, zip) from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike @@ -340,7 +340,7 @@ def test_constructor_dict_nan_tuple_key(self, value): result = DataFrame(data, index=idx, columns=cols) tm.assert_frame_equal(result, expected) - @pytest.mark.skipif(not PY36, reason='Insertion order for Python>=3.6') + @pytest.mark.skipif(not PY37, reason='Insertion order for Python>=3.6') def test_constructor_dict_order_insertion(self): # GH19018 # initialization ordering: by insertion order if python>= 3.6 diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index ec6123bae327e..38a3db5df764d 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -1,4 +1,4 @@ -from collections import deque +from collections import deque, OrderedDict import datetime as dt from datetime import datetime from decimal import Decimal @@ -2436,6 +2436,14 @@ def test_concat_different_extension_dtypes_upcasts(self): ], dtype=object) tm.assert_series_equal(result, expected) + def test_concat_odered_dict(self): + # GH 21510 + expected = pd.concat([pd.Series(range(3)), pd.Series(range(4))], + keys=['First', 'Another']) + result = pd.concat(OrderedDict([('First', pd.Series(range(3))), + ('Another', pd.Series(range(4)))])) + tm.assert_series_equal(result, expected) + @pytest.mark.parametrize('pdt', [pd.Series, pd.DataFrame, pd.Panel]) @pytest.mark.parametrize('dt', np.sctypes['float']) diff --git a/pandas/tests/sparse/frame/test_frame.py b/pandas/tests/sparse/frame/test_frame.py index bfb5103c97adc..5ce5d4eb1ff42 100644 --- a/pandas/tests/sparse/frame/test_frame.py +++ b/pandas/tests/sparse/frame/test_frame.py @@ -129,7 +129,7 @@ def test_constructor_dict_order(self): # order by value d = {'b': [2, 3], 'a': [0, 1]} frame = SparseDataFrame(data=d) - if compat.PY36: + if compat.PY37: expected = SparseDataFrame(data=d, columns=list('ba')) else: expected = SparseDataFrame(data=d, columns=list('ab')) From d6cec5c25a8dd14e4dc5ee930808549f67af79f8 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Fri, 8 Feb 2019 01:31:51 -0500 Subject: [PATCH 02/14] Fix pep8 issues --- pandas/core/common.py | 2 +- pandas/core/reshape/concat.py | 1 - pandas/tests/frame/test_constructors.py | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 731dfa94339d7..2f677e92e5904 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -14,7 +14,7 @@ from pandas._libs import lib, tslibs import pandas.compat as compat -from pandas.compat import PY36, PY37, iteritems +from pandas.compat import PY37, iteritems from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike from pandas.core.dtypes.common import ( diff --git a/pandas/core/reshape/concat.py b/pandas/core/reshape/concat.py index 2b9bb9a1e7829..c77bc0b71c952 100644 --- a/pandas/core/reshape/concat.py +++ b/pandas/core/reshape/concat.py @@ -6,7 +6,6 @@ import pandas.core.dtypes.concat as _concat -from collections import OrderedDict from pandas import DataFrame, Index, MultiIndex, Series, compat from pandas.core import common as com from pandas.core.arrays.categorical import ( diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index 1fad19da5ea3f..c4f62ab40d553 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -12,7 +12,8 @@ import pytest from pandas.compat import ( - PY3, PY36, PY37, is_platform_little_endian, lmap, long, lrange, lzip, range, zip) + PY3, PY36, PY37, is_platform_little_endian, lmap, long, lrange, lzip, + range, zip) from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike from pandas.core.dtypes.common import is_integer_dtype From d4e70efd75c3158f661c1cb723083d024a6dcb67 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Fri, 8 Feb 2019 01:55:45 -0500 Subject: [PATCH 03/14] Fix import order --- pandas/tests/reshape/test_concat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index 20d90dff2fd67..ac4c4aaa22c3f 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -1,4 +1,4 @@ -from collections import deque, OrderedDict +from collections import OrderedDict, deque import datetime as dt from datetime import datetime from decimal import Decimal From db82b071a081fc1b63b8ee34f455535707718afb Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Mon, 11 Feb 2019 19:45:39 -0500 Subject: [PATCH 04/14] Checks reversed to PY36 --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/core/common.py | 4 ++-- pandas/tests/frame/test_constructors.py | 5 ++--- pandas/tests/reshape/test_concat.py | 3 ++- pandas/tests/sparse/frame/test_frame.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index cbefae07b07f1..6b397d821c7ca 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -184,7 +184,7 @@ Reshaping - Bug in :func:`pandas.merge` adds a string of ``None`` if ``None`` is assigned in suffixes instead of remain the column name as-is (:issue:`24782`). - Bug in :func:`merge` when merging by index name would sometimes result in an incorrectly numbered index (:issue:`24212`) - :func:`to_records` now accepts dtypes to its `column_dtypes` parameter (:issue:`24895`) -- +- Bug in :func:`concat` order of OrderedDict is not respected (:issue:`21510`) Sparse diff --git a/pandas/core/common.py b/pandas/core/common.py index 2f677e92e5904..3574998140119 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -14,7 +14,7 @@ from pandas._libs import lib, tslibs import pandas.compat as compat -from pandas.compat import PY37, iteritems +from pandas.compat import PY36, PY37, iteritems from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike from pandas.core.dtypes.common import ( @@ -226,7 +226,7 @@ def try_sort(iterable): def dict_keys_to_ordered_list(mapping): # when pandas drops support for Python < 3.6, this function # can be replaced by a simple list(mapping.keys()) - if PY37 or isinstance(mapping, OrderedDict): + if PY36 or isinstance(mapping, OrderedDict): keys = list(mapping.keys()) else: keys = try_sort(mapping) diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index c4f62ab40d553..a8a78b26e317c 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -12,8 +12,7 @@ import pytest from pandas.compat import ( - PY3, PY36, PY37, is_platform_little_endian, lmap, long, lrange, lzip, - range, zip) + PY3, PY36, is_platform_little_endian, lmap, long, lrange, lzip, range, zip) from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike from pandas.core.dtypes.common import is_integer_dtype @@ -341,7 +340,7 @@ def test_constructor_dict_nan_tuple_key(self, value): result = DataFrame(data, index=idx, columns=cols) tm.assert_frame_equal(result, expected) - @pytest.mark.skipif(not PY37, reason='Insertion order for Python>=3.6') + @pytest.mark.skipif(not PY36, reason='Insertion order for Python>=3.6') def test_constructor_dict_order_insertion(self): # GH19018 # initialization ordering: by insertion order if python>= 3.6 diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index ac4c4aaa22c3f..12aa4faaa7fa0 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -18,6 +18,7 @@ from pandas import ( Categorical, DataFrame, DatetimeIndex, Index, MultiIndex, Panel, Series, Timestamp, concat, date_range, isna, read_csv) +from pandas.core.common import dict_keys_to_ordered_list from pandas.tests.extension.decimal import to_decimal from pandas.util import testing as tm from pandas.util.testing import assert_frame_equal, makeCustomDataframe as mkdf @@ -1163,7 +1164,7 @@ def test_concat_dict(self): 'baz': DataFrame(np.random.randn(4, 3)), 'qux': DataFrame(np.random.randn(4, 3))} - sorted_keys = sorted(frames) + sorted_keys = dict_keys_to_ordered_list(frames) result = concat(frames) expected = concat([frames[k] for k in sorted_keys], keys=sorted_keys) diff --git a/pandas/tests/sparse/frame/test_frame.py b/pandas/tests/sparse/frame/test_frame.py index 5ce5d4eb1ff42..bfb5103c97adc 100644 --- a/pandas/tests/sparse/frame/test_frame.py +++ b/pandas/tests/sparse/frame/test_frame.py @@ -129,7 +129,7 @@ def test_constructor_dict_order(self): # order by value d = {'b': [2, 3], 'a': [0, 1]} frame = SparseDataFrame(data=d) - if compat.PY37: + if compat.PY36: expected = SparseDataFrame(data=d, columns=list('ba')) else: expected = SparseDataFrame(data=d, columns=list('ab')) From 07722127c1e33780900c95fc156703e9f41523db Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Mon, 11 Feb 2019 19:50:02 -0500 Subject: [PATCH 05/14] Forgot to remove PY37 from imports --- pandas/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 3574998140119..5b83cb344b1e7 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -14,7 +14,7 @@ from pandas._libs import lib, tslibs import pandas.compat as compat -from pandas.compat import PY36, PY37, iteritems +from pandas.compat import PY36, iteritems from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike from pandas.core.dtypes.common import ( From 06d3472f4fcaf1b6fbd35e1d886281a8e0ff5ac9 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Mon, 11 Feb 2019 20:59:11 -0500 Subject: [PATCH 06/14] Change order of expected result in test_groupby_agg_ohlc_non_first() and remove an unwanted import --- pandas/tests/groupby/test_groupby.py | 6 +++--- pandas/tests/reshape/test_concat.py | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 1ae8efd2f6867..7d487f4d1301a 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1690,9 +1690,9 @@ def test_groupby_agg_ohlc_non_first(): [1, 1, 1, 1, 1], [1, 1, 1, 1, 1] ], columns=pd.MultiIndex.from_tuples(( - ('foo', 'ohlc', 'open'), ('foo', 'ohlc', 'high'), - ('foo', 'ohlc', 'low'), ('foo', 'ohlc', 'close'), - ('foo', 'sum', 'foo'))), index=pd.date_range( + ('foo', 'sum', 'foo'), ('foo', 'ohlc', 'open'), + ('foo', 'ohlc', 'high'), ('foo', 'ohlc', 'low'), + ('foo', 'ohlc', 'close'))), index=pd.date_range( '2018-01-01', periods=2, freq='D')) result = df.groupby(pd.Grouper(freq='D')).agg(['sum', 'ohlc']) diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index 12aa4faaa7fa0..e80225184cea1 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -18,7 +18,6 @@ from pandas import ( Categorical, DataFrame, DatetimeIndex, Index, MultiIndex, Panel, Series, Timestamp, concat, date_range, isna, read_csv) -from pandas.core.common import dict_keys_to_ordered_list from pandas.tests.extension.decimal import to_decimal from pandas.util import testing as tm from pandas.util.testing import assert_frame_equal, makeCustomDataframe as mkdf @@ -1164,8 +1163,7 @@ def test_concat_dict(self): 'baz': DataFrame(np.random.randn(4, 3)), 'qux': DataFrame(np.random.randn(4, 3))} - sorted_keys = dict_keys_to_ordered_list(frames) - + sorted_keys = pd.core.common.dict_keys_to_ordered_list(frames) result = concat(frames) expected = concat([frames[k] for k in sorted_keys], keys=sorted_keys) tm.assert_frame_equal(result, expected) From 2aee611521bce6f504efbf09a65534d71f159a3d Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Mon, 11 Feb 2019 21:04:58 -0500 Subject: [PATCH 07/14] Fixed accidental empty line removal --- pandas/tests/reshape/test_concat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index e80225184cea1..81b84e8a2ca80 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -1164,6 +1164,7 @@ def test_concat_dict(self): 'qux': DataFrame(np.random.randn(4, 3))} sorted_keys = pd.core.common.dict_keys_to_ordered_list(frames) + result = concat(frames) expected = concat([frames[k] for k in sorted_keys], keys=sorted_keys) tm.assert_frame_equal(result, expected) From feb7a3660a74dc10a90b9dbf0d9af6bfe3008f07 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Mon, 11 Feb 2019 22:38:06 -0500 Subject: [PATCH 08/14] Fix test_groupby_agg_ohlc_non_first() --- pandas/tests/groupby/test_groupby.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 7d487f4d1301a..f1fe0d0828d02 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1690,11 +1690,15 @@ def test_groupby_agg_ohlc_non_first(): [1, 1, 1, 1, 1], [1, 1, 1, 1, 1] ], columns=pd.MultiIndex.from_tuples(( - ('foo', 'sum', 'foo'), ('foo', 'ohlc', 'open'), - ('foo', 'ohlc', 'high'), ('foo', 'ohlc', 'low'), - ('foo', 'ohlc', 'close'))), index=pd.date_range( + ('foo', 'ohlc', 'open'), ('foo', 'ohlc', 'high'), + ('foo', 'ohlc', 'low'), ('foo', 'ohlc', 'close'), + ('foo', 'sum', 'foo'))), index=pd.date_range( '2018-01-01', periods=2, freq='D')) + if compat.PY36: + sorted_keys = sorted(expected, key=lambda x: x[1], reverse=True) + expected = expected.reindex(columns=sorted_keys) + result = df.groupby(pd.Grouper(freq='D')).agg(['sum', 'ohlc']) tm.assert_frame_equal(result, expected) From a11cfc8e32f154b7bb503a283ee0c5c63e27e285 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Thu, 14 Feb 2019 01:26:56 -0500 Subject: [PATCH 09/14] Testing if this will pass tests in <3.6 --- pandas/tests/groupby/test_groupby.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index f1fe0d0828d02..63a46db7989ae 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1695,10 +1695,6 @@ def test_groupby_agg_ohlc_non_first(): ('foo', 'sum', 'foo'))), index=pd.date_range( '2018-01-01', periods=2, freq='D')) - if compat.PY36: - sorted_keys = sorted(expected, key=lambda x: x[1], reverse=True) - expected = expected.reindex(columns=sorted_keys) - - result = df.groupby(pd.Grouper(freq='D')).agg(['sum', 'ohlc']) + result = df.groupby(pd.Grouper(freq='D')).agg(['ohlc', 'sum']) tm.assert_frame_equal(result, expected) From 4f6fc06fe28cff9d21d69eda599a8eabbeb953c4 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Thu, 14 Feb 2019 12:43:14 -0500 Subject: [PATCH 10/14] Revert back to compat check for now --- pandas/tests/groupby/test_groupby.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 63a46db7989ae..f1fe0d0828d02 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1695,6 +1695,10 @@ def test_groupby_agg_ohlc_non_first(): ('foo', 'sum', 'foo'))), index=pd.date_range( '2018-01-01', periods=2, freq='D')) - result = df.groupby(pd.Grouper(freq='D')).agg(['ohlc', 'sum']) + if compat.PY36: + sorted_keys = sorted(expected, key=lambda x: x[1], reverse=True) + expected = expected.reindex(columns=sorted_keys) + + result = df.groupby(pd.Grouper(freq='D')).agg(['sum', 'ohlc']) tm.assert_frame_equal(result, expected) From 451c43b33dd59ca961fa4a51a1b55cfed9f26bc4 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Sat, 16 Feb 2019 19:57:37 -0500 Subject: [PATCH 11/14] Requested changes --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/tests/groupby/test_groupby.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index e437e87dd512d..b0a836e086e5f 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -183,7 +183,7 @@ Reshaping - Bug in :func:`pandas.merge` adds a string of ``None`` if ``None`` is assigned in suffixes instead of remain the column name as-is (:issue:`24782`). - Bug in :func:`merge` when merging by index name would sometimes result in an incorrectly numbered index (:issue:`24212`) - :func:`to_records` now accepts dtypes to its `column_dtypes` parameter (:issue:`24895`) -- Bug in :func:`concat` order of OrderedDict is not respected (:issue:`21510`) +- Bug in :func:`concat` order of ``OrderedDict`` is not respected (:issue:`21510`) Sparse diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index f1fe0d0828d02..c38da28eb3214 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1696,8 +1696,7 @@ def test_groupby_agg_ohlc_non_first(): '2018-01-01', periods=2, freq='D')) if compat.PY36: - sorted_keys = sorted(expected, key=lambda x: x[1], reverse=True) - expected = expected.reindex(columns=sorted_keys) + expected = expected[sorted(expected, key=lambda x: x[1], reverse=True)] result = df.groupby(pd.Grouper(freq='D')).agg(['sum', 'ohlc']) From 154c6392028feb29df4db44a93705a58706d35db Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Sun, 3 Mar 2019 22:36:17 -0500 Subject: [PATCH 12/14] Remaining small requested fixes --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/tests/reshape/test_concat.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 160938844345f..a35bee113ff67 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -185,7 +185,7 @@ Reshaping - Bug in :func:`pandas.merge` adds a string of ``None`` if ``None`` is assigned in suffixes instead of remain the column name as-is (:issue:`24782`). - Bug in :func:`merge` when merging by index name would sometimes result in an incorrectly numbered index (:issue:`24212`) - :func:`to_records` now accepts dtypes to its `column_dtypes` parameter (:issue:`24895`) -- Bug in :func:`concat` order of ``OrderedDict`` is not respected (:issue:`21510`) +- Bug in :func:`concat` where order of ``OrderedDict`` is not respected (:issue:`21510`) Sparse diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index 81b84e8a2ca80..12aa4faaa7fa0 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -18,6 +18,7 @@ from pandas import ( Categorical, DataFrame, DatetimeIndex, Index, MultiIndex, Panel, Series, Timestamp, concat, date_range, isna, read_csv) +from pandas.core.common import dict_keys_to_ordered_list from pandas.tests.extension.decimal import to_decimal from pandas.util import testing as tm from pandas.util.testing import assert_frame_equal, makeCustomDataframe as mkdf @@ -1163,7 +1164,7 @@ def test_concat_dict(self): 'baz': DataFrame(np.random.randn(4, 3)), 'qux': DataFrame(np.random.randn(4, 3))} - sorted_keys = pd.core.common.dict_keys_to_ordered_list(frames) + sorted_keys = dict_keys_to_ordered_list(frames) result = concat(frames) expected = concat([frames[k] for k in sorted_keys], keys=sorted_keys) From b74dc5444fdb4bf1ade1aab1bb930bb82f3e0d8c Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Sun, 3 Mar 2019 23:15:30 -0500 Subject: [PATCH 13/14] Fix unwanted import error --- pandas/tests/reshape/test_concat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index 12aa4faaa7fa0..51313a2040731 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -18,7 +18,7 @@ from pandas import ( Categorical, DataFrame, DatetimeIndex, Index, MultiIndex, Panel, Series, Timestamp, concat, date_range, isna, read_csv) -from pandas.core.common import dict_keys_to_ordered_list +import pandas.core.common as com from pandas.tests.extension.decimal import to_decimal from pandas.util import testing as tm from pandas.util.testing import assert_frame_equal, makeCustomDataframe as mkdf @@ -1164,7 +1164,7 @@ def test_concat_dict(self): 'baz': DataFrame(np.random.randn(4, 3)), 'qux': DataFrame(np.random.randn(4, 3))} - sorted_keys = dict_keys_to_ordered_list(frames) + sorted_keys = com.dict_keys_to_ordered_list(frames) result = concat(frames) expected = concat([frames[k] for k in sorted_keys], keys=sorted_keys) From 3a20a1d5259344f5ccf784a74fda35298c5984a8 Mon Sep 17 00:00:00 2001 From: Alexander Ponomaroff Date: Tue, 12 Mar 2019 12:08:53 -0400 Subject: [PATCH 14/14] Use OrderedDict in groupby/generic.py --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/core/groupby/generic.py | 2 +- pandas/tests/groupby/test_groupby.py | 9 +++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 9826002138c6f..4e6a76c13c38a 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -241,7 +241,7 @@ Reshaping - Bug in :func:`pandas.merge` adds a string of ``None`` if ``None`` is assigned in suffixes instead of remain the column name as-is (:issue:`24782`). - Bug in :func:`merge` when merging by index name would sometimes result in an incorrectly numbered index (:issue:`24212`) - :func:`to_records` now accepts dtypes to its `column_dtypes` parameter (:issue:`24895`) -- Bug in :func:`concat` where order of ``OrderedDict`` is not respected (:issue:`21510`) +- Bug in :func:`concat` where order of ``OrderedDict`` (and ``dict`` in Python 3.6+) is not respected, when passed in as ``objs`` argument (:issue:`21510`) Sparse diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 683c21f7bd47a..c5f9e52e07ecf 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -822,7 +822,7 @@ def _aggregate_multiple_funcs(self, arg, _level): columns.append(com.get_callable_name(f)) arg = lzip(columns, arg) - results = {} + results = collections.OrderedDict() for name, func in arg: obj = self if name in results: diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 8aa1991704670..c9588c68076e5 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -1690,14 +1690,11 @@ def test_groupby_agg_ohlc_non_first(): [1, 1, 1, 1, 1], [1, 1, 1, 1, 1] ], columns=pd.MultiIndex.from_tuples(( - ('foo', 'ohlc', 'open'), ('foo', 'ohlc', 'high'), - ('foo', 'ohlc', 'low'), ('foo', 'ohlc', 'close'), - ('foo', 'sum', 'foo'))), index=pd.date_range( + ('foo', 'sum', 'foo'), ('foo', 'ohlc', 'open'), + ('foo', 'ohlc', 'high'), ('foo', 'ohlc', 'low'), + ('foo', 'ohlc', 'close'))), index=pd.date_range( '2018-01-01', periods=2, freq='D')) - if compat.PY36: - expected = expected[sorted(expected, key=lambda x: x[1], reverse=True)] - result = df.groupby(pd.Grouper(freq='D')).agg(['sum', 'ohlc']) tm.assert_frame_equal(result, expected)