From d556e6b04be87c7b2fb5048bb88443ef8110aaa0 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 22 Aug 2019 14:45:43 -0500 Subject: [PATCH 1/7] COMPAT: implement visit_Constant for 3.8 compat Closes https://github.com/pandas-dev/pandas/issues/27261 --- doc/source/whatsnew/v1.0.0.rst | 1 + pandas/core/computation/expr.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 4decc99087a9e..d2dca9fd11196 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -205,6 +205,7 @@ ExtensionArray Other ^^^^^ - Trying to set the ``display.precision``, ``display.max_rows`` or ``display.max_columns`` using :meth:`set_option` to anything but a ``None`` or a positive int will raise a ``ValueError`` (:issue:`23348`) +- Compatibility with Python 3.8 in :meth:`DataFrame.query` (:issue:`27261`) .. _whatsnew_1000.contributors: diff --git a/pandas/core/computation/expr.py b/pandas/core/computation/expr.py index a58f256cf61d4..4c164968575a1 100644 --- a/pandas/core/computation/expr.py +++ b/pandas/core/computation/expr.py @@ -582,6 +582,9 @@ def visit_NameConstant(self, node, **kwargs): def visit_Num(self, node, **kwargs): return self.const_type(node.n, self.env) + def visit_Constant(self, node, **kwargs): + return self.const_type(node.n, self.env) + def visit_Str(self, node, **kwargs): name = self.env.add_tmp(node.s) return self.term_type(name, self.env) From ae495a6c4df93fe66e57bcf908502aad559b205e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 23 Aug 2019 10:16:38 -0500 Subject: [PATCH 2/7] coverage for odd cases (cherry picked from commit 582e8967ba64c47d188e4849a897ec599e06b1e6) --- pandas/tests/test_expressions.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py index ca514f62f451d..609194d3e7fb5 100644 --- a/pandas/tests/test_expressions.py +++ b/pandas/tests/test_expressions.py @@ -5,7 +5,8 @@ from numpy.random import randn import pytest -from pandas.core.api import DataFrame +from pandas import compat +from pandas.core.api import DataFrame, Series from pandas.core.computation import expressions as expr import pandas.util.testing as tm from pandas.util.testing import ( @@ -456,3 +457,14 @@ def test_frame_series_axis(self, axis, arith): result = op_func(other, axis=axis) assert_frame_equal(expected, result) + + +@pytest.mark.parametrize("other", [ + "'x'", + pytest.param("...", marks=pytest.mark.xfail(not compat.PY38, reason="GH-28116")), +]) +def test_equals_various(other): + df = DataFrame({"A": ['a', 'b', 'c']}) + result = df.eval("A == {}".format(other)) + expected = Series([False, False, False]) + tm.assert_series_equal(result, expected) From a58f7e6d1abc86978d86258728f346ebd4cdfd54 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 23 Aug 2019 09:45:59 -0500 Subject: [PATCH 3/7] COMPAT: 3.8 compat for new datetime methods (cherry picked from commit 42893af2eef3b5ac4a342c5a7a287b4a0407005f) --- pandas/tests/scalar/test_nat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/tests/scalar/test_nat.py b/pandas/tests/scalar/test_nat.py index 5b1c4f92bf341..5eb69fb2952dc 100644 --- a/pandas/tests/scalar/test_nat.py +++ b/pandas/tests/scalar/test_nat.py @@ -252,6 +252,7 @@ def _get_overlap_public_nat_methods(klass, as_tuple=False): "day_name", "dst", "floor", + "fromisocalendar", "fromisoformat", "fromordinal", "fromtimestamp", @@ -296,6 +297,8 @@ def test_overlap_public_nat_methods(klass, expected): # "fromisoformat" was introduced in 3.7 if klass is Timestamp and not compat.PY37: expected.remove("fromisoformat") + if klass is Timestamp and not compat.PY38: + expected.remove("fromisocalendar") assert _get_overlap_public_nat_methods(klass) == expected From a61994bb385c759bbfbe07b029053ac57df3f84f Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 23 Aug 2019 09:45:37 -0500 Subject: [PATCH 4/7] COMPAT: 3.8 compat for parsers (cherry picked from commit ffe9b6fa7bf86a99b32fc50b131bfab62005f49d) --- pandas/tests/io/parser/test_common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/parser/test_common.py b/pandas/tests/io/parser/test_common.py index e5366a8357adb..e04535df56663 100644 --- a/pandas/tests/io/parser/test_common.py +++ b/pandas/tests/io/parser/test_common.py @@ -1898,7 +1898,10 @@ def test_null_byte_char(all_parsers): out = parser.read_csv(StringIO(data), names=names) tm.assert_frame_equal(out, expected) else: - msg = "NULL byte detected" + if compat.PY38: + msg = "line contains NUL" + else: + msg = "NULL byte detected" with pytest.raises(ParserError, match=msg): parser.read_csv(StringIO(data), names=names) From 55800f276ae80e917b502d762fc981b000200ba5 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 23 Aug 2019 08:20:28 -0500 Subject: [PATCH 5/7] COMPAT: 3.8 compat for expr (cherry picked from commit ca34ccddb47a102b25a2f23abeac28bc77d121ac) --- pandas/compat/__init__.py | 1 + pandas/tests/computation/test_eval.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index b32da8da3a1fb..9c778f68727c6 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -15,6 +15,7 @@ PY35 = sys.version_info[:2] == (3, 5) PY36 = sys.version_info >= (3, 6) PY37 = sys.version_info >= (3, 7) +PY38 = sys.version_info >= (3, 8) PYPY = platform.python_implementation() == "PyPy" diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index c500760fa1390..08c6e4b42eaeb 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -14,7 +14,7 @@ from pandas.core.dtypes.common import is_bool, is_list_like, is_scalar import pandas as pd -from pandas import DataFrame, Series, date_range +from pandas import DataFrame, Series, date_range, compat from pandas.core.computation import pytables from pandas.core.computation.check import _NUMEXPR_VERSION from pandas.core.computation.engines import NumExprClobberingError, _engines @@ -1267,7 +1267,10 @@ def test_assignment_column(self): msg = "left hand side of an assignment must be a single name" with pytest.raises(SyntaxError, match=msg): df.eval("d,c = a + b") - msg = "can't assign to function call" + if compat.PY38: + msg = "cannot assign to function call" + else: + msg = "can't assign to function call" with pytest.raises(SyntaxError, match=msg): df.eval('Timestamp("20131001") = a + b') From 07d723c261652dcd2b65ec69ab01a42c422e42ca Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 23 Aug 2019 10:25:45 -0500 Subject: [PATCH 6/7] update docs --- doc/source/whatsnew/v0.25.2.rst | 2 +- doc/source/whatsnew/v1.0.0.rst | 1 - pandas/tests/computation/test_eval.py | 18 +++++++++++++++++- pandas/tests/test_expressions.py | 14 +------------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/doc/source/whatsnew/v0.25.2.rst b/doc/source/whatsnew/v0.25.2.rst index 76473405374e8..403c02c3ff129 100644 --- a/doc/source/whatsnew/v0.25.2.rst +++ b/doc/source/whatsnew/v0.25.2.rst @@ -99,7 +99,7 @@ Sparse Other ^^^^^ -- +- Compatibility with Python 3.8 in :meth:`DataFrame.query` (:issue:`27261`) - .. _whatsnew_0.252.contributors: diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index d2dca9fd11196..4decc99087a9e 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -205,7 +205,6 @@ ExtensionArray Other ^^^^^ - Trying to set the ``display.precision``, ``display.max_rows`` or ``display.max_columns`` using :meth:`set_option` to anything but a ``None`` or a positive int will raise a ``ValueError`` (:issue:`23348`) -- Compatibility with Python 3.8 in :meth:`DataFrame.query` (:issue:`27261`) .. _whatsnew_1000.contributors: diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 08c6e4b42eaeb..bcaa3f2b24ca8 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -14,7 +14,7 @@ from pandas.core.dtypes.common import is_bool, is_list_like, is_scalar import pandas as pd -from pandas import DataFrame, Series, date_range, compat +from pandas import DataFrame, Series, compat, date_range from pandas.core.computation import pytables from pandas.core.computation.check import _NUMEXPR_VERSION from pandas.core.computation.engines import NumExprClobberingError, _engines @@ -1970,6 +1970,22 @@ def test_bool_ops_fails_on_scalars(lhs, cmp, rhs, engine, parser): pd.eval(ex, engine=engine, parser=parser) +@pytest.mark.parametrize( + "other", + [ + "'x'", + pytest.param( + "...", marks=pytest.mark.xfail(not compat.PY38, reason="GH-28116") + ), + ], +) +def test_equals_various(other): + df = DataFrame({"A": ["a", "b", "c"]}) + result = df.eval("A == {}".format(other)) + expected = Series([False, False, False]) + tm.assert_series_equal(result, expected) + + def test_inf(engine, parser): s = "inf + 1" expected = np.inf diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py index 609194d3e7fb5..ca514f62f451d 100644 --- a/pandas/tests/test_expressions.py +++ b/pandas/tests/test_expressions.py @@ -5,8 +5,7 @@ from numpy.random import randn import pytest -from pandas import compat -from pandas.core.api import DataFrame, Series +from pandas.core.api import DataFrame from pandas.core.computation import expressions as expr import pandas.util.testing as tm from pandas.util.testing import ( @@ -457,14 +456,3 @@ def test_frame_series_axis(self, axis, arith): result = op_func(other, axis=axis) assert_frame_equal(expected, result) - - -@pytest.mark.parametrize("other", [ - "'x'", - pytest.param("...", marks=pytest.mark.xfail(not compat.PY38, reason="GH-28116")), -]) -def test_equals_various(other): - df = DataFrame({"A": ['a', 'b', 'c']}) - result = df.eval("A == {}".format(other)) - expected = Series([False, False, False]) - tm.assert_series_equal(result, expected) From 3482be3523b7b98450877808ad006f8b6ed2af0b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 23 Aug 2019 13:55:30 -0500 Subject: [PATCH 7/7] workaround --- pandas/tests/computation/test_eval.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index bcaa3f2b24ca8..b6ffd8a83e409 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1982,7 +1982,11 @@ def test_bool_ops_fails_on_scalars(lhs, cmp, rhs, engine, parser): def test_equals_various(other): df = DataFrame({"A": ["a", "b", "c"]}) result = df.eval("A == {}".format(other)) - expected = Series([False, False, False]) + expected = Series([False, False, False], name="A") + if _USE_NUMEXPR: + # https://github.com/pandas-dev/pandas/issues/10239 + # lose name with numexpr engine. Remove when that's fixed. + expected.name = None tm.assert_series_equal(result, expected)