Skip to content

Commit d25a9f3

Browse files
committed
Merge pull request #10558 from sinhrks/numexpr_0dim
BUG: pd.eval with numexpr engine coerces 1 element numpy array to scalar
2 parents 5a9a9da + a36988b commit d25a9f3

File tree

3 files changed

+60
-28
lines changed

3 files changed

+60
-28
lines changed

doc/source/whatsnew/v0.17.0.txt

+1
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ Bug Fixes
366366

367367
- Bug that caused segfault when resampling an empty Series (:issue:`10228`)
368368
- Bug in ``DatetimeIndex`` and ``PeriodIndex.value_counts`` resets name from its result, but retains in result's ``Index``. (:issue:`10150`)
369+
- Bug in `pd.eval` using ``numexpr`` engine coerces 1 element numpy array to scalar (:issue:`10546`)
369370
- Bug in `pandas.concat` with ``axis=0`` when column is of dtype ``category`` (:issue:`10177`)
370371
- Bug in ``read_msgpack`` where input type is not always checked (:issue:`10369`)
371372
- Bug in `pandas.read_csv` with kwargs ``index_col=False``, ``index_col=['a', 'b']`` or ``dtype``

pandas/computation/align.py

+8-9
Original file line numberDiff line numberDiff line change
@@ -172,12 +172,11 @@ def _reconstruct_object(typ, obj, axes, dtype):
172172
ret_value = res_t.type(obj)
173173
else:
174174
ret_value = typ(obj).astype(res_t)
175-
176-
try:
177-
ret = ret_value.item()
178-
except (ValueError, IndexError):
179-
# XXX: we catch IndexError to absorb a
180-
# regression in numpy 1.7.0
181-
# fixed by numpy/numpy@04b89c63
182-
ret = ret_value
183-
return ret
175+
# The condition is to distinguish 0-dim array (returned in case of scalar)
176+
# and 1 element array
177+
# e.g. np.array(0) and np.array([0])
178+
if len(obj.shape) == 1 and len(obj) == 1:
179+
if not isinstance(ret_value, np.ndarray):
180+
ret_value = np.array([ret_value]).astype(res_t)
181+
182+
return ret_value

pandas/computation/tests/test_eval.py

+51-19
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
from numpy.random import randn, rand, randint
1212
import numpy as np
13-
from numpy.testing import assert_array_equal, assert_allclose
13+
from numpy.testing import assert_allclose
1414
from numpy.testing.decorators import slow
1515

1616
import pandas as pd
@@ -220,7 +220,7 @@ def check_complex_cmp_op(self, lhs, cmp1, rhs, binop, cmp2):
220220
expected = _eval_single_bin(
221221
lhs_new, binop, rhs_new, self.engine)
222222
result = pd.eval(ex, engine=self.engine, parser=self.parser)
223-
assert_array_equal(result, expected)
223+
tm.assert_numpy_array_equivalent(result, expected)
224224

225225
def check_chained_cmp_op(self, lhs, cmp1, mid, cmp2, rhs):
226226
skip_these = _scalar_skip
@@ -240,7 +240,7 @@ def check_operands(left, right, cmp_op):
240240
for ex in (ex1, ex2, ex3):
241241
result = pd.eval(ex, engine=self.engine,
242242
parser=self.parser)
243-
assert_array_equal(result, expected)
243+
tm.assert_numpy_array_equivalent(result, expected)
244244

245245
def check_simple_cmp_op(self, lhs, cmp1, rhs):
246246
ex = 'lhs {0} rhs'.format(cmp1)
@@ -251,13 +251,13 @@ def check_simple_cmp_op(self, lhs, cmp1, rhs):
251251
else:
252252
expected = _eval_single_bin(lhs, cmp1, rhs, self.engine)
253253
result = pd.eval(ex, engine=self.engine, parser=self.parser)
254-
assert_array_equal(result, expected)
254+
tm.assert_numpy_array_equivalent(result, expected)
255255

256256
def check_binary_arith_op(self, lhs, arith1, rhs):
257257
ex = 'lhs {0} rhs'.format(arith1)
258258
result = pd.eval(ex, engine=self.engine, parser=self.parser)
259259
expected = _eval_single_bin(lhs, arith1, rhs, self.engine)
260-
assert_array_equal(result, expected)
260+
tm.assert_numpy_array_equivalent(result, expected)
261261
ex = 'lhs {0} rhs {0} rhs'.format(arith1)
262262
result = pd.eval(ex, engine=self.engine, parser=self.parser)
263263
nlhs = _eval_single_bin(lhs, arith1, rhs,
@@ -273,7 +273,7 @@ def check_alignment(self, result, nlhs, ghs, op):
273273
pass
274274
else:
275275
expected = self.ne.evaluate('nlhs {0} ghs'.format(op))
276-
assert_array_equal(result, expected)
276+
tm.assert_numpy_array_equivalent(result, expected)
277277

278278
# modulus, pow, and floor division require special casing
279279

@@ -291,7 +291,7 @@ def check_floor_division(self, lhs, arith1, rhs):
291291
if self.engine == 'python':
292292
res = pd.eval(ex, engine=self.engine, parser=self.parser)
293293
expected = lhs // rhs
294-
assert_array_equal(res, expected)
294+
tm.assert_numpy_array_equivalent(res, expected)
295295
else:
296296
self.assertRaises(TypeError, pd.eval, ex, local_dict={'lhs': lhs,
297297
'rhs': rhs},
@@ -325,7 +325,7 @@ def check_pow(self, lhs, arith1, rhs):
325325

326326
if (np.isscalar(lhs) and np.isscalar(rhs) and
327327
_is_py3_complex_incompat(result, expected)):
328-
self.assertRaises(AssertionError, assert_array_equal, result,
328+
self.assertRaises(AssertionError, tm.assert_numpy_array_equivalent, result,
329329
expected)
330330
else:
331331
assert_allclose(result, expected)
@@ -345,11 +345,11 @@ def check_single_invert_op(self, lhs, cmp1, rhs):
345345
elb = np.array([bool(el)])
346346
expected = ~elb
347347
result = pd.eval('~elb', engine=self.engine, parser=self.parser)
348-
assert_array_equal(expected, result)
348+
tm.assert_numpy_array_equivalent(expected, result)
349349

350350
for engine in self.current_engines:
351351
tm.skip_if_no_ne(engine)
352-
assert_array_equal(result, pd.eval('~elb', engine=engine,
352+
tm.assert_numpy_array_equivalent(result, pd.eval('~elb', engine=engine,
353353
parser=self.parser))
354354

355355
def check_compound_invert_op(self, lhs, cmp1, rhs):
@@ -370,13 +370,13 @@ def check_compound_invert_op(self, lhs, cmp1, rhs):
370370
else:
371371
expected = ~expected
372372
result = pd.eval(ex, engine=self.engine, parser=self.parser)
373-
assert_array_equal(expected, result)
373+
tm.assert_numpy_array_equivalent(expected, result)
374374

375375
# make sure the other engines work the same as this one
376376
for engine in self.current_engines:
377377
tm.skip_if_no_ne(engine)
378378
ev = pd.eval(ex, engine=self.engine, parser=self.parser)
379-
assert_array_equal(ev, result)
379+
tm.assert_numpy_array_equivalent(ev, result)
380380

381381
def ex(self, op, var_name='lhs'):
382382
return '{0}{1}'.format(op, var_name)
@@ -620,6 +620,38 @@ def test_disallow_scalar_bool_ops(self):
620620
with tm.assertRaises(NotImplementedError):
621621
pd.eval(ex, engine=self.engine, parser=self.parser)
622622

623+
def test_identical(self):
624+
# GH 10546
625+
x = 1
626+
result = pd.eval('x', engine=self.engine, parser=self.parser)
627+
self.assertEqual(result, 1)
628+
self.assertTrue(np.isscalar(result))
629+
630+
x = 1.5
631+
result = pd.eval('x', engine=self.engine, parser=self.parser)
632+
self.assertEqual(result, 1.5)
633+
self.assertTrue(np.isscalar(result))
634+
635+
x = False
636+
result = pd.eval('x', engine=self.engine, parser=self.parser)
637+
self.assertEqual(result, False)
638+
self.assertTrue(np.isscalar(result))
639+
640+
x = np.array([1])
641+
result = pd.eval('x', engine=self.engine, parser=self.parser)
642+
tm.assert_numpy_array_equivalent(result, np.array([1]))
643+
self.assertEqual(result.shape, (1, ))
644+
645+
x = np.array([1.5])
646+
result = pd.eval('x', engine=self.engine, parser=self.parser)
647+
tm.assert_numpy_array_equivalent(result, np.array([1.5]))
648+
self.assertEqual(result.shape, (1, ))
649+
650+
x = np.array([False])
651+
result = pd.eval('x', engine=self.engine, parser=self.parser)
652+
tm.assert_numpy_array_equivalent(result, np.array([False]))
653+
self.assertEqual(result.shape, (1, ))
654+
623655

624656
class TestEvalNumexprPython(TestEvalNumexprPandas):
625657

@@ -675,7 +707,7 @@ def check_alignment(self, result, nlhs, ghs, op):
675707
pass
676708
else:
677709
expected = eval('nlhs {0} ghs'.format(op))
678-
assert_array_equal(result, expected)
710+
tm.assert_numpy_array_equivalent(result, expected)
679711

680712

681713
class TestEvalPythonPandas(TestEvalPythonPython):
@@ -1086,10 +1118,10 @@ def test_truediv(self):
10861118

10871119
if PY3:
10881120
res = self.eval(ex, truediv=False)
1089-
assert_array_equal(res, np.array([1.0]))
1121+
tm.assert_numpy_array_equivalent(res, np.array([1.0]))
10901122

10911123
res = self.eval(ex, truediv=True)
1092-
assert_array_equal(res, np.array([1.0]))
1124+
tm.assert_numpy_array_equivalent(res, np.array([1.0]))
10931125

10941126
res = self.eval('1 / 2', truediv=True)
10951127
expec = 0.5
@@ -1108,10 +1140,10 @@ def test_truediv(self):
11081140
self.assertEqual(res, expec)
11091141
else:
11101142
res = self.eval(ex, truediv=False)
1111-
assert_array_equal(res, np.array([1]))
1143+
tm.assert_numpy_array_equivalent(res, np.array([1]))
11121144

11131145
res = self.eval(ex, truediv=True)
1114-
assert_array_equal(res, np.array([1.0]))
1146+
tm.assert_numpy_array_equivalent(res, np.array([1.0]))
11151147

11161148
res = self.eval('1 / 2', truediv=True)
11171149
expec = 0.5
@@ -1414,8 +1446,8 @@ class TestScope(object):
14141446

14151447
def check_global_scope(self, e, engine, parser):
14161448
tm.skip_if_no_ne(engine)
1417-
assert_array_equal(_var_s * 2, pd.eval(e, engine=engine,
1418-
parser=parser))
1449+
tm.assert_numpy_array_equivalent(_var_s * 2, pd.eval(e, engine=engine,
1450+
parser=parser))
14191451

14201452
def test_global_scope(self):
14211453
e = '_var_s * 2'

0 commit comments

Comments
 (0)