TST: doc for inf_as_null. more testing to ensure inf not excluded by default. #2477

Author: wesm

doc/source/v0.10.0.txt

@@ -75,6 +75,20 @@ Note:
     # old behavior
     series.resample('D', how='sum', closed='right', label='right')
 
+- Infinity and negative infinity are no longer treated as NA by ``isnull`` and
+  ``notnull``. That they every were was a relic of early pandas. This behavior
+  can be re-enabled globally by the ``mode.use_inf_as_null`` option:
+
+.. ipython:: python
+
+    s = pd.Series([1.5, np.inf, 3.4, -np.inf])
+	pd.isnull(s)
+    s.fillna(0)
+    pd.set_option('use_inf_as_null', True)
+	pd.isnull(s)
+    s.fillna(0)
+	pd.reset_option('use_inf_as_null')
+
 - Methods with the ``inplace`` option now all return ``None`` instead of the
   calling object. E.g. code written like ``df = df.fillna(0, inplace=True)``
   may stop working. To fix, simply delete the unnecessary variable assignment.

pandas/core/common.py

@@ -41,8 +41,7 @@ class AmbiguousIndexError(PandasError, KeyError):
 
 def isnull(obj):
     '''
-    Replacement for numpy.isnan / -numpy.isfinite which is suitable
-    for use on object arrays.
+    Detect missing values (NaN in numeric arrays, None/NaN in object arrays)
 
     Parameters
     ----------
@@ -52,6 +51,9 @@ def isnull(obj):
     -------
     boolean ndarray or boolean
     '''
+    return _isnull(obj)
+
+def _isnull_new(obj):
     if lib.isscalar(obj):
         return lib.checknull(obj)
 
@@ -65,12 +67,10 @@ def isnull(obj):
         return _isnull_ndarraylike(obj)
     else:
         return obj is None
-isnull_new = isnull
 
-def isnull_old(obj):
+def _isnull_old(obj):
     '''
-    Replacement for numpy.isnan / -numpy.isfinite which is suitable
-    for use on object arrays.  Treat None, NaN, INF, -INF as null.
+    Detect missing values. Treat None, NaN, INF, -INF as null.
 
     Parameters
     ----------
@@ -88,12 +88,14 @@ def isnull_old(obj):
         return _isnull_ndarraylike_old(obj)
     elif isinstance(obj, PandasObject):
         # TODO: optimize for DataFrame, etc.
-        return obj.apply(isnull_old)
+        return obj.apply(_isnull_old)
     elif isinstance(obj, list) or hasattr(obj, '__array__'):
         return _isnull_ndarraylike_old(obj)
     else:
         return obj is None
 
+_isnull = _isnull_new
+
 def _use_inf_as_null(key):
     '''Option change callback for null/inf behaviour
     Choose which replacement for numpy.isnan / -numpy.isfinite is used.
@@ -115,9 +117,9 @@ def _use_inf_as_null(key):
     '''
     flag = get_option(key)
     if flag == True:
-        globals()['isnull'] = isnull_old
+        globals()['_isnull'] = _isnull_old
     else:
-        globals()['isnull'] = isnull_new
+        globals()['_isnull'] = _isnull_new
 
 
 
@@ -142,9 +144,11 @@ def _isnull_ndarraylike(obj):
         # this is the NaT pattern
         result = values.view('i8') == tslib.iNaT
     elif issubclass(values.dtype.type, np.timedelta64):
-        result = -np.isfinite(values.view('i8'))
+        # -np.isfinite(values.view('i8'))
+        result = np.ones(values.shape, dtype=bool)
     else:
-        result = -np.isfinite(obj)
+        # -np.isfinite(obj)
+        result = np.isnan(obj)
     return result
 
 

pandas/core/nanops.py

@@ -4,6 +4,7 @@
 
 from pandas.core.common import isnull, notnull
 import pandas.core.common as com
+import pandas.core.config as cf
 import pandas.lib as lib
 import pandas.algos as algos
 import pandas.hashtable as _hash

pandas/tests/test_common.py

@@ -34,10 +34,18 @@ def test_notnull():
         assert notnull(np.inf)
         assert notnull(-np.inf)
 
+        arr = np.array([1.5, np.inf, 3.5, -np.inf])
+        result = notnull(arr)
+        assert result.all()
+
     with cf.option_context("mode.use_inf_as_null",True):
         assert not notnull(np.inf)
         assert not notnull(-np.inf)
 
+        arr = np.array([1.5, np.inf, 3.5, -np.inf])
+        result = notnull(arr)
+        assert result.sum() == 2
+
     with cf.option_context("mode.use_inf_as_null",False):
         float_series = Series(np.random.randn(5))
         obj_series = Series(np.random.randn(5), dtype=object)
@@ -63,6 +71,7 @@ def test_isnull():
     expected = result.apply(isnull)
     tm.assert_frame_equal(result, expected)
 
+
 def test_isnull_lists():
     result = isnull([[False]])
     exp = np.array([[False]])

pandas/tests/test_series.py

@@ -15,6 +15,7 @@
                     bdate_range, date_range)
 from pandas.core.index import MultiIndex
 from pandas.tseries.index import Timestamp, DatetimeIndex
+import pandas.core.config as cf
 import pandas.core.series as smod
 import pandas.lib as lib
 
@@ -1200,18 +1201,28 @@ def test_sum(self):
         self._check_stat_op('sum', np.sum)
 
     def test_sum_inf(self):
+        import pandas.core.nanops as nanops
+
         s = Series(np.random.randn(10))
         s2 = s.copy()
+
         s[5:8] = np.inf
         s2[5:8] = np.nan
-        assert_almost_equal(s.sum(), s2.sum())
 
-        import pandas.core.nanops as nanops
+        self.assertTrue(np.isinf(s.sum()))
+
         arr = np.random.randn(100, 100).astype('f4')
         arr[:, 2] = np.inf
+
+        with cf.option_context("mode.use_inf_as_null", True):
+            assert_almost_equal(s.sum(), s2.sum())
+
+            res = nanops.nansum(arr, axis=1)
+            expected = nanops._nansum(arr, axis=1)
+            assert_almost_equal(res, expected)
+
         res = nanops.nansum(arr, axis=1)
-        expected = nanops._nansum(arr, axis=1)
-        assert_almost_equal(res, expected)
+        self.assertTrue(np.isinf(res).all())
 
     def test_mean(self):
         self._check_stat_op('mean', np.mean)