Fix missing data handling (geopandas#582)

Following on pandas-dev/pandas#17728

* Use None for missing values

Previously we used `Empty Polygon` for missing values.  Now we revert to
using NULL in GeometryArray (as before) and Python None when we convert
to shapely objects.

This makes it so that only Nones and NaNs are considered missing.

Author: mrocklin

geopandas/array.py

@@ -462,12 +462,19 @@ def _fill(self, idx, value):
 
         Returns a copy
         """
-        if not (isinstance(value, BaseGeometry) or value is None):
+        base = [self]
+        if isinstance(value, BaseGeometry):
+            base.append(value)
+            value = value.__geom__
+        elif value is None:
+            value = 0
+        else:
             raise TypeError("Value should be either a BaseGeometry or None, "
                             "got %s" % str(value))
-        # self.data[idx] = value
-        self.data[idx] = np.array([value], dtype=object)
-        return self
+
+        new = GeometryArray(self.data.copy(), base=base)
+        new.data[idx] = value
+        return new
 
     def fillna(self, value=None, method=None, limit=None):
         """ Fill NA/NaN values using the specified method.
@@ -826,7 +833,7 @@ def isna(self):
         """
         Boolean NumPy array indicating if each value is missing
         """
-        return np.array([g is None for g in self], dtype='bool')
+        return self.data == 0
 
     def unique(self):
         """Compute the ExtensionArray of unique values.

geopandas/geoseries.py

@@ -94,7 +94,7 @@ def __init__(self, *args, **kwargs):
         # by calling the Series init
         pass
 
-    def append(self, *args, **kwargs):
+    def append(self, *args):
         return self._wrapped_pandas_method('append', *args, **kwargs)
 
     @property
@@ -242,9 +242,8 @@ def isna(self):
         --------
         GeoSereies.notna : inverse of isna
         """
-        non_geo_null = super(GeoSeries, self).isnull()
-        val = self.apply(_is_empty)
-        return Series(np.logical_or(non_geo_null, val))
+        return pd.Series(self.array.data == 0, index=self.index,
+                         name=self.name)
 
     def isnull(self):
         """Alias for `isna` method. See `isna` for more detail."""
@@ -271,21 +270,24 @@ def notnull(self):
         """Alias for `notna` method. See `notna` for more detail."""
         return self.notna()
 
-    def fillna(self, value=None, method=None, inplace=False,
-               **kwargs):
-        """Fill NA/NaN values with a geometry (empty polygon by default).
-
-        "method" is currently not implemented for pandas <= 0.12.
-        """
+    def fillna(self, value=None):
+        """ Fill NA/NaN values with a geometry (empty polygon by default) """
         if value is None:
             value = BaseGeometry()
-        return super(GeoSeries, self).fillna(value=value, method=method,
-                                             inplace=inplace, **kwargs)
+        return GeoSeries(self.array.fillna(value), index=self.index,
+                         crs=self.crs, name=self.name)
+
+    def dropna(self):
+        """ Drop NA/NaN values
+
+        Note: the inplace keyword is not currently supported.
+        """
+        return GeoSeries(self.array[~self.isna()],
+                         index=self.index[~self.isna()],
+                         crs=self.crs, name=self.name)
 
     def align(self, other, join='outer', level=None, copy=True,
               fill_value=None, **kwargs):
-        if fill_value is None:
-            fill_value = BaseGeometry()
         left, right = super(GeoSeries, self).align(other, join=join,
                                                    level=level, copy=copy,
                                                    fill_value=fill_value,

geopandas/io/file.py

@@ -1,3 +1,5 @@
+import math
+from numbers import Number
 import os
 from distutils.version import LooseVersion
 

geopandas/tests/test_geocode.py

@@ -103,7 +103,7 @@ def test_prepare_result_none():
     assert 'address' in df
 
     row = df.loc['b']
-    assert len(row['geometry'].coords) == 0
+    assert not row['geometry']
     assert np.isnan(row['address'])
 
 

geopandas/tests/test_geoseries.py

@@ -101,9 +101,9 @@ def test_align(self):
         a1, a2 = self.a1.align(self.a2)
         assert isinstance(a1, GeoSeries)
         assert isinstance(a2, GeoSeries)
-        assert a2['A'].is_empty
+        assert a2['A'] is None
         assert a1['B'].equals(a2['B'])
-        assert a1['C'].is_empty
+        assert a1['C'] is None
 
     def test_align_crs(self):
         a1 = self.a1
@@ -180,7 +180,6 @@ def test_transform(self):
     def test_fillna(self):
         # default is to fill with empty geometry
         na = self.na_none.fillna()
-        assert isinstance(na[2], BaseGeometry)
         assert na[2].is_empty
         assert geom_equals(self.na_none[:2], na[:2])
         # XXX: method works inconsistently for different pandas versions
@@ -241,3 +240,14 @@ def test_construct_from_series():
     assert [a.equals(b) for a, b in zip(s, g)]
     assert s.name == g.name
     assert s.index is g.index
+
+
+def test_missing_values():
+    s = GeoSeries([Point(1, 1), None, np.nan, BaseGeometry(), Polygon()])
+
+    assert s.isna().tolist() == [False, True, True, False, False]
+    assert s.is_empty.tolist() == [False, False, False, True, True]
+    assert not s.fillna().isna().any()
+    assert len(s.fillna()) == 5
+    assert not s.dropna().isna().any()
+    assert len(s.dropna()) == 3

geopandas/tests/test_pandas_methods.py

@@ -8,6 +8,7 @@
 import pandas as pd
 import shapely
 from shapely.geometry import Point, Polygon
+from shapely.geometry.base import BaseGeometry
 
 from geopandas import GeoDataFrame, GeoSeries
 from geopandas.tests.util import assert_geoseries_equal
@@ -161,25 +162,21 @@ def test_select_dtypes(df):
 # Missing values
 
 
-@pytest.mark.xfail
 def test_fillna():
-    # this currently does not work (it seems to fill in the second coordinate
-    # of the point
     s2 = GeoSeries([Point(0, 0), None, Point(2, 2)])
     res = s2.fillna(Point(1, 1))
-    assert_geoseries_equal(res, s)
+    expected = GeoSeries([Point(0, 0), Point(1, 1), Point(2, 2)])
+    assert_geoseries_equal(res, expected)
 
 
-@pytest.mark.xfail
 def test_dropna():
-    # this currently does not work (doesn't drop)
     s2 = GeoSeries([Point(0, 0), None, Point(2, 2)])
     res = s2.dropna()
     exp = s2.loc[[0, 2]]
     assert_geoseries_equal(res, exp)
 
 
-@pytest.mark.parametrize("NA", [None, np.nan, Point(), Polygon()])
+@pytest.mark.parametrize("NA", [None, np.nan])
 def test_isna(NA):
     s2 = GeoSeries([Point(0, 0), NA, Point(2, 2)], index=[2, 4, 5], name='tt')
     exp = pd.Series([False, True, False], index=[2, 4, 5], name='tt')

geopandas/tests/test_vectorized.py

@@ -400,3 +400,14 @@ def test_unary_union():
 def test_coords():
     L = T.exterior.coords
     assert L == [tuple(t.exterior.coords) for t in triangles]
+
+
+def test_fill():
+    p = shapely.geometry.Point(1, 2)
+    P2 = P._fill([0, 3], p)
+    assert P2[0].equals(p)
+    assert P2[3].equals(p)
+    with pytest.raises(TypeError) as info:
+        P._fill([1, 2], 123)
+
+    assert '123' in str(info.value)

geopandas/tools/tests/test_sjoin.py

@@ -59,7 +59,9 @@ def test_sjoin(op, lsuffix, rsuffix, how, missing):
     right_touched = set()
     for left_index, left_row in left.iterrows():
         for right_index, right_row in right.iterrows():
-            if getattr(left_row['geometry'], op)(right_row['geometry']):
+            left_geom = left_row['geometry']
+            right_geom = right_row['geometry']
+            if left_geom and right_geom and getattr(left_row['geometry'], op)(right_row['geometry']):
                 left_out.append(left_index)
                 right_out.append(right_index)
 
@@ -75,15 +77,15 @@ def test_sjoin(op, lsuffix, rsuffix, how, missing):
         L = list(result.geometry)
         for t in triangles2:
             if t:
-                assert any(t.equals(t2) for t2 in L)
+                assert any(t2 and t.equals(t2) for t2 in L)
 
     if how == 'right':
         assert len(result) >= len(right_out)
         assert set(result.columns) == set(columns + ['index_left'])
         L = list(result.geometry)
         for p in points2:
             if p:
-                assert any(p.equals(p2) for p2 in L)
+                assert any(p2 and p.equals(p2) for p2 in L)
 
 
 def test_crs_mismatch():

geopandas/vectorized.pyx

@@ -59,8 +59,8 @@ cpdef get_element(np.ndarray[np.uintp_t, ndim=1, cast=True] geoms, int idx):
 
     handle = get_geos_context_handle()
 
-    if not geom:
-        geom = GEOSGeom_createEmptyPolygon_r(handle)
+    if geom is NULL:
+        return None
     else:
         geom = GEOSGeom_clone_r(handle, geom)  # create a copy rather than deal with gc
 
@@ -80,11 +80,11 @@ cpdef to_shapely(np.ndarray[np.uintp_t, ndim=1, cast=True] geoms):
         geom = <GEOSGeometry *> geoms[i]
 
         if not geom:
-            geom = GEOSGeom_createEmptyPolygon_r(handle)
+            out[i] = None
         else:
             geom = GEOSGeom_clone_r(handle, geom)  # create a copy rather than deal with gc
+            out[i] = geom_factory(<np.uintp_t> geom)
 
-        out[i] = geom_factory(<np.uintp_t> geom)
 
     return out