Fix missing data handling (geopandas#582)

Following on pandas-dev/pandas#17728

* Use None for missing values

Previously we used `Empty Polygon` for missing values.  Now we revert to
using NULL in GeometryArray (as before) and Python None when we convert
to shapely objects.

This makes it so that only Nones and NaNs are considered missing.

Author: mrocklin

geopandas/array.py

@@ -818,14 +818,23 @@ def _fill(self, idx, value):
         """ Fill index locations with value
 
         Value should be a BaseGeometry
+
+        Returns a copy
         """
-        if not (isinstance(value, BaseGeometry) or value is None):
+        base = [self]
+        if isinstance(value, BaseGeometry):
+            base.append(value)
+            value = value.__geom__
+        elif value is None:
+            value = 0
+        else:
             raise TypeError(
-                "Value should be either a BaseGeometry or None, got %s" % str(value)
+                "Value should be either a BaseGeometry or None, " "got %s" % str(value)
             )
-        # self.data[idx] = value
-        self.data[idx] = np.array([value], dtype=object)
-        return self
+
+        new = GeometryArray(self.data.copy(), base=base)
+        new.data[idx] = value
+        return new
 
     def fillna(self, value=None, method=None, limit=None):
         """ Fill NA/NaN values using the specified method.
@@ -897,7 +906,7 @@ def isna(self):
         """
         Boolean NumPy array indicating if each value is missing
         """
-        return np.array([g is None for g in self], dtype="bool")
+        return self.data == 0
 
     def unique(self):
         """Compute the ExtensionArray of unique values.

geopandas/geoseries.py

@@ -345,15 +345,24 @@ def notnull(self):
         """Alias for `notna` method. See `notna` for more detail."""
         return self.notna()
 
-    def fillna(self, value=None, method=None, inplace=False, **kwargs):
-        """Fill NA values with a geometry (empty polygon by default).
-
-        "method" is currently not implemented for pandas <= 0.12.
-        """
+    def fillna(self, value=None):
+        """Fill NA/NaN values with a geometry (empty polygon by default)"""
         if value is None:
             value = BaseGeometry()
-        return super(GeoSeries, self).fillna(
-            value=value, method=method, inplace=inplace, **kwargs
+        return GeoSeries(
+            self.array.fillna(value), index=self.index, crs=self.crs, name=self.name
+        )
+
+    def dropna(self):
+        """ Drop NA/NaN values
+
+        Note: the inplace keyword is not currently supported.
+        """
+        return GeoSeries(
+            self.array[~self.isna()],
+            index=self.index[~self.isna()],
+            crs=self.crs,
+            name=self.name,
         )
 
     def __contains__(self, other):

geopandas/tests/test_geocode.py

@@ -101,7 +101,7 @@ def test_prepare_result_none():
     assert "address" in df
 
     row = df.loc["b"]
-    assert len(row["geometry"].coords) == 0
+    assert not row["geometry"]
     assert np.isnan(row["address"])
 
 

geopandas/tests/test_geoseries.py

@@ -162,7 +162,6 @@ def test_transform(self):
     def test_fillna(self):
         # default is to fill with empty geometry
         na = self.na_none.fillna()
-        assert isinstance(na[2], BaseGeometry)
         assert na[2].is_empty
         assert geom_equals(self.na_none[:2], na[:2])
         # XXX: method works inconsistently for different pandas versions

geopandas/tests/test_pandas_methods.py

@@ -233,10 +233,11 @@ def test_select_dtypes(df):
 # Missing values
 
 
-def test_fillna(s):
+def test_fillna():
     s2 = GeoSeries([Point(0, 0), None, Point(2, 2)])
     res = s2.fillna(Point(1, 1))
-    assert_geoseries_equal(res, s)
+    expected = GeoSeries([Point(0, 0), Point(1, 1), Point(2, 2)])
+    assert_geoseries_equal(res, expected)
 
 
 def test_dropna():

geopandas/tests/test_vectorized.py

@@ -424,3 +424,14 @@ def test_unary_union():
 def test_coords():
     L = T.exterior.coords
     assert L == [tuple(t.exterior.coords) for t in triangles]
+
+
+def test_fill():
+    p = shapely.geometry.Point(1, 2)
+    P2 = P._fill([0, 3], p)
+    assert P2[0].equals(p)
+    assert P2[3].equals(p)
+    with pytest.raises(TypeError) as info:
+        P._fill([1, 2], 123)
+
+    assert "123" in str(info.value)

geopandas/tools/tests/test_sjoin.py

@@ -65,7 +65,13 @@ def test_sjoin(op, lsuffix, rsuffix, how, missing):
     right_touched = set()
     for left_index, left_row in left.iterrows():
         for right_index, right_row in right.iterrows():
-            if getattr(left_row["geometry"], op)(right_row["geometry"]):
+            left_geom = left_row["geometry"]
+            right_geom = right_row["geometry"]
+            if (
+                left_geom
+                and right_geom
+                and getattr(left_row["geometry"], op)(right_row["geometry"])
+            ):
                 left_out.append(left_index)
                 right_out.append(right_index)
 
@@ -81,15 +87,15 @@ def test_sjoin(op, lsuffix, rsuffix, how, missing):
         L = list(result.geometry)
         for t in triangles2:
             if t:
-                assert any(t.equals(t2) for t2 in L)
+                assert any(t2 and t.equals(t2) for t2 in L)
 
     if how == "right":
         assert len(result) >= len(right_out)
         assert set(result.columns) == set(columns + ["index_left"])
         L = list(result.geometry)
         for p in points2:
             if p:
-                assert any(p.equals(p2) for p2 in L)
+                assert any(p2 and p.equals(p2) for p2 in L)
 
 
 def test_crs_mismatch():

geopandas/vectorized.pyx

@@ -59,8 +59,8 @@ cpdef get_element(np.ndarray[np.uintp_t, ndim=1, cast=True] geoms, int idx):
 
     handle = get_geos_context_handle()
 
-    if not geom:
-        geom = GEOSGeom_createEmptyPolygon_r(handle)
+    if geom is NULL:
+        return None
     else:
         geom = GEOSGeom_clone_r(handle, geom)  # create a copy rather than deal with gc
 
@@ -80,11 +80,11 @@ cpdef to_shapely(np.ndarray[np.uintp_t, ndim=1, cast=True] geoms):
         geom = <GEOSGeometry *> geoms[i]
 
         if not geom:
-            geom = GEOSGeom_createEmptyPolygon_r(handle)
+            out[i] = None
         else:
             geom = GEOSGeom_clone_r(handle, geom)  # create a copy rather than deal with gc
+            out[i] = geom_factory(<np.uintp_t> geom)
 
-        out[i] = geom_factory(<np.uintp_t> geom)
 
     return out