Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-pytables-sigs

Author: jbrockmendel

ci/azure/posix.yml

@@ -44,15 +44,13 @@ jobs:
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
-        # Disabled for NumPy object-dtype warning.
-        # https://github.com/pandas-dev/pandas/issues/30043
-        # py37_np_dev:
-        #   ENV_FILE: ci/deps/azure-37-numpydev.yaml
-        #   CONDA_PY: "37"
-        #   PATTERN: "not slow and not network"
-        #   TEST_ARGS: "-W error"
-        #   PANDAS_TESTING_MODE: "deprecate"
-        #   EXTRA_APT: "xsel"
+        py37_np_dev:
+          ENV_FILE: ci/deps/azure-37-numpydev.yaml
+          CONDA_PY: "37"
+          PATTERN: "not slow and not network"
+          TEST_ARGS: "-W error"
+          PANDAS_TESTING_MODE: "deprecate"
+          EXTRA_APT: "xsel"
 
   steps:
     - script: |

pandas/io/pytables.py

@@ -1995,9 +1995,8 @@ def convert(self, values: np.ndarray, nan_rep, encoding: str, errors: str):
         self.values = new_pd_index
 
     def take_data(self):
-        """ return the values & release the memory """
-        self.values, values = None, self.values
-        return values
+        """ return the values"""
+        return self.values
 
     @property
     def attrs(self):
@@ -2250,9 +2249,8 @@ def set_data(self, data: Union[np.ndarray, ABCExtensionArray]):
         self.kind = _dtype_to_kind(dtype_name)
 
     def take_data(self):
-        """ return the data & release the memory """
-        self.data, data = None, self.data
-        return data
+        """ return the data """
+        return self.data
 
     @classmethod
     def _get_atom(cls, values: Union[np.ndarray, ABCExtensionArray]) -> "Col":

pandas/tests/frame/test_axis_select_reindex.py

@@ -929,6 +929,14 @@ def test_filter_corner(self):
         result = empty.filter(like="foo")
         tm.assert_frame_equal(result, empty)
 
+    def test_filter_regex_non_string(self):
+        # GH#5798 trying to filter on non-string columns should drop,
+        #  not raise
+        df = pd.DataFrame(np.random.random((3, 2)), columns=["STRING", 123])
+        result = df.filter(regex="STRING")
+        expected = df[["STRING"]]
+        tm.assert_frame_equal(result, expected)
+
     def test_take(self, float_frame):
         # homogeneous
         order = [3, 1, 2, 0]

pandas/tests/frame/test_constructors.py

@@ -1726,10 +1726,16 @@ def test_constructor_with_datetimes(self):
         tm.assert_frame_equal(df, expected)
 
     def test_constructor_datetimes_with_nulls(self):
-        # gh-15869
+        # gh-15869, GH#11220
         for arr in [
             np.array([None, None, None, None, datetime.now(), None]),
             np.array([None, None, datetime.now(), None]),
+            [[np.datetime64("NaT")], [None]],
+            [[np.datetime64("NaT")], [pd.NaT]],
+            [[None], [np.datetime64("NaT")]],
+            [[None], [pd.NaT]],
+            [[pd.NaT], [np.datetime64("NaT")]],
+            [[pd.NaT], [None]],
         ]:
             result = DataFrame(arr).dtypes
             expected = Series([np.dtype("datetime64[ns]")])

pandas/tests/frame/test_quantile.py

@@ -7,6 +7,16 @@
 
 
 class TestDataFrameQuantile:
+    def test_quantile_sparse(self):
+        # GH#17198
+        s = pd.Series(pd.SparseArray([1, 2]))
+        s1 = pd.Series(pd.SparseArray([3, 4]))
+        df = pd.DataFrame({0: s, 1: s1})
+        result = df.quantile()
+
+        expected = pd.Series([1.5, 3.5], name=0.5)
+        tm.assert_series_equal(result, expected)
+
     def test_quantile(self, datetime_frame):
         from numpy import percentile
 

pandas/tests/frame/test_timeseries.py

@@ -609,6 +609,14 @@ def test_first_last_valid(
         assert expected_first == df.first_valid_index()
         assert expected_last == df.last_valid_index()
 
+    @pytest.mark.parametrize("klass", [Series, DataFrame])
+    def test_first_valid_index_all_nan(self, klass):
+        # GH#9752 Series/DataFrame should both return None, not raise
+        obj = klass([np.nan])
+
+        assert obj.first_valid_index() is None
+        assert obj.iloc[:0].first_valid_index() is None
+
     def test_first_subset(self):
         ts = tm.makeTimeDataFrame(freq="12h")
         result = ts.first("10d")

pandas/tests/indexes/multi/test_constructor.py

@@ -577,6 +577,17 @@ def test_from_product_respects_none_names():
     tm.assert_index_equal(result, expected)
 
 
+def test_from_product_readonly():
+    # GH#15286 passing read-only array to from_product
+    a = np.array(range(3))
+    b = ["a", "b"]
+    expected = MultiIndex.from_product([a, b])
+
+    a.setflags(write=False)
+    result = MultiIndex.from_product([a, b])
+    tm.assert_index_equal(result, expected)
+
+
 def test_create_index_existing_name(idx):
 
     # GH11193, when an existing index is passed, and a new name is not

pandas/tests/io/formats/test_format.py

@@ -1106,6 +1106,15 @@ def test_truncate_with_different_dtypes(self):
             assert "None" in result
             assert "NaN" not in result
 
+    def test_truncate_with_different_dtypes_multiindex(self):
+        # GH#13000
+        df = DataFrame({"Vals": range(100)})
+        frame = pd.concat([df], keys=["Sweep"], names=["Sweep", "Index"])
+        result = repr(frame)
+
+        result2 = repr(frame.iloc[:5])
+        assert result.startswith(result2)
+
     def test_datetimelike_frame(self):
 
         # GH 12211

pandas/tests/series/test_missing.py

@@ -473,6 +473,13 @@ def test_fillna_consistency(self):
         s2[1] = "foo"
         tm.assert_series_equal(s2, expected)
 
+    def test_where_sparse(self):
+        # GH#17198 make sure we dont get an AttributeError for sp_index
+        ser = pd.Series(pd.SparseArray([1, 2]))
+        result = ser.where(ser >= 2, 0)
+        expected = pd.Series(pd.SparseArray([0, 2]))
+        tm.assert_series_equal(result, expected)
+
     def test_datetime64tz_fillna_round_issue(self):
         # GH 14872
 

pandas/tests/window/test_rolling.py

@@ -418,3 +418,11 @@ def test_rolling_window_as_string():
 
     expected = Series(expData, index=Index(days, name="DateCol"), name="metric")
     tm.assert_series_equal(result, expected)
+
+
+def test_min_periods1():
+    # GH#6795
+    df = pd.DataFrame([0, 1, 2, 1, 0], columns=["a"])
+    result = df["a"].rolling(3, center=True, min_periods=1).max()
+    expected = pd.Series([1.0, 2.0, 2.0, 2.0, 1.0], name="a")
+    tm.assert_series_equal(result, expected)

setup.py

@@ -545,12 +545,10 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
     return pjoin("pandas", subdir, name + suffix)
 
 
-common_include = ["pandas/_libs/src/klib", "pandas/_libs/src"]
-ts_include = ["pandas/_libs/tslibs/src", "pandas/_libs/tslibs"]
-
-
 lib_depends = ["pandas/_libs/src/parse_helper.h"]
 
+klib_include = ["pandas/_libs/src/klib"]
+
 np_datetime_headers = [
     "pandas/_libs/tslibs/src/datetime/np_datetime.h",
     "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h",
@@ -564,36 +562,42 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
 
 
 ext_data = {
-    "_libs.algos": {"pyxfile": "_libs/algos", "depends": _pxi_dep["algos"]},
+    "_libs.algos": {
+        "pyxfile": "_libs/algos",
+        "include": klib_include,
+        "depends": _pxi_dep["algos"],
+    },
     "_libs.groupby": {"pyxfile": "_libs/groupby"},
-    "_libs.hashing": {"pyxfile": "_libs/hashing", "include": [], "depends": []},
+    "_libs.hashing": {"pyxfile": "_libs/hashing", "depends": []},
     "_libs.hashtable": {
         "pyxfile": "_libs/hashtable",
+        "include": klib_include,
         "depends": (["pandas/_libs/src/klib/khash_python.h"] + _pxi_dep["hashtable"]),
     },
     "_libs.index": {
         "pyxfile": "_libs/index",
-        "include": common_include + ts_include,
+        "include": klib_include,
         "depends": _pxi_dep["index"],
         "sources": np_datetime_sources,
     },
     "_libs.indexing": {"pyxfile": "_libs/indexing"},
     "_libs.internals": {"pyxfile": "_libs/internals"},
-    "_libs.interval": {"pyxfile": "_libs/interval", "depends": _pxi_dep["interval"]},
-    "_libs.join": {"pyxfile": "_libs/join"},
+    "_libs.interval": {
+        "pyxfile": "_libs/interval",
+        "include": klib_include,
+        "depends": _pxi_dep["interval"],
+    },
+    "_libs.join": {"pyxfile": "_libs/join", "include": klib_include},
     "_libs.lib": {
         "pyxfile": "_libs/lib",
-        "include": common_include + ts_include,
         "depends": lib_depends + tseries_depends,
+        "include": klib_include,  # due to tokenizer import
         "sources": ["pandas/_libs/src/parser/tokenizer.c"],
     },
-    "_libs.missing": {
-        "pyxfile": "_libs/missing",
-        "include": common_include + ts_include,
-        "depends": tseries_depends,
-    },
+    "_libs.missing": {"pyxfile": "_libs/missing", "depends": tseries_depends},
     "_libs.parsers": {
         "pyxfile": "_libs/parsers",
+        "include": klib_include + ["pandas/_libs/src"],
         "depends": [
             "pandas/_libs/src/parser/tokenizer.h",
             "pandas/_libs/src/parser/io.h",
@@ -605,87 +609,76 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
     },
     "_libs.reduction": {"pyxfile": "_libs/reduction"},
     "_libs.ops": {"pyxfile": "_libs/ops"},
-    "_libs.properties": {"pyxfile": "_libs/properties", "include": []},
+    "_libs.properties": {"pyxfile": "_libs/properties"},
     "_libs.reshape": {"pyxfile": "_libs/reshape", "depends": []},
     "_libs.sparse": {"pyxfile": "_libs/sparse", "depends": _pxi_dep["sparse"]},
     "_libs.tslib": {
         "pyxfile": "_libs/tslib",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.c_timestamp": {
         "pyxfile": "_libs/tslibs/c_timestamp",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
-    "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar", "include": []},
+    "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar"},
     "_libs.tslibs.conversion": {
         "pyxfile": "_libs/tslibs/conversion",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.fields": {
         "pyxfile": "_libs/tslibs/fields",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
-    "_libs.tslibs.frequencies": {"pyxfile": "_libs/tslibs/frequencies", "include": []},
-    "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype", "include": []},
+    "_libs.tslibs.frequencies": {"pyxfile": "_libs/tslibs/frequencies"},
+    "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype"},
     "_libs.tslibs.np_datetime": {
         "pyxfile": "_libs/tslibs/np_datetime",
-        "include": ts_include,
         "depends": np_datetime_headers,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.offsets": {
         "pyxfile": "_libs/tslibs/offsets",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.parsing": {
         "pyxfile": "_libs/tslibs/parsing",
+        "include": klib_include,
         "depends": ["pandas/_libs/src/parser/tokenizer.h"],
         "sources": ["pandas/_libs/src/parser/tokenizer.c"],
     },
     "_libs.tslibs.period": {
         "pyxfile": "_libs/tslibs/period",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.resolution": {
         "pyxfile": "_libs/tslibs/resolution",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.strptime": {
         "pyxfile": "_libs/tslibs/strptime",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.timedeltas": {
         "pyxfile": "_libs/tslibs/timedeltas",
-        "include": ts_include,
         "depends": np_datetime_headers,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.timestamps": {
         "pyxfile": "_libs/tslibs/timestamps",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
-    "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones", "include": []},
+    "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones"},
     "_libs.tslibs.tzconversion": {
         "pyxfile": "_libs/tslibs/tzconversion",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
@@ -709,7 +702,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
 
     sources.extend(data.get("sources", []))
 
-    include = data.get("include", common_include)
+    include = data.get("include")
 
     obj = Extension(
         "pandas.{name}".format(name=name),