Merge remote-tracking branch 'upstream/master' into refactop-manager-ops

Author: jorisvandenbossche

doc/make.py

@@ -41,12 +41,14 @@ def __init__(
         self,
         num_jobs=0,
         include_api=True,
+        whatsnew=False,
         single_doc=None,
         verbosity=0,
         warnings_are_errors=False,
     ):
         self.num_jobs = num_jobs
         self.include_api = include_api
+        self.whatsnew = whatsnew
         self.verbosity = verbosity
         self.warnings_are_errors = warnings_are_errors
 
@@ -56,6 +58,8 @@ def __init__(
             os.environ["SPHINX_PATTERN"] = single_doc
         elif not include_api:
             os.environ["SPHINX_PATTERN"] = "-api"
+        elif whatsnew:
+            os.environ["SPHINX_PATTERN"] = "whatsnew"
 
         self.single_doc_html = None
         if single_doc and single_doc.endswith(".rst"):
@@ -235,6 +239,9 @@ def html(self):
                 self._open_browser(self.single_doc_html)
             else:
                 self._add_redirects()
+                if self.whatsnew:
+                    self._open_browser(os.path.join("whatsnew", "index.html"))
+
         return ret_code
 
     def latex(self, force=False):
@@ -302,6 +309,12 @@ def main():
     argparser.add_argument(
         "--no-api", default=False, help="omit api and autosummary", action="store_true"
     )
+    argparser.add_argument(
+        "--whatsnew",
+        default=False,
+        help="only build whatsnew (and api for links)",
+        action="store_true",
+    )
     argparser.add_argument(
         "--single",
         metavar="FILENAME",
@@ -353,6 +366,7 @@ def main():
     builder = DocBuilder(
         args.num_jobs,
         not args.no_api,
+        args.whatsnew,
         args.single,
         args.verbosity,
         args.warnings_are_errors,

doc/source/conf.py

@@ -91,8 +91,8 @@
 # (e.g. '10min.rst' or 'pandas.DataFrame.head')
 source_path = os.path.dirname(os.path.abspath(__file__))
 pattern = os.environ.get("SPHINX_PATTERN")
-single_doc = pattern is not None and pattern != "-api"
-include_api = pattern != "-api"
+single_doc = pattern is not None and pattern not in ("-api", "whatsnew")
+include_api = pattern is None or pattern == "whatsnew"
 if pattern:
     for dirname, dirs, fnames in os.walk(source_path):
         reldir = os.path.relpath(dirname, source_path)
@@ -104,7 +104,13 @@
                     continue
                 elif pattern == "-api" and reldir.startswith("reference"):
                     exclude_patterns.append(fname)
-                elif pattern != "-api" and fname != pattern:
+                elif (
+                    pattern == "whatsnew"
+                    and not reldir.startswith("reference")
+                    and reldir != "whatsnew"
+                ):
+                    exclude_patterns.append(fname)
+                elif single_doc and fname != pattern:
                     exclude_patterns.append(fname)
 
 with open(os.path.join(source_path, "index.rst.template")) as f:

doc/source/development/contributing.rst

@@ -604,6 +604,10 @@ reducing the turn-around time for checking your changes.
     python make.py clean
     python make.py --single pandas.DataFrame.join
 
+    # compile whatsnew and API section (to resolve links in the whatsnew)
+    python make.py clean
+    python make.py --whatsnew
+
 For comparison, a full documentation build may take 15 minutes, but a single
 section may take 15 seconds. Subsequent builds, which only process portions
 you have changed, will be faster.

doc/source/user_guide/visualization.rst

@@ -665,7 +665,7 @@ given by column ``z``. The bins are aggregated with NumPy's ``max`` function.
 .. ipython:: python
 
    df = pd.DataFrame(np.random.randn(1000, 2), columns=["a", "b"])
-   df["b"] = df["b"] = df["b"] + np.arange(1000)
+   df["b"] = df["b"] + np.arange(1000)
    df["z"] = np.random.uniform(0, 3, 1000)
 
    @savefig hexbin_plot_agg.png

doc/source/whatsnew/v1.3.0.rst

@@ -390,6 +390,7 @@ I/O
 - Bug in :func:`read_json` when ``orient="split"`` does not maintain numeric string index (:issue:`28556`)
 - :meth:`read_sql` returned an empty generator if ``chunksize`` was no-zero and the query returned no results. Now returns a generator with a single empty dataframe (:issue:`34411`)
 - Bug in :func:`read_hdf` returning unexpected records when filtering on categorical string columns using ``where`` parameter (:issue:`39189`)
+- Bug in :func:`read_sas` raising ``ValueError`` when ``datetimes`` were null (:issue:`39725`)
 
 Period
 ^^^^^^
@@ -430,6 +431,7 @@ Reshaping
 - Bug in :meth:`DataFrame.apply` would give incorrect results when used with a string argument and ``axis=1`` when the axis argument was not supported and now raises a ``ValueError`` instead (:issue:`39211`)
 - Bug in :meth:`DataFrame.sort_values` not reshaping index correctly after sorting on columns, when ``ignore_index=True`` (:issue:`39464`)
 - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``ExtensionDtype`` dtypes (:issue:`39454`)
+- Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``datetime64`` and ``timedelta64`` dtypes (:issue:`39574`)
 
 Sparse
 ^^^^^^
@@ -448,6 +450,8 @@ ExtensionArray
 Other
 ^^^^^
 - Bug in :class:`Index` constructor sometimes silently ignorning a specified ``dtype`` (:issue:`38879`)
+- Bug in :func:`pandas.api.types.infer_dtype` not recognizing Series, Index or array with a period dtype (:issue:`23553`)
+- Bug in :func:`pandas.api.types.infer_dtype` raising an error for general :class:`.ExtensionArray` objects. It will now return ``"unknown-array"`` instead of raising (:issue:`37367`)
 - Bug in constructing a :class:`Series` from a list and a :class:`PandasDtype` (:issue:`39357`)
 - Bug in :class:`Styler` which caused CSS to duplicate on multiple renders. (:issue:`39395`)
 - ``inspect.getmembers(Series)`` no longer raises an ``AbstractMethodError`` (:issue:`38782`)

pandas/_libs/lib.pyx

@@ -69,6 +69,7 @@ from pandas._libs cimport util
 from pandas._libs.util cimport INT64_MAX, INT64_MIN, UINT64_MAX, is_nan
 
 from pandas._libs.tslib import array_to_datetime
+from pandas._libs.tslibs.period import Period
 
 from pandas._libs.missing cimport (
     C_NA,
@@ -1082,6 +1083,7 @@ _TYPE_MAP = {
     "timedelta64[ns]": "timedelta64",
     "m": "timedelta64",
     "interval": "interval",
+    Period: "period",
 }
 
 # types only exist on certain platform
@@ -1233,8 +1235,8 @@ cdef object _try_infer_map(object dtype):
     cdef:
         object val
         str attr
-    for attr in ["name", "kind", "base"]:
-        val = getattr(dtype, attr)
+    for attr in ["name", "kind", "base", "type"]:
+        val = getattr(dtype, attr, None)
         if val in _TYPE_MAP:
             return _TYPE_MAP[val]
     return None
@@ -1275,6 +1277,7 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
     - time
     - period
     - mixed
+    - unknown-array
 
     Raises
     ------
@@ -1287,6 +1290,9 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
       specialized
     - 'mixed-integer-float' are floats and integers
     - 'mixed-integer' are integers mixed with non-integers
+    - 'unknown-array' is the catchall for something that *is* an array (has
+      a dtype attribute), but has a dtype unknown to pandas (e.g. external
+      extension array)
 
     Examples
     --------
@@ -1355,12 +1361,10 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
         # e.g. categoricals
         dtype = value.dtype
         if not isinstance(dtype, np.dtype):
-            value = _try_infer_map(value.dtype)
-            if value is not None:
-                return value
-
-            # its ndarray-like but we can't handle
-            raise ValueError(f"cannot infer type for {type(value)}")
+            inferred = _try_infer_map(value.dtype)
+            if inferred is not None:
+                return inferred
+            return "unknown-array"
 
         # Unwrap Series/Index
         values = np.asarray(value)

pandas/core/array_algos/putmask.py

@@ -1,18 +1,24 @@
 """
 EA-compatible analogue to to np.putmask
 """
-from typing import Any
+from typing import Any, Tuple
 import warnings
 
 import numpy as np
 
 from pandas._libs import lib
 from pandas._typing import ArrayLike
 
-from pandas.core.dtypes.cast import convert_scalar_for_putitemlike, find_common_type
+from pandas.core.dtypes.cast import (
+    convert_scalar_for_putitemlike,
+    find_common_type,
+    infer_dtype_from,
+)
 from pandas.core.dtypes.common import is_float_dtype, is_integer_dtype, is_list_like
 from pandas.core.dtypes.missing import isna_compat
 
+from pandas.core.arrays import ExtensionArray
+
 
 def putmask_inplace(values: ArrayLike, mask: np.ndarray, value: Any) -> None:
     """
@@ -22,7 +28,7 @@ def putmask_inplace(values: ArrayLike, mask: np.ndarray, value: Any) -> None:
     Parameters
     ----------
     mask : np.ndarray[bool]
-        We assume _extract_bool_array has already been called.
+        We assume extract_bool_array has already been called.
     value : Any
     """
 
@@ -152,3 +158,52 @@ def putmask_without_repeat(values: np.ndarray, mask: np.ndarray, new: Any) -> No
             raise ValueError("cannot assign mismatch length to masked array")
     else:
         np.putmask(values, mask, new)
+
+
+def validate_putmask(values: ArrayLike, mask: np.ndarray) -> Tuple[np.ndarray, bool]:
+    """
+    Validate mask and check if this putmask operation is a no-op.
+    """
+    mask = extract_bool_array(mask)
+    if mask.shape != values.shape:
+        raise ValueError("putmask: mask and data must be the same size")
+
+    noop = not mask.any()
+    return mask, noop
+
+
+def extract_bool_array(mask: ArrayLike) -> np.ndarray:
+    """
+    If we have a SparseArray or BooleanArray, convert it to ndarray[bool].
+    """
+    if isinstance(mask, ExtensionArray):
+        # We could have BooleanArray, Sparse[bool], ...
+        #  Except for BooleanArray, this is equivalent to just
+        #  np.asarray(mask, dtype=bool)
+        mask = mask.to_numpy(dtype=bool, na_value=False)
+
+    mask = np.asarray(mask, dtype=bool)
+    return mask
+
+
+def setitem_datetimelike_compat(values: np.ndarray, num_set: int, other):
+    """
+    Parameters
+    ----------
+    values : np.ndarray
+    num_set : int
+        For putmask, this is mask.sum()
+    other : Any
+    """
+    if values.dtype == object:
+        dtype, _ = infer_dtype_from(other, pandas_dtype=True)
+
+        if isinstance(dtype, np.dtype) and dtype.kind in ["m", "M"]:
+            # https://github.com/numpy/numpy/issues/12550
+            #  timedelta64 will incorrectly cast to int
+            if not is_list_like(other):
+                other = [other] * num_set
+            else:
+                other = list(other)
+
+    return other

pandas/core/dtypes/cast.py

@@ -24,7 +24,7 @@
 
 import numpy as np
 
-from pandas._libs import lib, missing as libmissing, tslib
+from pandas._libs import lib, tslib
 from pandas._libs.tslibs import (
     NaT,
     OutOfBoundsDatetime,
@@ -86,7 +86,12 @@
     ABCSeries,
 )
 from pandas.core.dtypes.inference import is_list_like
-from pandas.core.dtypes.missing import is_valid_na_for_dtype, isna, notna
+from pandas.core.dtypes.missing import (
+    is_valid_na_for_dtype,
+    isna,
+    na_value_for_dtype,
+    notna,
+)
 
 if TYPE_CHECKING:
     from pandas import Series
@@ -529,16 +534,26 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan):
         dtype = np.dtype(object)
         return dtype, fill_value
 
+    kinds = ["i", "u", "f", "c", "m", "M"]
+    if is_valid_na_for_dtype(fill_value, dtype) and dtype.kind in kinds:
+        dtype = ensure_dtype_can_hold_na(dtype)
+        fv = na_value_for_dtype(dtype)
+        return dtype, fv
+
+    elif isna(fill_value):
+        dtype = np.dtype(object)
+        if fill_value is None:
+            # but we retain e.g. pd.NA
+            fill_value = np.nan
+        return dtype, fill_value
+
     # returns tuple of (dtype, fill_value)
     if issubclass(dtype.type, np.datetime64):
         if isinstance(fill_value, datetime) and fill_value.tzinfo is not None:
             # Trying to insert tzaware into tznaive, have to cast to object
             dtype = np.dtype(np.object_)
-        elif is_integer(fill_value) or (is_float(fill_value) and not isna(fill_value)):
+        elif is_integer(fill_value) or is_float(fill_value):
             dtype = np.dtype(np.object_)
-        elif is_valid_na_for_dtype(fill_value, dtype):
-            # e.g. pd.NA, which is not accepted by Timestamp constructor
-            fill_value = np.datetime64("NaT", "ns")
         else:
             try:
                 fill_value = Timestamp(fill_value).to_datetime64()
@@ -547,14 +562,11 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan):
     elif issubclass(dtype.type, np.timedelta64):
         if (
             is_integer(fill_value)
-            or (is_float(fill_value) and not np.isnan(fill_value))
+            or is_float(fill_value)
             or isinstance(fill_value, str)
         ):
             # TODO: What about str that can be a timedelta?
             dtype = np.dtype(np.object_)
-        elif is_valid_na_for_dtype(fill_value, dtype):
-            # e.g pd.NA, which is not accepted by the  Timedelta constructor
-            fill_value = np.timedelta64("NaT", "ns")
         else:
             try:
                 fv = Timedelta(fill_value)
@@ -615,17 +627,6 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan):
                 # e.g. mst is np.complex128 and dtype is np.complex64
                 dtype = mst
 
-    elif fill_value is None or fill_value is libmissing.NA:
-        # Note: we already excluded dt64/td64 dtypes above
-        if is_float_dtype(dtype) or is_complex_dtype(dtype):
-            fill_value = np.nan
-        elif is_integer_dtype(dtype):
-            dtype = np.dtype(np.float64)
-            fill_value = np.nan
-        else:
-            dtype = np.dtype(np.object_)
-            if fill_value is not libmissing.NA:
-                fill_value = np.nan
     else:
         dtype = np.dtype(np.object_)
 

pandas/core/dtypes/concat.py

@@ -61,7 +61,7 @@ def _cast_to_common_type(arr: ArrayLike, dtype: DtypeObj) -> ArrayLike:
     return arr.astype(dtype, copy=False)
 
 
-def concat_compat(to_concat, axis: int = 0):
+def concat_compat(to_concat, axis: int = 0, ea_compat_axis: bool = False):
     """
     provide concatenation of an array of arrays each of which is a single
     'normalized' dtypes (in that for example, if it's object, then it is a
@@ -72,6 +72,9 @@ def concat_compat(to_concat, axis: int = 0):
     ----------
     to_concat : array of arrays
     axis : axis to provide concatenation
+    ea_compat_axis : bool, default False
+        For ExtensionArray compat, behave as if axis == 1 when determining
+        whether to drop empty arrays.
 
     Returns
     -------
@@ -91,7 +94,8 @@ def is_nonempty(x) -> bool:
     # marginal given that it would still require shape & dtype calculation and
     # np.concatenate which has them both implemented is compiled.
     non_empties = [x for x in to_concat if is_nonempty(x)]
-    if non_empties and axis == 0:
+    if non_empties and axis == 0 and not ea_compat_axis:
+        # ea_compat_axis see GH#39574
         to_concat = non_empties
 
     kinds = {obj.dtype.kind for obj in to_concat}