Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

doc/source/reference/io.rst

@@ -83,6 +83,11 @@ HDFStore: PyTables (HDF5)
    HDFStore.groups
    HDFStore.walk
 
+.. warning::
+
+   One can store a subclass of ``DataFrame`` or ``Series`` to HDF5,
+   but the type of the subclass is lost upon storing.
+
 Feather
 ~~~~~~~
 .. autosummary::

doc/source/whatsnew/v1.3.0.rst

@@ -176,7 +176,7 @@ Categorical
 Datetimelike
 ^^^^^^^^^^^^
 - Bug in :class:`DataFrame` and :class:`Series` constructors sometimes dropping nanoseconds from :class:`Timestamp` (resp. :class:`Timedelta`) ``data``, with ``dtype=datetime64[ns]`` (resp. ``timedelta64[ns]``) (:issue:`38032`)
--
+- Bug in :meth:`DataFrame.first` and :meth:`Series.first` returning two months for offset one month when first day is last calendar day (:issue:`29623`)
 -
 
 Timedelta
@@ -224,6 +224,7 @@ Indexing
 Missing
 ^^^^^^^
 
+- Bug in :class:`Grouper` now correctly propagates ``dropna`` argument and :meth:`DataFrameGroupBy.transform` now correctly handles missing values for ``dropna=True`` (:issue:`35612`)
 -
 -
 
@@ -241,7 +242,8 @@ I/O
 - Bug in :func:`read_csv` not accepting ``usecols`` with different length than ``names`` for ``engine="python"`` (:issue:`16469`)
 - Bug in :func:`read_csv` raising ``TypeError`` when ``names`` and ``parse_dates`` is specified for ``engine="c"`` (:issue:`33699`)
 - Allow custom error values for parse_dates argument of :func:`read_sql`, :func:`read_sql_query` and :func:`read_sql_table` (:issue:`35185`)
--
+- Bug in :func:`to_hdf` raising ``KeyError`` when trying to apply
+  for subclasses of ``DataFrame`` or ``Series`` (:issue:`33748`).
 
 Period
 ^^^^^^

environment.yml

@@ -31,7 +31,7 @@ dependencies:
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
   - gitdb
-  - sphinx
+  - sphinx=3.3.1
 
   # documentation (jupyter notebooks)
   - nbconvert>=5.4.1

pandas/_libs/tslibs/timedeltas.pyx

@@ -1091,11 +1091,9 @@ cdef class _Timedelta(timedelta):
         >>> td.isoformat()
         'P6DT0H50M3.010010012S'
         >>> pd.Timedelta(hours=1, seconds=10).isoformat()
-        'P0DT0H0M10S'
-        >>> pd.Timedelta(hours=1, seconds=10).isoformat()
-        'P0DT0H0M10S'
+        'P0DT1H0M10S'
         >>> pd.Timedelta(days=500.5).isoformat()
-        'P500DT12H0MS'
+        'P500DT12H0M0S'
         """
         components = self.components
         seconds = (f'{components.seconds}.'

pandas/_testing.py

@@ -10,7 +10,17 @@
 from shutil import rmtree
 import string
 import tempfile
-from typing import Any, Callable, ContextManager, List, Optional, Type, Union, cast
+from typing import (
+    Any,
+    Callable,
+    ContextManager,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 import warnings
 import zipfile
 
@@ -301,35 +311,25 @@ def write_to_compressed(compression, path, data, dest="test"):
     ------
     ValueError : An invalid compression value was passed in.
     """
+    args: Tuple[Any, ...] = (data,)
+    mode = "wb"
+    method = "write"
+    compress_method: Callable
+
     if compression == "zip":
         compress_method = zipfile.ZipFile
+        mode = "w"
+        args = (dest, data)
+        method = "writestr"
     elif compression == "gzip":
-        # pandas\_testing.py:288: error: Incompatible types in assignment
-        # (expression has type "Type[GzipFile]", variable has type
-        # "Type[ZipFile]")
-        compress_method = gzip.GzipFile  # type: ignore[assignment]
+        compress_method = gzip.GzipFile
     elif compression == "bz2":
-        # pandas\_testing.py:290: error: Incompatible types in assignment
-        # (expression has type "Type[BZ2File]", variable has type
-        # "Type[ZipFile]")
-        compress_method = bz2.BZ2File  # type: ignore[assignment]
+        compress_method = bz2.BZ2File
     elif compression == "xz":
         compress_method = get_lzma_file(lzma)
     else:
         raise ValueError(f"Unrecognized compression type: {compression}")
 
-    if compression == "zip":
-        mode = "w"
-        args = (dest, data)
-        method = "writestr"
-    else:
-        mode = "wb"
-        # pandas\_testing.py:302: error: Incompatible types in assignment
-        # (expression has type "Tuple[Any]", variable has type "Tuple[Any,
-        # Any]")
-        args = (data,)  # type: ignore[assignment]
-        method = "write"
-
     with compress_method(path, mode=mode) as f:
         getattr(f, method)(*args)
 

pandas/_typing.py

@@ -133,6 +133,8 @@
     "Resampler",
 ]
 
+PythonFuncType = Callable[[Any], Any]
+
 # filenames and file-like-objects
 Buffer = Union[IO[AnyStr], RawIOBase, BufferedIOBase, TextIOBase, TextIOWrapper, mmap]
 FileOrBuffer = Union[str, Buffer[T]]

pandas/core/apply.py

@@ -6,7 +6,7 @@
 
 from pandas._config import option_context
 
-from pandas._typing import Axis, FrameOrSeriesUnion
+from pandas._typing import AggFuncType, Axis, FrameOrSeriesUnion
 from pandas.util._decorators import cache_readonly
 
 from pandas.core.dtypes.common import (
@@ -27,7 +27,7 @@
 
 def frame_apply(
     obj: "DataFrame",
-    func,
+    func: AggFuncType,
     axis: Axis = 0,
     raw: bool = False,
     result_type: Optional[str] = None,

pandas/core/construction.py

@@ -449,6 +449,11 @@ def sanitize_array(
     # extract ndarray or ExtensionArray, ensure we have no PandasArray
     data = extract_array(data, extract_numpy=True)
 
+    if isinstance(data, np.ndarray) and data.ndim == 0:
+        if dtype is None:
+            dtype = data.dtype
+        data = lib.item_from_zerodim(data)
+
     # GH#846
     if isinstance(data, np.ndarray):
 
@@ -462,7 +467,7 @@ def sanitize_array(
                 else:
                     subarr = np.array(data, copy=False)
         else:
-            # we will try to copy be-definition here
+            # we will try to copy by-definition here
             subarr = _try_cast(data, dtype, copy, raise_cast_failure)
 
     elif isinstance(data, ABCExtensionArray):
@@ -491,46 +496,19 @@ def sanitize_array(
         # GH#16804
         arr = np.arange(data.start, data.stop, data.step, dtype="int64")
         subarr = _try_cast(arr, dtype, copy, raise_cast_failure)
-    elif lib.is_scalar(data) and index is not None and dtype is not None:
+
+    elif not is_list_like(data):
+        if index is None:
+            raise ValueError("index must be specified when data is not list-like")
         subarr = construct_1d_arraylike_from_scalar(data, len(index), dtype)
+
     else:
         subarr = _try_cast(data, dtype, copy, raise_cast_failure)
 
-    # scalar like, GH
-    if getattr(subarr, "ndim", 0) == 0:
-        if isinstance(data, list):  # pragma: no cover
-            subarr = np.array(data, dtype=object)
-        elif index is not None:
-            subarr = construct_1d_arraylike_from_scalar(data, len(index), dtype)
-
-        else:
-            return subarr.item()
-
-    # the result that we want
-    elif subarr.ndim == 1:
-        if index is not None:
-
-            # a 1-element ndarray
-            if len(subarr) != len(index) and len(subarr) == 1:
-                subarr = subarr.repeat(len(index))
-
-    elif subarr.ndim > 1:
-        if isinstance(data, np.ndarray):
-            raise ValueError("Data must be 1-dimensional")
-        else:
-            subarr = com.asarray_tuplesafe(data, dtype=dtype)
+    subarr = _sanitize_ndim(subarr, data, dtype, index)
 
     if not (is_extension_array_dtype(subarr.dtype) or is_extension_array_dtype(dtype)):
-        # This is to prevent mixed-type Series getting all casted to
-        # NumPy string type, e.g. NaN --> '-1#IND'.
-        if issubclass(subarr.dtype.type, str):
-            # GH#16605
-            # If not empty convert the data to dtype
-            # GH#19853: If data is a scalar, subarr has already the result
-            if not lib.is_scalar(data):
-                if not np.all(isna(data)):
-                    data = np.array(data, dtype=dtype, copy=False)
-                subarr = np.array(data, dtype=object, copy=copy)
+        subarr = _sanitize_str_dtypes(subarr, data, dtype, copy)
 
         is_object_or_str_dtype = is_object_dtype(dtype) or is_string_dtype(dtype)
         if is_object_dtype(subarr.dtype) and not is_object_or_str_dtype:
@@ -541,13 +519,65 @@ def sanitize_array(
     return subarr
 
 
+def _sanitize_ndim(
+    result: ArrayLike, data, dtype: Optional[DtypeObj], index: Optional[Index]
+) -> ArrayLike:
+    """
+    Ensure we have a 1-dimensional result array.
+    """
+    if getattr(result, "ndim", 0) == 0:
+        raise ValueError("result should be arraylike with ndim > 0")
+
+    elif result.ndim == 1:
+        # the result that we want
+        result = _maybe_repeat(result, index)
+
+    elif result.ndim > 1:
+        if isinstance(data, np.ndarray):
+            raise ValueError("Data must be 1-dimensional")
+        else:
+            result = com.asarray_tuplesafe(data, dtype=dtype)
+    return result
+
+
+def _sanitize_str_dtypes(
+    result: np.ndarray, data, dtype: Optional[DtypeObj], copy: bool
+) -> np.ndarray:
+    """
+    Ensure we have a dtype that is supported by pandas.
+    """
+
+    # This is to prevent mixed-type Series getting all casted to
+    # NumPy string type, e.g. NaN --> '-1#IND'.
+    if issubclass(result.dtype.type, str):
+        # GH#16605
+        # If not empty convert the data to dtype
+        # GH#19853: If data is a scalar, result has already the result
+        if not lib.is_scalar(data):
+            if not np.all(isna(data)):
+                data = np.array(data, dtype=dtype, copy=False)
+            result = np.array(data, dtype=object, copy=copy)
+    return result
+
+
+def _maybe_repeat(arr: ArrayLike, index: Optional[Index]) -> ArrayLike:
+    """
+    If we have a length-1 array and an index describing how long we expect
+    the result to be, repeat the array.
+    """
+    if index is not None:
+        if 1 == len(arr) != len(index):
+            arr = arr.repeat(len(index))
+    return arr
+
+
 def _try_cast(arr, dtype: Optional[DtypeObj], copy: bool, raise_cast_failure: bool):
     """
     Convert input to numpy ndarray and optionally cast to a given dtype.
 
     Parameters
     ----------
-    arr : ndarray, scalar, list, tuple, iterator (catchall)
+    arr : ndarray, list, tuple, iterator (catchall)
         Excludes: ExtensionArray, Series, Index.
     dtype : np.dtype, ExtensionDtype or None
     copy : bool

pandas/core/dtypes/cast.py

@@ -1543,7 +1543,10 @@ def construct_1d_arraylike_from_scalar(
     """
 
     if dtype is None:
-        dtype, value = infer_dtype_from_scalar(value, pandas_dtype=True)
+        try:
+            dtype, value = infer_dtype_from_scalar(value, pandas_dtype=True)
+        except OutOfBoundsDatetime:
+            dtype = np.dtype(object)
 
     if is_extension_array_dtype(dtype):
         cls = dtype.construct_array_type()

pandas/core/frame.py

@@ -62,6 +62,7 @@
     IndexLabel,
     Label,
     Level,
+    PythonFuncType,
     Renamer,
     StorageOptions,
     Suffixes,
@@ -7661,7 +7662,13 @@ def transform(
         return result
 
     def apply(
-        self, func, axis: Axis = 0, raw: bool = False, result_type=None, args=(), **kwds
+        self,
+        func: AggFuncType,
+        axis: Axis = 0,
+        raw: bool = False,
+        result_type=None,
+        args=(),
+        **kwds,
     ):
         """
         Apply a function along an axis of the DataFrame.
@@ -7807,7 +7814,9 @@ def apply(
         )
         return op.get_result()
 
-    def applymap(self, func, na_action: Optional[str] = None) -> DataFrame:
+    def applymap(
+        self, func: PythonFuncType, na_action: Optional[str] = None
+    ) -> DataFrame:
         """
         Apply a function to a Dataframe elementwise.
 

pandas/core/generic.py

@@ -2505,6 +2505,11 @@ def to_hdf(
         In order to add another DataFrame or Series to an existing HDF file
         please use append mode and a different a key.
 
+        .. warning::
+
+           One can store a subclass of ``DataFrame`` or ``Series`` to HDF5,
+           but the type of the subclass is lost upon storing.
+
         For more information see the :ref:`user guide <io.hdf5>`.
 
         Parameters
@@ -8424,7 +8429,12 @@ def first(self: FrameOrSeries, offset) -> FrameOrSeries:
             return self
 
         offset = to_offset(offset)
-        end_date = end = self.index[0] + offset
+        if not isinstance(offset, Tick) and offset.is_on_offset(self.index[0]):
+            # GH#29623 if first value is end of period, remove offset with n = 1
+            #  before adding the real offset
+            end_date = end = self.index[0] - offset.base + offset
+        else:
+            end_date = end = self.index[0] + offset
 
         # Tick-like, e.g. 3 weeks
         if isinstance(offset, Tick):

pandas/core/groupby/generic.py

@@ -553,7 +553,6 @@ def _transform_general(self, func, *args, **kwargs):
                 result = maybe_downcast_numeric(result, self._selected_obj.dtype)
 
         result.name = self._selected_obj.name
-        result.index = self._selected_obj.index
         return result
 
     def _transform_fast(self, result) -> Series: