Merge branch 'main' into issue-50977

Author: kostyafarber

.pre-commit-config.yaml

@@ -135,7 +135,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].276
+        - [email protected].284
     -   id: pyright_reportGeneralTypeIssues
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues

ci/code_checks.sh

@@ -187,7 +187,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.show_versions \
         pandas.test \
         pandas.NaT \
-        pandas.Timestamp.unit \
         pandas.Timestamp.as_unit \
         pandas.Timestamp.ctime \
         pandas.Timestamp.date \
@@ -579,13 +578,11 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
     MSG='Partially validate docstrings (EX02)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX02 --ignore_functions \
-        pandas.DataFrame.copy \
         pandas.DataFrame.plot.line \
         pandas.DataFrame.std \
         pandas.DataFrame.var \
         pandas.Index.factorize \
         pandas.Period.strftime \
-        pandas.Series.copy \
         pandas.Series.factorize \
         pandas.Series.floordiv \
         pandas.Series.plot.line \

doc/source/development/internals.rst

@@ -15,24 +15,24 @@ Indexing
 In pandas there are a few objects implemented which can serve as valid
 containers for the axis labels:
 
-* ``Index``: the generic "ordered set" object, an ndarray of object dtype
+* :class:`Index`: the generic "ordered set" object, an ndarray of object dtype
   assuming nothing about its contents. The labels must be hashable (and
   likely immutable) and unique. Populates a dict of label to location in
   Cython to do ``O(1)`` lookups.
 * ``Int64Index``: a version of ``Index`` highly optimized for 64-bit integer
   data, such as time stamps
 * ``Float64Index``: a version of ``Index`` highly optimized for 64-bit float data
-* ``MultiIndex``: the standard hierarchical index object
-* ``DatetimeIndex``: An Index object with ``Timestamp`` boxed elements (impl are the int64 values)
-* ``TimedeltaIndex``: An Index object with ``Timedelta`` boxed elements (impl are the in64 values)
-* ``PeriodIndex``: An Index object with Period elements
+* :class:`MultiIndex`: the standard hierarchical index object
+* :class:`DatetimeIndex`: An Index object with :class:`Timestamp` boxed elements (impl are the int64 values)
+* :class:`TimedeltaIndex`: An Index object with :class:`Timedelta` boxed elements (impl are the in64 values)
+* :class:`PeriodIndex`: An Index object with Period elements
 
 There are functions that make the creation of a regular index easy:
 
-* ``date_range``: fixed frequency date range generated from a time rule or
+* :func:`date_range`: fixed frequency date range generated from a time rule or
   DateOffset. An ndarray of Python datetime objects
-* ``period_range``: fixed frequency date range generated from a time rule or
-  DateOffset. An ndarray of ``Period`` objects, representing timespans
+* :func:`period_range`: fixed frequency date range generated from a time rule or
+  DateOffset. An ndarray of :class:`Period` objects, representing timespans
 
 The motivation for having an ``Index`` class in the first place was to enable
 different implementations of indexing. This means that it's possible for you,
@@ -43,28 +43,28 @@ From an internal implementation point of view, the relevant methods that an
 ``Index`` must define are one or more of the following (depending on how
 incompatible the new object internals are with the ``Index`` functions):
 
-* ``get_loc``: returns an "indexer" (an integer, or in some cases a
+* :meth:`~Index.get_loc`: returns an "indexer" (an integer, or in some cases a
   slice object) for a label
-* ``slice_locs``: returns the "range" to slice between two labels
-* ``get_indexer``: Computes the indexing vector for reindexing / data
+* :meth:`~Index.slice_locs`: returns the "range" to slice between two labels
+* :meth:`~Index.get_indexer`: Computes the indexing vector for reindexing / data
   alignment purposes. See the source / docstrings for more on this
-* ``get_indexer_non_unique``: Computes the indexing vector for reindexing / data
+* :meth:`~Index.get_indexer_non_unique`: Computes the indexing vector for reindexing / data
   alignment purposes when the index is non-unique. See the source / docstrings
   for more on this
-* ``reindex``: Does any pre-conversion of the input index then calls
+* :meth:`~Index.reindex`: Does any pre-conversion of the input index then calls
   ``get_indexer``
-* ``union``, ``intersection``: computes the union or intersection of two
+* :meth:`~Index.union`, :meth:`~Index.intersection`: computes the union or intersection of two
   Index objects
-* ``insert``: Inserts a new label into an Index, yielding a new object
-* ``delete``: Delete a label, yielding a new object
-* ``drop``: Deletes a set of labels
-* ``take``: Analogous to ndarray.take
+* :meth:`~Index.insert`: Inserts a new label into an Index, yielding a new object
+* :meth:`~Index.delete`: Delete a label, yielding a new object
+* :meth:`~Index.drop`: Deletes a set of labels
+* :meth:`~Index.take`: Analogous to ndarray.take
 
 MultiIndex
 ~~~~~~~~~~
 
-Internally, the ``MultiIndex`` consists of a few things: the **levels**, the
-integer **codes** (until version 0.24 named *labels*), and the level **names**:
+Internally, the :class:`MultiIndex` consists of a few things: the **levels**, the
+integer **codes**, and the level **names**:
 
 .. ipython:: python
 
@@ -80,13 +80,13 @@ You can probably guess that the codes determine which unique element is
 identified with that location at each layer of the index. It's important to
 note that sortedness is determined **solely** from the integer codes and does
 not check (or care) whether the levels themselves are sorted. Fortunately, the
-constructors ``from_tuples`` and ``from_arrays`` ensure that this is true, but
-if you compute the levels and codes yourself, please be careful.
+constructors :meth:`~MultiIndex.from_tuples` and :meth:`~MultiIndex.from_arrays` ensure
+that this is true, but if you compute the levels and codes yourself, please be careful.
 
 Values
 ~~~~~~
 
-pandas extends NumPy's type system with custom types, like ``Categorical`` or
+pandas extends NumPy's type system with custom types, like :class:`Categorical` or
 datetimes with a timezone, so we have multiple notions of "values". For 1-D
 containers (``Index`` classes and ``Series``) we have the following convention:
 

doc/source/user_guide/indexing.rst

@@ -231,7 +231,7 @@ You can also assign a ``dict`` to a row of a ``DataFrame``:
 
 You can use attribute access to modify an existing element of a Series or column of a DataFrame, but be careful;
 if you try to use attribute access to create a new column, it creates a new attribute rather than a
-new column. In 0.21.0 and later, this will raise a ``UserWarning``:
+new column and will this raise a ``UserWarning``:
 
 .. code-block:: ipython
 

doc/source/whatsnew/v2.0.0.rst

@@ -69,6 +69,9 @@ to select the nullable dtypes implementation.
 * :func:`read_html`
 * :func:`read_xml`
 * :func:`read_json`
+* :func:`read_sql`
+* :func:`read_sql_query`
+* :func:`read_sql_table`
 * :func:`read_parquet`
 * :func:`read_orc`
 * :func:`read_feather`
@@ -330,6 +333,36 @@ a supported dtype:
 
    pd.Series(["2016-01-01"], dtype="datetime64[D]")
 
+.. _whatsnew_200.api_breaking.value_counts:
+
+Value counts sets the resulting name to ``count``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+In past versions, when running :meth:`Series.value_counts`, the result would inherit
+the original object's name, and the result index would be nameless. This would cause
+confusion when resetting the index, and the column names would not correspond with the
+column values.
+Now, the result name will be ``'count'`` (or ``'proportion'`` if ``normalize=True`` was passed),
+and the index will be named after the original object (:issue:`49497`).
+
+*Previous behavior*:
+
+.. code-block:: ipython
+
+    In [8]: pd.Series(['quetzal', 'quetzal', 'elk'], name='animal').value_counts()
+
+    Out[2]:
+    quetzal    2
+    elk        1
+    Name: animal, dtype: int64
+
+*New behavior*:
+
+.. ipython:: python
+
+    pd.Series(['quetzal', 'quetzal', 'elk'], name='animal').value_counts()
+
+Likewise for other ``value_counts`` methods (for example, :meth:`DataFrame.value_counts`).
+
 .. _whatsnew_200.api_breaking.astype_to_unsupported_datetimelike:
 
 Disallow astype conversion to non-supported datetime64/timedelta64 dtypes
@@ -632,6 +665,7 @@ Other API changes
 
 Deprecations
 ~~~~~~~~~~~~
+- Deprecated parsing datetime strings with system-local timezone to ``tzlocal``, pass a ``tz`` keyword or explicitly call ``tz_localize`` instead (:issue:`50791`)
 - Deprecated argument ``infer_datetime_format`` in :func:`to_datetime` and :func:`read_csv`, as a strict version of it is now the default (:issue:`48621`)
 - Deprecated behavior of :func:`to_datetime` with ``unit`` when parsing strings, in a future version these will be parsed as datetimes (matching unit-less behavior) instead of cast to floats. To retain the old behavior, cast strings to numeric types before calling :func:`to_datetime` (:issue:`50735`)
 - Deprecated :func:`pandas.io.sql.execute` (:issue:`50185`)
@@ -959,6 +993,7 @@ Categorical
 - Bug in :meth:`Series.replace` with categorical dtype losing nullable dtypes of underlying categories (:issue:`49404`)
 - Bug in :meth:`DataFrame.groupby` and :meth:`Series.groupby` would reorder categories when used as a grouper (:issue:`48749`)
 - Bug in :class:`Categorical` constructor when constructing from a :class:`Categorical` object and ``dtype="category"`` losing ordered-ness (:issue:`49309`)
+- Bug in :meth:`SeriesGroupBy.min`, :meth:`SeriesGroupBy.max`, :meth:`DataFrameGroupBy.min`, and :meth:`DataFrameGroupBy.max` with unordered :class:`CategoricalDtype` with no groups failing to raise ``TypeError`` (:issue:`51034`)
 -
 
 Datetimelike

pandas/_libs/groupby.pyx

@@ -289,10 +289,8 @@ def group_cumprod(
 
                 if uses_mask:
                     isna_entry = mask[i, j]
-                elif int64float_t is float64_t or int64float_t is float32_t:
-                    isna_entry = val != val
                 else:
-                    isna_entry = False
+                    isna_entry = _treat_as_na(val, False)
 
                 if not isna_entry:
                     isna_prev = accum_mask[lab, j]
@@ -737,23 +735,10 @@ def group_sum(
                 for j in range(K):
                     val = values[i, j]
 
-                    # not nan
-                    # With dt64/td64 values, values have been cast to float64
-                    #  instead if int64 for group_sum, but the logic
-                    #  is otherwise the same as in _treat_as_na
                     if uses_mask:
                         isna_entry = mask[i, j]
-                    elif (
-                        sum_t is float32_t
-                        or sum_t is float64_t
-                        or sum_t is complex64_t
-                    ):
-                        # avoid warnings because of equality comparison
-                        isna_entry = not val == val
-                    elif sum_t is int64_t and is_datetimelike and val == NPY_NAT:
-                        isna_entry = True
                     else:
-                        isna_entry = False
+                        isna_entry = _treat_as_na(val, is_datetimelike)
 
                     if not isna_entry:
                         nobs[lab, j] += 1
@@ -831,10 +816,8 @@ def group_prod(
 
                 if uses_mask:
                     isna_entry = mask[i, j]
-                elif int64float_t is float32_t or int64float_t is float64_t:
-                    isna_entry = not val == val
                 else:
-                    isna_entry = False
+                    isna_entry = _treat_as_na(val, False)
 
                 if not isna_entry:
                     nobs[lab, j] += 1
@@ -906,7 +889,7 @@ def group_var(
                 if uses_mask:
                     isna_entry = mask[i, j]
                 else:
-                    isna_entry = not val == val
+                    isna_entry = _treat_as_na(val, False)
 
                 if not isna_entry:
                     nobs[lab, j] += 1
@@ -1008,9 +991,12 @@ def group_mean(
                 if uses_mask:
                     isna_entry = mask[i, j]
                 elif is_datetimelike:
+                    # With group_mean, we cannot just use _treat_as_na bc
+                    #  datetimelike dtypes get cast to float64 instead of
+                    #  to int64.
                     isna_entry = val == NPY_NAT
                 else:
-                    isna_entry = not val == val
+                    isna_entry = _treat_as_na(val, is_datetimelike)
 
                 if not isna_entry:
                     nobs[lab, j] += 1
@@ -1086,10 +1072,8 @@ def group_ohlc(
 
             if uses_mask:
                 isna_entry = mask[i, 0]
-            elif int64float_t is float32_t or int64float_t is float64_t:
-                isna_entry = val != val
             else:
-                isna_entry = False
+                isna_entry = _treat_as_na(val, False)
 
             if isna_entry:
                 continue
@@ -1231,15 +1215,26 @@ def group_quantile(
 # group_nth, group_last, group_rank
 # ----------------------------------------------------------------------
 
-cdef bint _treat_as_na(numeric_object_t val, bint is_datetimelike) nogil:
-    if numeric_object_t is object:
+ctypedef fused numeric_object_complex_t:
+    numeric_object_t
+    complex64_t
+    complex128_t
+
+
+cdef bint _treat_as_na(numeric_object_complex_t val, bint is_datetimelike) nogil:
+    if numeric_object_complex_t is object:
         # Should never be used, but we need to avoid the `val != val` below
         #  or else cython will raise about gil acquisition.
         raise NotImplementedError
 
-    elif numeric_object_t is int64_t:
+    elif numeric_object_complex_t is int64_t:
         return is_datetimelike and val == NPY_NAT
-    elif numeric_object_t is float32_t or numeric_object_t is float64_t:
+    elif (
+        numeric_object_complex_t is float32_t
+        or numeric_object_complex_t is float64_t
+        or numeric_object_complex_t is complex64_t
+        or numeric_object_complex_t is complex128_t
+    ):
         return val != val
     else:
         # non-datetimelike integer

pandas/_libs/internals.pyi

@@ -44,7 +44,9 @@ class BlockPlacement:
     @property
     def is_slice_like(self) -> bool: ...
     @overload
-    def __getitem__(self, loc: slice | Sequence[int]) -> BlockPlacement: ...
+    def __getitem__(
+        self, loc: slice | Sequence[int] | npt.NDArray[np.intp]
+    ) -> BlockPlacement: ...
     @overload
     def __getitem__(self, loc: int) -> int: ...
     def __iter__(self) -> Iterator[int]: ...

pandas/_libs/tslibs/conversion.pyx

@@ -50,28 +50,26 @@ from pandas._libs.tslibs.np_datetime cimport (
 
 from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
 
-from pandas._libs.tslibs.timezones cimport (
-    get_utcoffset,
-    is_utc,
-)
-from pandas._libs.tslibs.util cimport (
-    is_datetime64_object,
-    is_float_object,
-    is_integer_object,
-)
-
-from pandas._libs.tslibs.parsing import parse_datetime_string
-
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
     c_NaT as NaT,
     c_nat_strings as nat_strings,
 )
+from pandas._libs.tslibs.parsing cimport parse_datetime_string
 from pandas._libs.tslibs.timestamps cimport _Timestamp
+from pandas._libs.tslibs.timezones cimport (
+    get_utcoffset,
+    is_utc,
+)
 from pandas._libs.tslibs.tzconversion cimport (
     Localizer,
     tz_localize_to_utc_single,
 )
+from pandas._libs.tslibs.util cimport (
+    is_datetime64_object,
+    is_float_object,
+    is_integer_object,
+)
 
 # ----------------------------------------------------------------------
 # Constants
@@ -550,8 +548,10 @@ cdef _TSObject convert_str_to_tsobject(str ts, tzinfo tz, str unit,
                 return obj
 
         dt = parse_datetime_string(
-            ts, dayfirst=dayfirst, yearfirst=yearfirst
+            ts, dayfirst=dayfirst, yearfirst=yearfirst, out_bestunit=&out_bestunit
         )
+        reso = get_supported_reso(out_bestunit)
+        return convert_datetime_to_tsobject(dt, tz, nanos=0, reso=reso)
 
     return convert_datetime_to_tsobject(dt, tz)
 

pandas/_libs/tslibs/parsing.pxd

@@ -1,3 +1,14 @@
+from cpython.datetime cimport datetime
+
+from pandas._libs.tslibs.np_datetime cimport NPY_DATETIMEUNIT
+
 
 cpdef str get_rule_month(str source)
 cpdef quarter_to_myear(int year, int quarter, str freq)
+
+cdef datetime parse_datetime_string(
+    str date_string,
+    bint dayfirst,
+    bint yearfirst,
+    NPY_DATETIMEUNIT* out_bestunit
+)

pandas/_libs/tslibs/parsing.pyi

@@ -6,7 +6,7 @@ from pandas._typing import npt
 
 class DateParseError(ValueError): ...
 
-def parse_datetime_string(
+def py_parse_datetime_string(
     date_string: str,
     dayfirst: bool = ...,
     yearfirst: bool = ...,