Merge remote-tracking branch 'upstream/main' into clean/environment_yml

Author: mroeschke

.github/actions/setup-conda/action.yml

@@ -3,6 +3,12 @@ inputs:
   environment-file:
     description: Conda environment file to use.
     default: environment.yml
+  environment-name:
+    description: Name to use for the Conda environment
+    default: test
+  python-version:
+    description: Python version to install
+    required: false
   pyarrow-version:
     description: If set, overrides the PyArrow version in the Conda environment to the given string.
     required: false
@@ -18,10 +24,14 @@ runs:
       if: ${{ inputs.pyarrow-version }}
 
     - name: Install ${{ inputs.environment-file }}
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v2.1.1
       with:
         environment-file: ${{ inputs.environment-file }}
+        activate-environment: ${{ inputs.environment-name }}
+        python-version: ${{ inputs.python-version }}
         channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
         channels: conda-forge
-        mamba-version: "0.23"
+        mamba-version: "0.24"
         use-mamba: true
+        use-only-tar-bz2: true
+        condarc-file: ci/condarc.yml

.github/workflows/asv-bot.yml

@@ -41,12 +41,8 @@ jobs:
 
         # Although asv sets up its own env, deps are still needed
         # during discovery process
-      - uses: conda-incubator/[email protected]
-        with:
-          activate-environment: pandas-dev
-          channel-priority: strict
-          environment-file: ${{ env.ENV_FILE }}
-          use-only-tar-bz2: true
+      - name: Set up Conda
+        uses: ./.github/actions/setup-conda
 
       - name: Run benchmarks
         id: bench

.github/workflows/code-checks.yml

@@ -58,14 +58,8 @@ jobs:
         path: ~/conda_pkgs_dir
         key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - uses: conda-incubator/[email protected]
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
 
     - name: Build Pandas
       id: build
@@ -127,14 +121,8 @@ jobs:
         path: ~/conda_pkgs_dir
         key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - uses: conda-incubator/[email protected]
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
 
     - name: Build Pandas
       id: build

.github/workflows/sdist.yml

@@ -59,11 +59,12 @@ jobs:
         name: ${{matrix.python-version}}-sdist.gz
         path: dist/*.gz
 
-    - uses: conda-incubator/[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        activate-environment: pandas-sdist
-        channels: conda-forge
-        python-version: '${{ matrix.python-version }}'
+        environment-file: ""
+        environment-name: pandas-sdist
+        python-version: ${{ matrix.python-version }}
 
     - name: Install pandas from sdist
       run: |

.github/workflows/ubuntu.yml

@@ -147,18 +147,11 @@ jobs:
       # xsel for clipboard tests
       run: sudo apt-get update && sudo apt-get install -y xsel ${{ env.EXTRA_APT }}
 
-    - uses: conda-incubator/[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: flexible
         environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
-
-    - name: Upgrade Arrow version
-      run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }}
-      if: ${{ matrix.pyarrow_version }}
+        pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas

ci/condarc.yml

@@ -0,0 +1,32 @@
+# https://docs.conda.io/projects/conda/en/latest/configuration.html
+
+# always_yes (NoneType, bool)
+#   aliases: yes
+#   Automatically choose the 'yes' option whenever asked to proceed with a
+#   conda operation, such as when running `conda install`.
+#
+always_yes: true
+
+# remote_connect_timeout_secs (float)
+#   The number seconds conda will wait for your client to establish a
+#   connection to a remote url resource.
+#
+remote_connect_timeout_secs: 30.0
+
+# remote_max_retries (int)
+#   The maximum number of retries each HTTP connection should attempt.
+#
+remote_max_retries: 10
+
+# remote_backoff_factor (int)
+#   The factor determines the time HTTP connection should wait for
+#   attempt.
+#
+remote_backoff_factor: 3
+
+# remote_read_timeout_secs (float)
+#   Once conda has connected to a remote resource and sent an HTTP
+#   request, the read timeout is the number of seconds conda will wait for
+#   the server to send a response.
+#
+remote_read_timeout_secs: 60.0

doc/source/development/extending.rst

@@ -74,10 +74,11 @@ applies only to certain dtypes.
 Extension types
 ---------------
 
-.. warning::
+.. note::
 
-   The :class:`pandas.api.extensions.ExtensionDtype` and :class:`pandas.api.extensions.ExtensionArray` APIs are new and
-   experimental. They may change between versions without warning.
+   The :class:`pandas.api.extensions.ExtensionDtype` and :class:`pandas.api.extensions.ExtensionArray` APIs were
+   experimental prior to pandas 1.5. Starting with version 1.5, future changes will follow
+   the :ref:`pandas deprecation policy <policies.version>`.
 
 pandas defines an interface for implementing data types and arrays that *extend*
 NumPy's type system. pandas itself uses the extension system for some types

doc/source/user_guide/groupby.rst

@@ -761,7 +761,7 @@ different dtypes, then a common dtype will be determined in the same way as ``Da
 Transformation
 --------------
 
-The ``transform`` method returns an object that is indexed the same (same size)
+The ``transform`` method returns an object that is indexed the same
 as the one being grouped. The transform function must:
 
 * Return a result that is either the same size as the group chunk or
@@ -776,6 +776,14 @@ as the one being grouped. The transform function must:
 * (Optionally) operates on the entire group chunk. If this is supported, a
   fast path is used starting from the *second* chunk.
 
+.. deprecated:: 1.5.0
+
+    When using ``.transform`` on a grouped DataFrame and the transformation function
+    returns a DataFrame, currently pandas does not align the result's index
+    with the input's index. This behavior is deprecated and alignment will
+    be performed in a future version of pandas. You can apply ``.to_numpy()`` to the
+    result of the transformation function to avoid alignment.
+
 Similar to :ref:`groupby.aggregate.udfs`, the resulting dtype will reflect that of the
 transformation function. If the results from different groups have different dtypes, then
 a common dtype will be determined in the same way as ``DataFrame`` construction.

doc/source/whatsnew/v1.5.0.rst

@@ -133,6 +133,7 @@ Other enhancements
 - :meth:`MultiIndex.to_frame` now supports the argument ``allow_duplicates`` and raises on duplicate labels if it is missing or False (:issue:`45245`)
 - :class:`StringArray` now accepts array-likes containing nan-likes (``None``, ``np.nan``) for the ``values`` parameter in its constructor in addition to strings and :attr:`pandas.NA`. (:issue:`40839`)
 - Improved the rendering of ``categories`` in :class:`CategoricalIndex` (:issue:`45218`)
+- :meth:`DataFrame.plot` will now allow the ``subplots`` parameter to be a list of iterables specifying column groups, so that columns may be grouped together in the same subplot (:issue:`29688`).
 - :meth:`to_numeric` now preserves float64 arrays when downcasting would generate values not representable in float32 (:issue:`43693`)
 - :meth:`Series.reset_index` and :meth:`DataFrame.reset_index` now support the argument ``allow_duplicates`` (:issue:`44410`)
 - :meth:`.GroupBy.min` and :meth:`.GroupBy.max` now supports `Numba <https://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`45428`)
@@ -689,6 +690,8 @@ Other Deprecations
 - Deprecated the ``closed`` argument in :class:`intervaltree` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`ArrowInterval` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated allowing ``unit="M"`` or ``unit="Y"`` in :class:`Timestamp` constructor with a non-round float value (:issue:`47267`)
+- Deprecated the ``display.column_space`` global configuration option (:issue:`7576`)
+- Deprecated :meth:`DataFrameGroupBy.transform` not aligning the result when the UDF returned DataFrame (:issue:`45648`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -736,6 +739,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.tz_localize` localizing to UTC failing to make a copy of the underlying data (:issue:`46460`)
 - Bug in :meth:`DatetimeIndex.resolution` incorrectly returning "day" instead of "nanosecond" for nanosecond-resolution indexes (:issue:`46903`)
 - Bug in :class:`Timestamp` with an integer or float value and ``unit="Y"`` or ``unit="M"`` giving slightly-wrong results (:issue:`47266`)
+- Bug in :class:`DatetimeArray` construction when passed another :class:`DatetimeArray` and ``freq=None`` incorrectly inferring the freq from the given array (:issue:`47296`)
 -
 
 Timedelta
@@ -842,8 +846,10 @@ I/O
 - Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements (:issue:`45598`)
 - Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)
 - :meth:`to_html` now excludes the ``border`` attribute from ``<table>`` elements when ``border`` keyword is set to ``False``.
+- Bug in :func:`read_sas` with certain types of compressed SAS7BDAT files (:issue:`35545`)
 - Bug in :func:`read_sas` returned ``None`` rather than an empty DataFrame for SAS7BDAT files with zero rows (:issue:`18198`)
 - Bug in :class:`StataWriter` where value labels were always written with default encoding (:issue:`46750`)
+- Bug in :class:`StataWriterUTF8` where some valid characters were removed from variable names (:issue:`47276`)
 
 Period
 ^^^^^^

pandas/_libs/properties.pyi

@@ -1,9 +1,28 @@
-# pyright: reportIncompleteStub = false
-from typing import Any
+from typing import (
+    Sequence,
+    overload,
+)
+
+from pandas._typing import (
+    AnyArrayLike,
+    DataFrame,
+    Index,
+    Series,
+)
 
 # note: this is a lie to make type checkers happy (they special
 # case property). cache_readonly uses attribute names similar to
 # property (fget) but it does not provide fset and fdel.
 cache_readonly = property
 
-def __getattr__(name: str) -> Any: ...  # incomplete
+class AxisProperty:
+
+    axis: int
+    def __init__(self, axis: int = ..., doc: str = ...) -> None: ...
+    @overload
+    def __get__(self, obj: DataFrame | Series, type) -> Index: ...
+    @overload
+    def __get__(self, obj: None, type) -> AxisProperty: ...
+    def __set__(
+        self, obj: DataFrame | Series, value: AnyArrayLike | Sequence
+    ) -> None: ...

pandas/_libs/tslibs/conversion.pxd

@@ -8,7 +8,10 @@ from numpy cimport (
     ndarray,
 )
 
-from pandas._libs.tslibs.np_datetime cimport npy_datetimestruct
+from pandas._libs.tslibs.np_datetime cimport (
+    NPY_DATETIMEUNIT,
+    npy_datetimestruct,
+)
 
 
 cdef class _TSObject:
@@ -31,3 +34,5 @@ cdef int64_t get_datetime64_nanos(object val) except? -1
 cpdef datetime localize_pydatetime(datetime dt, tzinfo tz)
 cdef int64_t cast_from_unit(object ts, str unit) except? -1
 cpdef (int64_t, int) precision_from_unit(str unit)
+
+cdef maybe_localize_tso(_TSObject obj, tzinfo tz, NPY_DATETIMEUNIT reso)

pandas/_libs/tslibs/conversion.pyx

@@ -296,14 +296,18 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
         raise TypeError(f'Cannot convert input [{ts}] of type {type(ts)} to '
                         f'Timestamp')
 
+    maybe_localize_tso(obj, tz, NPY_FR_ns)
+    return obj
+
+
+cdef maybe_localize_tso(_TSObject obj, tzinfo tz, NPY_DATETIMEUNIT reso):
     if tz is not None:
-        _localize_tso(obj, tz)
+        _localize_tso(obj, tz, reso)
 
     if obj.value != NPY_NAT:
         # check_overflows needs to run after _localize_tso
-        check_dts_bounds(&obj.dts)
+        check_dts_bounds(&obj.dts, reso)
         check_overflows(obj)
-    return obj
 
 
 cdef _TSObject convert_datetime_to_tsobject(datetime ts, tzinfo tz,
@@ -401,7 +405,7 @@ cdef _TSObject _create_tsobject_tz_using_offset(npy_datetimestruct dts,
         return obj
 
     cdef:
-        Localizer info = Localizer(tz)
+        Localizer info = Localizer(tz, NPY_FR_ns)
 
     # Infer fold from offset-adjusted obj.value
     # see PEP 495 https://www.python.org/dev/peps/pep-0495/#the-fold-attribute
@@ -548,7 +552,7 @@ cdef inline check_overflows(_TSObject obj):
 # ----------------------------------------------------------------------
 # Localization
 
-cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
+cdef inline void _localize_tso(_TSObject obj, tzinfo tz, NPY_DATETIMEUNIT reso):
     """
     Given the UTC nanosecond timestamp in obj.value, find the wall-clock
     representation of that timestamp in the given timezone.
@@ -557,6 +561,7 @@ cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
     ----------
     obj : _TSObject
     tz : tzinfo
+    reso : NPY_DATETIMEUNIT
 
     Returns
     -------
@@ -569,7 +574,7 @@ cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
     cdef:
         int64_t local_val
         Py_ssize_t outpos = -1
-        Localizer info = Localizer(tz)
+        Localizer info = Localizer(tz, reso)
 
     assert obj.tzinfo is None
 
@@ -584,7 +589,7 @@ cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
             # infer we went through a pytz path, will have outpos!=-1
             tz = tz._tzinfos[tz._transition_info[outpos]]
 
-        dt64_to_dtstruct(local_val, &obj.dts)
+        pandas_datetime_to_datetimestruct(local_val, reso, &obj.dts)
 
     obj.tzinfo = tz
 

pandas/_libs/tslibs/timestamps.pyi

@@ -59,6 +59,10 @@ class Timestamp(datetime):
     # While Timestamp can return pd.NaT, having the constructor return
     # a Union with NaTType makes things awkward for users of pandas
     def _set_freq(self, freq: BaseOffset | None) -> None: ...
+    @classmethod
+    def _from_value_and_reso(
+        cls, value: int, reso: int, tz: _tzinfo | None
+    ) -> Timestamp: ...
     @property
     def year(self) -> int: ...
     @property