Merge remote-tracking branch 'upstream/main' into cow_inf_obj

Author: phofl

.github/workflows/32-bit-linux.yml

@@ -39,7 +39,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
           python -m pip install versioneer[toml] && \
-          python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python -m pip install cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.34.2 && \
           python setup.py build_ext -q -j1 && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
           python -m pip list && \

.github/workflows/python-dev.yml

@@ -73,10 +73,10 @@ jobs:
       run: |
         python --version
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
+        python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
         python -m pip install versioneer[toml]
-        python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
+        python -m pip install python-dateutil pytz cython hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
         python -m pip list
 
     # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs

.github/workflows/wheels.yml

@@ -168,7 +168,7 @@ jobs:
           # (1. Generate sdist 2. Build wheels from sdist)
           # This tests the sdists, and saves some build time
           python -m pip install dist/*.gz
-          pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17
+          pip install hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17
           cd .. # Not a good idea to test within the src tree
           python -c "import pandas; print(pandas.__version__);
           pandas.test(extra_args=['-m not clipboard and not single_cpu', '--skip-slow', '--skip-network', '--skip-db', '-n=2']);

asv_bench/asv.conf.json

@@ -41,7 +41,7 @@
     // pip (with all the conda available packages installed first,
     // followed by the pip installed packages).
     "matrix": {
-        "numpy": ["1.23.5"],  // https://github.com/pandas-dev/pandas/pull/50356
+        "numpy": [],
         "Cython": ["0.29.32"],
         "matplotlib": [],
         "sqlalchemy": [],

ci/deps/actions-310-numpydev.yaml

@@ -1,17 +1,17 @@
 name: pandas-dev
 channels:
-  - defaults
+  - conda-forge
 dependencies:
   - python=3.10
 
   # build dependencies
   - versioneer[toml]
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
+  - pytest-xdist>=2.2.0
+  - hypothesis>=6.34.2
   - pytest-asyncio>=0.17
 
   # pandas dependencies
@@ -22,5 +22,5 @@ dependencies:
     - "cython"
     - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple"
     - "--pre"
-    - "numpy<1.24"
+    - "numpy"
     - "scipy"

ci/deps/actions-310.yaml

@@ -9,16 +9,16 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
+  - pytest-xdist>=2.2.0
   - psutil
   - pytest-asyncio>=0.17
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<1.24
+  - numpy
   - pytz
 
   # optional dependencies

ci/deps/actions-38-downstream_compat.yaml

@@ -10,16 +10,16 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
+  - pytest-xdist>=2.2.0
   - psutil
   - pytest-asyncio>=0.17
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<1.24
+  - numpy
   - pytz
 
   # optional dependencies

ci/deps/actions-38-minimum_versions.yaml

@@ -11,9 +11,9 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
+  - pytest-xdist>=2.2.0
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -31,7 +31,7 @@ dependencies:
   - fastparquet=0.6.3
   - fsspec=2021.07.0
   - html5lib=1.1
-  - hypothesis=6.13.0
+  - hypothesis=6.34.2
   - gcsfs=2021.07.0
   - jinja2=3.0.0
   - lxml=4.6.3

ci/deps/actions-38.yaml

@@ -9,16 +9,16 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
+  - pytest-xdist>=2.2.0
   - psutil
   - pytest-asyncio>=0.17
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<1.24
+  - numpy
   - pytz
 
   # optional dependencies

ci/deps/actions-39.yaml

@@ -9,16 +9,16 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
+  - pytest-xdist>=2.2.0
   - psutil
   - pytest-asyncio>=0.17
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<1.24
+  - numpy
   - pytz
 
   # optional dependencies

ci/deps/actions-pypy-38.yaml

@@ -12,13 +12,13 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
   - pytest-asyncio
-  - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
+  - pytest-xdist>=2.2.0
+  - hypothesis>=6.34.2
 
   # required
-  - numpy<1.24
+  - numpy
   - python-dateutil
   - pytz

ci/deps/circle-38-arm64.yaml

@@ -9,16 +9,16 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
+  - pytest-xdist>=2.2.0
   - psutil
   - pytest-asyncio>=0.17
   - boto3
 
   # required dependencies
   - python-dateutil
-  - numpy<1.24
+  - numpy
   - pytz
 
   # optional dependencies

ci/test_wheels_windows.bat

@@ -4,6 +4,6 @@ pd.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-ne
 
 python --version
 pip install pytz six numpy python-dateutil
-pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17
+pip install hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17
 pip install --find-links=pandas/dist --no-index pandas
-python -c "%test_command%"
+python -c "%test_command%"

doc/source/getting_started/install.rst

@@ -208,8 +208,8 @@ pandas is equipped with an exhaustive set of unit tests, covering about 97% of
 the code base as of this writing. To run it on your machine to verify that
 everything is working (and that you have all of the dependencies, soft and hard,
 installed), make sure you have `pytest
-<https://docs.pytest.org/en/latest/>`__ >= 6.0 and `Hypothesis
-<https://hypothesis.readthedocs.io/en/latest/>`__ >= 6.13.0, then run:
+<https://docs.pytest.org/en/latest/>`__ >= 7.0 and `Hypothesis
+<https://hypothesis.readthedocs.io/en/latest/>`__ >= 6.34.2, then run:
 
 ::
 

doc/source/whatsnew/v1.5.3.rst

@@ -27,7 +27,9 @@ Fixed regressions
 
 Bug fixes
 ~~~~~~~~~
+- Bug in the Copy-on-Write implementation losing track of views when indexing a :class:`DataFrame` with another :class:`DataFrame` (:issue:`50630`)
 - Bug in :meth:`.Styler.to_excel` leading to error when unrecognized ``border-style`` (e.g. ``"hair"``) provided to Excel writers (:issue:`48649`)
+- Bug in :meth:`Series.quantile` emitting warning from NumPy when :class:`Series` has only ``NA`` values (:issue:`50681`)
 - Bug when chaining several :meth:`.Styler.concat` calls, only the last styler was concatenated (:issue:`49207`)
 - Fixed bug when instantiating a :class:`DataFrame` subclass inheriting from ``typing.Generic`` that triggered a ``UserWarning`` on python 3.11 (:issue:`49649`)
 -

doc/source/whatsnew/v2.0.0.rst

@@ -83,34 +83,40 @@ be set to ``"pyarrow"`` to return pyarrow-backed, nullable :class:`ArrowDtype` (
         df_pyarrow = pd.read_csv(data, use_nullable_dtypes=True, engine="pyarrow")
     df_pyarrow.dtypes
 
-Copy on write improvements
+Copy-on-Write improvements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A new lazy copy mechanism that defers the copy until the object in question is modified
-was added to the following methods:
-
-- :meth:`DataFrame.reset_index` / :meth:`Series.reset_index`
-- :meth:`DataFrame.set_index` / :meth:`Series.set_index`
-- :meth:`DataFrame.set_axis` / :meth:`Series.set_axis`
-- :meth:`DataFrame.rename_axis` / :meth:`Series.rename_axis`
-- :meth:`DataFrame.rename_columns`
-- :meth:`DataFrame.reindex` / :meth:`Series.reindex`
-- :meth:`DataFrame.reindex_like` / :meth:`Series.reindex_like`
-- :meth:`DataFrame.assign`
-- :meth:`DataFrame.drop`
-- :meth:`DataFrame.dropna` / :meth:`Series.dropna`
-- :meth:`DataFrame.select_dtypes`
-- :meth:`DataFrame.align` / :meth:`Series.align`
-- :meth:`Series.to_frame`
-- :meth:`DataFrame.rename` / :meth:`Series.rename`
-- :meth:`DataFrame.add_prefix` / :meth:`Series.add_prefix`
-- :meth:`DataFrame.add_suffix` / :meth:`Series.add_suffix`
-- :meth:`DataFrame.drop_duplicates` / :meth:`Series.drop_duplicates`
-- :meth:`DataFrame.reorder_levels` / :meth:`Series.reorder_levels`
-
-These methods return views when copy on write is enabled, which provides a significant
-performance improvement compared to the regular execution (:issue:`49473`). Copy on write
-can be enabled through
+- A new lazy copy mechanism that defers the copy until the object in question is modified
+  was added to the following methods:
+
+  - :meth:`DataFrame.reset_index` / :meth:`Series.reset_index`
+  - :meth:`DataFrame.set_index` / :meth:`Series.set_index`
+  - :meth:`DataFrame.set_axis` / :meth:`Series.set_axis`
+  - :meth:`DataFrame.rename_axis` / :meth:`Series.rename_axis`
+  - :meth:`DataFrame.rename_columns`
+  - :meth:`DataFrame.reindex` / :meth:`Series.reindex`
+  - :meth:`DataFrame.reindex_like` / :meth:`Series.reindex_like`
+  - :meth:`DataFrame.assign`
+  - :meth:`DataFrame.drop`
+  - :meth:`DataFrame.dropna` / :meth:`Series.dropna`
+  - :meth:`DataFrame.select_dtypes`
+  - :meth:`DataFrame.align` / :meth:`Series.align`
+  - :meth:`Series.to_frame`
+  - :meth:`DataFrame.rename` / :meth:`Series.rename`
+  - :meth:`DataFrame.add_prefix` / :meth:`Series.add_prefix`
+  - :meth:`DataFrame.add_suffix` / :meth:`Series.add_suffix`
+  - :meth:`DataFrame.drop_duplicates` / :meth:`Series.drop_duplicates`
+  - :meth:`DataFrame.reorder_levels` / :meth:`Series.reorder_levels`
+
+  These methods return views when Copy-on-Write is enabled, which provides a significant
+  performance improvement compared to the regular execution (:issue:`49473`).
+
+- Accessing a single column of a DataFrame as a Series (e.g. ``df["col"]``) now always
+  returns a new object every time it is constructed when Copy-on-Write is enabled (not
+  returning multiple times an identical, cached Series object). This ensures that those
+  Series objects correctly follow the Copy-on-Write rules (:issue:`49450`)
+
+Copy-on-Write can be enabled through
 
 .. code-block:: python
 
@@ -447,13 +453,19 @@ Increased minimum versions for dependencies
 Some minimum supported versions of dependencies were updated.
 If installed, we now require:
 
-+-----------------+-----------------+----------+---------+
-| Package         | Minimum Version | Required | Changed |
-+=================+=================+==========+=========+
-| mypy (dev)      | 0.991           |          |    X    |
-+-----------------+-----------------+----------+---------+
-| python-dateutil | 2.8.2           |    X     |    X    |
-+-----------------+-----------------+----------+---------+
++-------------------+-----------------+----------+---------+
+| Package           | Minimum Version | Required | Changed |
++===================+=================+==========+=========+
+| mypy (dev)        | 0.991           |          |    X    |
++-------------------+-----------------+----------+---------+
+| pytest (dev)      | 7.0.0           |          |    X    |
++-------------------+-----------------+----------+---------+
+| pytest-xdist (dev)| 2.2.0           |          |    X    |
++-------------------+-----------------+----------+---------+
+| hypothesis (dev)  | 6.34.2          |          |    X    |
++-------------------+-----------------+----------+---------+
+| python-dateutil   | 2.8.2           |    X     |    X    |
++-------------------+-----------------+----------+---------+
 
 For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
@@ -556,9 +568,12 @@ Other API changes
 Deprecations
 ~~~~~~~~~~~~
 - Deprecated argument ``infer_datetime_format`` in :func:`to_datetime` and :func:`read_csv`, as a strict version of it is now the default (:issue:`48621`)
+- Deprecated :func:`pandas.io.sql.execute`(:issue:`50185`)
+-
 
-.. ---------------------------------------------------------------------------
+- :meth:`Index.is_floating` has been deprecated. Use :func:`pandas.api.types.is_float_dtype` instead (:issue:`50042`)
 
+.. ---------------------------------------------------------------------------
 .. _whatsnew_200.prior_deprecations:
 
 Removal of prior version deprecations/changes

environment.yml

@@ -11,16 +11,16 @@ dependencies:
   - cython=0.29.32
 
   # test dependencies
-  - pytest>=6.0
+  - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=1.31
+  - pytest-xdist>=2.2.0
   - psutil
   - pytest-asyncio>=0.17
   - coverage
 
   # required dependencies
   - python-dateutil
-  - numpy<1.24
+  - numpy
   - pytz
 
   # optional dependencies

pandas/_libs/tslib.pyx

@@ -546,9 +546,6 @@ cpdef array_to_datetime(
                 else:
                     # coerce
                     # we now need to parse this as if unit='ns'
-                    # we can ONLY accept integers at this point
-                    # if we have previously (or in future accept
-                    # datetimes/strings, then we must coerce)
                     try:
                         iresult[i] = cast_from_unit(val, "ns")
                     except OverflowError:

pandas/_libs/tslibs/conversion.pxd

@@ -52,7 +52,7 @@ cdef tzinfo convert_timezone(
 )
 
 cdef int64_t parse_pydatetime(
-    object val,
+    datetime val,
     npy_datetimestruct *dts,
     bint utc_convert,
 ) except? -1