Merge branch 'pandas-dev:main' into main

Author: czroth

asv_bench/benchmarks/array.py

@@ -93,7 +93,7 @@ def time_setitem(self, multiple_chunks):
             self.array[i] = "foo"
 
     def time_setitem_list(self, multiple_chunks):
-        indexer = list(range(0, 50)) + list(range(-50, 0))
+        indexer = list(range(0, 50)) + list(range(-1000, 0, 50))
         self.array[indexer] = ["foo"] * len(indexer)
 
     def time_setitem_slice(self, multiple_chunks):

asv_bench/benchmarks/io/json.py

@@ -294,7 +294,8 @@ def time_float_longint_str_lines(self):
 class ToJSONMem:
     def setup_cache(self):
         df = DataFrame([[1]])
-        frames = {"int": df, "float": df.astype(float)}
+        df2 = DataFrame(range(8), date_range("1/1/2000", periods=8, freq="T"))
+        frames = {"int": df, "float": df.astype(float), "datetime": df2}
 
         return frames
 
@@ -308,5 +309,10 @@ def peakmem_float(self, frames):
         for _ in range(100_000):
             df.to_json()
 
+    def peakmem_time(self, frames):
+        df = frames["datetime"]
+        for _ in range(10_000):
+            df.to_json(orient="table")
+
 
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -85,27 +85,17 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
     MSG='Partially validate docstrings (RT02)' ; echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=RT02 --ignore_functions \
-    	    pandas.Series.align \
-	    pandas.Series.dt.total_seconds \
-	    pandas.Series.cat.rename_categories \
-	    pandas.Series.cat.reorder_categories \
-	    pandas.Series.cat.add_categories \
-	    pandas.Series.cat.remove_categories \
-	    pandas.Series.cat.remove_unused_categories \
 	    pandas.Index.all \
 	    pandas.Index.any \
 	    pandas.MultiIndex.drop \
 	    pandas.DatetimeIndex.to_pydatetime \
 	    pandas.TimedeltaIndex.to_pytimedelta \
-	    pandas.core.groupby.SeriesGroupBy.apply \
-	    pandas.core.groupby.DataFrameGroupBy.apply \
 	    pandas.io.formats.style.Styler.export \
 	    pandas.api.extensions.ExtensionArray.astype \
 	    pandas.api.extensions.ExtensionArray.dropna \
 	    pandas.api.extensions.ExtensionArray.isna \
 	    pandas.api.extensions.ExtensionArray.repeat \
-	    pandas.api.extensions.ExtensionArray.unique \
-	    pandas.DataFrame.align
+	    pandas.api.extensions.ExtensionArray.unique
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

doc/source/development/maintaining.rst

@@ -458,8 +458,8 @@ which will be triggered when the tag is pushed.
     git checkout master
     git pull --ff-only upstream master
     git checkout -B RLS-<version>
-    sed -i 's/BUILD_COMMIT: "v.*/BUILD_COMMIT: "'<version>'"/' azure/windows.yml azure/posix.yml
-    sed -i 's/BUILD_COMMIT="v.*/BUILD_COMMIT="'<version>'"/' .travis.yml
+    sed -i 's/BUILD_COMMIT: "v.*/BUILD_COMMIT: "'v<version>'"/' azure/windows.yml azure/posix.yml
+    sed -i 's/BUILD_COMMIT="v.*/BUILD_COMMIT="'v<version>'"/' .travis.yml
     git commit -am "RLS <version>"
     git push -u origin RLS-<version>
 
@@ -474,14 +474,21 @@ which will be triggered when the tag is pushed.
 Post-Release
 ````````````
 
-1. Close the milestone and the issue for the released version.
+1. Update symlink to stable documentation by logging in to our web server, and
+   editing ``/var/www/html/pandas-docs/stable`` to point to ``version/<latest-version>``.
 
-2. Create a new issue for the next release, with the estimated date or release.
+2. If releasing a major or minor release, open a PR in our source code to update
+   ``web/pandas/versions.json``, to have the desired versions in the documentation
+   dropdown menu.
 
-3. Open a PR with the placeholder for the release notes of the next version. See
+3. Close the milestone and the issue for the released version.
+
+4. Create a new issue for the next release, with the estimated date of release.
+
+5. Open a PR with the placeholder for the release notes of the next version. See
    for example [the PR for 1.5.3](https://github.com/pandas-dev/pandas/pull/49843/files).
 
-4. Announce the new release in the official channels (use previous announcements
+6. Announce the new release in the official channels (use previous announcements
    for reference):
 
     - The pandas-dev and pydata mailing lists

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.5
 .. toctree::
    :maxdepth: 2
 
+   v1.5.4
    v1.5.3
    v1.5.2
    v1.5.1

doc/source/whatsnew/v1.5.2.rst

@@ -43,4 +43,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.5.1..v1.5.2|HEAD
+.. contributors:: v1.5.1..v1.5.2

doc/source/whatsnew/v1.5.3.rst

@@ -55,4 +55,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.5.2..v1.5.3|HEAD
+.. contributors:: v1.5.2..v1.5.3

doc/source/whatsnew/v1.5.4.rst

@@ -0,0 +1,38 @@
+.. _whatsnew_154:
+
+What's new in 1.5.4 (March XX, 2023)
+--------------------------------------
+
+These are the changes in pandas 1.5.4. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.other:
+
+Other
+~~~~~
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.5.3..v1.5.4|HEAD

doc/source/whatsnew/v2.0.0.rst

@@ -161,6 +161,7 @@ Other enhancements
 - Added :meth:`Index.infer_objects` analogous to :meth:`Series.infer_objects` (:issue:`50034`)
 - Added ``copy`` parameter to :meth:`Series.infer_objects` and :meth:`DataFrame.infer_objects`, passing ``False`` will avoid making copies for series or columns that are already non-object or where no better dtype can be inferred (:issue:`50096`)
 - :meth:`DataFrame.plot.hist` now recognizes ``xlabel`` and ``ylabel`` arguments (:issue:`49793`)
+- :meth:`Series.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`48304`)
 - Improved error message in :func:`to_datetime` for non-ISO8601 formats, informing users about the position of the first error (:issue:`50361`)
 - Improved error message when trying to align :class:`DataFrame` objects (for example, in :func:`DataFrame.compare`) to clarify that "identically labelled" refers to both index and columns (:issue:`50083`)
 - Added :meth:`DatetimeIndex.as_unit` and :meth:`TimedeltaIndex.as_unit` to convert to different resolutions; supported resolutions are "s", "ms", "us", and "ns" (:issue:`50616`)
@@ -453,6 +454,37 @@ Now, the axes return an empty :class:`RangeIndex`.
    pd.Series().index
    pd.DataFrame().axes
 
+.. _whatsnew_200.api_breaking.to_latex:
+
+DataFrame to LaTeX has a new render engine
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The existing :meth:`DataFrame.to_latex` has been restructured to utilise the
+extended implementation previously available under :meth:`.Styler.to_latex`.
+The arguments signature is similar, albeit ``col_space`` has been removed since
+it is ignored by LaTeX engines. This render engine also requires ``jinja2`` as a
+dependency which needs to be installed, since rendering is based upon jinja2 templates.
+
+The pandas options below are no longer used and will be removed in future releases.
+The alternative options giving similar functionality are indicated below:
+
+- ``display.latex.escape``: replaced with ``styler.format.escape``,
+- ``display.latex.longtable``: replaced with ``styler.latex.environment``,
+- ``display.latex.multicolumn``, ``display.latex.multicolumn_format`` and
+  ``display.latex.multirow``: replaced with ``styler.sparse.rows``,
+  ``styler.sparse.columns``, ``styler.latex.multirow_align`` and
+  ``styler.latex.multicol_align``,
+- ``display.latex.repr``: replaced with ``styler.render.repr``,
+- ``display.max_rows`` and ``display.max_columns``: replace with
+  ``styler.render.max_rows``, ``styler.render.max_columns`` and
+  ``styler.render.max_elements``.
+
+Note that the behaviour of ``_repr_latex_`` is also changed. Previously
+setting ``display.latex.repr`` would generate LaTeX only when using nbconvert for a
+JupyterNotebook, and not when the user is running the notebook. Now the
+``styler.render.repr`` option allows control of the specific output
+within JupyterNotebooks for operations (not just on nbconvert). See :issue:`39911`.
+
 .. _whatsnew_200.api_breaking.deps:
 
 Increased minimum versions for dependencies
@@ -618,6 +650,7 @@ Removal of prior version deprecations/changes
 - Removed deprecated :meth:`.Styler.set_na_rep` and :meth:`.Styler.set_precision` (:issue:`49397`)
 - Removed deprecated :meth:`.Styler.where` (:issue:`49397`)
 - Removed deprecated :meth:`.Styler.render` (:issue:`49397`)
+- Removed deprecated argument ``col_space`` in :meth:`DataFrame.to_latex` (:issue:`47970`)
 - Removed deprecated argument ``null_color`` in :meth:`.Styler.highlight_null` (:issue:`49397`)
 - Removed deprecated argument ``check_less_precise`` in :meth:`.testing.assert_frame_equal`, :meth:`.testing.assert_extension_array_equal`, :meth:`.testing.assert_series_equal`,  :meth:`.testing.assert_index_equal` (:issue:`30562`)
 - Removed deprecated ``null_counts`` argument in :meth:`DataFrame.info`. Use ``show_counts`` instead (:issue:`37999`)
@@ -792,6 +825,7 @@ Removal of prior version deprecations/changes
 - Changed behavior of comparison of ``NaT`` with a ``datetime.date`` object; these now raise on inequality comparisons (:issue:`39196`)
 - Enforced deprecation of silently dropping columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a list or dictionary (:issue:`43740`)
 - Changed behavior of :meth:`DataFrame.apply` with list-like so that any partial failure will raise an error (:issue:`43740`)
+- Changed behaviour of :meth:`DataFrame.to_latex` to now use the Styler implementation via :meth:`.Styler.to_latex` (:issue:`47970`)
 - Changed behavior of :meth:`Series.__setitem__` with an integer key and a :class:`Float64Index` when the key is not present in the index; previously we treated the key as positional (behaving like ``series.iloc[key] = val``), now we treat it is a label (behaving like ``series.loc[key] = val``), consistent with :meth:`Series.__getitem__`` behavior (:issue:`33469`)
 - Removed ``na_sentinel`` argument from :func:`factorize`, :meth:`.Index.factorize`, and :meth:`.ExtensionArray.factorize` (:issue:`47157`)
 - Changed behavior of :meth:`Series.diff` and :meth:`DataFrame.diff` with :class:`ExtensionDtype` dtypes whose arrays do not implement ``diff``, these now raise ``TypeError`` rather than casting to numpy (:issue:`31025`)
@@ -850,7 +884,7 @@ Performance improvements
 - Performance improvement for :class:`~arrays.StringArray` constructor passing a numpy array with type ``np.str_`` (:issue:`49109`)
 - Performance improvement in :meth:`~arrays.IntervalArray.from_tuples` (:issue:`50620`)
 - Performance improvement in :meth:`~arrays.ArrowExtensionArray.factorize` (:issue:`49177`)
-- Performance improvement in :meth:`~arrays.ArrowExtensionArray.__setitem__` when key is a null slice (:issue:`50248`)
+- Performance improvement in :meth:`~arrays.ArrowExtensionArray.__setitem__` (:issue:`50248`, :issue:`50632`)
 - Performance improvement in :class:`~arrays.ArrowExtensionArray` comparison methods when array contains NA (:issue:`50524`)
 - Performance improvement in :meth:`~arrays.ArrowExtensionArray.to_numpy` (:issue:`49973`)
 - Performance improvement when parsing strings to :class:`BooleanDtype` (:issue:`50613`)
@@ -868,11 +902,15 @@ Performance improvements
 - Performance improvement in :func:`merge` when not merging on the index - the new index will now be :class:`RangeIndex` instead of :class:`Int64Index` (:issue:`49478`)
 - Performance improvement in :meth:`DataFrame.to_dict` and :meth:`Series.to_dict` when using any non-object dtypes (:issue:`46470`)
 - Performance improvement in :func:`read_html` when there are multiple tables (:issue:`49929`)
+- Performance improvement in :class:`Period` constructor when constructing from a string or integer (:issue:`38312`)
 - Performance improvement in :func:`to_datetime` when using ``'%Y%m%d'`` format (:issue:`17410`)
 - Performance improvement in :func:`to_datetime` when format is given or can be inferred (:issue:`50465`)
+- Performance improvement in :meth:`Series.median` for nullable dtypes (:issue:`50838`)
 - Performance improvement in :func:`read_csv` when passing :func:`to_datetime` lambda-function to ``date_parser`` and inputs have mixed timezone offsetes (:issue:`35296`)
+- Performance improvement in :func:`isna` and :func:`isnull` (:issue:`50658`)
 - Performance improvement in :meth:`.SeriesGroupBy.value_counts` with categorical dtype (:issue:`46202`)
 - Fixed a reference leak in :func:`read_hdf` (:issue:`37441`)
+- Fixed a memory leak in :meth:`DataFrame.to_json` and :meth:`Series.to_json` when serializing datetimes and timedeltas (:issue:`40443`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_200.bug_fixes:
@@ -1020,8 +1058,10 @@ I/O
 - Bug in :meth:`DataFrame.to_string` ignoring float formatter for extension arrays (:issue:`39336`)
 - Fixed memory leak which stemmed from the initialization of the internal JSON module (:issue:`49222`)
 - Fixed issue where :func:`json_normalize` would incorrectly remove leading characters from column names that matched the ``sep`` argument (:issue:`49861`)
+- Bug in :func:`read_csv` unnecessarily overflowing for extension array dtype when containing ``NA`` (:issue:`32134`)
 - Bug in :meth:`DataFrame.to_dict` not converting ``NA`` to ``None`` (:issue:`50795`)
 - Bug in :meth:`DataFrame.to_json` where it would segfault when failing to encode a string (:issue:`50307`)
+- Bug in :func:`read_xml` where file-like objects failed when iterparse is used (:issue:`50641`)
 
 Period
 ^^^^^^

pandas/_libs/algos.pyx

@@ -1366,7 +1366,7 @@ def rank_2d(
         nan_fill_val = get_rank_nan_fill_val(nans_rank_highest, <numeric_object_t>0)
 
         if numeric_object_t is object:
-            mask = missing.isnaobj2d(values).view(np.uint8)
+            mask = missing.isnaobj(values).view(np.uint8)
         elif numeric_object_t is float64_t or numeric_object_t is float32_t:
             mask = np.isnan(values).view(np.uint8)
         else:

pandas/_libs/missing.pyi

@@ -13,6 +13,5 @@ def isposinf_scalar(val: object) -> bool: ...
 def isneginf_scalar(val: object) -> bool: ...
 def checknull(val: object, inf_as_na: bool = ...) -> bool: ...
 def isnaobj(arr: np.ndarray, inf_as_na: bool = ...) -> npt.NDArray[np.bool_]: ...
-def isnaobj2d(arr: np.ndarray, inf_as_na: bool = ...) -> npt.NDArray[np.bool_]: ...
 def is_numeric_na(values: np.ndarray) -> npt.NDArray[np.bool_]: ...
 def is_float_nan(values: np.ndarray) -> npt.NDArray[np.bool_]: ...

pandas/_libs/missing.pyx

@@ -4,10 +4,12 @@ from sys import maxsize
 
 cimport cython
 from cython cimport Py_ssize_t
+
 import numpy as np
 
 cimport numpy as cnp
 from numpy cimport (
+    flatiter,
     float64_t,
     int64_t,
     ndarray,
@@ -197,56 +199,22 @@ cpdef ndarray[uint8_t] isnaobj(ndarray arr, bint inf_as_na=False):
     result : ndarray (dtype=np.bool_)
     """
     cdef:
-        Py_ssize_t i, n
+        Py_ssize_t i, n = arr.size
         object val
-        ndarray[uint8_t] result
-
-    assert arr.ndim == 1, "'arr' must be 1-D."
-
-    n = len(arr)
-    result = np.empty(n, dtype=np.uint8)
-    for i in range(n):
-        val = arr[i]
-        result[i] = checknull(val, inf_as_na=inf_as_na)
-    return result.view(np.bool_)
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj2d(arr: ndarray, inf_as_na: bool = False) -> ndarray:
-    """
-    Return boolean mask denoting which elements of a 2-D array are na-like,
-    according to the criteria defined in `checknull`:
-     - None
-     - nan
-     - NaT
-     - np.datetime64 representation of NaT
-     - np.timedelta64 representation of NaT
-     - NA
-     - Decimal("NaN")
-
-    Parameters
-    ----------
-    arr : ndarray
-
-    Returns
-    -------
-    result : ndarray (dtype=np.bool_)
-    """
-    cdef:
-        Py_ssize_t i, j, n, m
-        object val
-        ndarray[uint8_t, ndim=2] result
-
-    assert arr.ndim == 2, "'arr' must be 2-D."
+        bint is_null
+        ndarray result = np.empty((<object>arr).shape, dtype=np.uint8)
+        flatiter it = cnp.PyArray_IterNew(arr)
+        flatiter it2 = cnp.PyArray_IterNew(result)
 
-    n, m = (<object>arr).shape
-    result = np.zeros((n, m), dtype=np.uint8)
     for i in range(n):
-        for j in range(m):
-            val = arr[i, j]
-            if checknull(val, inf_as_na=inf_as_na):
-                result[i, j] = 1
+        # The PyArray_GETITEM and PyArray_ITER_NEXT are faster
+        #  equivalents to `val = values[i]`
+        val = cnp.PyArray_GETITEM(arr, cnp.PyArray_ITER_DATA(it))
+        cnp.PyArray_ITER_NEXT(it)
+        is_null = checknull(val, inf_as_na=inf_as_na)
+        # Dereference pointer (set value)
+        (<uint8_t *>(cnp.PyArray_ITER_DATA(it2)))[0] = <uint8_t>is_null
+        cnp.PyArray_ITER_NEXT(it2)
     return result.view(np.bool_)
 
 

pandas/_libs/src/ujson/python/objToJSON.c

@@ -350,13 +350,15 @@ static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc,
 static char *NpyDateTimeToIsoCallback(JSOBJ Py_UNUSED(unused),
                                       JSONTypeContext *tc, size_t *len) {
     NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit;
-    return int64ToIso(GET_TC(tc)->longValue, base, len);
+    GET_TC(tc)->cStr = int64ToIso(GET_TC(tc)->longValue, base, len);
+    return GET_TC(tc)->cStr;
 }
 
 /* JSON callback. returns a char* and mutates the pointer to *len */
 static char *NpyTimeDeltaToIsoCallback(JSOBJ Py_UNUSED(unused),
                                        JSONTypeContext *tc, size_t *len) {
-    return int64ToIsoDuration(GET_TC(tc)->longValue, len);
+    GET_TC(tc)->cStr = int64ToIsoDuration(GET_TC(tc)->longValue, len);
+    return GET_TC(tc)->cStr;
 }
 
 /* JSON callback */