Merge remote-tracking branch 'upstream/main' into same_val_counts_for_roll_skew_kurt

Author: auderson

.pre-commit-config.yaml

@@ -176,6 +176,13 @@ repos:
         files: ^pandas/core/
         exclude: ^pandas/core/api\.py$
         types: [python]
+    -   id: use-io-common-urlopen
+        name: Use pandas.io.common.urlopen instead of urllib.request.urlopen
+        language: python
+        entry: python scripts/use_io_common_urlopen.py
+        files: ^pandas/
+        exclude: ^pandas/tests/
+        types: [python]
     -   id: no-bool-in-core-generic
         name: Use bool_t instead of bool in pandas/core/generic.py
         entry: python scripts/no_bool_in_generic.py

LICENSES/KLIB_LICENSE

@@ -0,0 +1,23 @@
+The MIT License
+
+Copyright (c) 2008-     Attractive Chaos <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -1,4 +1,5 @@
 include RELEASE.md
+include versioneer.py
 
 graft doc
 prune doc/build
@@ -54,9 +55,6 @@ global-exclude *.pxi
 # exclude the whole directory to avoid running related tests in sdist
 prune pandas/tests/io/parser/data
 
-include versioneer.py
-include pandas/_version.py
-include pandas/io/formats/templates/*.tpl
-
+# Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src
 graft pandas/_libs/tslibs/src

ci/deps/actions-310.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

ci/deps/actions-38-downstream_compat.yaml

@@ -10,7 +10,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -28,6 +27,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

ci/deps/actions-38-minimum_versions.yaml

@@ -11,7 +11,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -29,6 +28,7 @@ dependencies:
   - fastparquet=0.4.0
   - fsspec=0.7.4
   - html5lib=1.1
+  - hypothesis=5.5.3
   - gcsfs=0.6.0
   - jinja2=2.11
   - lxml=4.5.0

ci/deps/actions-38.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

ci/deps/actions-39.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

ci/deps/circle-38-arm64.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

doc/source/development/code_style.rst

@@ -37,15 +37,14 @@ In addition to ``./ci/code_checks.sh``, some extra checks are run by
 ``pre-commit`` - see :ref:`here <contributing.pre-commit>` for how to
 run them.
 
-Additional standards are outlined on the :ref:`pandas code style guide <code_style>`.
-
 .. _contributing.pre-commit:
 
 Pre-commit
 ----------
 
 Additionally, :ref:`Continuous Integration <contributing.ci>` will run code formatting checks
-like ``black``, ``flake8``, ``isort``, and ``cpplint`` and more using `pre-commit hooks <https://pre-commit.com/>`_
+like ``black``, ``flake8`` (including a `pandas-dev-flaker <https://github.com/pandas-dev/pandas-dev-flaker>`_ plugin),
+``isort``, and ``cpplint`` and more using `pre-commit hooks <https://pre-commit.com/>`_
 Any warnings from these checks will cause the :ref:`Continuous Integration <contributing.ci>` to fail; therefore,
 it is helpful to run the check yourself before submitting code. This
 can be done by installing ``pre-commit``::

doc/source/development/contributing_codebase.rst

@@ -16,7 +16,6 @@ Development
     contributing_environment
     contributing_documentation
     contributing_codebase
-    code_style
     maintaining
     internals
     test_writing

doc/source/development/index.rst

@@ -664,7 +664,7 @@ Enhancements
      other = pd.DataFrame({'A': [1, 3, 3, 7], 'B': ['e', 'f', 'f', 'e']})
      mask = dfi.isin(other)
      mask
-     dfi[mask.any(1)]
+     dfi[mask.any(axis=1)]
 
 - ``Series`` now supports a ``to_frame`` method to convert it to a single-column DataFrame (:issue:`5164`)
 

doc/source/whatsnew/v0.13.0.rst

@@ -14,6 +14,7 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
+- Fixed regression in :meth:`DataFrame.nsmallest` led to wrong results when ``np.nan`` in the sorting column (:issue:`46589`)
 - Fixed regression in :func:`read_fwf` raising ``ValueError`` when ``widths`` was specified with ``usecols`` (:issue:`46580`)
 -
 

doc/source/whatsnew/v1.4.3.rst

@@ -94,7 +94,7 @@ Other enhancements
 - :meth:`DataFrame.reset_index` now accepts a ``names`` argument which renames the index names (:issue:`6878`)
 - :meth:`pd.concat` now raises when ``levels`` is given but ``keys`` is None (:issue:`46653`)
 - :meth:`pd.concat` now raises when ``levels`` contains duplicate values (:issue:`46653`)
--
+- Added ``numeric_only`` argument to :meth:`DataFrame.corr`, :meth:`DataFrame.corrwith`, and :meth:`DataFrame.cov` (:issue:`46560`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -429,8 +429,9 @@ Other Deprecations
 - Deprecated behavior of method :meth:`DataFrame.quantile`, attribute ``numeric_only`` will default False. Including datetime/timedelta columns in the result (:issue:`7308`).
 - Deprecated :attr:`Timedelta.freq` and :attr:`Timedelta.is_populated` (:issue:`46430`)
 - Deprecated :attr:`Timedelta.delta` (:issue:`46476`)
+- Deprecated passing arguments as positional in :meth:`DataFrame.any` and :meth:`Series.any` (:issue:`44802`)
 - Deprecated the ``closed`` argument in :meth:`interval_range` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
--
+- Deprecated the methods :meth:`DataFrame.mad`, :meth:`Series.mad`, and the corresponding groupby methods (:issue:`11787`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.performance:
@@ -498,6 +499,7 @@ Conversion
 - Bug in :meth:`Series.astype` and :meth:`DataFrame.astype` from floating dtype to unsigned integer dtype failing to raise in the presence of negative values (:issue:`45151`)
 - Bug in :func:`array` with ``FloatingDtype`` and values containing float-castable strings incorrectly raising (:issue:`45424`)
 - Bug when comparing string and datetime64ns objects causing ``OverflowError`` exception. (:issue:`45506`)
+- Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`)
 
 Strings
 ^^^^^^^
@@ -563,6 +565,7 @@ I/O
 - Bug in :func:`read_csv` not respecting a specified converter to index columns in all cases (:issue:`40589`)
 - Bug in :func:`read_parquet` when ``engine="pyarrow"`` which caused partial write to disk when column of unsupported datatype was passed (:issue:`44914`)
 - Bug in :func:`DataFrame.to_excel` and :class:`ExcelWriter` would raise when writing an empty DataFrame to a ``.ods`` file (:issue:`45793`)
+- Bug in :func:`read_html` where elements surrounding ``<br>`` were joined without a space between them (:issue:`29528`)
 - Bug in Parquet roundtrip for Interval dtype with ``datetime64[ns]`` subtype (:issue:`45881`)
 - Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements (:issue:`45598`)
 - Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)

doc/source/whatsnew/v1.5.0.rst

@@ -1,4 +1,7 @@
-from pandas._libs.dtypes cimport numeric_t
+from pandas._libs.dtypes cimport (
+    numeric_object_t,
+    numeric_t,
+)
 
 
 cdef numeric_t kth_smallest_c(numeric_t* arr, Py_ssize_t k, Py_ssize_t n) nogil
@@ -10,3 +13,10 @@ cdef enum TiebreakEnumType:
     TIEBREAK_FIRST
     TIEBREAK_FIRST_DESCENDING
     TIEBREAK_DENSE
+
+
+cdef numeric_object_t get_rank_nan_fill_val(
+    bint rank_nans_highest,
+    numeric_object_t val,
+    bint is_datetimelike=*,
+)

pandas/_libs/algos.pxd

@@ -822,13 +822,17 @@ def is_monotonic(ndarray[numeric_object_t, ndim=1] arr, bint timelike):
 
 cdef numeric_object_t get_rank_nan_fill_val(
     bint rank_nans_highest,
-    numeric_object_t[:] _=None
+    numeric_object_t val,
+    bint is_datetimelike=False,
 ):
     """
     Return the value we'll use to represent missing values when sorting depending
     on if we'd like missing values to end up at the top/bottom. (The second parameter
     is unused, but needed for fused type specialization)
     """
+    if numeric_object_t is int64_t and is_datetimelike and not rank_nans_highest:
+        return NPY_NAT + 1
+
     if rank_nans_highest:
         if numeric_object_t is object:
             return Infinity()
@@ -854,6 +858,9 @@ cdef numeric_object_t get_rank_nan_fill_val(
         if numeric_object_t is object:
             return NegInfinity()
         elif numeric_object_t is int64_t:
+            # Note(jbrockmendel) 2022-03-15 for reasons unknown, using util.INT64_MIN
+            #  instead of NPY_NAT here causes build warnings and failure in
+            #  test_cummax_i8_at_implementation_bound
             return NPY_NAT
         elif numeric_object_t is int32_t:
             return util.INT32_MIN
@@ -975,7 +982,7 @@ def rank_1d(
     # will flip the ordering to still end up with lowest rank.
     # Symmetric logic applies to `na_option == 'bottom'`
     nans_rank_highest = ascending ^ (na_option == 'top')
-    nan_fill_val = get_rank_nan_fill_val[numeric_object_t](nans_rank_highest)
+    nan_fill_val = get_rank_nan_fill_val(nans_rank_highest, <numeric_object_t>0)
     if nans_rank_highest:
         order = [masked_vals, mask]
     else:
@@ -1335,7 +1342,7 @@ def rank_2d(
 
     nans_rank_highest = ascending ^ (na_option == 'top')
     if check_mask:
-        nan_fill_val = get_rank_nan_fill_val[numeric_object_t](nans_rank_highest)
+        nan_fill_val = get_rank_nan_fill_val(nans_rank_highest, <numeric_object_t>0)
 
         if numeric_object_t is object:
             mask = missing.isnaobj2d(values).view(np.uint8)

pandas/_libs/algos.pyx

@@ -31,7 +31,10 @@ from numpy.math cimport NAN
 cnp.import_array()
 
 from pandas._libs cimport util
-from pandas._libs.algos cimport kth_smallest_c
+from pandas._libs.algos cimport (
+    get_rank_nan_fill_val,
+    kth_smallest_c,
+)
 
 from pandas._libs.algos import (
     ensure_platform_int,
@@ -989,36 +992,16 @@ cdef inline bint _treat_as_na(numeric_object_t val, bint is_datetimelike) nogil:
         return False
 
 
-cdef numeric_t _get_min_or_max(numeric_t val, bint compute_max, bint is_datetimelike):
+cdef numeric_object_t _get_min_or_max(numeric_object_t val, bint compute_max, bint is_datetimelike):
     """
-    Find either the min or the max supported by numeric_t; 'val' is a placeholder
-    to effectively make numeric_t an argument.
+    Find either the min or the max supported by numeric_object_t; 'val' is a
+    placeholder to effectively make numeric_object_t an argument.
     """
-    if numeric_t is int64_t:
-        if compute_max and is_datetimelike:
-            return -_int64_max
-        # Note(jbrockmendel) 2022-03-15 for reasons unknown, using util.INT64_MIN
-        #  instead of NPY_NAT here causes build warnings and failure in
-        #  test_cummax_i8_at_implementation_bound
-        return NPY_NAT if compute_max else util.INT64_MAX
-    elif numeric_t is int32_t:
-        return util.INT32_MIN if compute_max else util.INT32_MAX
-    elif numeric_t is int16_t:
-        return util.INT16_MIN if compute_max else util.INT16_MAX
-    elif numeric_t is int8_t:
-        return util.INT8_MIN if compute_max else util.INT8_MAX
-
-    elif numeric_t is uint64_t:
-        return 0 if compute_max else util.UINT64_MAX
-    elif numeric_t is uint32_t:
-        return 0 if compute_max else util.UINT32_MAX
-    elif numeric_t is uint16_t:
-        return 0 if compute_max else util.UINT16_MAX
-    elif numeric_t is uint8_t:
-        return 0 if compute_max else util.UINT8_MAX
-
-    else:
-        return -np.inf if compute_max else np.inf
+    return get_rank_nan_fill_val(
+        not compute_max,
+        val=val,
+        is_datetimelike=is_datetimelike,
+    )
 
 
 cdef numeric_t _get_na_val(numeric_t val, bint is_datetimelike):