Merge branch '1.3.x' of https://github.com/pandas-dev/pandas into 1.3.x

Author: jbrockmendel

.github/workflows/python-dev.yml

@@ -10,6 +10,12 @@ on:
     paths-ignore:
       - "doc/**"
 
+env:
+  PYTEST_WORKERS: "auto"
+  PANDAS_CI: 1
+  PATTERN: "not slow and not network and not clipboard"
+  COVERAGE: true
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -32,7 +38,7 @@ jobs:
         pip install git+https://github.com/numpy/numpy.git
         pip install git+https://github.com/pytest-dev/pytest.git
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest-xdist
+        pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov
         pip list
 
     - name: Build Pandas
@@ -46,7 +52,8 @@ jobs:
 
     - name: Test with pytest
       run: |
-        coverage run -m pytest -m 'not slow and not network and not clipboard' pandas
+        ci/run_tests.sh
+      # GH 41935
       continue-on-error: true
 
     - name: Publish test results

ci/azure/windows.yml

@@ -12,11 +12,13 @@ jobs:
         ENV_FILE: ci/deps/azure-windows-37.yaml
         CONDA_PY: "37"
         PATTERN: "not slow and not network"
+        PYTEST_WORKERS: 2  # GH-42236
 
       py38_np18:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
         PATTERN: "not slow and not network and not high_memory"
+        PYTEST_WORKERS: 2  # GH-42236
 
   steps:
     - powershell: |

doc/source/user_guide/io.rst

@@ -5526,13 +5526,23 @@ below and the SQLAlchemy `documentation <https://docs.sqlalchemy.org/en/latest/c
    # Create your engine.
    engine = create_engine("sqlite:///:memory:")
 
-If you want to manage your own connections you can pass one of those instead:
+If you want to manage your own connections you can pass one of those instead. The example below opens a
+connection to the database using a Python context manager that automatically closes the connection after
+the block has completed.
+See the `SQLAlchemy docs <https://docs.sqlalchemy.org/en/latest/core/connections.html#basic-usage>`__
+for an explanation of how the database connection is handled.
 
 .. code-block:: python
 
    with engine.connect() as conn, conn.begin():
        data = pd.read_sql_table("data", conn)
 
+.. warning::
+
+	When you open a connection to a database you are also responsible for closing it.
+	Side effects of leaving a connection open may include locking the database or
+	other breaking behaviour.
+
 Writing DataFrames
 ''''''''''''''''''
 

doc/source/whatsnew/v1.3.0.rst

@@ -707,6 +707,7 @@ Other API changes
 - Added new ``engine`` and ``**engine_kwargs`` parameters to :meth:`DataFrame.to_sql` to support other future "SQL engines". Currently we still only use ``SQLAlchemy`` under the hood, but more engines are planned to be supported such as `turbodbc <https://turbodbc.readthedocs.io/en/latest/>`_ (:issue:`36893`)
 - Removed redundant ``freq`` from :class:`PeriodIndex` string representation (:issue:`41653`)
 - :meth:`ExtensionDtype.construct_array_type` is now a required method instead of an optional one for :class:`ExtensionDtype` subclasses (:issue:`24860`)
+- Calling ``hash`` on non-hashable pandas objects will now raise ``TypeError`` with the built-in error message (e.g. ``unhashable type: 'Series'``). Previously it would raise a custom message such as ``'Series' objects are mutable, thus they cannot be hashed``. Furthermore, ``isinstance(<Series>, abc.collections.Hashable)`` will now return ``False`` (:issue:`40013`)
 - :meth:`.Styler.from_custom_template` now has two new arguments for template names, and removed the old ``name``, due to template inheritance having been introducing for better parsing (:issue:`42053`). Subclassing modifications to Styler attributes are also needed.
 
 .. _whatsnew_130.api_breaking.build:

pandas/core/arrays/base.py

@@ -1296,8 +1296,10 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
         """
         raise TypeError(f"cannot perform {name} with type {self.dtype}")
 
-    def __hash__(self) -> int:
-        raise TypeError(f"unhashable type: {repr(type(self).__name__)}")
+    # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
+    # Incompatible types in assignment (expression has type "None", base class
+    # "object" defined the type as "Callable[[object], int]")
+    __hash__: None  # type: ignore[assignment]
 
     # ------------------------------------------------------------------------
     # Non-Optimized Default Methods

pandas/core/frame.py

@@ -6186,7 +6186,10 @@ def f(vals) -> tuple[np.ndarray, int]:
             return labels.astype("i8", copy=False), len(shape)
 
         if subset is None:
-            subset = self.columns
+            # Incompatible types in assignment
+            # (expression has type "Index", variable has type "Sequence[Any]")
+            # (pending on https://github.com/pandas-dev/pandas/issues/28770)
+            subset = self.columns  # type: ignore[assignment]
         elif (
             not np.iterable(subset)
             or isinstance(subset, str)

pandas/core/generic.py

@@ -1873,11 +1873,10 @@ def _drop_labels_or_levels(self, keys, axis: int = 0):
     # ----------------------------------------------------------------------
     # Iteration
 
-    def __hash__(self) -> int:
-        raise TypeError(
-            f"{repr(type(self).__name__)} objects are mutable, "
-            f"thus they cannot be hashed"
-        )
+    # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
+    # Incompatible types in assignment (expression has type "None", base class
+    # "object" defined the type as "Callable[[object], int]")
+    __hash__: None  # type: ignore[assignment]
 
     def __iter__(self):
         """

pandas/core/indexes/base.py

@@ -3411,12 +3411,25 @@ def get_indexer(
             #  matched to Interval scalars
             return self._get_indexer_non_comparable(target, method=method, unique=True)
 
+        if is_categorical_dtype(self.dtype):
+            # _maybe_cast_listlike_indexer ensures target has our dtype
+            #  (could improve perf by doing _should_compare check earlier?)
+            assert is_dtype_equal(self.dtype, target.dtype)
+
+            indexer = self._engine.get_indexer(target.codes)
+            if self.hasnans and target.hasnans:
+                loc = self.get_loc(np.nan)
+                mask = target.isna()
+                indexer[mask] = loc
+            return indexer
+
         if is_categorical_dtype(target.dtype):
             # potential fastpath
             # get an indexer for unique categories then propagate to codes via take_nd
-            # Note: calling get_indexer instead of _get_indexer causes
-            #  RecursionError GH#42088
-            categories_indexer = self._get_indexer(target.categories)
+            # get_indexer instead of _get_indexer needed for MultiIndex cases
+            #  e.g. test_append_different_columns_types
+            categories_indexer = self.get_indexer(target.categories)
+
             indexer = algos.take_nd(categories_indexer, target.codes, fill_value=-1)
 
             if (not self._is_multi and self.hasnans) and target.hasnans:
@@ -4527,9 +4540,10 @@ def __contains__(self, key: Any) -> bool:
         except (OverflowError, TypeError, ValueError):
             return False
 
-    @final
-    def __hash__(self):
-        raise TypeError(f"unhashable type: {repr(type(self).__name__)}")
+    # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
+    # Incompatible types in assignment (expression has type "None", base class
+    # "object" defined the type as "Callable[[object], int]")
+    __hash__: None  # type: ignore[assignment]
 
     @final
     def __setitem__(self, key, value):

pandas/core/indexes/category.py

@@ -487,6 +487,21 @@ def _maybe_cast_indexer(self, key) -> int:
                 return -1
             raise
 
+    def _maybe_cast_listlike_indexer(self, values) -> CategoricalIndex:
+        if isinstance(values, CategoricalIndex):
+            values = values._data
+        if isinstance(values, Categorical):
+            # Indexing on codes is more efficient if categories are the same,
+            #  so we can apply some optimizations based on the degree of
+            #  dtype-matching.
+            cat = self._data._encode_with_my_categories(values)
+            codes = cat._codes
+        else:
+            codes = self.categories.get_indexer(values)
+            codes = codes.astype(self.codes.dtype, copy=False)
+            cat = self._data._from_backing_data(codes)
+        return type(self)._simple_new(cat)
+
     def _get_indexer(
         self,
         target: Index,

pandas/core/reshape/pivot.py

@@ -482,7 +482,7 @@ def pivot(
     if columns is None:
         raise TypeError("pivot() missing 1 required argument: 'columns'")
 
-    columns = com.convert_to_list_like(columns)
+    columns_listlike = com.convert_to_list_like(columns)
 
     if values is None:
         if index is not None:
@@ -494,28 +494,27 @@ def pivot(
         # error: Unsupported operand types for + ("List[Any]" and "ExtensionArray")
         # error: Unsupported left operand type for + ("ExtensionArray")
         indexed = data.set_index(
-            cols + columns, append=append  # type: ignore[operator]
+            cols + columns_listlike, append=append  # type: ignore[operator]
         )
     else:
         if index is None:
-            index = [Series(data.index, name=data.index.name)]
+            index_list = [Series(data.index, name=data.index.name)]
         else:
-            index = com.convert_to_list_like(index)
-            index = [data[idx] for idx in index]
+            index_list = [data[idx] for idx in com.convert_to_list_like(index)]
 
-        data_columns = [data[col] for col in columns]
-        index.extend(data_columns)
-        index = MultiIndex.from_arrays(index)
+        data_columns = [data[col] for col in columns_listlike]
+        index_list.extend(data_columns)
+        multiindex = MultiIndex.from_arrays(index_list)
 
         if is_list_like(values) and not isinstance(values, tuple):
             # Exclude tuple because it is seen as a single column name
             values = cast(Sequence[Hashable], values)
             indexed = data._constructor(
-                data[values]._values, index=index, columns=values
+                data[values]._values, index=multiindex, columns=values
             )
         else:
-            indexed = data._constructor_sliced(data[values]._values, index=index)
-    return indexed.unstack(columns)
+            indexed = data._constructor_sliced(data[values]._values, index=multiindex)
+    return indexed.unstack(columns_listlike)
 
 
 def crosstab(

pandas/core/series.py

@@ -305,7 +305,6 @@ class Series(base.IndexOpsMixin, generic.NDFrame):
     hasnans = property(  # type: ignore[assignment]
         base.IndexOpsMixin.hasnans.func, doc=base.IndexOpsMixin.hasnans.__doc__
     )
-    __hash__ = generic.NDFrame.__hash__
     _mgr: SingleManager
     div: Callable[[Series, Any], Series]
     rdiv: Callable[[Series, Any], Series]

pandas/io/excel/_odswriter.py

@@ -29,6 +29,7 @@ def __init__(
         storage_options: StorageOptions = None,
         if_sheet_exists: str | None = None,
         engine_kwargs: dict[str, Any] | None = None,
+        **kwargs,
     ):
         from odf.opendocument import OpenDocumentSpreadsheet
 

pandas/io/excel/_openpyxl.py

@@ -19,7 +19,10 @@
     BaseExcelReader,
     ExcelWriter,
 )
-from pandas.io.excel._util import validate_freeze_panes
+from pandas.io.excel._util import (
+    combine_kwargs,
+    validate_freeze_panes,
+)
 
 if TYPE_CHECKING:
     from openpyxl.descriptors.serialisable import Serialisable
@@ -39,10 +42,13 @@ def __init__(
         storage_options: StorageOptions = None,
         if_sheet_exists: str | None = None,
         engine_kwargs: dict[str, Any] | None = None,
+        **kwargs,
     ):
         # Use the openpyxl module as the Excel writer.
         from openpyxl.workbook import Workbook
 
+        engine_kwargs = combine_kwargs(engine_kwargs, kwargs)
+
         super().__init__(
             path,
             mode=mode,

pandas/io/excel/_util.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
 
-from typing import MutableMapping
+from typing import (
+    Any,
+    MutableMapping,
+)
 
 from pandas.compat._optional import import_optional_dependency
 
@@ -246,3 +249,30 @@ def pop_header_name(row, index_col):
     header_name = None if header_name == "" else header_name
 
     return header_name, row[:i] + [""] + row[i + 1 :]
+
+
+def combine_kwargs(engine_kwargs: dict[str, Any] | None, kwargs: dict) -> dict:
+    """
+    Used to combine two sources of kwargs for the backend engine.
+
+    Use of kwargs is deprecated, this function is solely for use in 1.3 and should
+    be removed in 1.4/2.0. Also _base.ExcelWriter.__new__ ensures either engine_kwargs
+    or kwargs must be None or empty respectively.
+
+    Parameters
+    ----------
+    engine_kwargs: dict
+        kwargs to be passed through to the engine.
+    kwargs: dict
+        kwargs to be psased through to the engine (deprecated)
+
+    Returns
+    -------
+    engine_kwargs combined with kwargs
+    """
+    if engine_kwargs is None:
+        result = {}
+    else:
+        result = engine_kwargs.copy()
+    result.update(kwargs)
+    return result

pandas/io/excel/_xlsxwriter.py

@@ -6,7 +6,10 @@
 from pandas._typing import StorageOptions
 
 from pandas.io.excel._base import ExcelWriter
-from pandas.io.excel._util import validate_freeze_panes
+from pandas.io.excel._util import (
+    combine_kwargs,
+    validate_freeze_panes,
+)
 
 
 class _XlsxStyler:
@@ -175,11 +178,12 @@ def __init__(
         storage_options: StorageOptions = None,
         if_sheet_exists: str | None = None,
         engine_kwargs: dict[str, Any] | None = None,
+        **kwargs,
     ):
         # Use the xlsxwriter module as the Excel writer.
         from xlsxwriter import Workbook
 
-        engine_kwargs = engine_kwargs or {}
+        engine_kwargs = combine_kwargs(engine_kwargs, kwargs)
 
         if mode == "a":
             raise ValueError("Append mode is not supported with xlsxwriter!")

pandas/io/excel/_xlwt.py

@@ -9,7 +9,10 @@
 from pandas._typing import StorageOptions
 
 from pandas.io.excel._base import ExcelWriter
-from pandas.io.excel._util import validate_freeze_panes
+from pandas.io.excel._util import (
+    combine_kwargs,
+    validate_freeze_panes,
+)
 
 if TYPE_CHECKING:
     from xlwt import XFStyle
@@ -30,10 +33,13 @@ def __init__(
         storage_options: StorageOptions = None,
         if_sheet_exists: str | None = None,
         engine_kwargs: dict[str, Any] | None = None,
+        **kwargs,
     ):
         # Use the xlwt module as the Excel writer.
         import xlwt
 
+        engine_kwargs = combine_kwargs(engine_kwargs, kwargs)
+
         if mode == "a":
             raise ValueError("Append mode is not supported with xlwt!")
 

pandas/tests/arrays/string_/test_string.py

@@ -2,9 +2,6 @@
 This module tests the functionality of StringArray and ArrowStringArray.
 Tests for the str accessors are in pandas/tests/strings/test_string_array.py
 """
-
-import re
-
 import numpy as np
 import pytest
 
@@ -314,7 +311,7 @@ def test_astype_int(dtype):
     tm.assert_numpy_array_equal(result, expected)
 
     arr = pd.array(["1", pd.NA, "3"], dtype=dtype)
-    msg = re.escape("int() argument must be a string, a bytes-like object or a number")
+    msg = r"int\(\) argument must be a string, a bytes-like object or a( real)? number"
     with pytest.raises(TypeError, match=msg):
         arr.astype("int64")
 

pandas/tests/extension/test_sparse.py

@@ -13,6 +13,7 @@
 be added to the array-specific tests in `pandas/tests/arrays/`.
 
 """
+
 import numpy as np
 import pytest