Merge branch 'master' into fillna-other-missing-values-not-modified

Author: Mike Phung

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,67 @@
+name: Bug Report
+description: Report incorrect behavior in the pandas library
+title: "BUG: "
+labels: [Bug, Needs Triage]
+
+body:
+  - type: checkboxes
+    id: checks
+    attributes:
+      options:
+        - label: >
+            I have checked that this issue has not already been reported.
+          required: true
+        - label: >
+            I have confirmed this bug exists on the
+            [latest version](https://pandas.pydata.org/docs/whatsnew/index.html) of pandas.
+          required: true
+        - label: >
+            I have confirmed this bug exists on the master branch of pandas.
+  - type: textarea
+    id: example
+    attributes:
+      label: Reproducible Example
+      description: >
+        Please follow [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) on how to
+        provide a minimal, copy-pastable example.
+      placeholder: >
+        import pandas as pd
+
+        df = pd.DataFrame(range(5))
+
+        ...
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: problem
+    attributes:
+      label: Issue Description
+      description: >
+        Please provide a description of the issue shown in the reproducible example.
+    validations:
+      required: true
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected Behavior
+      description: >
+        Please describe or show a code example of the expected behavior.
+    validations:
+      required: true
+  - type: textarea
+    id: version
+    attributes:
+      label: Installed Versions
+      description: >
+        Please paste the output of ``pd.show_versions()``
+      value: >
+        <details>
+
+
+        Replace this line with the output of pd.show_versions()
+
+
+        </details>
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,65 @@
+name: Installation Issue
+description: Report issues installing the pandas library on your system
+title: "BUILD: "
+labels: [Build, Needs Triage]
+
+body:
+  - type: checkboxes
+    id: checks
+    attributes:
+      options:
+        - label: >
+            I have read the [installation guide](https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#installing-pandas).
+          required: true
+  - type: input
+    id: platform
+    attributes:
+      label: Platform
+      description: >
+        Please provide the output of ``import platform; print(platform.platform())``
+    validations:
+      required: true
+  - type: dropdown
+    id: method
+    attributes:
+      label: Installation Method
+      description: >
+        Please provide how you tried to install pandas from a clean environment.
+      options:
+        - pip install
+        - conda install
+        - apt-get install
+        - Built from source
+        - Other
+    validations:
+      required: true
+  - type: input
+    id: pandas
+    attributes:
+      label: pandas Version
+      description: >
+        Please provide the version of pandas you are trying to install.
+    validations:
+      required: true
+  - type: input
+    id: python
+    attributes:
+      label: Python Version
+      description: >
+        Please provide the installed version of Python.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Installation Logs
+      description: >
+        If possible, please copy and paste the installation logs when attempting to install pandas.
+      value: >
+        <details>
+
+
+        Replace this line with the installation logs.
+
+
+        </details>

.github/ISSUE_TEMPLATE/installation_issue.yaml

@@ -0,0 +1,52 @@
+name: Performance Issue
+description: Report slow performance or memory issues when running pandas code
+title: "PERF: "
+labels: [Performance, Needs Triage]
+
+body:
+  - type: checkboxes
+    id: checks
+    attributes:
+      options:
+        - label: >
+            I have checked that this issue has not already been reported.
+          required: true
+        - label: >
+            I have confirmed this issue exists on the
+            [latest version](https://pandas.pydata.org/docs/whatsnew/index.html) of pandas.
+          required: true
+        - label: >
+            I have confirmed this issue exists on the master branch of pandas.
+  - type: textarea
+    id: example
+    attributes:
+      label: Reproducible Example
+      description: >
+        Please provide a minimal, copy-pastable example that quantifies
+        [slow runtime](https://docs.python.org/3/library/timeit.html) or
+        [memory](https://pypi.org/project/memory-profiler/) issues.
+    validations:
+      required: true
+  - type: textarea
+    id: version
+    attributes:
+      label: Installed Versions
+      description: >
+        Please paste the output of ``pd.show_versions()``
+      value: >
+        <details>
+
+
+        Replace this line with the output of pd.show_versions()
+
+
+        </details>
+    validations:
+      required: true
+  - type: textarea
+    id: prior-performance
+    attributes:
+      label: Prior Performance
+      description: >
+        If applicable, please provide the prior version of pandas and output
+        of the same reproducible example where the performance issue did not exist.

.github/ISSUE_TEMPLATE/performance_issue.yaml

@@ -26,6 +26,7 @@ Fixed regressions
 - Fixed regression in :func:`concat` where ``copy=False`` was not honored in ``axis=1`` Series concatenation (:issue:`42501`)
 - Regression in :meth:`Series.nlargest` and :meth:`Series.nsmallest` with nullable integer or float dtype (:issue:`42816`)
 - Fixed regression in :meth:`Series.quantile` with :class:`Int64Dtype` (:issue:`42626`)
+- Fixed regression in :meth:`Series.groupby` and :meth:`DataFrame.groupby` where supplying the ``by`` argument with a Series named with a tuple would incorrectly raise (:issue:`42731`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.3.2.rst

@@ -17,6 +17,7 @@ Fixed regressions
 - Fixed regression in :class:`DataFrame` constructor failing to broadcast for defined :class:`Index` and len one list of :class:`Timestamp` (:issue:`42810`)
 - Performance regression in :meth:`core.window.ewm.ExponentialMovingWindow.mean` (:issue:`42333`)
 - Fixed regression in :meth:`.GroupBy.agg` incorrectly raising in some cases (:issue:`42390`)
+- Fixed regression in :meth:`.GroupBy.apply` where ``nan`` values were dropped even with ``dropna=False`` (:issue:`43205`)
 - Fixed regression in :meth:`.GroupBy.quantile` which was failing with ``pandas.NA`` (:issue:`42849`)
 - Fixed regression in :meth:`merge` where ``on`` columns with ``ExtensionDtype`` or ``bool`` data types were cast to ``object`` in ``right`` and ``outer`` merge (:issue:`40073`)
 - Fixed regression in :meth:`RangeIndex.where` and :meth:`RangeIndex.putmask` raising ``AssertionError`` when result did not represent a :class:`RangeIndex` (:issue:`43240`)

doc/source/whatsnew/v1.3.3.rst

@@ -291,7 +291,7 @@ Performance improvements
 - Performance improvement in :func:`read_stata` (:issue:`43059`)
 - Performance improvement in :meth:`to_datetime` with ``uint`` dtypes (:issue:`42606`)
 - Performance improvement in :meth:`Series.sparse.to_coo` (:issue:`42880`)
--
+- Performance improvement in indexing with a :class:`MultiIndex` indexer on another :class:`MultiIndex` (:issue:43370`)
 
 .. ---------------------------------------------------------------------------
 
@@ -322,7 +322,7 @@ Timedelta
 
 Timezones
 ^^^^^^^^^
--
+- Bug in :meth:`Series.dt.tz_convert` resetting index in a :class:`Series` with :class:`CategoricalIndex` (:issue:`43080`)
 -
 
 Numeric
@@ -361,6 +361,7 @@ Indexing
 - Bug in :meth:`DataFrame.drop` where the error message did not show missing labels with commas when raising ``KeyError`` (:issue:`42881`)
 -
 
+
 Missing
 ^^^^^^^
 - Bug in :meth:`DataFrame.fillna` with limit and no method ignores axis='columns' or ``axis = 1`` (:issue:`40989`)
@@ -383,7 +384,7 @@ I/O
 - Bug in :func:`Series.to_json` and :func:`DataFrame.to_json` where some attributes were skipped when serialising plain Python objects to JSON (:issue:`42768`, :issue:`33043`)
 - Column headers are dropped when constructing a :class:`DataFrame` from a sqlalchemy's ``Row`` object (:issue:`40682`)
 - Bug in unpickling a :class:`Index` with object dtype incorrectly inferring numeric dtypes (:issue:`43188`)
--
+- Bug in :func:`read_csv` where reading multi-header input with unequal lengths incorrectly raising uncontrolled ``IndexError`` (:issue:`43102`)
 
 Period
 ^^^^^^
@@ -407,6 +408,7 @@ Groupby/resample/rolling
 - Bug in :meth:`pandas.DataFrame.rolling` operation along rows (``axis=1``) incorrectly omits columns containing ``float16`` and ``float32`` (:issue:`41779`)
 - Bug in :meth:`Resampler.aggregate` did not allow the use of Named Aggregation (:issue:`32803`)
 - Bug in :meth:`Series.rolling` when the :class:`Series` ``dtype`` was ``Int64`` (:issue:`43016`)
+- Bug in :meth:`DataFrame.groupby.rolling` when specifying ``on`` and calling ``__getitem__`` would subsequently return incorrect results (:issue:`43355`)
 
 Reshaping
 ^^^^^^^^^

doc/source/whatsnew/v1.4.0.rst

@@ -2,6 +2,8 @@ import numpy as np
 
 from pandas._typing import npt
 
+from pandas import MultiIndex
+
 class IndexEngine:
     over_size_threshold: bool
     def __init__(self, vgetter, n: int): ...
@@ -59,7 +61,7 @@ class BaseMultiIndexCodesEngine:
         self,
         target: npt.NDArray[np.object_],
     ) -> npt.NDArray[np.intp]: ...
-    def _extract_level_codes(self, target: object): ...
+    def _extract_level_codes(self, target: MultiIndex) -> np.ndarray: ...
     def get_indexer_with_fill(
         self,
         target: np.ndarray,  # np.ndarray[object] of tuples

pandas/_libs/index.pyi

@@ -622,23 +622,22 @@ cdef class BaseMultiIndexCodesEngine:
                        in zip(self.levels, zt)]
         return self._codes_to_ints(np.array(level_codes, dtype='uint64').T)
 
-    def get_indexer(self, target) -> np.ndarray:
+    def get_indexer(self, target: np.ndarray) -> np.ndarray:
         """
         Returns an array giving the positions of each value of `target` in
         `self.values`, where -1 represents a value in `target` which does not
         appear in `self.values`
 
         Parameters
         ----------
-        target : MultiIndex
+        target : np.ndarray
 
         Returns
         -------
         np.ndarray[intp_t, ndim=1] of the indexer of `target` into
         `self.values`
         """
-        lab_ints = self._extract_level_codes(target)
-        return self._base.get_indexer(self, lab_ints)
+        return self._base.get_indexer(self, target)
 
     def get_indexer_with_fill(self, ndarray target, ndarray values,
                               str method, object limit) -> np.ndarray:
@@ -741,10 +740,9 @@ cdef class BaseMultiIndexCodesEngine:
 
         return self._base.get_loc(self, lab_int)
 
-    def get_indexer_non_unique(self, target):
+    def get_indexer_non_unique(self, target: np.ndarray) -> np.ndarray:
         # target: MultiIndex
-        lab_ints = self._extract_level_codes(target)
-        indexer = self._base.get_indexer_non_unique(self, lab_ints)
+        indexer = self._base.get_indexer_non_unique(self, target)
 
         return indexer
 

pandas/_libs/index.pyx

@@ -69,6 +69,11 @@
 
     from pandas.io.formats.format import EngFormatter
     from pandas.tseries.offsets import DateOffset
+
+    # numpy compatible types
+    NumpyValueArrayLike = Union[npt._ScalarLike_co, npt.ArrayLike]
+    NumpySorter = Optional[npt._ArrayLikeInt_co]
+
 else:
     npt: Any = None
 
@@ -85,6 +90,7 @@
 PandasScalar = Union["Period", "Timestamp", "Timedelta", "Interval"]
 Scalar = Union[PythonScalar, PandasScalar]
 
+
 # timestamp and timedelta convertible types
 
 TimestampConvertibleTypes = Union[