Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -2,74 +2,81 @@ name: CI
 
 on:
   push:
-    branches: master
+    branches: [master]
   pull_request:
     branches:
       - master
       - 1.2.x
 
 env:
   ENV_FILE: environment.yml
+  PANDAS_CI: 1
 
 jobs:
   checks:
     name: Checks
     runs-on: ubuntu-latest
-    steps:
-
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
+    defaults:
+      run:
+        shell: bash -l {0}
 
+    steps:
     - name: Checkout
       uses: actions/checkout@v1
 
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
       if: always()
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
-      if: always()
+    - name: Cache conda
+      uses: actions/cache@v2
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - name: Linting
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ${{ env.ENV_FILE }}
+        use-only-tar-bz2: true
+
+    - name: Environment Detail
       run: |
-        source activate pandas-dev
-        ci/code_checks.sh lint
+        conda info
+        conda list
+
+    - name: Build Pandas
+      run: |
+        python setup.py build_ext -j 2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
+
+    - name: Linting
+      run: ci/code_checks.sh lint
       if: always()
 
     - name: Checks on imported code
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh code
+      run: ci/code_checks.sh code
       if: always()
 
     - name: Running doctests
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh doctests
+      run: ci/code_checks.sh doctests
       if: always()
 
     - name: Docstring validation
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh docstrings
+      run: ci/code_checks.sh docstrings
       if: always()
 
     - name: Typing validation
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh typing
+      run: ci/code_checks.sh typing
       if: always()
 
     - name: Testing docstring validation script
-      run: |
-        source activate pandas-dev
-        pytest --capture=no --strict-markers scripts
+      run: pytest --capture=no --strict-markers scripts
       if: always()
 
     - name: Running benchmarks
       run: |
-        source activate pandas-dev
         cd asv_bench
         asv check -E existing
         git remote add upstream https://github.com/pandas-dev/pandas.git
@@ -106,7 +113,6 @@ jobs:
       run: |
         source activate pandas-dev
         python web/pandas_web.py web/pandas --target-path=web/build
-
     - name: Build documentation
       run: |
         source activate pandas-dev

.pre-commit-config.yaml

@@ -145,11 +145,12 @@ repos:
         language: pygrep
         types_or: [python, cython]
     -   id: unwanted-typing
-        name: Check for use of comment-based annotation syntax and missing error codes
+        name: Check for outdated annotation syntax and missing error codes
         entry: |
             (?x)
             \#\ type:\ (?!ignore)|
-            \#\ type:\s?ignore(?!\[)
+            \#\ type:\s?ignore(?!\[)|
+            \)\ ->\ \"
         language: pygrep
         types: [python]
     -   id: np-bool

doc/source/reference/style.rst

@@ -39,6 +39,8 @@ Style application
    Styler.set_td_classes
    Styler.set_table_styles
    Styler.set_table_attributes
+   Styler.set_tooltips
+   Styler.set_tooltips_class
    Styler.set_caption
    Styler.set_properties
    Styler.set_uuid

doc/source/user_guide/enhancingperf.rst

@@ -247,7 +247,7 @@ We've gotten another big improvement. Let's check again where the time is spent:
 
 .. ipython:: python
 
-   %%prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy())
+   %prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy())
 
 As one might expect, the majority of the time is now spent in ``apply_integrate_f``,
 so if we wanted to make anymore efficiencies we must continue to concentrate our

doc/source/whatsnew/v0.8.0.rst

@@ -176,7 +176,7 @@ New plotting methods
 Vytautas Jancauskas, the 2012 GSOC participant, has added many new plot
 types. For example, ``'kde'`` is a new option:
 
-.. ipython:: python
+.. code-block:: python
 
    s = pd.Series(
        np.concatenate((np.random.randn(1000), np.random.randn(1000) * 0.5 + 3))

doc/source/whatsnew/v1.2.0.rst

@@ -751,7 +751,7 @@ Plotting
 
 - Bug in :meth:`DataFrame.plot` was rotating xticklabels when ``subplots=True``, even if the x-axis wasn't an irregular time series (:issue:`29460`)
 - Bug in :meth:`DataFrame.plot` where a marker letter in the ``style`` keyword sometimes caused a ``ValueError`` (:issue:`21003`)
-- Bug in :meth:`DataFrame.plot.bar` and :meth:`Series.plot.bar` where ticks positions were assigned by value order instead of using the actual value for numeric or a smart ordering for string (:issue:`26186`, :issue:`11465`)
+- Bug in :meth:`DataFrame.plot.bar` and :meth:`Series.plot.bar` where ticks positions were assigned by value order instead of using the actual value for numeric or a smart ordering for string (:issue:`26186`, :issue:`11465`). This fix has been reverted in pandas 1.2.1, see :doc:`v1.2.1`
 - Twinned axes were losing their tick labels which should only happen to all but the last row or column of 'externally' shared axes (:issue:`33819`)
 - Bug in :meth:`Series.plot` and :meth:`DataFrame.plot` was throwing a :exc:`ValueError` when the Series or DataFrame was
   indexed by a :class:`.TimedeltaIndex` with a fixed frequency and the x-axis lower limit was greater than the upper limit (:issue:`37454`)

doc/source/whatsnew/v1.2.1.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_121:
 
-What's new in 1.2.1 (January ??, 2021)
+What's new in 1.2.1 (January 18, 2021)
 --------------------------------------
 
 These are the changes in pandas 1.2.1. See :ref:`release` for a full changelog
@@ -14,23 +14,29 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
-- Fixed regression in :meth:`to_csv` that created corrupted zip files when there were more rows than ``chunksize`` (:issue:`38714`)
-- Fixed regression in ``groupby().rolling()`` where :class:`MultiIndex` levels were dropped (:issue:`38523`)
-- Fixed regression in repr of float-like strings of an ``object`` dtype having trailing 0's truncated after the decimal (:issue:`38708`)
-- Fixed regression in :meth:`DataFrame.groupby()` with :class:`Categorical` grouping column not showing unused categories for ``grouped.indices`` (:issue:`38642`)
-- Fixed regression in :meth:`DataFrame.any` and :meth:`DataFrame.all` not returning a result for tz-aware ``datetime64`` columns (:issue:`38723`)
-- Fixed regression in :meth:`DataFrame.__setitem__` raising ``ValueError`` when expanding :class:`DataFrame` and new column is from type ``"0 - name"`` (:issue:`39010`)
-- Fixed regression in :meth:`.GroupBy.sem` where the presence of non-numeric columns would cause an error instead of being dropped (:issue:`38774`)
-- Fixed regression in :meth:`DataFrame.loc.__setitem__` raising ``ValueError`` when :class:`DataFrame` has unsorted :class:`MultiIndex` columns and indexer is a scalar (:issue:`38601`)
-- Fixed regression in :func:`read_excel` with non-rawbyte file handles (:issue:`38788`)
-- Fixed regression in :meth:`Rolling.skew` and :meth:`Rolling.kurt` modifying the object inplace (:issue:`38908`)
+- Fixed regression in :meth:`~DataFrame.to_csv` that created corrupted zip files when there were more rows than ``chunksize`` (:issue:`38714`)
 - Fixed regression in :meth:`read_csv` and other read functions were the encoding error policy (``errors``) did not default to ``"replace"`` when no encoding was specified (:issue:`38989`)
+- Fixed regression in :func:`read_excel` with non-rawbyte file handles (:issue:`38788`)
+- Fixed regression in :meth:`DataFrame.to_stata` not removing the created file when an error occured (:issue:`39202`)
+- Fixed regression in ``DataFrame.__setitem__`` raising ``ValueError`` when expanding :class:`DataFrame` and new column is from type ``"0 - name"`` (:issue:`39010`)
+- Fixed regression in setting with :meth:`DataFrame.loc`  raising ``ValueError`` when :class:`DataFrame` has unsorted :class:`MultiIndex` columns and indexer is a scalar (:issue:`38601`)
+- Fixed regression in setting with :meth:`DataFrame.loc` raising ``KeyError`` with :class:`MultiIndex` and list-like columns indexer enlarging :class:`DataFrame` (:issue:`39147`)
+- Fixed regression in :meth:`~DataFrame.groupby()` with :class:`Categorical` grouping column not showing unused categories for ``grouped.indices`` (:issue:`38642`)
+- Fixed regression in :meth:`.GroupBy.sem` where the presence of non-numeric columns would cause an error instead of being dropped (:issue:`38774`)
+- Fixed regression in :meth:`.DataFrameGroupBy.diff` raising for ``int8`` and ``int16`` columns (:issue:`39050`)
+- Fixed regression in :meth:`DataFrame.groupby` when aggregating an ``ExtensionDType`` that could fail for non-numeric values (:issue:`38980`)
+- Fixed regression in :meth:`.Rolling.skew` and :meth:`.Rolling.kurt` modifying the object inplace (:issue:`38908`)
+- Fixed regression in :meth:`DataFrame.any` and :meth:`DataFrame.all` not returning a result for tz-aware ``datetime64`` columns (:issue:`38723`)
+- Fixed regression in :meth:`DataFrame.apply` with ``axis=1`` using str accessor in apply function (:issue:`38979`)
 - Fixed regression in :meth:`DataFrame.replace` raising ``ValueError`` when :class:`DataFrame` has dtype ``bytes`` (:issue:`38900`)
-- Fixed regression in :meth:`DataFrameGroupBy.diff` raising for ``int8`` and ``int16`` columns (:issue:`39050`)
+- Fixed regression in :meth:`Series.fillna` that raised ``RecursionError`` with ``datetime64[ns, UTC]`` dtype (:issue:`38851`)
+- Fixed regression in comparisons between ``NaT`` and ``datetime.date`` objects incorrectly returning ``True`` (:issue:`39151`)
+- Fixed regression in repr of float-like strings of an ``object`` dtype having trailing 0's truncated after the decimal (:issue:`38708`)
 - Fixed regression that raised ``AttributeError`` with PyArrow versions [0.16.0, 1.0.0) (:issue:`38801`)
-- Fixed regression in :meth:`DataFrame.groupby` when aggregating an :class:`ExtensionDType` that could fail for non-numeric values (:issue:`38980`)
--
--
+- Fixed regression in :func:`pandas.testing.assert_frame_equal` raising ``TypeError`` with ``check_like=True`` when :class:`Index` or columns have mixed dtype (:issue:`39168`)
+
+We have reverted a commit that resulted in several plotting related regressions in pandas 1.2.0 (:issue:`38969`, :issue:`38736`, :issue:`38865`, :issue:`38947` and :issue:`39126`).
+As a result, bugs reported as fixed in pandas 1.2.0 related to inconsistent tick labeling in bar plots are again present (:issue:`26186` and :issue:`11465`)
 
 .. ---------------------------------------------------------------------------
 
@@ -41,7 +47,7 @@ Bug fixes
 
 - Bug in :meth:`read_csv` with ``float_precision="high"`` caused segfault or wrong parsing of long exponent strings. This resulted in a regression in some cases as the default for ``float_precision`` was changed in pandas 1.2.0 (:issue:`38753`)
 - Bug in :func:`read_csv` not closing an opened file handle when a ``csv.Error`` or ``UnicodeDecodeError`` occurred while initializing (:issue:`39024`)
--
+- Bug in :func:`pandas.testing.assert_index_equal` raising ``TypeError`` with ``check_order=False`` when :class:`Index` has mixed dtype (:issue:`39168`)
 
 .. ---------------------------------------------------------------------------
 
@@ -55,8 +61,6 @@ Other
 - Bumped minimum pymysql version to 0.8.1 to avoid test failures (:issue:`38344`)
 - Fixed build failure on MacOS 11 in Python 3.9.1 (:issue:`38766`)
 - Added reference to backwards incompatible ``check_freq`` arg of :func:`testing.assert_frame_equal` and :func:`testing.assert_series_equal` in :ref:`pandas 1.1.0 whats new <whatsnew_110.api_breaking.testing.check_freq>` (:issue:`34050`)
--
--
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.3.0.rst

@@ -50,7 +50,9 @@ Other enhancements
 - :func:`pandas.read_excel` can now auto detect .xlsb files (:issue:`35416`)
 - :meth:`.Rolling.sum`, :meth:`.Expanding.sum`, :meth:`.Rolling.mean`, :meth:`.Expanding.mean`, :meth:`.Rolling.median`, :meth:`.Expanding.median`, :meth:`.Rolling.max`, :meth:`.Expanding.max`, :meth:`.Rolling.min`, and :meth:`.Expanding.min` now support ``Numba`` execution with the ``engine`` keyword (:issue:`38895`)
 - :meth:`DataFrame.apply` can now accept NumPy unary operators as strings, e.g. ``df.apply("sqrt")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
-- :meth:`DataFrame.apply` can now accept non-callable :class:`DataFrame` properties as strings, e.g. ``df.apply("size")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
+- :meth:`DataFrame.apply` can now accept non-callable DataFrame properties as strings, e.g. ``df.apply("size")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
+- :meth:`Series.apply` can now accept list-like or dictionary-like arguments that aren't lists or dictionaries, e.g. ``ser.apply(np.array(["sum", "mean"]))``, which was already the case for :meth:`DataFrame.apply` (:issue:`39140`)
+- :meth:`.Styler.set_tooltips` allows on hover tooltips to be added to styled HTML dataframes.
 
 .. ---------------------------------------------------------------------------
 
@@ -62,6 +64,36 @@ Notable bug fixes
 These are bug fixes that might have notable behavior changes.
 
 
+Preserve dtypes in  :meth:`~pandas.DataFrame.combine_first`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:meth:`~pandas.DataFrame.combine_first` will now preserve dtypes (:issue:`7509`)
+
+.. ipython:: python
+
+   df1 = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=[0, 1, 2])
+   df1
+   df2 = pd.DataFrame({"B": [4, 5, 6], "C": [1, 2, 3]}, index=[2, 3, 4])
+   df2
+   combined = df1.combine_first(df2)
+
+*pandas 1.2.x*
+
+.. code-block:: ipython
+
+   In [1]: combined.dtypes
+   Out[2]:
+   A    float64
+   B    float64
+   C    float64
+   dtype: object
+
+*pandas 1.3.0*
+
+.. ipython:: python
+
+   combined.dtypes
+
 
 .. _whatsnew_130.api_breaking.deps:
 
@@ -186,6 +218,8 @@ Categorical
 - Bug in ``CategoricalIndex.reindex`` failed when ``Index`` passed with elements all in category (:issue:`28690`)
 - Bug where constructing a :class:`Categorical` from an object-dtype array of ``date`` objects did not round-trip correctly with ``astype`` (:issue:`38552`)
 - Bug in constructing a :class:`DataFrame` from an ``ndarray`` and a :class:`CategoricalDtype` (:issue:`38857`)
+- Bug in :meth:`DataFrame.reindex` was throwing ``IndexError`` when new index contained duplicates and old index was :class:`CategoricalIndex` (:issue:`38906`)
+- Bug in setting categorical values into an object-dtype column in a :class:`DataFrame` (:issue:`39136`)
 - Bug in :meth:`DataFrame.reindex` was raising ``IndexError`` when new index contained duplicates and old index was :class:`CategoricalIndex` (:issue:`38906`)
 
 Datetimelike
@@ -246,6 +280,7 @@ Indexing
 - Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)
 - Bug in :meth:`DataFrame.loc` dropping levels of :class:`MultiIndex` when :class:`DataFrame` used as input has only one row (:issue:`10521`)
 - Bug in setting ``timedelta64`` values into numeric :class:`Series` failing to cast to object dtype (:issue:`39086`)
+- Bug in setting :class:`Interval` values into a :class:`Series` or :class:`DataFrame` with mismatched :class:`IntervalDtype` incorrectly casting the new values to the existing dtype (:issue:`39120`)
 
 Missing
 ^^^^^^^
@@ -281,6 +316,7 @@ I/O
 - :func:`read_excel` now respects :func:`set_option` (:issue:`34252`)
 - Bug in :func:`read_csv` not switching ``true_values`` and ``false_values`` for nullable ``boolean`` dtype (:issue:`34655`)
 - Bug in :func:`read_json` when ``orient="split"`` does not maintain numeric string index (:issue:`28556`)
+- :meth:`read_sql` returned an empty generator if ``chunksize`` was no-zero and the query returned no results. Now returns a generator with a single empty dataframe (:issue:`34411`)
 
 Period
 ^^^^^^

environment.yml

@@ -68,7 +68,7 @@ dependencies:
 
   # unused (required indirectly may be?)
   - ipywidgets
-  - nbformat
+  - nbformat=5.0.8
   - notebook>=5.7.5
   - pip
 

pandas/_libs/tslibs/nattype.pyx

@@ -1,4 +1,7 @@
+import warnings
+
 from cpython.datetime cimport (
+    PyDate_Check,
     PyDateTime_Check,
     PyDateTime_IMPORT,
     PyDelta_Check,
@@ -125,6 +128,21 @@ cdef class _NaT(datetime):
                 return NotImplemented
             return result
 
+        elif PyDate_Check(other):
+            # GH#39151 don't defer to datetime.date object
+            if op == Py_EQ:
+                return False
+            if op == Py_NE:
+                return True
+            warnings.warn(
+                "Comparison of NaT with datetime.date is deprecated in "
+                "order to match the standard library behavior.  "
+                "In a future version these will be considered non-comparable.",
+                FutureWarning,
+                stacklevel=1,
+            )
+            return False
+
         return NotImplemented
 
     def __add__(self, other):

pandas/_testing/__init__.py

@@ -977,3 +977,11 @@ def loc(x):
 
 def iloc(x):
     return x.iloc
+
+
+def at(x):
+    return x.at
+
+
+def iat(x):
+    return x.iat

pandas/_testing/asserters.py

@@ -29,7 +29,7 @@
     Series,
     TimedeltaIndex,
 )
-from pandas.core.algorithms import take_1d
+from pandas.core.algorithms import safe_sort, take_1d
 from pandas.core.arrays import (
     DatetimeArray,
     ExtensionArray,
@@ -344,8 +344,8 @@ def _get_ilevel_values(index, level):
 
     # If order doesn't matter then sort the index entries
     if not check_order:
-        left = left.sort_values()
-        right = right.sort_values()
+        left = Index(safe_sort(left))
+        right = Index(safe_sort(right))
 
     # MultiIndex special comparison for little-friendly error messages
     if left.nlevels > 1: