Merge remote-tracking branch 'upstream/master' into typ_c_parser

Author: phofl

.github/workflows/ci.yml

@@ -23,8 +23,9 @@ jobs:
         shell: bash -l {0}
 
     concurrency:
-      group: ${{ github.ref }}-checks
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-checks
+      cancel-in-progress: true
 
     steps:
     - name: Checkout
@@ -102,7 +103,8 @@ jobs:
     runs-on: ubuntu-latest
 
     concurrency:
-      group: ${{ github.ref }}-web-docs
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs
       cancel-in-progress: true
 
     steps:
@@ -160,11 +162,20 @@ jobs:
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
     strategy:
       matrix:
         pattern: ["not slow and not network and not clipboard", "slow"]
     concurrency:
-      group: ${{ github.ref }}-data_manager-${{ matrix.pattern }}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager-${{ matrix.pattern }}
       cancel-in-progress: true
 
     steps:

.github/workflows/database.yml

@@ -31,8 +31,9 @@ jobs:
       fail-fast: false
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.ENV_FILE }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.ENV_FILE }}
+      cancel-in-progress: true
 
     services:
       mysql:
@@ -62,6 +63,14 @@ jobs:
         ports:
           - 5432:5432
 
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
+
     steps:
     - name: Checkout
       uses: actions/checkout@v2

.github/workflows/posix.yml

@@ -46,8 +46,17 @@ jobs:
       TEST_ARGS: ${{ matrix.settings[6] }}
       PYTEST_TARGET:  pandas
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.settings[0] }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}
+      cancel-in-progress: true
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
 
     steps:
     - name: Checkout

.github/workflows/pre-commit.yml

@@ -11,8 +11,9 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     concurrency:
-      group: ${{ github.ref }}-pre-commit
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
+      cancel-in-progress: true
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.github/workflows/python-dev.yml

@@ -31,8 +31,9 @@ jobs:
     timeout-minutes: 80
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      #https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/sdist.yml

@@ -25,8 +25,9 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
     concurrency:
-      group: ${{github.ref}}-${{matrix.python-version}}-sdist
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/python/black
-    rev: 21.10b0
+    rev: 21.11b1
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -55,11 +55,11 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.0
+    rev: v2.29.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]

asv_bench/benchmarks/frame_ctor.py

@@ -19,7 +19,10 @@
     )
 except ImportError:
     # For compatibility with older versions
-    from pandas.core.datetools import *  # noqa
+    from pandas.core.datetools import (
+        Hour,
+        Nano,
+    )
 
 
 class FromDicts:

ci/deps/actions-38-db-min.yaml

@@ -38,8 +38,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
-  - flask
 
   # sql
   - psycopg2=2.8.4

ci/deps/actions-38-db.yaml

@@ -14,6 +14,7 @@ dependencies:
   # pandas dependencies
   - aiobotocore<2.0.0
   - beautifulsoup4
+  - boto3
   - botocore>=1.11
   - dask
   - fastparquet>=0.4.0
@@ -22,8 +23,6 @@ dependencies:
   - geopandas
   - html5lib
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - nomkl
   - numexpr
   - numpy=1.18

ci/deps/actions-38-locale.yaml

@@ -14,14 +14,12 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - flask
   - html5lib
   - ipython
   - jinja2
   - jedi
   - lxml
   - matplotlib<3.3.0
-  - moto
   - nomkl
   - numexpr
   - numpy<1.20  # GH#39541 compat with pyarrow<3
@@ -34,7 +32,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
   - pyarrow=1.0.1
   - pip
   - pip:

ci/deps/actions-38-slow.yaml

@@ -13,6 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
+  - boto3
   - fsspec>=0.7.4
   - html5lib
   - lxml
@@ -27,12 +28,9 @@ dependencies:
   - python-dateutil
   - pytz
   - s3fs>=0.4.0
-  - moto>=1.3.14
   - scipy
   - sqlalchemy
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
-  - flask
   - numba

ci/deps/actions-39-slow.yaml

@@ -15,14 +15,13 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
+  - boto3
   - fsspec>=0.8.0
   - gcsfs
   - html5lib
   - jinja2
   - lxml
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - numba
   - numexpr
   - numpy

ci/deps/actions-39.yaml

@@ -14,14 +14,13 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
+  - boto3
   - fsspec>=0.8.0
   - gcsfs
   - html5lib
   - jinja2
   - lxml
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - numba
   - numexpr
   - numpy

ci/deps/azure-windows-38.yaml

@@ -16,10 +16,8 @@ dependencies:
   - blosc
   - bottleneck
   - fastparquet>=0.4.0
-  - flask
   - fsspec>=0.8.0
   - matplotlib=3.3.2
-  - moto>=1.3.14
   - numba
   - numexpr
   - numpy=1.18

ci/deps/azure-windows-39.yaml

@@ -21,8 +21,6 @@ dependencies:
   - jinja2
   - lxml
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - numba
   - numexpr
   - numpy

doc/source/ecosystem.rst

@@ -35,11 +35,13 @@ Data cleaning and validation
 
 Pyjanitor provides a clean API for cleaning data, using method chaining.
 
-`Engarde <https://engarde.readthedocs.io/en/latest/>`__
+`Pandera <https://pandera.readthedocs.io/en/stable/>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Engarde is a lightweight library used to explicitly state assumptions about your datasets
-and check that they're *actually* true.
+Pandera provides a flexible and expressive API for performing data validation on dataframes
+to make data processing pipelines more readable and robust.
+Dataframes contain information that pandera explicitly validates at runtime. This is useful in
+production-critical data pipelines or reproducible research settings.
 
 `pandas-path <https://github.com/drivendataorg/pandas-path/>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/user_guide/io.rst

@@ -161,6 +161,18 @@ squeeze : boolean, default ``False``
      the data.
 prefix : str, default ``None``
   Prefix to add to column numbers when no header, e.g. 'X' for X0, X1, ...
+
+  .. deprecated:: 1.4.0
+     Use a list comprehension on the DataFrame's columns after calling ``read_csv``.
+
+  .. ipython:: python
+
+     data = "col1,col2,col3\na,b,1"
+
+     df = pd.read_csv(StringIO(data))
+     df.columns = [f"pre_{col}" for col in df.columns]
+     df
+
 mangle_dupe_cols : boolean, default ``True``
   Duplicate columns will be specified as 'X', 'X.1'...'X.N', rather than 'X'...'X'.
   Passing in ``False`` will cause data to be overwritten if there are duplicate

doc/source/whatsnew/v0.5.0.rst

@@ -28,7 +28,6 @@ New features
 - :ref:`Added <indexing.set_index>` convenience ``set_index`` function for creating a DataFrame index from its existing columns
 - :ref:`Implemented <groupby.multiindex>` ``groupby`` hierarchical index level name  (:issue:`223`)
 - :ref:`Added <io.store_in_csv>` support for different delimiters in ``DataFrame.to_csv`` (:issue:`244`)
-- TODO: DOCS ABOUT TAKE METHODS
 
 Performance enhancements
 ~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/whatsnew/v1.4.0.rst

@@ -217,6 +217,7 @@ Other enhancements
 - Added "Juneteenth National Independence Day" to
   ``USFederalHolidayCalendar``.  See also `Other API changes`_.
 - :meth:`.Rolling.var`, :meth:`.Expanding.var`, :meth:`.Rolling.std`, :meth:`.Expanding.std` now support `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`44461`)
+- :meth:`Series.info` has been added, for compatibility with :meth:`DataFrame.info` (:issue:`5167`)
 
 
 .. ---------------------------------------------------------------------------
@@ -516,8 +517,10 @@ Other Deprecations
 - Deprecated casting behavior when setting timezone-aware value(s) into a timezone-aware :class:`Series` or :class:`DataFrame` column when the timezones do not match. Previously this cast to object dtype. In a future version, the values being inserted will be converted to the series or column's existing timezone (:issue:`37605`)
 - Deprecated casting behavior when passing an item with mismatched-timezone to :meth:`DatetimeIndex.insert`, :meth:`DatetimeIndex.putmask`, :meth:`DatetimeIndex.where` :meth:`DatetimeIndex.fillna`, :meth:`Series.mask`, :meth:`Series.where`, :meth:`Series.fillna`, :meth:`Series.shift`, :meth:`Series.replace`, :meth:`Series.reindex` (and :class:`DataFrame` column analogues). In the past this has cast to object dtype. In a future version, these will cast the passed item to the index or series's timezone (:issue:`37605`)
 - Deprecated the 'errors' keyword argument in :meth:`Series.where`, :meth:`DataFrame.where`, :meth:`Series.mask`, and meth:`DataFrame.mask`; in a future version the argument will be removed (:issue:`44294`)
+- Deprecated the ``prefix`` keyword argument in :func:`read_csv` and :func:`read_table`, in a future version the argument will be removed (:issue:`43396`)
 - Deprecated :meth:`PeriodIndex.astype` to ``datetime64[ns]`` or ``DatetimeTZDtype``, use ``obj.to_timestamp(how).tz_localize(dtype.tz)`` instead (:issue:`44398`)
 - Deprecated passing non boolean argument to sort in :func:`concat` (:issue:`41518`)
+- Deprecated passing arguments as positional for :func:`read_fwf` other than ``filepath_or_buffer`` (:issue:`41485`):
 - Deprecated passing ``skipna=None`` for :meth:`DataFrame.mad` and :meth:`Series.mad`, pass ``skipna=True`` instead (:issue:`44580`)
 - Deprecated :meth:`DateOffset.apply`, use ``offset + other`` instead (:issue:`44522`)
 - A deprecation warning is now shown for :meth:`DataFrame.to_latex` indicating the arguments signature may change and emulate more the arguments to :meth:`.Styler.to_latex` in future versions (:issue:`44411`)
@@ -621,12 +624,14 @@ Timezones
 
 Numeric
 ^^^^^^^
+- Bug in floor-dividing a list or tuple of integers by a :class:`Series` incorrectly raising (:issue:`44674`)
 - Bug in :meth:`DataFrame.rank` raising ``ValueError`` with ``object`` columns and ``method="first"`` (:issue:`41931`)
 - Bug in :meth:`DataFrame.rank` treating missing values and extreme values as equal (for example ``np.nan`` and ``np.inf``), causing incorrect results when ``na_option="bottom"`` or ``na_option="top`` used (:issue:`41931`)
 - Bug in ``numexpr`` engine still being used when the option ``compute.use_numexpr`` is set to ``False`` (:issue:`32556`)
 - Bug in :class:`DataFrame` arithmetic ops with a subclass whose :meth:`_constructor` attribute is a callable other than the subclass itself (:issue:`43201`)
 - Bug in arithmetic operations involving :class:`RangeIndex` where the result would have the incorrect ``name`` (:issue:`43962`)
 - Bug in arithmetic operations involving :class:`Series` where the result could have the incorrect ``name`` when the operands having matching NA or matching tuple names (:issue:`44459`)
+- Bug in division with ``IntegerDtype`` or ``BooleanDtype`` array and NA scalar incorrectly raising (:issue:`44685`)
 -
 
 Conversion
@@ -677,6 +682,7 @@ Indexing
 - Bug in :meth:`Series.reset_index` not ignoring ``name`` argument when ``drop`` and ``inplace`` are set to ``True`` (:issue:`44575`)
 - Bug in :meth:`DataFrame.loc.__setitem__` and :meth:`DataFrame.iloc.__setitem__` with mixed dtypes sometimes failing to operate in-place (:issue:`44345`)
 - Bug in :meth:`DataFrame.loc.__getitem__` incorrectly raising ``KeyError`` when selecting a single column with a boolean key (:issue:`44322`).
+- Bug in setting :meth:`DataFrame.iloc` with a single ``ExtensionDtype`` column and setting 2D values e.g. ``df.iloc[:] = df.values`` incorrectly raising (:issue:`44514`)
 - Bug in indexing on columns with ``loc`` or ``iloc`` using a slice with a negative step with ``ExtensionDtype`` columns incorrectly raising (:issue:`44551`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` returning boolean mask instead of array of integers for a non unique and non monotonic index (:issue:`44084`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` not handling targets of ``dtype`` 'object' with NaNs correctly (:issue:`44482`)
@@ -796,7 +802,9 @@ ExtensionArray
 - NumPy ufuncs ``np.abs``, ``np.positive``, ``np.negative`` now correctly preserve dtype when called on ExtensionArrays that implement ``__abs__, __pos__, __neg__``, respectively. In particular this is fixed for :class:`TimedeltaArray` (:issue:`43899`)
 - NumPy ufuncs ``np.minimum.reduce`` and ``np.maximum.reduce`` now work correctly instead of raising ``NotImplementedError`` on :class:`Series` with ``IntegerDtype`` or ``FloatDtype`` (:issue:`43923`)
 - Avoid raising ``PerformanceWarning`` about fragmented DataFrame when using many columns with an extension dtype (:issue:`44098`)
+- Bug in :class:`IntegerArray` and :class:`FloatingArray` construction incorrectly coercing mismatched NA values (e.g. ``np.timedelta64("NaT")``) to numeric NA (:issue:`44514`)
 - Bug in :meth:`BooleanArray.__eq__` and :meth:`BooleanArray.__ne__` raising ``TypeError`` on comparison with an incompatible type (like a string). This caused :meth:`DataFrame.replace` to sometimes raise a ``TypeError`` if a nullable boolean column was included (:issue:`44499`)
+- Bug in :meth:`Series.where` with ``ExtensionDtype`` when ``other`` is a NA scalar incompatible with the series dtype (e.g. ``NaT`` with a numeric dtype) incorrectly casting to a compatible NA value (:issue:`44697`)
 -
 
 Styler
@@ -822,7 +830,7 @@ Other
 - Bug in :meth:`FloatingArray.equals` failing to consider two arrays equal if they contain ``np.nan`` values (:issue:`44382`)
 - Bug in :meth:`DataFrame.shift` with ``axis=1`` and ``ExtensionDtype`` columns incorrectly raising when an incompatible ``fill_value`` is passed (:issue:`44564`)
 - Bug in :meth:`DataFrame.diff` when passing a NumPy integer object instead of an ``int`` object (:issue:`44572`)
--
+- Bug in :meth:`Series.replace` raising ``ValueError`` when using ``regex=True`` with a :class:`Series` containing ``np.nan`` values (:issue:`43344`)
 
 .. ***DO NOT USE THIS SECTION***