Merge branch 'master' into mypy/base

Author: jreback

.github/workflows/ci.yml

@@ -154,7 +154,9 @@ jobs:
         source activate pandas-dev
         pytest pandas/tests/frame/methods --array-manager
 
-        # indexing iset related (temporary since other tests don't pass yet)
+        # indexing subset (temporary since other tests don't pass yet)
+        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean --array-manager
+        pytest pandas/tests/frame/indexing/test_where.py --array-manager
         pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_multi_index --array-manager
         pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns --array-manager
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups --array-manager

MANIFEST.in

@@ -1,9 +1,4 @@
-include MANIFEST.in
-include LICENSE
 include RELEASE.md
-include README.md
-include setup.py
-include pyproject.toml
 
 graft doc
 prune doc/build
@@ -16,10 +11,12 @@ global-exclude *.bz2
 global-exclude *.csv
 global-exclude *.dta
 global-exclude *.feather
+global-exclude *.tar
 global-exclude *.gz
 global-exclude *.h5
 global-exclude *.html
 global-exclude *.json
+global-exclude *.jsonl
 global-exclude *.pickle
 global-exclude *.png
 global-exclude *.pyc
@@ -40,6 +37,11 @@ global-exclude .DS_Store
 global-exclude .git*
 global-exclude \#*
 
+# GH 39321
+# csv_dir_path fixture checks the existence of the directory
+# exclude the whole directory to avoid running related tests in sdist
+prune pandas/tests/io/parser/data
+
 include versioneer.py
 include pandas/_version.py
 include pandas/io/formats/templates/*.tpl

asv_bench/benchmarks/arithmetic.py

@@ -4,7 +4,13 @@
 import numpy as np
 
 import pandas as pd
-from pandas import DataFrame, Series, Timestamp, date_range, to_timedelta
+from pandas import (
+    DataFrame,
+    Series,
+    Timestamp,
+    date_range,
+    to_timedelta,
+)
 import pandas._testing as tm
 from pandas.core.algorithms import checked_add_with_arr
 

asv_bench/benchmarks/ctors.py

@@ -1,6 +1,12 @@
 import numpy as np
 
-from pandas import DatetimeIndex, Index, MultiIndex, Series, Timestamp
+from pandas import (
+    DatetimeIndex,
+    Index,
+    MultiIndex,
+    Series,
+    Timestamp,
+)
 
 from .pandas_vb_common import tm
 

asv_bench/benchmarks/dtypes.py

@@ -5,7 +5,10 @@
 import pandas as pd
 from pandas import DataFrame
 import pandas._testing as tm
-from pandas.api.types import is_extension_array_dtype, pandas_dtype
+from pandas.api.types import (
+    is_extension_array_dtype,
+    pandas_dtype,
+)
 
 from .pandas_vb_common import (
     datetime_dtypes,

asv_bench/benchmarks/frame_ctor.py

@@ -1,12 +1,21 @@
 import numpy as np
 
 import pandas as pd
-from pandas import DataFrame, MultiIndex, Series, Timestamp, date_range
+from pandas import (
+    DataFrame,
+    MultiIndex,
+    Series,
+    Timestamp,
+    date_range,
+)
 
 from .pandas_vb_common import tm
 
 try:
-    from pandas.tseries.offsets import Hour, Nano
+    from pandas.tseries.offsets import (
+        Hour,
+        Nano,
+    )
 except ImportError:
     # For compatibility with older versions
     from pandas.core.datetools import *  # noqa

asv_bench/benchmarks/frame_methods.py

@@ -3,7 +3,15 @@
 
 import numpy as np
 
-from pandas import DataFrame, MultiIndex, NaT, Series, date_range, isnull, period_range
+from pandas import (
+    DataFrame,
+    MultiIndex,
+    NaT,
+    Series,
+    date_range,
+    isnull,
+    period_range,
+)
 
 from .pandas_vb_common import tm
 

asv_bench/benchmarks/gil.py

@@ -1,6 +1,12 @@
 import numpy as np
 
-from pandas import DataFrame, Series, date_range, factorize, read_csv
+from pandas import (
+    DataFrame,
+    Series,
+    date_range,
+    factorize,
+    read_csv,
+)
 from pandas.core.algorithms import take_nd
 
 from .pandas_vb_common import tm

asv_bench/benchmarks/groupby.py

@@ -461,6 +461,29 @@ def time_dtype_as_field(self, dtype, method, application):
         self.as_field_method()
 
 
+class GroupByCythonAgg:
+    """
+    Benchmarks specifically targetting our cython aggregation algorithms
+    (using a big enough dataframe with simple key, so a large part of the
+    time is actually spent in the grouped aggregation).
+    """
+
+    param_names = ["dtype", "method"]
+    params = [
+        ["float64"],
+        ["sum", "prod", "min", "max", "mean", "median", "var", "first", "last"],
+    ]
+
+    def setup(self, dtype, method):
+        N = 1_000_000
+        df = DataFrame(np.random.randn(N, 10), columns=list("abcdefghij"))
+        df["key"] = np.random.randint(0, 100, size=N)
+        self.df = df
+
+    def time_frame_agg(self, dtype, method):
+        self.df.groupby("key").agg(method)
+
+
 class RankWithTies:
     # GH 21237
     param_names = ["dtype", "tie_method"]

asv_bench/benchmarks/inference.py

@@ -1,8 +1,14 @@
 import numpy as np
 
-from pandas import Series, to_numeric
-
-from .pandas_vb_common import lib, tm
+from pandas import (
+    Series,
+    to_numeric,
+)
+
+from .pandas_vb_common import (
+    lib,
+    tm,
+)
 
 
 class ToNumeric:

asv_bench/benchmarks/io/csv.py

@@ -1,12 +1,24 @@
-from io import BytesIO, StringIO
+from io import (
+    BytesIO,
+    StringIO,
+)
 import random
 import string
 
 import numpy as np
 
-from pandas import Categorical, DataFrame, date_range, read_csv, to_datetime
-
-from ..pandas_vb_common import BaseIO, tm
+from pandas import (
+    Categorical,
+    DataFrame,
+    date_range,
+    read_csv,
+    to_datetime,
+)
+
+from ..pandas_vb_common import (
+    BaseIO,
+    tm,
+)
 
 
 class ToCSV(BaseIO):

asv_bench/benchmarks/io/excel.py

@@ -2,10 +2,19 @@
 
 import numpy as np
 from odf.opendocument import OpenDocumentSpreadsheet
-from odf.table import Table, TableCell, TableRow
+from odf.table import (
+    Table,
+    TableCell,
+    TableRow,
+)
 from odf.text import P
 
-from pandas import DataFrame, ExcelWriter, date_range, read_excel
+from pandas import (
+    DataFrame,
+    ExcelWriter,
+    date_range,
+    read_excel,
+)
 
 from ..pandas_vb_common import tm
 

asv_bench/benchmarks/io/hdf.py

@@ -1,8 +1,16 @@
 import numpy as np
 
-from pandas import DataFrame, HDFStore, date_range, read_hdf
-
-from ..pandas_vb_common import BaseIO, tm
+from pandas import (
+    DataFrame,
+    HDFStore,
+    date_range,
+    read_hdf,
+)
+
+from ..pandas_vb_common import (
+    BaseIO,
+    tm,
+)
 
 
 class HDFStoreDataFrame(BaseIO):

asv_bench/benchmarks/io/json.py

@@ -2,9 +2,18 @@
 
 import numpy as np
 
-from pandas import DataFrame, concat, date_range, read_json, timedelta_range
-
-from ..pandas_vb_common import BaseIO, tm
+from pandas import (
+    DataFrame,
+    concat,
+    date_range,
+    read_json,
+    timedelta_range,
+)
+
+from ..pandas_vb_common import (
+    BaseIO,
+    tm,
+)
 
 
 class ReadJSON(BaseIO):

asv_bench/benchmarks/io/pickle.py

@@ -1,8 +1,15 @@
 import numpy as np
 
-from pandas import DataFrame, date_range, read_pickle
-
-from ..pandas_vb_common import BaseIO, tm
+from pandas import (
+    DataFrame,
+    date_range,
+    read_pickle,
+)
+
+from ..pandas_vb_common import (
+    BaseIO,
+    tm,
+)
 
 
 class Pickle(BaseIO):

asv_bench/benchmarks/io/sql.py

@@ -3,7 +3,12 @@
 import numpy as np
 from sqlalchemy import create_engine
 
-from pandas import DataFrame, date_range, read_sql_query, read_sql_table
+from pandas import (
+    DataFrame,
+    date_range,
+    read_sql_query,
+    read_sql_table,
+)
 
 from ..pandas_vb_common import tm
 

asv_bench/benchmarks/io/stata.py

@@ -1,8 +1,15 @@
 import numpy as np
 
-from pandas import DataFrame, date_range, read_stata
-
-from ..pandas_vb_common import BaseIO, tm
+from pandas import (
+    DataFrame,
+    date_range,
+    read_stata,
+)
+
+from ..pandas_vb_common import (
+    BaseIO,
+    tm,
+)
 
 
 class Stata(BaseIO):

asv_bench/benchmarks/join_merge.py

@@ -2,7 +2,15 @@
 
 import numpy as np
 
-from pandas import DataFrame, MultiIndex, Series, concat, date_range, merge, merge_asof
+from pandas import (
+    DataFrame,
+    MultiIndex,
+    Series,
+    concat,
+    date_range,
+    merge,
+    merge_asof,
+)
 
 from .pandas_vb_common import tm
 

asv_bench/benchmarks/multiindex_object.py

@@ -2,7 +2,12 @@
 
 import numpy as np
 
-from pandas import DataFrame, MultiIndex, RangeIndex, date_range
+from pandas import (
+    DataFrame,
+    MultiIndex,
+    RangeIndex,
+    date_range,
+)
 
 from .pandas_vb_common import tm
 

asv_bench/benchmarks/period.py

@@ -2,7 +2,14 @@
 Period benchmarks with non-tslibs dependencies.  See
 benchmarks.tslibs.period for benchmarks that rely only on tslibs.
 """
-from pandas import DataFrame, Period, PeriodIndex, Series, date_range, period_range
+from pandas import (
+    DataFrame,
+    Period,
+    PeriodIndex,
+    Series,
+    date_range,
+    period_range,
+)
 
 from pandas.tseries.frequencies import to_offset
 

asv_bench/benchmarks/plotting.py

@@ -1,7 +1,12 @@
 import matplotlib
 import numpy as np
 
-from pandas import DataFrame, DatetimeIndex, Series, date_range
+from pandas import (
+    DataFrame,
+    DatetimeIndex,
+    Series,
+    date_range,
+)
 
 try:
     from pandas.plotting import andrews_curves