Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,4 @@
 - [ ] closes #xxxx
 - [ ] tests added / passed
-- [ ] passes `black pandas`
-- [ ] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
+- [ ] Ensure all linting tests pass, see [here](https://pandas.pydata.org/pandas-docs/dev/development/contributing.html#code-standards) for how to run them
 - [ ] whatsnew entry

.pre-commit-config.yaml

@@ -1,4 +1,5 @@
 minimum_pre_commit_version: 2.9.2
+exclude: ^LICENSES/|\.(html|csv|svg)$
 repos:
 -   repo: https://github.com/python/black
     rev: 20.8b1
@@ -19,11 +20,9 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.6.4
+    rev: 5.7.0
     hooks:
     -   id: isort
-        types: [text] # overwrite upstream `types: [python]`
-        types_or: [python, cython]
 -   repo: https://github.com/asottile/pyupgrade
     rev: v2.7.4
     hooks:
@@ -121,6 +120,13 @@ repos:
         entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
         types: [python]
         exclude: ^(asv_bench|pandas/tests|doc)/
+    -   id: unwanted-patterns-bare-pytest-raises
+        name: Check for use of bare pytest raises
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
+        types: [python]
+        files: ^pandas/tests/
+        exclude: ^pandas/tests/extension/
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace in tests'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py
@@ -137,7 +143,7 @@ repos:
         name: Check for use of foo.__class__ instead of type(foo)
         entry: \.__class__
         language: pygrep
-        files: \.(py|pyx)$
+        types_or: [python, cython]
     -   id: unwanted-typing
         name: Check for use of comment-based annotation syntax and missing error codes
         entry: |
@@ -165,9 +171,8 @@ repos:
     rev: v3.4.0
     hooks:
     -   id: end-of-file-fixer
-        exclude: ^LICENSES/|\.(html|csv|txt|svg|py)$
+        exclude: \.txt$
     -   id: trailing-whitespace
-        exclude: \.(html|svg)$
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.0.0
     hooks:

asv_bench/benchmarks/arithmetic.py

@@ -122,18 +122,18 @@ def setup(self, op):
         n_rows = 500
 
         # construct dataframe with 2 blocks
-        arr1 = np.random.randn(n_rows, int(n_cols / 2)).astype("f8")
-        arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("f4")
+        arr1 = np.random.randn(n_rows, n_cols // 2).astype("f8")
+        arr2 = np.random.randn(n_rows, n_cols // 2).astype("f4")
         df = pd.concat(
             [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True
         )
         # should already be the case, but just to be sure
         df._consolidate_inplace()
 
         # TODO: GH#33198 the setting here shoudlnt need two steps
-        arr1 = np.random.randn(n_rows, int(n_cols / 4)).astype("f8")
-        arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("i8")
-        arr3 = np.random.randn(n_rows, int(n_cols / 4)).astype("f8")
+        arr1 = np.random.randn(n_rows, n_cols // 4).astype("f8")
+        arr2 = np.random.randn(n_rows, n_cols // 2).astype("i8")
+        arr3 = np.random.randn(n_rows, n_cols // 4).astype("f8")
         df2 = pd.concat(
             [pd.DataFrame(arr1), pd.DataFrame(arr2), pd.DataFrame(arr3)],
             axis=1,

asv_bench/benchmarks/frame_methods.py

@@ -263,7 +263,7 @@ class Repr:
     def setup(self):
         nrows = 10000
         data = np.random.randn(nrows, 10)
-        arrays = np.tile(np.random.randn(3, int(nrows / 100)), 100)
+        arrays = np.tile(np.random.randn(3, nrows // 100), 100)
         idx = MultiIndex.from_arrays(arrays)
         self.df3 = DataFrame(data, index=idx)
         self.df4 = DataFrame(data, index=np.random.randn(nrows))
@@ -648,9 +648,9 @@ class Describe:
     def setup(self):
         self.df = DataFrame(
             {
-                "a": np.random.randint(0, 100, int(1e6)),
-                "b": np.random.randint(0, 100, int(1e6)),
-                "c": np.random.randint(0, 100, int(1e6)),
+                "a": np.random.randint(0, 100, 10 ** 6),
+                "b": np.random.randint(0, 100, 10 ** 6),
+                "c": np.random.randint(0, 100, 10 ** 6),
             }
         )
 

asv_bench/benchmarks/groupby.py

@@ -126,6 +126,9 @@ def setup(self, data, key):
     def time_series_groups(self, data, key):
         self.ser.groupby(self.ser).groups
 
+    def time_series_indices(self, data, key):
+        self.ser.groupby(self.ser).indices
+
 
 class GroupManyLabels:
 

asv_bench/benchmarks/hash_functions.py

@@ -103,9 +103,9 @@ class Float64GroupIndex:
     # GH28303
     def setup(self):
         self.df = pd.date_range(
-            start="1/1/2018", end="1/2/2018", periods=1e6
+            start="1/1/2018", end="1/2/2018", periods=10 ** 6
         ).to_frame()
-        self.group_index = np.round(self.df.index.astype(int) / 1e9)
+        self.group_index = np.round(self.df.index.astype(int) / 10 ** 9)
 
     def time_groupby(self):
         self.df.groupby(self.group_index).last()

asv_bench/benchmarks/inference.py

@@ -42,7 +42,7 @@ class ToNumericDowncast:
     ]
 
     N = 500000
-    N2 = int(N / 2)
+    N2 = N // 2
 
     data_dict = {
         "string-int": ["1"] * N2 + [2] * N2,

asv_bench/benchmarks/join_merge.py

@@ -158,7 +158,7 @@ def setup(self):
         daily_dates = date_index.to_period("D").to_timestamp("S", "S")
         self.fracofday = date_index.values - daily_dates.values
         self.fracofday = self.fracofday.astype("timedelta64[ns]")
-        self.fracofday = self.fracofday.astype(np.float64) / 86400000000000.0
+        self.fracofday = self.fracofday.astype(np.float64) / 86_400_000_000_000
         self.fracofday = Series(self.fracofday, daily_dates)
         index = date_range(date_index.min(), date_index.max(), freq="D")
         self.temp = Series(1.0, index)[self.fracofday.index]

asv_bench/benchmarks/rolling.py

@@ -50,20 +50,24 @@ class Engine:
         ["int", "float"],
         [np.sum, lambda x: np.sum(x) + 5],
         ["cython", "numba"],
+        ["sum", "max", "min", "median", "mean"],
     )
-    param_names = ["constructor", "dtype", "function", "engine"]
+    param_names = ["constructor", "dtype", "function", "engine", "method"]
 
-    def setup(self, constructor, dtype, function, engine):
+    def setup(self, constructor, dtype, function, engine, method):
         N = 10 ** 3
         arr = (100 * np.random.random(N)).astype(dtype)
         self.data = getattr(pd, constructor)(arr)
 
-    def time_rolling_apply(self, constructor, dtype, function, engine):
+    def time_rolling_apply(self, constructor, dtype, function, engine, method):
         self.data.rolling(10).apply(function, raw=True, engine=engine)
 
-    def time_expanding_apply(self, constructor, dtype, function, engine):
+    def time_expanding_apply(self, constructor, dtype, function, engine, method):
         self.data.expanding().apply(function, raw=True, engine=engine)
 
+    def time_rolling_methods(self, constructor, dtype, function, engine, method):
+        getattr(self.data.rolling(10), method)(engine=engine)
+
 
 class ExpandingMethods:
 
@@ -171,7 +175,7 @@ class PeakMemFixedWindowMinMax:
     params = ["min", "max"]
 
     def setup(self, operation):
-        N = int(1e6)
+        N = 10 ** 6
         arr = np.random.random(N)
         self.roll = pd.Series(arr).rolling(2)
 
@@ -233,7 +237,7 @@ class GroupbyLargeGroups:
 
     def setup(self):
         N = 100000
-        self.df = pd.DataFrame({"A": [1, 2] * int(N / 2), "B": np.random.randn(N)})
+        self.df = pd.DataFrame({"A": [1, 2] * (N // 2), "B": np.random.randn(N)})
 
     def time_rolling_multiindex_creation(self):
         self.df.groupby("A").rolling(3).mean()

asv_bench/benchmarks/series_methods.py

@@ -284,7 +284,7 @@ def time_dir_strings(self):
 class SeriesGetattr:
     # https://github.com/pandas-dev/pandas/issues/19764
     def setup(self):
-        self.s = Series(1, index=date_range("2012-01-01", freq="s", periods=int(1e6)))
+        self.s = Series(1, index=date_range("2012-01-01", freq="s", periods=10 ** 6))
 
     def time_series_datetimeindex_repr(self):
         getattr(self.s, "a", None)

asv_bench/benchmarks/timeseries.py

@@ -346,7 +346,7 @@ def time_iso8601_tz_spaceformat(self):
 class ToDatetimeNONISO8601:
     def setup(self):
         N = 10000
-        half = int(N / 2)
+        half = N // 2
         ts_string_1 = "March 1, 2018 12:00:00+0400"
         ts_string_2 = "March 1, 2018 12:00:00+0500"
         self.same_offset = [ts_string_1] * N
@@ -376,7 +376,7 @@ def setup(self):
         self.same_offset = ["10/11/2018 00:00:00.045-07:00"] * N
         self.diff_offset = [
             f"10/11/2018 00:00:00.045-0{offset}:00" for offset in range(10)
-        ] * int(N / 10)
+        ] * (N // 10)
 
     def time_exact(self):
         to_datetime(self.s2, format="%d%b%y")

ci/deps/actions-37-locale.yaml

@@ -30,7 +30,7 @@ dependencies:
   - openpyxl
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
-  - pyarrow>=0.17
+  - pyarrow=0.17 # GH 38803
   - pytables>=3.5.1
   - scipy
   - xarray=0.12.3

doc/source/conf.py

@@ -432,7 +432,7 @@
 
 
 ipython_warning_is_error = False
-ipython_exec_lines = [
+ipython_execlines = [
     "import numpy as np",
     "import pandas as pd",
     # This ensures correct rendering on system with console encoding != utf8

doc/source/development/test_writing.rst

@@ -149,13 +149,6 @@ be located.
             ``frame_or_series`` fixture, by convention it goes in the
             ``tests.frame`` file.
 
-      - tests.generic.methods.test_mymethod
-
-        .. note::
-
-           The generic/methods/ directory is only for methods with tests
-           that are fully parametrized over Series/DataFrame
-
 7. Is your test for an Index method, not depending on Series/DataFrame?
    This test likely belongs in one of: