Merge branch 'master' of https://github.com/pandas-dev/pandas into BUG29751

Author: kasim95

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,4 @@
 - [ ] closes #xxxx
 - [ ] tests added / passed
-- [ ] passes `black pandas`
-- [ ] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
+- [ ] Ensure all linting tests pass, see [here](https://pandas.pydata.org/pandas-docs/dev/development/contributing.html#code-standards) for how to run them
 - [ ] whatsnew entry

.pre-commit-config.yaml

@@ -1,4 +1,5 @@
 minimum_pre_commit_version: 2.9.2
+exclude: ^LICENSES/|\.(html|csv|svg)$
 repos:
 -   repo: https://github.com/python/black
     rev: 20.8b1
@@ -121,6 +122,13 @@ repos:
         entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
         types: [python]
         exclude: ^(asv_bench|pandas/tests|doc)/
+    -   id: unwanted-patterns-bare-pytest-raises
+        name: Check for use of bare pytest raises
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
+        types: [python]
+        files: ^pandas/tests/
+        exclude: ^pandas/tests/(computation|extension|io)/
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace in tests'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py
@@ -137,7 +145,7 @@ repos:
         name: Check for use of foo.__class__ instead of type(foo)
         entry: \.__class__
         language: pygrep
-        files: \.(py|pyx)$
+        types_or: [python, cython]
     -   id: unwanted-typing
         name: Check for use of comment-based annotation syntax and missing error codes
         entry: |
@@ -165,9 +173,8 @@ repos:
     rev: v3.4.0
     hooks:
     -   id: end-of-file-fixer
-        exclude: ^LICENSES/|\.(html|csv|txt|svg|py)$
+        exclude: \.txt$
     -   id: trailing-whitespace
-        exclude: \.(html|svg)$
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.0.0
     hooks:

asv_bench/benchmarks/arithmetic.py

@@ -122,18 +122,18 @@ def setup(self, op):
         n_rows = 500
 
         # construct dataframe with 2 blocks
-        arr1 = np.random.randn(n_rows, int(n_cols / 2)).astype("f8")
-        arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("f4")
+        arr1 = np.random.randn(n_rows, n_cols // 2).astype("f8")
+        arr2 = np.random.randn(n_rows, n_cols // 2).astype("f4")
         df = pd.concat(
             [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True
         )
         # should already be the case, but just to be sure
         df._consolidate_inplace()
 
         # TODO: GH#33198 the setting here shoudlnt need two steps
-        arr1 = np.random.randn(n_rows, int(n_cols / 4)).astype("f8")
-        arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("i8")
-        arr3 = np.random.randn(n_rows, int(n_cols / 4)).astype("f8")
+        arr1 = np.random.randn(n_rows, n_cols // 4).astype("f8")
+        arr2 = np.random.randn(n_rows, n_cols // 2).astype("i8")
+        arr3 = np.random.randn(n_rows, n_cols // 4).astype("f8")
         df2 = pd.concat(
             [pd.DataFrame(arr1), pd.DataFrame(arr2), pd.DataFrame(arr3)],
             axis=1,

asv_bench/benchmarks/frame_methods.py

@@ -263,7 +263,7 @@ class Repr:
     def setup(self):
         nrows = 10000
         data = np.random.randn(nrows, 10)
-        arrays = np.tile(np.random.randn(3, int(nrows / 100)), 100)
+        arrays = np.tile(np.random.randn(3, nrows // 100), 100)
         idx = MultiIndex.from_arrays(arrays)
         self.df3 = DataFrame(data, index=idx)
         self.df4 = DataFrame(data, index=np.random.randn(nrows))
@@ -597,6 +597,19 @@ def time_frame_quantile(self, axis):
         self.df.quantile([0.1, 0.5], axis=axis)
 
 
+class Rank:
+    param_names = ["dtype"]
+    params = [
+        ["int", "uint", "float", "object"],
+    ]
+
+    def setup(self, dtype):
+        self.df = DataFrame(np.random.randn(10000, 10), columns=range(10), dtype=dtype)
+
+    def time_rank(self, dtype):
+        self.df.rank()
+
+
 class GetDtypeCounts:
     # 2807
     def setup(self):
@@ -635,9 +648,9 @@ class Describe:
     def setup(self):
         self.df = DataFrame(
             {
-                "a": np.random.randint(0, 100, int(1e6)),
-                "b": np.random.randint(0, 100, int(1e6)),
-                "c": np.random.randint(0, 100, int(1e6)),
+                "a": np.random.randint(0, 100, 10 ** 6),
+                "b": np.random.randint(0, 100, 10 ** 6),
+                "c": np.random.randint(0, 100, 10 ** 6),
             }
         )
 

asv_bench/benchmarks/groupby.py

@@ -29,7 +29,6 @@
         "skew",
         "cumprod",
         "cummax",
-        "rank",
         "pct_change",
         "min",
         "var",
@@ -127,6 +126,9 @@ def setup(self, data, key):
     def time_series_groups(self, data, key):
         self.ser.groupby(self.ser).groups
 
+    def time_series_indices(self, data, key):
+        self.ser.groupby(self.ser).indices
+
 
 class GroupManyLabels:
 

asv_bench/benchmarks/hash_functions.py

@@ -103,9 +103,9 @@ class Float64GroupIndex:
     # GH28303
     def setup(self):
         self.df = pd.date_range(
-            start="1/1/2018", end="1/2/2018", periods=1e6
+            start="1/1/2018", end="1/2/2018", periods=10 ** 6
         ).to_frame()
-        self.group_index = np.round(self.df.index.astype(int) / 1e9)
+        self.group_index = np.round(self.df.index.astype(int) / 10 ** 9)
 
     def time_groupby(self):
         self.df.groupby(self.group_index).last()

asv_bench/benchmarks/inference.py

@@ -42,7 +42,7 @@ class ToNumericDowncast:
     ]
 
     N = 500000
-    N2 = int(N / 2)
+    N2 = N // 2
 
     data_dict = {
         "string-int": ["1"] * N2 + [2] * N2,

asv_bench/benchmarks/join_merge.py

@@ -158,7 +158,7 @@ def setup(self):
         daily_dates = date_index.to_period("D").to_timestamp("S", "S")
         self.fracofday = date_index.values - daily_dates.values
         self.fracofday = self.fracofday.astype("timedelta64[ns]")
-        self.fracofday = self.fracofday.astype(np.float64) / 86400000000000.0
+        self.fracofday = self.fracofday.astype(np.float64) / 86_400_000_000_000
         self.fracofday = Series(self.fracofday, daily_dates)
         index = date_range(date_index.min(), date_index.max(), freq="D")
         self.temp = Series(1.0, index)[self.fracofday.index]

asv_bench/benchmarks/rolling.py

@@ -171,7 +171,7 @@ class PeakMemFixedWindowMinMax:
     params = ["min", "max"]
 
     def setup(self, operation):
-        N = int(1e6)
+        N = 10 ** 6
         arr = np.random.random(N)
         self.roll = pd.Series(arr).rolling(2)
 
@@ -233,7 +233,7 @@ class GroupbyLargeGroups:
 
     def setup(self):
         N = 100000
-        self.df = pd.DataFrame({"A": [1, 2] * int(N / 2), "B": np.random.randn(N)})
+        self.df = pd.DataFrame({"A": [1, 2] * (N // 2), "B": np.random.randn(N)})
 
     def time_rolling_multiindex_creation(self):
         self.df.groupby("A").rolling(3).mean()

asv_bench/benchmarks/series_methods.py

@@ -284,7 +284,7 @@ def time_dir_strings(self):
 class SeriesGetattr:
     # https://github.com/pandas-dev/pandas/issues/19764
     def setup(self):
-        self.s = Series(1, index=date_range("2012-01-01", freq="s", periods=int(1e6)))
+        self.s = Series(1, index=date_range("2012-01-01", freq="s", periods=10 ** 6))
 
     def time_series_datetimeindex_repr(self):
         getattr(self.s, "a", None)
@@ -349,4 +349,18 @@ def time_func(self, func, N, dtype):
         self.func()
 
 
+class Rank:
+
+    param_names = ["dtype"]
+    params = [
+        ["int", "uint", "float", "object"],
+    ]
+
+    def setup(self, dtype):
+        self.s = Series(np.random.randint(0, 1000, size=100000), dtype=dtype)
+
+    def time_rank(self, dtype):
+        self.s.rank()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/timeseries.py

@@ -346,7 +346,7 @@ def time_iso8601_tz_spaceformat(self):
 class ToDatetimeNONISO8601:
     def setup(self):
         N = 10000
-        half = int(N / 2)
+        half = N // 2
         ts_string_1 = "March 1, 2018 12:00:00+0400"
         ts_string_2 = "March 1, 2018 12:00:00+0500"
         self.same_offset = [ts_string_1] * N
@@ -376,7 +376,7 @@ def setup(self):
         self.same_offset = ["10/11/2018 00:00:00.045-07:00"] * N
         self.diff_offset = [
             f"10/11/2018 00:00:00.045-0{offset}:00" for offset in range(10)
-        ] * int(N / 10)
+        ] * (N // 10)
 
     def time_exact(self):
         to_datetime(self.s2, format="%d%b%y")

doc/source/conf.py

@@ -68,7 +68,12 @@
     "contributors",  # custom pandas extension
 ]
 
-exclude_patterns = ["**.ipynb_checkpoints"]
+exclude_patterns = [
+    "**.ipynb_checkpoints",
+    # to ensure that include files (partial pages) aren't built, exclude them
+    # https://github.com/sphinx-doc/sphinx/issues/1965#issuecomment-124732907
+    "**/includes/**",
+]
 try:
     import nbconvert
 except ImportError:

doc/source/development/test_writing.rst

@@ -149,13 +149,6 @@ be located.
             ``frame_or_series`` fixture, by convention it goes in the
             ``tests.frame`` file.
 
-      - tests.generic.methods.test_mymethod
-
-        .. note::
-
-           The generic/methods/ directory is only for methods with tests
-           that are fully parametrized over Series/DataFrame
-
 7. Is your test for an Index method, not depending on Series/DataFrame?
    This test likely belongs in one of: