Merge branch 'pandas-dev:master' into fillna-other-missing-values-not-modified

Author: Mike Phung

.circleci/config.yml

@@ -0,0 +1,21 @@
+version: 2.1
+
+jobs:
+  test-arm:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.medium
+    environment:
+      ENV_FILE: ci/deps/circle-38-arm64.yaml
+      PYTEST_WORKERS: auto
+      PATTERN: "not slow and not network and not clipboard and not arm_slow"
+      PYTEST_TARGET: "pandas"
+    steps:
+      - checkout
+      - run: ci/setup_env.sh
+      - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
+
+workflows:
+  test:
+    jobs:
+      - test-arm

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
 - [ ] closes #xxxx
 - [ ] tests added / passed
-- [ ] Ensure all linting tests pass, see [here](https://pandas.pydata.org/pandas-docs/dev/development/contributing.html#code-standards) for how to run them
+- [ ] Ensure all linting tests pass, see [here](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit) for how to run them
 - [ ] whatsnew entry

.github/workflows/ci.yml

@@ -45,6 +45,15 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
+    - name: Install node.js (for pyright)
+      uses: actions/setup-node@v2
+      with:
+        node-version: "16"
+
+    - name: Install pyright
+      # note: keep version in sync with .pre-commit-config.yaml
+      run: npm install -g [email protected]
+
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
@@ -168,6 +177,7 @@ jobs:
         PANDAS_DATA_MANAGER: array
         PATTERN: ${{ matrix.pattern }}
         PYTEST_WORKERS: "auto"
+        PYTEST_TARGET: pandas
       run: |
         source activate pandas-dev
         ci/run_tests.sh

.github/workflows/database.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
@@ -79,7 +80,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-dev
-        channel-priority: flexible
+        channel-priority: strict
         environment-file: ${{ matrix.ENV_FILE }}
         use-only-tar-bz2: true
 

.github/workflows/posix.yml

@@ -44,6 +44,7 @@ jobs:
       LC_ALL: ${{ matrix.settings[4] }}
       PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
       TEST_ARGS: ${{ matrix.settings[6] }}
+      PYTEST_TARGET:  pandas
     concurrency:
       group: ${{ github.ref }}-${{ matrix.settings[0] }}
       cancel-in-progress: ${{github.event_name == 'pull_request'}}

.github/workflows/pre-commit.yml

@@ -13,6 +13,8 @@ jobs:
     concurrency:
       group: ${{ github.ref }}-pre-commit
       cancel-in-progress: ${{github.event_name == 'pull_request'}}
+    env:
+      SKIP: pyright
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.github/workflows/python-dev.yml

@@ -17,6 +17,7 @@ env:
   PANDAS_CI: 1
   PATTERN: "not slow and not network and not clipboard"
   COVERAGE: true
+  PYTEST_TARGET:  pandas
 
 jobs:
   build:

.pre-commit-config.yaml

@@ -81,6 +81,16 @@ repos:
             - flake8-comprehensions==3.1.0
             - flake8-bugbear==21.3.2
             - pandas-dev-flaker==0.2.0
+-   repo: local
+    hooks:
+    -   id: pyright
+        name: pyright
+        entry: pyright
+        language: node
+        pass_filenames: false
+        types: [python]
+        # note: keep version in sync with .github/workflows/ci.yml
+        additional_dependencies: ['[email protected]']
 -   repo: local
     hooks:
     -   id: flake8-rst

README.md

@@ -12,7 +12,7 @@
 [![License](https://img.shields.io/pypi/l/pandas.svg)](https://github.com/pandas-dev/pandas/blob/master/LICENSE)
 [![Azure Build Status](https://dev.azure.com/pandas-dev/pandas/_apis/build/status/pandas-dev.pandas?branch=master)](https://dev.azure.com/pandas-dev/pandas/_build/latest?definitionId=1&branch=master)
 [![Coverage](https://codecov.io/github/pandas-dev/pandas/coverage.svg?branch=master)](https://codecov.io/gh/pandas-dev/pandas)
-[![Downloads](https://anaconda.org/conda-forge/pandas/badges/downloads.svg)](https://pandas.pydata.org)
+[![Downloads](https://static.pepy.tech/personalized-badge/pandas?period=month&units=international_system&left_color=black&right_color=orange&left_text=PyPI%20downloads%20per%20month)](https://pepy.tech/project/pandas)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pydata/pandas)
 [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](https://numfocus.org)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)

asv_bench/benchmarks/algorithms.py

@@ -44,9 +44,9 @@ def setup(self, unique, sort, dtype):
                 raise NotImplementedError
 
         data = {
-            "int": pd.Int64Index(np.arange(N)),
-            "uint": pd.UInt64Index(np.arange(N)),
-            "float": pd.Float64Index(np.random.randn(N)),
+            "int": pd.Index(np.arange(N), dtype="int64"),
+            "uint": pd.Index(np.arange(N), dtype="uint64"),
+            "float": pd.Index(np.random.randn(N), dtype="float64"),
             "object": string_index,
             "datetime64[ns]": pd.date_range("2011-01-01", freq="H", periods=N),
             "datetime64[ns, tz]": pd.date_range(
@@ -76,9 +76,9 @@ class Duplicated:
     def setup(self, unique, keep, dtype):
         N = 10 ** 5
         data = {
-            "int": pd.Int64Index(np.arange(N)),
-            "uint": pd.UInt64Index(np.arange(N)),
-            "float": pd.Float64Index(np.random.randn(N)),
+            "int": pd.Index(np.arange(N), dtype="int64"),
+            "uint": pd.Index(np.arange(N), dtype="uint64"),
+            "float": pd.Index(np.random.randn(N), dtype="float64"),
             "string": tm.makeStringIndex(N),
             "datetime64[ns]": pd.date_range("2011-01-01", freq="H", periods=N),
             "datetime64[ns, tz]": pd.date_range(

asv_bench/benchmarks/groupby.py

@@ -454,6 +454,16 @@ def setup(self, dtype, method, application, ncols):
             # DataFrameGroupBy doesn't have these methods
             raise NotImplementedError
 
+        if application == "transformation" and method in [
+            "head",
+            "tail",
+            "unique",
+            "value_counts",
+            "size",
+        ]:
+            # DataFrameGroupBy doesn't have these methods
+            raise NotImplementedError
+
         ngroups = 1000
         size = ngroups * 2
         rng = np.arange(ngroups).reshape(-1, 1)
@@ -480,7 +490,7 @@ def setup(self, dtype, method, application, ncols):
         if len(cols) == 1:
             cols = cols[0]
 
-        if application == "transform":
+        if application == "transformation":
             if method == "describe":
                 raise NotImplementedError
 
@@ -593,6 +603,38 @@ def time_sum(self):
         self.df.groupby(["a"])["b"].sum()
 
 
+class String:
+    # GH#41596
+    param_names = ["dtype", "method"]
+    params = [
+        ["str", "string[python]"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
+    ]
+
+    def setup(self, dtype, method):
+        cols = list("abcdefghjkl")
+        self.df = DataFrame(
+            np.random.randint(0, 100, size=(1_000_000, len(cols))),
+            columns=cols,
+            dtype=dtype,
+        )
+
+    def time_str_func(self, dtype, method):
+        self.df.groupby("a")[self.df.columns[1:]].agg(method)
+
+
 class Categories:
     def setup(self):
         N = 10 ** 5

asv_bench/benchmarks/indexing_engines.py

@@ -48,7 +48,7 @@ def setup(self, engine_and_dtype, index_type):
             "non_monotonic": np.array([1, 2, 3] * N, dtype=dtype),
         }[index_type]
 
-        self.data = engine(lambda: arr, len(arr))
+        self.data = engine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc(2)
 
@@ -70,7 +70,7 @@ def setup(self, index_type):
             "non_monotonic": np.array(list("abc") * N, dtype=object),
         }[index_type]
 
-        self.data = libindex.ObjectEngine(lambda: arr, len(arr))
+        self.data = libindex.ObjectEngine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc("b")
 

asv_bench/benchmarks/inference.py

@@ -173,6 +173,7 @@ def setup(self):
         self.strings_tz_space = [
             x.strftime("%Y-%m-%d %H:%M:%S") + " -0800" for x in rng
         ]
+        self.strings_zero_tz = [x.strftime("%Y-%m-%d %H:%M:%S") + "Z" for x in rng]
 
     def time_iso8601(self):
         to_datetime(self.strings)
@@ -189,6 +190,10 @@ def time_iso8601_format_no_sep(self):
     def time_iso8601_tz_spaceformat(self):
         to_datetime(self.strings_tz_space)
 
+    def time_iso8601_infer_zero_tz_fromat(self):
+        # GH 41047
+        to_datetime(self.strings_zero_tz, infer_datetime_format=True)
+
 
 class ToDatetimeNONISO8601:
     def setup(self):

asv_bench/benchmarks/io/style.py

@@ -34,6 +34,14 @@ def peakmem_classes_render(self, cols, rows):
         self._style_classes()
         self.st._render_html(True, True)
 
+    def time_tooltips_render(self, cols, rows):
+        self._style_tooltips()
+        self.st._render_html(True, True)
+
+    def peakmem_tooltips_render(self, cols, rows):
+        self._style_tooltips()
+        self.st._render_html(True, True)
+
     def time_format_render(self, cols, rows):
         self._style_format()
         self.st._render_html(True, True)
@@ -42,6 +50,14 @@ def peakmem_format_render(self, cols, rows):
         self._style_format()
         self.st._render_html(True, True)
 
+    def time_apply_format_hide_render(self, cols, rows):
+        self._style_apply_format_hide()
+        self.st._render_html(True, True)
+
+    def peakmem_apply_format_hide_render(self, cols, rows):
+        self._style_apply_format_hide()
+        self.st._render_html(True, True)
+
     def _style_apply(self):
         def _apply_func(s):
             return [
@@ -63,3 +79,15 @@ def _style_format(self):
         self.st = self.df.style.format(
             "{:,.3f}", subset=IndexSlice["row_1":f"row_{ir}", "float_1":f"float_{ic}"]
         )
+
+    def _style_apply_format_hide(self):
+        self.st = self.df.style.applymap(lambda v: "color: red;")
+        self.st.format("{:.3f}")
+        self.st.hide_index(self.st.index[1:])
+        self.st.hide_columns(self.st.columns[1:])
+
+    def _style_tooltips(self):
+        ttips = DataFrame("abc", index=self.df.index[::2], columns=self.df.columns[::2])
+        self.st = self.df.style.set_tooltips(ttips)
+        self.st.hide_index(self.st.index[12:])
+        self.st.hide_columns(self.st.columns[12:])

asv_bench/benchmarks/rolling.py

@@ -180,6 +180,33 @@ def time_quantile(self, constructor, window, dtype, percentile, interpolation):
         self.roll.quantile(percentile, interpolation=interpolation)
 
 
+class Rank:
+    params = (
+        ["DataFrame", "Series"],
+        [10, 1000],
+        ["int", "float"],
+        [True, False],
+        [True, False],
+        ["min", "max", "average"],
+    )
+    param_names = [
+        "constructor",
+        "window",
+        "dtype",
+        "percentile",
+        "ascending",
+        "method",
+    ]
+
+    def setup(self, constructor, window, dtype, percentile, ascending, method):
+        N = 10 ** 5
+        arr = np.random.random(N).astype(dtype)
+        self.roll = getattr(pd, constructor)(arr).rolling(window)
+
+    def time_rank(self, constructor, window, dtype, percentile, ascending, method):
+        self.roll.rank(pct=percentile, ascending=ascending, method=method)
+
+
 class PeakMemFixedWindowMinMax:
 
     params = ["min", "max"]

asv_bench/benchmarks/series_methods.py

@@ -28,6 +28,19 @@ def time_constructor(self, data):
         Series(data=self.data, index=self.idx)
 
 
+class ToFrame:
+    params = [["int64", "datetime64[ns]", "category", "Int64"], [None, "foo"]]
+    param_names = ["dtype", "name"]
+
+    def setup(self, dtype, name):
+        arr = np.arange(10 ** 5)
+        ser = Series(arr, dtype=dtype)
+        self.ser = ser
+
+    def time_to_frame(self, dtype, name):
+        self.ser.to_frame(name)
+
+
 class NSort:
 
     params = ["first", "last", "all"]