Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -22,7 +22,9 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
@@ -94,7 +96,9 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Set up pandas
       uses: ./.github/actions/setup
@@ -147,7 +151,9 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Set up pandas
       uses: ./.github/actions/setup

.github/workflows/database.yml

@@ -56,10 +56,12 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Cache conda
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
         CACHE_NUMBER: 0
       with:
@@ -70,7 +72,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-dev
-        channel-priority: strict
+        channel-priority: flexible
         environment-file: ${{ matrix.ENV_FILE }}
         use-only-tar-bz2: true
 

.github/workflows/posix.yml

@@ -44,10 +44,12 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Cache conda
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       env:
         CACHE_NUMBER: 0
       with:

.pre-commit-config.yaml

@@ -21,6 +21,7 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.4.0
     hooks:
+    -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
     -   id: trailing-whitespace
@@ -35,7 +36,7 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.1
+    rev: 3.9.2
     hooks:
     -   id: flake8
         additional_dependencies:
@@ -75,7 +76,7 @@ repos:
     hooks:
     -   id: yesqa
         additional_dependencies:
-            - flake8==3.9.1
+            - flake8==3.9.2
             - flake8-comprehensions==3.1.0
             - flake8-bugbear==21.3.2
             - pandas-dev-flaker==0.2.0

asv_bench/benchmarks/io/style.py

@@ -20,19 +20,19 @@ def setup(self, cols, rows):
 
     def time_apply_render(self, cols, rows):
         self._style_apply()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def peakmem_apply_render(self, cols, rows):
         self._style_apply()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def time_classes_render(self, cols, rows):
         self._style_classes()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def peakmem_classes_render(self, cols, rows):
         self._style_classes()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def time_format_render(self, cols, rows):
         self._style_format()

asv_bench/benchmarks/strings.py

@@ -11,6 +11,19 @@
 from .pandas_vb_common import tm
 
 
+class Dtypes:
+    params = ["str", "string", "arrow_string"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        from pandas.core.arrays.string_arrow import ArrowStringDtype  # noqa: F401
+
+        try:
+            self.s = Series(tm.makeStringIndex(10 ** 5), dtype=dtype)
+        except ImportError:
+            raise NotImplementedError
+
+
 class Construction:
 
     params = ["str", "string"]
@@ -49,18 +62,7 @@ def peakmem_cat_frame_construction(self, dtype):
         DataFrame(self.frame_cat_arr, dtype=dtype)
 
 
-class Methods:
-    params = ["str", "string", "arrow_string"]
-    param_names = ["dtype"]
-
-    def setup(self, dtype):
-        from pandas.core.arrays.string_arrow import ArrowStringDtype  # noqa: F401
-
-        try:
-            self.s = Series(tm.makeStringIndex(10 ** 5), dtype=dtype)
-        except ImportError:
-            raise NotImplementedError
-
+class Methods(Dtypes):
     def time_center(self, dtype):
         self.s.str.center(100)
 
@@ -83,6 +85,9 @@ def time_find(self, dtype):
     def time_rfind(self, dtype):
         self.s.str.rfind("[A-Z]+")
 
+    def time_fullmatch(self, dtype):
+        self.s.str.fullmatch("A")
+
     def time_get(self, dtype):
         self.s.str.get(0)
 
@@ -211,43 +216,53 @@ def time_cat(self, other_cols, sep, na_rep, na_frac):
         self.s.str.cat(others=self.others, sep=sep, na_rep=na_rep)
 
 
-class Contains:
+class Contains(Dtypes):
 
-    params = (["str", "string", "arrow_string"], [True, False])
+    params = (Dtypes.params, [True, False])
     param_names = ["dtype", "regex"]
 
     def setup(self, dtype, regex):
-        from pandas.core.arrays.string_arrow import ArrowStringDtype  # noqa: F401
-
-        try:
-            self.s = Series(tm.makeStringIndex(10 ** 5), dtype=dtype)
-        except ImportError:
-            raise NotImplementedError
+        super().setup(dtype)
 
     def time_contains(self, dtype, regex):
         self.s.str.contains("A", regex=regex)
 
 
-class Split:
+class Split(Dtypes):
 
-    params = [True, False]
-    param_names = ["expand"]
+    params = (Dtypes.params, [True, False])
+    param_names = ["dtype", "expand"]
 
-    def setup(self, expand):
-        self.s = Series(tm.makeStringIndex(10 ** 5)).str.join("--")
+    def setup(self, dtype, expand):
+        super().setup(dtype)
+        self.s = self.s.str.join("--")
 
-    def time_split(self, expand):
+    def time_split(self, dtype, expand):
         self.s.str.split("--", expand=expand)
 
-    def time_rsplit(self, expand):
+    def time_rsplit(self, dtype, expand):
         self.s.str.rsplit("--", expand=expand)
 
 
-class Dummies:
-    def setup(self):
-        self.s = Series(tm.makeStringIndex(10 ** 5)).str.join("|")
+class Extract(Dtypes):
+
+    params = (Dtypes.params, [True, False])
+    param_names = ["dtype", "expand"]
+
+    def setup(self, dtype, expand):
+        super().setup(dtype)
+
+    def time_extract_single_group(self, dtype, expand):
+        with warnings.catch_warnings(record=True):
+            self.s.str.extract("(\\w*)A", expand=expand)
 
-    def time_get_dummies(self):
+
+class Dummies(Dtypes):
+    def setup(self, dtype):
+        super().setup(dtype)
+        self.s = self.s.str.join("|")
+
+    def time_get_dummies(self, dtype):
         self.s.str.get_dummies("|")
 
 
@@ -266,3 +281,9 @@ def setup(self):
     def time_vector_slice(self):
         # GH 2602
         self.s.str[:5]
+
+
+class Iter(Dtypes):
+    def time_iter(self, dtype):
+        for i in self.s:
+            pass

ci/deps/actions-37-db-min.yaml

@@ -31,7 +31,8 @@ dependencies:
   - openpyxl
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
-  - pyarrow=0.17 # GH 38803
+  - protobuf>=3.12.4
+  - pyarrow=0.17.1 # GH 38803
   - pytables>=3.5.1
   - scipy
   - xarray=0.12.3

ci/deps/actions-37-db.yaml

@@ -15,7 +15,7 @@ dependencies:
   - beautifulsoup4
   - botocore>=1.11
   - dask
-  - fastparquet>=0.4.0, <=0.5.0
+  - fastparquet>=0.4.0
   - fsspec>=0.7.4
   - gcsfs>=0.6.0
   - geopandas
@@ -31,7 +31,7 @@ dependencies:
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
   - psycopg2
-  - pyarrow>=0.15.0
+  - pyarrow>=0.17.0
   - pymysql
   - pytables
   - python-snappy

ci/deps/actions-37-minimum_versions.yaml

@@ -17,13 +17,13 @@ dependencies:
   - bottleneck=1.2.1
   - jinja2=2.10
   - numba=0.46.0
-  - numexpr=2.6.8
+  - numexpr=2.7.0
   - numpy=1.17.3
   - openpyxl=3.0.0
   - pytables=3.5.1
   - python-dateutil=2.7.3
   - pytz=2017.3
-  - pyarrow=0.15
+  - pyarrow=0.17.0
   - scipy=1.2
   - xlrd=1.2.0
   - xlsxwriter=1.0.2

ci/deps/actions-37.yaml

@@ -18,7 +18,7 @@ dependencies:
   - numpy=1.19
   - python-dateutil
   - nomkl
-  - pyarrow=0.15.1
+  - pyarrow
   - pytz
   - s3fs>=0.4.0
   - moto>=1.3.14

ci/deps/azure-macos-37.yaml

@@ -1,6 +1,7 @@
 name: pandas-dev
 channels:
   - defaults
+  - conda-forge
 dependencies:
   - python=3.7.*
 
@@ -21,7 +22,7 @@ dependencies:
   - numexpr
   - numpy=1.17.3
   - openpyxl
-  - pyarrow=0.15.1
+  - pyarrow=0.17.0
   - pytables
   - python-dateutil==2.7.3
   - pytz

ci/deps/azure-windows-37.yaml

@@ -26,7 +26,7 @@ dependencies:
   - numexpr
   - numpy=1.17.*
   - openpyxl
-  - pyarrow=0.15
+  - pyarrow=0.17.0
   - pytables
   - python-dateutil
   - pytz

ci/deps/azure-windows-38.yaml

@@ -15,7 +15,7 @@ dependencies:
   # pandas dependencies
   - blosc
   - bottleneck
-  - fastparquet>=0.4.0, <=0.5.0
+  - fastparquet>=0.4.0
   - flask
   - fsspec>=0.8.0
   - matplotlib=3.1.3
@@ -25,7 +25,7 @@ dependencies:
   - numpy=1.18.*
   - openpyxl
   - jinja2
-  - pyarrow>=0.15.0
+  - pyarrow>=0.17.0
   - pytables
   - python-dateutil
   - pytz

ci/run_tests.sh

@@ -24,7 +24,7 @@ PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TE
 if [[ $(uname) != "Linux"  && $(uname) != "Darwin" ]]; then
     # GH#37455 windows py38 build appears to be running out of memory
     #  skip collection of window tests
-    PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/window/ --ignore=pandas/tests/plotting/"
+    PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/window/moments --ignore=pandas/tests/plotting/"
 fi
 
 echo $PYTEST_CMD

doc/redirects.csv

@@ -1197,6 +1197,7 @@ generated/pandas.Series.str.extractall,../reference/api/pandas.Series.str.extrac
 generated/pandas.Series.str.extract,../reference/api/pandas.Series.str.extract
 generated/pandas.Series.str.findall,../reference/api/pandas.Series.str.findall
 generated/pandas.Series.str.find,../reference/api/pandas.Series.str.find
+generated/pandas.Series.str.fullmatch,../reference/api/pandas.Series.str.fullmatch
 generated/pandas.Series.str.get_dummies,../reference/api/pandas.Series.str.get_dummies
 generated/pandas.Series.str.get,../reference/api/pandas.Series.str.get
 generated/pandas.Series.str,../reference/api/pandas.Series.str

doc/source/development/code_style.rst

@@ -57,7 +57,8 @@ xfail during the testing phase. To do so, use the ``request`` fixture:
     import pytest
 
     def test_xfail(request):
-        request.node.add_marker(pytest.mark.xfail(reason="Indicate why here"))
+        mark = pytest.mark.xfail(raises=TypeError, reason="Indicate why here")
+        request.node.add_marker(mark)
 
 xfail is not to be used for tests involving failure due to invalid user arguments.
 For these tests, we need to verify the correct exception type and error message