Merge remote-tracking branch 'upstream/master' into test-min_period

Author: horaceklai

.github/workflows/ci.yml

@@ -32,10 +32,6 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Looking for unwanted patterns
-      run: ci/code_checks.sh patterns
-      if: always()
-
     - name: Cache conda
       uses: actions/cache@v2
       with:

.pre-commit-config.yaml

@@ -102,7 +102,29 @@ repos:
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
+
+            # Check for deprecated messages without sphinx directive
+            |(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)
         types_or: [python, cython, rst]
+    -   id: incorrect-backticks
+        name: Check for backticks incorrectly rendering because of missing spaces
+        language: pygrep
+        entry: '[a-zA-Z0-9]\`\`?[a-zA-Z0-9]'
+        types: [rst]
+        files: ^doc/source/
+    -   id: seed-check-asv
+        name: Check for unnecessary random seeds in asv benchmarks
+        language: pygrep
+        entry: 'np\.random\.seed'
+        files: ^asv_bench/benchmarks
+        exclude: ^asv_bench/benchmarks/pandas_vb_common\.py
+    -   id: invalid-ea-testing
+        name: Check for invalid EA testing
+        language: pygrep
+        entry: 'tm\.assert_(series|frame)_equal'
+        files: ^pandas/tests/extension/base
+        types: [python]
+        exclude: ^pandas/tests/extension/base/base\.py
     -   id: pip-to-conda
         name: Generate pip dependency from conda
         description: This hook checks if the conda environment.yml and requirements-dev.txt are equal

ci/code_checks.sh

@@ -11,14 +11,13 @@
 # Usage:
 #   $ ./ci/code_checks.sh               # run all checks
 #   $ ./ci/code_checks.sh lint          # run linting only
-#   $ ./ci/code_checks.sh patterns      # check for patterns that should not exist
 #   $ ./ci/code_checks.sh code          # checks on imported code
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh typing        # run static type analysis
 
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" ]] || \
-    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|typing]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" ]] || \
+    { echo "Unknown command $1. Usage: $0 [lint|code|doctests|docstrings|typing]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -58,28 +57,6 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
 fi
 
-### PATTERNS ###
-if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
-
-    # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal`
-    MSG='Check for invalid EA testing' ; echo $MSG
-    invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for deprecated messages without sphinx directive' ; echo $MSG
-    invgrep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for backticks incorrectly rendering because of missing spaces' ; echo $MSG
-    invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for unnecessary random seeds in asv benchmarks' ; echo $MSG
-    invgrep -R --exclude pandas_vb_common.py -E 'np.random.seed' asv_bench/benchmarks/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-fi
-
 ### CODE ###
 if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
 

ci/deps/actions-38-db.yaml

@@ -15,7 +15,7 @@ dependencies:
   - beautifulsoup4
   - botocore>=1.11
   - dask
-  - fastparquet>=0.4.0, < 0.7.0
+  - fastparquet>=0.4.0
   - fsspec>=0.7.4, <2021.6.0
   - gcsfs>=0.6.0
   - geopandas

ci/deps/azure-windows-38.yaml

@@ -15,7 +15,7 @@ dependencies:
   # pandas dependencies
   - blosc
   - bottleneck
-  - fastparquet>=0.4.0, <0.7.0
+  - fastparquet>=0.4.0
   - flask
   - fsspec>=0.8.0, <2021.6.0
   - matplotlib=3.3.2

doc/source/user_guide/duplicates.rst

@@ -28,6 +28,7 @@ duplicates present. The output can't be determined, and so pandas raises.
 
 .. ipython:: python
    :okexcept:
+   :okwarning:
 
    s1 = pd.Series([0, 1, 2], index=["a", "b", "b"])
    s1.reindex(["a", "b", "c"])

doc/source/whatsnew/v1.3.2.rst

@@ -32,7 +32,9 @@ Bug fixes
 ~~~~~~~~~
 - Bug in :meth:`pandas.read_excel` modifies the dtypes dictionary when reading a file with duplicate columns (:issue:`42462`)
 - 1D slices over extension types turn into N-dimensional slices over ExtensionArrays (:issue:`42430`)
+- Fixed bug in :meth:`Series.rolling` and :meth:`DataFrame.rolling` not calculating window bounds correctly for the first row when ``center=True`` and ``window`` is an offset that covers all the rows (:issue:`42753`)
 - :meth:`.Styler.hide_columns` now hides the index name header row as well as column headers (:issue:`42101`)
+- :meth:`.Styler.set_sticky` has amended CSS to control the column/index names and ensure the correct sticky positions (:issue:`42537`)
 - Bug in de-serializing datetime indexes in PYTHONOPTIMIZED mode (:issue:`42866`)
 -
 
@@ -42,7 +44,7 @@ Bug fixes
 
 Other
 ~~~~~
--
+- :meth:`pandas.read_parquet` now supports reading nullable dtypes with ``fastparquet`` versions above 0.7.1.
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.4.0.rst

@@ -162,6 +162,7 @@ Deprecations
 - Deprecated ignoring missing labels when indexing with a sequence of labels on a level of a MultiIndex (:issue:`42351`)
 - Creating an empty Series without a dtype will now raise a more visible ``FutureWarning`` instead of a ``DeprecationWarning`` (:issue:`30017`)
 - Deprecated the 'kind' argument in :meth:`Index.get_slice_bound`, :meth:`Index.slice_indexer`, :meth:`Index.slice_locs`; in a future version passing 'kind' will raise (:issue:`42857`)
+- Deprecated :meth:`Index.reindex` with a non-unique index (:issue:`42568`)
 -
 
 .. ---------------------------------------------------------------------------

environment.yml

@@ -99,7 +99,7 @@ dependencies:
   - xlwt
   - odfpy
 
-  - fastparquet>=0.4.0, <0.7.0  # pandas.read_parquet, DataFrame.to_parquet
+  - fastparquet>=0.4.0  # pandas.read_parquet, DataFrame.to_parquet
   - pyarrow>=0.17.0  # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather
   - python-snappy  # required by pyarrow
 

pandas/_libs/window/indexers.pyx

@@ -79,12 +79,11 @@ def calculate_variable_window_bounds(
     else:
         end[0] = 0
     if center:
-        for j in range(0, num_values + 1):
-            if (index[j] == index[0] + index_growth_sign * window_size / 2 and
-               right_closed):
+        end_bound = index[0] + index_growth_sign * window_size / 2
+        for j in range(0, num_values):
+            if (index[j] < end_bound) or (index[j] == end_bound and right_closed):
                 end[0] = j + 1
-                break
-            elif index[j] >= index[0] + index_growth_sign * window_size / 2:
+            elif index[j] >= end_bound:
                 end[0] = j
                 break
 

pandas/core/indexes/base.py

@@ -3915,6 +3915,15 @@ def reindex(
                         )
                     indexer, _ = self.get_indexer_non_unique(target)
 
+                if not self.is_unique:
+                    # GH#42568
+                    warnings.warn(
+                        "reindexing with a non-unique Index is deprecated and "
+                        "will raise in a future version",
+                        FutureWarning,
+                        stacklevel=2,
+                    )
+
         target = self._wrap_reindex_result(target, indexer, preserve_names)
         return target, indexer
 

pandas/core/indexes/category.py

@@ -426,6 +426,14 @@ def reindex(
             missing = np.array([], dtype=np.intp)
         else:
             indexer, missing = self.get_indexer_non_unique(target)
+            if not self.is_unique:
+                # GH#42568
+                warnings.warn(
+                    "reindexing with a non-unique Index is deprecated and will "
+                    "raise in a future version",
+                    FutureWarning,
+                    stacklevel=2,
+                )
 
         if len(self) and indexer is not None:
             new_target = self.take(indexer)

pandas/core/internals/concat.py

@@ -94,12 +94,15 @@ def _concatenate_array_managers(
             concat_arrays([mgrs[i].arrays[j] for i in range(len(mgrs))])
             for j in range(len(mgrs[0].arrays))
         ]
-        return ArrayManager(arrays, [axes[1], axes[0]], verify_integrity=False)
     else:
         # concatting along the columns -> combine reindexed arrays in a single manager
         assert concat_axis == 0
         arrays = list(itertools.chain.from_iterable([mgr.arrays for mgr in mgrs]))
-        return ArrayManager(arrays, [axes[1], axes[0]], verify_integrity=False)
+        if copy:
+            arrays = [x.copy() for x in arrays]
+
+    new_mgr = ArrayManager(arrays, [axes[1], axes[0]], verify_integrity=False)
+    return new_mgr
 
 
 def concat_arrays(to_concat: list) -> ArrayLike:

pandas/io/formats/style.py

@@ -1534,24 +1534,24 @@ def set_sticky(
         may produce strange behaviour due to CSS controls with missing elements.
         """
         if axis in [0, "index"]:
-            axis, obj, tag, pos = 0, self.data.index, "tbody", "left"
+            axis, obj = 0, self.data.index
             pixel_size = 75 if not pixel_size else pixel_size
         elif axis in [1, "columns"]:
-            axis, obj, tag, pos = 1, self.data.columns, "thead", "top"
+            axis, obj = 1, self.data.columns
             pixel_size = 25 if not pixel_size else pixel_size
         else:
             raise ValueError("`axis` must be one of {0, 1, 'index', 'columns'}")
 
+        props = "position:sticky; background-color:white;"
         if not isinstance(obj, pd.MultiIndex):
             # handling MultiIndexes requires different CSS
-            props = "position:sticky; background-color:white;"
 
             if axis == 1:
                 # stick the first <tr> of <head> and, if index names, the second <tr>
                 # if self._hide_columns then no <thead><tr> here will exist: no conflict
                 styles: CSSStyles = [
                     {
-                        "selector": "thead tr:first-child",
+                        "selector": "thead tr:nth-child(1) th",
                         "props": props + "top:0px; z-index:2;",
                     }
                 ]
@@ -1561,7 +1561,7 @@ def set_sticky(
                     )
                     styles.append(
                         {
-                            "selector": "thead tr:nth-child(2)",
+                            "selector": "thead tr:nth-child(2) th",
                             "props": props
                             + f"top:{pixel_size}px; z-index:2; height:{pixel_size}px; ",
                         }
@@ -1572,34 +1572,67 @@ def set_sticky(
                 # but <th> will exist in <thead>: conflict with initial element
                 styles = [
                     {
-                        "selector": "tr th:first-child",
+                        "selector": "thead tr th:nth-child(1)",
+                        "props": props + "left:0px; z-index:3 !important;",
+                    },
+                    {
+                        "selector": "tbody tr th:nth-child(1)",
                         "props": props + "left:0px; z-index:1;",
-                    }
+                    },
                 ]
 
-            return self.set_table_styles(styles, overwrite=False)
-
         else:
+            # handle the MultiIndex case
             range_idx = list(range(obj.nlevels))
+            levels = sorted(levels) if levels else range_idx
 
-        levels = sorted(levels) if levels else range_idx
-        for i, level in enumerate(levels):
-            self.set_table_styles(
-                [
-                    {
-                        "selector": f"{tag} th.level{level}",
-                        "props": f"position: sticky; "
-                        f"{pos}: {i * pixel_size}px; "
-                        f"{f'height: {pixel_size}px; ' if axis == 1 else ''}"
-                        f"{f'min-width: {pixel_size}px; ' if axis == 0 else ''}"
-                        f"{f'max-width: {pixel_size}px; ' if axis == 0 else ''}"
-                        f"background-color: white;",
-                    }
-                ],
-                overwrite=False,
-            )
+            if axis == 1:
+                styles = []
+                for i, level in enumerate(levels):
+                    styles.append(
+                        {
+                            "selector": f"thead tr:nth-child({level+1}) th",
+                            "props": props
+                            + (
+                                f"top:{i * pixel_size}px; height:{pixel_size}px; "
+                                "z-index:2;"
+                            ),
+                        }
+                    )
+                if not all(name is None for name in self.index.names):
+                    styles.append(
+                        {
+                            "selector": f"thead tr:nth-child({obj.nlevels+1}) th",
+                            "props": props
+                            + (
+                                f"top:{(i+1) * pixel_size}px; height:{pixel_size}px; "
+                                "z-index:2;"
+                            ),
+                        }
+                    )
 
-        return self
+            else:
+                styles = []
+                for i, level in enumerate(levels):
+                    props_ = props + (
+                        f"left:{i * pixel_size}px; "
+                        f"min-width:{pixel_size}px; "
+                        f"max-width:{pixel_size}px; "
+                    )
+                    styles.extend(
+                        [
+                            {
+                                "selector": f"thead tr th:nth-child({level+1})",
+                                "props": props_ + "z-index:3 !important;",
+                            },
+                            {
+                                "selector": f"tbody tr th.level{level}",
+                                "props": props_ + "z-index:1;",
+                            },
+                        ]
+                    )
+
+        return self.set_table_styles(styles, overwrite=False)
 
     def set_table_styles(
         self,

pandas/io/parquet.py

@@ -309,14 +309,21 @@ def write(
     def read(
         self, path, columns=None, storage_options: StorageOptions = None, **kwargs
     ):
+        parquet_kwargs = {}
         use_nullable_dtypes = kwargs.pop("use_nullable_dtypes", False)
-        if use_nullable_dtypes:
-            raise ValueError(
-                "The 'use_nullable_dtypes' argument is not supported for the "
-                "fastparquet engine"
-            )
+        # Technically works with 0.7.0, but was incorrect
+        # so lets just require 0.7.1
+        if Version(self.api.__version__) >= Version("0.7.1"):
+            # Need to set even for use_nullable_dtypes = False,
+            # since our defaults differ
+            parquet_kwargs["pandas_nulls"] = use_nullable_dtypes
+        else:
+            if use_nullable_dtypes:
+                raise ValueError(
+                    "The 'use_nullable_dtypes' argument is not supported for the "
+                    "fastparquet engine for fastparquet versions less than 0.7.1"
+                )
         path = stringify_path(path)
-        parquet_kwargs = {}
         handles = None
         if is_fsspec_url(path):
             fsspec = import_optional_dependency("fsspec")
@@ -337,6 +344,7 @@ def read(
                 path, "rb", is_text=False, storage_options=storage_options
             )
             path = handles.handle
+
         parquet_file = self.api.ParquetFile(path, **parquet_kwargs)
 
         result = parquet_file.to_pandas(columns=columns, **kwargs)
@@ -470,14 +478,18 @@ def read_parquet(
 
     use_nullable_dtypes : bool, default False
         If True, use dtypes that use ``pd.NA`` as missing value indicator
-        for the resulting DataFrame (only applicable for ``engine="pyarrow"``).
+        for the resulting DataFrame.
         As new dtypes are added that support ``pd.NA`` in the future, the
         output with this option will change to use those dtypes.
         Note: this is an experimental option, and behaviour (e.g. additional
         support dtypes) may change without notice.
 
         .. versionadded:: 1.2.0
 
+        .. versionchanged:: 1.3.2
+            ``use_nullable_dtypes`` now works with the the ``fastparquet`` engine
+            if ``fastparquet`` is version 0.7.1 or higher.
+
     **kwargs
         Any additional kwargs are passed to the engine.