Merge pull request #152 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

doc/source/user_guide/io.rst

@@ -5240,6 +5240,7 @@ Write to a feather file.
 Read from a feather file.
 
 .. ipython:: python
+   :okwarning:
 
    result = pd.read_feather("example.feather")
    result
@@ -5323,6 +5324,7 @@ Write to a parquet file.
 Read from a parquet file.
 
 .. ipython:: python
+   :okwarning:
 
    result = pd.read_parquet("example_fp.parquet", engine="fastparquet")
    result = pd.read_parquet("example_pa.parquet", engine="pyarrow")

doc/source/whatsnew/v1.3.0.rst

@@ -299,6 +299,24 @@ cast to ``dtype=object`` (:issue:`38709`)
    ser
    ser2
 
+
+.. _whatsnew_130.notable_bug_fixes.rolling_var_precision:
+
+Removed artificial truncation in rolling variance and standard deviation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:meth:`core.window.Rolling.std` and :meth:`core.window.Rolling.var` will no longer
+artificially truncate results that are less than ``~1e-8`` and ``~1e-15`` respectively to
+zero (:issue:`37051`, :issue:`40448`, :issue:`39872`).
+
+However, floating point artifacts may now exist in the results when rolling over larger values.
+
+.. ipython:: python
+
+   s = pd.Series([7, 5, 5, 5])
+   s.rolling(3).var()
+
+
 .. _whatsnew_130.api_breaking.deps:
 
 Increased minimum versions for dependencies
@@ -661,6 +679,7 @@ Other
 - Bug in :class:`Styler` where rendered HTML was missing a column class identifier for certain header cells (:issue:`39716`)
 - Bug in :meth:`Styler.background_gradient` where text-color was not determined correctly (:issue:`39888`)
 - Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
+- Bug in :class:`.Styler` where copying from Jupyter dropped top left cell and misaligned headers (:issue:`12147`)
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
 - Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
 - Bug in :meth:`DataFrame.convert_dtypes` incorrectly raised ValueError when called on an empty DataFrame (:issue:`40393`)

pandas/_libs/window/aggregations.pyx

@@ -283,10 +283,6 @@ cdef inline float64_t calc_var(int64_t minp, int ddof, float64_t nobs,
             result = 0
         else:
             result = ssqdm_x / (nobs - <float64_t>ddof)
-            # Fix for numerical imprecision.
-            # Can be result < 0 once Kahan Summation is implemented
-            if result < 1e-14:
-                result = 0
     else:
         result = NaN
 

pandas/_typing.py

@@ -80,7 +80,7 @@
 # scalars
 
 PythonScalar = Union[str, int, float, bool]
-DatetimeLikeScalar = TypeVar("DatetimeLikeScalar", "Period", "Timestamp", "Timedelta")
+DatetimeLikeScalar = Union["Period", "Timestamp", "Timedelta"]
 PandasScalar = Union["Period", "Timestamp", "Timedelta", "Interval"]
 Scalar = Union[PythonScalar, PandasScalar]
 

pandas/core/arrays/datetimelike.py

@@ -563,8 +563,7 @@ def _validate_comparison_value(self, other):
                 raise InvalidComparison(other)
 
         if isinstance(other, self._recognized_scalars) or other is NaT:
-            # error: Too many arguments for "object"
-            other = self._scalar_type(other)  # type: ignore[call-arg]
+            other = self._scalar_type(other)
             try:
                 self._check_compatible_with(other)
             except (TypeError, IncompatibleFrequency) as err:
@@ -614,16 +613,14 @@ def _validate_shift_value(self, fill_value):
         if is_valid_na_for_dtype(fill_value, self.dtype):
             fill_value = NaT
         elif isinstance(fill_value, self._recognized_scalars):
-            # error: Too many arguments for "object"
-            fill_value = self._scalar_type(fill_value)  # type: ignore[call-arg]
+            fill_value = self._scalar_type(fill_value)
         else:
             # only warn if we're not going to raise
             if self._scalar_type is Period and lib.is_integer(fill_value):
                 # kludge for #31971 since Period(integer) tries to cast to str
                 new_fill = Period._from_ordinal(fill_value, freq=self.freq)
             else:
-                # error: Too many arguments for "object"
-                new_fill = self._scalar_type(fill_value)  # type: ignore[call-arg]
+                new_fill = self._scalar_type(fill_value)
 
             # stacklevel here is chosen to be correct when called from
             #  DataFrame.shift or Series.shift
@@ -684,8 +681,7 @@ def _validate_scalar(
             raise TypeError(msg)
 
         elif isinstance(value, self._recognized_scalars):
-            # error: Too many arguments for "object"
-            value = self._scalar_type(value)  # type: ignore[call-arg]
+            value = self._scalar_type(value)
 
         else:
             msg = self._validation_error_message(value, allow_listlike)

pandas/core/construction.py

@@ -33,7 +33,7 @@
 
 from pandas.core.dtypes.base import (
     ExtensionDtype,
-    registry,
+    _registry as registry,
 )
 from pandas.core.dtypes.cast import (
     construct_1d_arraylike_from_scalar,

pandas/core/dtypes/base.py

@@ -388,7 +388,7 @@ def register_extension_dtype(cls: Type[ExtensionDtype]) -> Type[ExtensionDtype]:
     ... class MyExtensionDtype(ExtensionDtype):
     ...     name = "myextension"
     """
-    registry.register(cls)
+    _registry.register(cls)
     return cls
 
 
@@ -452,4 +452,4 @@ def find(
         return None
 
 
-registry = Registry()
+_registry = Registry()

pandas/core/dtypes/common.py

@@ -23,7 +23,7 @@
     Optional,
 )
 
-from pandas.core.dtypes.base import registry
+from pandas.core.dtypes.base import _registry as registry
 from pandas.core.dtypes.dtypes import (
     CategoricalDtype,
     DatetimeTZDtype,

pandas/core/reshape/merge.py

@@ -10,6 +10,8 @@
 import string
 from typing import (
     TYPE_CHECKING,
+    Hashable,
+    List,
     Optional,
     Tuple,
     cast,
@@ -124,14 +126,13 @@ def merge(
     merge.__doc__ = _merge_doc % "\nleft : DataFrame"
 
 
-def _groupby_and_merge(by, on, left: DataFrame, right: DataFrame, merge_pieces):
+def _groupby_and_merge(by, left: DataFrame, right: DataFrame, merge_pieces):
     """
     groupby & merge; we are always performing a left-by type operation
 
     Parameters
     ----------
     by: field to group
-    on: duplicates field
     left: DataFrame
     right: DataFrame
     merge_pieces: function for merging
@@ -307,17 +308,15 @@ def _merger(x, y):
         check = set(left_by).difference(left.columns)
         if len(check) != 0:
             raise KeyError(f"{check} not found in left columns")
-        result, _ = _groupby_and_merge(
-            left_by, on, left, right, lambda x, y: _merger(x, y)
-        )
+        result, _ = _groupby_and_merge(left_by, left, right, lambda x, y: _merger(x, y))
     elif right_by is not None:
         if isinstance(right_by, str):
             right_by = [right_by]
         check = set(right_by).difference(right.columns)
         if len(check) != 0:
             raise KeyError(f"{check} not found in right columns")
         result, _ = _groupby_and_merge(
-            right_by, on, right, left, lambda x, y: _merger(y, x)
+            right_by, right, left, lambda x, y: _merger(y, x)
         )
     else:
         result = _merger(left, right)
@@ -708,7 +707,7 @@ def __init__(
         if validate is not None:
             self._validate(validate)
 
-    def get_result(self):
+    def get_result(self) -> DataFrame:
         if self.indicator:
             self.left, self.right = self._indicator_pre_merge(self.left, self.right)
 
@@ -774,7 +773,7 @@ def _indicator_pre_merge(
 
         return left, right
 
-    def _indicator_post_merge(self, result):
+    def _indicator_post_merge(self, result: DataFrame) -> DataFrame:
 
         result["_left_indicator"] = result["_left_indicator"].fillna(0)
         result["_right_indicator"] = result["_right_indicator"].fillna(0)
@@ -790,7 +789,7 @@ def _indicator_post_merge(self, result):
         result = result.drop(labels=["_left_indicator", "_right_indicator"], axis=1)
         return result
 
-    def _maybe_restore_index_levels(self, result):
+    def _maybe_restore_index_levels(self, result: DataFrame) -> None:
         """
         Restore index levels specified as `on` parameters
 
@@ -949,7 +948,6 @@ def _get_join_info(self):
                         self.left.index,
                         self.right.index,
                         left_indexer,
-                        right_indexer,
                         how="right",
                     )
                 else:
@@ -961,7 +959,6 @@ def _get_join_info(self):
                         self.right.index,
                         self.left.index,
                         right_indexer,
-                        left_indexer,
                         how="left",
                     )
                 else:
@@ -979,9 +976,8 @@ def _create_join_index(
         index: Index,
         other_index: Index,
         indexer,
-        other_indexer,
         how: str = "left",
-    ):
+    ) -> Index:
         """
         Create a join index by rearranging one index to match another
 
@@ -1126,7 +1122,7 @@ def _get_merge_keys(self):
 
         return left_keys, right_keys, join_names
 
-    def _maybe_coerce_merge_keys(self):
+    def _maybe_coerce_merge_keys(self) -> None:
         # we have valid merges but we may have to further
         # coerce these if they are originally incompatible types
         #
@@ -1285,7 +1281,7 @@ def _create_cross_configuration(
             cross_col,
         )
 
-    def _validate_specification(self):
+    def _validate_specification(self) -> None:
         if self.how == "cross":
             if (
                 self.left_index
@@ -1372,7 +1368,7 @@ def _validate_specification(self):
         if self.how != "cross" and len(self.right_on) != len(self.left_on):
             raise ValueError("len(right_on) must equal len(left_on)")
 
-    def _validate(self, validate: str):
+    def _validate(self, validate: str) -> None:
 
         # Check uniqueness of each
         if self.left_index:
@@ -1479,10 +1475,10 @@ def restore_dropped_levels_multijoin(
     left: MultiIndex,
     right: MultiIndex,
     dropped_level_names,
-    join_index,
-    lindexer,
-    rindexer,
-):
+    join_index: Index,
+    lindexer: np.ndarray,
+    rindexer: np.ndarray,
+) -> Tuple[List[Index], np.ndarray, List[Hashable]]:
     """
     *this is an internal non-public method*
 
@@ -1500,7 +1496,7 @@ def restore_dropped_levels_multijoin(
         right index
     dropped_level_names : str array
         list of non-common level names
-    join_index : MultiIndex
+    join_index : Index
         the index of the join between the
         common levels of left and right
     lindexer : intp array
@@ -1514,8 +1510,8 @@ def restore_dropped_levels_multijoin(
         levels of combined multiindexes
     labels : intp array
         labels of combined multiindexes
-    names : str array
-        names of combined multiindexes
+    names : List[Hashable]
+        names of combined multiindex levels
 
     """
 
@@ -1604,7 +1600,7 @@ def __init__(
             sort=True,  # factorize sorts
         )
 
-    def get_result(self):
+    def get_result(self) -> DataFrame:
         join_index, left_indexer, right_indexer = self._get_join_info()
 
         llabels, rlabels = _items_overlap_with_suffix(
@@ -1653,7 +1649,7 @@ def _asof_by_function(direction: str):
 }
 
 
-def _get_cython_type_upcast(dtype):
+def _get_cython_type_upcast(dtype) -> str:
     """ Upcast a dtype to 'int64_t', 'double', or 'object' """
     if is_integer_dtype(dtype):
         return "int64_t"

pandas/core/window/rolling.py

@@ -1882,21 +1882,24 @@ def median(
         The default ``ddof`` of 1 used in :meth:`Series.std` is different
         than the default ``ddof`` of 0 in :func:`numpy.std`.
 
-        A minimum of one period is required for the rolling calculation.\n
+        A minimum of one period is required for the rolling calculation.
+
+        The implementation is susceptible to floating point imprecision as
+        shown in the example below.\n
         """
         ).replace("\n", "", 1),
         create_section_header("Examples"),
         dedent(
             """
         >>> s = pd.Series([5, 5, 6, 7, 5, 5, 5])
         >>> s.rolling(3).std()
-        0         NaN
-        1         NaN
-        2    0.577350
-        3    1.000000
-        4    1.000000
-        5    1.154701
-        6    0.000000
+        0             NaN
+        1             NaN
+        2    5.773503e-01
+        3    1.000000e+00
+        4    1.000000e+00
+        5    1.154701e+00
+        6    2.580957e-08
         dtype: float64
         """
         ).replace("\n", "", 1),
@@ -1931,21 +1934,24 @@ def std(self, ddof: int = 1, *args, **kwargs):
         The default ``ddof`` of 1 used in :meth:`Series.var` is different
         than the default ``ddof`` of 0 in :func:`numpy.var`.
 
-        A minimum of one period is required for the rolling calculation.\n
+        A minimum of one period is required for the rolling calculation.
+
+        The implementation is susceptible to floating point imprecision as
+        shown in the example below.\n
         """
         ).replace("\n", "", 1),
         create_section_header("Examples"),
         dedent(
             """
         >>> s = pd.Series([5, 5, 6, 7, 5, 5, 5])
         >>> s.rolling(3).var()
-        0         NaN
-        1         NaN
-        2    0.333333
-        3    1.000000
-        4    1.000000
-        5    1.333333
-        6    0.000000
+        0             NaN
+        1             NaN
+        2    3.333333e-01
+        3    1.000000e+00
+        4    1.000000e+00
+        5    1.333333e+00
+        6    6.661338e-16
         dtype: float64
         """
         ).replace("\n", "", 1),

pandas/io/formats/style.py

@@ -359,7 +359,7 @@ def _translate(self):
 
         DATA_CLASS = "data"
         BLANK_CLASS = "blank"
-        BLANK_VALUE = ""
+        BLANK_VALUE = " "
 
         # mapping variables
         ctx = self.ctx  # td css styles from apply() and applymap()