From ffdc947e857871f521ebc7c0089714d81e7f46f3 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Fri, 3 Jan 2020 19:17:21 -0500 Subject: [PATCH 01/11] Fixed errors='ignore' being ignored Signed-off-by: nbonnin --- pandas/core/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index f846d5883a8b5..bcf1922f23515 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -5576,7 +5576,7 @@ def astype(self, dtype, copy: bool_t = True, errors: str = "raise"): """ if is_dict_like(dtype): if self.ndim == 1: # i.e. Series - if len(dtype) > 1 or self.name not in dtype: + if len(dtype) > 1 or self.name not in dtype and errors == "raise": raise KeyError( "Only the Series name can be used for " "the key in Series dtype mappings." @@ -5585,7 +5585,7 @@ def astype(self, dtype, copy: bool_t = True, errors: str = "raise"): return self.astype(new_type, copy, errors) for col_name in dtype.keys(): - if col_name not in self: + if col_name not in self and errors == "raise": raise KeyError( "Only a column name can be used for the " "key in a dtype mappings argument." From c88b98e7f279bf667ee039f297d20cf1e4c835ad Mon Sep 17 00:00:00 2001 From: nbonnin Date: Tue, 7 Jan 2020 20:54:41 -0500 Subject: [PATCH 02/11] Added comments to the tests. Test is already in place, unsure why it was not working before. Signed-off-by: nbonnin --- pandas/tests/frame/test_dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_dtypes.py b/pandas/tests/frame/test_dtypes.py index d8d56e90a2f31..010eb2c64a5d2 100644 --- a/pandas/tests/frame/test_dtypes.py +++ b/pandas/tests/frame/test_dtypes.py @@ -595,7 +595,7 @@ def test_astype_str_float(self): @pytest.mark.parametrize("dtype_class", [dict, Series]) def test_astype_dict_like(self, dtype_class): - # GH7271 & GH16717 + # GH7271 & GH16717 & GH30324 a = Series(date_range("2010-01-04", periods=5)) b = Series(range(5)) c = Series([0.0, 0.2, 0.4, 0.6, 0.8]) From b73279ed9fb1de69bbb97e03e244da9d172755c6 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Thu, 9 Jan 2020 07:54:13 -0500 Subject: [PATCH 03/11] Added a test for a not covered case. Added whats new entry. Signed-off-by: nbonnin --- doc/source/whatsnew/v1.0.0.rst | 1 + pandas/tests/frame/test_dtypes.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 9023cf2ab1b4f..5061726676755 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -895,6 +895,7 @@ Other - Fixed :class:`IntegerArray` returning ``inf`` rather than ``NaN`` for operations dividing by 0 (:issue:`27398`) - Fixed ``pow`` operations for :class:`IntegerArray` when the other value is ``0`` or ``1`` (:issue:`29997`) - Bug in :meth:`Series.count` raises if use_inf_as_na is enabled (:issue:`29478`) +- Bug in :meth:`DataFrame.astype` that raised ``KeyError`` when errors='ignore' is ``True`` (:issue:`30324`) .. _whatsnew_1000.contributors: diff --git a/pandas/tests/frame/test_dtypes.py b/pandas/tests/frame/test_dtypes.py index 010eb2c64a5d2..55cf65dc7349e 100644 --- a/pandas/tests/frame/test_dtypes.py +++ b/pandas/tests/frame/test_dtypes.py @@ -661,6 +661,15 @@ def test_astype_dict_like(self, dtype_class): tm.assert_frame_equal(df, equiv) tm.assert_frame_equal(df, original) + # GH 30324 + # If errors=='ignore' than the resulting DataFrame + # should be the same as the original DataFrame + dt8 = dtype_class({"b": str, 2: str}) + dt9 = dtype_class({"e": str}) + df.astype(dt8, errors="ignore") + df.astype(dt9, errors="ignore") + tm.assert_frame_equal(df, original) + def test_astype_duplicate_col(self): a1 = Series([1, 2, 3, 4, 5], name="a") b = Series([0.1, 0.2, 0.4, 0.6, 0.8], name="b") From 88aa2f3236f37e37587c21d72b461a1a9a410315 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Thu, 9 Jan 2020 08:57:40 -0500 Subject: [PATCH 04/11] Updated whatsnew to the latest version, added my own whatsnew entry Signed-off-by: nbonnin --- doc/source/whatsnew/v1.0.0.rst | 650 +++++++++++++++++++++------------ 1 file changed, 415 insertions(+), 235 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 5061726676755..6de324b7b2eef 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -1,40 +1,20 @@ -.. _whatsnew_1000: +.. _whatsnew_100: What's new in 1.0.0 (??) ------------------------ -.. warning:: - - Starting with the 1.x series of releases, pandas only supports Python 3.6.1 and higher. +These are the changes in pandas 1.0.0. See :ref:`release` for a full changelog +including other versions of pandas. New Deprecation Policy ~~~~~~~~~~~~~~~~~~~~~~ -Starting with Pandas 1.0.0, pandas will adopt a version of `SemVer`_. - -Historically, pandas has used a "rolling" deprecation policy, with occasional -outright breaking API changes. Where possible, we would deprecate the behavior -we'd like to change, giving an option to adopt the new behavior (via a keyword -or an alternative method), and issuing a warning for users of the old behavior. -Sometimes, a deprecation was not possible, and we would make an outright API -breaking change. - -We'll continue to *introduce* deprecations in major and minor releases (e.g. -1.0.0, 1.1.0, ...). Those deprecations will be *enforced* in the next major -release. +Starting with Pandas 1.0.0, pandas will adopt a variant of `SemVer`_ to +version releases. Briefly, -Note that *behavior changes* and *API breaking changes* are not identical. API -breaking changes will only be released in major versions. If we consider a -behavior to be a bug, and fixing that bug induces a behavior change, we'll -release that change in a minor release. This is a sometimes difficult judgment -call that we'll do our best on. - -This doesn't mean that pandas' pace of development will slow down. In the `2019 -Pandas User Survey`_, about 95% of the respondents said they considered pandas -"stable enough". This indicates there's an appetite for new features, even if it -comes at the cost of break API. The difference is that now API breaking changes -will be accompanied with a bump in the major version number (e.g. pandas 1.5.1 --> 2.0.0). +* Deprecations will be introduced in minor releases (e.g. 1.1.0, 1.2.0, 2.1.0, ...) +* Deprecations will be enforced in major releases (e.g. 1.0.0, 2.0,0, 3.0.0, ...) +* API-breaking changes will be made only in major releases See :ref:`policies.version` for more. @@ -43,20 +23,63 @@ See :ref:`policies.version` for more. {{ header }} -These are the changes in pandas 1.0.0. See :ref:`release` for a full changelog -including other versions of pandas. - +.. --------------------------------------------------------------------------- Enhancements ~~~~~~~~~~~~ +.. _whatsnew_100.NA: + +Experimental ``NA`` scalar to denote missing values +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A new ``pd.NA`` value (singleton) is introduced to represent scalar missing +values. Up to now, pandas used several values to represent missing data: ``np.nan`` is used for this for float data, ``np.nan`` or +``None`` for object-dtype data and ``pd.NaT`` for datetime-like data. The +goal of ``pd.NA`` is to provide a "missing" indicator that can be used +consistently across data types. ``pd.NA`` is currently used by the nullable integer and boolean +data types and the new string data type (:issue:`28095`). + +.. warning:: + + Experimental: the behaviour of ``pd.NA`` can still change without warning. + +For example, creating a Series using the nullable integer dtype: + +.. ipython:: python + + s = pd.Series([1, 2, None], dtype="Int64") + s + s[2] + +Compared to ``np.nan``, ``pd.NA`` behaves differently in certain operations. +In addition to arithmetic operations, ``pd.NA`` also propagates as "missing" +or "unknown" in comparison operations: + +.. ipython:: python + + np.nan > 1 + pd.NA > 1 + +For logical operations, ``pd.NA`` follows the rules of the +`three-valued logic `__ (or +*Kleene logic*). For example: + +.. ipython:: python + + pd.NA | True + +For more, see :ref:`NA section ` in the user guide on missing +data. + + .. _whatsnew_100.string: Dedicated string data type ^^^^^^^^^^^^^^^^^^^^^^^^^^ We've added :class:`StringDtype`, an extension type dedicated to string data. -Previously, strings were typically stored in object-dtype NumPy arrays. +Previously, strings were typically stored in object-dtype NumPy arrays. (:issue:`29975`) .. warning:: @@ -102,59 +125,15 @@ String accessor methods returning integers will return a value with :class:`Int6 We recommend explicitly using the ``string`` data type when working with strings. See :ref:`text.types` for more. -.. _whatsnew_100.NA: - -Experimental ``NA`` scalar to denote missing values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A new ``pd.NA`` value (singleton) is introduced to represent scalar missing -values. Up to now, ``np.nan`` is used for this for float data, ``np.nan`` or -``None`` for object-dtype data and ``pd.NaT`` for datetime-like data. The -goal of ``pd.NA`` is provide a "missing" indicator that can be used -consistently across data types. For now, the nullable integer and boolean -data types and the new string data type make use of ``pd.NA`` (:issue:`28095`). - -.. warning:: - - Experimental: the behaviour of ``pd.NA`` can still change without warning. - -For example, creating a Series using the nullable integer dtype: - -.. ipython:: python - - s = pd.Series([1, 2, None], dtype="Int64") - s - s[2] - -Compared to ``np.nan``, ``pd.NA`` behaves differently in certain operations. -In addition to arithmetic operations, ``pd.NA`` also propagates as "missing" -or "unknown" in comparison operations: - -.. ipython:: python - - np.nan > 1 - pd.NA > 1 - -For logical operations, ``pd.NA`` follows the rules of the -`three-valued logic `__ (or -*Kleene logic*). For example: - -.. ipython:: python - - pd.NA | True - -For more, see :ref:`NA section ` in the user guide on missing -data. - .. _whatsnew_100.boolean: Boolean data type with missing values support ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We've added :class:`BooleanDtype` / :class:`~arrays.BooleanArray`, an extension -type dedicated to boolean data that can hold missing values. With the default -``'bool`` data type based on a numpy bool array, the column can only hold -True or False values and not missing values. This new :class:`BooleanDtype` +type dedicated to boolean data that can hold missing values. The default +``bool`` data type based on a bool-dtype NumPy array, the column can only hold +``True`` or ``False``, and not missing values. This new :class:`~arrays.BooleanArray` can store missing values as well by keeping track of this in a separate mask. (:issue:`29555`, :issue:`30095`) @@ -169,7 +148,18 @@ You can use the alias ``"boolean"`` as well. s = pd.Series([True, False, None], dtype="boolean") s -.. _whatsnew_1000.custom_window: +.. _whatsnew_100.numba_rolling_apply: + +Using Numba in ``rolling.apply`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We've added an ``engine`` keyword to :meth:`~core.window.rolling.Rolling.apply` that allows the user to execute the +routine using `Numba `__ instead of Cython. Using the Numba engine +can yield significant performance gains if the apply function can operate on numpy arrays and +the data set is larger (1 million rows or greater). For more details, see +:ref:`rolling apply documentation ` (:issue:`28987`) + +.. _whatsnew_100.custom_window: Defining custom windows for rolling operations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -180,12 +170,25 @@ method on a :func:`pandas.api.indexers.BaseIndexer` subclass that will generate indices used for each window during the rolling aggregation. For more details and example usage, see the :ref:`custom window rolling documentation ` -.. _whatsnew_1000.enhancements.other: +.. _whatsnew_100.to_markdown: + +Converting to Markdown +^^^^^^^^^^^^^^^^^^^^^^ + +We've added :meth:`~DataFrame.to_markdown` for creating a markdown table (:issue:`11052`) + +.. ipython:: python + + df = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=['a', 'a', 'b']) + print(df.to_markdown()) + +.. _whatsnew_100.enhancements.other: Other enhancements ^^^^^^^^^^^^^^^^^^ - :meth:`DataFrame.to_string` added the ``max_colwidth`` parameter to control when wide columns are truncated (:issue:`9784`) +- Added the ``na_value`` argument to :meth:`Series.to_numpy`, :meth:`Index.to_numpy` and :meth:`DataFrame.to_numpy` to control the value used for missing data (:issue:`30322`) - :meth:`MultiIndex.from_product` infers level names from inputs if not explicitly provided (:issue:`27292`) - :meth:`DataFrame.to_latex` now accepts ``caption`` and ``label`` arguments (:issue:`25436`) - The :ref:`integer dtype ` with support for missing values and the @@ -201,12 +204,23 @@ Other enhancements - Added ``encoding`` argument to :func:`DataFrame.to_html` for non-ascii text (:issue:`28663`) - :meth:`Styler.background_gradient` now accepts ``vmin`` and ``vmax`` arguments (:issue:`12145`) - :meth:`Styler.format` added the ``na_rep`` parameter to help format the missing values (:issue:`21527`, :issue:`28358`) -- Roundtripping DataFrames with nullable integer or string data types to parquet +- Roundtripping DataFrames with nullable integer, string and period data types to parquet (:meth:`~DataFrame.to_parquet` / :func:`read_parquet`) using the `'pyarrow'` engine - now preserve those data types with pyarrow >= 1.0.0 (:issue:`20612`). + now preserve those data types with pyarrow >= 0.16.0 (:issue:`20612`, :issue:`28371`). - The ``partition_cols`` argument in :meth:`DataFrame.to_parquet` now accepts a string (:issue:`27117`) -- :func:`to_parquet` now appropriately handles the ``schema`` argument for user defined schemas in the pyarrow engine. (:issue: `30270`) +- :func:`pandas.read_json` now parses ``NaN``, ``Infinity`` and ``-Infinity`` (:issue:`12213`) +- The ``pandas.np`` submodule is now deprecated. Import numpy directly instead (:issue:`30296`) +- :func:`to_parquet` now appropriately handles the ``schema`` argument for user defined schemas in the pyarrow engine. (:issue:`30270`) - DataFrame constructor preserve `ExtensionArray` dtype with `ExtensionArray` (:issue:`11363`) +- :meth:`DataFrame.sort_values` and :meth:`Series.sort_values` have gained ``ignore_index`` keyword to be able to reset index after sorting (:issue:`30114`) +- :meth:`DataFrame.sort_index` and :meth:`Series.sort_index` have gained ``ignore_index`` keyword to reset index (:issue:`30114`) +- :meth:`DataFrame.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`30114`) +- Added new writer for exporting Stata dta files in version 118, ``StataWriter118``. This format supports exporting strings containing Unicode characters (:issue:`23573`) +- :meth:`Series.map` now accepts ``collections.abc.Mapping`` subclasses as a mapper (:issue:`29733`) +- The ``pandas.datetime`` class is now deprecated. Import from ``datetime`` instead (:issue:`30296`) +- Added an experimental :attr:`~DataFrame.attrs` for storing global metadata about a dataset (:issue:`29062`) +- :meth:`Timestamp.fromisocalendar` is now compatible with python 3.8 and above (:issue:`28115`) +- :meth:`DataFrame.to_pickle` and :func:`read_pickle` now accept URL (:issue:`30163`) Build Changes @@ -217,12 +231,14 @@ cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :i a built distribution (wheel) or via conda, this shouldn't have any effect on you. If you're building pandas from source, you should no longer need to install Cython into your build environment before calling ``pip install pandas``. -.. _whatsnew_1000.api_breaking: +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.api_breaking: Backwards incompatible API changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _whatsnew_1000.api_breaking.MultiIndex._names: +.. _whatsnew_100.api_breaking.MultiIndex._names: Avoid using names from ``MultiIndex.levels`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -240,10 +256,10 @@ For backwards compatibility, you can still *access* the names via the levels. mi.levels[0].name However, it is no longer possible to *update* the names of the ``MultiIndex`` -via the name of the level. The following will **silently** fail to update the -name of the ``MultiIndex`` +via the level. .. ipython:: python + :okexcept: mi.levels[0].name = "new name" mi.names @@ -270,52 +286,42 @@ New repr for :class:`~pandas.arrays.IntervalArray` closed='right', dtype='interval[int64]') - *pandas 1.0.0* .. ipython:: python pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)]) +Extended verbose info output for :class:`~pandas.DataFrame` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -All :class:`SeriesGroupBy` aggregation methods now respect the ``observed`` keyword -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The following methods now also correctly output values for unobserved categories when called through ``groupby(..., observed=False)`` (:issue:`17605`) - -- :meth:`SeriesGroupBy.count` -- :meth:`SeriesGroupBy.size` -- :meth:`SeriesGroupBy.nunique` -- :meth:`SeriesGroupBy.nth` - -.. ipython:: python - - df = pd.DataFrame({ - "cat_1": pd.Categorical(list("AABB"), categories=list("ABC")), - "cat_2": pd.Categorical(list("AB") * 2, categories=list("ABC")), - "value": [0.1] * 4, - }) - df - +- :meth:`DataFrame.info` now shows line numbers for the columns summary (:issue:`17304`) *pandas 0.25.x* -.. code-block:: ipython - - In [2]: df.groupby(["cat_1", "cat_2"], observed=False)["value"].count() - Out[2]: - cat_1 cat_2 - A A 1 - B 1 - B A 1 - B 1 - Name: value, dtype: int64 +.. code-block:: python + >>> df = pd.DataFrame({"int_col": [1, 2, 3], + ... "text_col": ["a", "b", "c"], + ... "float_col": [0.0, 0.1, 0.2]}) + >>> df.info(verbose=True) + + RangeIndex: 3 entries, 0 to 2 + Data columns (total 3 columns): + int_col 3 non-null int64 + text_col 3 non-null object + float_col 3 non-null float64 + dtypes: float64(1), int64(1), object(1) + memory usage: 152.0+ bytes *pandas 1.0.0* .. ipython:: python - df.groupby(["cat_1", "cat_2"], observed=False)["value"].count() + df = pd.DataFrame({"int_col": [1, 2, 3], + "text_col": ["a", "b", "c"], + "float_col": [0.0, 0.1, 0.2]}) + df.info(verbose=True) :meth:`pandas.array` inference changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -350,6 +356,93 @@ The following methods now also correctly output values for unobserved categories As a reminder, you can specify the ``dtype`` to disable all inference. +:class:`arrays.IntegerArray` now uses :attr:`pandas.NA` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:class:`arrays.IntegerArray` now uses :attr:`pandas.NA` rather than +:attr:`numpy.nan` as its missing value marker (:issue:`29964`). + +*pandas 0.25.x* + +.. code-block:: python + + >>> a = pd.array([1, 2, None], dtype="Int64") + >>> a + + [1, 2, NaN] + Length: 3, dtype: Int64 + + >>> a[2] + nan + +*pandas 1.0.0* + +.. ipython:: python + + a = pd.array([1, 2, None], dtype="Int64") + a + a[2] + +This has a few API-breaking consequences. + +**Converting to a NumPy ndarray** + +When converting to a NumPy array missing values will be ``pd.NA``, which cannot +be converted to a float. So calling ``np.asarray(integer_array, dtype="float")`` +will now raise. + +*pandas 0.25.x* + +.. code-block:: python + + >>> np.asarray(a, dtype="float") + array([ 1., 2., nan]) + +*pandas 1.0.0* + +.. ipython:: python + :okexcept: + + np.asarray(a, dtype="float") + +Use :meth:`arrays.IntegerArray.to_numpy` with an explicit ``na_value`` instead. + +.. ipython:: python + + a.to_numpy(dtype="float", na_value=np.nan) + +See :ref:`missing_data.NA` for more on the differences between :attr:`pandas.NA` +and :attr:`numpy.nan`. + +:class:`arrays.IntegerArray` comparisons return :class:`arrays.BooleanArray` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Comparison operations on a :class:`arrays.IntegerArray` now returns a +:class:`arrays.BooleanArray` rather than a NumPy array (:issue:`29964`). + +*pandas 0.25.x* + +.. code-block:: python + + >>> a = pd.array([1, 2, None], dtype="Int64") + >>> a + + [1, 2, NaN] + Length: 3, dtype: Int64 + + >>> a > 1 + array([False, True, False]) + +*pandas 1.0.0* + +.. ipython:: python + + a = pd.array([1, 2, None], dtype="Int64") + a > 1 + +Note that missing values now propagate, rather than always comparing unequal +like :attr:`numpy.nan`. See :ref:`missing_data.NA` for more. + By default :meth:`Categorical.min` now returns the minimum instead of np.nan ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -387,7 +480,14 @@ consistent with the behaviour of :class:`DataFrame` and :class:`Index`. DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning. Series([], dtype: float64) -.. _whatsnew_1000.api_breaking.deps: +.. _whatsnew_100.api_breaking.python: + +Increased minimum version for Python +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Pandas 1.0.0 supports Python 3.6.1 and higher (:issue:`29212`). + +.. _whatsnew_100.api_breaking.deps: Increased minimum versions for dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -428,9 +528,11 @@ Optional libraries below the lowest tested version may still work, but are not c +-----------------+-----------------+---------+ | matplotlib | 2.2.2 | | +-----------------+-----------------+---------+ +| numba | 0.46.0 | X | ++-----------------+-----------------+---------+ | openpyxl | 2.5.7 | X | +-----------------+-----------------+---------+ -| pyarrow | 0.12.0 | X | +| pyarrow | 0.13.0 | X | +-----------------+-----------------+---------+ | pymysql | 0.7.1 | | +-----------------+-----------------+---------+ @@ -454,13 +556,13 @@ Optional libraries below the lowest tested version may still work, but are not c See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more. -.. _whatsnew_1000.api.other: +.. _whatsnew_100.api.other: Other API changes ^^^^^^^^^^^^^^^^^ - Bumped the minimum supported version of ``s3fs`` from 0.0.8 to 0.3.0 (:issue:`28616`) -- :class:`pandas.core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`) +- :class:`core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`) - :meth:`pandas.api.types.infer_dtype` will now return "integer-na" for integer and ``np.nan`` mix (:issue:`27283`) - :meth:`MultiIndex.from_arrays` will no longer infer names from arrays if ``names=None`` is explicitly provided (:issue:`27292`) - In order to improve tab-completion, Pandas does not include most deprecated attributes when introspecting a pandas object using ``dir`` (e.g. ``dir(df)``). @@ -469,22 +571,24 @@ Other API changes - Changed the default configuration value for ``options.matplotlib.register_converters`` from ``True`` to ``"auto"`` (:issue:`18720`). Now, pandas custom formatters will only be applied to plots created by pandas, through :meth:`~DataFrame.plot`. Previously, pandas' formatters would be applied to all plots created *after* a :meth:`~DataFrame.plot`. - See :ref:`units registration ` for more. + See :ref:`units registration ` for more. - :meth:`Series.dropna` has dropped its ``**kwargs`` argument in favor of a single ``how`` parameter. Supplying anything else than ``how`` to ``**kwargs`` raised a ``TypeError`` previously (:issue:`29388`) - When testing pandas, the new minimum required version of pytest is 5.0.1 (:issue:`29664`) - :meth:`Series.str.__iter__` was deprecated and will be removed in future releases (:issue:`28277`). -.. _whatsnew_1000.api.documentation: +.. _whatsnew_100.api.documentation: Documentation Improvements ^^^^^^^^^^^^^^^^^^^^^^^^^^ - Added new section on :ref:`scale` (:issue:`28315`). -- Added sub-section Query MultiIndex in IO tools user guide (:issue:`28791`) +- Added sub-section on :ref:`io.query_multi` for HDF5 datasets (:issue:`28791`). + +.. --------------------------------------------------------------------------- -.. _whatsnew_1000.deprecations: +.. _whatsnew_100.deprecations: Deprecations ~~~~~~~~~~~~ @@ -495,31 +599,67 @@ Deprecations is equivalent to ``arr[idx.get_loc(idx_val)] = val``, which should be used instead (:issue:`28621`). - :func:`is_extension_type` is deprecated, :func:`is_extension_array_dtype` should be used instead (:issue:`29457`) - :func:`eval` keyword argument "truediv" is deprecated and will be removed in a future version (:issue:`29812`) -- :meth:`Categorical.take_nd` is deprecated, use :meth:`Categorical.take` instead (:issue:`27745`) +- :meth:`DateOffset.isAnchored` and :meth:`DatetOffset.onOffset` are deprecated and will be removed in a future version, use :meth:`DateOffset.is_anchored` and :meth:`DateOffset.is_on_offset` instead (:issue:`30340`) +- ``pandas.tseries.frequencies.get_offset`` is deprecated and will be removed in a future version, use ``pandas.tseries.frequencies.to_offset`` instead (:issue:`4205`) +- :meth:`Categorical.take_nd` and :meth:`CategoricalIndex.take_nd` are deprecated, use :meth:`Categorical.take` and :meth:`CategoricalIndex.take` instead (:issue:`27745`) - The parameter ``numeric_only`` of :meth:`Categorical.min` and :meth:`Categorical.max` is deprecated and replaced with ``skipna`` (:issue:`25303`) - The parameter ``label`` in :func:`lreshape` has been deprecated and will be removed in a future version (:issue:`29742`) - ``pandas.core.index`` has been deprecated and will be removed in a future version, the public classes are available in the top-level namespace (:issue:`19711`) - :func:`pandas.json_normalize` is now exposed in the top-level namespace. Usage of ``json_normalize`` as ``pandas.io.json.json_normalize`` is now deprecated and it is recommended to use ``json_normalize`` as :func:`pandas.json_normalize` instead (:issue:`27586`). +- The ``numpy`` argument of :meth:`pandas.read_json` is deprecated (:issue:`28512`). - :meth:`DataFrame.to_stata`, :meth:`DataFrame.to_feather`, and :meth:`DataFrame.to_parquet` argument "fname" is deprecated, use "path" instead (:issue:`23574`) +- The deprecated internal attributes ``_start``, ``_stop`` and ``_step`` of :class:`RangeIndex` now raise a ``FutureWarning`` instead of a ``DeprecationWarning`` (:issue:`26581`) +- The ``pandas.util.testing`` module has been deprecated. Use the public API in ``pandas.testing`` documented at :ref:`api.general.testing` (:issue:`16232`). +- ``pandas.SparseArray`` has been deprecated. Use ``pandas.arrays.SparseArray`` (:class:`arrays.SparseArray`) instead. (:issue:`30642`) +- The parameter ``is_copy`` of :meth:`DataFrame.take` has been deprecated and will be removed in a future version. (:issue:`27357`) + +**Selecting Columns from a Grouped DataFrame** +When selecting columns from a :class:`DataFrameGroupBy` object, passing individual keys (or a tuple of keys) inside single brackets is deprecated, +a list of items should be used instead. (:issue:`23566`) For example: -.. _whatsnew_1000.prior_deprecations: +.. code-block:: ipython + df = pd.DataFrame({ + "A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"], + "B": np.random.randn(8), + "C": np.random.randn(8), + }) + g = df.groupby('A') -Removed SparseSeries and SparseDataFrame -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # single key, returns SeriesGroupBy + g['B'] + + # tuple of single key, returns SeriesGroupBy + g[('B',)] + + # tuple of multiple keys, returns DataFrameGroupBy, raises FutureWarning + g[('B', 'C')] + + # multiple keys passed directly, returns DataFrameGroupBy, raises FutureWarning + # (implicitly converts the passed strings into a single tuple) + g['B', 'C'] + + # proper way, returns DataFrameGroupBy + g[['B', 'C']] + +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.prior_deprecations: + +Removal of prior version deprecations/changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Removed SparseSeries and SparseDataFrame** ``SparseSeries``, ``SparseDataFrame`` and the ``DataFrame.to_sparse`` method have been removed (:issue:`28425`). We recommend using a ``Series`` or ``DataFrame`` with sparse values instead. See :ref:`sparse.migration` for help with migrating existing code. -Removal of prior version deprecations/changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. _whatsnew_1000.matplotlib_units: +.. _whatsnew_100.matplotlib_units: **Matplotlib unit registration** @@ -540,122 +680,125 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more. **Other removals** -- Removed the previously deprecated "index" keyword from :func:`read_stata`, :class:`StataReader`, and :meth:`StataReader.read`, use "index_col" instead (:issue:`17328`) -- Removed the previously deprecated :meth:`StataReader.data` method, use :meth:`StataReader.read` instead (:issue:`9493`) -- Removed the previously deprecated :func:`pandas.plotting._matplotlib.tsplot`, use :meth:`Series.plot` instead (:issue:`19980`) -- :func:`pandas.tseries.converter.register` has been moved to :func:`pandas.plotting.register_matplotlib_converters` (:issue:`18307`) +- Removed the previously deprecated keyword "index" from :func:`read_stata`, :class:`StataReader`, and :meth:`StataReader.read`, use "index_col" instead (:issue:`17328`) +- Removed ``StataReader.data`` method, use :meth:`StataReader.read` instead (:issue:`9493`) +- Removed ``pandas.plotting._matplotlib.tsplot``, use :meth:`Series.plot` instead (:issue:`19980`) +- ``pandas.tseries.converter.register`` has been moved to :func:`pandas.plotting.register_matplotlib_converters` (:issue:`18307`) - :meth:`Series.plot` no longer accepts positional arguments, pass keyword arguments instead (:issue:`30003`) - :meth:`DataFrame.hist` and :meth:`Series.hist` no longer allows ``figsize="default"``, specify figure size by passinig a tuple instead (:issue:`30003`) - Floordiv of integer-dtyped array by :class:`Timedelta` now raises ``TypeError`` (:issue:`21036`) - :class:`TimedeltaIndex` and :class:`DatetimeIndex` no longer accept non-nanosecond dtype strings like "timedelta64" or "datetime64", use "timedelta64[ns]" and "datetime64[ns]" instead (:issue:`24806`) -- :func:`pandas.api.types.infer_dtype` argument ``skipna`` defaults to ``True`` instead of ``False`` (:issue:`24050`) -- Removed the previously deprecated :attr:`Series.ix` and :attr:`DataFrame.ix` (:issue:`26438`) -- Removed the previously deprecated :meth:`Index.summary` (:issue:`18217`) -- Removed the previously deprecated "fastpath" keyword from the :class:`Index` constructor (:issue:`23110`) -- Removed the previously deprecated :meth:`Series.get_value`, :meth:`Series.set_value`, :meth:`DataFrame.get_value`, :meth:`DataFrame.set_value` (:issue:`17739`) -- Removed the previously deprecated :meth:`Series.compound` and :meth:`DataFrame.compound` (:issue:`26405`) -- Changed the the default value of `inplace` in :meth:`DataFrame.set_index` and :meth:`Series.set_axis`. It now defaults to ``False`` (:issue:`27600`) -- Removed the previously deprecated :attr:`Series.cat.categorical`, :attr:`Series.cat.index`, :attr:`Series.cat.name` (:issue:`24751`) -- :func:`to_datetime` and :func:`to_timedelta` no longer accept "box" argument, always returns :class:`DatetimeIndex`, :class:`TimedeltaIndex`, :class:`Index`, :class:`Series`, or :class:`DataFrame` (:issue:`24486`) +- Changed the default "skipna" argument in :func:`pandas.api.types.infer_dtype` from ``False`` to ``True`` (:issue:`24050`) +- Removed ``Series.ix`` and ``DataFrame.ix`` (:issue:`26438`) +- Removed ``Index.summary`` (:issue:`18217`) +- Removed the previously deprecated keyword "fastpath" from the :class:`Index` constructor (:issue:`23110`) +- Removed ``Series.get_value``, ``Series.set_value``, ``DataFrame.get_value``, ``DataFrame.set_value`` (:issue:`17739`) +- Removed ``Series.compound`` and ``DataFrame.compound`` (:issue:`26405`) +- Changed the default "inplace" argument in :meth:`DataFrame.set_index` and :meth:`Series.set_axis` from ``None`` to ``False`` (:issue:`27600`) +- Removed ``Series.cat.categorical``, ``Series.cat.index``, ``Series.cat.name`` (:issue:`24751`) +- Removed the previously deprecated keyword "box" from :func:`to_datetime` and :func:`to_timedelta`; in addition these now always returns :class:`DatetimeIndex`, :class:`TimedeltaIndex`, :class:`Index`, :class:`Series`, or :class:`DataFrame` (:issue:`24486`) - :func:`to_timedelta`, :class:`Timedelta`, and :class:`TimedeltaIndex` no longer allow "M", "y", or "Y" for the "unit" argument (:issue:`23264`) -- Removed the previously deprecated ``time_rule`` keyword from (non-public) :func:`offsets.generate_range`, which has been moved to :func:`core.arrays._ranges.generate_range` (:issue:`24157`) +- Removed the previously deprecated keyword "time_rule" from (non-public) ``offsets.generate_range``, which has been moved to :func:`core.arrays._ranges.generate_range` (:issue:`24157`) - :meth:`DataFrame.loc` or :meth:`Series.loc` with listlike indexers and missing labels will no longer reindex (:issue:`17295`) - :meth:`DataFrame.to_excel` and :meth:`Series.to_excel` with non-existent columns will no longer reindex (:issue:`17295`) -- :func:`concat` parameter "join_axes" has been removed, use ``reindex_like`` on the result instead (:issue:`22318`) -- Removed the previously deprecated "by" keyword from :meth:`DataFrame.sort_index`, use :meth:`DataFrame.sort_values` instead (:issue:`10726`) -- Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`DataFrameGroupBy.aggregate`, :meth:`SeriesGroupBy.aggregate`, :meth:`Rolling.aggregate` (:issue:`18529`) +- Removed the previously deprecated keyword "join_axes" from :func:`concat`; use ``reindex_like`` on the result instead (:issue:`22318`) +- Removed the previously deprecated keyword "by" from :meth:`DataFrame.sort_index`, use :meth:`DataFrame.sort_values` instead (:issue:`10726`) +- Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`core.groupby.DataFrameGroupBy.aggregate`, :meth:`core.groupby.SeriesGroupBy.aggregate`, :meth:`core.window.rolling.Rolling.aggregate` (:issue:`18529`) - Passing ``datetime64`` data to :class:`TimedeltaIndex` or ``timedelta64`` data to ``DatetimeIndex`` now raises ``TypeError`` (:issue:`23539`, :issue:`23937`) - Passing ``int64`` values to :class:`DatetimeIndex` and a timezone now interprets the values as nanosecond timestamps in UTC, not wall times in the given timezone (:issue:`24559`) - A tuple passed to :meth:`DataFrame.groupby` is now exclusively treated as a single key (:issue:`18314`) -- Removed the previously deprecated :meth:`Index.contains`, use ``key in index`` instead (:issue:`30103`) +- Removed ``Index.contains``, use ``key in index`` instead (:issue:`30103`) - Addition and subtraction of ``int`` or integer-arrays is no longer allowed in :class:`Timestamp`, :class:`DatetimeIndex`, :class:`TimedeltaIndex`, use ``obj + n * obj.freq`` instead of ``obj + n`` (:issue:`22535`) -- Removed :meth:`Series.ptp` (:issue:`21614`) -- Removed :meth:`Series.from_array` (:issue:`18258`) -- Removed :meth:`DataFrame.from_items` (:issue:`18458`) -- Removed :meth:`DataFrame.as_matrix`, :meth:`Series.as_matrix` (:issue:`18458`) -- Removed :meth:`Series.asobject` (:issue:`18477`) -- Removed :meth:`DataFrame.as_blocks`, :meth:`Series.as_blocks`, `DataFrame.blocks`, :meth:`Series.blocks` (:issue:`17656`) +- Removed ``Series.ptp`` (:issue:`21614`) +- Removed ``Series.from_array`` (:issue:`18258`) +- Removed ``DataFrame.from_items`` (:issue:`18458`) +- Removed ``DataFrame.as_matrix``, ``Series.as_matrix`` (:issue:`18458`) +- Removed ``Series.asobject`` (:issue:`18477`) +- Removed ``DataFrame.as_blocks``, ``Series.as_blocks``, ``DataFrame.blocks``, ``Series.blocks`` (:issue:`17656`) - :meth:`pandas.Series.str.cat` now defaults to aligning ``others``, using ``join='left'`` (:issue:`27611`) - :meth:`pandas.Series.str.cat` does not accept list-likes *within* list-likes anymore (:issue:`27611`) - :meth:`Series.where` with ``Categorical`` dtype (or :meth:`DataFrame.where` with ``Categorical`` column) no longer allows setting new categories (:issue:`24114`) -- :class:`DatetimeIndex`, :class:`TimedeltaIndex`, and :class:`PeriodIndex` constructors no longer allow ``start``, ``end``, and ``periods`` keywords, use :func:`date_range`, :func:`timedelta_range`, and :func:`period_range` instead (:issue:`23919`) -- :class:`DatetimeIndex` and :class:`TimedeltaIndex` constructors no longer have a ``verify_integrity`` keyword argument (:issue:`23919`) -- ``pandas.core.internals.blocks.make_block`` no longer accepts the "fastpath" keyword(:issue:`19265`) -- :meth:`Block.make_block_same_class` no longer accepts the "dtype" keyword(:issue:`19434`) -- Removed the previously deprecated :meth:`ExtensionArray._formatting_values`. Use :attr:`ExtensionArray._formatter` instead. (:issue:`23601`) -- Removed the previously deprecated :meth:`MultiIndex.to_hierarchical` (:issue:`21613`) -- Removed the previously deprecated :attr:`MultiIndex.labels`, use :attr:`MultiIndex.codes` instead (:issue:`23752`) -- Removed the previously deprecated "labels" keyword from the :class:`MultiIndex` constructor, use "codes" instead (:issue:`23752`) -- Removed the previously deprecated :meth:`MultiIndex.set_labels`, use :meth:`MultiIndex.set_codes` instead (:issue:`23752`) -- Removed the previously deprecated "labels" keyword from :meth:`MultiIndex.set_codes`, :meth:`MultiIndex.copy`, :meth:`MultiIndex.drop`, use "codes" instead (:issue:`23752`) +- Removed the previously deprecated keywords "start", "end", and "periods" from the :class:`DatetimeIndex`, :class:`TimedeltaIndex`, and :class:`PeriodIndex` constructors; use :func:`date_range`, :func:`timedelta_range`, and :func:`period_range` instead (:issue:`23919`) +- Removed the previously deprecated keyword "verify_integrity" from the :class:`DatetimeIndex` and :class:`TimedeltaIndex` constructors (:issue:`23919`) +- Removed the previously deprecated keyword "fastpath" from ``pandas.core.internals.blocks.make_block`` (:issue:`19265`) +- Removed the previously deprecated keyword "dtype" from :meth:`Block.make_block_same_class` (:issue:`19434`) +- Removed ``ExtensionArray._formatting_values``. Use :attr:`ExtensionArray._formatter` instead. (:issue:`23601`) +- Removed ``MultiIndex.to_hierarchical`` (:issue:`21613`) +- Removed ``MultiIndex.labels``, use :attr:`MultiIndex.codes` instead (:issue:`23752`) +- Removed the previously deprecated keyword "labels" from the :class:`MultiIndex` constructor, use "codes" instead (:issue:`23752`) +- Removed ``MultiIndex.set_labels``, use :meth:`MultiIndex.set_codes` instead (:issue:`23752`) +- Removed the previously deprecated keyword "labels" from :meth:`MultiIndex.set_codes`, :meth:`MultiIndex.copy`, :meth:`MultiIndex.drop`, use "codes" instead (:issue:`23752`) - Removed support for legacy HDF5 formats (:issue:`29787`) - Passing a dtype alias (e.g. 'datetime64[ns, UTC]') to :class:`DatetimeTZDtype` is no longer allowed, use :meth:`DatetimeTZDtype.construct_from_string` instead (:issue:`23990`) -- :func:`read_excel` removed support for "skip_footer" argument, use "skipfooter" instead (:issue:`18836`) +- Removed the previously deprecated keyword "skip_footer" from :func:`read_excel`; use "skipfooter" instead (:issue:`18836`) - :func:`read_excel` no longer allows an integer value for the parameter ``usecols``, instead pass a list of integers from 0 to ``usecols`` inclusive (:issue:`23635`) -- :meth:`DataFrame.to_records` no longer supports the argument "convert_datetime64" (:issue:`18902`) -- Removed the previously deprecated ``IntervalIndex.from_intervals`` in favor of the :class:`IntervalIndex` constructor (:issue:`19263`) -- Changed the default value for the "keep_tz" argument in :meth:`DatetimeIndex.to_series` to ``True`` (:issue:`23739`) -- Removed the previously deprecated :func:`api.types.is_period` and :func:`api.types.is_datetimetz` (:issue:`23917`) +- Removed the previously deprecated keyword "convert_datetime64" from :meth:`DataFrame.to_records` (:issue:`18902`) +- Removed ``IntervalIndex.from_intervals`` in favor of the :class:`IntervalIndex` constructor (:issue:`19263`) +- Changed the default "keep_tz" argument in :meth:`DatetimeIndex.to_series` from ``None`` to ``True`` (:issue:`23739`) +- Removed ``api.types.is_period`` and ``api.types.is_datetimetz`` (:issue:`23917`) - Ability to read pickles containing :class:`Categorical` instances created with pre-0.16 version of pandas has been removed (:issue:`27538`) -- Removed previously deprecated :func:`pandas.tseries.plotting.tsplot` (:issue:`18627`) -- Removed the previously deprecated ``reduce`` and ``broadcast`` arguments from :meth:`DataFrame.apply` (:issue:`18577`) -- Removed the previously deprecated ``assert_raises_regex`` function in ``pandas.util.testing`` (:issue:`29174`) +- Removed ``pandas.tseries.plotting.tsplot`` (:issue:`18627`) +- Removed the previously deprecated keywords "reduce" and "broadcast" from :meth:`DataFrame.apply` (:issue:`18577`) +- Removed the previously deprecated ``assert_raises_regex`` function in ``pandas._testing`` (:issue:`29174`) - Removed the previously deprecated ``FrozenNDArray`` class in ``pandas.core.indexes.frozen`` (:issue:`29335`) -- Removed previously deprecated "nthreads" argument from :func:`read_feather`, use "use_threads" instead (:issue:`23053`) -- Removed :meth:`Index.is_lexsorted_for_tuple` (:issue:`29305`) -- Removed support for nexted renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`DataFrameGroupBy.aggregate`, :meth:`SeriesGroupBy.aggregate`, :meth:`Rolling.aggregate` (:issue:`29608`) -- Removed the previously deprecated :meth:`Series.valid`; use :meth:`Series.dropna` instead (:issue:`18800`) -- Removed the previously properties :attr:`DataFrame.is_copy`, :attr:`Series.is_copy` (:issue:`18812`) -- Removed the previously deprecated :meth:`DataFrame.get_ftype_counts`, :meth:`Series.get_ftype_counts` (:issue:`18243`) -- Removed the previously deprecated :meth:`DataFrame.ftypes`, :meth:`Series.ftypes`, :meth:`Series.ftype` (:issue:`26744`) -- Removed the previously deprecated :meth:`Index.get_duplicates`, use ``idx[idx.duplicated()].unique()`` instead (:issue:`20239`) -- Removed the previously deprecated :meth:`Series.clip_upper`, :meth:`Series.clip_lower`, :meth:`DataFrame.clip_upper`, :meth:`DataFrame.clip_lower` (:issue:`24203`) +- Removed the previously deprecated keyword "nthreads" from :func:`read_feather`, use "use_threads" instead (:issue:`23053`) +- Removed ``Index.is_lexsorted_for_tuple`` (:issue:`29305`) +- Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`core.groupby.DataFrameGroupBy.aggregate`, :meth:`core.groupby.SeriesGroupBy.aggregate`, :meth:`core.window.rolling.Rolling.aggregate` (:issue:`29608`) +- Removed ``Series.valid``; use :meth:`Series.dropna` instead (:issue:`18800`) +- Removed ``DataFrame.is_copy``, ``Series.is_copy`` (:issue:`18812`) +- Removed ``DataFrame.get_ftype_counts``, ``Series.get_ftype_counts`` (:issue:`18243`) +- Removed ``DataFrame.ftypes``, ``Series.ftypes``, ``Series.ftype`` (:issue:`26744`) +- Removed ``Index.get_duplicates``, use ``idx[idx.duplicated()].unique()`` instead (:issue:`20239`) +- Removed ``Series.clip_upper``, ``Series.clip_lower``, ``DataFrame.clip_upper``, ``DataFrame.clip_lower`` (:issue:`24203`) - Removed the ability to alter :attr:`DatetimeIndex.freq`, :attr:`TimedeltaIndex.freq`, or :attr:`PeriodIndex.freq` (:issue:`20772`) -- Removed the previously deprecated :attr:`DatetimeIndex.offset` (:issue:`20730`) -- Removed the previously deprecated :meth:`DatetimeIndex.asobject`, :meth:`TimedeltaIndex.asobject`, :meth:`PeriodIndex.asobject`, use ``astype(object)`` instead (:issue:`29801`) -- Removed previously deprecated "order" argument from :func:`factorize` (:issue:`19751`) -- :func:`read_stata` and :meth:`DataFrame.to_stata` no longer supports the "encoding" argument (:issue:`21400`) -- In :func:`concat` the default value for ``sort`` has been changed from ``None`` to ``False`` (:issue:`20613`) -- Removed previously deprecated "raise_conflict" argument from :meth:`DataFrame.update`, use "errors" instead (:issue:`23585`) -- Removed previously deprecated keyword "n" from :meth:`DatetimeIndex.shift`, :meth:`TimedeltaIndex.shift`, :meth:`PeriodIndex.shift`, use "periods" instead (:issue:`22458`) -- Removed previously deprecated keywords ``how``, ``fill_method``, and ``limit`` from :meth:`DataFrame.resample` (:issue:`30139`) +- Removed ``DatetimeIndex.offset`` (:issue:`20730`) +- Removed ``DatetimeIndex.asobject``, ``TimedeltaIndex.asobject``, ``PeriodIndex.asobject``, use ``astype(object)`` instead (:issue:`29801`) +- Removed the previously deprecated keyword "order" from :func:`factorize` (:issue:`19751`) +- Removed the previously deprecated keyword "encoding" from :func:`read_stata` and :meth:`DataFrame.to_stata` (:issue:`21400`) +- Changed the default "sort" argument in :func:`concat` from ``None`` to ``False`` (:issue:`20613`) +- Removed the previously deprecated keyword "raise_conflict" from :meth:`DataFrame.update`, use "errors" instead (:issue:`23585`) +- Removed the previously deprecated keyword "n" from :meth:`DatetimeIndex.shift`, :meth:`TimedeltaIndex.shift`, :meth:`PeriodIndex.shift`, use "periods" instead (:issue:`22458`) +- Removed the previously deprecated keywords "how", "fill_method", and "limit" from :meth:`DataFrame.resample` (:issue:`30139`) - Passing an integer to :meth:`Series.fillna` or :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtype now raises ``TypeError`` (:issue:`24694`) - Passing multiple axes to :meth:`DataFrame.dropna` is no longer supported (:issue:`20995`) -- Removed previously deprecated :meth:`Series.nonzero`, use `to_numpy().nonzero()` instead (:issue:`24048`) +- Removed ``Series.nonzero``, use ``to_numpy().nonzero()`` instead (:issue:`24048`) - Passing floating dtype ``codes`` to :meth:`Categorical.from_codes` is no longer supported, pass ``codes.astype(np.int64)`` instead (:issue:`21775`) -- :meth:`Series.str.partition` and :meth:`Series.str.rpartition` no longer accept "pat" keyword, use "sep" instead (:issue:`23767`) -- Removed the previously deprecated :meth:`Series.put` (:issue:`27106`) -- Removed the previously deprecated :attr:`Series.real`, :attr:`Series.imag` (:issue:`27106`) -- Removed the previously deprecated :meth:`Series.to_dense`, :meth:`DataFrame.to_dense` (:issue:`26684`) -- Removed the previously deprecated :meth:`Index.dtype_str`, use ``str(index.dtype)`` instead (:issue:`27106`) +- Removed the previously deprecated keyword "pat" from :meth:`Series.str.partition` and :meth:`Series.str.rpartition`, use "sep" instead (:issue:`23767`) +- Removed ``Series.put`` (:issue:`27106`) +- Removed ``Series.real``, ``Series.imag`` (:issue:`27106`) +- Removed ``Series.to_dense``, ``DataFrame.to_dense`` (:issue:`26684`) +- Removed ``Index.dtype_str``, use ``str(index.dtype)`` instead (:issue:`27106`) - :meth:`Categorical.ravel` returns a :class:`Categorical` instead of a ``ndarray`` (:issue:`27199`) - The 'outer' method on Numpy ufuncs, e.g. ``np.subtract.outer`` operating on :class:`Series` objects is no longer supported, and will raise ``NotImplementedError`` (:issue:`27198`) -- Removed previously deprecated :meth:`Series.get_dtype_counts` and :meth:`DataFrame.get_dtype_counts` (:issue:`27145`) -- Changed the default ``fill_value`` in :meth:`Categorical.take` from ``True`` to ``False`` (:issue:`20841`) -- Changed the default value for the `raw` argument in :func:`Series.rolling().apply() `, :func:`DataFrame.rolling().apply() `, -- :func:`Series.expanding().apply() `, and :func:`DataFrame.expanding().apply() ` to ``False`` (:issue:`20584`) +- Removed ``Series.get_dtype_counts`` and ``DataFrame.get_dtype_counts`` (:issue:`27145`) +- Changed the default "fill_value" argument in :meth:`Categorical.take` from ``True`` to ``False`` (:issue:`20841`) +- Changed the default value for the `raw` argument in :func:`Series.rolling().apply() `, :func:`DataFrame.rolling().apply() `, :func:`Series.expanding().apply() `, and :func:`DataFrame.expanding().apply() ` from ``None`` to ``False`` (:issue:`20584`) - Removed deprecated behavior of :meth:`Series.argmin` and :meth:`Series.argmax`, use :meth:`Series.idxmin` and :meth:`Series.idxmax` for the old behavior (:issue:`16955`) - Passing a tz-aware ``datetime.datetime`` or :class:`Timestamp` into the :class:`Timestamp` constructor with the ``tz`` argument now raises a ``ValueError`` (:issue:`23621`) -- Removed the previously deprecated :attr:`Series.base`, :attr:`Index.base`, :attr:`Categorical.base`, :attr:`Series.flags`, :attr:`Index.flags`, :attr:`PeriodArray.flags`, :attr:`Series.strides`, :attr:`Index.strides`, :attr:`Series.itemsize`, :attr:`Index.itemsize`, :attr:`Series.data`, :attr:`Index.data` (:issue:`20721`) +- Removed ``Series.base``, ``Index.base``, ``Categorical.base``, ``Series.flags``, ``Index.flags``, ``PeriodArray.flags``, ``Series.strides``, ``Index.strides``, ``Series.itemsize``, ``Index.itemsize``, ``Series.data``, ``Index.data`` (:issue:`20721`) - Changed :meth:`Timedelta.resolution` to match the behavior of the standard library ``datetime.timedelta.resolution``, for the old behavior, use :meth:`Timedelta.resolution_string` (:issue:`26839`) -- Removed previously deprecated :attr:`Timestamp.weekday_name`, :attr:`DatetimeIndex.weekday_name`, and :attr:`Series.dt.weekday_name` (:issue:`18164`) -- Removed previously deprecated ``errors`` argument in :meth:`Timestamp.tz_localize`, :meth:`DatetimeIndex.tz_localize`, and :meth:`Series.tz_localize` (:issue:`22644`) -- Changed the default value for ``ordered`` in :class:`CategoricalDtype` from ``None`` to ``False`` (:issue:`26336`) +- Removed ``Timestamp.weekday_name``, ``DatetimeIndex.weekday_name``, and ``Series.dt.weekday_name`` (:issue:`18164`) +- Removed the previously deprecated keyword "errors" in :meth:`Timestamp.tz_localize`, :meth:`DatetimeIndex.tz_localize`, and :meth:`Series.tz_localize` (:issue:`22644`) +- Changed the default "ordered" argument in :class:`CategoricalDtype` from ``None`` to ``False`` (:issue:`26336`) - :meth:`Series.set_axis` and :meth:`DataFrame.set_axis` now require "labels" as the first argument and "axis" as an optional named parameter (:issue:`30089`) -- Removed the previously deprecated :func:`to_msgpack`, :func:`read_msgpack`, :meth:`DataFrame.to_msgpack`, :meth:`Series.to_msgpack` (:issue:`27103`) -- +- Removed ``to_msgpack``, ``read_msgpack``, ``DataFrame.to_msgpack``, ``Series.to_msgpack`` (:issue:`27103`) +- Removed ``Series.compress`` (:issue:`21930`) - Removed the previously deprecated keyword "fill_value" from :meth:`Categorical.fillna`, use "value" instead (:issue:`19269`) - Removed the previously deprecated keyword "data" from :func:`andrews_curves`, use "frame" instead (:issue:`6956`) - Removed the previously deprecated keyword "data" from :func:`parallel_coordinates`, use "frame" instead (:issue:`6956`) - Removed the previously deprecated keyword "colors" from :func:`parallel_coordinates`, use "color" instead (:issue:`6956`) - Removed the previously deprecated keywords "verbose" and "private_key" from :func:`read_gbq` (:issue:`30200`) +- Calling ``np.array`` and ``np.asarray`` on tz-aware :class:`Series` and :class:`DatetimeIndex` will now return an object array of tz-aware :class:`Timestamp` (:issue:`24596`) - -.. _whatsnew_1000.performance: +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.performance: Performance improvements ~~~~~~~~~~~~~~~~~~~~~~~~ +- Performance improvement in :class:`DataFrame` arithmetic and comparison operations with scalars (:issue:`24990`, :issue:`29853`) - Performance improvement in indexing with a non-unique :class:`IntervalIndex` (:issue:`27489`) - Performance improvement in :attr:`MultiIndex.is_monotonic` (:issue:`27495`) - Performance improvement in :func:`cut` when ``bins`` is an :class:`IntervalIndex` (:issue:`27668`) @@ -670,7 +813,9 @@ Performance improvements - Performance improvement in :meth:`Index.equals` and :meth:`MultiIndex.equals` (:issue:`29134`) - Performance improvement in :func:`~pandas.api.types.infer_dtype` when ``skipna`` is ``True`` (:issue:`28814`) -.. _whatsnew_1000.bug_fixes: +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.bug_fixes: Bug fixes ~~~~~~~~~ @@ -692,6 +837,11 @@ Categorical :class:`Categorical` with duplicate entries, the accessor was skipping duplicates (:issue:`27952`) - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` that would give incorrect results on categorical data (:issue:`26988`) - Bug where calling :meth:`Categorical.min` or :meth:`Categorical.max` on an empty Categorical would raise a numpy exception (:issue:`30227`) +- The following methods now also correctly output values for unobserved categories when called through ``groupby(..., observed=False)`` (:issue:`17605`) + * :meth:`core.groupby.SeriesGroupBy.count` + * :meth:`core.groupby.SeriesGroupBy.size` + * :meth:`core.groupby.SeriesGroupBy.nunique` + * :meth:`core.groupby.SeriesGroupBy.nth` Datetimelike @@ -700,16 +850,18 @@ Datetimelike - Bug in :meth:`Series.dt` property lookups when the underlying data is read-only (:issue:`27529`) - Bug in ``HDFStore.__getitem__`` incorrectly reading tz attribute created in Python 2 (:issue:`26443`) - Bug in :func:`to_datetime` where passing arrays of malformed ``str`` with errors="coerce" could incorrectly lead to raising ``ValueError`` (:issue:`28299`) -- Bug in :meth:`pandas.core.groupby.SeriesGroupBy.nunique` where ``NaT`` values were interfering with the count of unique values (:issue:`27951`) +- Bug in :meth:`core.groupby.SeriesGroupBy.nunique` where ``NaT`` values were interfering with the count of unique values (:issue:`27951`) - Bug in :class:`Timestamp` subtraction when subtracting a :class:`Timestamp` from a ``np.datetime64`` object incorrectly raising ``TypeError`` (:issue:`28286`) - Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`) - Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`) +- Bug in :meth:`Series.astype`, :meth:`Index.astype`, and :meth:`DataFrame.astype` failing to handle ``NaT`` when casting to an integer dtype (:issue:`28492`) - Bug in :class:`Week` with ``weekday`` incorrectly raising ``AttributeError`` instead of ``TypeError`` when adding or subtracting an invalid type (:issue:`28530`) - Bug in :class:`DataFrame` arithmetic operations when operating with a :class:`Series` with dtype `'timedelta64[ns]'` (:issue:`28049`) -- Bug in :func:`pandas.core.groupby.generic.SeriesGroupBy.apply` raising ``ValueError`` when a column in the original DataFrame is a datetime and the column labels are not standard integers (:issue:`28247`) +- Bug in :func:`core.groupby.generic.SeriesGroupBy.apply` raising ``ValueError`` when a column in the original DataFrame is a datetime and the column labels are not standard integers (:issue:`28247`) - Bug in :func:`pandas._config.localization.get_locales` where the ``locales -a`` encodes the locales list as windows-1252 (:issue:`23638`, :issue:`24760`, :issue:`27368`) - Bug in :meth:`Series.var` failing to raise ``TypeError`` when called with ``timedelta64[ns]`` dtype (:issue:`28289`) - Bug in :meth:`DatetimeIndex.strftime` and :meth:`Series.dt.strftime` where ``NaT`` was converted to the string ``'NaT'`` instead of ``np.nan`` (:issue:`29578`) +- Bug in masking datetime-like arrays with a boolean mask of an incorrect length not raising an ``IndexError`` (:issue:`30308`) - Bug in :attr:`Timestamp.resolution` being a property instead of a class attribute (:issue:`29910`) - Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`) - Bug in :func:`pandas.to_datetime` failing for `deques` when using ``cache=True`` (the default) (:issue:`29403`) @@ -717,7 +869,12 @@ Datetimelike - Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`) - Bug in :meth:`DataFrame.drop` where attempting to drop non-existent values from a DatetimeIndex would yield a confusing error message (:issue:`30399`) - Bug in :meth:`DataFrame.append` would remove the timezone-awareness of new data (:issue:`30238`) - +- Bug in :meth:`Series.cummin` and :meth:`Series.cummax` with timezone-aware dtype incorrectly dropping its timezone (:issue:`15553`) +- Bug in :class:`DatetimeArray`, :class:`TimedeltaArray`, and :class:`PeriodArray` where inplace addition and subtraction did not actually operate inplace (:issue:`24115`) +- Bug in :func:`pandas.to_datetime` when called with ``Series`` storing ``IntegerArray`` raising ``TypeError`` instead of returning ``Series`` (:issue:`30050`) +- Bug in :func:`date_range` with custom business hours as ``freq`` and given number of ``periods`` (:issue:`30593`) +- Bug in :class:`PeriodIndex` comparisons with incorrectly casting integers to :class:`Period` objects, inconsistent with the :class:`Period` comparison behavior (:issue:`30722`) +- Bug in :meth:`DatetimeIndex.insert` raising a ``ValueError`` instead of a ``TypeError`` when trying to insert a timezone-aware :class:`Timestamp` into a timezone-naive :class:`DatetimeIndex`, or vice-versa (:issue:`30806`) Timedelta ^^^^^^^^^ @@ -746,6 +903,8 @@ Numeric - Bug in :class:`NumericIndex` construction that caused :class:`UInt64Index` to be casted to :class:`Float64Index` when integers in the ``np.uint64`` range were used to index a :class:`DataFrame` (:issue:`28279`) - Bug in :meth:`Series.interpolate` when using method=`index` with an unsorted index, would previously return incorrect results. (:issue:`21037`) - Bug in :meth:`DataFrame.round` where a :class:`DataFrame` with a :class:`CategoricalIndex` of :class:`IntervalIndex` columns would incorrectly raise a ``TypeError`` (:issue:`30063`) +- Bug in :meth:`Series.pct_change` and :meth:`DataFrame.pct_change` when there are duplicated indices (:issue:`30463`) +- Bug in :class:`DataFrame` cumulative operations (e.g. cumsum, cummax) incorrect casting to object-dtype (:issue:`19296`) Conversion ^^^^^^^^^^ @@ -756,7 +915,7 @@ Conversion Strings ^^^^^^^ -- Calling :meth:`Series.str.isalnum` (and other "ismethods") on an empty Series would return an object dtype instead of bool (:issue:`29624`) +- Calling :meth:`Series.str.isalnum` (and other "ismethods") on an empty ``Series`` would return an ``object`` dtype instead of ``bool`` (:issue:`29624`) - @@ -765,6 +924,9 @@ Interval - Bug in :meth:`IntervalIndex.get_indexer` where a :class:`Categorical` or :class:`CategoricalIndex` ``target`` would incorrectly raise a ``TypeError`` (:issue:`30063`) - Bug in ``pandas.core.dtypes.cast.infer_dtype_from_scalar`` where passing ``pandas_dtype=True`` did not infer :class:`IntervalDtype` (:issue:`30337`) +- Bug in :class:`Series` constructor where constructing a ``Series`` from a ``list`` of :class:`Interval` objects resulted in ``object`` dtype instead of :class:`IntervalDtype` (:issue:`23563`) +- Bug in :class:`IntervalDtype` where the ``kind`` attribute was incorrectly set as ``None`` instead of ``"O"`` (:issue:`30568`) +- Bug in :class:`IntervalIndex`, :class:`~arrays.IntervalArray`, and :class:`Series` with interval data where equality comparisons were incorrect (:issue:`24112`) Indexing ^^^^^^^^ @@ -776,8 +938,11 @@ Indexing - Bug in :meth:`Float64Index.astype` where ``np.inf`` was not handled properly when casting to an integer dtype (:issue:`28475`) - :meth:`Index.union` could fail when the left contained duplicates (:issue:`28257`) - Bug when indexing with ``.loc`` where the index was a :class:`CategoricalIndex` with non-string categories didn't work (:issue:`17569`, :issue:`30225`) -- :meth:`Index.get_indexer_non_unique` could fail with `TypeError` in some cases, such as when searching for ints in a string index (:issue:`28257`) +- :meth:`Index.get_indexer_non_unique` could fail with ``TypeError`` in some cases, such as when searching for ints in a string index (:issue:`28257`) - Bug in :meth:`Float64Index.get_loc` incorrectly raising ``TypeError`` instead of ``KeyError`` (:issue:`29189`) +- :meth:`MultiIndex.get_loc` can't find missing values when input includes missing values (:issue:`19132`) +- Bug in :meth:`Series.__setitem__` incorrectly assigning values with boolean indexer when the length of new data matches the number of ``True`` values and new data is not a ``Series`` or an ``np.array`` (:issue:`30567`) +- Bug in indexing with a :class:`PeriodIndex` incorrectly accepting integers representing years, use e.g. ``ser.loc["2007"]`` instead of ``ser.loc[2007]`` (:issue:`30763`) Missing ^^^^^^^ @@ -788,8 +953,8 @@ Missing MultiIndex ^^^^^^^^^^ -- Constructior for :class:`MultiIndex` verifies that the given ``sortorder`` is compatible with the actual ``lexsort_depth`` if ``verify_integrity`` parameter is ``True`` (the default) (:issue:`28735`) -- +- Constructor for :class:`MultiIndex` verifies that the given ``sortorder`` is compatible with the actual ``lexsort_depth`` if ``verify_integrity`` parameter is ``True`` (the default) (:issue:`28735`) +- Series and MultiIndex `.drop` with `MultiIndex` raise exception if labels not in given in level (:issue:`8594`) - I/O @@ -813,41 +978,47 @@ I/O - Bug in :func:`read_json` where default encoding was not set to ``utf-8`` (:issue:`29565`) - Bug in :class:`PythonParser` where str and bytes were being mixed when dealing with the decimal field (:issue:`29650`) - :meth:`read_gbq` now accepts ``progress_bar_type`` to display progress bar while the data downloads. (:issue:`29857`) +- Bug in :func:`pandas.io.json.json_normalize` where a missing value in the location specified by `record_path` would raise a ``TypeError`` (:issue:`30148`) +- :func:`read_excel` now accepts binary data (:issue:`15914`) +- Bug in :meth:`read_csv` in which encoding handling was limited to just the string `utf-16` for the C engine (:issue:`24130`) Plotting ^^^^^^^^ - Bug in :meth:`Series.plot` not able to plot boolean values (:issue:`23719`) -- - Bug in :meth:`DataFrame.plot` not able to plot when no rows (:issue:`27758`) - Bug in :meth:`DataFrame.plot` producing incorrect legend markers when plotting multiple series on the same axis (:issue:`18222`) - Bug in :meth:`DataFrame.plot` when ``kind='box'`` and data contains datetime or timedelta data. These types are now automatically dropped (:issue:`22799`) - Bug in :meth:`DataFrame.plot.line` and :meth:`DataFrame.plot.area` produce wrong xlim in x-axis (:issue:`27686`, :issue:`25160`, :issue:`24784`) -- Bug where :meth:`DataFrame.boxplot` would not accept a `color` parameter like `DataFrame.plot.box` (:issue:`26214`) +- Bug where :meth:`DataFrame.boxplot` would not accept a ``color`` parameter like :meth:`DataFrame.plot.box` (:issue:`26214`) - Bug in the ``xticks`` argument being ignored for :meth:`DataFrame.plot.bar` (:issue:`14119`) - :func:`set_option` now validates that the plot backend provided to ``'plotting.backend'`` implements the backend when the option is set, rather than when a plot is created (:issue:`28163`) - :meth:`DataFrame.plot` now allow a ``backend`` keyword argument to allow changing between backends in one session (:issue:`28619`). - Bug in color validation incorrectly raising for non-color styles (:issue:`29122`). +- Allow :meth:`DataFrame.plot.scatter` to plot ``objects`` and ``datetime`` type data (:issue:`18755`, :issue:`30391`) +- Bug in :meth:`DataFrame.hist`, ``xrot=0`` does not work with ``by`` and subplots (:issue:`30288`). Groupby/resample/rolling ^^^^^^^^^^^^^^^^^^^^^^^^ -- +- Bug in :meth:`core.groupby.DataFrameGroupBy.apply` only showing output from a single group when function returns an :class:`Index` (:issue:`28652`) - Bug in :meth:`DataFrame.groupby` with multiple groups where an ``IndexError`` would be raised if any group contained all NA values (:issue:`20519`) -- Bug in :meth:`pandas.core.resample.Resampler.size` and :meth:`pandas.core.resample.Resampler.count` returning wrong dtype when used with an empty series or dataframe (:issue:`28427`) +- Bug in :meth:`pandas.core.resample.Resampler.size` and :meth:`pandas.core.resample.Resampler.count` returning wrong dtype when used with an empty :class:`Series` or :class:`DataFrame` (:issue:`28427`) - Bug in :meth:`DataFrame.rolling` not allowing for rolling over datetimes when ``axis=1`` (:issue:`28192`) - Bug in :meth:`DataFrame.rolling` not allowing rolling over multi-index levels (:issue:`15584`). - Bug in :meth:`DataFrame.rolling` not allowing rolling on monotonic decreasing time indexes (:issue:`19248`). - Bug in :meth:`DataFrame.groupby` not offering selection by column name when ``axis=1`` (:issue:`27614`) -- Bug in :meth:`DataFrameGroupby.agg` not able to use lambda function with named aggregation (:issue:`27519`) +- Bug in :meth:`core.groupby.DataFrameGroupby.agg` not able to use lambda function with named aggregation (:issue:`27519`) - Bug in :meth:`DataFrame.groupby` losing column name information when grouping by a categorical column (:issue:`28787`) - Remove error raised due to duplicated input functions in named aggregation in :meth:`DataFrame.groupby` and :meth:`Series.groupby`. Previously error will be raised if the same function is applied on the same column and now it is allowed if new assigned names are different. (:issue:`28426`) -- :meth:`SeriesGroupBy.value_counts` will be able to handle the case even when the :class:`Grouper` makes empty groups (:issue: 28479) -- Bug in :meth:`DataFrameGroupBy.rolling().quantile()` ignoring ``interpolation`` keyword argument (:issue:`28779`) +- :meth:`core.groupby.SeriesGroupBy.value_counts` will be able to handle the case even when the :class:`Grouper` makes empty groups (:issue:`28479`) +- Bug in :meth:`core.window.rolling.Rolling.quantile` ignoring ``interpolation`` keyword argument when used within a groupby (:issue:`28779`) - Bug in :meth:`DataFrame.groupby` where ``any``, ``all``, ``nunique`` and transform functions would incorrectly handle duplicate column labels (:issue:`21668`) -- Bug in :meth:`DataFrameGroupBy.agg` with timezone-aware datetime64 column incorrectly casting results to the original dtype (:issue:`29641`) +- Bug in :meth:`core.groupby.DataFrameGroupBy.agg` with timezone-aware datetime64 column incorrectly casting results to the original dtype (:issue:`29641`) - Bug in :meth:`DataFrame.groupby` when using axis=1 and having a single level columns index (:issue:`30208`) - Bug in :meth:`DataFrame.groupby` when using nunique on axis=1 (:issue:`30253`) +- Bug in :meth:`GroupBy.quantile` with multiple list-like q value and integer column names (:issue:`30289`) +- Bug in :meth:`GroupBy.pct_change` and :meth:`core.groupby.SeriesGroupBy.pct_change` causes ``TypeError`` when ``fill_method`` is ``None`` (:issue:`30463`) Reshaping ^^^^^^^^^ @@ -860,17 +1031,19 @@ Reshaping - :func:`qcut` and :func:`cut` now handle boolean input (:issue:`20303`) - Fix to ensure all int dtypes can be used in :func:`merge_asof` when using a tolerance value. Previously every non-int64 type would raise an erroneous ``MergeError`` (:issue:`28870`). - Better error message in :func:`get_dummies` when `columns` isn't a list-like value (:issue:`28383`) -- Bug :meth:`Series.pct_change` where supplying an anchored frequency would throw a ValueError (:issue:`28664`) +- Bug in :meth:`Index.join` that caused infinite recursion error for mismatched ``MultiIndex`` name orders. (:issue:`25760`, :issue:`28956`) +- Bug :meth:`Series.pct_change` where supplying an anchored frequency would throw a ``ValueError`` (:issue:`28664`) - Bug where :meth:`DataFrame.equals` returned True incorrectly in some cases when two DataFrames had the same columns in different orders (:issue:`28839`) - Bug in :meth:`DataFrame.replace` that caused non-numeric replacer's dtype not respected (:issue:`26632`) - Bug in :func:`melt` where supplying mixed strings and numeric values for ``id_vars`` or ``value_vars`` would incorrectly raise a ``ValueError`` (:issue:`29718`) +- Dtypes are now preserved when transposing a ``DataFrame`` where each column is the same extension dtype (:issue:`30091`) - Bug in :func:`merge_asof` merging on a tz-aware ``left_index`` and ``right_on`` a tz-aware column (:issue:`29864`) -- +- Improved error message and docstring in :func:`cut` and :func:`qcut` when `labels=True` (:issue:`13318`) Sparse ^^^^^^ - Bug in :class:`SparseDataFrame` arithmetic operations incorrectly casting inputs to float (:issue:`28107`) -- +- Bug in ``DataFrame.sparse`` returning a ``Series`` when there was a column named ``sparse`` rather than the accessor (:issue:`30758`) - ExtensionArray @@ -878,7 +1051,7 @@ ExtensionArray - Bug in :class:`arrays.PandasArray` when setting a scalar string (:issue:`28118`, :issue:`28150`). - Bug where nullable integers could not be compared to strings (:issue:`28930`) -- Bug where :class:`DataFrame` constructor raised ValueError with list-like data and ``dtype`` specified (:issue:`30280`) +- Bug where :class:`DataFrame` constructor raised ``ValueError`` with list-like data and ``dtype`` specified (:issue:`30280`) Other @@ -889,16 +1062,23 @@ Other - Bug in :meth:`Series.diff` where a boolean series would incorrectly raise a ``TypeError`` (:issue:`17294`) - :meth:`Series.append` will no longer raise a ``TypeError`` when passed a tuple of ``Series`` (:issue:`28410`) - Fix corrupted error message when calling ``pandas.libs._json.encode()`` on a 0d array (:issue:`18878`) +- Backtick quoting in :meth:`DataFrame.query` and :meth:`DataFrame.eval` can now also be used to use invalid identifiers like names that start with a digit, are python keywords, or are using single character operators. (:issue:`27017`) +- Bug in ``pd.core.util.hashing.hash_pandas_object`` where arrays containing tuples were incorrectly treated as non-hashable (:issue:`28969`) - Bug in :meth:`DataFrame.append` that raised ``IndexError`` when appending with empty list (:issue:`28769`) - Fix :class:`AbstractHolidayCalendar` to return correct results for years after 2030 (now goes up to 2200) (:issue:`27790`) -- Fixed :class:`IntegerArray` returning ``inf`` rather than ``NaN`` for operations dividing by 0 (:issue:`27398`) -- Fixed ``pow`` operations for :class:`IntegerArray` when the other value is ``0`` or ``1`` (:issue:`29997`) +- Fixed :class:`~arrays.IntegerArray` returning ``inf`` rather than ``NaN`` for operations dividing by ``0`` (:issue:`27398`) +- Fixed ``pow`` operations for :class:`~arrays.IntegerArray` when the other value is ``0`` or ``1`` (:issue:`29997`) - Bug in :meth:`Series.count` raises if use_inf_as_na is enabled (:issue:`29478`) -- Bug in :meth:`DataFrame.astype` that raised ``KeyError`` when errors='ignore' is ``True`` (:issue:`30324`) +- Bug in :class:`Index` where a non-hashable name could be set without raising ``TypeError`` (:issue:`29069`) +- Bug in :class:`DataFrame` constructor when passing a 2D ``ndarray`` and an extension dtype (:issue:`12513`) +- Bug in :meth:`DataFrame.to_csv` when supplied a series with a ``dtype="string"`` and a ``na_rep``, the ``na_rep`` was being truncated to 2 characters. (:issue:`29975`) +- Bug where :meth:`DataFrame.itertuples` would incorrectly determine whether or not namedtuples could be used for dataframes of 255 columns (:issue:`28282`) +- Bug in :meth:`DataFrame.astype` where when errors='ignore' was true, a ``KeyError`` would be raised (:issue:`30324`) +.. --------------------------------------------------------------------------- -.. _whatsnew_1000.contributors: +.. _whatsnew_100.contributors: Contributors -~~~~~~~~~~~~ +~~~~~~~~~~~~ \ No newline at end of file From 79ff6e33a461e60dd5d379f52887ad1106153961 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Wed, 4 Mar 2020 18:07:11 -0500 Subject: [PATCH 05/11] Fix whats new entry --- doc/source/whatsnew/v1.0.0.rst | 329 +++++++++++++++++++++++++++------ 1 file changed, 269 insertions(+), 60 deletions(-) mode change 100644 => 100755 doc/source/whatsnew/v1.0.0.rst diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst old mode 100644 new mode 100755 index 6de324b7b2eef..6597b764581a4 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -1,11 +1,20 @@ .. _whatsnew_100: -What's new in 1.0.0 (??) ------------------------- +What's new in 1.0.0 (January 29, 2020) +-------------------------------------- These are the changes in pandas 1.0.0. See :ref:`release` for a full changelog including other versions of pandas. +.. note:: + + The pandas 1.0 release removed a lot of functionality that was deprecated + in previous releases (see :ref:`below ` + for an overview). It is recommended to first upgrade to pandas 0.25 and to + ensure your code is working without warnings, before upgrading to pandas + 1.0. + + New Deprecation Policy ~~~~~~~~~~~~~~~~~~~~~~ @@ -13,8 +22,8 @@ Starting with Pandas 1.0.0, pandas will adopt a variant of `SemVer`_ to version releases. Briefly, * Deprecations will be introduced in minor releases (e.g. 1.1.0, 1.2.0, 2.1.0, ...) -* Deprecations will be enforced in major releases (e.g. 1.0.0, 2.0,0, 3.0.0, ...) -* API-breaking changes will be made only in major releases +* Deprecations will be enforced in major releases (e.g. 1.0.0, 2.0.0, 3.0.0, ...) +* API-breaking changes will be made only in major releases (except for experimental features) See :ref:`policies.version` for more. @@ -28,6 +37,43 @@ See :ref:`policies.version` for more. Enhancements ~~~~~~~~~~~~ +.. _whatsnew_100.numba_rolling_apply: + +Using Numba in ``rolling.apply`` and ``expanding.apply`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We've added an ``engine`` keyword to :meth:`~core.window.rolling.Rolling.apply` and :meth:`~core.window.expanding.Expanding.apply` +that allows the user to execute the routine using `Numba `__ instead of Cython. +Using the Numba engine can yield significant performance gains if the apply function can operate on numpy arrays and +the data set is larger (1 million rows or greater). For more details, see +:ref:`rolling apply documentation ` (:issue:`28987`, :issue:`30936`) + +.. _whatsnew_100.custom_window: + +Defining custom windows for rolling operations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We've added a :func:`pandas.api.indexers.BaseIndexer` class that allows users to define how +window bounds are created during ``rolling`` operations. Users can define their own ``get_window_bounds`` +method on a :func:`pandas.api.indexers.BaseIndexer` subclass that will generate the start and end +indices used for each window during the rolling aggregation. For more details and example usage, see +the :ref:`custom window rolling documentation ` + +.. _whatsnew_100.to_markdown: + +Converting to Markdown +^^^^^^^^^^^^^^^^^^^^^^ + +We've added :meth:`~DataFrame.to_markdown` for creating a markdown table (:issue:`11052`) + +.. ipython:: python + + df = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=['a', 'a', 'b']) + print(df.to_markdown()) + +Experimental new features +~~~~~~~~~~~~~~~~~~~~~~~~~ + .. _whatsnew_100.NA: Experimental ``NA`` scalar to denote missing values @@ -135,7 +181,7 @@ type dedicated to boolean data that can hold missing values. The default ``bool`` data type based on a bool-dtype NumPy array, the column can only hold ``True`` or ``False``, and not missing values. This new :class:`~arrays.BooleanArray` can store missing values as well by keeping track of this in a separate mask. -(:issue:`29555`, :issue:`30095`) +(:issue:`29555`, :issue:`30095`, :issue:`31131`) .. ipython:: python @@ -148,55 +194,52 @@ You can use the alias ``"boolean"`` as well. s = pd.Series([True, False, None], dtype="boolean") s -.. _whatsnew_100.numba_rolling_apply: +.. _whatsnew_100.convert_dtypes: -Using Numba in ``rolling.apply`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``convert_dtypes`` method to ease use of supported extension dtypes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We've added an ``engine`` keyword to :meth:`~core.window.rolling.Rolling.apply` that allows the user to execute the -routine using `Numba `__ instead of Cython. Using the Numba engine -can yield significant performance gains if the apply function can operate on numpy arrays and -the data set is larger (1 million rows or greater). For more details, see -:ref:`rolling apply documentation ` (:issue:`28987`) +In order to encourage use of the extension dtypes ``StringDtype``, +``BooleanDtype``, ``Int64Dtype``, ``Int32Dtype``, etc., that support ``pd.NA``, the +methods :meth:`DataFrame.convert_dtypes` and :meth:`Series.convert_dtypes` +have been introduced. (:issue:`29752`) (:issue:`30929`) -.. _whatsnew_100.custom_window: - -Defining custom windows for rolling operations -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Example: -We've added a :func:`pandas.api.indexers.BaseIndexer` class that allows users to define how -window bounds are created during ``rolling`` operations. Users can define their own ``get_window_bounds`` -method on a :func:`pandas.api.indexers.BaseIndexer` subclass that will generate the start and end -indices used for each window during the rolling aggregation. For more details and example usage, see -the :ref:`custom window rolling documentation ` +.. ipython:: python -.. _whatsnew_100.to_markdown: + df = pd.DataFrame({'x': ['abc', None, 'def'], + 'y': [1, 2, np.nan], + 'z': [True, False, True]}) + df + df.dtypes -Converting to Markdown -^^^^^^^^^^^^^^^^^^^^^^ +.. ipython:: python -We've added :meth:`~DataFrame.to_markdown` for creating a markdown table (:issue:`11052`) + converted = df.convert_dtypes() + converted + converted.dtypes -.. ipython:: python +This is especially useful after reading in data using readers such as :func:`read_csv` +and :func:`read_excel`. +See :ref:`here ` for a description. - df = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=['a', 'a', 'b']) - print(df.to_markdown()) .. _whatsnew_100.enhancements.other: Other enhancements -^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~ - :meth:`DataFrame.to_string` added the ``max_colwidth`` parameter to control when wide columns are truncated (:issue:`9784`) - Added the ``na_value`` argument to :meth:`Series.to_numpy`, :meth:`Index.to_numpy` and :meth:`DataFrame.to_numpy` to control the value used for missing data (:issue:`30322`) - :meth:`MultiIndex.from_product` infers level names from inputs if not explicitly provided (:issue:`27292`) - :meth:`DataFrame.to_latex` now accepts ``caption`` and ``label`` arguments (:issue:`25436`) -- The :ref:`integer dtype ` with support for missing values and the - new :ref:`string dtype ` can now be converted to ``pyarrow`` (>= - 0.15.0), which means that it is supported in writing to the Parquet file - format when using the ``pyarrow`` engine. It is currently not yet supported - when converting back to pandas, so it will become an integer or float - (depending on the presence of missing data) or object dtype column. (:issue:`28368`) +- DataFrames with :ref:`nullable integer `, the :ref:`new string dtype ` + and period data type can now be converted to ``pyarrow`` (>=0.15.0), which means that it is + supported in writing to the Parquet file format when using the ``pyarrow`` engine (:issue:`28368`). + Full roundtrip to parquet (writing and reading back in with :meth:`~DataFrame.to_parquet` / :func:`read_parquet`) + is supported starting with pyarrow >= 0.16 (:issue:`20612`). +- :func:`to_parquet` now appropriately handles the ``schema`` argument for user defined schemas in the pyarrow engine. (:issue:`30270`) - :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`) - :meth:`read_stata` can read Stata 119 dta files. (:issue:`28250`) - Implemented :meth:`pandas.core.window.Window.var` and :meth:`pandas.core.window.Window.std` functions (:issue:`26597`) @@ -204,32 +247,20 @@ Other enhancements - Added ``encoding`` argument to :func:`DataFrame.to_html` for non-ascii text (:issue:`28663`) - :meth:`Styler.background_gradient` now accepts ``vmin`` and ``vmax`` arguments (:issue:`12145`) - :meth:`Styler.format` added the ``na_rep`` parameter to help format the missing values (:issue:`21527`, :issue:`28358`) -- Roundtripping DataFrames with nullable integer, string and period data types to parquet - (:meth:`~DataFrame.to_parquet` / :func:`read_parquet`) using the `'pyarrow'` engine - now preserve those data types with pyarrow >= 0.16.0 (:issue:`20612`, :issue:`28371`). +- :func:`read_excel` now can read binary Excel (``.xlsb``) files by passing ``engine='pyxlsb'``. For more details and example usage, see the :ref:`Binary Excel files documentation `. Closes :issue:`8540`. - The ``partition_cols`` argument in :meth:`DataFrame.to_parquet` now accepts a string (:issue:`27117`) - :func:`pandas.read_json` now parses ``NaN``, ``Infinity`` and ``-Infinity`` (:issue:`12213`) -- The ``pandas.np`` submodule is now deprecated. Import numpy directly instead (:issue:`30296`) -- :func:`to_parquet` now appropriately handles the ``schema`` argument for user defined schemas in the pyarrow engine. (:issue:`30270`) - DataFrame constructor preserve `ExtensionArray` dtype with `ExtensionArray` (:issue:`11363`) - :meth:`DataFrame.sort_values` and :meth:`Series.sort_values` have gained ``ignore_index`` keyword to be able to reset index after sorting (:issue:`30114`) - :meth:`DataFrame.sort_index` and :meth:`Series.sort_index` have gained ``ignore_index`` keyword to reset index (:issue:`30114`) - :meth:`DataFrame.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`30114`) -- Added new writer for exporting Stata dta files in version 118, ``StataWriter118``. This format supports exporting strings containing Unicode characters (:issue:`23573`) +- Added new writer for exporting Stata dta files in versions 118 and 119, ``StataWriterUTF8``. These files formats support exporting strings containing Unicode characters. Format 119 supports data sets with more than 32,767 variables (:issue:`23573`, :issue:`30959`) - :meth:`Series.map` now accepts ``collections.abc.Mapping`` subclasses as a mapper (:issue:`29733`) -- The ``pandas.datetime`` class is now deprecated. Import from ``datetime`` instead (:issue:`30296`) - Added an experimental :attr:`~DataFrame.attrs` for storing global metadata about a dataset (:issue:`29062`) - :meth:`Timestamp.fromisocalendar` is now compatible with python 3.8 and above (:issue:`28115`) - :meth:`DataFrame.to_pickle` and :func:`read_pickle` now accept URL (:issue:`30163`) -Build Changes -^^^^^^^^^^^^^ - -Pandas has added a `pyproject.toml `_ file and will no longer include -cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing -a built distribution (wheel) or via conda, this shouldn't have any effect on you. If you're building pandas from -source, you should no longer need to install Cython into your build environment before calling ``pip install pandas``. .. --------------------------------------------------------------------------- @@ -274,7 +305,7 @@ To update, use ``MultiIndex.set_names``, which returns a new ``MultiIndex``. New repr for :class:`~pandas.arrays.IntervalArray` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- :class:`pandas.arrays.IntervalArray` adopts a new ``__repr__`` in accordance with other array classes (:issue:`25022`) +:class:`pandas.arrays.IntervalArray` adopts a new ``__repr__`` in accordance with other array classes (:issue:`25022`) *pandas 0.25.x* @@ -292,10 +323,85 @@ New repr for :class:`~pandas.arrays.IntervalArray` pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)]) +``DataFrame.rename`` now only accepts one positional argument +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:meth:`DataFrame.rename` would previously accept positional arguments that would lead +to ambiguous or undefined behavior. From pandas 1.0, only the very first argument, which +maps labels to their new names along the default axis, is allowed to be passed by position +(:issue:`29136`). + +.. ipython:: python + :suppress: + + df = pd.DataFrame([[1]]) + +*pandas 0.25.x* + +.. code-block:: python + + >>> df = pd.DataFrame([[1]]) + >>> df.rename({0: 1}, {0: 2}) + FutureWarning: ...Use named arguments to resolve ambiguity... + 2 + 1 1 + +*pandas 1.0.0* + +.. code-block:: python + + >>> df.rename({0: 1}, {0: 2}) + Traceback (most recent call last): + ... + TypeError: rename() takes from 1 to 2 positional arguments but 3 were given + +Note that errors will now be raised when conflicting or potentially ambiguous arguments are provided. + +*pandas 0.25.x* + +.. code-block:: python + + >>> df.rename({0: 1}, index={0: 2}) + 0 + 1 1 + + >>> df.rename(mapper={0: 1}, index={0: 2}) + 0 + 2 1 + +*pandas 1.0.0* + +.. code-block:: python + + >>> df.rename({0: 1}, index={0: 2}) + Traceback (most recent call last): + ... + TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns' + + >>> df.rename(mapper={0: 1}, index={0: 2}) + Traceback (most recent call last): + ... + TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns' + +You can still change the axis along which the first positional argument is applied by +supplying the ``axis`` keyword argument. + +.. ipython:: python + + df.rename({0: 1}) + df.rename({0: 1}, axis=1) + +If you would like to update both the index and column labels, be sure to use the respective +keywords. + +.. ipython:: python + + df.rename(index={0: 1}, columns={0: 2}) + Extended verbose info output for :class:`~pandas.DataFrame` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- :meth:`DataFrame.info` now shows line numbers for the columns summary (:issue:`17304`) +:meth:`DataFrame.info` now shows line numbers for the columns summary (:issue:`17304`) *pandas 0.25.x* @@ -411,6 +517,43 @@ Use :meth:`arrays.IntegerArray.to_numpy` with an explicit ``na_value`` instead. a.to_numpy(dtype="float", na_value=np.nan) +**Reductions can return ``pd.NA``** + +When performing a reduction such as a sum with ``skipna=False``, the result +will now be ``pd.NA`` instead of ``np.nan`` in presence of missing values +(:issue:`30958`). + +*pandas 0.25.x* + +.. code-block:: python + + >>> pd.Series(a).sum(skipna=False) + nan + +*pandas 1.0.0* + +.. ipython:: python + + pd.Series(a).sum(skipna=False) + +**value_counts returns a nullable integer dtype** + +:meth:`Series.value_counts` with a nullable integer dtype now returns a nullable +integer dtype for the values. + +*pandas 0.25.x* + +.. code-block:: python + + >>> pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype + dtype('int64') + +*pandas 1.0.0* + +.. ipython:: python + + pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype + See :ref:`missing_data.NA` for more on the differences between :attr:`pandas.NA` and :attr:`numpy.nan`. @@ -480,6 +623,54 @@ consistent with the behaviour of :class:`DataFrame` and :class:`Index`. DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning. Series([], dtype: float64) +Result dtype inference changes for resample operations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The rules for the result dtype in :meth:`DataFrame.resample` aggregations have changed for extension types (:issue:`31359`). +Previously, pandas would attempt to convert the result back to the original dtype, falling back to the usual +inference rules if that was not possible. Now, pandas will only return a result of the original dtype if the +scalar values in the result are instances of the extension dtype's scalar type. + +.. ipython:: python + + df = pd.DataFrame({"A": ['a', 'b']}, dtype='category', + index=pd.date_range('2000', periods=2)) + df + + +*pandas 0.25.x* + +.. code-block:: python + + >>> df.resample("2D").agg(lambda x: 'a').A.dtype + CategoricalDtype(categories=['a', 'b'], ordered=False) + +*pandas 1.0.0* + +.. ipython:: python + + df.resample("2D").agg(lambda x: 'a').A.dtype + +This fixes an inconsistency between ``resample`` and ``groupby``. +This also fixes a potential bug, where the **values** of the result might change +depending on how the results are cast back to the original dtype. + +*pandas 0.25.x* + +.. code-block:: python + + >>> df.resample("2D").agg(lambda x: 'c') + + A + 0 NaN + +*pandas 1.0.0* + +.. ipython:: python + + df.resample("2D").agg(lambda x: 'c') + + .. _whatsnew_100.api_breaking.python: Increased minimum version for Python @@ -555,19 +746,26 @@ Optional libraries below the lowest tested version may still work, but are not c See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more. +Build Changes +^^^^^^^^^^^^^ + +Pandas has added a `pyproject.toml `_ file and will no longer include +cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing +a built distribution (wheel) or via conda, this shouldn't have any effect on you. If you're building pandas from +source, you should no longer need to install Cython into your build environment before calling ``pip install pandas``. + .. _whatsnew_100.api.other: Other API changes ^^^^^^^^^^^^^^^^^ -- Bumped the minimum supported version of ``s3fs`` from 0.0.8 to 0.3.0 (:issue:`28616`) - :class:`core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`) - :meth:`pandas.api.types.infer_dtype` will now return "integer-na" for integer and ``np.nan`` mix (:issue:`27283`) - :meth:`MultiIndex.from_arrays` will no longer infer names from arrays if ``names=None`` is explicitly provided (:issue:`27292`) - In order to improve tab-completion, Pandas does not include most deprecated attributes when introspecting a pandas object using ``dir`` (e.g. ``dir(df)``). To see which attributes are excluded, see an object's ``_deprecations`` attribute, for example ``pd.DataFrame._deprecations`` (:issue:`28805`). -- The returned dtype of ::func:`pd.unique` now matches the input dtype. (:issue:`27874`) +- The returned dtype of :func:`unique` now matches the input dtype. (:issue:`27874`) - Changed the default configuration value for ``options.matplotlib.register_converters`` from ``True`` to ``"auto"`` (:issue:`18720`). Now, pandas custom formatters will only be applied to plots created by pandas, through :meth:`~DataFrame.plot`. Previously, pandas' formatters would be applied to all plots created *after* a :meth:`~DataFrame.plot`. @@ -576,7 +774,7 @@ Other API changes Supplying anything else than ``how`` to ``**kwargs`` raised a ``TypeError`` previously (:issue:`29388`) - When testing pandas, the new minimum required version of pytest is 5.0.1 (:issue:`29664`) - :meth:`Series.str.__iter__` was deprecated and will be removed in future releases (:issue:`28277`). - +- Added ```` to the list of default NA values for :meth:`read_csv` (:issue:`30821`) .. _whatsnew_100.api.documentation: @@ -613,7 +811,11 @@ Deprecations - The deprecated internal attributes ``_start``, ``_stop`` and ``_step`` of :class:`RangeIndex` now raise a ``FutureWarning`` instead of a ``DeprecationWarning`` (:issue:`26581`) - The ``pandas.util.testing`` module has been deprecated. Use the public API in ``pandas.testing`` documented at :ref:`api.general.testing` (:issue:`16232`). - ``pandas.SparseArray`` has been deprecated. Use ``pandas.arrays.SparseArray`` (:class:`arrays.SparseArray`) instead. (:issue:`30642`) -- The parameter ``is_copy`` of :meth:`DataFrame.take` has been deprecated and will be removed in a future version. (:issue:`27357`) +- The parameter ``is_copy`` of :meth:`Series.take` and :meth:`DataFrame.take` has been deprecated and will be removed in a future version. (:issue:`27357`) +- Support for multi-dimensional indexing (e.g. ``index[:, None]``) on a :class:`Index` is deprecated and will be removed in a future version, convert to a numpy array before indexing instead (:issue:`30588`) +- The ``pandas.np`` submodule is now deprecated. Import numpy directly instead (:issue:`30296`) +- The ``pandas.datetime`` class is now deprecated. Import from ``datetime`` instead (:issue:`30610`) +- :class:`~DataFrame.diff` will raise a ``TypeError`` rather than implicitly losing the dtype of extension types in the future. Convert to the correct dtype before calling ``diff`` instead (:issue:`31025`) **Selecting Columns from a Grouped DataFrame** @@ -905,6 +1107,8 @@ Numeric - Bug in :meth:`DataFrame.round` where a :class:`DataFrame` with a :class:`CategoricalIndex` of :class:`IntervalIndex` columns would incorrectly raise a ``TypeError`` (:issue:`30063`) - Bug in :meth:`Series.pct_change` and :meth:`DataFrame.pct_change` when there are duplicated indices (:issue:`30463`) - Bug in :class:`DataFrame` cumulative operations (e.g. cumsum, cummax) incorrect casting to object-dtype (:issue:`19296`) +- Bug in :class:`~DataFrame.diff` losing the dtype for extension types (:issue:`30889`) +- Bug in :class:`DataFrame.diff` raising an ``IndexError`` when one of the columns was a nullable integer dtype (:issue:`30967`) Conversion ^^^^^^^^^^ @@ -940,6 +1144,7 @@ Indexing - Bug when indexing with ``.loc`` where the index was a :class:`CategoricalIndex` with non-string categories didn't work (:issue:`17569`, :issue:`30225`) - :meth:`Index.get_indexer_non_unique` could fail with ``TypeError`` in some cases, such as when searching for ints in a string index (:issue:`28257`) - Bug in :meth:`Float64Index.get_loc` incorrectly raising ``TypeError`` instead of ``KeyError`` (:issue:`29189`) +- Bug in :meth:`DataFrame.loc` with incorrect dtype when setting Categorical value in 1-row DataFrame (:issue:`25495`) - :meth:`MultiIndex.get_loc` can't find missing values when input includes missing values (:issue:`19132`) - Bug in :meth:`Series.__setitem__` incorrectly assigning values with boolean indexer when the length of new data matches the number of ``True`` values and new data is not a ``Series`` or an ``np.array`` (:issue:`30567`) - Bug in indexing with a :class:`PeriodIndex` incorrectly accepting integers representing years, use e.g. ``ser.loc["2007"]`` instead of ``ser.loc[2007]`` (:issue:`30763`) @@ -1019,6 +1224,7 @@ Groupby/resample/rolling - Bug in :meth:`DataFrame.groupby` when using nunique on axis=1 (:issue:`30253`) - Bug in :meth:`GroupBy.quantile` with multiple list-like q value and integer column names (:issue:`30289`) - Bug in :meth:`GroupBy.pct_change` and :meth:`core.groupby.SeriesGroupBy.pct_change` causes ``TypeError`` when ``fill_method`` is ``None`` (:issue:`30463`) +- Bug in :meth:`Rolling.count` and :meth:`Expanding.count` argument where ``min_periods`` was ignored (:issue:`26996`) Reshaping ^^^^^^^^^ @@ -1039,12 +1245,13 @@ Reshaping - Dtypes are now preserved when transposing a ``DataFrame`` where each column is the same extension dtype (:issue:`30091`) - Bug in :func:`merge_asof` merging on a tz-aware ``left_index`` and ``right_on`` a tz-aware column (:issue:`29864`) - Improved error message and docstring in :func:`cut` and :func:`qcut` when `labels=True` (:issue:`13318`) +- Bug in missing `fill_na` parameter to :meth:`DataFrame.unstack` with list of levels (:issue:`30740`) Sparse ^^^^^^ - Bug in :class:`SparseDataFrame` arithmetic operations incorrectly casting inputs to float (:issue:`28107`) - Bug in ``DataFrame.sparse`` returning a ``Series`` when there was a column named ``sparse`` rather than the accessor (:issue:`30758`) -- +- Fixed :meth:`operator.xor` with a boolean-dtype ``SparseArray``. Now returns a sparse result, rather than object dtype (:issue:`31025`) ExtensionArray ^^^^^^^^^^^^^^ @@ -1053,7 +1260,6 @@ ExtensionArray - Bug where nullable integers could not be compared to strings (:issue:`28930`) - Bug where :class:`DataFrame` constructor raised ``ValueError`` with list-like data and ``dtype`` specified (:issue:`30280`) - Other ^^^^^ - Trying to set the ``display.precision``, ``display.max_rows`` or ``display.max_columns`` using :meth:`set_option` to anything but a ``None`` or a positive int will raise a ``ValueError`` (:issue:`23348`) @@ -1074,11 +1280,14 @@ Other - Bug in :class:`DataFrame` constructor when passing a 2D ``ndarray`` and an extension dtype (:issue:`12513`) - Bug in :meth:`DataFrame.to_csv` when supplied a series with a ``dtype="string"`` and a ``na_rep``, the ``na_rep`` was being truncated to 2 characters. (:issue:`29975`) - Bug where :meth:`DataFrame.itertuples` would incorrectly determine whether or not namedtuples could be used for dataframes of 255 columns (:issue:`28282`) -- Bug in :meth:`DataFrame.astype` where when errors='ignore' was true, a ``KeyError`` would be raised (:issue:`30324`) +- Handle nested NumPy ``object`` arrays in :func:`testing.assert_series_equal` for ExtensionArray implementations (:issue:`30841`) +- Bug in :class:`Index` constructor incorrectly allowing 2-dimensional input arrays (:issue:`13601`, :issue:`27125`) .. --------------------------------------------------------------------------- .. _whatsnew_100.contributors: Contributors -~~~~~~~~~~~~ \ No newline at end of file +~~~~~~~~~~~~ + +.. contributors:: v0.25.3..v1.0.0 From 0cd4929af205382bcd47a6e8aae807a77921af99 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Wed, 4 Mar 2020 20:12:02 -0500 Subject: [PATCH 06/11] Fix whats new entry --- doc/source/whatsnew/v1.0.1.rst | 79 ++++++++ doc/source/whatsnew/v1.0.2.rst | 94 +++++++++ doc/source/whatsnew/v1.1.0.rst | 359 +++++++++++++++++++++++++++++++++ 3 files changed, 532 insertions(+) create mode 100644 doc/source/whatsnew/v1.0.1.rst create mode 100644 doc/source/whatsnew/v1.0.2.rst create mode 100644 doc/source/whatsnew/v1.1.0.rst diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst new file mode 100644 index 0000000000000..ef3bb8161d13f --- /dev/null +++ b/doc/source/whatsnew/v1.0.1.rst @@ -0,0 +1,79 @@ +.. _whatsnew_101: + +What's new in 1.0.1 (February 5, 2020) +-------------------------------------- + +These are the changes in pandas 1.0.1. See :ref:`release` for a full changelog +including other versions of pandas. + +{{ header }} + +.. --------------------------------------------------------------------------- + +.. _whatsnew_101.regressions: + +Fixed regressions +~~~~~~~~~~~~~~~~~ + +- Fixed regression in :class:`DataFrame` setting values with a slice (e.g. ``df[-4:] = 1``) indexing by label instead of position (:issue:`31469`) +- Fixed regression when indexing a ``Series`` or ``DataFrame`` indexed by ``DatetimeIndex`` with a slice containg a :class:`datetime.date` (:issue:`31501`) +- Fixed regression in ``DataFrame.__setitem__`` raising an ``AttributeError`` with a :class:`MultiIndex` and a non-monotonic indexer (:issue:`31449`) +- Fixed regression in :class:`Series` multiplication when multiplying a numeric :class:`Series` with >10000 elements with a timedelta-like scalar (:issue:`31457`) +- Fixed regression in ``.groupby().agg()`` raising an ``AssertionError`` for some reductions like ``min`` on object-dtype columns (:issue:`31522`) +- Fixed regression in ``.groupby()`` aggregations with categorical dtype using Cythonized reduction functions (e.g. ``first``) (:issue:`31450`) +- Fixed regression in :meth:`GroupBy.apply` if called with a function which returned a non-pandas non-scalar object (e.g. a list or numpy array) (:issue:`31441`) +- Fixed regression in :meth:`DataFrame.groupby` whereby taking the minimum or maximum of a column with period dtype would raise a ``TypeError``. (:issue:`31471`) +- Fixed regression in :meth:`DataFrame.groupby` with an empty DataFrame grouping by a level of a MultiIndex (:issue:`31670`). +- Fixed regression in :meth:`DataFrame.apply` with object dtype and non-reducing function (:issue:`31505`) +- Fixed regression in :meth:`to_datetime` when parsing non-nanosecond resolution datetimes (:issue:`31491`) +- Fixed regression in :meth:`~DataFrame.to_csv` where specifying an ``na_rep`` might truncate the values written (:issue:`31447`) +- Fixed regression in :class:`Categorical` construction with ``numpy.str_`` categories (:issue:`31499`) +- Fixed regression in :meth:`DataFrame.loc` and :meth:`DataFrame.iloc` when selecting a row containing a single ``datetime64`` or ``timedelta64`` column (:issue:`31649`) +- Fixed regression where setting :attr:`pd.options.display.max_colwidth` was not accepting negative integer. In addition, this behavior has been deprecated in favor of using ``None`` (:issue:`31532`) +- Fixed regression in objTOJSON.c fix return-type warning (:issue:`31463`) +- Fixed regression in :meth:`qcut` when passed a nullable integer. (:issue:`31389`) +- Fixed regression in assigning to a :class:`Series` using a nullable integer dtype (:issue:`31446`) +- Fixed performance regression when indexing a ``DataFrame`` or ``Series`` with a :class:`MultiIndex` for the index using a list of labels (:issue:`31648`) +- Fixed regression in :meth:`read_csv` used in file like object ``RawIOBase`` is not recognize ``encoding`` option (:issue:`31575`) + +.. --------------------------------------------------------------------------- + +.. _whatsnew_101.deprecations: + +Deprecations +~~~~~~~~~~~~ + +- Support for negative integer for :attr:`pd.options.display.max_colwidth` is deprecated in favor of using ``None`` (:issue:`31532`) + +.. --------------------------------------------------------------------------- + +.. _whatsnew_101.bug_fixes: + +Bug fixes +~~~~~~~~~ + +**Datetimelike** + +- Fixed bug in :meth:`to_datetime` raising when ``cache=True`` and out-of-bound values are present (:issue:`31491`) + +**Numeric** + +- Bug in dtypes being lost in ``DataFrame.__invert__`` (``~`` operator) with mixed dtypes (:issue:`31183`) + and for extension-array backed ``Series`` and ``DataFrame`` (:issue:`23087`) + +**Plotting** + +- Plotting tz-aware timeseries no longer gives UserWarning (:issue:`31205`) + +**Interval** + +- Bug in :meth:`Series.shift` with ``interval`` dtype raising a ``TypeError`` when shifting an interval array of integers or datetimes (:issue:`34195`) + +.. --------------------------------------------------------------------------- + +.. _whatsnew_101.contributors: + +Contributors +~~~~~~~~~~~~ + +.. contributors:: v1.0.0..v1.0.1|HEAD diff --git a/doc/source/whatsnew/v1.0.2.rst b/doc/source/whatsnew/v1.0.2.rst new file mode 100644 index 0000000000000..808e6ae709ce9 --- /dev/null +++ b/doc/source/whatsnew/v1.0.2.rst @@ -0,0 +1,94 @@ +.. _whatsnew_102: + +What's new in 1.0.2 (February ??, 2020) +--------------------------------------- + +These are the changes in pandas 1.0.2. See :ref:`release` for a full changelog +including other versions of pandas. + +{{ header }} + +.. --------------------------------------------------------------------------- + +.. _whatsnew_102.regressions: + +Fixed regressions +~~~~~~~~~~~~~~~~~ + +- Fixed regression in :meth:`DataFrame.to_excel` when ``columns`` kwarg is passed (:issue:`31677`) +- Fixed regression in :meth:`Series.align` when ``other`` is a DataFrame and ``method`` is not None (:issue:`31785`) +- Fixed regression in :meth:`pandas.core.groupby.RollingGroupby.apply` where the ``raw`` parameter was ignored (:issue:`31754`) +- Fixed regression in :meth:`rolling(..).corr() ` when using a time offset (:issue:`31789`) +- Fixed regression in :meth:`DataFrameGroupBy.nunique` which was modifying the original values if ``NaN`` values were present (:issue:`31950`) +- Fixed regression where :func:`read_pickle` raised a ``UnicodeDecodeError`` when reading a py27 pickle with :class:`MultiIndex` column (:issue:`31988`). +- Fixed regression in :class:`DataFrame` arithmetic operations with mis-matched columns (:issue:`31623`) +- Fixed regression in :meth:`GroupBy.agg` calling a user-provided function an extra time on an empty input (:issue:`31760`) +- Joining on :class:`DatetimeIndex` or :class:`TimedeltaIndex` will preserve ``freq`` in simple cases (:issue:`32166`) +- + +.. --------------------------------------------------------------------------- + +Indexing with Nullable Boolean Arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Previously indexing with a nullable Boolean array containing ``NA`` would raise a ``ValueError``, however this is now permitted with ``NA`` being treated as ``False``. (:issue:`31503`) + +.. ipython:: python + + s = pd.Series([1, 2, 3, 4]) + mask = pd.array([True, True, False, None], dtype="boolean") + s + mask + +*pandas 1.0.0-1.0.1* + +.. code-block:: python + + >>> s[mask] + Traceback (most recent call last): + ... + ValueError: cannot mask with array containing NA / NaN values + +*pandas 1.0.2* + +.. ipython:: python + + s[mask] + +.. _whatsnew_102.bug_fixes: + +Bug fixes +~~~~~~~~~ + +**Datetimelike** + +- Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with a tz-aware index (:issue:`26683`) +- Bug where :func:`to_datetime` would raise when passed ``pd.NA`` (:issue:`32213`) + +**Categorical** + +- Fixed bug where :meth:`Categorical.from_codes` improperly raised a ``ValueError`` when passed nullable integer codes. (:issue:`31779`) +- Fixed bug where :meth:`Categorical` constructor would raise a ``TypeError`` when given a numpy array containing ``pd.NA``. (:issue:`31927`) +- Bug in :class:`Categorical` that would ignore or crash when calling :meth:`Series.replace` with a list-like ``to_replace`` (:issue:`31720`) + +**I/O** + +- Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`) +- Fixed pickling of ``pandas.NA``. Previously a new object was returned, which broke computations relying on ``NA`` being a singleton (:issue:`31847`) +- Fixed bug in parquet roundtrip with nullable unsigned integer dtypes (:issue:`31896`). + +**Experimental dtypes** + +- Fix bug in :meth:`DataFrame.convert_dtypes` for columns that were already using the ``"string"`` dtype (:issue:`31731`). +- Fixed bug in setting values using a slice indexer with string dtype (:issue:`31772`) +- Fixed bug where :meth:`GroupBy.first` and :meth:`GroupBy.last` would raise a ``TypeError`` when groups contained ``pd.NA`` in a column of object dtype (:issue:`32123`) +- Fix bug in :meth:`Series.convert_dtypes` for series with mix of integers and strings (:issue:`32117`) + +.. --------------------------------------------------------------------------- + +.. _whatsnew_102.contributors: + +Contributors +~~~~~~~~~~~~ + +.. contributors:: v1.0.1..v1.0.2|HEAD diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst new file mode 100644 index 0000000000000..5310419bfc100 --- /dev/null +++ b/doc/source/whatsnew/v1.1.0.rst @@ -0,0 +1,359 @@ +.. _whatsnew_110: + +What's new in 1.1.0 (??) +------------------------ + +These are the changes in pandas 1.1.0. See :ref:`release` for a full changelog +including other versions of pandas. + +{{ header }} + +.. --------------------------------------------------------------------------- + +Enhancements +~~~~~~~~~~~~ + +.. _whatsnew_110.period_index_partial_string_slicing: + +Nonmonotonic PeriodIndex Partial String Slicing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:class:`PeriodIndex` now supports partial string slicing for non-monotonic indexes, mirroring :class:`DatetimeIndex` behavior (:issue:`31096`) + +For example: + +.. ipython:: python + + dti = pd.date_range("2014-01-01", periods=30, freq="30D") + pi = dti.to_period("D") + ser_monotonic = pd.Series(np.arange(30), index=pi) + shuffler = list(range(0, 30, 2)) + list(range(1, 31, 2)) + ser = ser_monotonic[shuffler] + ser + +.. ipython:: python + + ser["2014"] + ser.loc["May 2015"] + +.. _whatsnew_110.timestamp_fold_support: + +Fold argument support in Timestamp constructor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:class:`Timestamp:` now supports the keyword-only fold argument according to `PEP 495 `_ similar to parent ``datetime.datetime`` class. It supports both accepting fold as an initialization argument and inferring fold from other constructor arguments (:issue:`25057`, :issue:`31338`). Support is limited to ``dateutil`` timezones as ``pytz`` doesn't support fold. + +For example: + +.. ipython:: python + + ts = pd.Timestamp("2019-10-27 01:30:00+00:00") + ts.fold + +.. ipython:: python + + ts = pd.Timestamp(year=2019, month=10, day=27, hour=1, minute=30, + tz="dateutil/Europe/London", fold=1) + ts + +For more on working with fold, see :ref:`Fold subsection ` in the user guide. + +.. _whatsnew_110.enhancements.other: + +Other enhancements +^^^^^^^^^^^^^^^^^^ + +- :class:`Styler` may now render CSS more efficiently where multiple cells have the same styling (:issue:`30876`) +- When writing directly to a sqlite connection :func:`to_sql` now supports the ``multi`` method (:issue:`29921`) +- `OptionError` is now exposed in `pandas.errors` (:issue:`27553`) +- :func:`timedelta_range` will now infer a frequency when passed ``start``, ``stop``, and ``periods`` (:issue:`32377`) +- + +.. --------------------------------------------------------------------------- + +.. _whatsnew_110.api.other: + +Other API changes +^^^^^^^^^^^^^^^^^ + +- :meth:`Series.describe` will now show distribution percentiles for ``datetime`` dtypes, statistics ``first`` and ``last`` + will now be ``min`` and ``max`` to match with numeric dtypes in :meth:`DataFrame.describe` (:issue:`30164`) +- Added :meth:`DataFrame.value_counts` (:issue:`5377`) +- :meth:`Groupby.groups` now returns an abbreviated representation when called on large dataframes (:issue:`1135`) +- ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`) +- + +Backwards incompatible API changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- :meth:`DataFrame.swaplevels` now raises a ``TypeError`` if the axis is not a :class:`MultiIndex`. + Previously a ``AttributeError`` was raised (:issue:`31126`) +- :meth:`DataFrameGroupby.mean` and :meth:`SeriesGroupby.mean` (and similarly for :meth:`~DataFrameGroupby.median`, :meth:`~DataFrameGroupby.std`` and :meth:`~DataFrameGroupby.var``) + now raise a ``TypeError`` if a not-accepted keyword argument is passed into it. + Previously a ``UnsupportedFunctionCall`` was raised (``AssertionError`` if ``min_count`` passed into :meth:`~DataFrameGroupby.median``) (:issue:`31485`) +- :meth:`DataFrame.at` and :meth:`Series.at` will raise a ``TypeError`` instead of a ``ValueError`` if an incompatible key is passed, and ``KeyError`` if a missing key is passed, matching the behavior of ``.loc[]`` (:issue:`31722`) + +.. _whatsnew_110.api_breaking.indexing_raises_key_errors: + +Failed Label-Based Lookups Always Raise KeyError +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Label lookups ``series[key]``, ``series.loc[key]`` and ``frame.loc[key]`` +used to raises either ``KeyError`` or ``TypeError`` depending on the type of +key and type of :class:`Index`. These now consistently raise ``KeyError`` (:issue:`31867`) + +.. ipython:: python + + ser1 = pd.Series(range(3), index=[0, 1, 2]) + ser2 = pd.Series(range(3), index=pd.date_range("2020-02-01", periods=3)) + +*Previous behavior*: + +.. code-block:: ipython + + In [3]: ser1[1.5] + ... + TypeError: cannot do label indexing on Int64Index with these indexers [1.5] of type float + + In [4] ser1["foo"] + ... + KeyError: 'foo' + + In [5]: ser1.loc[1.5] + ... + TypeError: cannot do label indexing on Int64Index with these indexers [1.5] of type float + + In [6]: ser1.loc["foo"] + ... + KeyError: 'foo' + + In [7]: ser2.loc[1] + ... + TypeError: cannot do label indexing on DatetimeIndex with these indexers [1] of type int + + In [8]: ser2.loc[pd.Timestamp(0)] + ... + KeyError: Timestamp('1970-01-01 00:00:00') + +*New behavior*: + +.. code-block:: ipython + + In [3]: ser1[1.5] + ... + KeyError: 1.5 + + In [4] ser1["foo"] + ... + KeyError: 'foo' + + In [5]: ser1.loc[1.5] + ... + KeyError: 1.5 + + In [6]: ser1.loc["foo"] + ... + KeyError: 'foo' + + In [7]: ser2.loc[1] + ... + KeyError: 1 + + In [8]: ser2.loc[pd.Timestamp(0)] + ... + KeyError: Timestamp('1970-01-01 00:00:00') + +.. --------------------------------------------------------------------------- + +.. _whatsnew_110.deprecations: + +Deprecations +~~~~~~~~~~~~ +- Lookups on a :class:`Series` with a single-item list containing a slice (e.g. ``ser[[slice(0, 4)]]``) are deprecated, will raise in a future version. Either convert the list to tuple, or pass the slice directly instead (:issue:`31333`) +- :meth:`DataFrame.mean` and :meth:`DataFrame.median` with ``numeric_only=None`` will include datetime64 and datetime64tz columns in a future version (:issue:`29941`) +- +- + +.. --------------------------------------------------------------------------- + + +.. _whatsnew_110.performance: + +Performance improvements +~~~~~~~~~~~~~~~~~~~~~~~~ + +- Performance improvement in :class:`Timedelta` constructor (:issue:`30543`) +- Performance improvement in :class:`Timestamp` constructor (:issue:`30543`) +- +- + +.. --------------------------------------------------------------------------- + +.. _whatsnew_110.bug_fixes: + +Bug fixes +~~~~~~~~~ + + +Categorical +^^^^^^^^^^^ + +- Bug where :func:`merge` was unable to join on non-unique categorical indices (:issue:`28189`) +- Bug when passing categorical data to :class:`Index` constructor along with ``dtype=object`` incorrectly returning a :class:`CategoricalIndex` instead of object-dtype :class:`Index` (:issue:`32167`) +- Bug where :class:`Categorical` comparison operator ``__ne__`` would incorrectly evaluate to ``False`` when either element was missing (:issue:`32276`) +- + +Datetimelike +^^^^^^^^^^^^ + +- Bug in :class:`Timestamp` where constructing :class:`Timestamp` from ambiguous epoch time and calling constructor again changed :meth:`Timestamp.value` property (:issue:`24329`) +- :meth:`DatetimeArray.searchsorted`, :meth:`TimedeltaArray.searchsorted`, :meth:`PeriodArray.searchsorted` not recognizing non-pandas scalars and incorrectly raising ``ValueError`` instead of ``TypeError`` (:issue:`30950`) +- Bug in :class:`Timestamp` where constructing :class:`Timestamp` with dateutil timezone less than 128 nanoseconds before daylight saving time switch from winter to summer would result in nonexistent time (:issue:`31043`) +- Bug in :meth:`Period.to_timestamp`, :meth:`Period.start_time` with microsecond frequency returning a timestamp one nanosecond earlier than the correct time (:issue:`31475`) +- :class:`Timestamp` raising confusing error message when year, month or day is missing (:issue:`31200`) + +Timedelta +^^^^^^^^^ + +- Bug in constructing a :class:`Timedelta` with a high precision integer that would round the :class:`Timedelta` components (:issue:`31354`) +- Bug in dividing ``np.nan`` or ``None`` by :class:`Timedelta`` incorrectly returning ``NaT`` (:issue:`31869`) +- + +Timezones +^^^^^^^^^ + +- +- + + +Numeric +^^^^^^^ +- Bug in :meth:`DataFrame.floordiv` with ``axis=0`` not treating division-by-zero like :meth:`Series.floordiv` (:issue:`31271`) +- +- + +Conversion +^^^^^^^^^^ +- Bug in :class:`Series` construction from NumPy array with big-endian ``datetime64`` dtype (:issue:`29684`) +- Bug in :class:`Timedelta` construction with large nanoseconds keyword value (:issue:`34202`) +- + +Strings +^^^^^^^ + +- +- + + +Interval +^^^^^^^^ + +- +- + +Indexing +^^^^^^^^ +- Bug in slicing on a :class:`DatetimeIndex` with a partial-timestamp dropping high-resolution indices near the end of a year, quarter, or month (:issue:`31064`) +- Bug in :meth:`PeriodIndex.get_loc` treating higher-resolution strings differently from :meth:`PeriodIndex.get_value` (:issue:`31172`) +- Bug in :meth:`Series.at` and :meth:`DataFrame.at` not matching ``.loc`` behavior when looking up an integer in a :class:`Float64Index` (:issue:`31329`) +- Bug in :meth:`PeriodIndex.is_monotonic` incorrectly returning ``True`` when containing leading ``NaT`` entries (:issue:`31437`) +- Bug in :meth:`DatetimeIndex.get_loc` raising ``KeyError`` with converted-integer key instead of the user-passed key (:issue:`31425`) +- Bug in :meth:`Series.xs` incorrectly returning ``Timestamp`` instead of ``datetime64`` in some object-dtype cases (:issue:`31630`) +- Bug in :meth:`DataFrame.iat` incorrectly returning ``Timestamp`` instead of ``datetime`` in some object-dtype cases (:issue:`32809`) +- Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`) +- + +Missing +^^^^^^^ + +- +- + +MultiIndex +^^^^^^^^^^ +- Bug in :meth:`Dataframe.loc` when used with a :class:`MultiIndex`. The returned values were not in the same order as the given inputs (:issue:`22797`) + +.. ipython:: python + + df = pd.DataFrame(np.arange(4), + index=[["a", "a", "b", "b"], [1, 2, 1, 2]]) + # Rows are now ordered as the requested keys + df.loc[(['b', 'a'], [2, 1]), :] + +- Bug in :meth:`MultiIndex.intersection` was not guaranteed to preserve order when ``sort=False``. (:issue:`31325`) + +.. ipython:: python + + left = pd.MultiIndex.from_arrays([["b", "a"], [2, 1]]) + right = pd.MultiIndex.from_arrays([["a", "b", "c"], [1, 2, 3]]) + # Common elements are now guaranteed to be ordered by the left side + left.intersection(right, sort=False) + +- + +I/O +^^^ +- Bug in :meth:`read_json` where integer overflow was occuring when json contains big number strings. (:issue:`30320`) +- `read_csv` will now raise a ``ValueError`` when the arguments `header` and `prefix` both are not `None`. (:issue:`27394`) +- Bug in :meth:`DataFrame.to_json` was raising ``NotFoundError`` when ``path_or_buf`` was an S3 URI (:issue:`28375`) +- Bug in :meth:`DataFrame.to_parquet` overwriting pyarrow's default for + ``coerce_timestamps``; following pyarrow's default allows writing nanosecond + timestamps with ``version="2.0"`` (:issue:`31652`). +- Bug in :meth:`read_csv` was raising `TypeError` when `sep=None` was used in combination with `comment` keyword (:issue:`31396`) +- Bug in :class:`HDFStore` that caused it to set to ``int64`` the dtype of a ``datetime64`` column when reading a DataFrame in Python 3 from fixed format written in Python 2 (:issue:`31750`) + + +Plotting +^^^^^^^^ + +- :func:`.plot` for line/bar now accepts color by dictonary (:issue:`8193`). +- +- Bug in :meth:`DataFrame.boxplot` and :meth:`DataFrame.plot.boxplot` lost color attributes of ``medianprops``, ``whiskerprops``, ``capprops`` and ``medianprops`` (:issue:`30346`) + + +Groupby/resample/rolling +^^^^^^^^^^^^^^^^^^^^^^^^ + +- Bug in :meth:`GroupBy.apply` raises ``ValueError`` when the ``by`` axis is not sorted and has duplicates and the applied ``func`` does not mutate passed in objects (:issue:`30667`) +- Bug in :meth:`DataFrameGroupby.transform` produces incorrect result with transformation functions (:issue:`30918`) + +Reshaping +^^^^^^^^^ + +- Bug effecting all numeric and boolean reduction methods not returning subclassed data type. (:issue:`25596`) +- Bug in :meth:`DataFrame.pivot_table` when only MultiIndexed columns is set (:issue:`17038`) +- Bug in :meth:`DataFrame.unstack` and :meth:`Series.unstack` can take tuple names in MultiIndexed data (:issue:`19966`) +- Bug in :meth:`DataFrame.pivot_table` when ``margin`` is ``True`` and only ``column`` is defined (:issue:`31016`) +- Fix incorrect error message in :meth:`DataFrame.pivot` when ``columns`` is set to ``None``. (:issue:`30924`) +- Bug in :func:`crosstab` when inputs are two Series and have tuple names, the output will keep dummy MultiIndex as columns. (:issue:`18321`) +- :meth:`DataFrame.pivot` can now take lists for ``index`` and ``columns`` arguments (:issue:`21425`) +- Bug in :func:`concat` where the resulting indices are not copied when ``copy=True`` (:issue:`29879`) +- :meth:`Series.append` will now raise a ``TypeError`` when passed a DataFrame or a sequence containing Dataframe (:issue:`31413`) +- :meth:`DataFrame.replace` and :meth:`Series.replace` will raise a ``TypeError`` if ``to_replace`` is not an expected type. Previously the ``replace`` would fail silently (:issue:`18634`) + + +Sparse +^^^^^^ + +- +- + +ExtensionArray +^^^^^^^^^^^^^^ + +- +- + + +Other +^^^^^ +- Appending a dictionary to a :class:`DataFrame` without passing ``ignore_index=True`` will raise ``TypeError: Can only append a dict if ignore_index=True`` + instead of ``TypeError: Can only append a Series if ignore_index=True or if the Series has a name`` (:issue:`30871`) +- Set operations on an object-dtype :class:`Index` now always return object-dtype results (:issue:`31401`) +- Bug in :meth:`AbstractHolidayCalendar.holidays` when no rules were defined (:issue:`31415`) + +.. --------------------------------------------------------------------------- + +.. _whatsnew_110.contributors: + +Contributors +~~~~~~~~~~~~ From 94eb49eb2ff2d10cb67122d85cd93c4a8e6036f1 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Wed, 4 Mar 2020 21:44:56 -0500 Subject: [PATCH 07/11] Trying again to fix whats new entry --- doc/source/whatsnew/v1.0.1.rst | 79 -------- doc/source/whatsnew/v1.0.2.rst | 94 --------- doc/source/whatsnew/v1.1.0.rst | 359 --------------------------------- 3 files changed, 532 deletions(-) delete mode 100644 doc/source/whatsnew/v1.0.1.rst delete mode 100644 doc/source/whatsnew/v1.0.2.rst delete mode 100644 doc/source/whatsnew/v1.1.0.rst diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst deleted file mode 100644 index ef3bb8161d13f..0000000000000 --- a/doc/source/whatsnew/v1.0.1.rst +++ /dev/null @@ -1,79 +0,0 @@ -.. _whatsnew_101: - -What's new in 1.0.1 (February 5, 2020) --------------------------------------- - -These are the changes in pandas 1.0.1. See :ref:`release` for a full changelog -including other versions of pandas. - -{{ header }} - -.. --------------------------------------------------------------------------- - -.. _whatsnew_101.regressions: - -Fixed regressions -~~~~~~~~~~~~~~~~~ - -- Fixed regression in :class:`DataFrame` setting values with a slice (e.g. ``df[-4:] = 1``) indexing by label instead of position (:issue:`31469`) -- Fixed regression when indexing a ``Series`` or ``DataFrame`` indexed by ``DatetimeIndex`` with a slice containg a :class:`datetime.date` (:issue:`31501`) -- Fixed regression in ``DataFrame.__setitem__`` raising an ``AttributeError`` with a :class:`MultiIndex` and a non-monotonic indexer (:issue:`31449`) -- Fixed regression in :class:`Series` multiplication when multiplying a numeric :class:`Series` with >10000 elements with a timedelta-like scalar (:issue:`31457`) -- Fixed regression in ``.groupby().agg()`` raising an ``AssertionError`` for some reductions like ``min`` on object-dtype columns (:issue:`31522`) -- Fixed regression in ``.groupby()`` aggregations with categorical dtype using Cythonized reduction functions (e.g. ``first``) (:issue:`31450`) -- Fixed regression in :meth:`GroupBy.apply` if called with a function which returned a non-pandas non-scalar object (e.g. a list or numpy array) (:issue:`31441`) -- Fixed regression in :meth:`DataFrame.groupby` whereby taking the minimum or maximum of a column with period dtype would raise a ``TypeError``. (:issue:`31471`) -- Fixed regression in :meth:`DataFrame.groupby` with an empty DataFrame grouping by a level of a MultiIndex (:issue:`31670`). -- Fixed regression in :meth:`DataFrame.apply` with object dtype and non-reducing function (:issue:`31505`) -- Fixed regression in :meth:`to_datetime` when parsing non-nanosecond resolution datetimes (:issue:`31491`) -- Fixed regression in :meth:`~DataFrame.to_csv` where specifying an ``na_rep`` might truncate the values written (:issue:`31447`) -- Fixed regression in :class:`Categorical` construction with ``numpy.str_`` categories (:issue:`31499`) -- Fixed regression in :meth:`DataFrame.loc` and :meth:`DataFrame.iloc` when selecting a row containing a single ``datetime64`` or ``timedelta64`` column (:issue:`31649`) -- Fixed regression where setting :attr:`pd.options.display.max_colwidth` was not accepting negative integer. In addition, this behavior has been deprecated in favor of using ``None`` (:issue:`31532`) -- Fixed regression in objTOJSON.c fix return-type warning (:issue:`31463`) -- Fixed regression in :meth:`qcut` when passed a nullable integer. (:issue:`31389`) -- Fixed regression in assigning to a :class:`Series` using a nullable integer dtype (:issue:`31446`) -- Fixed performance regression when indexing a ``DataFrame`` or ``Series`` with a :class:`MultiIndex` for the index using a list of labels (:issue:`31648`) -- Fixed regression in :meth:`read_csv` used in file like object ``RawIOBase`` is not recognize ``encoding`` option (:issue:`31575`) - -.. --------------------------------------------------------------------------- - -.. _whatsnew_101.deprecations: - -Deprecations -~~~~~~~~~~~~ - -- Support for negative integer for :attr:`pd.options.display.max_colwidth` is deprecated in favor of using ``None`` (:issue:`31532`) - -.. --------------------------------------------------------------------------- - -.. _whatsnew_101.bug_fixes: - -Bug fixes -~~~~~~~~~ - -**Datetimelike** - -- Fixed bug in :meth:`to_datetime` raising when ``cache=True`` and out-of-bound values are present (:issue:`31491`) - -**Numeric** - -- Bug in dtypes being lost in ``DataFrame.__invert__`` (``~`` operator) with mixed dtypes (:issue:`31183`) - and for extension-array backed ``Series`` and ``DataFrame`` (:issue:`23087`) - -**Plotting** - -- Plotting tz-aware timeseries no longer gives UserWarning (:issue:`31205`) - -**Interval** - -- Bug in :meth:`Series.shift` with ``interval`` dtype raising a ``TypeError`` when shifting an interval array of integers or datetimes (:issue:`34195`) - -.. --------------------------------------------------------------------------- - -.. _whatsnew_101.contributors: - -Contributors -~~~~~~~~~~~~ - -.. contributors:: v1.0.0..v1.0.1|HEAD diff --git a/doc/source/whatsnew/v1.0.2.rst b/doc/source/whatsnew/v1.0.2.rst deleted file mode 100644 index 808e6ae709ce9..0000000000000 --- a/doc/source/whatsnew/v1.0.2.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. _whatsnew_102: - -What's new in 1.0.2 (February ??, 2020) ---------------------------------------- - -These are the changes in pandas 1.0.2. See :ref:`release` for a full changelog -including other versions of pandas. - -{{ header }} - -.. --------------------------------------------------------------------------- - -.. _whatsnew_102.regressions: - -Fixed regressions -~~~~~~~~~~~~~~~~~ - -- Fixed regression in :meth:`DataFrame.to_excel` when ``columns`` kwarg is passed (:issue:`31677`) -- Fixed regression in :meth:`Series.align` when ``other`` is a DataFrame and ``method`` is not None (:issue:`31785`) -- Fixed regression in :meth:`pandas.core.groupby.RollingGroupby.apply` where the ``raw`` parameter was ignored (:issue:`31754`) -- Fixed regression in :meth:`rolling(..).corr() ` when using a time offset (:issue:`31789`) -- Fixed regression in :meth:`DataFrameGroupBy.nunique` which was modifying the original values if ``NaN`` values were present (:issue:`31950`) -- Fixed regression where :func:`read_pickle` raised a ``UnicodeDecodeError`` when reading a py27 pickle with :class:`MultiIndex` column (:issue:`31988`). -- Fixed regression in :class:`DataFrame` arithmetic operations with mis-matched columns (:issue:`31623`) -- Fixed regression in :meth:`GroupBy.agg` calling a user-provided function an extra time on an empty input (:issue:`31760`) -- Joining on :class:`DatetimeIndex` or :class:`TimedeltaIndex` will preserve ``freq`` in simple cases (:issue:`32166`) -- - -.. --------------------------------------------------------------------------- - -Indexing with Nullable Boolean Arrays -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Previously indexing with a nullable Boolean array containing ``NA`` would raise a ``ValueError``, however this is now permitted with ``NA`` being treated as ``False``. (:issue:`31503`) - -.. ipython:: python - - s = pd.Series([1, 2, 3, 4]) - mask = pd.array([True, True, False, None], dtype="boolean") - s - mask - -*pandas 1.0.0-1.0.1* - -.. code-block:: python - - >>> s[mask] - Traceback (most recent call last): - ... - ValueError: cannot mask with array containing NA / NaN values - -*pandas 1.0.2* - -.. ipython:: python - - s[mask] - -.. _whatsnew_102.bug_fixes: - -Bug fixes -~~~~~~~~~ - -**Datetimelike** - -- Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with a tz-aware index (:issue:`26683`) -- Bug where :func:`to_datetime` would raise when passed ``pd.NA`` (:issue:`32213`) - -**Categorical** - -- Fixed bug where :meth:`Categorical.from_codes` improperly raised a ``ValueError`` when passed nullable integer codes. (:issue:`31779`) -- Fixed bug where :meth:`Categorical` constructor would raise a ``TypeError`` when given a numpy array containing ``pd.NA``. (:issue:`31927`) -- Bug in :class:`Categorical` that would ignore or crash when calling :meth:`Series.replace` with a list-like ``to_replace`` (:issue:`31720`) - -**I/O** - -- Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`) -- Fixed pickling of ``pandas.NA``. Previously a new object was returned, which broke computations relying on ``NA`` being a singleton (:issue:`31847`) -- Fixed bug in parquet roundtrip with nullable unsigned integer dtypes (:issue:`31896`). - -**Experimental dtypes** - -- Fix bug in :meth:`DataFrame.convert_dtypes` for columns that were already using the ``"string"`` dtype (:issue:`31731`). -- Fixed bug in setting values using a slice indexer with string dtype (:issue:`31772`) -- Fixed bug where :meth:`GroupBy.first` and :meth:`GroupBy.last` would raise a ``TypeError`` when groups contained ``pd.NA`` in a column of object dtype (:issue:`32123`) -- Fix bug in :meth:`Series.convert_dtypes` for series with mix of integers and strings (:issue:`32117`) - -.. --------------------------------------------------------------------------- - -.. _whatsnew_102.contributors: - -Contributors -~~~~~~~~~~~~ - -.. contributors:: v1.0.1..v1.0.2|HEAD diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst deleted file mode 100644 index 5310419bfc100..0000000000000 --- a/doc/source/whatsnew/v1.1.0.rst +++ /dev/null @@ -1,359 +0,0 @@ -.. _whatsnew_110: - -What's new in 1.1.0 (??) ------------------------- - -These are the changes in pandas 1.1.0. See :ref:`release` for a full changelog -including other versions of pandas. - -{{ header }} - -.. --------------------------------------------------------------------------- - -Enhancements -~~~~~~~~~~~~ - -.. _whatsnew_110.period_index_partial_string_slicing: - -Nonmonotonic PeriodIndex Partial String Slicing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`PeriodIndex` now supports partial string slicing for non-monotonic indexes, mirroring :class:`DatetimeIndex` behavior (:issue:`31096`) - -For example: - -.. ipython:: python - - dti = pd.date_range("2014-01-01", periods=30, freq="30D") - pi = dti.to_period("D") - ser_monotonic = pd.Series(np.arange(30), index=pi) - shuffler = list(range(0, 30, 2)) + list(range(1, 31, 2)) - ser = ser_monotonic[shuffler] - ser - -.. ipython:: python - - ser["2014"] - ser.loc["May 2015"] - -.. _whatsnew_110.timestamp_fold_support: - -Fold argument support in Timestamp constructor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`Timestamp:` now supports the keyword-only fold argument according to `PEP 495 `_ similar to parent ``datetime.datetime`` class. It supports both accepting fold as an initialization argument and inferring fold from other constructor arguments (:issue:`25057`, :issue:`31338`). Support is limited to ``dateutil`` timezones as ``pytz`` doesn't support fold. - -For example: - -.. ipython:: python - - ts = pd.Timestamp("2019-10-27 01:30:00+00:00") - ts.fold - -.. ipython:: python - - ts = pd.Timestamp(year=2019, month=10, day=27, hour=1, minute=30, - tz="dateutil/Europe/London", fold=1) - ts - -For more on working with fold, see :ref:`Fold subsection ` in the user guide. - -.. _whatsnew_110.enhancements.other: - -Other enhancements -^^^^^^^^^^^^^^^^^^ - -- :class:`Styler` may now render CSS more efficiently where multiple cells have the same styling (:issue:`30876`) -- When writing directly to a sqlite connection :func:`to_sql` now supports the ``multi`` method (:issue:`29921`) -- `OptionError` is now exposed in `pandas.errors` (:issue:`27553`) -- :func:`timedelta_range` will now infer a frequency when passed ``start``, ``stop``, and ``periods`` (:issue:`32377`) -- - -.. --------------------------------------------------------------------------- - -.. _whatsnew_110.api.other: - -Other API changes -^^^^^^^^^^^^^^^^^ - -- :meth:`Series.describe` will now show distribution percentiles for ``datetime`` dtypes, statistics ``first`` and ``last`` - will now be ``min`` and ``max`` to match with numeric dtypes in :meth:`DataFrame.describe` (:issue:`30164`) -- Added :meth:`DataFrame.value_counts` (:issue:`5377`) -- :meth:`Groupby.groups` now returns an abbreviated representation when called on large dataframes (:issue:`1135`) -- ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`) -- - -Backwards incompatible API changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- :meth:`DataFrame.swaplevels` now raises a ``TypeError`` if the axis is not a :class:`MultiIndex`. - Previously a ``AttributeError`` was raised (:issue:`31126`) -- :meth:`DataFrameGroupby.mean` and :meth:`SeriesGroupby.mean` (and similarly for :meth:`~DataFrameGroupby.median`, :meth:`~DataFrameGroupby.std`` and :meth:`~DataFrameGroupby.var``) - now raise a ``TypeError`` if a not-accepted keyword argument is passed into it. - Previously a ``UnsupportedFunctionCall`` was raised (``AssertionError`` if ``min_count`` passed into :meth:`~DataFrameGroupby.median``) (:issue:`31485`) -- :meth:`DataFrame.at` and :meth:`Series.at` will raise a ``TypeError`` instead of a ``ValueError`` if an incompatible key is passed, and ``KeyError`` if a missing key is passed, matching the behavior of ``.loc[]`` (:issue:`31722`) - -.. _whatsnew_110.api_breaking.indexing_raises_key_errors: - -Failed Label-Based Lookups Always Raise KeyError -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Label lookups ``series[key]``, ``series.loc[key]`` and ``frame.loc[key]`` -used to raises either ``KeyError`` or ``TypeError`` depending on the type of -key and type of :class:`Index`. These now consistently raise ``KeyError`` (:issue:`31867`) - -.. ipython:: python - - ser1 = pd.Series(range(3), index=[0, 1, 2]) - ser2 = pd.Series(range(3), index=pd.date_range("2020-02-01", periods=3)) - -*Previous behavior*: - -.. code-block:: ipython - - In [3]: ser1[1.5] - ... - TypeError: cannot do label indexing on Int64Index with these indexers [1.5] of type float - - In [4] ser1["foo"] - ... - KeyError: 'foo' - - In [5]: ser1.loc[1.5] - ... - TypeError: cannot do label indexing on Int64Index with these indexers [1.5] of type float - - In [6]: ser1.loc["foo"] - ... - KeyError: 'foo' - - In [7]: ser2.loc[1] - ... - TypeError: cannot do label indexing on DatetimeIndex with these indexers [1] of type int - - In [8]: ser2.loc[pd.Timestamp(0)] - ... - KeyError: Timestamp('1970-01-01 00:00:00') - -*New behavior*: - -.. code-block:: ipython - - In [3]: ser1[1.5] - ... - KeyError: 1.5 - - In [4] ser1["foo"] - ... - KeyError: 'foo' - - In [5]: ser1.loc[1.5] - ... - KeyError: 1.5 - - In [6]: ser1.loc["foo"] - ... - KeyError: 'foo' - - In [7]: ser2.loc[1] - ... - KeyError: 1 - - In [8]: ser2.loc[pd.Timestamp(0)] - ... - KeyError: Timestamp('1970-01-01 00:00:00') - -.. --------------------------------------------------------------------------- - -.. _whatsnew_110.deprecations: - -Deprecations -~~~~~~~~~~~~ -- Lookups on a :class:`Series` with a single-item list containing a slice (e.g. ``ser[[slice(0, 4)]]``) are deprecated, will raise in a future version. Either convert the list to tuple, or pass the slice directly instead (:issue:`31333`) -- :meth:`DataFrame.mean` and :meth:`DataFrame.median` with ``numeric_only=None`` will include datetime64 and datetime64tz columns in a future version (:issue:`29941`) -- -- - -.. --------------------------------------------------------------------------- - - -.. _whatsnew_110.performance: - -Performance improvements -~~~~~~~~~~~~~~~~~~~~~~~~ - -- Performance improvement in :class:`Timedelta` constructor (:issue:`30543`) -- Performance improvement in :class:`Timestamp` constructor (:issue:`30543`) -- -- - -.. --------------------------------------------------------------------------- - -.. _whatsnew_110.bug_fixes: - -Bug fixes -~~~~~~~~~ - - -Categorical -^^^^^^^^^^^ - -- Bug where :func:`merge` was unable to join on non-unique categorical indices (:issue:`28189`) -- Bug when passing categorical data to :class:`Index` constructor along with ``dtype=object`` incorrectly returning a :class:`CategoricalIndex` instead of object-dtype :class:`Index` (:issue:`32167`) -- Bug where :class:`Categorical` comparison operator ``__ne__`` would incorrectly evaluate to ``False`` when either element was missing (:issue:`32276`) -- - -Datetimelike -^^^^^^^^^^^^ - -- Bug in :class:`Timestamp` where constructing :class:`Timestamp` from ambiguous epoch time and calling constructor again changed :meth:`Timestamp.value` property (:issue:`24329`) -- :meth:`DatetimeArray.searchsorted`, :meth:`TimedeltaArray.searchsorted`, :meth:`PeriodArray.searchsorted` not recognizing non-pandas scalars and incorrectly raising ``ValueError`` instead of ``TypeError`` (:issue:`30950`) -- Bug in :class:`Timestamp` where constructing :class:`Timestamp` with dateutil timezone less than 128 nanoseconds before daylight saving time switch from winter to summer would result in nonexistent time (:issue:`31043`) -- Bug in :meth:`Period.to_timestamp`, :meth:`Period.start_time` with microsecond frequency returning a timestamp one nanosecond earlier than the correct time (:issue:`31475`) -- :class:`Timestamp` raising confusing error message when year, month or day is missing (:issue:`31200`) - -Timedelta -^^^^^^^^^ - -- Bug in constructing a :class:`Timedelta` with a high precision integer that would round the :class:`Timedelta` components (:issue:`31354`) -- Bug in dividing ``np.nan`` or ``None`` by :class:`Timedelta`` incorrectly returning ``NaT`` (:issue:`31869`) -- - -Timezones -^^^^^^^^^ - -- -- - - -Numeric -^^^^^^^ -- Bug in :meth:`DataFrame.floordiv` with ``axis=0`` not treating division-by-zero like :meth:`Series.floordiv` (:issue:`31271`) -- -- - -Conversion -^^^^^^^^^^ -- Bug in :class:`Series` construction from NumPy array with big-endian ``datetime64`` dtype (:issue:`29684`) -- Bug in :class:`Timedelta` construction with large nanoseconds keyword value (:issue:`34202`) -- - -Strings -^^^^^^^ - -- -- - - -Interval -^^^^^^^^ - -- -- - -Indexing -^^^^^^^^ -- Bug in slicing on a :class:`DatetimeIndex` with a partial-timestamp dropping high-resolution indices near the end of a year, quarter, or month (:issue:`31064`) -- Bug in :meth:`PeriodIndex.get_loc` treating higher-resolution strings differently from :meth:`PeriodIndex.get_value` (:issue:`31172`) -- Bug in :meth:`Series.at` and :meth:`DataFrame.at` not matching ``.loc`` behavior when looking up an integer in a :class:`Float64Index` (:issue:`31329`) -- Bug in :meth:`PeriodIndex.is_monotonic` incorrectly returning ``True`` when containing leading ``NaT`` entries (:issue:`31437`) -- Bug in :meth:`DatetimeIndex.get_loc` raising ``KeyError`` with converted-integer key instead of the user-passed key (:issue:`31425`) -- Bug in :meth:`Series.xs` incorrectly returning ``Timestamp`` instead of ``datetime64`` in some object-dtype cases (:issue:`31630`) -- Bug in :meth:`DataFrame.iat` incorrectly returning ``Timestamp`` instead of ``datetime`` in some object-dtype cases (:issue:`32809`) -- Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`) -- - -Missing -^^^^^^^ - -- -- - -MultiIndex -^^^^^^^^^^ -- Bug in :meth:`Dataframe.loc` when used with a :class:`MultiIndex`. The returned values were not in the same order as the given inputs (:issue:`22797`) - -.. ipython:: python - - df = pd.DataFrame(np.arange(4), - index=[["a", "a", "b", "b"], [1, 2, 1, 2]]) - # Rows are now ordered as the requested keys - df.loc[(['b', 'a'], [2, 1]), :] - -- Bug in :meth:`MultiIndex.intersection` was not guaranteed to preserve order when ``sort=False``. (:issue:`31325`) - -.. ipython:: python - - left = pd.MultiIndex.from_arrays([["b", "a"], [2, 1]]) - right = pd.MultiIndex.from_arrays([["a", "b", "c"], [1, 2, 3]]) - # Common elements are now guaranteed to be ordered by the left side - left.intersection(right, sort=False) - -- - -I/O -^^^ -- Bug in :meth:`read_json` where integer overflow was occuring when json contains big number strings. (:issue:`30320`) -- `read_csv` will now raise a ``ValueError`` when the arguments `header` and `prefix` both are not `None`. (:issue:`27394`) -- Bug in :meth:`DataFrame.to_json` was raising ``NotFoundError`` when ``path_or_buf`` was an S3 URI (:issue:`28375`) -- Bug in :meth:`DataFrame.to_parquet` overwriting pyarrow's default for - ``coerce_timestamps``; following pyarrow's default allows writing nanosecond - timestamps with ``version="2.0"`` (:issue:`31652`). -- Bug in :meth:`read_csv` was raising `TypeError` when `sep=None` was used in combination with `comment` keyword (:issue:`31396`) -- Bug in :class:`HDFStore` that caused it to set to ``int64`` the dtype of a ``datetime64`` column when reading a DataFrame in Python 3 from fixed format written in Python 2 (:issue:`31750`) - - -Plotting -^^^^^^^^ - -- :func:`.plot` for line/bar now accepts color by dictonary (:issue:`8193`). -- -- Bug in :meth:`DataFrame.boxplot` and :meth:`DataFrame.plot.boxplot` lost color attributes of ``medianprops``, ``whiskerprops``, ``capprops`` and ``medianprops`` (:issue:`30346`) - - -Groupby/resample/rolling -^^^^^^^^^^^^^^^^^^^^^^^^ - -- Bug in :meth:`GroupBy.apply` raises ``ValueError`` when the ``by`` axis is not sorted and has duplicates and the applied ``func`` does not mutate passed in objects (:issue:`30667`) -- Bug in :meth:`DataFrameGroupby.transform` produces incorrect result with transformation functions (:issue:`30918`) - -Reshaping -^^^^^^^^^ - -- Bug effecting all numeric and boolean reduction methods not returning subclassed data type. (:issue:`25596`) -- Bug in :meth:`DataFrame.pivot_table` when only MultiIndexed columns is set (:issue:`17038`) -- Bug in :meth:`DataFrame.unstack` and :meth:`Series.unstack` can take tuple names in MultiIndexed data (:issue:`19966`) -- Bug in :meth:`DataFrame.pivot_table` when ``margin`` is ``True`` and only ``column`` is defined (:issue:`31016`) -- Fix incorrect error message in :meth:`DataFrame.pivot` when ``columns`` is set to ``None``. (:issue:`30924`) -- Bug in :func:`crosstab` when inputs are two Series and have tuple names, the output will keep dummy MultiIndex as columns. (:issue:`18321`) -- :meth:`DataFrame.pivot` can now take lists for ``index`` and ``columns`` arguments (:issue:`21425`) -- Bug in :func:`concat` where the resulting indices are not copied when ``copy=True`` (:issue:`29879`) -- :meth:`Series.append` will now raise a ``TypeError`` when passed a DataFrame or a sequence containing Dataframe (:issue:`31413`) -- :meth:`DataFrame.replace` and :meth:`Series.replace` will raise a ``TypeError`` if ``to_replace`` is not an expected type. Previously the ``replace`` would fail silently (:issue:`18634`) - - -Sparse -^^^^^^ - -- -- - -ExtensionArray -^^^^^^^^^^^^^^ - -- -- - - -Other -^^^^^ -- Appending a dictionary to a :class:`DataFrame` without passing ``ignore_index=True`` will raise ``TypeError: Can only append a dict if ignore_index=True`` - instead of ``TypeError: Can only append a Series if ignore_index=True or if the Series has a name`` (:issue:`30871`) -- Set operations on an object-dtype :class:`Index` now always return object-dtype results (:issue:`31401`) -- Bug in :meth:`AbstractHolidayCalendar.holidays` when no rules were defined (:issue:`31415`) - -.. --------------------------------------------------------------------------- - -.. _whatsnew_110.contributors: - -Contributors -~~~~~~~~~~~~ From 061e02ae8582b2f079eace479d752749b1ca52d4 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Wed, 4 Mar 2020 21:59:05 -0500 Subject: [PATCH 08/11] Updated file permissions --- doc/source/whatsnew/v1.0.0.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 doc/source/whatsnew/v1.0.0.rst diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst old mode 100755 new mode 100644 From 7a76a717bc7b38de1564a60c887c68bc560d3ea4 Mon Sep 17 00:00:00 2001 From: nbonnin Date: Wed, 4 Mar 2020 22:10:39 -0500 Subject: [PATCH 09/11] Trying yet again to fix the whatsnew --- doc/source/whatsnew/v1.0.0.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 doc/source/whatsnew/v1.0.0.rst diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst old mode 100644 new mode 100755 From 1272c8afca4e01515d6c5011d639fa783c4ec95c Mon Sep 17 00:00:00 2001 From: nbonnin Date: Wed, 4 Mar 2020 22:33:45 -0500 Subject: [PATCH 10/11] Tried downloading directly from github. --- doc/source/whatsnew/v1.0.0.rst | 3435 ++++++++++++++++++++++---------- 1 file changed, 2330 insertions(+), 1105 deletions(-) mode change 100755 => 100644 doc/source/whatsnew/v1.0.0.rst diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst old mode 100755 new mode 100644 index 6597b764581a4..77fca5ce0b9c7 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -1,1293 +1,2518 @@ -.. _whatsnew_100: -What's new in 1.0.0 (January 29, 2020) --------------------------------------- -These are the changes in pandas 1.0.0. See :ref:`release` for a full changelog -including other versions of pandas. -.. note:: - The pandas 1.0 release removed a lot of functionality that was deprecated - in previous releases (see :ref:`below ` - for an overview). It is recommended to first upgrade to pandas 0.25 and to - ensure your code is working without warnings, before upgrading to pandas - 1.0. -New Deprecation Policy -~~~~~~~~~~~~~~~~~~~~~~ + + + + + + + + + + + -Starting with Pandas 1.0.0, pandas will adopt a variant of `SemVer`_ to -version releases. Briefly, -* Deprecations will be introduced in minor releases (e.g. 1.1.0, 1.2.0, 2.1.0, ...) -* Deprecations will be enforced in major releases (e.g. 1.0.0, 2.0.0, 3.0.0, ...) -* API-breaking changes will be made only in major releases (except for experimental features) -See :ref:`policies.version` for more. + + + + + + + -.. _2019 Pandas User Survey: http://dev.pandas.io/pandas-blog/2019-pandas-user-survey.html -.. _SemVer: https://semver.org -{{ header }} + + + pandas/v1.0.0.rst at master · pandas-dev/pandas + + + + -.. --------------------------------------------------------------------------- + + -Enhancements -~~~~~~~~~~~~ + + + -.. _whatsnew_100.numba_rolling_apply: + -Using Numba in ``rolling.apply`` and ``expanding.apply`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We've added an ``engine`` keyword to :meth:`~core.window.rolling.Rolling.apply` and :meth:`~core.window.expanding.Expanding.apply` -that allows the user to execute the routine using `Numba `__ instead of Cython. -Using the Numba engine can yield significant performance gains if the apply function can operate on numpy arrays and -the data set is larger (1 million rows or greater). For more details, see -:ref:`rolling apply documentation ` (:issue:`28987`, :issue:`30936`) -.. _whatsnew_100.custom_window: + -Defining custom windows for rolling operations -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + -We've added a :func:`pandas.api.indexers.BaseIndexer` class that allows users to define how -window bounds are created during ``rolling`` operations. Users can define their own ``get_window_bounds`` -method on a :func:`pandas.api.indexers.BaseIndexer` subclass that will generate the start and end -indices used for each window during the rolling aggregation. For more details and example usage, see -the :ref:`custom window rolling documentation ` + -.. _whatsnew_100.to_markdown: + + + -Converting to Markdown -^^^^^^^^^^^^^^^^^^^^^^ + + -We've added :meth:`~DataFrame.to_markdown` for creating a markdown table (:issue:`11052`) -.. ipython:: python - df = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=['a', 'a', 'b']) - print(df.to_markdown()) + -Experimental new features -~~~~~~~~~~~~~~~~~~~~~~~~~ + -.. _whatsnew_100.NA: + -Experimental ``NA`` scalar to denote missing values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A new ``pd.NA`` value (singleton) is introduced to represent scalar missing -values. Up to now, pandas used several values to represent missing data: ``np.nan`` is used for this for float data, ``np.nan`` or -``None`` for object-dtype data and ``pd.NaT`` for datetime-like data. The -goal of ``pd.NA`` is to provide a "missing" indicator that can be used -consistently across data types. ``pd.NA`` is currently used by the nullable integer and boolean -data types and the new string data type (:issue:`28095`). -.. warning:: + - Experimental: the behaviour of ``pd.NA`` can still change without warning. + + -For example, creating a Series using the nullable integer dtype: + -.. ipython:: python + - s = pd.Series([1, 2, None], dtype="Int64") - s - s[2] + -Compared to ``np.nan``, ``pd.NA`` behaves differently in certain operations. -In addition to arithmetic operations, ``pd.NA`` also propagates as "missing" -or "unknown" in comparison operations: + + -.. ipython:: python + - np.nan > 1 - pd.NA > 1 + -For logical operations, ``pd.NA`` follows the rules of the -`three-valued logic `__ (or -*Kleene logic*). For example: + -.. ipython:: python - pd.NA | True + -For more, see :ref:`NA section ` in the user guide on missing -data. + -.. _whatsnew_100.string: + -Dedicated string data type -^^^^^^^^^^^^^^^^^^^^^^^^^^ + + -We've added :class:`StringDtype`, an extension type dedicated to string data. -Previously, strings were typically stored in object-dtype NumPy arrays. (:issue:`29975`) + -.. warning:: - ``StringDtype`` is currently considered experimental. The implementation - and parts of the API may change without warning. + -The ``'string'`` extension type solves several issues with object-dtype NumPy arrays: + -1. You can accidentally store a *mixture* of strings and non-strings in an - ``object`` dtype array. A ``StringArray`` can only store strings. -2. ``object`` dtype breaks dtype-specific operations like :meth:`DataFrame.select_dtypes`. - There isn't a clear way to select *just* text while excluding non-text, - but still object-dtype columns. -3. When reading code, the contents of an ``object`` dtype array is less clear - than ``string``. + + +
+ Skip to content + + + -.. ipython:: python + + - pd.Series(['abc', None, 'def'], dtype=pd.StringDtype()) -You can use the alias ``"string"`` as well. -.. ipython:: python + -You can use the alias ``"boolean"`` as well. + -.. ipython:: python +
- s = pd.Series([True, False, None], dtype="boolean") - s +
-.. _whatsnew_100.convert_dtypes: -``convert_dtypes`` method to ease use of supported extension dtypes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +
-In order to encourage use of the extension dtypes ``StringDtype``, -``BooleanDtype``, ``Int64Dtype``, ``Int32Dtype``, etc., that support ``pd.NA``, the -methods :meth:`DataFrame.convert_dtypes` and :meth:`Series.convert_dtypes` -have been introduced. (:issue:`29752`) (:issue:`30929`) +
-Example: -.. ipython:: python + - df = pd.DataFrame({'x': ['abc', None, 'def'], - 'y': [1, 2, np.nan], - 'z': [True, False, True]}) - df - df.dtypes + -.. ipython:: python - converted = df.convert_dtypes() - converted - converted.dtypes -This is especially useful after reading in data using readers such as :func:`read_csv` -and :func:`read_excel`. -See :ref:`here ` for a description. +
+
+
+ -.. _whatsnew_100.enhancements.other: + -Other enhancements -~~~~~~~~~~~~~~~~~~ -- :meth:`DataFrame.to_string` added the ``max_colwidth`` parameter to control when wide columns are truncated (:issue:`9784`) -- Added the ``na_value`` argument to :meth:`Series.to_numpy`, :meth:`Index.to_numpy` and :meth:`DataFrame.to_numpy` to control the value used for missing data (:issue:`30322`) -- :meth:`MultiIndex.from_product` infers level names from inputs if not explicitly provided (:issue:`27292`) -- :meth:`DataFrame.to_latex` now accepts ``caption`` and ``label`` arguments (:issue:`25436`) -- DataFrames with :ref:`nullable integer `, the :ref:`new string dtype ` - and period data type can now be converted to ``pyarrow`` (>=0.15.0), which means that it is - supported in writing to the Parquet file format when using the ``pyarrow`` engine (:issue:`28368`). - Full roundtrip to parquet (writing and reading back in with :meth:`~DataFrame.to_parquet` / :func:`read_parquet`) - is supported starting with pyarrow >= 0.16 (:issue:`20612`). -- :func:`to_parquet` now appropriately handles the ``schema`` argument for user defined schemas in the pyarrow engine. (:issue:`30270`) -- :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`) -- :meth:`read_stata` can read Stata 119 dta files. (:issue:`28250`) -- Implemented :meth:`pandas.core.window.Window.var` and :meth:`pandas.core.window.Window.std` functions (:issue:`26597`) -- Added ``encoding`` argument to :meth:`DataFrame.to_string` for non-ascii text (:issue:`28766`) -- Added ``encoding`` argument to :func:`DataFrame.to_html` for non-ascii text (:issue:`28663`) -- :meth:`Styler.background_gradient` now accepts ``vmin`` and ``vmax`` arguments (:issue:`12145`) -- :meth:`Styler.format` added the ``na_rep`` parameter to help format the missing values (:issue:`21527`, :issue:`28358`) -- :func:`read_excel` now can read binary Excel (``.xlsb``) files by passing ``engine='pyxlsb'``. For more details and example usage, see the :ref:`Binary Excel files documentation `. Closes :issue:`8540`. -- The ``partition_cols`` argument in :meth:`DataFrame.to_parquet` now accepts a string (:issue:`27117`) -- :func:`pandas.read_json` now parses ``NaN``, ``Infinity`` and ``-Infinity`` (:issue:`12213`) -- DataFrame constructor preserve `ExtensionArray` dtype with `ExtensionArray` (:issue:`11363`) -- :meth:`DataFrame.sort_values` and :meth:`Series.sort_values` have gained ``ignore_index`` keyword to be able to reset index after sorting (:issue:`30114`) -- :meth:`DataFrame.sort_index` and :meth:`Series.sort_index` have gained ``ignore_index`` keyword to reset index (:issue:`30114`) -- :meth:`DataFrame.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`30114`) -- Added new writer for exporting Stata dta files in versions 118 and 119, ``StataWriterUTF8``. These files formats support exporting strings containing Unicode characters. Format 119 supports data sets with more than 32,767 variables (:issue:`23573`, :issue:`30959`) -- :meth:`Series.map` now accepts ``collections.abc.Mapping`` subclasses as a mapper (:issue:`29733`) -- Added an experimental :attr:`~DataFrame.attrs` for storing global metadata about a dataset (:issue:`29062`) -- :meth:`Timestamp.fromisocalendar` is now compatible with python 3.8 and above (:issue:`28115`) -- :meth:`DataFrame.to_pickle` and :func:`read_pickle` now accept URL (:issue:`30163`) + -.. --------------------------------------------------------------------------- -.. _whatsnew_100.api_breaking: -Backwards incompatible API changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _whatsnew_100.api_breaking.MultiIndex._names: -Avoid using names from ``MultiIndex.levels`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As part of a larger refactor to :class:`MultiIndex` the level names are now -stored separately from the levels (:issue:`27242`). We recommend using -:attr:`MultiIndex.names` to access the names, and :meth:`Index.set_names` -to update the names. -For backwards compatibility, you can still *access* the names via the levels. -.. ipython:: python - mi = pd.MultiIndex.from_product([[1, 2], ['a', 'b']], names=['x', 'y']) - mi.levels[0].name +
-However, it is no longer possible to *update* the names of the ``MultiIndex`` -via the level. +
-.. ipython:: python - :okexcept: +
+

+ + + / + + pandas + + +

- mi.levels[0].name = "new name" - mi.names -To update, use ``MultiIndex.set_names``, which returns a new ``MultiIndex``. +
+ +
    + +
  • + +
    • + + +
  • + +
    + + + Used by +
    +     + +
    + Loading... +

    + Loading dependents... +

    +
    +     +
    + + +
    • + +
  • + +
    + +
    + + + Watch + + +
    + Notifications +
    +
    + + + + + + + +
    +     +
    + +
    +
    • + +
  • +
    +
    + + +
    +
    + + +
    + +
    • + +
  • +
    + + Fork + +
    + +

    Fork pandas

    +
    +
    + + +

    If this dialog fails to load, you can visit the fork page directly.

    +     +
    +     +
    + + +
    • +
+ +
+ + + +
+ +
+ + +
+ + + + + + +
+
+ + + + + + + + + Permalink + + + + +
+ + +
+ + Branch: + master + + + + +
+ + + +
+ +
+ +
+ + Find file + + + Copy path + +
+ +

+ pandas/doc/source/whatsnew/v1.0.0.rst +

+ +
+ + Find file + + + Copy path + +
+
+ + + + + + +
+
+ + @jorisvandenbossche + jorisvandenbossche + + DOC: move whatnew entry for invert from 1.0.0 t 1.0.1 (#31494) + + + + db4da03 + Jan 31, 2020 + +
+ +
+
+ + 115 contributors + + +
+ +

+ Users who have contributed to this file +

+
+ + +
+ + + @jbrockmendel + + @TomAugspurger + + @jorisvandenbossche + + @charlesdong1991 + + @topper-123 + + @MarcoGorelli + + @jschendel + + @mroeschke + + @dsaxton + + @WillAyd + + @fujiaxiang + + @ryankarlos + + @proost + + @christopherzimmerman + + @Dr-Irv + + @immaxchen + + @gfyoung + + @bashtage + + @Unprocessable + + @simonjayhawkins + + @OliverHofkens + + @oguzhanogreden + + @nlepleux + + @SaturnFromTitan + + @makbigc + + @JustinZhengBC + + + + +
+
+ + + + + + +
+ +
+
+ + executable file + + 1293 lines (966 sloc) + + 71.6 KB +
+ +
+ +
+ Raw + Blame + History +
+ + +
+ + + + +
+ +
+
+ +
+
+
+ + + + + +
+

What's new in 1.0.0 (January 29, 2020)

+

These are the changes in pandas 1.0.0. See :ref:`release` for a full changelog +including other versions of pandas.

+
+

Note

+

The pandas 1.0 release removed a lot of functionality that was deprecated +in previous releases (see :ref:`below <whatsnew_100.prior_deprecations>` +for an overview). It is recommended to first upgrade to pandas 0.25 and to +ensure your code is working without warnings, before upgrading to pandas +1.0.

+
+ +

New Deprecation Policy

+

Starting with Pandas 1.0.0, pandas will adopt a variant of SemVer to +version releases. Briefly,

+
    +
  • Deprecations will be introduced in minor releases (e.g. 1.1.0, 1.2.0, 2.1.0, ...)
    • +
  • Deprecations will be enforced in major releases (e.g. 1.0.0, 2.0.0, 3.0.0, ...)
    • +
  • API-breaking changes will be made only in major releases (except for experimental features)
    • +
+

See :ref:`policies.version` for more.

+

{{ header }}

+ +

Enhancements

+ +

Using Numba in rolling.apply and expanding.apply

+

We've added an engine keyword to :meth:`~core.window.rolling.Rolling.apply` and :meth:`~core.window.expanding.Expanding.apply` +that allows the user to execute the routine using Numba instead of Cython. +Using the Numba engine can yield significant performance gains if the apply function can operate on numpy arrays and +the data set is larger (1 million rows or greater). For more details, see +:ref:`rolling apply documentation <stats.rolling_apply>` (:issue:`28987`, :issue:`30936`)

+ +

Defining custom windows for rolling operations

+

We've added a :func:`pandas.api.indexers.BaseIndexer` class that allows users to define how +window bounds are created during rolling operations. Users can define their own get_window_bounds +method on a :func:`pandas.api.indexers.BaseIndexer` subclass that will generate the start and end +indices used for each window during the rolling aggregation. For more details and example usage, see +the :ref:`custom window rolling documentation <stats.custom_rolling_window>`

+ +

Converting to Markdown

+

We've added :meth:`~DataFrame.to_markdown` for creating a markdown table (:issue:`11052`)

+
.. ipython:: python
+
+   df = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=['a', 'a', 'b'])
+   print(df.to_markdown())
 
-.. ipython:: python
+
+ +

Experimental new features

+ +

Experimental NA scalar to denote missing values

+

A new pd.NA value (singleton) is introduced to represent scalar missing +values. Up to now, pandas used several values to represent missing data: np.nan is used for this for float data, np.nan or +None for object-dtype data and pd.NaT for datetime-like data. The +goal of pd.NA is to provide a "missing" indicator that can be used +consistently across data types. pd.NA is currently used by the nullable integer and boolean +data types and the new string data type (:issue:`28095`).

+
+

Warning

+

Experimental: the behaviour of pd.NA can still change without warning.

+
+

For example, creating a Series using the nullable integer dtype:

+
.. ipython:: python
 
-   mi2 = mi.set_names("new name", level=0)
-   mi2.names
+    s = pd.Series([1, 2, None], dtype="Int64")
+    s
+    s[2]
 
-New repr for :class:`~pandas.arrays.IntervalArray`
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+

Compared to np.nan, pd.NA behaves differently in certain operations. +In addition to arithmetic operations, pd.NA also propagates as "missing" +or "unknown" in comparison operations:

+
.. ipython:: python
 
-:class:`pandas.arrays.IntervalArray` adopts a new ``__repr__`` in accordance with other array classes (:issue:`25022`)
+    np.nan > 1
+    pd.NA > 1
 
-*pandas 0.25.x*
+
+

For logical operations, pd.NA follows the rules of the +three-valued logic (or +Kleene logic). For example:

+
.. ipython:: python
 
-.. code-block:: ipython
+    pd.NA | True
 
-   In [1]: pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
-   Out[2]:
-   IntervalArray([(0, 1], (2, 3]],
-                 closed='right',
-                 dtype='interval[int64]')
+
+

For more, see :ref:`NA section <missing_data.NA>` in the user guide on missing +data.

+ +

Dedicated string data type

+

We've added :class:`StringDtype`, an extension type dedicated to string data. +Previously, strings were typically stored in object-dtype NumPy arrays. (:issue:`29975`)

+
+

Warning

+

StringDtype is currently considered experimental. The implementation +and parts of the API may change without warning.

+
+

The 'string' extension type solves several issues with object-dtype NumPy arrays:

+
    +
  1. You can accidentally store a mixture of strings and non-strings in an +object dtype array. A StringArray can only store strings.
    2. +
  2. object dtype breaks dtype-specific operations like :meth:`DataFrame.select_dtypes`. +There isn't a clear way to select just text while excluding non-text, +but still object-dtype columns.
    3. +
  3. When reading code, the contents of an object dtype array is less clear +than string.
    4. +
+
.. ipython:: python
 
-*pandas 1.0.0*
+   pd.Series(['abc', None, 'def'], dtype=pd.StringDtype())
 
-.. ipython:: python
+
+

You can use the alias "string" as well.

+
.. ipython:: python
 
-   pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
+   s = pd.Series(['abc', None, 'def'], dtype="string")
+   s
 
-``DataFrame.rename`` now only accepts one positional argument
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+

The usual string accessor methods work. Where appropriate, the return type +of the Series or columns of a DataFrame will also have string dtype.

+
.. ipython:: python
 
-:meth:`DataFrame.rename` would previously accept positional arguments that would lead
-to ambiguous or undefined behavior. From pandas 1.0, only the very first argument, which
-maps labels to their new names along the default axis, is allowed to be passed by position
-(:issue:`29136`).
+   s.str.upper()
+   s.str.split('b', expand=True).dtypes
 
-.. ipython:: python
-   :suppress:
+
+

String accessor methods returning integers will return a value with :class:`Int64Dtype`

+
.. ipython:: python
 
-   df = pd.DataFrame([[1]])
+   s.str.count("a")
 
-*pandas 0.25.x*
+
+

We recommend explicitly using the string data type when working with strings. +See :ref:`text.types` for more.

+ +

Boolean data type with missing values support

+

We've added :class:`BooleanDtype` / :class:`~arrays.BooleanArray`, an extension +type dedicated to boolean data that can hold missing values. The default +bool data type based on a bool-dtype NumPy array, the column can only hold +True or False, and not missing values. This new :class:`~arrays.BooleanArray` +can store missing values as well by keeping track of this in a separate mask. +(:issue:`29555`, :issue:`30095`, :issue:`31131`)

+
.. ipython:: python
 
-.. code-block:: python
+   pd.Series([True, False, None], dtype=pd.BooleanDtype())
 
-   >>> df = pd.DataFrame([[1]])
-   >>> df.rename({0: 1}, {0: 2})
-   FutureWarning: ...Use named arguments to resolve ambiguity...
-      2
-   1  1
+
+

You can use the alias "boolean" as well.

+
.. ipython:: python
 
-*pandas 1.0.0*
+   s = pd.Series([True, False, None], dtype="boolean")
+   s
 
-.. code-block:: python
+
+ +

convert_dtypes method to ease use of supported extension dtypes

+

In order to encourage use of the extension dtypes StringDtype, +BooleanDtype, Int64Dtype, Int32Dtype, etc., that support pd.NA, the +methods :meth:`DataFrame.convert_dtypes` and :meth:`Series.convert_dtypes` +have been introduced. (:issue:`29752`) (:issue:`30929`)

+

Example:

+
.. ipython:: python
 
-   >>> df.rename({0: 1}, {0: 2})
-   Traceback (most recent call last):
-   ...
-   TypeError: rename() takes from 1 to 2 positional arguments but 3 were given
+   df = pd.DataFrame({'x': ['abc', None, 'def'],
+                      'y': [1, 2, np.nan],
+                      'z': [True, False, True]})
+   df
+   df.dtypes
 
-Note that errors will now be raised when conflicting or potentially ambiguous arguments are provided.
+
+
.. ipython:: python
 
-*pandas 0.25.x*
+   converted = df.convert_dtypes()
+   converted
+   converted.dtypes
 
-.. code-block:: python
+
+

This is especially useful after reading in data using readers such as :func:`read_csv` +and :func:`read_excel`. +See :ref:`here <missing_data.NA.conversion>` for a description.

+ +

Other enhancements

+ + +

Backwards incompatible API changes

+ +

Avoid using names from MultiIndex.levels

+

As part of a larger refactor to :class:`MultiIndex` the level names are now +stored separately from the levels (:issue:`27242`). We recommend using +:attr:`MultiIndex.names` to access the names, and :meth:`Index.set_names` +to update the names.

+

For backwards compatibility, you can still access the names via the levels.

+
.. ipython:: python
 
-   >>> df.rename({0: 1}, index={0: 2})
-      0
-   1  1
+   mi = pd.MultiIndex.from_product([[1, 2], ['a', 'b']], names=['x', 'y'])
+   mi.levels[0].name
 
-   >>> df.rename(mapper={0: 1}, index={0: 2})
-      0
-   2  1
+
+

However, it is no longer possible to update the names of the MultiIndex +via the level.

+
.. ipython:: python
+   :okexcept:
 
-*pandas 1.0.0*
+   mi.levels[0].name = "new name"
+   mi.names
+
+
+

To update, use MultiIndex.set_names, which returns a new MultiIndex.

+
.. ipython:: python
+
+   mi2 = mi.set_names("new name", level=0)
+   mi2.names
 
-.. code-block:: python
+
+ +

New repr for :class:`~pandas.arrays.IntervalArray`

+

:class:`pandas.arrays.IntervalArray` adopts a new __repr__ in accordance with other array classes (:issue:`25022`)

+

pandas 0.25.x

+
In [1]: pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
+Out[2]:
+IntervalArray([(0, 1], (2, 3]],
+              closed='right',
+              dtype='interval[int64]')
+
+

pandas 1.0.0

+
.. ipython:: python
 
-   >>> df.rename({0: 1}, index={0: 2})
-   Traceback (most recent call last):
-   ...
-   TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
+   pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
 
-   >>> df.rename(mapper={0: 1}, index={0: 2})
-   Traceback (most recent call last):
-   ...
-   TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
+
+ +

DataFrame.rename now only accepts one positional argument

+

:meth:`DataFrame.rename` would previously accept positional arguments that would lead +to ambiguous or undefined behavior. From pandas 1.0, only the very first argument, which +maps labels to their new names along the default axis, is allowed to be passed by position +(:issue:`29136`).

+
.. ipython:: python
+   :suppress:
 
-You can still change the axis along which the first positional argument is applied by
-supplying the ``axis`` keyword argument.
+   df = pd.DataFrame([[1]])
 
-.. ipython:: python
+
+

pandas 0.25.x

+
>>> df = pd.DataFrame([[1]])
+>>> df.rename({0: 1}, {0: 2})
+FutureWarning: ...Use named arguments to resolve ambiguity...
+   2
+1  1
+

pandas 1.0.0

+
>>> df.rename({0: 1}, {0: 2})
+Traceback (most recent call last):
+...
+TypeError: rename() takes from 1 to 2 positional arguments but 3 were given
+

Note that errors will now be raised when conflicting or potentially ambiguous arguments are provided.

+

pandas 0.25.x

+
>>> df.rename({0: 1}, index={0: 2})
+   0
+1  1
+
+>>> df.rename(mapper={0: 1}, index={0: 2})
+   0
+2  1
+

pandas 1.0.0

+
>>> df.rename({0: 1}, index={0: 2})
+Traceback (most recent call last):
+...
+TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
+
+>>> df.rename(mapper={0: 1}, index={0: 2})
+Traceback (most recent call last):
+...
+TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
+

You can still change the axis along which the first positional argument is applied by +supplying the axis keyword argument.

+
.. ipython:: python
 
    df.rename({0: 1})
    df.rename({0: 1}, axis=1)
 
-If you would like to update both the index and column labels, be sure to use the respective
-keywords.
-
-.. ipython:: python
+
+

If you would like to update both the index and column labels, be sure to use the respective +keywords.

+
.. ipython:: python
 
    df.rename(index={0: 1}, columns={0: 2})
 
-Extended verbose info output for :class:`~pandas.DataFrame`
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-:meth:`DataFrame.info` now shows line numbers for the columns summary (:issue:`17304`)
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-   >>> df = pd.DataFrame({"int_col": [1, 2, 3],
-   ...                    "text_col": ["a", "b", "c"],
-   ...                    "float_col": [0.0, 0.1, 0.2]})
-   >>> df.info(verbose=True)
-   
-   RangeIndex: 3 entries, 0 to 2
-   Data columns (total 3 columns):
-   int_col      3 non-null int64
-   text_col     3 non-null object
-   float_col    3 non-null float64
-   dtypes: float64(1), int64(1), object(1)
-   memory usage: 152.0+ bytes
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+ +

Extended verbose info output for :class:`~pandas.DataFrame`

+

:meth:`DataFrame.info` now shows line numbers for the columns summary (:issue:`17304`)

+

pandas 0.25.x

+
>>> df = pd.DataFrame({"int_col": [1, 2, 3],
+...                    "text_col": ["a", "b", "c"],
+...                    "float_col": [0.0, 0.1, 0.2]})
+>>> df.info(verbose=True)
+<class 'pandas.core.frame.DataFrame'>
+RangeIndex: 3 entries, 0 to 2
+Data columns (total 3 columns):
+int_col      3 non-null int64
+text_col     3 non-null object
+float_col    3 non-null float64
+dtypes: float64(1), int64(1), object(1)
+memory usage: 152.0+ bytes
+

pandas 1.0.0

+
.. ipython:: python
 
    df = pd.DataFrame({"int_col": [1, 2, 3],
                       "text_col": ["a", "b", "c"],
                       "float_col": [0.0, 0.1, 0.2]})
    df.info(verbose=True)
 
-:meth:`pandas.array` inference changes
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-:meth:`pandas.array` now infers pandas' new extension types in several cases (:issue:`29791`):
-
-1. String data (including missing values) now returns a :class:`arrays.StringArray`.
-2. Integer data (including missing values) now returns a :class:`arrays.IntegerArray`.
-3. Boolean data (including missing values) now returns the new :class:`arrays.BooleanArray`
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-   >>> pd.array(["a", None])
-   
-   ['a', None]
-   Length: 2, dtype: object
-
-   >>> pd.array([1, None])
-   
-   [1, None]
-   Length: 2, dtype: object
-
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+ +

:meth:`pandas.array` inference changes

+

:meth:`pandas.array` now infers pandas' new extension types in several cases (:issue:`29791`):

+
    +
  1. String data (including missing values) now returns a :class:`arrays.StringArray`.
    2. +
  2. Integer data (including missing values) now returns a :class:`arrays.IntegerArray`.
    3. +
  3. Boolean data (including missing values) now returns the new :class:`arrays.BooleanArray`
    4. +
+

pandas 0.25.x

+
>>> pd.array(["a", None])
+<PandasArray>
+['a', None]
+Length: 2, dtype: object
+
+>>> pd.array([1, None])
+<PandasArray>
+[1, None]
+Length: 2, dtype: object
+

pandas 1.0.0

+
.. ipython:: python
 
    pd.array(["a", None])
    pd.array([1, None])
 
-As a reminder, you can specify the ``dtype`` to disable all inference.
-
-:class:`arrays.IntegerArray` now uses :attr:`pandas.NA`
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-:class:`arrays.IntegerArray` now uses :attr:`pandas.NA` rather than
-:attr:`numpy.nan` as its missing value marker (:issue:`29964`).
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-   >>> a = pd.array([1, 2, None], dtype="Int64")
-   >>> a
-   
-   [1, 2, NaN]
-   Length: 3, dtype: Int64
-
-   >>> a[2]
-   nan
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+

As a reminder, you can specify the dtype to disable all inference.

+ +

:class:`arrays.IntegerArray` now uses :attr:`pandas.NA`

+

:class:`arrays.IntegerArray` now uses :attr:`pandas.NA` rather than +:attr:`numpy.nan` as its missing value marker (:issue:`29964`).

+

pandas 0.25.x

+
>>> a = pd.array([1, 2, None], dtype="Int64")
+>>> a
+<IntegerArray>
+[1, 2, NaN]
+Length: 3, dtype: Int64
+
+>>> a[2]
+nan
+

pandas 1.0.0

+
.. ipython:: python
 
    a = pd.array([1, 2, None], dtype="Int64")
    a
    a[2]
 
-This has a few API-breaking consequences.
-
-**Converting to a NumPy ndarray**
-
-When converting to a NumPy array missing values will be ``pd.NA``, which cannot
-be converted to a float. So calling ``np.asarray(integer_array, dtype="float")``
-will now raise.
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-    >>> np.asarray(a, dtype="float")
-    array([ 1.,  2., nan])
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+

This has a few API-breaking consequences.

+

Converting to a NumPy ndarray

+

When converting to a NumPy array missing values will be pd.NA, which cannot +be converted to a float. So calling np.asarray(integer_array, dtype="float") +will now raise.

+

pandas 0.25.x

+
>>> np.asarray(a, dtype="float")
+array([ 1.,  2., nan])
+

pandas 1.0.0

+
.. ipython:: python
    :okexcept:
 
    np.asarray(a, dtype="float")
 
-Use :meth:`arrays.IntegerArray.to_numpy` with an explicit ``na_value`` instead.
-
-.. ipython:: python
+
+

Use :meth:`arrays.IntegerArray.to_numpy` with an explicit na_value instead.

+
.. ipython:: python
 
    a.to_numpy(dtype="float", na_value=np.nan)
 
-**Reductions can return ``pd.NA``**
-
-When performing a reduction such as a sum with ``skipna=False``, the result
-will now be ``pd.NA`` instead of ``np.nan`` in presence of missing values
-(:issue:`30958`).
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-    >>> pd.Series(a).sum(skipna=False)
-    nan
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+

Reductions can return ``pd.NA``

+

When performing a reduction such as a sum with skipna=False, the result +will now be pd.NA instead of np.nan in presence of missing values +(:issue:`30958`).

+

pandas 0.25.x

+
>>> pd.Series(a).sum(skipna=False)
+nan
+

pandas 1.0.0

+
.. ipython:: python
 
    pd.Series(a).sum(skipna=False)
 
-**value_counts returns a nullable integer dtype**
-
-:meth:`Series.value_counts` with a nullable integer dtype now returns a nullable
-integer dtype for the values.
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-   >>> pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype
-   dtype('int64')
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+

value_counts returns a nullable integer dtype

+

:meth:`Series.value_counts` with a nullable integer dtype now returns a nullable +integer dtype for the values.

+

pandas 0.25.x

+
>>> pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype
+dtype('int64')
+

pandas 1.0.0

+
.. ipython:: python
 
    pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype
 
-See :ref:`missing_data.NA` for more on the differences between :attr:`pandas.NA`
-and :attr:`numpy.nan`.
-
-:class:`arrays.IntegerArray` comparisons return :class:`arrays.BooleanArray`
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Comparison operations on a :class:`arrays.IntegerArray` now returns a
-:class:`arrays.BooleanArray` rather than a NumPy array (:issue:`29964`).
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-   >>> a = pd.array([1, 2, None], dtype="Int64")
-   >>> a
-   
-   [1, 2, NaN]
-   Length: 3, dtype: Int64
-
-   >>> a > 1
-   array([False,  True, False])
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+

See :ref:`missing_data.NA` for more on the differences between :attr:`pandas.NA` +and :attr:`numpy.nan`.

+ +

:class:`arrays.IntegerArray` comparisons return :class:`arrays.BooleanArray`

+

Comparison operations on a :class:`arrays.IntegerArray` now returns a +:class:`arrays.BooleanArray` rather than a NumPy array (:issue:`29964`).

+

pandas 0.25.x

+
>>> a = pd.array([1, 2, None], dtype="Int64")
+>>> a
+<IntegerArray>
+[1, 2, NaN]
+Length: 3, dtype: Int64
+
+>>> a > 1
+array([False,  True, False])
+

pandas 1.0.0

+
.. ipython:: python
 
    a = pd.array([1, 2, None], dtype="Int64")
-   a > 1
-
-Note that missing values now propagate, rather than always comparing unequal
-like :attr:`numpy.nan`. See :ref:`missing_data.NA` for more.
-
-By default :meth:`Categorical.min` now returns the minimum instead of np.nan
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-When :class:`Categorical` contains ``np.nan``,
-:meth:`Categorical.min` no longer return ``np.nan`` by default (skipna=True) (:issue:`25303`)
-
-*pandas 0.25.x*
-
-.. code-block:: ipython
-
-   In [1]: pd.Categorical([1, 2, np.nan], ordered=True).min()
-   Out[1]: nan
-
-
-*pandas 1.0.0*
-
-.. ipython:: python
+   a > 1
+
+
+

Note that missing values now propagate, rather than always comparing unequal +like :attr:`numpy.nan`. See :ref:`missing_data.NA` for more.

+ +

By default :meth:`Categorical.min` now returns the minimum instead of np.nan

+

When :class:`Categorical` contains np.nan, +:meth:`Categorical.min` no longer return np.nan by default (skipna=True) (:issue:`25303`)

+

pandas 0.25.x

+
In [1]: pd.Categorical([1, 2, np.nan], ordered=True).min()
+Out[1]: nan
+
+

pandas 1.0.0

+
.. ipython:: python
 
    pd.Categorical([1, 2, np.nan], ordered=True).min()
 
 
-Default dtype of empty :class:`pandas.Series`
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Initialising an empty :class:`pandas.Series` without specifying a dtype will raise a `DeprecationWarning` now
-(:issue:`17261`). The default dtype will change from ``float64`` to ``object`` in future releases so that it is
-consistent with the behaviour of :class:`DataFrame` and :class:`Index`.
-
-*pandas 1.0.0*
-
-.. code-block:: ipython
-
-   In [1]: pd.Series()
-   Out[2]:
-   DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.
-   Series([], dtype: float64)
-
-Result dtype inference changes for resample operations
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The rules for the result dtype in :meth:`DataFrame.resample` aggregations have changed for extension types (:issue:`31359`).
+
+ +

Default dtype of empty :class:`pandas.Series`

+

Initialising an empty :class:`pandas.Series` without specifying a dtype will raise a DeprecationWarning now +(:issue:`17261`). The default dtype will change from float64 to object in future releases so that it is +consistent with the behaviour of :class:`DataFrame` and :class:`Index`.

+

pandas 1.0.0

+
In [1]: pd.Series()
+Out[2]:
+DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.
+Series([], dtype: float64)
+
+ +

Result dtype inference changes for resample operations

+

The rules for the result dtype in :meth:`DataFrame.resample` aggregations have changed for extension types (:issue:`31359`). Previously, pandas would attempt to convert the result back to the original dtype, falling back to the usual inference rules if that was not possible. Now, pandas will only return a result of the original dtype if the -scalar values in the result are instances of the extension dtype's scalar type. - -.. ipython:: python +scalar values in the result are instances of the extension dtype's scalar type.

+
.. ipython:: python
 
    df = pd.DataFrame({"A": ['a', 'b']}, dtype='category',
                      index=pd.date_range('2000', periods=2))
    df
 
 
-*pandas 0.25.x*
-
-.. code-block:: python
-
-   >>> df.resample("2D").agg(lambda x: 'a').A.dtype
-   CategoricalDtype(categories=['a', 'b'], ordered=False)
-
-*pandas 1.0.0*
-
-.. ipython:: python
+
+

pandas 0.25.x

+
>>> df.resample("2D").agg(lambda x: 'a').A.dtype
+CategoricalDtype(categories=['a', 'b'], ordered=False)
+

pandas 1.0.0

+
.. ipython:: python
 
    df.resample("2D").agg(lambda x: 'a').A.dtype
 
-This fixes an inconsistency between ``resample`` and ``groupby``.
-This also fixes a potential bug, where the **values** of the result might change
-depending on how the results are cast back to the original dtype.
-
-*pandas 0.25.x*
-
-.. code-block:: python
-
-   >>> df.resample("2D").agg(lambda x: 'c')
-
-        A
-   0  NaN
-
-*pandas 1.0.0*
+
+

This fixes an inconsistency between resample and groupby. +This also fixes a potential bug, where the values of the result might change +depending on how the results are cast back to the original dtype.

+

pandas 0.25.x

+
>>> df.resample("2D").agg(lambda x: 'c')
 
-.. ipython:: python
+     A
+0  NaN
+

pandas 1.0.0

+
.. ipython:: python
 
    df.resample("2D").agg(lambda x: 'c')
 
 
-.. _whatsnew_100.api_breaking.python:
-
-Increased minimum version for Python
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Pandas 1.0.0 supports Python 3.6.1 and higher (:issue:`29212`).
-
-.. _whatsnew_100.api_breaking.deps:
-
-Increased minimum versions for dependencies
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Some minimum supported versions of dependencies were updated (:issue:`29766`, :issue:`29723`).
-If installed, we now require:
-
-+-----------------+-----------------+----------+---------+
-| Package         | Minimum Version | Required | Changed |
-+=================+=================+==========+=========+
-| numpy           | 1.13.3          |    X     |         |
-+-----------------+-----------------+----------+---------+
-| pytz            | 2015.4          |    X     |         |
-+-----------------+-----------------+----------+---------+
-| python-dateutil | 2.6.1           |    X     |         |
-+-----------------+-----------------+----------+---------+
-| bottleneck      | 1.2.1           |          |         |
-+-----------------+-----------------+----------+---------+
-| numexpr         | 2.6.2           |          |         |
-+-----------------+-----------------+----------+---------+
-| pytest (dev)    | 4.0.2           |          |         |
-+-----------------+-----------------+----------+---------+
-
-For `optional libraries `_ the general recommendation is to use the latest version.
+
+ +

Increased minimum version for Python

+

Pandas 1.0.0 supports Python 3.6.1 and higher (:issue:`29212`).

+ +

Increased minimum versions for dependencies

+

Some minimum supported versions of dependencies were updated (:issue:`29766`, :issue:`29723`). +If installed, we now require:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageMinimum VersionRequiredChanged
numpy1.13.3X 
pytz2015.4X 
python-dateutil2.6.1X 
bottleneck1.2.1  
numexpr2.6.2  
pytest (dev)4.0.2  
+

For optional libraries the general recommendation is to use the latest version. The following table lists the lowest version per library that is currently being tested throughout the development of pandas. -Optional libraries below the lowest tested version may still work, but are not considered supported. - -+-----------------+-----------------+---------+ -| Package | Minimum Version | Changed | -+=================+=================+=========+ -| beautifulsoup4 | 4.6.0 | | -+-----------------+-----------------+---------+ -| fastparquet | 0.3.2 | X | -+-----------------+-----------------+---------+ -| gcsfs | 0.2.2 | | -+-----------------+-----------------+---------+ -| lxml | 3.8.0 | | -+-----------------+-----------------+---------+ -| matplotlib | 2.2.2 | | -+-----------------+-----------------+---------+ -| numba | 0.46.0 | X | -+-----------------+-----------------+---------+ -| openpyxl | 2.5.7 | X | -+-----------------+-----------------+---------+ -| pyarrow | 0.13.0 | X | -+-----------------+-----------------+---------+ -| pymysql | 0.7.1 | | -+-----------------+-----------------+---------+ -| pytables | 3.4.2 | | -+-----------------+-----------------+---------+ -| s3fs | 0.3.0 | X | -+-----------------+-----------------+---------+ -| scipy | 0.19.0 | | -+-----------------+-----------------+---------+ -| sqlalchemy | 1.1.4 | | -+-----------------+-----------------+---------+ -| xarray | 0.8.2 | | -+-----------------+-----------------+---------+ -| xlrd | 1.1.0 | | -+-----------------+-----------------+---------+ -| xlsxwriter | 0.9.8 | | -+-----------------+-----------------+---------+ -| xlwt | 1.2.0 | | -+-----------------+-----------------+---------+ - -See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more. - -Build Changes -^^^^^^^^^^^^^ - -Pandas has added a `pyproject.toml `_ file and will no longer include -cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing +Optional libraries below the lowest tested version may still work, but are not considered supported.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageMinimum VersionChanged
beautifulsoup44.6.0 
fastparquet0.3.2X
gcsfs0.2.2 
lxml3.8.0 
matplotlib2.2.2 
numba0.46.0X
openpyxl2.5.7X
pyarrow0.13.0X
pymysql0.7.1 
pytables3.4.2 
s3fs0.3.0X
scipy0.19.0 
sqlalchemy1.1.4 
xarray0.8.2 
xlrd1.1.0 
xlsxwriter0.9.8 
xlwt1.2.0 
+

See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.

+ +

Build Changes

+

Pandas has added a pyproject.toml file and will no longer include +cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing a built distribution (wheel) or via conda, this shouldn't have any effect on you. If you're building pandas from -source, you should no longer need to install Cython into your build environment before calling ``pip install pandas``. - - -.. _whatsnew_100.api.other: - -Other API changes -^^^^^^^^^^^^^^^^^ - -- :class:`core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`) -- :meth:`pandas.api.types.infer_dtype` will now return "integer-na" for integer and ``np.nan`` mix (:issue:`27283`) -- :meth:`MultiIndex.from_arrays` will no longer infer names from arrays if ``names=None`` is explicitly provided (:issue:`27292`) -- In order to improve tab-completion, Pandas does not include most deprecated attributes when introspecting a pandas object using ``dir`` (e.g. ``dir(df)``). - To see which attributes are excluded, see an object's ``_deprecations`` attribute, for example ``pd.DataFrame._deprecations`` (:issue:`28805`). -- The returned dtype of :func:`unique` now matches the input dtype. (:issue:`27874`) -- Changed the default configuration value for ``options.matplotlib.register_converters`` from ``True`` to ``"auto"`` (:issue:`18720`). - Now, pandas custom formatters will only be applied to plots created by pandas, through :meth:`~DataFrame.plot`. - Previously, pandas' formatters would be applied to all plots created *after* a :meth:`~DataFrame.plot`. - See :ref:`units registration ` for more. -- :meth:`Series.dropna` has dropped its ``**kwargs`` argument in favor of a single ``how`` parameter. - Supplying anything else than ``how`` to ``**kwargs`` raised a ``TypeError`` previously (:issue:`29388`) -- When testing pandas, the new minimum required version of pytest is 5.0.1 (:issue:`29664`) -- :meth:`Series.str.__iter__` was deprecated and will be removed in future releases (:issue:`28277`). -- Added ```` to the list of default NA values for :meth:`read_csv` (:issue:`30821`) - -.. _whatsnew_100.api.documentation: - -Documentation Improvements -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- Added new section on :ref:`scale` (:issue:`28315`). -- Added sub-section on :ref:`io.query_multi` for HDF5 datasets (:issue:`28791`). - -.. --------------------------------------------------------------------------- - -.. _whatsnew_100.deprecations: - -Deprecations -~~~~~~~~~~~~ - -- :meth:`Series.item` and :meth:`Index.item` have been _undeprecated_ (:issue:`29250`) -- ``Index.set_value`` has been deprecated. For a given index ``idx``, array ``arr``, - value in ``idx`` of ``idx_val`` and a new value of ``val``, ``idx.set_value(arr, idx_val, val)`` - is equivalent to ``arr[idx.get_loc(idx_val)] = val``, which should be used instead (:issue:`28621`). -- :func:`is_extension_type` is deprecated, :func:`is_extension_array_dtype` should be used instead (:issue:`29457`) -- :func:`eval` keyword argument "truediv" is deprecated and will be removed in a future version (:issue:`29812`) -- :meth:`DateOffset.isAnchored` and :meth:`DatetOffset.onOffset` are deprecated and will be removed in a future version, use :meth:`DateOffset.is_anchored` and :meth:`DateOffset.is_on_offset` instead (:issue:`30340`) -- ``pandas.tseries.frequencies.get_offset`` is deprecated and will be removed in a future version, use ``pandas.tseries.frequencies.to_offset`` instead (:issue:`4205`) -- :meth:`Categorical.take_nd` and :meth:`CategoricalIndex.take_nd` are deprecated, use :meth:`Categorical.take` and :meth:`CategoricalIndex.take` instead (:issue:`27745`) -- The parameter ``numeric_only`` of :meth:`Categorical.min` and :meth:`Categorical.max` is deprecated and replaced with ``skipna`` (:issue:`25303`) -- The parameter ``label`` in :func:`lreshape` has been deprecated and will be removed in a future version (:issue:`29742`) -- ``pandas.core.index`` has been deprecated and will be removed in a future version, the public classes are available in the top-level namespace (:issue:`19711`) -- :func:`pandas.json_normalize` is now exposed in the top-level namespace. - Usage of ``json_normalize`` as ``pandas.io.json.json_normalize`` is now deprecated and - it is recommended to use ``json_normalize`` as :func:`pandas.json_normalize` instead (:issue:`27586`). -- The ``numpy`` argument of :meth:`pandas.read_json` is deprecated (:issue:`28512`). -- :meth:`DataFrame.to_stata`, :meth:`DataFrame.to_feather`, and :meth:`DataFrame.to_parquet` argument "fname" is deprecated, use "path" instead (:issue:`23574`) -- The deprecated internal attributes ``_start``, ``_stop`` and ``_step`` of :class:`RangeIndex` now raise a ``FutureWarning`` instead of a ``DeprecationWarning`` (:issue:`26581`) -- The ``pandas.util.testing`` module has been deprecated. Use the public API in ``pandas.testing`` documented at :ref:`api.general.testing` (:issue:`16232`). -- ``pandas.SparseArray`` has been deprecated. Use ``pandas.arrays.SparseArray`` (:class:`arrays.SparseArray`) instead. (:issue:`30642`) -- The parameter ``is_copy`` of :meth:`Series.take` and :meth:`DataFrame.take` has been deprecated and will be removed in a future version. (:issue:`27357`) -- Support for multi-dimensional indexing (e.g. ``index[:, None]``) on a :class:`Index` is deprecated and will be removed in a future version, convert to a numpy array before indexing instead (:issue:`30588`) -- The ``pandas.np`` submodule is now deprecated. Import numpy directly instead (:issue:`30296`) -- The ``pandas.datetime`` class is now deprecated. Import from ``datetime`` instead (:issue:`30610`) -- :class:`~DataFrame.diff` will raise a ``TypeError`` rather than implicitly losing the dtype of extension types in the future. Convert to the correct dtype before calling ``diff`` instead (:issue:`31025`) - -**Selecting Columns from a Grouped DataFrame** - -When selecting columns from a :class:`DataFrameGroupBy` object, passing individual keys (or a tuple of keys) inside single brackets is deprecated, -a list of items should be used instead. (:issue:`23566`) For example: - -.. code-block:: ipython - - df = pd.DataFrame({ - "A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"], - "B": np.random.randn(8), - "C": np.random.randn(8), - }) - g = df.groupby('A') - - # single key, returns SeriesGroupBy - g['B'] - - # tuple of single key, returns SeriesGroupBy - g[('B',)] - - # tuple of multiple keys, returns DataFrameGroupBy, raises FutureWarning - g[('B', 'C')] - - # multiple keys passed directly, returns DataFrameGroupBy, raises FutureWarning - # (implicitly converts the passed strings into a single tuple) - g['B', 'C'] - - # proper way, returns DataFrameGroupBy - g[['B', 'C']] - -.. --------------------------------------------------------------------------- - -.. _whatsnew_100.prior_deprecations: - -Removal of prior version deprecations/changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Removed SparseSeries and SparseDataFrame** - -``SparseSeries``, ``SparseDataFrame`` and the ``DataFrame.to_sparse`` method -have been removed (:issue:`28425`). We recommend using a ``Series`` or -``DataFrame`` with sparse values instead. See :ref:`sparse.migration` for help -with migrating existing code. - -.. _whatsnew_100.matplotlib_units: - -**Matplotlib unit registration** - -Previously, pandas would register converters with matplotlib as a side effect of importing pandas (:issue:`18720`). +source, you should no longer need to install Cython into your build environment before calling pip install pandas.

+ +

Other API changes

+ + +

Documentation Improvements

+ + +

Deprecations

+ +

Selecting Columns from a Grouped DataFrame

+

When selecting columns from a :class:`DataFrameGroupBy` object, passing individual keys (or a tuple of keys) inside single brackets is deprecated, +a list of items should be used instead. (:issue:`23566`) For example:

+
df = pd.DataFrame({
+    "A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"],
+    "B": np.random.randn(8),
+    "C": np.random.randn(8),
+})
+g = df.groupby('A')
+
+# single key, returns SeriesGroupBy
+g['B']
+
+# tuple of single key, returns SeriesGroupBy
+g[('B',)]
+
+# tuple of multiple keys, returns DataFrameGroupBy, raises FutureWarning
+g[('B', 'C')]
+
+# multiple keys passed directly, returns DataFrameGroupBy, raises FutureWarning
+# (implicitly converts the passed strings into a single tuple)
+g['B', 'C']
+
+# proper way, returns DataFrameGroupBy
+g[['B', 'C']]
+
+ +

Removal of prior version deprecations/changes

+

Removed SparseSeries and SparseDataFrame

+

SparseSeries, SparseDataFrame and the DataFrame.to_sparse method +have been removed (:issue:`28425`). We recommend using a Series or +DataFrame with sparse values instead. See :ref:`sparse.migration` for help +with migrating existing code.

+

Matplotlib unit registration

+

Previously, pandas would register converters with matplotlib as a side effect of importing pandas (:issue:`18720`). This changed the output of plots made via matplotlib plots after pandas was imported, even if you were using -matplotlib directly rather than :meth:`~DataFrame.plot`. - -To use pandas formatters with a matplotlib plot, specify - -.. code-block:: python - - >>> import pandas as pd - >>> pd.options.plotting.matplotlib.register_converters = True - -Note that plots created by :meth:`DataFrame.plot` and :meth:`Series.plot` *do* register the converters -automatically. The only behavior change is when plotting a date-like object via ``matplotlib.pyplot.plot`` -or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more. - -**Other removals** - -- Removed the previously deprecated keyword "index" from :func:`read_stata`, :class:`StataReader`, and :meth:`StataReader.read`, use "index_col" instead (:issue:`17328`) -- Removed ``StataReader.data`` method, use :meth:`StataReader.read` instead (:issue:`9493`) -- Removed ``pandas.plotting._matplotlib.tsplot``, use :meth:`Series.plot` instead (:issue:`19980`) -- ``pandas.tseries.converter.register`` has been moved to :func:`pandas.plotting.register_matplotlib_converters` (:issue:`18307`) -- :meth:`Series.plot` no longer accepts positional arguments, pass keyword arguments instead (:issue:`30003`) -- :meth:`DataFrame.hist` and :meth:`Series.hist` no longer allows ``figsize="default"``, specify figure size by passinig a tuple instead (:issue:`30003`) -- Floordiv of integer-dtyped array by :class:`Timedelta` now raises ``TypeError`` (:issue:`21036`) -- :class:`TimedeltaIndex` and :class:`DatetimeIndex` no longer accept non-nanosecond dtype strings like "timedelta64" or "datetime64", use "timedelta64[ns]" and "datetime64[ns]" instead (:issue:`24806`) -- Changed the default "skipna" argument in :func:`pandas.api.types.infer_dtype` from ``False`` to ``True`` (:issue:`24050`) -- Removed ``Series.ix`` and ``DataFrame.ix`` (:issue:`26438`) -- Removed ``Index.summary`` (:issue:`18217`) -- Removed the previously deprecated keyword "fastpath" from the :class:`Index` constructor (:issue:`23110`) -- Removed ``Series.get_value``, ``Series.set_value``, ``DataFrame.get_value``, ``DataFrame.set_value`` (:issue:`17739`) -- Removed ``Series.compound`` and ``DataFrame.compound`` (:issue:`26405`) -- Changed the default "inplace" argument in :meth:`DataFrame.set_index` and :meth:`Series.set_axis` from ``None`` to ``False`` (:issue:`27600`) -- Removed ``Series.cat.categorical``, ``Series.cat.index``, ``Series.cat.name`` (:issue:`24751`) -- Removed the previously deprecated keyword "box" from :func:`to_datetime` and :func:`to_timedelta`; in addition these now always returns :class:`DatetimeIndex`, :class:`TimedeltaIndex`, :class:`Index`, :class:`Series`, or :class:`DataFrame` (:issue:`24486`) -- :func:`to_timedelta`, :class:`Timedelta`, and :class:`TimedeltaIndex` no longer allow "M", "y", or "Y" for the "unit" argument (:issue:`23264`) -- Removed the previously deprecated keyword "time_rule" from (non-public) ``offsets.generate_range``, which has been moved to :func:`core.arrays._ranges.generate_range` (:issue:`24157`) -- :meth:`DataFrame.loc` or :meth:`Series.loc` with listlike indexers and missing labels will no longer reindex (:issue:`17295`) -- :meth:`DataFrame.to_excel` and :meth:`Series.to_excel` with non-existent columns will no longer reindex (:issue:`17295`) -- Removed the previously deprecated keyword "join_axes" from :func:`concat`; use ``reindex_like`` on the result instead (:issue:`22318`) -- Removed the previously deprecated keyword "by" from :meth:`DataFrame.sort_index`, use :meth:`DataFrame.sort_values` instead (:issue:`10726`) -- Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`core.groupby.DataFrameGroupBy.aggregate`, :meth:`core.groupby.SeriesGroupBy.aggregate`, :meth:`core.window.rolling.Rolling.aggregate` (:issue:`18529`) -- Passing ``datetime64`` data to :class:`TimedeltaIndex` or ``timedelta64`` data to ``DatetimeIndex`` now raises ``TypeError`` (:issue:`23539`, :issue:`23937`) -- Passing ``int64`` values to :class:`DatetimeIndex` and a timezone now interprets the values as nanosecond timestamps in UTC, not wall times in the given timezone (:issue:`24559`) -- A tuple passed to :meth:`DataFrame.groupby` is now exclusively treated as a single key (:issue:`18314`) -- Removed ``Index.contains``, use ``key in index`` instead (:issue:`30103`) -- Addition and subtraction of ``int`` or integer-arrays is no longer allowed in :class:`Timestamp`, :class:`DatetimeIndex`, :class:`TimedeltaIndex`, use ``obj + n * obj.freq`` instead of ``obj + n`` (:issue:`22535`) -- Removed ``Series.ptp`` (:issue:`21614`) -- Removed ``Series.from_array`` (:issue:`18258`) -- Removed ``DataFrame.from_items`` (:issue:`18458`) -- Removed ``DataFrame.as_matrix``, ``Series.as_matrix`` (:issue:`18458`) -- Removed ``Series.asobject`` (:issue:`18477`) -- Removed ``DataFrame.as_blocks``, ``Series.as_blocks``, ``DataFrame.blocks``, ``Series.blocks`` (:issue:`17656`) -- :meth:`pandas.Series.str.cat` now defaults to aligning ``others``, using ``join='left'`` (:issue:`27611`) -- :meth:`pandas.Series.str.cat` does not accept list-likes *within* list-likes anymore (:issue:`27611`) -- :meth:`Series.where` with ``Categorical`` dtype (or :meth:`DataFrame.where` with ``Categorical`` column) no longer allows setting new categories (:issue:`24114`) -- Removed the previously deprecated keywords "start", "end", and "periods" from the :class:`DatetimeIndex`, :class:`TimedeltaIndex`, and :class:`PeriodIndex` constructors; use :func:`date_range`, :func:`timedelta_range`, and :func:`period_range` instead (:issue:`23919`) -- Removed the previously deprecated keyword "verify_integrity" from the :class:`DatetimeIndex` and :class:`TimedeltaIndex` constructors (:issue:`23919`) -- Removed the previously deprecated keyword "fastpath" from ``pandas.core.internals.blocks.make_block`` (:issue:`19265`) -- Removed the previously deprecated keyword "dtype" from :meth:`Block.make_block_same_class` (:issue:`19434`) -- Removed ``ExtensionArray._formatting_values``. Use :attr:`ExtensionArray._formatter` instead. (:issue:`23601`) -- Removed ``MultiIndex.to_hierarchical`` (:issue:`21613`) -- Removed ``MultiIndex.labels``, use :attr:`MultiIndex.codes` instead (:issue:`23752`) -- Removed the previously deprecated keyword "labels" from the :class:`MultiIndex` constructor, use "codes" instead (:issue:`23752`) -- Removed ``MultiIndex.set_labels``, use :meth:`MultiIndex.set_codes` instead (:issue:`23752`) -- Removed the previously deprecated keyword "labels" from :meth:`MultiIndex.set_codes`, :meth:`MultiIndex.copy`, :meth:`MultiIndex.drop`, use "codes" instead (:issue:`23752`) -- Removed support for legacy HDF5 formats (:issue:`29787`) -- Passing a dtype alias (e.g. 'datetime64[ns, UTC]') to :class:`DatetimeTZDtype` is no longer allowed, use :meth:`DatetimeTZDtype.construct_from_string` instead (:issue:`23990`) -- Removed the previously deprecated keyword "skip_footer" from :func:`read_excel`; use "skipfooter" instead (:issue:`18836`) -- :func:`read_excel` no longer allows an integer value for the parameter ``usecols``, instead pass a list of integers from 0 to ``usecols`` inclusive (:issue:`23635`) -- Removed the previously deprecated keyword "convert_datetime64" from :meth:`DataFrame.to_records` (:issue:`18902`) -- Removed ``IntervalIndex.from_intervals`` in favor of the :class:`IntervalIndex` constructor (:issue:`19263`) -- Changed the default "keep_tz" argument in :meth:`DatetimeIndex.to_series` from ``None`` to ``True`` (:issue:`23739`) -- Removed ``api.types.is_period`` and ``api.types.is_datetimetz`` (:issue:`23917`) -- Ability to read pickles containing :class:`Categorical` instances created with pre-0.16 version of pandas has been removed (:issue:`27538`) -- Removed ``pandas.tseries.plotting.tsplot`` (:issue:`18627`) -- Removed the previously deprecated keywords "reduce" and "broadcast" from :meth:`DataFrame.apply` (:issue:`18577`) -- Removed the previously deprecated ``assert_raises_regex`` function in ``pandas._testing`` (:issue:`29174`) -- Removed the previously deprecated ``FrozenNDArray`` class in ``pandas.core.indexes.frozen`` (:issue:`29335`) -- Removed the previously deprecated keyword "nthreads" from :func:`read_feather`, use "use_threads" instead (:issue:`23053`) -- Removed ``Index.is_lexsorted_for_tuple`` (:issue:`29305`) -- Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`core.groupby.DataFrameGroupBy.aggregate`, :meth:`core.groupby.SeriesGroupBy.aggregate`, :meth:`core.window.rolling.Rolling.aggregate` (:issue:`29608`) -- Removed ``Series.valid``; use :meth:`Series.dropna` instead (:issue:`18800`) -- Removed ``DataFrame.is_copy``, ``Series.is_copy`` (:issue:`18812`) -- Removed ``DataFrame.get_ftype_counts``, ``Series.get_ftype_counts`` (:issue:`18243`) -- Removed ``DataFrame.ftypes``, ``Series.ftypes``, ``Series.ftype`` (:issue:`26744`) -- Removed ``Index.get_duplicates``, use ``idx[idx.duplicated()].unique()`` instead (:issue:`20239`) -- Removed ``Series.clip_upper``, ``Series.clip_lower``, ``DataFrame.clip_upper``, ``DataFrame.clip_lower`` (:issue:`24203`) -- Removed the ability to alter :attr:`DatetimeIndex.freq`, :attr:`TimedeltaIndex.freq`, or :attr:`PeriodIndex.freq` (:issue:`20772`) -- Removed ``DatetimeIndex.offset`` (:issue:`20730`) -- Removed ``DatetimeIndex.asobject``, ``TimedeltaIndex.asobject``, ``PeriodIndex.asobject``, use ``astype(object)`` instead (:issue:`29801`) -- Removed the previously deprecated keyword "order" from :func:`factorize` (:issue:`19751`) -- Removed the previously deprecated keyword "encoding" from :func:`read_stata` and :meth:`DataFrame.to_stata` (:issue:`21400`) -- Changed the default "sort" argument in :func:`concat` from ``None`` to ``False`` (:issue:`20613`) -- Removed the previously deprecated keyword "raise_conflict" from :meth:`DataFrame.update`, use "errors" instead (:issue:`23585`) -- Removed the previously deprecated keyword "n" from :meth:`DatetimeIndex.shift`, :meth:`TimedeltaIndex.shift`, :meth:`PeriodIndex.shift`, use "periods" instead (:issue:`22458`) -- Removed the previously deprecated keywords "how", "fill_method", and "limit" from :meth:`DataFrame.resample` (:issue:`30139`) -- Passing an integer to :meth:`Series.fillna` or :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtype now raises ``TypeError`` (:issue:`24694`) -- Passing multiple axes to :meth:`DataFrame.dropna` is no longer supported (:issue:`20995`) -- Removed ``Series.nonzero``, use ``to_numpy().nonzero()`` instead (:issue:`24048`) -- Passing floating dtype ``codes`` to :meth:`Categorical.from_codes` is no longer supported, pass ``codes.astype(np.int64)`` instead (:issue:`21775`) -- Removed the previously deprecated keyword "pat" from :meth:`Series.str.partition` and :meth:`Series.str.rpartition`, use "sep" instead (:issue:`23767`) -- Removed ``Series.put`` (:issue:`27106`) -- Removed ``Series.real``, ``Series.imag`` (:issue:`27106`) -- Removed ``Series.to_dense``, ``DataFrame.to_dense`` (:issue:`26684`) -- Removed ``Index.dtype_str``, use ``str(index.dtype)`` instead (:issue:`27106`) -- :meth:`Categorical.ravel` returns a :class:`Categorical` instead of a ``ndarray`` (:issue:`27199`) -- The 'outer' method on Numpy ufuncs, e.g. ``np.subtract.outer`` operating on :class:`Series` objects is no longer supported, and will raise ``NotImplementedError`` (:issue:`27198`) -- Removed ``Series.get_dtype_counts`` and ``DataFrame.get_dtype_counts`` (:issue:`27145`) -- Changed the default "fill_value" argument in :meth:`Categorical.take` from ``True`` to ``False`` (:issue:`20841`) -- Changed the default value for the `raw` argument in :func:`Series.rolling().apply() `, :func:`DataFrame.rolling().apply() `, :func:`Series.expanding().apply() `, and :func:`DataFrame.expanding().apply() ` from ``None`` to ``False`` (:issue:`20584`) -- Removed deprecated behavior of :meth:`Series.argmin` and :meth:`Series.argmax`, use :meth:`Series.idxmin` and :meth:`Series.idxmax` for the old behavior (:issue:`16955`) -- Passing a tz-aware ``datetime.datetime`` or :class:`Timestamp` into the :class:`Timestamp` constructor with the ``tz`` argument now raises a ``ValueError`` (:issue:`23621`) -- Removed ``Series.base``, ``Index.base``, ``Categorical.base``, ``Series.flags``, ``Index.flags``, ``PeriodArray.flags``, ``Series.strides``, ``Index.strides``, ``Series.itemsize``, ``Index.itemsize``, ``Series.data``, ``Index.data`` (:issue:`20721`) -- Changed :meth:`Timedelta.resolution` to match the behavior of the standard library ``datetime.timedelta.resolution``, for the old behavior, use :meth:`Timedelta.resolution_string` (:issue:`26839`) -- Removed ``Timestamp.weekday_name``, ``DatetimeIndex.weekday_name``, and ``Series.dt.weekday_name`` (:issue:`18164`) -- Removed the previously deprecated keyword "errors" in :meth:`Timestamp.tz_localize`, :meth:`DatetimeIndex.tz_localize`, and :meth:`Series.tz_localize` (:issue:`22644`) -- Changed the default "ordered" argument in :class:`CategoricalDtype` from ``None`` to ``False`` (:issue:`26336`) -- :meth:`Series.set_axis` and :meth:`DataFrame.set_axis` now require "labels" as the first argument and "axis" as an optional named parameter (:issue:`30089`) -- Removed ``to_msgpack``, ``read_msgpack``, ``DataFrame.to_msgpack``, ``Series.to_msgpack`` (:issue:`27103`) -- Removed ``Series.compress`` (:issue:`21930`) -- Removed the previously deprecated keyword "fill_value" from :meth:`Categorical.fillna`, use "value" instead (:issue:`19269`) -- Removed the previously deprecated keyword "data" from :func:`andrews_curves`, use "frame" instead (:issue:`6956`) -- Removed the previously deprecated keyword "data" from :func:`parallel_coordinates`, use "frame" instead (:issue:`6956`) -- Removed the previously deprecated keyword "colors" from :func:`parallel_coordinates`, use "color" instead (:issue:`6956`) -- Removed the previously deprecated keywords "verbose" and "private_key" from :func:`read_gbq` (:issue:`30200`) -- Calling ``np.array`` and ``np.asarray`` on tz-aware :class:`Series` and :class:`DatetimeIndex` will now return an object array of tz-aware :class:`Timestamp` (:issue:`24596`) -- - -.. --------------------------------------------------------------------------- - -.. _whatsnew_100.performance: - -Performance improvements -~~~~~~~~~~~~~~~~~~~~~~~~ - -- Performance improvement in :class:`DataFrame` arithmetic and comparison operations with scalars (:issue:`24990`, :issue:`29853`) -- Performance improvement in indexing with a non-unique :class:`IntervalIndex` (:issue:`27489`) -- Performance improvement in :attr:`MultiIndex.is_monotonic` (:issue:`27495`) -- Performance improvement in :func:`cut` when ``bins`` is an :class:`IntervalIndex` (:issue:`27668`) -- Performance improvement when initializing a :class:`DataFrame` using a ``range`` (:issue:`30171`) -- Performance improvement in :meth:`DataFrame.corr` when ``method`` is ``"spearman"`` (:issue:`28139`) -- Performance improvement in :meth:`DataFrame.replace` when provided a list of values to replace (:issue:`28099`) -- Performance improvement in :meth:`DataFrame.select_dtypes` by using vectorization instead of iterating over a loop (:issue:`28317`) -- Performance improvement in :meth:`Categorical.searchsorted` and :meth:`CategoricalIndex.searchsorted` (:issue:`28795`) -- Performance improvement when comparing a :class:`Categorical` with a scalar and the scalar is not found in the categories (:issue:`29750`) -- Performance improvement when checking if values in a :class:`Categorical` are equal, equal or larger or larger than a given scalar. - The improvement is not present if checking if the :class:`Categorical` is less than or less than or equal than the scalar (:issue:`29820`) -- Performance improvement in :meth:`Index.equals` and :meth:`MultiIndex.equals` (:issue:`29134`) -- Performance improvement in :func:`~pandas.api.types.infer_dtype` when ``skipna`` is ``True`` (:issue:`28814`) - -.. --------------------------------------------------------------------------- - -.. _whatsnew_100.bug_fixes: - -Bug fixes -~~~~~~~~~ - - -Categorical -^^^^^^^^^^^ - -- Added test to assert the :func:`fillna` raises the correct ``ValueError`` message when the value isn't a value from categories (:issue:`13628`) -- Bug in :meth:`Categorical.astype` where ``NaN`` values were handled incorrectly when casting to int (:issue:`28406`) -- :meth:`DataFrame.reindex` with a :class:`CategoricalIndex` would fail when the targets contained duplicates, and wouldn't fail if the source contained duplicates (:issue:`28107`) -- Bug in :meth:`Categorical.astype` not allowing for casting to extension dtypes (:issue:`28668`) -- Bug where :func:`merge` was unable to join on categorical and extension dtype columns (:issue:`28668`) -- :meth:`Categorical.searchsorted` and :meth:`CategoricalIndex.searchsorted` now work on unordered categoricals also (:issue:`21667`) -- Added test to assert roundtripping to parquet with :func:`DataFrame.to_parquet` or :func:`read_parquet` will preserve Categorical dtypes for string types (:issue:`27955`) -- Changed the error message in :meth:`Categorical.remove_categories` to always show the invalid removals as a set (:issue:`28669`) -- Using date accessors on a categorical dtyped :class:`Series` of datetimes was not returning an object of the - same type as if one used the :meth:`.str.` / :meth:`.dt.` on a :class:`Series` of that type. E.g. when accessing :meth:`Series.dt.tz_localize` on a - :class:`Categorical` with duplicate entries, the accessor was skipping duplicates (:issue:`27952`) -- Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` that would give incorrect results on categorical data (:issue:`26988`) -- Bug where calling :meth:`Categorical.min` or :meth:`Categorical.max` on an empty Categorical would raise a numpy exception (:issue:`30227`) -- The following methods now also correctly output values for unobserved categories when called through ``groupby(..., observed=False)`` (:issue:`17605`) - * :meth:`core.groupby.SeriesGroupBy.count` - * :meth:`core.groupby.SeriesGroupBy.size` - * :meth:`core.groupby.SeriesGroupBy.nunique` - * :meth:`core.groupby.SeriesGroupBy.nth` - - -Datetimelike -^^^^^^^^^^^^ -- Bug in :meth:`Series.__setitem__` incorrectly casting ``np.timedelta64("NaT")`` to ``np.datetime64("NaT")`` when inserting into a :class:`Series` with datetime64 dtype (:issue:`27311`) -- Bug in :meth:`Series.dt` property lookups when the underlying data is read-only (:issue:`27529`) -- Bug in ``HDFStore.__getitem__`` incorrectly reading tz attribute created in Python 2 (:issue:`26443`) -- Bug in :func:`to_datetime` where passing arrays of malformed ``str`` with errors="coerce" could incorrectly lead to raising ``ValueError`` (:issue:`28299`) -- Bug in :meth:`core.groupby.SeriesGroupBy.nunique` where ``NaT`` values were interfering with the count of unique values (:issue:`27951`) -- Bug in :class:`Timestamp` subtraction when subtracting a :class:`Timestamp` from a ``np.datetime64`` object incorrectly raising ``TypeError`` (:issue:`28286`) -- Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`) -- Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`) -- Bug in :meth:`Series.astype`, :meth:`Index.astype`, and :meth:`DataFrame.astype` failing to handle ``NaT`` when casting to an integer dtype (:issue:`28492`) -- Bug in :class:`Week` with ``weekday`` incorrectly raising ``AttributeError`` instead of ``TypeError`` when adding or subtracting an invalid type (:issue:`28530`) -- Bug in :class:`DataFrame` arithmetic operations when operating with a :class:`Series` with dtype `'timedelta64[ns]'` (:issue:`28049`) -- Bug in :func:`core.groupby.generic.SeriesGroupBy.apply` raising ``ValueError`` when a column in the original DataFrame is a datetime and the column labels are not standard integers (:issue:`28247`) -- Bug in :func:`pandas._config.localization.get_locales` where the ``locales -a`` encodes the locales list as windows-1252 (:issue:`23638`, :issue:`24760`, :issue:`27368`) -- Bug in :meth:`Series.var` failing to raise ``TypeError`` when called with ``timedelta64[ns]`` dtype (:issue:`28289`) -- Bug in :meth:`DatetimeIndex.strftime` and :meth:`Series.dt.strftime` where ``NaT`` was converted to the string ``'NaT'`` instead of ``np.nan`` (:issue:`29578`) -- Bug in masking datetime-like arrays with a boolean mask of an incorrect length not raising an ``IndexError`` (:issue:`30308`) -- Bug in :attr:`Timestamp.resolution` being a property instead of a class attribute (:issue:`29910`) -- Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`) -- Bug in :func:`pandas.to_datetime` failing for `deques` when using ``cache=True`` (the default) (:issue:`29403`) -- Bug in :meth:`Series.item` with ``datetime64`` or ``timedelta64`` dtype, :meth:`DatetimeIndex.item`, and :meth:`TimedeltaIndex.item` returning an integer instead of a :class:`Timestamp` or :class:`Timedelta` (:issue:`30175`) -- Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`) -- Bug in :meth:`DataFrame.drop` where attempting to drop non-existent values from a DatetimeIndex would yield a confusing error message (:issue:`30399`) -- Bug in :meth:`DataFrame.append` would remove the timezone-awareness of new data (:issue:`30238`) -- Bug in :meth:`Series.cummin` and :meth:`Series.cummax` with timezone-aware dtype incorrectly dropping its timezone (:issue:`15553`) -- Bug in :class:`DatetimeArray`, :class:`TimedeltaArray`, and :class:`PeriodArray` where inplace addition and subtraction did not actually operate inplace (:issue:`24115`) -- Bug in :func:`pandas.to_datetime` when called with ``Series`` storing ``IntegerArray`` raising ``TypeError`` instead of returning ``Series`` (:issue:`30050`) -- Bug in :func:`date_range` with custom business hours as ``freq`` and given number of ``periods`` (:issue:`30593`) -- Bug in :class:`PeriodIndex` comparisons with incorrectly casting integers to :class:`Period` objects, inconsistent with the :class:`Period` comparison behavior (:issue:`30722`) -- Bug in :meth:`DatetimeIndex.insert` raising a ``ValueError`` instead of a ``TypeError`` when trying to insert a timezone-aware :class:`Timestamp` into a timezone-naive :class:`DatetimeIndex`, or vice-versa (:issue:`30806`) - -Timedelta -^^^^^^^^^ -- Bug in subtracting a :class:`TimedeltaIndex` or :class:`TimedeltaArray` from a ``np.datetime64`` object (:issue:`29558`) -- -- - -Timezones -^^^^^^^^^ - -- -- - - -Numeric -^^^^^^^ -- Bug in :meth:`DataFrame.quantile` with zero-column :class:`DataFrame` incorrectly raising (:issue:`23925`) -- :class:`DataFrame` flex inequality comparisons methods (:meth:`DataFrame.lt`, :meth:`DataFrame.le`, :meth:`DataFrame.gt`, :meth:`DataFrame.ge`) with object-dtype and ``complex`` entries failing to raise ``TypeError`` like their :class:`Series` counterparts (:issue:`28079`) -- Bug in :class:`DataFrame` logical operations (`&`, `|`, `^`) not matching :class:`Series` behavior by filling NA values (:issue:`28741`) -- Bug in :meth:`DataFrame.interpolate` where specifying axis by name references variable before it is assigned (:issue:`29142`) -- Bug in :meth:`Series.var` not computing the right value with a nullable integer dtype series not passing through ddof argument (:issue:`29128`) -- Improved error message when using `frac` > 1 and `replace` = False (:issue:`27451`) -- Bug in numeric indexes resulted in it being possible to instantiate an :class:`Int64Index`, :class:`UInt64Index`, or :class:`Float64Index` with an invalid dtype (e.g. datetime-like) (:issue:`29539`) -- Bug in :class:`UInt64Index` precision loss while constructing from a list with values in the ``np.uint64`` range (:issue:`29526`) -- Bug in :class:`NumericIndex` construction that caused indexing to fail when integers in the ``np.uint64`` range were used (:issue:`28023`) -- Bug in :class:`NumericIndex` construction that caused :class:`UInt64Index` to be casted to :class:`Float64Index` when integers in the ``np.uint64`` range were used to index a :class:`DataFrame` (:issue:`28279`) -- Bug in :meth:`Series.interpolate` when using method=`index` with an unsorted index, would previously return incorrect results. (:issue:`21037`) -- Bug in :meth:`DataFrame.round` where a :class:`DataFrame` with a :class:`CategoricalIndex` of :class:`IntervalIndex` columns would incorrectly raise a ``TypeError`` (:issue:`30063`) -- Bug in :meth:`Series.pct_change` and :meth:`DataFrame.pct_change` when there are duplicated indices (:issue:`30463`) -- Bug in :class:`DataFrame` cumulative operations (e.g. cumsum, cummax) incorrect casting to object-dtype (:issue:`19296`) -- Bug in :class:`~DataFrame.diff` losing the dtype for extension types (:issue:`30889`) -- Bug in :class:`DataFrame.diff` raising an ``IndexError`` when one of the columns was a nullable integer dtype (:issue:`30967`) - -Conversion -^^^^^^^^^^ - -- -- - -Strings -^^^^^^^ - -- Calling :meth:`Series.str.isalnum` (and other "ismethods") on an empty ``Series`` would return an ``object`` dtype instead of ``bool`` (:issue:`29624`) -- - - -Interval -^^^^^^^^ - -- Bug in :meth:`IntervalIndex.get_indexer` where a :class:`Categorical` or :class:`CategoricalIndex` ``target`` would incorrectly raise a ``TypeError`` (:issue:`30063`) -- Bug in ``pandas.core.dtypes.cast.infer_dtype_from_scalar`` where passing ``pandas_dtype=True`` did not infer :class:`IntervalDtype` (:issue:`30337`) -- Bug in :class:`Series` constructor where constructing a ``Series`` from a ``list`` of :class:`Interval` objects resulted in ``object`` dtype instead of :class:`IntervalDtype` (:issue:`23563`) -- Bug in :class:`IntervalDtype` where the ``kind`` attribute was incorrectly set as ``None`` instead of ``"O"`` (:issue:`30568`) -- Bug in :class:`IntervalIndex`, :class:`~arrays.IntervalArray`, and :class:`Series` with interval data where equality comparisons were incorrect (:issue:`24112`) - -Indexing -^^^^^^^^ - -- Bug in assignment using a reverse slicer (:issue:`26939`) -- Bug in :meth:`DataFrame.explode` would duplicate frame in the presence of duplicates in the index (:issue:`28010`) -- Bug in reindexing a :meth:`PeriodIndex` with another type of index that contained a `Period` (:issue:`28323`) (:issue:`28337`) -- Fix assignment of column via `.loc` with numpy non-ns datetime type (:issue:`27395`) -- Bug in :meth:`Float64Index.astype` where ``np.inf`` was not handled properly when casting to an integer dtype (:issue:`28475`) -- :meth:`Index.union` could fail when the left contained duplicates (:issue:`28257`) -- Bug when indexing with ``.loc`` where the index was a :class:`CategoricalIndex` with non-string categories didn't work (:issue:`17569`, :issue:`30225`) -- :meth:`Index.get_indexer_non_unique` could fail with ``TypeError`` in some cases, such as when searching for ints in a string index (:issue:`28257`) -- Bug in :meth:`Float64Index.get_loc` incorrectly raising ``TypeError`` instead of ``KeyError`` (:issue:`29189`) -- Bug in :meth:`DataFrame.loc` with incorrect dtype when setting Categorical value in 1-row DataFrame (:issue:`25495`) -- :meth:`MultiIndex.get_loc` can't find missing values when input includes missing values (:issue:`19132`) -- Bug in :meth:`Series.__setitem__` incorrectly assigning values with boolean indexer when the length of new data matches the number of ``True`` values and new data is not a ``Series`` or an ``np.array`` (:issue:`30567`) -- Bug in indexing with a :class:`PeriodIndex` incorrectly accepting integers representing years, use e.g. ``ser.loc["2007"]`` instead of ``ser.loc[2007]`` (:issue:`30763`) - -Missing -^^^^^^^ - -- -- - -MultiIndex -^^^^^^^^^^ - -- Constructor for :class:`MultiIndex` verifies that the given ``sortorder`` is compatible with the actual ``lexsort_depth`` if ``verify_integrity`` parameter is ``True`` (the default) (:issue:`28735`) -- Series and MultiIndex `.drop` with `MultiIndex` raise exception if labels not in given in level (:issue:`8594`) -- - -I/O -^^^ - -- :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`) -- Bug in :meth:`DataFrame.to_json` where using a Tuple as a column or index value and using ``orient="columns"`` or ``orient="index"`` would produce invalid JSON (:issue:`20500`) -- Improve infinity parsing. :meth:`read_csv` now interprets ``Infinity``, ``+Infinity``, ``-Infinity`` as floating point values (:issue:`10065`) -- Bug in :meth:`DataFrame.to_csv` where values were truncated when the length of ``na_rep`` was shorter than the text input data. (:issue:`25099`) -- Bug in :func:`DataFrame.to_string` where values were truncated using display options instead of outputting the full content (:issue:`9784`) -- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in ISO format with ``orient="table"`` (:issue:`28130`) -- Bug in :func:`DataFrame.to_parquet` where writing to GCS would fail with `engine='fastparquet'` if the file did not already exist (:issue:`28326`) -- Bug in :func:`read_hdf` closing stores that it didn't open when Exceptions are raised (:issue:`28699`) -- Bug in :meth:`DataFrame.read_json` where using ``orient="index"`` would not maintain the order (:issue:`28557`) -- Bug in :meth:`DataFrame.to_html` where the length of the ``formatters`` argument was not verified (:issue:`28469`) -- Bug in :meth:`DataFrame.read_excel` with ``engine='ods'`` when ``sheet_name`` argument references a non-existent sheet (:issue:`27676`) -- Bug in :meth:`pandas.io.formats.style.Styler` formatting for floating values not displaying decimals correctly (:issue:`13257`) -- Bug in :meth:`DataFrame.to_html` when using ``formatters=`` and ``max_cols`` together. (:issue:`25955`) -- Bug in :meth:`Styler.background_gradient` not able to work with dtype ``Int64`` (:issue:`28869`) -- Bug in :meth:`DataFrame.to_clipboard` which did not work reliably in ipython (:issue:`22707`) -- Bug in :func:`read_json` where default encoding was not set to ``utf-8`` (:issue:`29565`) -- Bug in :class:`PythonParser` where str and bytes were being mixed when dealing with the decimal field (:issue:`29650`) -- :meth:`read_gbq` now accepts ``progress_bar_type`` to display progress bar while the data downloads. (:issue:`29857`) -- Bug in :func:`pandas.io.json.json_normalize` where a missing value in the location specified by `record_path` would raise a ``TypeError`` (:issue:`30148`) -- :func:`read_excel` now accepts binary data (:issue:`15914`) -- Bug in :meth:`read_csv` in which encoding handling was limited to just the string `utf-16` for the C engine (:issue:`24130`) - -Plotting -^^^^^^^^ - -- Bug in :meth:`Series.plot` not able to plot boolean values (:issue:`23719`) -- Bug in :meth:`DataFrame.plot` not able to plot when no rows (:issue:`27758`) -- Bug in :meth:`DataFrame.plot` producing incorrect legend markers when plotting multiple series on the same axis (:issue:`18222`) -- Bug in :meth:`DataFrame.plot` when ``kind='box'`` and data contains datetime or timedelta data. These types are now automatically dropped (:issue:`22799`) -- Bug in :meth:`DataFrame.plot.line` and :meth:`DataFrame.plot.area` produce wrong xlim in x-axis (:issue:`27686`, :issue:`25160`, :issue:`24784`) -- Bug where :meth:`DataFrame.boxplot` would not accept a ``color`` parameter like :meth:`DataFrame.plot.box` (:issue:`26214`) -- Bug in the ``xticks`` argument being ignored for :meth:`DataFrame.plot.bar` (:issue:`14119`) -- :func:`set_option` now validates that the plot backend provided to ``'plotting.backend'`` implements the backend when the option is set, rather than when a plot is created (:issue:`28163`) -- :meth:`DataFrame.plot` now allow a ``backend`` keyword argument to allow changing between backends in one session (:issue:`28619`). -- Bug in color validation incorrectly raising for non-color styles (:issue:`29122`). -- Allow :meth:`DataFrame.plot.scatter` to plot ``objects`` and ``datetime`` type data (:issue:`18755`, :issue:`30391`) -- Bug in :meth:`DataFrame.hist`, ``xrot=0`` does not work with ``by`` and subplots (:issue:`30288`). - -Groupby/resample/rolling -^^^^^^^^^^^^^^^^^^^^^^^^ - -- Bug in :meth:`core.groupby.DataFrameGroupBy.apply` only showing output from a single group when function returns an :class:`Index` (:issue:`28652`) -- Bug in :meth:`DataFrame.groupby` with multiple groups where an ``IndexError`` would be raised if any group contained all NA values (:issue:`20519`) -- Bug in :meth:`pandas.core.resample.Resampler.size` and :meth:`pandas.core.resample.Resampler.count` returning wrong dtype when used with an empty :class:`Series` or :class:`DataFrame` (:issue:`28427`) -- Bug in :meth:`DataFrame.rolling` not allowing for rolling over datetimes when ``axis=1`` (:issue:`28192`) -- Bug in :meth:`DataFrame.rolling` not allowing rolling over multi-index levels (:issue:`15584`). -- Bug in :meth:`DataFrame.rolling` not allowing rolling on monotonic decreasing time indexes (:issue:`19248`). -- Bug in :meth:`DataFrame.groupby` not offering selection by column name when ``axis=1`` (:issue:`27614`) -- Bug in :meth:`core.groupby.DataFrameGroupby.agg` not able to use lambda function with named aggregation (:issue:`27519`) -- Bug in :meth:`DataFrame.groupby` losing column name information when grouping by a categorical column (:issue:`28787`) -- Remove error raised due to duplicated input functions in named aggregation in :meth:`DataFrame.groupby` and :meth:`Series.groupby`. Previously error will be raised if the same function is applied on the same column and now it is allowed if new assigned names are different. (:issue:`28426`) -- :meth:`core.groupby.SeriesGroupBy.value_counts` will be able to handle the case even when the :class:`Grouper` makes empty groups (:issue:`28479`) -- Bug in :meth:`core.window.rolling.Rolling.quantile` ignoring ``interpolation`` keyword argument when used within a groupby (:issue:`28779`) -- Bug in :meth:`DataFrame.groupby` where ``any``, ``all``, ``nunique`` and transform functions would incorrectly handle duplicate column labels (:issue:`21668`) -- Bug in :meth:`core.groupby.DataFrameGroupBy.agg` with timezone-aware datetime64 column incorrectly casting results to the original dtype (:issue:`29641`) -- Bug in :meth:`DataFrame.groupby` when using axis=1 and having a single level columns index (:issue:`30208`) -- Bug in :meth:`DataFrame.groupby` when using nunique on axis=1 (:issue:`30253`) -- Bug in :meth:`GroupBy.quantile` with multiple list-like q value and integer column names (:issue:`30289`) -- Bug in :meth:`GroupBy.pct_change` and :meth:`core.groupby.SeriesGroupBy.pct_change` causes ``TypeError`` when ``fill_method`` is ``None`` (:issue:`30463`) -- Bug in :meth:`Rolling.count` and :meth:`Expanding.count` argument where ``min_periods`` was ignored (:issue:`26996`) - -Reshaping -^^^^^^^^^ - -- Bug in :meth:`DataFrame.apply` that caused incorrect output with empty :class:`DataFrame` (:issue:`28202`, :issue:`21959`) -- Bug in :meth:`DataFrame.stack` not handling non-unique indexes correctly when creating MultiIndex (:issue:`28301`) -- Bug in :meth:`pivot_table` not returning correct type ``float`` when ``margins=True`` and ``aggfunc='mean'`` (:issue:`24893`) -- Bug :func:`merge_asof` could not use :class:`datetime.timedelta` for ``tolerance`` kwarg (:issue:`28098`) -- Bug in :func:`merge`, did not append suffixes correctly with MultiIndex (:issue:`28518`) -- :func:`qcut` and :func:`cut` now handle boolean input (:issue:`20303`) -- Fix to ensure all int dtypes can be used in :func:`merge_asof` when using a tolerance value. Previously every non-int64 type would raise an erroneous ``MergeError`` (:issue:`28870`). -- Better error message in :func:`get_dummies` when `columns` isn't a list-like value (:issue:`28383`) -- Bug in :meth:`Index.join` that caused infinite recursion error for mismatched ``MultiIndex`` name orders. (:issue:`25760`, :issue:`28956`) -- Bug :meth:`Series.pct_change` where supplying an anchored frequency would throw a ``ValueError`` (:issue:`28664`) -- Bug where :meth:`DataFrame.equals` returned True incorrectly in some cases when two DataFrames had the same columns in different orders (:issue:`28839`) -- Bug in :meth:`DataFrame.replace` that caused non-numeric replacer's dtype not respected (:issue:`26632`) -- Bug in :func:`melt` where supplying mixed strings and numeric values for ``id_vars`` or ``value_vars`` would incorrectly raise a ``ValueError`` (:issue:`29718`) -- Dtypes are now preserved when transposing a ``DataFrame`` where each column is the same extension dtype (:issue:`30091`) -- Bug in :func:`merge_asof` merging on a tz-aware ``left_index`` and ``right_on`` a tz-aware column (:issue:`29864`) -- Improved error message and docstring in :func:`cut` and :func:`qcut` when `labels=True` (:issue:`13318`) -- Bug in missing `fill_na` parameter to :meth:`DataFrame.unstack` with list of levels (:issue:`30740`) - -Sparse -^^^^^^ -- Bug in :class:`SparseDataFrame` arithmetic operations incorrectly casting inputs to float (:issue:`28107`) -- Bug in ``DataFrame.sparse`` returning a ``Series`` when there was a column named ``sparse`` rather than the accessor (:issue:`30758`) -- Fixed :meth:`operator.xor` with a boolean-dtype ``SparseArray``. Now returns a sparse result, rather than object dtype (:issue:`31025`) - -ExtensionArray -^^^^^^^^^^^^^^ - -- Bug in :class:`arrays.PandasArray` when setting a scalar string (:issue:`28118`, :issue:`28150`). -- Bug where nullable integers could not be compared to strings (:issue:`28930`) -- Bug where :class:`DataFrame` constructor raised ``ValueError`` with list-like data and ``dtype`` specified (:issue:`30280`) - -Other -^^^^^ -- Trying to set the ``display.precision``, ``display.max_rows`` or ``display.max_columns`` using :meth:`set_option` to anything but a ``None`` or a positive int will raise a ``ValueError`` (:issue:`23348`) -- Using :meth:`DataFrame.replace` with overlapping keys in a nested dictionary will no longer raise, now matching the behavior of a flat dictionary (:issue:`27660`) -- :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` now support dicts as ``compression`` argument with key ``'method'`` being the compression method and others as additional compression options when the compression method is ``'zip'``. (:issue:`26023`) -- Bug in :meth:`Series.diff` where a boolean series would incorrectly raise a ``TypeError`` (:issue:`17294`) -- :meth:`Series.append` will no longer raise a ``TypeError`` when passed a tuple of ``Series`` (:issue:`28410`) -- Fix corrupted error message when calling ``pandas.libs._json.encode()`` on a 0d array (:issue:`18878`) -- Backtick quoting in :meth:`DataFrame.query` and :meth:`DataFrame.eval` can now also be used to use invalid identifiers like names that start with a digit, are python keywords, or are using single character operators. (:issue:`27017`) -- Bug in ``pd.core.util.hashing.hash_pandas_object`` where arrays containing tuples were incorrectly treated as non-hashable (:issue:`28969`) -- Bug in :meth:`DataFrame.append` that raised ``IndexError`` when appending with empty list (:issue:`28769`) -- Fix :class:`AbstractHolidayCalendar` to return correct results for - years after 2030 (now goes up to 2200) (:issue:`27790`) -- Fixed :class:`~arrays.IntegerArray` returning ``inf`` rather than ``NaN`` for operations dividing by ``0`` (:issue:`27398`) -- Fixed ``pow`` operations for :class:`~arrays.IntegerArray` when the other value is ``0`` or ``1`` (:issue:`29997`) -- Bug in :meth:`Series.count` raises if use_inf_as_na is enabled (:issue:`29478`) -- Bug in :class:`Index` where a non-hashable name could be set without raising ``TypeError`` (:issue:`29069`) -- Bug in :class:`DataFrame` constructor when passing a 2D ``ndarray`` and an extension dtype (:issue:`12513`) -- Bug in :meth:`DataFrame.to_csv` when supplied a series with a ``dtype="string"`` and a ``na_rep``, the ``na_rep`` was being truncated to 2 characters. (:issue:`29975`) -- Bug where :meth:`DataFrame.itertuples` would incorrectly determine whether or not namedtuples could be used for dataframes of 255 columns (:issue:`28282`) -- Handle nested NumPy ``object`` arrays in :func:`testing.assert_series_equal` for ExtensionArray implementations (:issue:`30841`) -- Bug in :class:`Index` constructor incorrectly allowing 2-dimensional input arrays (:issue:`13601`, :issue:`27125`) - -.. --------------------------------------------------------------------------- - -.. _whatsnew_100.contributors: - -Contributors -~~~~~~~~~~~~ - -.. contributors:: v0.25.3..v1.0.0 +matplotlib directly rather than :meth:`~DataFrame.plot`.

+

To use pandas formatters with a matplotlib plot, specify

+
>>> import pandas as pd
+>>> pd.options.plotting.matplotlib.register_converters = True
+

Note that plots created by :meth:`DataFrame.plot` and :meth:`Series.plot` do register the converters +automatically. The only behavior change is when plotting a date-like object via matplotlib.pyplot.plot +or matplotlib.Axes.plot. See :ref:`plotting.formatters` for more.

+

Other removals

+ + +

Performance improvements

+ + +

Bug fixes

+ +

Categorical

+ + +

Datetimelike

+ + +

Timedelta

+ + +

Timezones

+
    +
  • +
  • +
+ +

Numeric

+ + +

Conversion

+
    +
  • +
  • +
+ +

Strings

+ + +

Interval

+ + +

Indexing

+ + +

Missing

+
    +
  • +
  • +
+ +

MultiIndex

+
    +
  • Constructor for :class:`MultiIndex` verifies that the given sortorder is compatible with the actual lexsort_depth if verify_integrity parameter is True (the default) (:issue:`28735`)
    • +
  • Series and MultiIndex .drop with MultiIndex raise exception if labels not in given in level (:issue:`8594`)
    • +
  • +
+ +

I/O

+ + +

Plotting

+ + +

Groupby/resample/rolling

+ + +

Reshaping

+ + +

Sparse

+ + +

ExtensionArray

+ + +

Other

+ + +

Contributors

+
.. contributors:: v0.25.3..v1.0.0
+
+ +
+
+ +
+ + + +
+ + +
+ + +
 +
+ + + +
+
+ +
+
+ + +
+ + + + + + +
+ + + You can’t perform that action at this time. +
+ + + + + + + + + + + + + + + +
+ + + + From 5010732b51a09dac9981dd12098ead872271c94b Mon Sep 17 00:00:00 2001 From: nbonnin Date: Wed, 4 Mar 2020 22:35:23 -0500 Subject: [PATCH 11/11] Tried downloading directly from github. --- doc/source/whatsnew/v1.0.0.rst | 3435 ++++++++++---------------------- 1 file changed, 1105 insertions(+), 2330 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 77fca5ce0b9c7..6597b764581a4 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -1,2518 +1,1293 @@ +.. _whatsnew_100: +What's new in 1.0.0 (January 29, 2020) +-------------------------------------- +These are the changes in pandas 1.0.0. See :ref:`release` for a full changelog +including other versions of pandas. +.. note:: + The pandas 1.0 release removed a lot of functionality that was deprecated + in previous releases (see :ref:`below ` + for an overview). It is recommended to first upgrade to pandas 0.25 and to + ensure your code is working without warnings, before upgrading to pandas + 1.0. - - - - - - - - - - - +New Deprecation Policy +~~~~~~~~~~~~~~~~~~~~~~ +Starting with Pandas 1.0.0, pandas will adopt a variant of `SemVer`_ to +version releases. Briefly, +* Deprecations will be introduced in minor releases (e.g. 1.1.0, 1.2.0, 2.1.0, ...) +* Deprecations will be enforced in major releases (e.g. 1.0.0, 2.0.0, 3.0.0, ...) +* API-breaking changes will be made only in major releases (except for experimental features) - - - - - - - +See :ref:`policies.version` for more. +.. _2019 Pandas User Survey: http://dev.pandas.io/pandas-blog/2019-pandas-user-survey.html +.. _SemVer: https://semver.org - - - pandas/v1.0.0.rst at master · pandas-dev/pandas - - - - +{{ header }} - - +.. --------------------------------------------------------------------------- - - - +Enhancements +~~~~~~~~~~~~ - +.. _whatsnew_100.numba_rolling_apply: +Using Numba in ``rolling.apply`` and ``expanding.apply`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +We've added an ``engine`` keyword to :meth:`~core.window.rolling.Rolling.apply` and :meth:`~core.window.expanding.Expanding.apply` +that allows the user to execute the routine using `Numba `__ instead of Cython. +Using the Numba engine can yield significant performance gains if the apply function can operate on numpy arrays and +the data set is larger (1 million rows or greater). For more details, see +:ref:`rolling apply documentation ` (:issue:`28987`, :issue:`30936`) - - - - - +.. _whatsnew_100.custom_window: - - - +Defining custom windows for rolling operations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - +We've added a :func:`pandas.api.indexers.BaseIndexer` class that allows users to define how +window bounds are created during ``rolling`` operations. Users can define their own ``get_window_bounds`` +method on a :func:`pandas.api.indexers.BaseIndexer` subclass that will generate the start and end +indices used for each window during the rolling aggregation. For more details and example usage, see +the :ref:`custom window rolling documentation ` +.. _whatsnew_100.to_markdown: +Converting to Markdown +^^^^^^^^^^^^^^^^^^^^^^ - +We've added :meth:`~DataFrame.to_markdown` for creating a markdown table (:issue:`11052`) - +.. ipython:: python - + df = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=['a', 'a', 'b']) + print(df.to_markdown()) +Experimental new features +~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _whatsnew_100.NA: - +Experimental ``NA`` scalar to denote missing values +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - +A new ``pd.NA`` value (singleton) is introduced to represent scalar missing +values. Up to now, pandas used several values to represent missing data: ``np.nan`` is used for this for float data, ``np.nan`` or +``None`` for object-dtype data and ``pd.NaT`` for datetime-like data. The +goal of ``pd.NA`` is to provide a "missing" indicator that can be used +consistently across data types. ``pd.NA`` is currently used by the nullable integer and boolean +data types and the new string data type (:issue:`28095`). - +.. warning:: - + Experimental: the behaviour of ``pd.NA`` can still change without warning. - +For example, creating a Series using the nullable integer dtype: - - +.. ipython:: python - + s = pd.Series([1, 2, None], dtype="Int64") + s + s[2] - +Compared to ``np.nan``, ``pd.NA`` behaves differently in certain operations. +In addition to arithmetic operations, ``pd.NA`` also propagates as "missing" +or "unknown" in comparison operations: - +.. ipython:: python + np.nan > 1 + pd.NA > 1 - +For logical operations, ``pd.NA`` follows the rules of the +`three-valued logic `__ (or +*Kleene logic*). For example: +.. ipython:: python - + pd.NA | True - +For more, see :ref:`NA section ` in the user guide on missing +data. - - - +.. _whatsnew_100.string: +Dedicated string data type +^^^^^^^^^^^^^^^^^^^^^^^^^^ - +We've added :class:`StringDtype`, an extension type dedicated to string data. +Previously, strings were typically stored in object-dtype NumPy arrays. (:issue:`29975`) - +.. warning:: - - + ``StringDtype`` is currently considered experimental. The implementation + and parts of the API may change without warning. -
- Skip to content - - - +The ``'string'`` extension type solves several issues with object-dtype NumPy arrays: - - +1. You can accidentally store a *mixture* of strings and non-strings in an + ``object`` dtype array. A ``StringArray`` can only store strings. +2. ``object`` dtype breaks dtype-specific operations like :meth:`DataFrame.select_dtypes`. + There isn't a clear way to select *just* text while excluding non-text, + but still object-dtype columns. +3. When reading code, the contents of an ``object`` dtype array is less clear + than ``string``. +.. ipython:: python - +We've added :class:`BooleanDtype` / :class:`~arrays.BooleanArray`, an extension +type dedicated to boolean data that can hold missing values. The default +``bool`` data type based on a bool-dtype NumPy array, the column can only hold +``True`` or ``False``, and not missing values. This new :class:`~arrays.BooleanArray` +can store missing values as well by keeping track of this in a separate mask. +(:issue:`29555`, :issue:`30095`, :issue:`31131`) - +.. ipython:: python -
+ pd.Series([True, False, None], dtype=pd.BooleanDtype()) -
+You can use the alias ``"boolean"`` as well. +.. ipython:: python -
+ s = pd.Series([True, False, None], dtype="boolean") + s -
+.. _whatsnew_100.convert_dtypes: +``convert_dtypes`` method to ease use of supported extension dtypes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - +In order to encourage use of the extension dtypes ``StringDtype``, +``BooleanDtype``, ``Int64Dtype``, ``Int32Dtype``, etc., that support ``pd.NA``, the +methods :meth:`DataFrame.convert_dtypes` and :meth:`Series.convert_dtypes` +have been introduced. (:issue:`29752`) (:issue:`30929`) - +Example: +.. ipython:: python + df = pd.DataFrame({'x': ['abc', None, 'def'], + 'y': [1, 2, np.nan], + 'z': [True, False, True]}) + df + df.dtypes +.. ipython:: python -
-
-
- + converted = df.convert_dtypes() + converted + converted.dtypes - +This is especially useful after reading in data using readers such as :func:`read_csv` +and :func:`read_excel`. +See :ref:`here ` for a description. +.. _whatsnew_100.enhancements.other: +Other enhancements +~~~~~~~~~~~~~~~~~~ - +- :meth:`DataFrame.to_string` added the ``max_colwidth`` parameter to control when wide columns are truncated (:issue:`9784`) +- Added the ``na_value`` argument to :meth:`Series.to_numpy`, :meth:`Index.to_numpy` and :meth:`DataFrame.to_numpy` to control the value used for missing data (:issue:`30322`) +- :meth:`MultiIndex.from_product` infers level names from inputs if not explicitly provided (:issue:`27292`) +- :meth:`DataFrame.to_latex` now accepts ``caption`` and ``label`` arguments (:issue:`25436`) +- DataFrames with :ref:`nullable integer `, the :ref:`new string dtype ` + and period data type can now be converted to ``pyarrow`` (>=0.15.0), which means that it is + supported in writing to the Parquet file format when using the ``pyarrow`` engine (:issue:`28368`). + Full roundtrip to parquet (writing and reading back in with :meth:`~DataFrame.to_parquet` / :func:`read_parquet`) + is supported starting with pyarrow >= 0.16 (:issue:`20612`). +- :func:`to_parquet` now appropriately handles the ``schema`` argument for user defined schemas in the pyarrow engine. (:issue:`30270`) +- :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`) +- :meth:`read_stata` can read Stata 119 dta files. (:issue:`28250`) +- Implemented :meth:`pandas.core.window.Window.var` and :meth:`pandas.core.window.Window.std` functions (:issue:`26597`) +- Added ``encoding`` argument to :meth:`DataFrame.to_string` for non-ascii text (:issue:`28766`) +- Added ``encoding`` argument to :func:`DataFrame.to_html` for non-ascii text (:issue:`28663`) +- :meth:`Styler.background_gradient` now accepts ``vmin`` and ``vmax`` arguments (:issue:`12145`) +- :meth:`Styler.format` added the ``na_rep`` parameter to help format the missing values (:issue:`21527`, :issue:`28358`) +- :func:`read_excel` now can read binary Excel (``.xlsb``) files by passing ``engine='pyxlsb'``. For more details and example usage, see the :ref:`Binary Excel files documentation `. Closes :issue:`8540`. +- The ``partition_cols`` argument in :meth:`DataFrame.to_parquet` now accepts a string (:issue:`27117`) +- :func:`pandas.read_json` now parses ``NaN``, ``Infinity`` and ``-Infinity`` (:issue:`12213`) +- DataFrame constructor preserve `ExtensionArray` dtype with `ExtensionArray` (:issue:`11363`) +- :meth:`DataFrame.sort_values` and :meth:`Series.sort_values` have gained ``ignore_index`` keyword to be able to reset index after sorting (:issue:`30114`) +- :meth:`DataFrame.sort_index` and :meth:`Series.sort_index` have gained ``ignore_index`` keyword to reset index (:issue:`30114`) +- :meth:`DataFrame.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`30114`) +- Added new writer for exporting Stata dta files in versions 118 and 119, ``StataWriterUTF8``. These files formats support exporting strings containing Unicode characters. Format 119 supports data sets with more than 32,767 variables (:issue:`23573`, :issue:`30959`) +- :meth:`Series.map` now accepts ``collections.abc.Mapping`` subclasses as a mapper (:issue:`29733`) +- Added an experimental :attr:`~DataFrame.attrs` for storing global metadata about a dataset (:issue:`29062`) +- :meth:`Timestamp.fromisocalendar` is now compatible with python 3.8 and above (:issue:`28115`) +- :meth:`DataFrame.to_pickle` and :func:`read_pickle` now accept URL (:issue:`30163`) +.. --------------------------------------------------------------------------- +.. _whatsnew_100.api_breaking: +Backwards incompatible API changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _whatsnew_100.api_breaking.MultiIndex._names: +Avoid using names from ``MultiIndex.levels`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +As part of a larger refactor to :class:`MultiIndex` the level names are now +stored separately from the levels (:issue:`27242`). We recommend using +:attr:`MultiIndex.names` to access the names, and :meth:`Index.set_names` +to update the names. -
+For backwards compatibility, you can still *access* the names via the levels. -
+.. ipython:: python -
-

- - - / - - pandas - - -

+ mi = pd.MultiIndex.from_product([[1, 2], ['a', 'b']], names=['x', 'y']) + mi.levels[0].name +However, it is no longer possible to *update* the names of the ``MultiIndex`` +via the level. -
- -
    - -
  • - -
    • - - -
  • - -
    - - - Used by -
    -     - -
    - Loading... -

    - Loading dependents... -

    -
    -     -
    - - -
    • - -
  • - -
    - -
    - - - Watch - - -
    - Notifications -
    -
    - - - - - - - -
    -     -
    - -
    -
    • - -
  • -
    -
    - - -
    -
    - - -
    - -
    • - -
  • -
    - - Fork - -
    - -

    Fork pandas

    -
    -
    - - -

    If this dialog fails to load, you can visit the fork page directly.

    -     -
    -     -
    - - -
    • -
- -
- - - -
- -
- - -
- - - - - - -
-
- - - - - - - - - Permalink - - - - -
- - -
- - Branch: - master - - - - -
- - - -
- -
- -
- - Find file - - - Copy path - -
- -

- pandas/doc/source/whatsnew/v1.0.0.rst -

- -
- - Find file - - - Copy path - -
-
- - - - - - -
-
- - @jorisvandenbossche - jorisvandenbossche - - DOC: move whatnew entry for invert from 1.0.0 t 1.0.1 (#31494) - - - - db4da03 - Jan 31, 2020 - -
- -
-
- - 115 contributors - - -
- -

- Users who have contributed to this file -

-
- - -
- - - @jbrockmendel - - @TomAugspurger - - @jorisvandenbossche - - @charlesdong1991 - - @topper-123 - - @MarcoGorelli - - @jschendel - - @mroeschke - - @dsaxton - - @WillAyd - - @fujiaxiang - - @ryankarlos - - @proost - - @christopherzimmerman - - @Dr-Irv - - @immaxchen - - @gfyoung - - @bashtage - - @Unprocessable - - @simonjayhawkins - - @OliverHofkens - - @oguzhanogreden - - @nlepleux - - @SaturnFromTitan - - @makbigc - - @JustinZhengBC - - - - -
-
- - - - - - -
- -
-
- - executable file - - 1293 lines (966 sloc) - - 71.6 KB -
- -
- -
- Raw - Blame - History -
- - -
- - - - -
- -
-
- -
-
-
- - - - - -
-

What's new in 1.0.0 (January 29, 2020)

-

These are the changes in pandas 1.0.0. See :ref:`release` for a full changelog -including other versions of pandas.

-
-

Note

-

The pandas 1.0 release removed a lot of functionality that was deprecated -in previous releases (see :ref:`below <whatsnew_100.prior_deprecations>` -for an overview). It is recommended to first upgrade to pandas 0.25 and to -ensure your code is working without warnings, before upgrading to pandas -1.0.

-
- -

New Deprecation Policy

-

Starting with Pandas 1.0.0, pandas will adopt a variant of SemVer to -version releases. Briefly,

-
    -
  • Deprecations will be introduced in minor releases (e.g. 1.1.0, 1.2.0, 2.1.0, ...)
    • -
  • Deprecations will be enforced in major releases (e.g. 1.0.0, 2.0.0, 3.0.0, ...)
    • -
  • API-breaking changes will be made only in major releases (except for experimental features)
    • -
-

See :ref:`policies.version` for more.

-

{{ header }}

- -

Enhancements

- -

Using Numba in rolling.apply and expanding.apply

-

We've added an engine keyword to :meth:`~core.window.rolling.Rolling.apply` and :meth:`~core.window.expanding.Expanding.apply` -that allows the user to execute the routine using Numba instead of Cython. -Using the Numba engine can yield significant performance gains if the apply function can operate on numpy arrays and -the data set is larger (1 million rows or greater). For more details, see -:ref:`rolling apply documentation <stats.rolling_apply>` (:issue:`28987`, :issue:`30936`)

- -

Defining custom windows for rolling operations

-

We've added a :func:`pandas.api.indexers.BaseIndexer` class that allows users to define how -window bounds are created during rolling operations. Users can define their own get_window_bounds -method on a :func:`pandas.api.indexers.BaseIndexer` subclass that will generate the start and end -indices used for each window during the rolling aggregation. For more details and example usage, see -the :ref:`custom window rolling documentation <stats.custom_rolling_window>`

- -

Converting to Markdown

-

We've added :meth:`~DataFrame.to_markdown` for creating a markdown table (:issue:`11052`)

-
.. ipython:: python
-
-   df = pd.DataFrame({"A": [1, 2, 3], "B": [1, 2, 3]}, index=['a', 'a', 'b'])
-   print(df.to_markdown())
+.. ipython:: python
+   :okexcept:
 
-
- -

Experimental new features

- -

Experimental NA scalar to denote missing values

-

A new pd.NA value (singleton) is introduced to represent scalar missing -values. Up to now, pandas used several values to represent missing data: np.nan is used for this for float data, np.nan or -None for object-dtype data and pd.NaT for datetime-like data. The -goal of pd.NA is to provide a "missing" indicator that can be used -consistently across data types. pd.NA is currently used by the nullable integer and boolean -data types and the new string data type (:issue:`28095`).

-
-

Warning

-

Experimental: the behaviour of pd.NA can still change without warning.

-
-

For example, creating a Series using the nullable integer dtype:

-
.. ipython:: python
+   mi.levels[0].name = "new name"
+   mi.names
 
-    s = pd.Series([1, 2, None], dtype="Int64")
-    s
-    s[2]
+To update, use ``MultiIndex.set_names``, which returns a new ``MultiIndex``.
 
-
-

Compared to np.nan, pd.NA behaves differently in certain operations. -In addition to arithmetic operations, pd.NA also propagates as "missing" -or "unknown" in comparison operations:

-
.. ipython:: python
+.. ipython:: python
 
-    np.nan > 1
-    pd.NA > 1
+   mi2 = mi.set_names("new name", level=0)
+   mi2.names
 
-
-

For logical operations, pd.NA follows the rules of the -three-valued logic (or -Kleene logic). For example:

-
.. ipython:: python
+New repr for :class:`~pandas.arrays.IntervalArray`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-    pd.NA | True
+:class:`pandas.arrays.IntervalArray` adopts a new ``__repr__`` in accordance with other array classes (:issue:`25022`)
 
-
-

For more, see :ref:`NA section <missing_data.NA>` in the user guide on missing -data.

- -

Dedicated string data type

-

We've added :class:`StringDtype`, an extension type dedicated to string data. -Previously, strings were typically stored in object-dtype NumPy arrays. (:issue:`29975`)

-
-

Warning

-

StringDtype is currently considered experimental. The implementation -and parts of the API may change without warning.

-
-

The 'string' extension type solves several issues with object-dtype NumPy arrays:

-
    -
  1. You can accidentally store a mixture of strings and non-strings in an -object dtype array. A StringArray can only store strings.
    2. -
  2. object dtype breaks dtype-specific operations like :meth:`DataFrame.select_dtypes`. -There isn't a clear way to select just text while excluding non-text, -but still object-dtype columns.
    3. -
  3. When reading code, the contents of an object dtype array is less clear -than string.
    4. -
-
.. ipython:: python
+*pandas 0.25.x*
 
-   pd.Series(['abc', None, 'def'], dtype=pd.StringDtype())
+.. code-block:: ipython
 
-
-

You can use the alias "string" as well.

-
.. ipython:: python
+   In [1]: pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
+   Out[2]:
+   IntervalArray([(0, 1], (2, 3]],
+                 closed='right',
+                 dtype='interval[int64]')
 
-   s = pd.Series(['abc', None, 'def'], dtype="string")
-   s
+*pandas 1.0.0*
 
-
-

The usual string accessor methods work. Where appropriate, the return type -of the Series or columns of a DataFrame will also have string dtype.

-
.. ipython:: python
+.. ipython:: python
 
-   s.str.upper()
-   s.str.split('b', expand=True).dtypes
+   pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
 
-
-

String accessor methods returning integers will return a value with :class:`Int64Dtype`

-
.. ipython:: python
+``DataFrame.rename`` now only accepts one positional argument
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-   s.str.count("a")
+:meth:`DataFrame.rename` would previously accept positional arguments that would lead
+to ambiguous or undefined behavior. From pandas 1.0, only the very first argument, which
+maps labels to their new names along the default axis, is allowed to be passed by position
+(:issue:`29136`).
 
-
-

We recommend explicitly using the string data type when working with strings. -See :ref:`text.types` for more.

- -

Boolean data type with missing values support

-

We've added :class:`BooleanDtype` / :class:`~arrays.BooleanArray`, an extension -type dedicated to boolean data that can hold missing values. The default -bool data type based on a bool-dtype NumPy array, the column can only hold -True or False, and not missing values. This new :class:`~arrays.BooleanArray` -can store missing values as well by keeping track of this in a separate mask. -(:issue:`29555`, :issue:`30095`, :issue:`31131`)

-
.. ipython:: python
+.. ipython:: python
+   :suppress:
 
-   pd.Series([True, False, None], dtype=pd.BooleanDtype())
+   df = pd.DataFrame([[1]])
 
-
-

You can use the alias "boolean" as well.

-
.. ipython:: python
+*pandas 0.25.x*
 
-   s = pd.Series([True, False, None], dtype="boolean")
-   s
+.. code-block:: python
 
-
- -

convert_dtypes method to ease use of supported extension dtypes

-

In order to encourage use of the extension dtypes StringDtype, -BooleanDtype, Int64Dtype, Int32Dtype, etc., that support pd.NA, the -methods :meth:`DataFrame.convert_dtypes` and :meth:`Series.convert_dtypes` -have been introduced. (:issue:`29752`) (:issue:`30929`)

-

Example:

-
.. ipython:: python
+   >>> df = pd.DataFrame([[1]])
+   >>> df.rename({0: 1}, {0: 2})
+   FutureWarning: ...Use named arguments to resolve ambiguity...
+      2
+   1  1
 
-   df = pd.DataFrame({'x': ['abc', None, 'def'],
-                      'y': [1, 2, np.nan],
-                      'z': [True, False, True]})
-   df
-   df.dtypes
+*pandas 1.0.0*
 
-
-
.. ipython:: python
+.. code-block:: python
 
-   converted = df.convert_dtypes()
-   converted
-   converted.dtypes
+   >>> df.rename({0: 1}, {0: 2})
+   Traceback (most recent call last):
+   ...
+   TypeError: rename() takes from 1 to 2 positional arguments but 3 were given
 
-
-

This is especially useful after reading in data using readers such as :func:`read_csv` -and :func:`read_excel`. -See :ref:`here <missing_data.NA.conversion>` for a description.

- -

Other enhancements

- - -

Backwards incompatible API changes

- -

Avoid using names from MultiIndex.levels

-

As part of a larger refactor to :class:`MultiIndex` the level names are now -stored separately from the levels (:issue:`27242`). We recommend using -:attr:`MultiIndex.names` to access the names, and :meth:`Index.set_names` -to update the names.

-

For backwards compatibility, you can still access the names via the levels.

-
.. ipython:: python
+Note that errors will now be raised when conflicting or potentially ambiguous arguments are provided.
 
-   mi = pd.MultiIndex.from_product([[1, 2], ['a', 'b']], names=['x', 'y'])
-   mi.levels[0].name
+*pandas 0.25.x*
 
-
-

However, it is no longer possible to update the names of the MultiIndex -via the level.

-
.. ipython:: python
-   :okexcept:
+.. code-block:: python
 
-   mi.levels[0].name = "new name"
-   mi.names
+   >>> df.rename({0: 1}, index={0: 2})
+      0
+   1  1
 
-
-

To update, use MultiIndex.set_names, which returns a new MultiIndex.

-
.. ipython:: python
+   >>> df.rename(mapper={0: 1}, index={0: 2})
+      0
+   2  1
 
-   mi2 = mi.set_names("new name", level=0)
-   mi2.names
+*pandas 1.0.0*
 
-
- -

New repr for :class:`~pandas.arrays.IntervalArray`

-

:class:`pandas.arrays.IntervalArray` adopts a new __repr__ in accordance with other array classes (:issue:`25022`)

-

pandas 0.25.x

-
In [1]: pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
-Out[2]:
-IntervalArray([(0, 1], (2, 3]],
-              closed='right',
-              dtype='interval[int64]')
-
-

pandas 1.0.0

-
.. ipython:: python
+.. code-block:: python
 
-   pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
+   >>> df.rename({0: 1}, index={0: 2})
+   Traceback (most recent call last):
+   ...
+   TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
 
-
- -

DataFrame.rename now only accepts one positional argument

-

:meth:`DataFrame.rename` would previously accept positional arguments that would lead -to ambiguous or undefined behavior. From pandas 1.0, only the very first argument, which -maps labels to their new names along the default axis, is allowed to be passed by position -(:issue:`29136`).

-
.. ipython:: python
-   :suppress:
+   >>> df.rename(mapper={0: 1}, index={0: 2})
+   Traceback (most recent call last):
+   ...
+   TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
 
-   df = pd.DataFrame([[1]])
+You can still change the axis along which the first positional argument is applied by
+supplying the ``axis`` keyword argument.
 
-
-

pandas 0.25.x

-
>>> df = pd.DataFrame([[1]])
->>> df.rename({0: 1}, {0: 2})
-FutureWarning: ...Use named arguments to resolve ambiguity...
-   2
-1  1
-

pandas 1.0.0

-
>>> df.rename({0: 1}, {0: 2})
-Traceback (most recent call last):
-...
-TypeError: rename() takes from 1 to 2 positional arguments but 3 were given
-

Note that errors will now be raised when conflicting or potentially ambiguous arguments are provided.

-

pandas 0.25.x

-
>>> df.rename({0: 1}, index={0: 2})
-   0
-1  1
-
->>> df.rename(mapper={0: 1}, index={0: 2})
-   0
-2  1
-

pandas 1.0.0

-
>>> df.rename({0: 1}, index={0: 2})
-Traceback (most recent call last):
-...
-TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
-
->>> df.rename(mapper={0: 1}, index={0: 2})
-Traceback (most recent call last):
-...
-TypeError: Cannot specify both 'mapper' and any of 'index' or 'columns'
-

You can still change the axis along which the first positional argument is applied by -supplying the axis keyword argument.

-
.. ipython:: python
+.. ipython:: python
 
    df.rename({0: 1})
    df.rename({0: 1}, axis=1)
 
-
-

If you would like to update both the index and column labels, be sure to use the respective -keywords.

-
.. ipython:: python
+If you would like to update both the index and column labels, be sure to use the respective
+keywords.
+
+.. ipython:: python
 
    df.rename(index={0: 1}, columns={0: 2})
 
-
- -

Extended verbose info output for :class:`~pandas.DataFrame`

-

:meth:`DataFrame.info` now shows line numbers for the columns summary (:issue:`17304`)

-

pandas 0.25.x

-
>>> df = pd.DataFrame({"int_col": [1, 2, 3],
-...                    "text_col": ["a", "b", "c"],
-...                    "float_col": [0.0, 0.1, 0.2]})
->>> df.info(verbose=True)
-<class 'pandas.core.frame.DataFrame'>
-RangeIndex: 3 entries, 0 to 2
-Data columns (total 3 columns):
-int_col      3 non-null int64
-text_col     3 non-null object
-float_col    3 non-null float64
-dtypes: float64(1), int64(1), object(1)
-memory usage: 152.0+ bytes
-

pandas 1.0.0

-
.. ipython:: python
+Extended verbose info output for :class:`~pandas.DataFrame`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:meth:`DataFrame.info` now shows line numbers for the columns summary (:issue:`17304`)
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> df = pd.DataFrame({"int_col": [1, 2, 3],
+   ...                    "text_col": ["a", "b", "c"],
+   ...                    "float_col": [0.0, 0.1, 0.2]})
+   >>> df.info(verbose=True)
+   
+   RangeIndex: 3 entries, 0 to 2
+   Data columns (total 3 columns):
+   int_col      3 non-null int64
+   text_col     3 non-null object
+   float_col    3 non-null float64
+   dtypes: float64(1), int64(1), object(1)
+   memory usage: 152.0+ bytes
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    df = pd.DataFrame({"int_col": [1, 2, 3],
                       "text_col": ["a", "b", "c"],
                       "float_col": [0.0, 0.1, 0.2]})
    df.info(verbose=True)
 
-
- -

:meth:`pandas.array` inference changes

-

:meth:`pandas.array` now infers pandas' new extension types in several cases (:issue:`29791`):

-
    -
  1. String data (including missing values) now returns a :class:`arrays.StringArray`.
    2. -
  2. Integer data (including missing values) now returns a :class:`arrays.IntegerArray`.
    3. -
  3. Boolean data (including missing values) now returns the new :class:`arrays.BooleanArray`
    4. -
-

pandas 0.25.x

-
>>> pd.array(["a", None])
-<PandasArray>
-['a', None]
-Length: 2, dtype: object
-
->>> pd.array([1, None])
-<PandasArray>
-[1, None]
-Length: 2, dtype: object
-

pandas 1.0.0

-
.. ipython:: python
+:meth:`pandas.array` inference changes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:meth:`pandas.array` now infers pandas' new extension types in several cases (:issue:`29791`):
+
+1. String data (including missing values) now returns a :class:`arrays.StringArray`.
+2. Integer data (including missing values) now returns a :class:`arrays.IntegerArray`.
+3. Boolean data (including missing values) now returns the new :class:`arrays.BooleanArray`
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> pd.array(["a", None])
+   
+   ['a', None]
+   Length: 2, dtype: object
+
+   >>> pd.array([1, None])
+   
+   [1, None]
+   Length: 2, dtype: object
+
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    pd.array(["a", None])
    pd.array([1, None])
 
-
-

As a reminder, you can specify the dtype to disable all inference.

- -

:class:`arrays.IntegerArray` now uses :attr:`pandas.NA`

-

:class:`arrays.IntegerArray` now uses :attr:`pandas.NA` rather than -:attr:`numpy.nan` as its missing value marker (:issue:`29964`).

-

pandas 0.25.x

-
>>> a = pd.array([1, 2, None], dtype="Int64")
->>> a
-<IntegerArray>
-[1, 2, NaN]
-Length: 3, dtype: Int64
-
->>> a[2]
-nan
-

pandas 1.0.0

-
.. ipython:: python
+As a reminder, you can specify the ``dtype`` to disable all inference.
+
+:class:`arrays.IntegerArray` now uses :attr:`pandas.NA`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:class:`arrays.IntegerArray` now uses :attr:`pandas.NA` rather than
+:attr:`numpy.nan` as its missing value marker (:issue:`29964`).
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> a = pd.array([1, 2, None], dtype="Int64")
+   >>> a
+   
+   [1, 2, NaN]
+   Length: 3, dtype: Int64
+
+   >>> a[2]
+   nan
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    a = pd.array([1, 2, None], dtype="Int64")
    a
    a[2]
 
-
-

This has a few API-breaking consequences.

-

Converting to a NumPy ndarray

-

When converting to a NumPy array missing values will be pd.NA, which cannot -be converted to a float. So calling np.asarray(integer_array, dtype="float") -will now raise.

-

pandas 0.25.x

-
>>> np.asarray(a, dtype="float")
-array([ 1.,  2., nan])
-

pandas 1.0.0

-
.. ipython:: python
+This has a few API-breaking consequences.
+
+**Converting to a NumPy ndarray**
+
+When converting to a NumPy array missing values will be ``pd.NA``, which cannot
+be converted to a float. So calling ``np.asarray(integer_array, dtype="float")``
+will now raise.
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+    >>> np.asarray(a, dtype="float")
+    array([ 1.,  2., nan])
+
+*pandas 1.0.0*
+
+.. ipython:: python
    :okexcept:
 
    np.asarray(a, dtype="float")
 
-
-

Use :meth:`arrays.IntegerArray.to_numpy` with an explicit na_value instead.

-
.. ipython:: python
+Use :meth:`arrays.IntegerArray.to_numpy` with an explicit ``na_value`` instead.
+
+.. ipython:: python
 
    a.to_numpy(dtype="float", na_value=np.nan)
 
-
-

Reductions can return ``pd.NA``

-

When performing a reduction such as a sum with skipna=False, the result -will now be pd.NA instead of np.nan in presence of missing values -(:issue:`30958`).

-

pandas 0.25.x

-
>>> pd.Series(a).sum(skipna=False)
-nan
-

pandas 1.0.0

-
.. ipython:: python
+**Reductions can return ``pd.NA``**
+
+When performing a reduction such as a sum with ``skipna=False``, the result
+will now be ``pd.NA`` instead of ``np.nan`` in presence of missing values
+(:issue:`30958`).
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+    >>> pd.Series(a).sum(skipna=False)
+    nan
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    pd.Series(a).sum(skipna=False)
 
-
-

value_counts returns a nullable integer dtype

-

:meth:`Series.value_counts` with a nullable integer dtype now returns a nullable -integer dtype for the values.

-

pandas 0.25.x

-
>>> pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype
-dtype('int64')
-

pandas 1.0.0

-
.. ipython:: python
+**value_counts returns a nullable integer dtype**
+
+:meth:`Series.value_counts` with a nullable integer dtype now returns a nullable
+integer dtype for the values.
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype
+   dtype('int64')
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    pd.Series([2, 1, 1, None], dtype="Int64").value_counts().dtype
 
-
-

See :ref:`missing_data.NA` for more on the differences between :attr:`pandas.NA` -and :attr:`numpy.nan`.

- -

:class:`arrays.IntegerArray` comparisons return :class:`arrays.BooleanArray`

-

Comparison operations on a :class:`arrays.IntegerArray` now returns a -:class:`arrays.BooleanArray` rather than a NumPy array (:issue:`29964`).

-

pandas 0.25.x

-
>>> a = pd.array([1, 2, None], dtype="Int64")
->>> a
-<IntegerArray>
-[1, 2, NaN]
-Length: 3, dtype: Int64
-
->>> a > 1
-array([False,  True, False])
-

pandas 1.0.0

-
.. ipython:: python
+See :ref:`missing_data.NA` for more on the differences between :attr:`pandas.NA`
+and :attr:`numpy.nan`.
+
+:class:`arrays.IntegerArray` comparisons return :class:`arrays.BooleanArray`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Comparison operations on a :class:`arrays.IntegerArray` now returns a
+:class:`arrays.BooleanArray` rather than a NumPy array (:issue:`29964`).
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> a = pd.array([1, 2, None], dtype="Int64")
+   >>> a
+   
+   [1, 2, NaN]
+   Length: 3, dtype: Int64
+
+   >>> a > 1
+   array([False,  True, False])
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    a = pd.array([1, 2, None], dtype="Int64")
-   a > 1
-
-
-

Note that missing values now propagate, rather than always comparing unequal -like :attr:`numpy.nan`. See :ref:`missing_data.NA` for more.

- -

By default :meth:`Categorical.min` now returns the minimum instead of np.nan

-

When :class:`Categorical` contains np.nan, -:meth:`Categorical.min` no longer return np.nan by default (skipna=True) (:issue:`25303`)

-

pandas 0.25.x

-
In [1]: pd.Categorical([1, 2, np.nan], ordered=True).min()
-Out[1]: nan
-
-

pandas 1.0.0

-
.. ipython:: python
+   a > 1
+
+Note that missing values now propagate, rather than always comparing unequal
+like :attr:`numpy.nan`. See :ref:`missing_data.NA` for more.
+
+By default :meth:`Categorical.min` now returns the minimum instead of np.nan
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When :class:`Categorical` contains ``np.nan``,
+:meth:`Categorical.min` no longer return ``np.nan`` by default (skipna=True) (:issue:`25303`)
+
+*pandas 0.25.x*
+
+.. code-block:: ipython
+
+   In [1]: pd.Categorical([1, 2, np.nan], ordered=True).min()
+   Out[1]: nan
+
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    pd.Categorical([1, 2, np.nan], ordered=True).min()
 
 
-
- -

Default dtype of empty :class:`pandas.Series`

-

Initialising an empty :class:`pandas.Series` without specifying a dtype will raise a DeprecationWarning now -(:issue:`17261`). The default dtype will change from float64 to object in future releases so that it is -consistent with the behaviour of :class:`DataFrame` and :class:`Index`.

-

pandas 1.0.0

-
In [1]: pd.Series()
-Out[2]:
-DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.
-Series([], dtype: float64)
-
- -

Result dtype inference changes for resample operations

-

The rules for the result dtype in :meth:`DataFrame.resample` aggregations have changed for extension types (:issue:`31359`). +Default dtype of empty :class:`pandas.Series` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Initialising an empty :class:`pandas.Series` without specifying a dtype will raise a `DeprecationWarning` now +(:issue:`17261`). The default dtype will change from ``float64`` to ``object`` in future releases so that it is +consistent with the behaviour of :class:`DataFrame` and :class:`Index`. + +*pandas 1.0.0* + +.. code-block:: ipython + + In [1]: pd.Series() + Out[2]: + DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning. + Series([], dtype: float64) + +Result dtype inference changes for resample operations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The rules for the result dtype in :meth:`DataFrame.resample` aggregations have changed for extension types (:issue:`31359`). Previously, pandas would attempt to convert the result back to the original dtype, falling back to the usual inference rules if that was not possible. Now, pandas will only return a result of the original dtype if the -scalar values in the result are instances of the extension dtype's scalar type.

-
.. ipython:: python
+scalar values in the result are instances of the extension dtype's scalar type.
+
+.. ipython:: python
 
    df = pd.DataFrame({"A": ['a', 'b']}, dtype='category',
                      index=pd.date_range('2000', periods=2))
    df
 
 
-
-

pandas 0.25.x

-
>>> df.resample("2D").agg(lambda x: 'a').A.dtype
-CategoricalDtype(categories=['a', 'b'], ordered=False)
-

pandas 1.0.0

-
.. ipython:: python
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> df.resample("2D").agg(lambda x: 'a').A.dtype
+   CategoricalDtype(categories=['a', 'b'], ordered=False)
+
+*pandas 1.0.0*
+
+.. ipython:: python
 
    df.resample("2D").agg(lambda x: 'a').A.dtype
 
-
-

This fixes an inconsistency between resample and groupby. -This also fixes a potential bug, where the values of the result might change -depending on how the results are cast back to the original dtype.

-

pandas 0.25.x

-
>>> df.resample("2D").agg(lambda x: 'c')
+This fixes an inconsistency between ``resample`` and ``groupby``.
+This also fixes a potential bug, where the **values** of the result might change
+depending on how the results are cast back to the original dtype.
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> df.resample("2D").agg(lambda x: 'c')
+
+        A
+   0  NaN
+
+*pandas 1.0.0*
 
-     A
-0  NaN
-

pandas 1.0.0

-
.. ipython:: python
+.. ipython:: python
 
    df.resample("2D").agg(lambda x: 'c')
 
 
-
- -

Increased minimum version for Python

-

Pandas 1.0.0 supports Python 3.6.1 and higher (:issue:`29212`).

- -

Increased minimum versions for dependencies

-

Some minimum supported versions of dependencies were updated (:issue:`29766`, :issue:`29723`). -If installed, we now require:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageMinimum VersionRequiredChanged
numpy1.13.3X 
pytz2015.4X 
python-dateutil2.6.1X 
bottleneck1.2.1  
numexpr2.6.2  
pytest (dev)4.0.2  
-

For optional libraries the general recommendation is to use the latest version. +.. _whatsnew_100.api_breaking.python: + +Increased minimum version for Python +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Pandas 1.0.0 supports Python 3.6.1 and higher (:issue:`29212`). + +.. _whatsnew_100.api_breaking.deps: + +Increased minimum versions for dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some minimum supported versions of dependencies were updated (:issue:`29766`, :issue:`29723`). +If installed, we now require: + ++-----------------+-----------------+----------+---------+ +| Package | Minimum Version | Required | Changed | ++=================+=================+==========+=========+ +| numpy | 1.13.3 | X | | ++-----------------+-----------------+----------+---------+ +| pytz | 2015.4 | X | | ++-----------------+-----------------+----------+---------+ +| python-dateutil | 2.6.1 | X | | ++-----------------+-----------------+----------+---------+ +| bottleneck | 1.2.1 | | | ++-----------------+-----------------+----------+---------+ +| numexpr | 2.6.2 | | | ++-----------------+-----------------+----------+---------+ +| pytest (dev) | 4.0.2 | | | ++-----------------+-----------------+----------+---------+ + +For `optional libraries `_ the general recommendation is to use the latest version. The following table lists the lowest version per library that is currently being tested throughout the development of pandas. -Optional libraries below the lowest tested version may still work, but are not considered supported.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageMinimum VersionChanged
beautifulsoup44.6.0 
fastparquet0.3.2X
gcsfs0.2.2 
lxml3.8.0 
matplotlib2.2.2 
numba0.46.0X
openpyxl2.5.7X
pyarrow0.13.0X
pymysql0.7.1 
pytables3.4.2 
s3fs0.3.0X
scipy0.19.0 
sqlalchemy1.1.4 
xarray0.8.2 
xlrd1.1.0 
xlsxwriter0.9.8 
xlwt1.2.0 
-

See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.

- -

Build Changes

-

Pandas has added a pyproject.toml file and will no longer include -cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing +Optional libraries below the lowest tested version may still work, but are not considered supported. + ++-----------------+-----------------+---------+ +| Package | Minimum Version | Changed | ++=================+=================+=========+ +| beautifulsoup4 | 4.6.0 | | ++-----------------+-----------------+---------+ +| fastparquet | 0.3.2 | X | ++-----------------+-----------------+---------+ +| gcsfs | 0.2.2 | | ++-----------------+-----------------+---------+ +| lxml | 3.8.0 | | ++-----------------+-----------------+---------+ +| matplotlib | 2.2.2 | | ++-----------------+-----------------+---------+ +| numba | 0.46.0 | X | ++-----------------+-----------------+---------+ +| openpyxl | 2.5.7 | X | ++-----------------+-----------------+---------+ +| pyarrow | 0.13.0 | X | ++-----------------+-----------------+---------+ +| pymysql | 0.7.1 | | ++-----------------+-----------------+---------+ +| pytables | 3.4.2 | | ++-----------------+-----------------+---------+ +| s3fs | 0.3.0 | X | ++-----------------+-----------------+---------+ +| scipy | 0.19.0 | | ++-----------------+-----------------+---------+ +| sqlalchemy | 1.1.4 | | ++-----------------+-----------------+---------+ +| xarray | 0.8.2 | | ++-----------------+-----------------+---------+ +| xlrd | 1.1.0 | | ++-----------------+-----------------+---------+ +| xlsxwriter | 0.9.8 | | ++-----------------+-----------------+---------+ +| xlwt | 1.2.0 | | ++-----------------+-----------------+---------+ + +See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more. + +Build Changes +^^^^^^^^^^^^^ + +Pandas has added a `pyproject.toml `_ file and will no longer include +cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing a built distribution (wheel) or via conda, this shouldn't have any effect on you. If you're building pandas from -source, you should no longer need to install Cython into your build environment before calling pip install pandas.

- -

Other API changes

- - -

Documentation Improvements

- - -

Deprecations

- -

Selecting Columns from a Grouped DataFrame

-

When selecting columns from a :class:`DataFrameGroupBy` object, passing individual keys (or a tuple of keys) inside single brackets is deprecated, -a list of items should be used instead. (:issue:`23566`) For example:

-
df = pd.DataFrame({
-    "A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"],
-    "B": np.random.randn(8),
-    "C": np.random.randn(8),
-})
-g = df.groupby('A')
-
-# single key, returns SeriesGroupBy
-g['B']
-
-# tuple of single key, returns SeriesGroupBy
-g[('B',)]
-
-# tuple of multiple keys, returns DataFrameGroupBy, raises FutureWarning
-g[('B', 'C')]
-
-# multiple keys passed directly, returns DataFrameGroupBy, raises FutureWarning
-# (implicitly converts the passed strings into a single tuple)
-g['B', 'C']
-
-# proper way, returns DataFrameGroupBy
-g[['B', 'C']]
-
- -

Removal of prior version deprecations/changes

-

Removed SparseSeries and SparseDataFrame

-

SparseSeries, SparseDataFrame and the DataFrame.to_sparse method -have been removed (:issue:`28425`). We recommend using a Series or -DataFrame with sparse values instead. See :ref:`sparse.migration` for help -with migrating existing code.

-

Matplotlib unit registration

-

Previously, pandas would register converters with matplotlib as a side effect of importing pandas (:issue:`18720`). -This changed the output of plots made via matplotlib plots after pandas was imported, even if you were using -matplotlib directly rather than :meth:`~DataFrame.plot`.

-

To use pandas formatters with a matplotlib plot, specify

-
>>> import pandas as pd
->>> pd.options.plotting.matplotlib.register_converters = True
-

Note that plots created by :meth:`DataFrame.plot` and :meth:`Series.plot` do register the converters -automatically. The only behavior change is when plotting a date-like object via matplotlib.pyplot.plot -or matplotlib.Axes.plot. See :ref:`plotting.formatters` for more.

-

Other removals

- - -

Performance improvements

- - -

Bug fixes

- -

Categorical

- - -

Datetimelike

- - -

Timedelta

- - -

Timezones

-
    -
  • -
  • -
- -

Numeric

- - -

Conversion

-
    -
  • -
  • -
- -

Strings

- - -

Interval

- - -

Indexing

- - -

Missing

-
    -
  • -
  • -
- -

MultiIndex

-
    -
  • Constructor for :class:`MultiIndex` verifies that the given sortorder is compatible with the actual lexsort_depth if verify_integrity parameter is True (the default) (:issue:`28735`)
    • -
  • Series and MultiIndex .drop with MultiIndex raise exception if labels not in given in level (:issue:`8594`)
    • -
  • -
- -

I/O

- - -

Plotting

- - -

Groupby/resample/rolling

- - -

Reshaping

- - -

Sparse

- - -

ExtensionArray

- - -

Other

- - -

Contributors

-
.. contributors:: v0.25.3..v1.0.0
-
- -
-
- -
- - - -
- - -
- - -
 -
- - - -
-
- -
-
- - -
- - - - - - -
- - - You can’t perform that action at this time. -
- - - - - - - - - - - - - - - -
- - - +source, you should no longer need to install Cython into your build environment before calling ``pip install pandas``. + + +.. _whatsnew_100.api.other: + +Other API changes +^^^^^^^^^^^^^^^^^ + +- :class:`core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`) +- :meth:`pandas.api.types.infer_dtype` will now return "integer-na" for integer and ``np.nan`` mix (:issue:`27283`) +- :meth:`MultiIndex.from_arrays` will no longer infer names from arrays if ``names=None`` is explicitly provided (:issue:`27292`) +- In order to improve tab-completion, Pandas does not include most deprecated attributes when introspecting a pandas object using ``dir`` (e.g. ``dir(df)``). + To see which attributes are excluded, see an object's ``_deprecations`` attribute, for example ``pd.DataFrame._deprecations`` (:issue:`28805`). +- The returned dtype of :func:`unique` now matches the input dtype. (:issue:`27874`) +- Changed the default configuration value for ``options.matplotlib.register_converters`` from ``True`` to ``"auto"`` (:issue:`18720`). + Now, pandas custom formatters will only be applied to plots created by pandas, through :meth:`~DataFrame.plot`. + Previously, pandas' formatters would be applied to all plots created *after* a :meth:`~DataFrame.plot`. + See :ref:`units registration ` for more. +- :meth:`Series.dropna` has dropped its ``**kwargs`` argument in favor of a single ``how`` parameter. + Supplying anything else than ``how`` to ``**kwargs`` raised a ``TypeError`` previously (:issue:`29388`) +- When testing pandas, the new minimum required version of pytest is 5.0.1 (:issue:`29664`) +- :meth:`Series.str.__iter__` was deprecated and will be removed in future releases (:issue:`28277`). +- Added ```` to the list of default NA values for :meth:`read_csv` (:issue:`30821`) + +.. _whatsnew_100.api.documentation: + +Documentation Improvements +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Added new section on :ref:`scale` (:issue:`28315`). +- Added sub-section on :ref:`io.query_multi` for HDF5 datasets (:issue:`28791`). + +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.deprecations: + +Deprecations +~~~~~~~~~~~~ + +- :meth:`Series.item` and :meth:`Index.item` have been _undeprecated_ (:issue:`29250`) +- ``Index.set_value`` has been deprecated. For a given index ``idx``, array ``arr``, + value in ``idx`` of ``idx_val`` and a new value of ``val``, ``idx.set_value(arr, idx_val, val)`` + is equivalent to ``arr[idx.get_loc(idx_val)] = val``, which should be used instead (:issue:`28621`). +- :func:`is_extension_type` is deprecated, :func:`is_extension_array_dtype` should be used instead (:issue:`29457`) +- :func:`eval` keyword argument "truediv" is deprecated and will be removed in a future version (:issue:`29812`) +- :meth:`DateOffset.isAnchored` and :meth:`DatetOffset.onOffset` are deprecated and will be removed in a future version, use :meth:`DateOffset.is_anchored` and :meth:`DateOffset.is_on_offset` instead (:issue:`30340`) +- ``pandas.tseries.frequencies.get_offset`` is deprecated and will be removed in a future version, use ``pandas.tseries.frequencies.to_offset`` instead (:issue:`4205`) +- :meth:`Categorical.take_nd` and :meth:`CategoricalIndex.take_nd` are deprecated, use :meth:`Categorical.take` and :meth:`CategoricalIndex.take` instead (:issue:`27745`) +- The parameter ``numeric_only`` of :meth:`Categorical.min` and :meth:`Categorical.max` is deprecated and replaced with ``skipna`` (:issue:`25303`) +- The parameter ``label`` in :func:`lreshape` has been deprecated and will be removed in a future version (:issue:`29742`) +- ``pandas.core.index`` has been deprecated and will be removed in a future version, the public classes are available in the top-level namespace (:issue:`19711`) +- :func:`pandas.json_normalize` is now exposed in the top-level namespace. + Usage of ``json_normalize`` as ``pandas.io.json.json_normalize`` is now deprecated and + it is recommended to use ``json_normalize`` as :func:`pandas.json_normalize` instead (:issue:`27586`). +- The ``numpy`` argument of :meth:`pandas.read_json` is deprecated (:issue:`28512`). +- :meth:`DataFrame.to_stata`, :meth:`DataFrame.to_feather`, and :meth:`DataFrame.to_parquet` argument "fname" is deprecated, use "path" instead (:issue:`23574`) +- The deprecated internal attributes ``_start``, ``_stop`` and ``_step`` of :class:`RangeIndex` now raise a ``FutureWarning`` instead of a ``DeprecationWarning`` (:issue:`26581`) +- The ``pandas.util.testing`` module has been deprecated. Use the public API in ``pandas.testing`` documented at :ref:`api.general.testing` (:issue:`16232`). +- ``pandas.SparseArray`` has been deprecated. Use ``pandas.arrays.SparseArray`` (:class:`arrays.SparseArray`) instead. (:issue:`30642`) +- The parameter ``is_copy`` of :meth:`Series.take` and :meth:`DataFrame.take` has been deprecated and will be removed in a future version. (:issue:`27357`) +- Support for multi-dimensional indexing (e.g. ``index[:, None]``) on a :class:`Index` is deprecated and will be removed in a future version, convert to a numpy array before indexing instead (:issue:`30588`) +- The ``pandas.np`` submodule is now deprecated. Import numpy directly instead (:issue:`30296`) +- The ``pandas.datetime`` class is now deprecated. Import from ``datetime`` instead (:issue:`30610`) +- :class:`~DataFrame.diff` will raise a ``TypeError`` rather than implicitly losing the dtype of extension types in the future. Convert to the correct dtype before calling ``diff`` instead (:issue:`31025`) + +**Selecting Columns from a Grouped DataFrame** + +When selecting columns from a :class:`DataFrameGroupBy` object, passing individual keys (or a tuple of keys) inside single brackets is deprecated, +a list of items should be used instead. (:issue:`23566`) For example: +.. code-block:: ipython + + df = pd.DataFrame({ + "A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"], + "B": np.random.randn(8), + "C": np.random.randn(8), + }) + g = df.groupby('A') + + # single key, returns SeriesGroupBy + g['B'] + + # tuple of single key, returns SeriesGroupBy + g[('B',)] + + # tuple of multiple keys, returns DataFrameGroupBy, raises FutureWarning + g[('B', 'C')] + + # multiple keys passed directly, returns DataFrameGroupBy, raises FutureWarning + # (implicitly converts the passed strings into a single tuple) + g['B', 'C'] + + # proper way, returns DataFrameGroupBy + g[['B', 'C']] + +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.prior_deprecations: + +Removal of prior version deprecations/changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Removed SparseSeries and SparseDataFrame** + +``SparseSeries``, ``SparseDataFrame`` and the ``DataFrame.to_sparse`` method +have been removed (:issue:`28425`). We recommend using a ``Series`` or +``DataFrame`` with sparse values instead. See :ref:`sparse.migration` for help +with migrating existing code. + +.. _whatsnew_100.matplotlib_units: + +**Matplotlib unit registration** + +Previously, pandas would register converters with matplotlib as a side effect of importing pandas (:issue:`18720`). +This changed the output of plots made via matplotlib plots after pandas was imported, even if you were using +matplotlib directly rather than :meth:`~DataFrame.plot`. + +To use pandas formatters with a matplotlib plot, specify + +.. code-block:: python + + >>> import pandas as pd + >>> pd.options.plotting.matplotlib.register_converters = True + +Note that plots created by :meth:`DataFrame.plot` and :meth:`Series.plot` *do* register the converters +automatically. The only behavior change is when plotting a date-like object via ``matplotlib.pyplot.plot`` +or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more. + +**Other removals** + +- Removed the previously deprecated keyword "index" from :func:`read_stata`, :class:`StataReader`, and :meth:`StataReader.read`, use "index_col" instead (:issue:`17328`) +- Removed ``StataReader.data`` method, use :meth:`StataReader.read` instead (:issue:`9493`) +- Removed ``pandas.plotting._matplotlib.tsplot``, use :meth:`Series.plot` instead (:issue:`19980`) +- ``pandas.tseries.converter.register`` has been moved to :func:`pandas.plotting.register_matplotlib_converters` (:issue:`18307`) +- :meth:`Series.plot` no longer accepts positional arguments, pass keyword arguments instead (:issue:`30003`) +- :meth:`DataFrame.hist` and :meth:`Series.hist` no longer allows ``figsize="default"``, specify figure size by passinig a tuple instead (:issue:`30003`) +- Floordiv of integer-dtyped array by :class:`Timedelta` now raises ``TypeError`` (:issue:`21036`) +- :class:`TimedeltaIndex` and :class:`DatetimeIndex` no longer accept non-nanosecond dtype strings like "timedelta64" or "datetime64", use "timedelta64[ns]" and "datetime64[ns]" instead (:issue:`24806`) +- Changed the default "skipna" argument in :func:`pandas.api.types.infer_dtype` from ``False`` to ``True`` (:issue:`24050`) +- Removed ``Series.ix`` and ``DataFrame.ix`` (:issue:`26438`) +- Removed ``Index.summary`` (:issue:`18217`) +- Removed the previously deprecated keyword "fastpath" from the :class:`Index` constructor (:issue:`23110`) +- Removed ``Series.get_value``, ``Series.set_value``, ``DataFrame.get_value``, ``DataFrame.set_value`` (:issue:`17739`) +- Removed ``Series.compound`` and ``DataFrame.compound`` (:issue:`26405`) +- Changed the default "inplace" argument in :meth:`DataFrame.set_index` and :meth:`Series.set_axis` from ``None`` to ``False`` (:issue:`27600`) +- Removed ``Series.cat.categorical``, ``Series.cat.index``, ``Series.cat.name`` (:issue:`24751`) +- Removed the previously deprecated keyword "box" from :func:`to_datetime` and :func:`to_timedelta`; in addition these now always returns :class:`DatetimeIndex`, :class:`TimedeltaIndex`, :class:`Index`, :class:`Series`, or :class:`DataFrame` (:issue:`24486`) +- :func:`to_timedelta`, :class:`Timedelta`, and :class:`TimedeltaIndex` no longer allow "M", "y", or "Y" for the "unit" argument (:issue:`23264`) +- Removed the previously deprecated keyword "time_rule" from (non-public) ``offsets.generate_range``, which has been moved to :func:`core.arrays._ranges.generate_range` (:issue:`24157`) +- :meth:`DataFrame.loc` or :meth:`Series.loc` with listlike indexers and missing labels will no longer reindex (:issue:`17295`) +- :meth:`DataFrame.to_excel` and :meth:`Series.to_excel` with non-existent columns will no longer reindex (:issue:`17295`) +- Removed the previously deprecated keyword "join_axes" from :func:`concat`; use ``reindex_like`` on the result instead (:issue:`22318`) +- Removed the previously deprecated keyword "by" from :meth:`DataFrame.sort_index`, use :meth:`DataFrame.sort_values` instead (:issue:`10726`) +- Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`core.groupby.DataFrameGroupBy.aggregate`, :meth:`core.groupby.SeriesGroupBy.aggregate`, :meth:`core.window.rolling.Rolling.aggregate` (:issue:`18529`) +- Passing ``datetime64`` data to :class:`TimedeltaIndex` or ``timedelta64`` data to ``DatetimeIndex`` now raises ``TypeError`` (:issue:`23539`, :issue:`23937`) +- Passing ``int64`` values to :class:`DatetimeIndex` and a timezone now interprets the values as nanosecond timestamps in UTC, not wall times in the given timezone (:issue:`24559`) +- A tuple passed to :meth:`DataFrame.groupby` is now exclusively treated as a single key (:issue:`18314`) +- Removed ``Index.contains``, use ``key in index`` instead (:issue:`30103`) +- Addition and subtraction of ``int`` or integer-arrays is no longer allowed in :class:`Timestamp`, :class:`DatetimeIndex`, :class:`TimedeltaIndex`, use ``obj + n * obj.freq`` instead of ``obj + n`` (:issue:`22535`) +- Removed ``Series.ptp`` (:issue:`21614`) +- Removed ``Series.from_array`` (:issue:`18258`) +- Removed ``DataFrame.from_items`` (:issue:`18458`) +- Removed ``DataFrame.as_matrix``, ``Series.as_matrix`` (:issue:`18458`) +- Removed ``Series.asobject`` (:issue:`18477`) +- Removed ``DataFrame.as_blocks``, ``Series.as_blocks``, ``DataFrame.blocks``, ``Series.blocks`` (:issue:`17656`) +- :meth:`pandas.Series.str.cat` now defaults to aligning ``others``, using ``join='left'`` (:issue:`27611`) +- :meth:`pandas.Series.str.cat` does not accept list-likes *within* list-likes anymore (:issue:`27611`) +- :meth:`Series.where` with ``Categorical`` dtype (or :meth:`DataFrame.where` with ``Categorical`` column) no longer allows setting new categories (:issue:`24114`) +- Removed the previously deprecated keywords "start", "end", and "periods" from the :class:`DatetimeIndex`, :class:`TimedeltaIndex`, and :class:`PeriodIndex` constructors; use :func:`date_range`, :func:`timedelta_range`, and :func:`period_range` instead (:issue:`23919`) +- Removed the previously deprecated keyword "verify_integrity" from the :class:`DatetimeIndex` and :class:`TimedeltaIndex` constructors (:issue:`23919`) +- Removed the previously deprecated keyword "fastpath" from ``pandas.core.internals.blocks.make_block`` (:issue:`19265`) +- Removed the previously deprecated keyword "dtype" from :meth:`Block.make_block_same_class` (:issue:`19434`) +- Removed ``ExtensionArray._formatting_values``. Use :attr:`ExtensionArray._formatter` instead. (:issue:`23601`) +- Removed ``MultiIndex.to_hierarchical`` (:issue:`21613`) +- Removed ``MultiIndex.labels``, use :attr:`MultiIndex.codes` instead (:issue:`23752`) +- Removed the previously deprecated keyword "labels" from the :class:`MultiIndex` constructor, use "codes" instead (:issue:`23752`) +- Removed ``MultiIndex.set_labels``, use :meth:`MultiIndex.set_codes` instead (:issue:`23752`) +- Removed the previously deprecated keyword "labels" from :meth:`MultiIndex.set_codes`, :meth:`MultiIndex.copy`, :meth:`MultiIndex.drop`, use "codes" instead (:issue:`23752`) +- Removed support for legacy HDF5 formats (:issue:`29787`) +- Passing a dtype alias (e.g. 'datetime64[ns, UTC]') to :class:`DatetimeTZDtype` is no longer allowed, use :meth:`DatetimeTZDtype.construct_from_string` instead (:issue:`23990`) +- Removed the previously deprecated keyword "skip_footer" from :func:`read_excel`; use "skipfooter" instead (:issue:`18836`) +- :func:`read_excel` no longer allows an integer value for the parameter ``usecols``, instead pass a list of integers from 0 to ``usecols`` inclusive (:issue:`23635`) +- Removed the previously deprecated keyword "convert_datetime64" from :meth:`DataFrame.to_records` (:issue:`18902`) +- Removed ``IntervalIndex.from_intervals`` in favor of the :class:`IntervalIndex` constructor (:issue:`19263`) +- Changed the default "keep_tz" argument in :meth:`DatetimeIndex.to_series` from ``None`` to ``True`` (:issue:`23739`) +- Removed ``api.types.is_period`` and ``api.types.is_datetimetz`` (:issue:`23917`) +- Ability to read pickles containing :class:`Categorical` instances created with pre-0.16 version of pandas has been removed (:issue:`27538`) +- Removed ``pandas.tseries.plotting.tsplot`` (:issue:`18627`) +- Removed the previously deprecated keywords "reduce" and "broadcast" from :meth:`DataFrame.apply` (:issue:`18577`) +- Removed the previously deprecated ``assert_raises_regex`` function in ``pandas._testing`` (:issue:`29174`) +- Removed the previously deprecated ``FrozenNDArray`` class in ``pandas.core.indexes.frozen`` (:issue:`29335`) +- Removed the previously deprecated keyword "nthreads" from :func:`read_feather`, use "use_threads" instead (:issue:`23053`) +- Removed ``Index.is_lexsorted_for_tuple`` (:issue:`29305`) +- Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`core.groupby.DataFrameGroupBy.aggregate`, :meth:`core.groupby.SeriesGroupBy.aggregate`, :meth:`core.window.rolling.Rolling.aggregate` (:issue:`29608`) +- Removed ``Series.valid``; use :meth:`Series.dropna` instead (:issue:`18800`) +- Removed ``DataFrame.is_copy``, ``Series.is_copy`` (:issue:`18812`) +- Removed ``DataFrame.get_ftype_counts``, ``Series.get_ftype_counts`` (:issue:`18243`) +- Removed ``DataFrame.ftypes``, ``Series.ftypes``, ``Series.ftype`` (:issue:`26744`) +- Removed ``Index.get_duplicates``, use ``idx[idx.duplicated()].unique()`` instead (:issue:`20239`) +- Removed ``Series.clip_upper``, ``Series.clip_lower``, ``DataFrame.clip_upper``, ``DataFrame.clip_lower`` (:issue:`24203`) +- Removed the ability to alter :attr:`DatetimeIndex.freq`, :attr:`TimedeltaIndex.freq`, or :attr:`PeriodIndex.freq` (:issue:`20772`) +- Removed ``DatetimeIndex.offset`` (:issue:`20730`) +- Removed ``DatetimeIndex.asobject``, ``TimedeltaIndex.asobject``, ``PeriodIndex.asobject``, use ``astype(object)`` instead (:issue:`29801`) +- Removed the previously deprecated keyword "order" from :func:`factorize` (:issue:`19751`) +- Removed the previously deprecated keyword "encoding" from :func:`read_stata` and :meth:`DataFrame.to_stata` (:issue:`21400`) +- Changed the default "sort" argument in :func:`concat` from ``None`` to ``False`` (:issue:`20613`) +- Removed the previously deprecated keyword "raise_conflict" from :meth:`DataFrame.update`, use "errors" instead (:issue:`23585`) +- Removed the previously deprecated keyword "n" from :meth:`DatetimeIndex.shift`, :meth:`TimedeltaIndex.shift`, :meth:`PeriodIndex.shift`, use "periods" instead (:issue:`22458`) +- Removed the previously deprecated keywords "how", "fill_method", and "limit" from :meth:`DataFrame.resample` (:issue:`30139`) +- Passing an integer to :meth:`Series.fillna` or :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtype now raises ``TypeError`` (:issue:`24694`) +- Passing multiple axes to :meth:`DataFrame.dropna` is no longer supported (:issue:`20995`) +- Removed ``Series.nonzero``, use ``to_numpy().nonzero()`` instead (:issue:`24048`) +- Passing floating dtype ``codes`` to :meth:`Categorical.from_codes` is no longer supported, pass ``codes.astype(np.int64)`` instead (:issue:`21775`) +- Removed the previously deprecated keyword "pat" from :meth:`Series.str.partition` and :meth:`Series.str.rpartition`, use "sep" instead (:issue:`23767`) +- Removed ``Series.put`` (:issue:`27106`) +- Removed ``Series.real``, ``Series.imag`` (:issue:`27106`) +- Removed ``Series.to_dense``, ``DataFrame.to_dense`` (:issue:`26684`) +- Removed ``Index.dtype_str``, use ``str(index.dtype)`` instead (:issue:`27106`) +- :meth:`Categorical.ravel` returns a :class:`Categorical` instead of a ``ndarray`` (:issue:`27199`) +- The 'outer' method on Numpy ufuncs, e.g. ``np.subtract.outer`` operating on :class:`Series` objects is no longer supported, and will raise ``NotImplementedError`` (:issue:`27198`) +- Removed ``Series.get_dtype_counts`` and ``DataFrame.get_dtype_counts`` (:issue:`27145`) +- Changed the default "fill_value" argument in :meth:`Categorical.take` from ``True`` to ``False`` (:issue:`20841`) +- Changed the default value for the `raw` argument in :func:`Series.rolling().apply() `, :func:`DataFrame.rolling().apply() `, :func:`Series.expanding().apply() `, and :func:`DataFrame.expanding().apply() ` from ``None`` to ``False`` (:issue:`20584`) +- Removed deprecated behavior of :meth:`Series.argmin` and :meth:`Series.argmax`, use :meth:`Series.idxmin` and :meth:`Series.idxmax` for the old behavior (:issue:`16955`) +- Passing a tz-aware ``datetime.datetime`` or :class:`Timestamp` into the :class:`Timestamp` constructor with the ``tz`` argument now raises a ``ValueError`` (:issue:`23621`) +- Removed ``Series.base``, ``Index.base``, ``Categorical.base``, ``Series.flags``, ``Index.flags``, ``PeriodArray.flags``, ``Series.strides``, ``Index.strides``, ``Series.itemsize``, ``Index.itemsize``, ``Series.data``, ``Index.data`` (:issue:`20721`) +- Changed :meth:`Timedelta.resolution` to match the behavior of the standard library ``datetime.timedelta.resolution``, for the old behavior, use :meth:`Timedelta.resolution_string` (:issue:`26839`) +- Removed ``Timestamp.weekday_name``, ``DatetimeIndex.weekday_name``, and ``Series.dt.weekday_name`` (:issue:`18164`) +- Removed the previously deprecated keyword "errors" in :meth:`Timestamp.tz_localize`, :meth:`DatetimeIndex.tz_localize`, and :meth:`Series.tz_localize` (:issue:`22644`) +- Changed the default "ordered" argument in :class:`CategoricalDtype` from ``None`` to ``False`` (:issue:`26336`) +- :meth:`Series.set_axis` and :meth:`DataFrame.set_axis` now require "labels" as the first argument and "axis" as an optional named parameter (:issue:`30089`) +- Removed ``to_msgpack``, ``read_msgpack``, ``DataFrame.to_msgpack``, ``Series.to_msgpack`` (:issue:`27103`) +- Removed ``Series.compress`` (:issue:`21930`) +- Removed the previously deprecated keyword "fill_value" from :meth:`Categorical.fillna`, use "value" instead (:issue:`19269`) +- Removed the previously deprecated keyword "data" from :func:`andrews_curves`, use "frame" instead (:issue:`6956`) +- Removed the previously deprecated keyword "data" from :func:`parallel_coordinates`, use "frame" instead (:issue:`6956`) +- Removed the previously deprecated keyword "colors" from :func:`parallel_coordinates`, use "color" instead (:issue:`6956`) +- Removed the previously deprecated keywords "verbose" and "private_key" from :func:`read_gbq` (:issue:`30200`) +- Calling ``np.array`` and ``np.asarray`` on tz-aware :class:`Series` and :class:`DatetimeIndex` will now return an object array of tz-aware :class:`Timestamp` (:issue:`24596`) +- + +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.performance: + +Performance improvements +~~~~~~~~~~~~~~~~~~~~~~~~ + +- Performance improvement in :class:`DataFrame` arithmetic and comparison operations with scalars (:issue:`24990`, :issue:`29853`) +- Performance improvement in indexing with a non-unique :class:`IntervalIndex` (:issue:`27489`) +- Performance improvement in :attr:`MultiIndex.is_monotonic` (:issue:`27495`) +- Performance improvement in :func:`cut` when ``bins`` is an :class:`IntervalIndex` (:issue:`27668`) +- Performance improvement when initializing a :class:`DataFrame` using a ``range`` (:issue:`30171`) +- Performance improvement in :meth:`DataFrame.corr` when ``method`` is ``"spearman"`` (:issue:`28139`) +- Performance improvement in :meth:`DataFrame.replace` when provided a list of values to replace (:issue:`28099`) +- Performance improvement in :meth:`DataFrame.select_dtypes` by using vectorization instead of iterating over a loop (:issue:`28317`) +- Performance improvement in :meth:`Categorical.searchsorted` and :meth:`CategoricalIndex.searchsorted` (:issue:`28795`) +- Performance improvement when comparing a :class:`Categorical` with a scalar and the scalar is not found in the categories (:issue:`29750`) +- Performance improvement when checking if values in a :class:`Categorical` are equal, equal or larger or larger than a given scalar. + The improvement is not present if checking if the :class:`Categorical` is less than or less than or equal than the scalar (:issue:`29820`) +- Performance improvement in :meth:`Index.equals` and :meth:`MultiIndex.equals` (:issue:`29134`) +- Performance improvement in :func:`~pandas.api.types.infer_dtype` when ``skipna`` is ``True`` (:issue:`28814`) + +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.bug_fixes: + +Bug fixes +~~~~~~~~~ + + +Categorical +^^^^^^^^^^^ + +- Added test to assert the :func:`fillna` raises the correct ``ValueError`` message when the value isn't a value from categories (:issue:`13628`) +- Bug in :meth:`Categorical.astype` where ``NaN`` values were handled incorrectly when casting to int (:issue:`28406`) +- :meth:`DataFrame.reindex` with a :class:`CategoricalIndex` would fail when the targets contained duplicates, and wouldn't fail if the source contained duplicates (:issue:`28107`) +- Bug in :meth:`Categorical.astype` not allowing for casting to extension dtypes (:issue:`28668`) +- Bug where :func:`merge` was unable to join on categorical and extension dtype columns (:issue:`28668`) +- :meth:`Categorical.searchsorted` and :meth:`CategoricalIndex.searchsorted` now work on unordered categoricals also (:issue:`21667`) +- Added test to assert roundtripping to parquet with :func:`DataFrame.to_parquet` or :func:`read_parquet` will preserve Categorical dtypes for string types (:issue:`27955`) +- Changed the error message in :meth:`Categorical.remove_categories` to always show the invalid removals as a set (:issue:`28669`) +- Using date accessors on a categorical dtyped :class:`Series` of datetimes was not returning an object of the + same type as if one used the :meth:`.str.` / :meth:`.dt.` on a :class:`Series` of that type. E.g. when accessing :meth:`Series.dt.tz_localize` on a + :class:`Categorical` with duplicate entries, the accessor was skipping duplicates (:issue:`27952`) +- Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` that would give incorrect results on categorical data (:issue:`26988`) +- Bug where calling :meth:`Categorical.min` or :meth:`Categorical.max` on an empty Categorical would raise a numpy exception (:issue:`30227`) +- The following methods now also correctly output values for unobserved categories when called through ``groupby(..., observed=False)`` (:issue:`17605`) + * :meth:`core.groupby.SeriesGroupBy.count` + * :meth:`core.groupby.SeriesGroupBy.size` + * :meth:`core.groupby.SeriesGroupBy.nunique` + * :meth:`core.groupby.SeriesGroupBy.nth` + + +Datetimelike +^^^^^^^^^^^^ +- Bug in :meth:`Series.__setitem__` incorrectly casting ``np.timedelta64("NaT")`` to ``np.datetime64("NaT")`` when inserting into a :class:`Series` with datetime64 dtype (:issue:`27311`) +- Bug in :meth:`Series.dt` property lookups when the underlying data is read-only (:issue:`27529`) +- Bug in ``HDFStore.__getitem__`` incorrectly reading tz attribute created in Python 2 (:issue:`26443`) +- Bug in :func:`to_datetime` where passing arrays of malformed ``str`` with errors="coerce" could incorrectly lead to raising ``ValueError`` (:issue:`28299`) +- Bug in :meth:`core.groupby.SeriesGroupBy.nunique` where ``NaT`` values were interfering with the count of unique values (:issue:`27951`) +- Bug in :class:`Timestamp` subtraction when subtracting a :class:`Timestamp` from a ``np.datetime64`` object incorrectly raising ``TypeError`` (:issue:`28286`) +- Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`) +- Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`) +- Bug in :meth:`Series.astype`, :meth:`Index.astype`, and :meth:`DataFrame.astype` failing to handle ``NaT`` when casting to an integer dtype (:issue:`28492`) +- Bug in :class:`Week` with ``weekday`` incorrectly raising ``AttributeError`` instead of ``TypeError`` when adding or subtracting an invalid type (:issue:`28530`) +- Bug in :class:`DataFrame` arithmetic operations when operating with a :class:`Series` with dtype `'timedelta64[ns]'` (:issue:`28049`) +- Bug in :func:`core.groupby.generic.SeriesGroupBy.apply` raising ``ValueError`` when a column in the original DataFrame is a datetime and the column labels are not standard integers (:issue:`28247`) +- Bug in :func:`pandas._config.localization.get_locales` where the ``locales -a`` encodes the locales list as windows-1252 (:issue:`23638`, :issue:`24760`, :issue:`27368`) +- Bug in :meth:`Series.var` failing to raise ``TypeError`` when called with ``timedelta64[ns]`` dtype (:issue:`28289`) +- Bug in :meth:`DatetimeIndex.strftime` and :meth:`Series.dt.strftime` where ``NaT`` was converted to the string ``'NaT'`` instead of ``np.nan`` (:issue:`29578`) +- Bug in masking datetime-like arrays with a boolean mask of an incorrect length not raising an ``IndexError`` (:issue:`30308`) +- Bug in :attr:`Timestamp.resolution` being a property instead of a class attribute (:issue:`29910`) +- Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`) +- Bug in :func:`pandas.to_datetime` failing for `deques` when using ``cache=True`` (the default) (:issue:`29403`) +- Bug in :meth:`Series.item` with ``datetime64`` or ``timedelta64`` dtype, :meth:`DatetimeIndex.item`, and :meth:`TimedeltaIndex.item` returning an integer instead of a :class:`Timestamp` or :class:`Timedelta` (:issue:`30175`) +- Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`) +- Bug in :meth:`DataFrame.drop` where attempting to drop non-existent values from a DatetimeIndex would yield a confusing error message (:issue:`30399`) +- Bug in :meth:`DataFrame.append` would remove the timezone-awareness of new data (:issue:`30238`) +- Bug in :meth:`Series.cummin` and :meth:`Series.cummax` with timezone-aware dtype incorrectly dropping its timezone (:issue:`15553`) +- Bug in :class:`DatetimeArray`, :class:`TimedeltaArray`, and :class:`PeriodArray` where inplace addition and subtraction did not actually operate inplace (:issue:`24115`) +- Bug in :func:`pandas.to_datetime` when called with ``Series`` storing ``IntegerArray`` raising ``TypeError`` instead of returning ``Series`` (:issue:`30050`) +- Bug in :func:`date_range` with custom business hours as ``freq`` and given number of ``periods`` (:issue:`30593`) +- Bug in :class:`PeriodIndex` comparisons with incorrectly casting integers to :class:`Period` objects, inconsistent with the :class:`Period` comparison behavior (:issue:`30722`) +- Bug in :meth:`DatetimeIndex.insert` raising a ``ValueError`` instead of a ``TypeError`` when trying to insert a timezone-aware :class:`Timestamp` into a timezone-naive :class:`DatetimeIndex`, or vice-versa (:issue:`30806`) + +Timedelta +^^^^^^^^^ +- Bug in subtracting a :class:`TimedeltaIndex` or :class:`TimedeltaArray` from a ``np.datetime64`` object (:issue:`29558`) +- +- + +Timezones +^^^^^^^^^ + +- +- + + +Numeric +^^^^^^^ +- Bug in :meth:`DataFrame.quantile` with zero-column :class:`DataFrame` incorrectly raising (:issue:`23925`) +- :class:`DataFrame` flex inequality comparisons methods (:meth:`DataFrame.lt`, :meth:`DataFrame.le`, :meth:`DataFrame.gt`, :meth:`DataFrame.ge`) with object-dtype and ``complex`` entries failing to raise ``TypeError`` like their :class:`Series` counterparts (:issue:`28079`) +- Bug in :class:`DataFrame` logical operations (`&`, `|`, `^`) not matching :class:`Series` behavior by filling NA values (:issue:`28741`) +- Bug in :meth:`DataFrame.interpolate` where specifying axis by name references variable before it is assigned (:issue:`29142`) +- Bug in :meth:`Series.var` not computing the right value with a nullable integer dtype series not passing through ddof argument (:issue:`29128`) +- Improved error message when using `frac` > 1 and `replace` = False (:issue:`27451`) +- Bug in numeric indexes resulted in it being possible to instantiate an :class:`Int64Index`, :class:`UInt64Index`, or :class:`Float64Index` with an invalid dtype (e.g. datetime-like) (:issue:`29539`) +- Bug in :class:`UInt64Index` precision loss while constructing from a list with values in the ``np.uint64`` range (:issue:`29526`) +- Bug in :class:`NumericIndex` construction that caused indexing to fail when integers in the ``np.uint64`` range were used (:issue:`28023`) +- Bug in :class:`NumericIndex` construction that caused :class:`UInt64Index` to be casted to :class:`Float64Index` when integers in the ``np.uint64`` range were used to index a :class:`DataFrame` (:issue:`28279`) +- Bug in :meth:`Series.interpolate` when using method=`index` with an unsorted index, would previously return incorrect results. (:issue:`21037`) +- Bug in :meth:`DataFrame.round` where a :class:`DataFrame` with a :class:`CategoricalIndex` of :class:`IntervalIndex` columns would incorrectly raise a ``TypeError`` (:issue:`30063`) +- Bug in :meth:`Series.pct_change` and :meth:`DataFrame.pct_change` when there are duplicated indices (:issue:`30463`) +- Bug in :class:`DataFrame` cumulative operations (e.g. cumsum, cummax) incorrect casting to object-dtype (:issue:`19296`) +- Bug in :class:`~DataFrame.diff` losing the dtype for extension types (:issue:`30889`) +- Bug in :class:`DataFrame.diff` raising an ``IndexError`` when one of the columns was a nullable integer dtype (:issue:`30967`) + +Conversion +^^^^^^^^^^ + +- +- + +Strings +^^^^^^^ + +- Calling :meth:`Series.str.isalnum` (and other "ismethods") on an empty ``Series`` would return an ``object`` dtype instead of ``bool`` (:issue:`29624`) +- + + +Interval +^^^^^^^^ + +- Bug in :meth:`IntervalIndex.get_indexer` where a :class:`Categorical` or :class:`CategoricalIndex` ``target`` would incorrectly raise a ``TypeError`` (:issue:`30063`) +- Bug in ``pandas.core.dtypes.cast.infer_dtype_from_scalar`` where passing ``pandas_dtype=True`` did not infer :class:`IntervalDtype` (:issue:`30337`) +- Bug in :class:`Series` constructor where constructing a ``Series`` from a ``list`` of :class:`Interval` objects resulted in ``object`` dtype instead of :class:`IntervalDtype` (:issue:`23563`) +- Bug in :class:`IntervalDtype` where the ``kind`` attribute was incorrectly set as ``None`` instead of ``"O"`` (:issue:`30568`) +- Bug in :class:`IntervalIndex`, :class:`~arrays.IntervalArray`, and :class:`Series` with interval data where equality comparisons were incorrect (:issue:`24112`) + +Indexing +^^^^^^^^ + +- Bug in assignment using a reverse slicer (:issue:`26939`) +- Bug in :meth:`DataFrame.explode` would duplicate frame in the presence of duplicates in the index (:issue:`28010`) +- Bug in reindexing a :meth:`PeriodIndex` with another type of index that contained a `Period` (:issue:`28323`) (:issue:`28337`) +- Fix assignment of column via `.loc` with numpy non-ns datetime type (:issue:`27395`) +- Bug in :meth:`Float64Index.astype` where ``np.inf`` was not handled properly when casting to an integer dtype (:issue:`28475`) +- :meth:`Index.union` could fail when the left contained duplicates (:issue:`28257`) +- Bug when indexing with ``.loc`` where the index was a :class:`CategoricalIndex` with non-string categories didn't work (:issue:`17569`, :issue:`30225`) +- :meth:`Index.get_indexer_non_unique` could fail with ``TypeError`` in some cases, such as when searching for ints in a string index (:issue:`28257`) +- Bug in :meth:`Float64Index.get_loc` incorrectly raising ``TypeError`` instead of ``KeyError`` (:issue:`29189`) +- Bug in :meth:`DataFrame.loc` with incorrect dtype when setting Categorical value in 1-row DataFrame (:issue:`25495`) +- :meth:`MultiIndex.get_loc` can't find missing values when input includes missing values (:issue:`19132`) +- Bug in :meth:`Series.__setitem__` incorrectly assigning values with boolean indexer when the length of new data matches the number of ``True`` values and new data is not a ``Series`` or an ``np.array`` (:issue:`30567`) +- Bug in indexing with a :class:`PeriodIndex` incorrectly accepting integers representing years, use e.g. ``ser.loc["2007"]`` instead of ``ser.loc[2007]`` (:issue:`30763`) + +Missing +^^^^^^^ + +- +- + +MultiIndex +^^^^^^^^^^ + +- Constructor for :class:`MultiIndex` verifies that the given ``sortorder`` is compatible with the actual ``lexsort_depth`` if ``verify_integrity`` parameter is ``True`` (the default) (:issue:`28735`) +- Series and MultiIndex `.drop` with `MultiIndex` raise exception if labels not in given in level (:issue:`8594`) +- + +I/O +^^^ + +- :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`) +- Bug in :meth:`DataFrame.to_json` where using a Tuple as a column or index value and using ``orient="columns"`` or ``orient="index"`` would produce invalid JSON (:issue:`20500`) +- Improve infinity parsing. :meth:`read_csv` now interprets ``Infinity``, ``+Infinity``, ``-Infinity`` as floating point values (:issue:`10065`) +- Bug in :meth:`DataFrame.to_csv` where values were truncated when the length of ``na_rep`` was shorter than the text input data. (:issue:`25099`) +- Bug in :func:`DataFrame.to_string` where values were truncated using display options instead of outputting the full content (:issue:`9784`) +- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in ISO format with ``orient="table"`` (:issue:`28130`) +- Bug in :func:`DataFrame.to_parquet` where writing to GCS would fail with `engine='fastparquet'` if the file did not already exist (:issue:`28326`) +- Bug in :func:`read_hdf` closing stores that it didn't open when Exceptions are raised (:issue:`28699`) +- Bug in :meth:`DataFrame.read_json` where using ``orient="index"`` would not maintain the order (:issue:`28557`) +- Bug in :meth:`DataFrame.to_html` where the length of the ``formatters`` argument was not verified (:issue:`28469`) +- Bug in :meth:`DataFrame.read_excel` with ``engine='ods'`` when ``sheet_name`` argument references a non-existent sheet (:issue:`27676`) +- Bug in :meth:`pandas.io.formats.style.Styler` formatting for floating values not displaying decimals correctly (:issue:`13257`) +- Bug in :meth:`DataFrame.to_html` when using ``formatters=`` and ``max_cols`` together. (:issue:`25955`) +- Bug in :meth:`Styler.background_gradient` not able to work with dtype ``Int64`` (:issue:`28869`) +- Bug in :meth:`DataFrame.to_clipboard` which did not work reliably in ipython (:issue:`22707`) +- Bug in :func:`read_json` where default encoding was not set to ``utf-8`` (:issue:`29565`) +- Bug in :class:`PythonParser` where str and bytes were being mixed when dealing with the decimal field (:issue:`29650`) +- :meth:`read_gbq` now accepts ``progress_bar_type`` to display progress bar while the data downloads. (:issue:`29857`) +- Bug in :func:`pandas.io.json.json_normalize` where a missing value in the location specified by `record_path` would raise a ``TypeError`` (:issue:`30148`) +- :func:`read_excel` now accepts binary data (:issue:`15914`) +- Bug in :meth:`read_csv` in which encoding handling was limited to just the string `utf-16` for the C engine (:issue:`24130`) + +Plotting +^^^^^^^^ + +- Bug in :meth:`Series.plot` not able to plot boolean values (:issue:`23719`) +- Bug in :meth:`DataFrame.plot` not able to plot when no rows (:issue:`27758`) +- Bug in :meth:`DataFrame.plot` producing incorrect legend markers when plotting multiple series on the same axis (:issue:`18222`) +- Bug in :meth:`DataFrame.plot` when ``kind='box'`` and data contains datetime or timedelta data. These types are now automatically dropped (:issue:`22799`) +- Bug in :meth:`DataFrame.plot.line` and :meth:`DataFrame.plot.area` produce wrong xlim in x-axis (:issue:`27686`, :issue:`25160`, :issue:`24784`) +- Bug where :meth:`DataFrame.boxplot` would not accept a ``color`` parameter like :meth:`DataFrame.plot.box` (:issue:`26214`) +- Bug in the ``xticks`` argument being ignored for :meth:`DataFrame.plot.bar` (:issue:`14119`) +- :func:`set_option` now validates that the plot backend provided to ``'plotting.backend'`` implements the backend when the option is set, rather than when a plot is created (:issue:`28163`) +- :meth:`DataFrame.plot` now allow a ``backend`` keyword argument to allow changing between backends in one session (:issue:`28619`). +- Bug in color validation incorrectly raising for non-color styles (:issue:`29122`). +- Allow :meth:`DataFrame.plot.scatter` to plot ``objects`` and ``datetime`` type data (:issue:`18755`, :issue:`30391`) +- Bug in :meth:`DataFrame.hist`, ``xrot=0`` does not work with ``by`` and subplots (:issue:`30288`). + +Groupby/resample/rolling +^^^^^^^^^^^^^^^^^^^^^^^^ + +- Bug in :meth:`core.groupby.DataFrameGroupBy.apply` only showing output from a single group when function returns an :class:`Index` (:issue:`28652`) +- Bug in :meth:`DataFrame.groupby` with multiple groups where an ``IndexError`` would be raised if any group contained all NA values (:issue:`20519`) +- Bug in :meth:`pandas.core.resample.Resampler.size` and :meth:`pandas.core.resample.Resampler.count` returning wrong dtype when used with an empty :class:`Series` or :class:`DataFrame` (:issue:`28427`) +- Bug in :meth:`DataFrame.rolling` not allowing for rolling over datetimes when ``axis=1`` (:issue:`28192`) +- Bug in :meth:`DataFrame.rolling` not allowing rolling over multi-index levels (:issue:`15584`). +- Bug in :meth:`DataFrame.rolling` not allowing rolling on monotonic decreasing time indexes (:issue:`19248`). +- Bug in :meth:`DataFrame.groupby` not offering selection by column name when ``axis=1`` (:issue:`27614`) +- Bug in :meth:`core.groupby.DataFrameGroupby.agg` not able to use lambda function with named aggregation (:issue:`27519`) +- Bug in :meth:`DataFrame.groupby` losing column name information when grouping by a categorical column (:issue:`28787`) +- Remove error raised due to duplicated input functions in named aggregation in :meth:`DataFrame.groupby` and :meth:`Series.groupby`. Previously error will be raised if the same function is applied on the same column and now it is allowed if new assigned names are different. (:issue:`28426`) +- :meth:`core.groupby.SeriesGroupBy.value_counts` will be able to handle the case even when the :class:`Grouper` makes empty groups (:issue:`28479`) +- Bug in :meth:`core.window.rolling.Rolling.quantile` ignoring ``interpolation`` keyword argument when used within a groupby (:issue:`28779`) +- Bug in :meth:`DataFrame.groupby` where ``any``, ``all``, ``nunique`` and transform functions would incorrectly handle duplicate column labels (:issue:`21668`) +- Bug in :meth:`core.groupby.DataFrameGroupBy.agg` with timezone-aware datetime64 column incorrectly casting results to the original dtype (:issue:`29641`) +- Bug in :meth:`DataFrame.groupby` when using axis=1 and having a single level columns index (:issue:`30208`) +- Bug in :meth:`DataFrame.groupby` when using nunique on axis=1 (:issue:`30253`) +- Bug in :meth:`GroupBy.quantile` with multiple list-like q value and integer column names (:issue:`30289`) +- Bug in :meth:`GroupBy.pct_change` and :meth:`core.groupby.SeriesGroupBy.pct_change` causes ``TypeError`` when ``fill_method`` is ``None`` (:issue:`30463`) +- Bug in :meth:`Rolling.count` and :meth:`Expanding.count` argument where ``min_periods`` was ignored (:issue:`26996`) + +Reshaping +^^^^^^^^^ + +- Bug in :meth:`DataFrame.apply` that caused incorrect output with empty :class:`DataFrame` (:issue:`28202`, :issue:`21959`) +- Bug in :meth:`DataFrame.stack` not handling non-unique indexes correctly when creating MultiIndex (:issue:`28301`) +- Bug in :meth:`pivot_table` not returning correct type ``float`` when ``margins=True`` and ``aggfunc='mean'`` (:issue:`24893`) +- Bug :func:`merge_asof` could not use :class:`datetime.timedelta` for ``tolerance`` kwarg (:issue:`28098`) +- Bug in :func:`merge`, did not append suffixes correctly with MultiIndex (:issue:`28518`) +- :func:`qcut` and :func:`cut` now handle boolean input (:issue:`20303`) +- Fix to ensure all int dtypes can be used in :func:`merge_asof` when using a tolerance value. Previously every non-int64 type would raise an erroneous ``MergeError`` (:issue:`28870`). +- Better error message in :func:`get_dummies` when `columns` isn't a list-like value (:issue:`28383`) +- Bug in :meth:`Index.join` that caused infinite recursion error for mismatched ``MultiIndex`` name orders. (:issue:`25760`, :issue:`28956`) +- Bug :meth:`Series.pct_change` where supplying an anchored frequency would throw a ``ValueError`` (:issue:`28664`) +- Bug where :meth:`DataFrame.equals` returned True incorrectly in some cases when two DataFrames had the same columns in different orders (:issue:`28839`) +- Bug in :meth:`DataFrame.replace` that caused non-numeric replacer's dtype not respected (:issue:`26632`) +- Bug in :func:`melt` where supplying mixed strings and numeric values for ``id_vars`` or ``value_vars`` would incorrectly raise a ``ValueError`` (:issue:`29718`) +- Dtypes are now preserved when transposing a ``DataFrame`` where each column is the same extension dtype (:issue:`30091`) +- Bug in :func:`merge_asof` merging on a tz-aware ``left_index`` and ``right_on`` a tz-aware column (:issue:`29864`) +- Improved error message and docstring in :func:`cut` and :func:`qcut` when `labels=True` (:issue:`13318`) +- Bug in missing `fill_na` parameter to :meth:`DataFrame.unstack` with list of levels (:issue:`30740`) + +Sparse +^^^^^^ +- Bug in :class:`SparseDataFrame` arithmetic operations incorrectly casting inputs to float (:issue:`28107`) +- Bug in ``DataFrame.sparse`` returning a ``Series`` when there was a column named ``sparse`` rather than the accessor (:issue:`30758`) +- Fixed :meth:`operator.xor` with a boolean-dtype ``SparseArray``. Now returns a sparse result, rather than object dtype (:issue:`31025`) + +ExtensionArray +^^^^^^^^^^^^^^ + +- Bug in :class:`arrays.PandasArray` when setting a scalar string (:issue:`28118`, :issue:`28150`). +- Bug where nullable integers could not be compared to strings (:issue:`28930`) +- Bug where :class:`DataFrame` constructor raised ``ValueError`` with list-like data and ``dtype`` specified (:issue:`30280`) + +Other +^^^^^ +- Trying to set the ``display.precision``, ``display.max_rows`` or ``display.max_columns`` using :meth:`set_option` to anything but a ``None`` or a positive int will raise a ``ValueError`` (:issue:`23348`) +- Using :meth:`DataFrame.replace` with overlapping keys in a nested dictionary will no longer raise, now matching the behavior of a flat dictionary (:issue:`27660`) +- :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` now support dicts as ``compression`` argument with key ``'method'`` being the compression method and others as additional compression options when the compression method is ``'zip'``. (:issue:`26023`) +- Bug in :meth:`Series.diff` where a boolean series would incorrectly raise a ``TypeError`` (:issue:`17294`) +- :meth:`Series.append` will no longer raise a ``TypeError`` when passed a tuple of ``Series`` (:issue:`28410`) +- Fix corrupted error message when calling ``pandas.libs._json.encode()`` on a 0d array (:issue:`18878`) +- Backtick quoting in :meth:`DataFrame.query` and :meth:`DataFrame.eval` can now also be used to use invalid identifiers like names that start with a digit, are python keywords, or are using single character operators. (:issue:`27017`) +- Bug in ``pd.core.util.hashing.hash_pandas_object`` where arrays containing tuples were incorrectly treated as non-hashable (:issue:`28969`) +- Bug in :meth:`DataFrame.append` that raised ``IndexError`` when appending with empty list (:issue:`28769`) +- Fix :class:`AbstractHolidayCalendar` to return correct results for + years after 2030 (now goes up to 2200) (:issue:`27790`) +- Fixed :class:`~arrays.IntegerArray` returning ``inf`` rather than ``NaN`` for operations dividing by ``0`` (:issue:`27398`) +- Fixed ``pow`` operations for :class:`~arrays.IntegerArray` when the other value is ``0`` or ``1`` (:issue:`29997`) +- Bug in :meth:`Series.count` raises if use_inf_as_na is enabled (:issue:`29478`) +- Bug in :class:`Index` where a non-hashable name could be set without raising ``TypeError`` (:issue:`29069`) +- Bug in :class:`DataFrame` constructor when passing a 2D ``ndarray`` and an extension dtype (:issue:`12513`) +- Bug in :meth:`DataFrame.to_csv` when supplied a series with a ``dtype="string"`` and a ``na_rep``, the ``na_rep`` was being truncated to 2 characters. (:issue:`29975`) +- Bug where :meth:`DataFrame.itertuples` would incorrectly determine whether or not namedtuples could be used for dataframes of 255 columns (:issue:`28282`) +- Handle nested NumPy ``object`` arrays in :func:`testing.assert_series_equal` for ExtensionArray implementations (:issue:`30841`) +- Bug in :class:`Index` constructor incorrectly allowing 2-dimensional input arrays (:issue:`13601`, :issue:`27125`) + +.. --------------------------------------------------------------------------- + +.. _whatsnew_100.contributors: + +Contributors +~~~~~~~~~~~~ + +.. contributors:: v0.25.3..v1.0.0