Merge branch 'master' of https://github.com/pandas-dev/pandas into git_version

Author: atulagrwl

.coveragerc

@@ -2,6 +2,7 @@
 [run]
 branch = False
 omit = */tests/*
+plugins = Cython.Coverage
 
 [report]
 # Regexes for lines to exclude from consideration
@@ -22,6 +23,7 @@ exclude_lines =
     if __name__ == .__main__.:
 
 ignore_errors = False
+show_missing = True
 
 [html]
 directory = coverage_html_report

appveyor.yml

@@ -20,12 +20,14 @@ environment:
   matrix:
 
     - CONDA_ROOT: "C:\\Miniconda3_64"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
       CONDA_PY: "36"
       CONDA_NPY: "113"
 
     - CONDA_ROOT: "C:\\Miniconda3_64"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
       PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
       CONDA_PY: "27"

asv_bench/benchmarks/groupby.py

@@ -142,7 +142,7 @@ def time_frame_nth(self, dtype):
     def time_series_nth_any(self, dtype):
         self.df['values'].groupby(self.df['key']).nth(0, dropna='any')
 
-    def time_groupby_nth_all(self, dtype):
+    def time_series_nth_all(self, dtype):
         self.df['values'].groupby(self.df['key']).nth(0, dropna='all')
 
     def time_series_nth(self, dtype):

asv_bench/benchmarks/reshape.py

@@ -1,7 +1,9 @@
+import string
 from itertools import product
 
 import numpy as np
 from pandas import DataFrame, MultiIndex, date_range, melt, wide_to_long
+import pandas as pd
 
 from .pandas_vb_common import setup  # noqa
 
@@ -132,3 +134,19 @@ def setup(self):
 
     def time_pivot_table(self):
         self.df.pivot_table(index='key1', columns=['key2', 'key3'])
+
+
+class GetDummies(object):
+    goal_time = 0.2
+
+    def setup(self):
+        categories = list(string.ascii_letters[:12])
+        s = pd.Series(np.random.choice(categories, size=1_000_000),
+                      dtype=pd.api.types.CategoricalDtype(categories))
+        self.s = s
+
+    def time_get_dummies_1d(self):
+        pd.get_dummies(self.s, sparse=False)
+
+    def time_get_dummies_1d_sparse(self):
+        pd.get_dummies(self.s, sparse=True)

ci/requirements-optional-conda.txt

@@ -22,6 +22,7 @@ s3fs
 scipy
 seaborn
 sqlalchemy
+statsmodels
 xarray
 xlrd
 xlsxwriter

ci/requirements-optional-pip.txt

@@ -24,6 +24,7 @@ s3fs
 scipy
 seaborn
 sqlalchemy
+statsmodels
 xarray
 xlrd
 xlsxwriter

doc/make.py

@@ -363,6 +363,10 @@ def main():
     sys.path.append(args.python_path)
     globals()['pandas'] = importlib.import_module('pandas')
 
+    # Set the matplotlib backend to the non-interactive Agg backend for all
+    # child processes.
+    os.environ['MPLBACKEND'] = 'module://matplotlib.backends.backend_agg'
+
     builder = DocBuilder(args.num_jobs, not args.no_api, args.single,
                          args.verbosity)
     getattr(builder, args.command)()

doc/source/api.rst

@@ -444,6 +444,7 @@ Reindexing / Selection / Label manipulation
 
    Series.align
    Series.drop
+   Series.droplevel
    Series.drop_duplicates
    Series.duplicated
    Series.equals
@@ -1063,6 +1064,7 @@ Reshaping, sorting, transposing
 .. autosummary::
    :toctree: generated/
 
+   DataFrame.droplevel
    DataFrame.pivot
    DataFrame.pivot_table
    DataFrame.reorder_levels
@@ -1870,8 +1872,6 @@ Methods
     PeriodIndex.asfreq
     PeriodIndex.strftime
     PeriodIndex.to_timestamp
-    PeriodIndex.tz_convert
-    PeriodIndex.tz_localize
 
 Scalars
 -------

doc/source/basics.rst

@@ -1924,11 +1924,24 @@ untouched. If the data is modified, it is because you did so explicitly.
 dtypes
 ------
 
-The main types stored in pandas objects are ``float``, ``int``, ``bool``,
-``datetime64[ns]`` and ``datetime64[ns, tz]``, ``timedelta[ns]``,
-``category`` and ``object``. In addition these dtypes have item sizes, e.g.
-``int64`` and ``int32``. See :ref:`Series with TZ <timeseries.timezone_series>`
-for more detail on ``datetime64[ns, tz]`` dtypes.
+For the most part, pandas uses NumPy arrays and dtypes for Series or individual
+columns of a DataFrame. The main types allowed in pandas objects are ``float``,
+``int``, ``bool``, and ``datetime64[ns]`` (note that NumPy does not support
+timezone-aware datetimes).
+
+In addition to NumPy's types, pandas :ref:`extends <extending.extension-types>`
+NumPy's type-system for a few cases.
+
+* :ref:`Categorical <categorical>`
+* :ref:`Datetime with Timezone <timeseries.timezone_series>`
+* :ref:`Period <timeseries.periods>`
+* :ref:`Interval <advanced.indexing.intervallindex>`
+
+Pandas uses the ``object`` dtype for storing strings.
+
+Finally, arbitrary objects may be stored using the ``object`` dtype, but should
+be avoided to the extent possible (for performance and interoperability with
+other libraries and methods. See :ref:`basics.object_conversion`).
 
 A convenient :attr:`~DataFrame.dtypes` attribute for DataFrame returns a Series
 with the data type of each column.

doc/source/merging.rst

@@ -506,8 +506,8 @@ You can also pass a list of dicts or Series:
 
 .. _merging.join:
 
-Database-style DataFrame joining/merging
-----------------------------------------
+Database-style DataFrame or named Series joining/merging
+--------------------------------------------------------
 
 pandas has full-featured, **high performance** in-memory join operations
 idiomatically very similar to relational databases like SQL. These methods
@@ -522,7 +522,7 @@ Users who are familiar with SQL but new to pandas might be interested in a
 :ref:`comparison with SQL<compare_with_sql.join>`.
 
 pandas provides a single function, :func:`~pandas.merge`, as the entry point for 
-all standard database join operations between ``DataFrame`` objects:
+all standard database join operations between ``DataFrame`` or named ``Series`` objects:
 
 ::
 
@@ -531,40 +531,40 @@ all standard database join operations between ``DataFrame`` objects:
              suffixes=('_x', '_y'), copy=True, indicator=False,
              validate=None)
 
-* ``left``: A DataFrame object.
-* ``right``: Another DataFrame object.
+* ``left``: A DataFrame or named Series object.
+* ``right``: Another DataFrame or named Series object.
 * ``on``: Column or index level names to join on. Must be found in both the left
-  and right DataFrame objects. If not passed and ``left_index`` and
+  and right DataFrame and/or Series objects. If not passed and ``left_index`` and
   ``right_index`` are ``False``, the intersection of the columns in the
-  DataFrames will be inferred to be the join keys.
-* ``left_on``: Columns or index levels from the left DataFrame to use as
+  DataFrames and/or Series will be inferred to be the join keys.
+* ``left_on``: Columns or index levels from the left DataFrame or Series to use as
   keys. Can either be column names, index level names, or arrays with length
-  equal to the length of the DataFrame.
-* ``right_on``: Columns or index levels from the right DataFrame to use as
+  equal to the length of the DataFrame or Series.
+* ``right_on``: Columns or index levels from the right DataFrame or Series to use as
   keys. Can either be column names, index level names, or arrays with length
-  equal to the length of the DataFrame.
+  equal to the length of the DataFrame or Series.
 * ``left_index``: If ``True``, use the index (row labels) from the left
-  DataFrame as its join key(s). In the case of a DataFrame with a MultiIndex
+  DataFrame or Series as its join key(s). In the case of a DataFrame or Series with a MultiIndex
   (hierarchical), the number of levels must match the number of join keys
-  from the right DataFrame.
-* ``right_index``: Same usage as ``left_index`` for the right DataFrame
+  from the right DataFrame or Series.
+* ``right_index``: Same usage as ``left_index`` for the right DataFrame or Series
 * ``how``: One of ``'left'``, ``'right'``, ``'outer'``, ``'inner'``. Defaults
   to ``inner``. See below for more detailed description of each method.
 * ``sort``: Sort the result DataFrame by the join keys in lexicographical
   order. Defaults to ``True``, setting to ``False`` will improve performance
   substantially in many cases.
 * ``suffixes``: A tuple of string suffixes to apply to overlapping
   columns. Defaults to ``('_x', '_y')``.
-* ``copy``: Always copy data (default ``True``) from the passed DataFrame
+* ``copy``: Always copy data (default ``True``) from the passed DataFrame or named Series
   objects, even when reindexing is not necessary. Cannot be avoided in many
   cases but may improve performance / memory usage. The cases where copying
   can be avoided are somewhat pathological but this option is provided
   nonetheless.
 * ``indicator``: Add a column to the output DataFrame called ``_merge``
   with information on the source of each row. ``_merge`` is Categorical-type
   and takes on a value of ``left_only`` for observations whose merge key
-  only appears in ``'left'`` DataFrame, ``right_only`` for observations whose
-  merge key only appears in ``'right'`` DataFrame, and ``both`` if the
+  only appears in ``'left'`` DataFrame or Series, ``right_only`` for observations whose
+  merge key only appears in ``'right'`` DataFrame or Series, and ``both`` if the
   observation's merge key is found in both.
 
 * ``validate`` : string, default None.
@@ -584,10 +584,10 @@ all standard database join operations between ``DataFrame`` objects:
 
    Support for specifying index levels as the ``on``, ``left_on``, and
    ``right_on`` parameters was added in version 0.23.0.
+   Support for merging named ``Series`` objects was added in version 0.24.0.
 
-The return type will be the same as ``left``. If ``left`` is a ``DataFrame``
-and ``right`` is a subclass of DataFrame, the return type will still be
-``DataFrame``.
+The return type will be the same as ``left``. If ``left`` is a ``DataFrame`` or named ``Series``
+and ``right`` is a subclass of ``DataFrame``, the return type will still be ``DataFrame``.
 
 ``merge`` is a function in the pandas namespace, and it is also available as a
 ``DataFrame`` instance method :meth:`~DataFrame.merge`, with the calling 

doc/source/style.ipynb

@@ -985,7 +985,10 @@
     "- `vertical-align`\n",
     "- `white-space: nowrap`\n",
     "\n",
-    "Only CSS2 named colors and hex colors of the form `#rgb` or `#rrggbb` are currently supported."
+    "Only CSS2 named colors and hex colors of the form `#rgb` or `#rrggbb` are currently supported.\n",
+    "\n",
+    "The following pseudo CSS properties are also available to set excel specific style properties:\n",
+    "- `number-format`\n"
    ]
   },
   {

doc/source/whatsnew.rst

@@ -20,6 +20,8 @@ These are new features and improvements of note in each release.
 
 .. include:: whatsnew/v0.24.0.txt
 
+.. include:: whatsnew/v0.23.3.txt
+
 .. include:: whatsnew/v0.23.2.txt
 
 .. include:: whatsnew/v0.23.1.txt

doc/source/whatsnew/v0.23.0.txt

@@ -1245,7 +1245,6 @@ Offsets
 - Bug in :class:`FY5253` where ``datetime`` addition and subtraction incremented incorrectly for dates on the year-end but not normalized to midnight (:issue:`18854`)
 - Bug in :class:`FY5253` where date offsets could incorrectly raise an ``AssertionError`` in arithmetic operations (:issue:`14774`)
 
-
 Numeric
 ^^^^^^^
 - Bug in :class:`Series` constructor with an int or float list where specifying ``dtype=str``, ``dtype='str'`` or ``dtype='U'`` failed to convert the data elements to strings (:issue:`16605`)

doc/source/whatsnew/v0.23.1.txt

@@ -6,6 +6,11 @@ v0.23.1 (June 12, 2018)
 This is a minor bug-fix release in the 0.23.x series and includes some small regression fixes
 and bug fixes. We recommend that all users upgrade to this version.
 
+.. warning::
+
+   Starting January 1, 2019, pandas feature releases will support Python 3 only.
+   See :ref:`install.dropping-27` for more.
+
 .. contents:: What's new in v0.23.1
     :local:
     :backlinks: none

doc/source/whatsnew/v0.23.2.txt

@@ -11,6 +11,10 @@ and bug fixes. We recommend that all users upgrade to this version.
    Pandas 0.23.2 is first pandas release that's compatible with
    Python 3.7 (:issue:`20552`)
 
+.. warning::
+
+   Starting January 1, 2019, pandas feature releases will support Python 3 only.
+   See :ref:`install.dropping-27` for more.
 
 .. contents:: What's new in v0.23.2
     :local:

doc/source/whatsnew/v0.23.4.txt

@@ -6,6 +6,10 @@ v0.23.4
 This is a minor bug-fix release in the 0.23.x series and includes some small regression fixes
 and bug fixes. We recommend that all users upgrade to this version.
 
+.. warning::
+
+   Starting January 1, 2019, pandas feature releases will support Python 3 only.
+   See :ref:`install.dropping-27` for more.
 
 .. contents:: What's new in v0.23.4
     :local:
@@ -16,7 +20,7 @@ and bug fixes. We recommend that all users upgrade to this version.
 Fixed Regressions
 ~~~~~~~~~~~~~~~~~
 
-- 
+- Python 3.7 with Windows gave all missing values for rolling variance calculations (:issue:`21813`)
 -
 
 .. _whatsnew_0234.bug_fixes:
@@ -27,6 +31,7 @@ Bug Fixes
 **Groupby/Resample/Rolling**
 
 - Bug where calling :func:`DataFrameGroupBy.agg` with a list of functions including ``ohlc`` as the non-initial element would raise a ``ValueError`` (:issue:`21716`)
+- Bug in ``roll_quantile`` caused a memory leak when calling ``.rolling(...).quantile(q)`` with ``q`` in (0,1) (:issue:`21965`)
 -
 
 **Conversion**
@@ -58,3 +63,7 @@ Bug Fixes
 
 -
 -
+
+**Missing**
+
+- Bug in :func:`Series.clip` and :func:`DataFrame.clip` cannot accept list-like threshold containing ``NaN`` (:issue:`19992`)