Merge branch 'master' of https://github.com/pandas-dev/pandas

Author: Si Wei How

.travis.yml

@@ -86,6 +86,14 @@ install:
   - ci/submit_cython_cache.sh
   - echo "install done"
 
+before_script:
+  # display server (for clipboard functionality) needs to be started here,
+  # does not work if done in install:setup_env.sh (GH-26103)
+  - export DISPLAY=":99.0"
+  - echo "sh -e /etc/init.d/xvfb start"
+  - sh -e /etc/init.d/xvfb start
+  - sleep 3
+
 script:
   - echo "script start"
   - source activate pandas-dev

ci/azure/windows.yml

@@ -17,16 +17,15 @@ jobs:
         CONDA_PY: "37"
 
   steps:
-    - task: CondaEnvironment@1
-      inputs:
-        updateConda: no
-        packageSpecs: ''
-
-    - script: |
-        ci\\incremental\\setup_conda_environment.cmd
-      displayName: 'Before Install'
+    - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+      displayName: Add conda to PATH
+    - script: conda update -q -n base conda
+      displayName: Update conda
+    - script: conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
+      displayName: Create anaconda environment
     - script: |
         call activate pandas-dev
+        call conda list
         ci\\incremental\\build.cmd
       displayName: 'Build'
     - script: |

ci/code_checks.sh

@@ -169,15 +169,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -r -E --include '*.py' '(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)' pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    # Check that we use pytest.raises only as a context manager
-    #
-    # For any flake8-compliant code, the only way this regex gets
-    # matched is if there is no "with" statement preceding "pytest.raises"
-    MSG='Check for pytest.raises as context manager (a line starting with `pytest.raises` is invalid, needs a `with` to precede it)' ; echo $MSG
-    MSG='TODO: This check is currently skipped because so many files fail this. Please enable when all are corrected (xref gh-24332)' ; echo $MSG
-    # invgrep -R --include '*.py' -E '[[:space:]] pytest.raises' pandas/tests
-    # RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     MSG='Check for wrong space after code-block directive and before colon (".. code-block ::" instead of ".. code-block::")' ; echo $MSG
     invgrep -R --include="*.rst" ".. code-block ::" doc/source
     RET=$(($RET + $?)) ; echo $MSG "DONE"
@@ -239,6 +230,10 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     pytest -q --doctest-modules pandas/core/groupby/groupby.py -k"-cumcount -describe -pipe"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Doctests datetimes.py' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/tools/datetimes.py
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Doctests top-level reshaping functions' ; echo $MSG
     pytest -q --doctest-modules \
         pandas/core/reshape/concat.py \

ci/deps/azure-windows-37.yaml

@@ -1,9 +1,11 @@
 name: pandas-dev
 channels:
   - defaults
+  - conda-forge
 dependencies:
   - beautifulsoup4
   - bottleneck
+  - gcsfs
   - html5lib
   - jinja2
   - lxml

ci/incremental/setup_conda_environment.cmd

@@ -118,16 +118,10 @@ echo "conda list"
 conda list
 
 # Install DB for Linux
-export DISPLAY=":99."
 if [ ${TRAVIS_OS_NAME} == "linux" ]; then
   echo "installing dbs"
   mysql -e 'create database pandas_nosetest;'
   psql -c 'create database pandas_nosetest;' -U postgres
-
-  echo
-  echo "sh -e /etc/init.d/xvfb start"
-  sh -e /etc/init.d/xvfb start
-  sleep 3
 else
    echo "not using dbs on non-linux"
 fi

ci/setup_env.sh

@@ -285,6 +285,11 @@ provides a familiar ``DataFrame`` interface for out-of-core, parallel and distri
 
 Dask-ML enables parallel and distributed machine learning using Dask alongside existing machine learning libraries like Scikit-Learn, XGBoost, and TensorFlow.
 
+`Koalas <https://koalas.readthedocs.io/en/latest/>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Koalas provides a familiar pandas DataFrame interface on top of Apache Spark. It enables users to leverage multi-cores on one machine or a cluster of machines to speed up or scale their DataFrame code.
+
 `Odo <http://odo.pydata.org>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/ecosystem.rst

@@ -865,7 +865,7 @@ which is equivalent to using weights
 
 The difference between the above two variants arises because we are
 dealing with series which have finite history. Consider a series of infinite
-history:
+history, with ``adjust=True``:
 
 .. math::
 
@@ -884,10 +884,11 @@ and a ratio of :math:`1 - \alpha` we have
     &= \alpha x_t + (1 - \alpha)[x_{t-1} + (1 - \alpha) x_{t-2} + ...]\alpha\\
     &= \alpha x_t + (1 - \alpha) y_{t-1}
 
-which shows the equivalence of the above two variants for infinite series.
-When ``adjust=True`` we have :math:`y_0 = x_0` and from the last
-representation above we have :math:`y_t = \alpha x_t + (1 - \alpha) y_{t-1}`,
-therefore there is an assumption that :math:`x_0` is not an ordinary value
+which is the same expression as ``adjust=False`` above and therefore
+shows the equivalence of the two variants for infinite series.
+When ``adjust=False``, we have :math:`y_0 = x_0` and
+:math:`y_t = \alpha x_t + (1 - \alpha) y_{t-1}`.
+Therefore, there is an assumption that :math:`x_0` is not an ordinary value
 but rather an exponentially weighted moment of the infinite series up to that
 point.
 

doc/source/user_guide/computation.rst

@@ -238,14 +238,9 @@ Enhancements
 
     - support ``read_hdf/to_hdf`` API similar to ``read_csv/to_csv``
 
-      .. ipython:: python
-          :suppress:
-
-          from pandas.compat import lrange
-
       .. ipython:: python
 
-          df = pd.DataFrame({'A': lrange(5), 'B': lrange(5)})
+          df = pd.DataFrame({'A': range(5), 'B': range(5)})
           df.to_hdf('store.h5', 'table', append=True)
           pd.read_hdf('store.h5', 'table', where=['index > 2'])
 

doc/source/whatsnew/v0.11.0.rst

@@ -83,13 +83,8 @@ API changes
     ``iloc`` API to be *purely* positional based.
 
     .. ipython:: python
-       :suppress:
 
-       from pandas.compat import lrange
-
-    .. ipython:: python
-
-       df = pd.DataFrame(lrange(5), list('ABCDE'), columns=['a'])
+       df = pd.DataFrame(range(5), index=list('ABCDE'), columns=['a'])
        mask = (df.a % 2 == 0)
        mask
 

doc/source/whatsnew/v0.12.0.rst

@@ -250,14 +250,19 @@ Other API Changes
 - Comparing :class:`Timestamp` with unsupported objects now returns :py:obj:`NotImplemented` instead of raising ``TypeError``. This implies that unsupported rich comparisons are delegated to the other object, and are now consistent with Python 3 behavior for ``datetime`` objects (:issue:`24011`)
 - Bug in :meth:`DatetimeIndex.snap` which didn't preserving the ``name`` of the input :class:`Index` (:issue:`25575`)
 - The ``arg`` argument in :meth:`pandas.core.groupby.DataFrameGroupBy.agg` has been renamed to ``func`` (:issue:`26089`)
+- The ``arg`` argument in :meth:`pandas.core.window._Window.aggregate` has been renamed to ``func`` (:issue:`26372`)
+- Most Pandas classes had a ``__bytes__`` method, which was used for getting a python2-style bytestring representation of the object. This method has been removed as a part of dropping Python2 (:issue:`26447`)
 
 .. _whatsnew_0250.deprecations:
 
 Deprecations
 ~~~~~~~~~~~~
 
+- The deprecated ``.ix[]`` indexer now raises a more visible FutureWarning instead of DeprecationWarning (:issue:`26438`).
 - Deprecated the ``units=M`` (months) and ``units=Y`` (year) parameters for ``units`` of :func:`pandas.to_timedelta`, :func:`pandas.Timedelta` and :func:`pandas.TimedeltaIndex` (:issue:`16344`)
 - The functions :func:`pandas.to_datetime` and :func:`pandas.to_timedelta` have deprecated the ``box`` keyword. Instead, use :meth:`to_numpy` or :meth:`Timestamp.to_datetime64` or :meth:`Timedelta.to_timedelta64`. (:issue:`24416`)
+- The :meth:`DataFrame.compound` and :meth:`Series.compound` methods are deprecated and will be removed in a future version.
+
 
 .. _whatsnew_0250.prior_deprecations:
 
@@ -294,11 +299,10 @@ Bug Fixes
 ~~~~~~~~~
 
 
-
 Categorical
 ^^^^^^^^^^^
 
--
+- Bug in :func:`DataFrame.at` and :func:`Series.at` that would raise exception if the index was a :class:`CategoricalIndex` (:issue:`20629`)
 -
 -
 
@@ -318,7 +322,7 @@ Timedelta
 
 - Bug in :func:`TimedeltaIndex.intersection` where for non-monotonic indices in some cases an empty ``Index`` was returned when in fact an intersection existed (:issue:`25913`)
 - Bug with comparisons between :class:`Timedelta` and ``NaT`` raising ``TypeError`` (:issue:`26039`)
--
+- Bug when adding or subtracting a :class:`BusinessHour` to a :class:`Timestamp` with the resulting time landing in a following or prior day respectively (:issue:`26381`)
 
 Timezones
 ^^^^^^^^^
@@ -374,7 +378,9 @@ Indexing
 - Improved exception message when calling :meth:`DataFrame.iloc` with a list of non-numeric objects (:issue:`25753`).
 - Bug in :meth:`DataFrame.loc` and :meth:`Series.loc` where ``KeyError`` was not raised for a ``MultiIndex`` when the key was less than or equal to the number of levels in the :class:`MultiIndex` (:issue:`14885`).
 - Bug in which :meth:`DataFrame.append` produced an erroneous warning indicating that a ``KeyError`` will be thrown in the future when the data to be appended contains new columns (:issue:`22252`).
--
+- Bug in which :meth:`DataFrame.to_csv` caused a segfault for a reindexed data frame, when the indices were single-level :class:`MultiIndex` (:issue:`26303`).
+- Fixed bug where assigning a :class:`arrays.PandasArray` to a :class:`pandas.core.frame.DataFrame` would raise error (:issue:`26390`)
+- Allow keyword arguments for callable local reference used in the :method:`DataFrame.query` string (:issue:`26426`)
 
 
 Missing
@@ -441,7 +447,9 @@ Groupby/Resample/Rolling
 - Bug in :meth:`pandas.core.groupby.GroupBy.idxmax` and :meth:`pandas.core.groupby.GroupBy.idxmin` with datetime column would return incorrect dtype (:issue:`25444`, :issue:`15306`)
 - Bug in :meth:`pandas.core.groupby.GroupBy.cumsum`, :meth:`pandas.core.groupby.GroupBy.cumprod`, :meth:`pandas.core.groupby.GroupBy.cummin` and :meth:`pandas.core.groupby.GroupBy.cummax` with categorical column having absent categories, would return incorrect result or segfault (:issue:`16771`)
 - Bug in :meth:`pandas.core.groupby.GroupBy.nth` where NA values in the grouping would return incorrect results (:issue:`26011`)
-
+- Bug in :meth:`pandas.core.groupby.SeriesGroupBy.transform` where transforming an empty group would raise error (:issue:`26208`)
+- Bug in :meth:`pandas.core.frame.DataFrame.groupby` where passing a :class:`pandas.core.groupby.grouper.Grouper` would return incorrect groups when using the ``.groups`` accessor (:issue:`26326`)
+- Bug in :meth:`pandas.core.groupby.GroupBy.agg` where incorrect results are returned for uint64 columns. (:issue:`26310`)
 
 Reshaping
 ^^^^^^^^^
@@ -453,8 +461,10 @@ Reshaping
 - Bug in :func:`pivot_table` where columns with ``NaN`` values are dropped even if ``dropna`` argument is ``False``, when the ``aggfunc`` argument contains a ``list`` (:issue:`22159`)
 - Bug in :func:`concat` where the resulting ``freq`` of two :class:`DatetimeIndex` with the same ``freq`` would be dropped (:issue:`3232`).
 - Bug in :func:`merge` where merging with equivalent Categorical dtypes was raising an error (:issue:`22501`)
+- bug in :class:`DataFrame` instantiating with a dict of iterators or generators (e.g. ``pd.DataFrame({'A': reversed(range(3))})``) raised an error (:issue:`26349`).
 - bug in :class:`DataFrame` instantiating with a ``range`` (e.g. ``pd.DataFrame(range(3))``) raised an error (:issue:`26342`).
 - Bug in :class:`DataFrame` constructor when passing non-empty tuples would cause a segmentation fault (:issue:`25691`)
+- Bug in :func:`Series.apply` failed when the series is a timezone aware :class:`DatetimeIndex` (:issue:`25959`)
 - Bug in :func:`pandas.cut` where large bins could incorrectly raise an error due to an integer overflow (:issue:`26045`)
 - Bug in :func:`DataFrame.sort_index` where an error is thrown when a multi-indexed DataFrame is sorted on all levels with the initial level sorted last (:issue:`26053`)
 - Bug in :meth:`Series.nlargest` treats ``True`` as smaller than ``False`` (:issue:`26154`)
@@ -474,7 +484,6 @@ Other
 - Bug in :func:`factorize` when passing an ``ExtensionArray`` with a custom ``na_sentinel`` (:issue:`25696`).
 - Allow :class:`Index` and :class:`RangeIndex` to be passed to numpy ``min`` and ``max`` functions.
 
-
 .. _whatsnew_0.250.contributors:
 
 Contributors

doc/source/whatsnew/v0.25.0.rst

@@ -15,43 +15,4 @@ ignore_errors=True
 ignore_errors=True
 
 [mypy-pandas.core.indexes.timedeltas]
-ignore_errors=True
-
-[mypy-pandas.core.indexing]
-ignore_errors=True
-
-[mypy-pandas.core.internals.blocks]
-ignore_errors=True
-
-[mypy-pandas.core.ops]
-ignore_errors=True
-
-[mypy-pandas.core.panel]
-ignore_errors=True
-
-[mypy-pandas.core.resample]
-ignore_errors=True
-
-[mypy-pandas.core.reshape.merge]
-ignore_errors=True
-
-[mypy-pandas.core.reshape.reshape]
-ignore_errors=True
-
-[mypy-pandas.core.series]
-ignore_errors=True
-
-[mypy-pandas.core.util.hashing]
-ignore_errors=True
-
-[mypy-pandas.core.window]
-ignore_errors=True
-
-[mypy-pandas.io.pytables]
-ignore_errors=True
-
-[mypy-pandas.util._doctools]
-ignore_errors=True
-
-[mypy-pandas.util.testing]
-ignore_errors=True
+ignore_errors=True

mypy.ini

@@ -149,9 +149,6 @@ cdef extern from "parser/tokenizer.h":
         int skipinitialspace       # ignore spaces following delimiter? */
         int quoting                # style of quoting to write */
 
-        # hmm =/
-        # int numeric_field
-
         char commentchar
         int allow_embedded_newline
         int strict                 # raise exception on bad CSV */

pandas/_libs/parsers.pyx

@@ -10,7 +10,8 @@ The full license is in the LICENSE file, distributed with this software.
 #ifndef PANDAS__LIBS_SRC_PARSER_IO_H_
 #define PANDAS__LIBS_SRC_PARSER_IO_H_
 
-#include "Python.h"
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
 #include "tokenizer.h"
 
 typedef struct _file_source {
@@ -37,8 +38,6 @@ typedef struct _memory_map {
     size_t position;
 } memory_map;
 
-#define MM(src) ((memory_map *)src)
-
 void *new_mmap(char *fname);
 
 int del_mmap(void *src);