Merge branch 'master' into doc-fix-flake8-issue-in-groupby.rst-pandas-dev#24178

Author: LJArendse

asv_bench/benchmarks/frame_methods.py

@@ -103,6 +103,7 @@ def setup(self):
         self.df2 = DataFrame(np.random.randn(N * 50, 10))
         self.df3 = DataFrame(np.random.randn(N, 5 * N),
                              columns=['C' + str(c) for c in range(N * 5)])
+        self.df4 = DataFrame(np.random.randn(N * 1000, 10))
 
     def time_iteritems(self):
         # (monitor no-copying behaviour)
@@ -119,10 +120,70 @@ def time_iteritems_indexing(self):
         for col in self.df3:
             self.df3[col]
 
+    def time_itertuples_start(self):
+        self.df4.itertuples()
+
+    def time_itertuples_read_first(self):
+        next(self.df4.itertuples())
+
     def time_itertuples(self):
-        for row in self.df2.itertuples():
+        for row in self.df4.itertuples():
+            pass
+
+    def time_itertuples_to_list(self):
+        list(self.df4.itertuples())
+
+    def mem_itertuples_start(self):
+        return self.df4.itertuples()
+
+    def peakmem_itertuples_start(self):
+        self.df4.itertuples()
+
+    def mem_itertuples_read_first(self):
+        return next(self.df4.itertuples())
+
+    def peakmem_itertuples(self):
+        for row in self.df4.itertuples():
+            pass
+
+    def mem_itertuples_to_list(self):
+        return list(self.df4.itertuples())
+
+    def peakmem_itertuples_to_list(self):
+        list(self.df4.itertuples())
+
+    def time_itertuples_raw_start(self):
+        self.df4.itertuples(index=False, name=None)
+
+    def time_itertuples_raw_read_first(self):
+        next(self.df4.itertuples(index=False, name=None))
+
+    def time_itertuples_raw_tuples(self):
+        for row in self.df4.itertuples(index=False, name=None):
             pass
 
+    def time_itertuples_raw_tuples_to_list(self):
+        list(self.df4.itertuples(index=False, name=None))
+
+    def mem_itertuples_raw_start(self):
+        return self.df4.itertuples(index=False, name=None)
+
+    def peakmem_itertuples_raw_start(self):
+        self.df4.itertuples(index=False, name=None)
+
+    def peakmem_itertuples_raw_read_first(self):
+        next(self.df4.itertuples(index=False, name=None))
+
+    def peakmem_itertuples_raw(self):
+        for row in self.df4.itertuples(index=False, name=None):
+            pass
+
+    def mem_itertuples_raw_to_list(self):
+        return list(self.df4.itertuples(index=False, name=None))
+
+    def peakmem_itertuples_raw_to_list(self):
+        list(self.df4.itertuples(index=False, name=None))
+
     def time_iterrows(self):
         for row in self.df.iterrows():
             pass

asv_bench/benchmarks/join_merge.py

@@ -50,7 +50,7 @@ def setup(self, axis):
         self.empty_right = [df, DataFrame()]
 
     def time_concat_series(self, axis):
-        concat(self.series, axis=axis)
+        concat(self.series, axis=axis, sort=False)
 
     def time_concat_small_frames(self, axis):
         concat(self.small_frames, axis=axis)

asv_bench/benchmarks/panel_ctor.py

@@ -1,7 +1,7 @@
 import warnings
 from datetime import datetime, timedelta
 
-from pandas import DataFrame, Panel, DatetimeIndex, date_range
+from pandas import DataFrame, Panel, date_range
 
 
 class DifferentIndexes(object):
@@ -23,9 +23,9 @@ def time_from_dict(self):
 class SameIndexes(object):
 
     def setup(self):
-        idx = DatetimeIndex(start=datetime(1990, 1, 1),
-                            end=datetime(2012, 1, 1),
-                            freq='D')
+        idx = date_range(start=datetime(1990, 1, 1),
+                         end=datetime(2012, 1, 1),
+                         freq='D')
         df = DataFrame({'a': 0, 'b': 1, 'c': 2}, index=idx)
         self.data_frames = dict(enumerate([df] * 100))
 
@@ -40,10 +40,10 @@ def setup(self):
         start = datetime(1990, 1, 1)
         end = datetime(2012, 1, 1)
         df1 = DataFrame({'a': 0, 'b': 1, 'c': 2},
-                        index=DatetimeIndex(start=start, end=end, freq='D'))
+                        index=date_range(start=start, end=end, freq='D'))
         end += timedelta(days=1)
         df2 = DataFrame({'a': 0, 'b': 1, 'c': 2},
-                        index=DatetimeIndex(start=start, end=end, freq='D'))
+                        index=date_range(start=start, end=end, freq='D'))
         dfs = [df1] * 50 + [df2] * 50
         self.data_frames = dict(enumerate(dfs))
 

asv_bench/benchmarks/reindex.py

@@ -1,14 +1,14 @@
 import numpy as np
 import pandas.util.testing as tm
-from pandas import (DataFrame, Series, DatetimeIndex, MultiIndex, Index,
+from pandas import (DataFrame, Series, MultiIndex, Index,
                     date_range)
 from .pandas_vb_common import lib
 
 
 class Reindex(object):
 
     def setup(self):
-        rng = DatetimeIndex(start='1/1/1970', periods=10000, freq='1min')
+        rng = date_range(start='1/1/1970', periods=10000, freq='1min')
         self.df = DataFrame(np.random.rand(10000, 10), index=rng,
                             columns=range(10))
         self.df['foo'] = 'bar'

asv_bench/benchmarks/timedelta.py

@@ -1,8 +1,9 @@
 import datetime
 
 import numpy as np
-from pandas import Series, timedelta_range, to_timedelta, Timestamp, \
-    Timedelta, TimedeltaIndex, DataFrame
+
+from pandas import (
+    DataFrame, Series, Timedelta, Timestamp, timedelta_range, to_timedelta)
 
 
 class TimedeltaConstructor(object):
@@ -122,8 +123,8 @@ def time_timedelta_nanoseconds(self, series):
 class TimedeltaIndexing(object):
 
     def setup(self):
-        self.index = TimedeltaIndex(start='1985', periods=1000, freq='D')
-        self.index2 = TimedeltaIndex(start='1986', periods=1000, freq='D')
+        self.index = timedelta_range(start='1985', periods=1000, freq='D')
+        self.index2 = timedelta_range(start='1986', periods=1000, freq='D')
         self.series = Series(range(1000), index=self.index)
         self.timedelta = self.index[500]
 

asv_bench/benchmarks/timestamp.py

@@ -1,8 +1,9 @@
 import datetime
 
-from pandas import Timestamp
-import pytz
 import dateutil
+import pytz
+
+from pandas import Timestamp
 
 
 class TimestampConstruction(object):
@@ -46,7 +47,7 @@ def time_dayofweek(self, tz, freq):
         self.ts.dayofweek
 
     def time_weekday_name(self, tz, freq):
-        self.ts.weekday_name
+        self.ts.day_name
 
     def time_dayofyear(self, tz, freq):
         self.ts.dayofyear

azure-pipelines.yml

@@ -43,6 +43,7 @@ jobs:
       ci/incremental/install_miniconda.sh
       ci/incremental/setup_conda_environment.sh
     displayName: 'Set up environment'
+    condition: true
 
   # Do not require pandas
   - script: |

ci/code_checks.sh

@@ -145,7 +145,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check that the deprecated `assert_raises_regex` is not used (`pytest.raises(match=pattern)` should be used instead)' ; echo $MSG
-    invgrep -R --exclude=*.pyc --exclude=testing.py --exclude=test_testing.py assert_raises_regex pandas
+    invgrep -R --exclude=*.pyc --exclude=testing.py --exclude=test_util.py assert_raises_regex pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Check that we use pytest.raises only as a context manager
@@ -158,7 +158,12 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     # RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check that no file in the repo contains tailing whitespaces' ; echo $MSG
-    invgrep --exclude="*.svg" -RI "\s$" *
+    set -o pipefail
+    if [[ "$AZURE" == "true" ]]; then
+        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+    else
+        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+    fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
 

doc/source/advanced.rst

@@ -778,12 +778,12 @@ a ``Categorical`` will return a ``CategoricalIndex``, indexed according to the c
 of the **passed** ``Categorical`` dtype. This allows one to arbitrarily index these even with
 values **not** in the categories, similarly to how you can reindex **any** pandas index.
 
-.. ipython :: python
+.. ipython:: python
 
-   df2.reindex(['a','e'])
-   df2.reindex(['a','e']).index
-   df2.reindex(pd.Categorical(['a','e'],categories=list('abcde')))
-   df2.reindex(pd.Categorical(['a','e'],categories=list('abcde'))).index
+   df2.reindex(['a', 'e'])
+   df2.reindex(['a', 'e']).index
+   df2.reindex(pd.Categorical(['a', 'e'], categories=list('abcde')))
+   df2.reindex(pd.Categorical(['a', 'e'], categories=list('abcde'))).index
 
 .. warning::
 
@@ -1040,7 +1040,8 @@ than integer locations. Therefore, with an integer axis index *only*
 label-based indexing is possible with the standard tools like ``.loc``. The
 following code will generate exceptions:
 
-.. code-block:: python
+.. ipython:: python
+   :okexcept:
 
    s = pd.Series(range(5))
    s[-1]
@@ -1130,7 +1131,7 @@ index can be somewhat complicated. For example, the following does not work:
 
 ::
 
-    s.loc['c':'e'+1]
+    s.loc['c':'e' + 1]
 
 A very common use case is to limit a time series to start and end at two
 specific dates. To enable this, we made the design to make label-based

doc/source/basics.rst

@@ -374,9 +374,7 @@ To evaluate single-element pandas objects in a boolean context, use the method
 
       >>> df and df2
 
-   These will both raise errors, as you are trying to compare multiple values.
-
-   .. code-block:: python-traceback
+   These will both raise errors, as you are trying to compare multiple values.::
 
        ValueError: The truth value of an array is ambiguous. Use a.empty, a.any() or a.all().
 

doc/source/categorical.rst

@@ -977,21 +977,17 @@ categorical (categories and ordering). So if you read back the CSV file you have
 relevant columns back to `category` and assign the right categories and categories ordering.
 
 .. ipython:: python
-    :suppress:
 
-
-.. ipython:: python
-
-    from pandas.compat import StringIO
+    import io
     s = pd.Series(pd.Categorical(['a', 'b', 'b', 'a', 'a', 'd']))
     # rename the categories
     s.cat.categories = ["very good", "good", "bad"]
     # reorder the categories and add missing categories
     s = s.cat.set_categories(["very bad", "bad", "medium", "good", "very good"])
     df = pd.DataFrame({"cats": s, "vals": [1, 2, 3, 4, 5, 6]})
-    csv = StringIO()
+    csv = io.StringIO()
     df.to_csv(csv)
-    df2 = pd.read_csv(StringIO(csv.getvalue()))
+    df2 = pd.read_csv(io.StringIO(csv.getvalue()))
     df2.dtypes
     df2["cats"]
     # Redo the category
@@ -1145,7 +1141,8 @@ dtype in apply
 
 Pandas currently does not preserve the dtype in apply functions: If you apply along rows you get
 a `Series` of ``object`` `dtype` (same as getting a row -> getting one element will return a
-basic type) and applying along columns will also convert to object.
+basic type) and applying along columns will also convert to object. ``NaN`` values are unaffected.
+You can use ``fillna`` to handle missing values before applying a function.
 
 .. ipython:: python
 
@@ -1205,6 +1202,7 @@ Use ``copy=True`` to prevent such a behaviour or simply don't reuse ``Categorica
     cat
 
 .. note::
+
     This also happens in some cases when you supply a NumPy array instead of a ``Categorical``:
     using an int array (e.g. ``np.array([1,2,3,4])``) will exhibit the same behavior, while using
     a string array (e.g. ``np.array(["a","b","c","a"])``) will not.

doc/source/conf.py

@@ -296,7 +296,10 @@
    np.random.seed(123456)
    np.set_printoptions(precision=4, suppress=True)
    pd.options.display.max_rows = 15
-"""
+
+   import os
+   os.chdir('{}')
+""".format(os.path.dirname(os.path.dirname(__file__)))
 
 
 html_context = {

doc/source/contributing.rst

@@ -125,21 +125,25 @@ requires a C compiler and Python environment. If you're making documentation
 changes, you can skip to :ref:`contributing.documentation` but you won't be able
 to build the documentation locally before pushing your changes.
 
-.. _contributiong.dev_c:
+.. _contributing.dev_c:
 
 Installing a C Compiler
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 Pandas uses C extensions (mostly written using Cython) to speed up certain
 operations. To install pandas from source, you need to compile these C
 extensions, which means you need a C compiler. This process depends on which
-platform you're using. Follow the `CPython contributing guidelines
-<https://docs.python.org/devguide/setup.html#build-dependencies>`_ for getting a
+platform you're using. Follow the `CPython contributing guide
+<https://devguide.python.org/setup/#compile-and-build>`_ for getting a
 compiler installed. You don't need to do any of the ``./configure`` or ``make``
 steps; you only need to install the compiler.
 
-For Windows developers, the following links may be helpful.
+For Windows developers, when using Python 3.5 and later, it is sufficient to
+install `Visual Studio 2017 <https://visualstudio.com/>`_ with the
+**Python development workload** and the **Python native development tools**
+option. Otherwise, the following links may be helpful.
 
+* https://blogs.msdn.microsoft.com/pythonengineering/2017/03/07/python-support-in-vs2017/
 * https://blogs.msdn.microsoft.com/pythonengineering/2016/04/11/unable-to-find-vcvarsall-bat/
 * https://github.com/conda/conda-recipes/wiki/Building-from-Source-on-Windows-32-bit-and-64-bit
 * https://cowboyprogrammer.org/building-python-wheels-for-windows/
@@ -149,7 +153,7 @@ For Windows developers, the following links may be helpful.
 Let us know if you have any difficulties by opening an issue or reaching out on
 `Gitter`_.
 
-.. _contributiong.dev_python:
+.. _contributing.dev_python:
 
 Creating a Python Environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/cookbook.rst

@@ -1236,7 +1236,7 @@ the following Python code will read the binary file ``'binary.dat'`` into a
 pandas ``DataFrame``, where each element of the struct corresponds to a column
 in the frame:
 
-.. code-block:: python
+.. ipython:: python
 
    names = 'count', 'avg', 'scale'
 
@@ -1399,7 +1399,6 @@ of the data values:
 
 .. ipython:: python
 
-
    def expand_grid(data_dict):
        rows = itertools.product(*data_dict.values())
        return pd.DataFrame.from_records(rows, columns=data_dict.keys())

doc/source/gotchas.rst

@@ -301,9 +301,7 @@ Byte-Ordering Issues
 --------------------
 Occasionally you may have to deal with data that were created on a machine with
 a different byte order than the one on which you are running Python. A common
-symptom of this issue is an error like:
-
-.. code-block:: python-traceback
+symptom of this issue is an error like:::
 
     Traceback
         ...