Merge remote-tracking branch 'upstream/master' into disown-tz-only-rebased

Author: TomAugspurger

asv_bench/benchmarks/frame_methods.py

@@ -103,6 +103,7 @@ def setup(self):
         self.df2 = DataFrame(np.random.randn(N * 50, 10))
         self.df3 = DataFrame(np.random.randn(N, 5 * N),
                              columns=['C' + str(c) for c in range(N * 5)])
+        self.df4 = DataFrame(np.random.randn(N * 1000, 10))
 
     def time_iteritems(self):
         # (monitor no-copying behaviour)
@@ -119,10 +120,70 @@ def time_iteritems_indexing(self):
         for col in self.df3:
             self.df3[col]
 
+    def time_itertuples_start(self):
+        self.df4.itertuples()
+
+    def time_itertuples_read_first(self):
+        next(self.df4.itertuples())
+
     def time_itertuples(self):
-        for row in self.df2.itertuples():
+        for row in self.df4.itertuples():
+            pass
+
+    def time_itertuples_to_list(self):
+        list(self.df4.itertuples())
+
+    def mem_itertuples_start(self):
+        return self.df4.itertuples()
+
+    def peakmem_itertuples_start(self):
+        self.df4.itertuples()
+
+    def mem_itertuples_read_first(self):
+        return next(self.df4.itertuples())
+
+    def peakmem_itertuples(self):
+        for row in self.df4.itertuples():
+            pass
+
+    def mem_itertuples_to_list(self):
+        return list(self.df4.itertuples())
+
+    def peakmem_itertuples_to_list(self):
+        list(self.df4.itertuples())
+
+    def time_itertuples_raw_start(self):
+        self.df4.itertuples(index=False, name=None)
+
+    def time_itertuples_raw_read_first(self):
+        next(self.df4.itertuples(index=False, name=None))
+
+    def time_itertuples_raw_tuples(self):
+        for row in self.df4.itertuples(index=False, name=None):
             pass
 
+    def time_itertuples_raw_tuples_to_list(self):
+        list(self.df4.itertuples(index=False, name=None))
+
+    def mem_itertuples_raw_start(self):
+        return self.df4.itertuples(index=False, name=None)
+
+    def peakmem_itertuples_raw_start(self):
+        self.df4.itertuples(index=False, name=None)
+
+    def peakmem_itertuples_raw_read_first(self):
+        next(self.df4.itertuples(index=False, name=None))
+
+    def peakmem_itertuples_raw(self):
+        for row in self.df4.itertuples(index=False, name=None):
+            pass
+
+    def mem_itertuples_raw_to_list(self):
+        return list(self.df4.itertuples(index=False, name=None))
+
+    def peakmem_itertuples_raw_to_list(self):
+        list(self.df4.itertuples(index=False, name=None))
+
     def time_iterrows(self):
         for row in self.df.iterrows():
             pass

azure-pipelines.yml

@@ -43,6 +43,7 @@ jobs:
       ci/incremental/install_miniconda.sh
       ci/incremental/setup_conda_environment.sh
     displayName: 'Set up environment'
+    condition: true
 
   # Do not require pandas
   - script: |

ci/code_checks.sh

@@ -158,7 +158,12 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     # RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check that no file in the repo contains tailing whitespaces' ; echo $MSG
-    invgrep --exclude="*.svg" -RI "\s$" *
+    set -o pipefail
+    if [[ "$AZURE" == "true" ]]; then
+        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+    else
+        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+    fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
 

doc/source/advanced.rst

@@ -778,12 +778,12 @@ a ``Categorical`` will return a ``CategoricalIndex``, indexed according to the c
 of the **passed** ``Categorical`` dtype. This allows one to arbitrarily index these even with
 values **not** in the categories, similarly to how you can reindex **any** pandas index.
 
-.. ipython :: python
+.. ipython:: python
 
-   df2.reindex(['a','e'])
-   df2.reindex(['a','e']).index
-   df2.reindex(pd.Categorical(['a','e'],categories=list('abcde')))
-   df2.reindex(pd.Categorical(['a','e'],categories=list('abcde'))).index
+   df2.reindex(['a', 'e'])
+   df2.reindex(['a', 'e']).index
+   df2.reindex(pd.Categorical(['a', 'e'], categories=list('abcde')))
+   df2.reindex(pd.Categorical(['a', 'e'], categories=list('abcde'))).index
 
 .. warning::
 
@@ -1040,7 +1040,8 @@ than integer locations. Therefore, with an integer axis index *only*
 label-based indexing is possible with the standard tools like ``.loc``. The
 following code will generate exceptions:
 
-.. code-block:: python
+.. ipython:: python
+   :okexcept:
 
    s = pd.Series(range(5))
    s[-1]
@@ -1130,7 +1131,7 @@ index can be somewhat complicated. For example, the following does not work:
 
 ::
 
-    s.loc['c':'e'+1]
+    s.loc['c':'e' + 1]
 
 A very common use case is to limit a time series to start and end at two
 specific dates. To enable this, we made the design to make label-based

doc/source/basics.rst

@@ -374,9 +374,7 @@ To evaluate single-element pandas objects in a boolean context, use the method
 
       >>> df and df2
 
-   These will both raise errors, as you are trying to compare multiple values.
-
-   .. code-block:: python-traceback
+   These will both raise errors, as you are trying to compare multiple values.::
 
        ValueError: The truth value of an array is ambiguous. Use a.empty, a.any() or a.all().
 

doc/source/categorical.rst

@@ -977,21 +977,17 @@ categorical (categories and ordering). So if you read back the CSV file you have
 relevant columns back to `category` and assign the right categories and categories ordering.
 
 .. ipython:: python
-    :suppress:
 
-
-.. ipython:: python
-
-    from pandas.compat import StringIO
+    import io
     s = pd.Series(pd.Categorical(['a', 'b', 'b', 'a', 'a', 'd']))
     # rename the categories
     s.cat.categories = ["very good", "good", "bad"]
     # reorder the categories and add missing categories
     s = s.cat.set_categories(["very bad", "bad", "medium", "good", "very good"])
     df = pd.DataFrame({"cats": s, "vals": [1, 2, 3, 4, 5, 6]})
-    csv = StringIO()
+    csv = io.StringIO()
     df.to_csv(csv)
-    df2 = pd.read_csv(StringIO(csv.getvalue()))
+    df2 = pd.read_csv(io.StringIO(csv.getvalue()))
     df2.dtypes
     df2["cats"]
     # Redo the category
@@ -1206,6 +1202,7 @@ Use ``copy=True`` to prevent such a behaviour or simply don't reuse ``Categorica
     cat
 
 .. note::
+
     This also happens in some cases when you supply a NumPy array instead of a ``Categorical``:
     using an int array (e.g. ``np.array([1,2,3,4])``) will exhibit the same behavior, while using
     a string array (e.g. ``np.array(["a","b","c","a"])``) will not.

doc/source/conf.py

@@ -296,7 +296,10 @@
    np.random.seed(123456)
    np.set_printoptions(precision=4, suppress=True)
    pd.options.display.max_rows = 15
-"""
+
+   import os
+   os.chdir('{}')
+""".format(os.path.dirname(os.path.dirname(__file__)))
 
 
 html_context = {

doc/source/cookbook.rst

@@ -1236,7 +1236,7 @@ the following Python code will read the binary file ``'binary.dat'`` into a
 pandas ``DataFrame``, where each element of the struct corresponds to a column
 in the frame:
 
-.. code-block:: python
+.. ipython:: python
 
    names = 'count', 'avg', 'scale'
 
@@ -1399,7 +1399,6 @@ of the data values:
 
 .. ipython:: python
 
-
    def expand_grid(data_dict):
        rows = itertools.product(*data_dict.values())
        return pd.DataFrame.from_records(rows, columns=data_dict.keys())

doc/source/gotchas.rst

@@ -301,9 +301,7 @@ Byte-Ordering Issues
 --------------------
 Occasionally you may have to deal with data that were created on a machine with
 a different byte order than the one on which you are running Python. A common
-symptom of this issue is an error like:
-
-.. code-block:: python-traceback
+symptom of this issue is an error like:::
 
     Traceback
         ...

doc/source/io.rst

@@ -1,5 +1,8 @@
 .. _io:
 
+.. currentmodule:: pandas
+
+
 {{ header }}
 
 .. ipython:: python
@@ -4876,7 +4879,7 @@ below and the SQLAlchemy `documentation <https://docs.sqlalchemy.org/en/latest/c
 
 If you want to manage your own connections you can pass one of those instead:
 
-.. code-block:: python
+.. ipython:: python
 
    with engine.connect() as conn, conn.begin():
        data = pd.read_sql_table('data', conn)

doc/source/merging.rst

@@ -1122,6 +1122,8 @@ This is equivalent but less verbose and more memory efficient / faster than this
           labels=['left', 'right'], vertical=False);
    plt.close('all');
 
+.. _merging.join_with_two_multi_indexes:
+
 Joining with two MultiIndexes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~