Merge remote-tracking branch 'upstream/master' into bug26390

Author: shantanu-gontia

.travis.yml

@@ -86,6 +86,14 @@ install:
   - ci/submit_cython_cache.sh
   - echo "install done"
 
+before_script:
+  # display server (for clipboard functionality) needs to be started here,
+  # does not work if done in install:setup_env.sh (GH-26103)
+  - export DISPLAY=":99.0"
+  - echo "sh -e /etc/init.d/xvfb start"
+  - sh -e /etc/init.d/xvfb start
+  - sleep 3
+
 script:
   - echo "script start"
   - source activate pandas-dev

ci/azure/windows.yml

@@ -19,10 +19,13 @@ jobs:
   steps:
     - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
       displayName: Add conda to PATH
-    - script: conda env create --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
+    - script: conda update -q -n base conda
+      displayName: Update conda
+    - script: conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
       displayName: Create anaconda environment
     - script: |
         call activate pandas-dev
+        call conda list
         ci\\incremental\\build.cmd
       displayName: 'Build'
     - script: |

ci/deps/azure-windows-37.yaml

@@ -1,9 +1,11 @@
 name: pandas-dev
 channels:
   - defaults
+  - conda-forge
 dependencies:
   - beautifulsoup4
   - bottleneck
+  - gcsfs
   - html5lib
   - jinja2
   - lxml

ci/setup_env.sh

@@ -118,16 +118,10 @@ echo "conda list"
 conda list
 
 # Install DB for Linux
-export DISPLAY=":99."
 if [ ${TRAVIS_OS_NAME} == "linux" ]; then
   echo "installing dbs"
   mysql -e 'create database pandas_nosetest;'
   psql -c 'create database pandas_nosetest;' -U postgres
-
-  echo
-  echo "sh -e /etc/init.d/xvfb start"
-  sh -e /etc/init.d/xvfb start
-  sleep 3
 else
    echo "not using dbs on non-linux"
 fi

doc/source/whatsnew/v0.11.0.rst

@@ -238,14 +238,9 @@ Enhancements
 
     - support ``read_hdf/to_hdf`` API similar to ``read_csv/to_csv``
 
-      .. ipython:: python
-          :suppress:
-
-          from pandas.compat import lrange
-
       .. ipython:: python
 
-          df = pd.DataFrame({'A': lrange(5), 'B': lrange(5)})
+          df = pd.DataFrame({'A': range(5), 'B': range(5)})
           df.to_hdf('store.h5', 'table', append=True)
           pd.read_hdf('store.h5', 'table', where=['index > 2'])
 

doc/source/whatsnew/v0.12.0.rst

@@ -83,13 +83,8 @@ API changes
     ``iloc`` API to be *purely* positional based.
 
     .. ipython:: python
-       :suppress:
 
-       from pandas.compat import lrange
-
-    .. ipython:: python
-
-       df = pd.DataFrame(lrange(5), list('ABCDE'), columns=['a'])
+       df = pd.DataFrame(range(5), index=list('ABCDE'), columns=['a'])
        mask = (df.a % 2 == 0)
        mask
 

doc/source/whatsnew/v0.25.0.rst

@@ -257,6 +257,8 @@ Deprecations
 
 - Deprecated the ``units=M`` (months) and ``units=Y`` (year) parameters for ``units`` of :func:`pandas.to_timedelta`, :func:`pandas.Timedelta` and :func:`pandas.TimedeltaIndex` (:issue:`16344`)
 - The functions :func:`pandas.to_datetime` and :func:`pandas.to_timedelta` have deprecated the ``box`` keyword. Instead, use :meth:`to_numpy` or :meth:`Timestamp.to_datetime64` or :meth:`Timedelta.to_timedelta64`. (:issue:`24416`)
+- The :meth:`DataFrame.compound` and :meth:`Series.compound` methods are deprecated and will be removed in a future version.
+
 
 .. _whatsnew_0250.prior_deprecations:
 
@@ -373,7 +375,7 @@ Indexing
 - Improved exception message when calling :meth:`DataFrame.iloc` with a list of non-numeric objects (:issue:`25753`).
 - Bug in :meth:`DataFrame.loc` and :meth:`Series.loc` where ``KeyError`` was not raised for a ``MultiIndex`` when the key was less than or equal to the number of levels in the :class:`MultiIndex` (:issue:`14885`).
 - Bug in which :meth:`DataFrame.append` produced an erroneous warning indicating that a ``KeyError`` will be thrown in the future when the data to be appended contains new columns (:issue:`22252`).
--
+- Bug in which :meth:`DataFrame.to_csv` caused a segfault for a reindexed data frame, when the indices were single-level :class:`MultiIndex` (:issue:`26303`).
 
 
 Missing

mypy.ini

@@ -23,15 +23,9 @@ ignore_errors=True
 [mypy-pandas.core.internals.blocks]
 ignore_errors=True
 
-[mypy-pandas.core.ops]
-ignore_errors=True
-
 [mypy-pandas.core.panel]
 ignore_errors=True
 
-[mypy-pandas.core.resample]
-ignore_errors=True
-
 [mypy-pandas.core.reshape.merge]
 ignore_errors=True
 

pandas/_libs/tslibs/period.pyx

@@ -2208,10 +2208,6 @@ cdef class _Period:
     def now(cls, freq=None):
         return Period(datetime.now(), freq=freq)
 
-    # HACK IT UP AND YOU BETTER FIX IT SOON
-    def __str__(self):
-        return self.__unicode__()
-
     @property
     def freqstr(self):
         return self.freq.freqstr
@@ -2221,9 +2217,9 @@ cdef class _Period:
         formatted = period_format(self.ordinal, base)
         return "Period('%s', '%s')" % (formatted, self.freqstr)
 
-    def __unicode__(self):
+    def __str__(self):
         """
-        Return a unicode string representation for a particular DataFrame
+        Return a string representation for a particular DataFrame
         """
         base, mult = get_freq_code(self.freq)
         formatted = period_format(self.ordinal, base)

pandas/core/dtypes/dtypes.py

@@ -126,14 +126,11 @@ class PandasExtensionDtype(ExtensionDtype):
     isnative = 0
     _cache = {}  # type: Dict[str_type, 'PandasExtensionDtype']
 
-    def __unicode__(self):
-        return self.name
-
     def __str__(self):
         """
         Return a string representation for a particular Object
         """
-        return self.__unicode__()
+        return self.name
 
     def __bytes__(self):
         """
@@ -142,7 +139,7 @@ def __bytes__(self):
         from pandas._config import get_option
 
         encoding = get_option("display.encoding")
-        return self.__unicode__().encode(encoding, 'replace')
+        return str(self).encode(encoding, 'replace')
 
     def __repr__(self):
         """
@@ -707,7 +704,7 @@ def construct_from_string(cls, string):
 
         raise TypeError("Could not construct DatetimeTZDtype")
 
-    def __unicode__(self):
+    def __str__(self):
         return "datetime64[{unit}, {tz}]".format(unit=self.unit, tz=self.tz)
 
     @property
@@ -837,12 +834,12 @@ def construct_from_string(cls, string):
                 pass
         raise TypeError("could not construct PeriodDtype")
 
-    def __unicode__(self):
-        return str(self.name)
+    def __str__(self):
+        return self.name
 
     @property
     def name(self):
-        return str("period[{freq}]".format(freq=self.freq.freqstr))
+        return "period[{freq}]".format(freq=self.freq.freqstr)
 
     @property
     def na_value(self):
@@ -1007,7 +1004,7 @@ def construct_from_string(cls, string):
     def type(self):
         return Interval
 
-    def __unicode__(self):
+    def __str__(self):
         if self.subtype is None:
             return "interval"
         return "interval[{subtype}]".format(subtype=self.subtype)

pandas/core/generic.py

@@ -10177,11 +10177,14 @@ def mad(self, axis=None, skipna=None, level=None):
             nanops.nanstd)
 
         @Substitution(desc="Return the compound percentage of the values for "
-                      "the requested axis.", name1=name, name2=name2,
-                      axis_descr=axis_descr,
+                      "the requested axis.\n\n.. deprecated:: 0.25.0",
+                      name1=name, name2=name2, axis_descr=axis_descr,
                       min_count='', see_also='', examples='')
         @Appender(_num_doc)
         def compound(self, axis=None, skipna=None, level=None):
+            msg = ("The 'compound' method is deprecated and will be"
+                   "removed in a future version.")
+            warnings.warn(msg, FutureWarning, stacklevel=2)
             if skipna is None:
                 skipna = True
             return (1 + self).prod(axis=axis, skipna=skipna, level=level) - 1

pandas/core/groupby/grouper.py

@@ -3,6 +3,7 @@
 split-apply-combine paradigm.
 """
 
+from typing import Tuple
 import warnings
 
 import numpy as np
@@ -84,7 +85,8 @@ class Grouper:
 
     >>> df.groupby(Grouper(level='date', freq='60s', axis=1))
     """
-    _attributes = ('key', 'level', 'freq', 'axis', 'sort')
+    _attributes = ('key', 'level', 'freq', 'axis',
+                   'sort')  # type: Tuple[str, ...]
 
     def __new__(cls, *args, **kwargs):
         if kwargs.get('freq') is not None:

pandas/core/indexes/multi.py

@@ -946,7 +946,9 @@ def _format_native_types(self, na_rep='nan', **kwargs):
             new_codes.append(level_codes)
 
         if len(new_levels) == 1:
-            return Index(new_levels[0])._format_native_types()
+            # a single-level multi-index
+            return Index(new_levels[0].take(
+                new_codes[0]))._format_native_types()
         else:
             # reconstruct the multi-index
             mi = MultiIndex(levels=new_levels, codes=new_codes,

pandas/core/ops.py

@@ -6,6 +6,7 @@
 import datetime
 import operator
 import textwrap
+from typing import Dict, Optional
 import warnings
 
 import numpy as np
@@ -625,15 +626,13 @@ def _get_op_name(op, special):
            'desc': 'Greater than or equal to',
            'reverse': None,
            'series_examples': None}
-}
+}  # type: Dict[str, Dict[str, Optional[str]]]
 
 _op_names = list(_op_descriptions.keys())
 for key in _op_names:
-    _op_descriptions[key]['reversed'] = False
     reverse_op = _op_descriptions[key]['reverse']
     if reverse_op is not None:
         _op_descriptions[reverse_op] = _op_descriptions[key].copy()
-        _op_descriptions[reverse_op]['reversed'] = True
         _op_descriptions[reverse_op]['reverse'] = key
 
 _flex_doc_SERIES = """
@@ -1010,7 +1009,7 @@ def _make_flex_doc(op_name, typ):
     op_name = op_name.replace('__', '')
     op_desc = _op_descriptions[op_name]
 
-    if op_desc['reversed']:
+    if op_name.startswith('r'):
         equiv = 'other ' + op_desc['op'] + ' ' + typ
     else:
         equiv = typ + ' ' + op_desc['op'] + ' other'

pandas/core/resample.py

@@ -1,6 +1,7 @@
 import copy
 from datetime import timedelta
 from textwrap import dedent
+from typing import Dict, no_type_check
 import warnings
 
 import numpy as np
@@ -31,7 +32,7 @@
 from pandas.tseries.frequencies import to_offset
 from pandas.tseries.offsets import DateOffset, Day, Nano, Tick
 
-_shared_docs_kwargs = dict()
+_shared_docs_kwargs = dict()  # type: Dict[str, str]
 
 
 class Resampler(_GroupBy):
@@ -873,25 +874,25 @@ def f(self, _method=method, min_count=0, *args, **kwargs):
 for method in ['min', 'max', 'first', 'last', 'mean', 'sem',
                'median', 'ohlc']:
 
-    def f(self, _method=method, *args, **kwargs):
+    def g(self, _method=method, *args, **kwargs):
         nv.validate_resampler_func(_method, args, kwargs)
         return self._downsample(_method)
-    f.__doc__ = getattr(GroupBy, method).__doc__
-    setattr(Resampler, method, f)
+    g.__doc__ = getattr(GroupBy, method).__doc__
+    setattr(Resampler, method, g)
 
 # groupby & aggregate methods
 for method in ['count']:
-    def f(self, _method=method):
+    def h(self, _method=method):
         return self._downsample(_method)
-    f.__doc__ = getattr(GroupBy, method).__doc__
-    setattr(Resampler, method, f)
+    h.__doc__ = getattr(GroupBy, method).__doc__
+    setattr(Resampler, method, h)
 
 # series only methods
 for method in ['nunique']:
-    def f(self, _method=method):
+    def h(self, _method=method):
         return self._downsample(_method)
-    f.__doc__ = getattr(SeriesGroupBy, method).__doc__
-    setattr(Resampler, method, f)
+    h.__doc__ = getattr(SeriesGroupBy, method).__doc__
+    setattr(Resampler, method, h)
 
 
 def _maybe_process_deprecations(r, how=None, fill_method=None, limit=None):
@@ -964,6 +965,7 @@ def __init__(self, obj, *args, **kwargs):
         self._groupby.grouper.mutated = True
         self.groupby = copy.copy(parent.groupby)
 
+    @no_type_check
     def _apply(self, f, grouper=None, *args, **kwargs):
         """
         Dispatch to _upsample; we are stripping all of the _upsample kwargs and

pandas/tests/frame/test_to_csv.py

@@ -1230,6 +1230,15 @@ def test_multi_index_header(self):
         expected = tm.convert_rows_list_to_csv_str(expected_rows)
         assert result == expected
 
+    def test_to_csv_single_level_multi_index(self):
+        # see gh-26303
+        index = pd.Index([(1,), (2,), (3,)])
+        df = pd.DataFrame([[1, 2, 3]], columns=index)
+        df = df.reindex(columns=[(1,), (3,)])
+        expected = ",1,3\n0,1,3\n"
+        result = df.to_csv(line_terminator='\n')
+        assert_almost_equal(result, expected)
+
     def test_gz_lineend(self):
         # GH 25311
         df = pd.DataFrame({'a': [1, 2]})

pandas/tests/series/test_analytics.py

@@ -1145,6 +1145,15 @@ def test_validate_stat_keepdims(self):
         with pytest.raises(ValueError, match=msg):
             np.sum(s, keepdims=True)
 
+    def test_compound_deprecated(self):
+        s = Series([.1, .2, .3, .4])
+        with tm.assert_produces_warning(FutureWarning):
+            s.compound()
+
+        df = pd.DataFrame({'s': s})
+        with tm.assert_produces_warning(FutureWarning):
+            df.compound()
+
 
 main_dtypes = [
     'datetime',