Merge branch 'master' into gh15077

Author: mralgos

ci/requirements-3.4_SLOW.build

@@ -1,4 +1,4 @@
 python-dateutil
 pytz
-numpy=1.9.3
+numpy=1.10*
 cython

ci/requirements-3.4_SLOW.run

@@ -1,6 +1,6 @@
 python-dateutil
 pytz
-numpy=1.9.3
+numpy=1.10*
 openpyxl
 xlsxwriter
 xlrd

ci/requirements-3.4_SLOW.sh

@@ -4,4 +4,4 @@ source activate pandas
 
 echo "install 34_slow"
 
-conda install -n pandas -c conda-forge/label/rc -c conda-forge matplotlib
+conda install -n pandas -c conda-forge matplotlib

doc/source/groupby.rst

@@ -810,7 +810,7 @@ next). This enables some operations to be carried out rather succinctly:
    tsdf = pd.DataFrame(np.random.randn(1000, 3),
                        index=pd.date_range('1/1/2000', periods=1000),
                        columns=['A', 'B', 'C'])
-   tsdf.ix[::2] = np.nan
+   tsdf.iloc[::2] = np.nan
    grouped = tsdf.groupby(lambda x: x.year)
    grouped.fillna(method='pad')
 

doc/source/indexing.rst

@@ -557,13 +557,18 @@ IX Indexer is Deprecated
 
 .. warning::
 
-  Starting in 0.20.0, the ``.ix`` indexer is deprecated, in favor of the more strict ``.iloc``
-and ``.loc`` indexers. ``.ix`` offers a lot of magic on the inference of what the user wants to
-do. To wit, ``.ix`` can decide to index *positionally* OR via *labels*. This has caused
-quite a bit of user confusion over the years.
+   Starting in 0.20.0, the ``.ix`` indexer is deprecated, in favor of the more strict ``.iloc``
+   and ``.loc`` indexers.
+
+``.ix`` offers a lot of magic on the inference of what the user wants to do. To wit, ``.ix`` can decide
+to index *positionally* OR via *labels* depending on the data type of the index. This has caused quite a
+bit of user confusion over the years.
 
 The recommended methods of indexing are:
 
+- ``.loc`` if you want to *label* index
+- ``.iloc`` if you want to *positionally* index.
+
 .. ipython:: python
 
   dfd = pd.DataFrame({'A': [1, 2, 3],

doc/source/io.rst

@@ -357,94 +357,6 @@ warn_bad_lines : boolean, default ``True``
   If error_bad_lines is ``False``, and warn_bad_lines is ``True``, a warning for
   each "bad line" will be output (only valid with C parser).
 
-.. ipython:: python
-   :suppress:
-
-   f = open('foo.csv','w')
-   f.write('date,A,B,C\n20090101,a,1,2\n20090102,b,3,4\n20090103,c,4,5')
-   f.close()
-
-Consider a typical CSV file containing, in this case, some time series data:
-
-.. ipython:: python
-
-   print(open('foo.csv').read())
-
-The default for `read_csv` is to create a DataFrame with simple numbered rows:
-
-.. ipython:: python
-
-   pd.read_csv('foo.csv')
-
-In the case of indexed data, you can pass the column number or column name you
-wish to use as the index:
-
-.. ipython:: python
-
-   pd.read_csv('foo.csv', index_col=0)
-
-.. ipython:: python
-
-   pd.read_csv('foo.csv', index_col='date')
-
-You can also use a list of columns to create a hierarchical index:
-
-.. ipython:: python
-
-   pd.read_csv('foo.csv', index_col=[0, 'A'])
-
-.. _io.dialect:
-
-The ``dialect`` keyword gives greater flexibility in specifying the file format.
-By default it uses the Excel dialect but you can specify either the dialect name
-or a :class:`python:csv.Dialect` instance.
-
-.. ipython:: python
-   :suppress:
-
-   data = ('label1,label2,label3\n'
-           'index1,"a,c,e\n'
-           'index2,b,d,f')
-
-Suppose you had data with unenclosed quotes:
-
-.. ipython:: python
-
-   print(data)
-
-By default, ``read_csv`` uses the Excel dialect and treats the double quote as
-the quote character, which causes it to fail when it finds a newline before it
-finds the closing double quote.
-
-We can get around this using ``dialect``
-
-.. ipython:: python
-
-   dia = csv.excel()
-   dia.quoting = csv.QUOTE_NONE
-   pd.read_csv(StringIO(data), dialect=dia)
-
-All of the dialect options can be specified separately by keyword arguments:
-
-.. ipython:: python
-
-    data = 'a,b,c~1,2,3~4,5,6'
-    pd.read_csv(StringIO(data), lineterminator='~')
-
-Another common dialect option is ``skipinitialspace``, to skip any whitespace
-after a delimiter:
-
-.. ipython:: python
-
-   data = 'a, b, c\n1, 2, 3\n4, 5, 6'
-   print(data)
-   pd.read_csv(StringIO(data), skipinitialspace=True)
-
-The parsers make every attempt to "do the right thing" and not be very
-fragile. Type inference is a pretty big deal. So if a column can be coerced to
-integer dtype without altering the contents, it will do so. Any non-numeric
-columns will come through as object dtype as with the rest of pandas objects.
-
 .. _io.dtypes:
 
 Specifying column data types
@@ -1238,6 +1150,62 @@ data that appear in some lines but not others:
     1  4  5   6
     2  8  9  10
 
+.. _io.dialect:
+
+Dialect
+'''''''
+
+The ``dialect`` keyword gives greater flexibility in specifying the file format.
+By default it uses the Excel dialect but you can specify either the dialect name
+or a :class:`python:csv.Dialect` instance.
+
+.. ipython:: python
+   :suppress:
+
+   data = ('label1,label2,label3\n'
+           'index1,"a,c,e\n'
+           'index2,b,d,f')
+
+Suppose you had data with unenclosed quotes:
+
+.. ipython:: python
+
+   print(data)
+
+By default, ``read_csv`` uses the Excel dialect and treats the double quote as
+the quote character, which causes it to fail when it finds a newline before it
+finds the closing double quote.
+
+We can get around this using ``dialect``
+
+.. ipython:: python
+   :okwarning:
+
+   dia = csv.excel()
+   dia.quoting = csv.QUOTE_NONE
+   pd.read_csv(StringIO(data), dialect=dia)
+
+All of the dialect options can be specified separately by keyword arguments:
+
+.. ipython:: python
+
+    data = 'a,b,c~1,2,3~4,5,6'
+    pd.read_csv(StringIO(data), lineterminator='~')
+
+Another common dialect option is ``skipinitialspace``, to skip any whitespace
+after a delimiter:
+
+.. ipython:: python
+
+   data = 'a, b, c\n1, 2, 3\n4, 5, 6'
+   print(data)
+   pd.read_csv(StringIO(data), skipinitialspace=True)
+
+The parsers make every attempt to "do the right thing" and not be very
+fragile. Type inference is a pretty big deal. So if a column can be coerced to
+integer dtype without altering the contents, it will do so. Any non-numeric
+columns will come through as object dtype as with the rest of pandas objects.
+
 .. _io.quoting:
 
 Quoting and Escape Characters
@@ -1400,7 +1368,7 @@ returned object:
 
    df = pd.read_csv("data/mindex_ex.csv", index_col=[0,1])
    df
-   df.iloc[1978]
+   df.loc[1978]
 
 .. _io.multi_index_columns:
 

doc/source/timedeltas.rst

@@ -310,6 +310,20 @@ similarly to the ``Series``. These are the *displayed* values of the ``Timedelta
    td.dt.components
    td.dt.components.seconds
 
+.. _timedeltas.isoformat:
+
+You can convert a ``Timedelta`` to an ISO 8601 Duration string with the
+``.isoformat`` method
+
+.. versionadded:: 0.20.0
+
+.. ipython:: python
+    pd.Timedelta(days=6, minutes=50, seconds=3,
+                 milliseconds=10, microseconds=10,
+                 nanoseconds=12).isoformat()
+
+.. _ISO 8601 duration: https://en.wikipedia.org/wiki/ISO_8601#Durations
+
 .. _timedeltas.index:
 
 TimedeltaIndex

doc/source/whatsnew/v0.20.0.txt

@@ -133,11 +133,14 @@ Other enhancements
 - The ``skiprows`` argument in ``pd.read_csv`` now accepts a callable function as a value  (:issue:`10882`)
 - ``pd.DataFrame.plot`` now prints a title above each subplot if ``suplots=True`` and ``title`` is a list of strings (:issue:`14753`)
 - ``pd.Series.interpolate`` now supports timedelta as an index type with ``method='time'`` (:issue:`6424`)
+- ``Timedelta.isoformat`` method added for formatting Timedeltas as an `ISO 8601 duration`_. See the :ref:`Timedelta docs <timedeltas.isoformat>` (:issue:`15136`)
 - ``pandas.io.json.json_normalize()`` gained the option ``errors='ignore'|'raise'``; the default is ``errors='raise'`` which is backward compatible. (:issue:`14583`)
 
 - ``.select_dtypes()`` now allows the string 'datetimetz' to generically select datetimes with tz (:issue:`14910`)
 - ``pd.merge_asof()`` gained the option ``direction='backward'|'forward'|'nearest'`` (:issue:`14887`)
 
+.. _ISO 8601 duration: https://en.wikipedia.org/wiki/ISO_8601#Durations
+
 
 .. _whatsnew_0200.api_breaking:
 
@@ -149,7 +152,7 @@ Backwards incompatible API changes
 Deprecate .ix
 ^^^^^^^^^^^^^
 
-The ``.ix`` indexer is deprecated, in favor of the more strict ``.iloc`` and ``.loc`` indexers. ``.ix`` offers a lot of magic on the inference of what the user wants to do. To wit, ``.ix`` can decide to index *positionally* OR via *labels*. This has caused quite a bit of user confusion over the years. The full indexing documentation are :ref:`here <indexing>`. (:issue:`14218`)
+The ``.ix`` indexer is deprecated, in favor of the more strict ``.iloc`` and ``.loc`` indexers. ``.ix`` offers a lot of magic on the inference of what the user wants to do. To wit, ``.ix`` can decide to index *positionally* OR via *labels*, depending on the data type of the index. This has caused quite a bit of user confusion over the years. The full indexing documentation are :ref:`here <indexing>`. (:issue:`14218`)
 
 
 The recommended methods of indexing are:
@@ -388,10 +391,11 @@ Bug Fixes
 
 - Bug in compat for passing long integers to ``Timestamp.replace`` (:issue:`15030`)
 - Bug in ``.loc`` that would not return the correct dtype for scalar access for a DataFrame (:issue:`11617`)
+- Bug in ``GroupBy.get_group()`` failing with a categorical grouper (:issue:`15155`)
 
 
 
-
+- Bug in ``.groupby(...).rolling(...)`` when ``on`` is specified and using a ``DatetimeIndex`` (:issue:`15130`)
 
 
 
@@ -435,4 +439,10 @@ Bug Fixes
 - Bug in ``pd.read_csv()`` for the C engine where ``usecols`` were being indexed incorrectly with ``parse_dates`` (:issue:`14792`)
 - Incorrect dtyped ``Series`` was returned by comparison methods (e.g., ``lt``, ``gt``, ...) against a constant for an empty ``DataFrame`` (:issue:`15077`)
 - Bug in ``Series.dt.round`` inconsistent behaviour on NAT's with different arguments (:issue:`14940`)
-- Bug in ``.read_json()`` for Python 2 where ``lines=True`` and contents contain non-ascii unicode characters (:issue:`15132`)
+
+
+- Bug in ``.read_json()`` for Python 2 where ``lines=True`` and contents contain non-ascii unicode characters (:issue:`15132`)
+
+- Bug in ``pd.read_csv()`` with ``float_precision='round_trip'`` which caused a segfault when a text entry is parsed (:issue:`15140`)
+
+- Bug in ``DataFrame.boxplot`` where ``fontsize`` was not applied to the tick labels on both axes (:issue:`15108`)

pandas/core/window.py

@@ -1025,19 +1025,8 @@ def validate(self):
         if (self.is_datetimelike and
                 isinstance(self.window, (compat.string_types, DateOffset))):
 
-            # must be monotonic for on
-            if not self._on.is_monotonic:
-                formatted = self.on or 'index'
-                raise ValueError("{0} must be "
-                                 "monotonic".format(formatted))
-
-            from pandas.tseries.frequencies import to_offset
-            try:
-                freq = to_offset(self.window)
-            except (TypeError, ValueError):
-                raise ValueError("passed window {0} in not "
-                                 "compat with a datetimelike "
-                                 "index".format(self.window))
+            self._validate_monotonic()
+            freq = self._validate_freq()
 
             # we don't allow center
             if self.center:
@@ -1058,6 +1047,23 @@ def validate(self):
         elif self.window < 0:
             raise ValueError("window must be non-negative")
 
+    def _validate_monotonic(self):
+        """ validate on is monotonic """
+        if not self._on.is_monotonic:
+            formatted = self.on or 'index'
+            raise ValueError("{0} must be "
+                             "monotonic".format(formatted))
+
+    def _validate_freq(self):
+        """ validate & return our freq """
+        from pandas.tseries.frequencies import to_offset
+        try:
+            return to_offset(self.window)
+        except (TypeError, ValueError):
+            raise ValueError("passed window {0} in not "
+                             "compat with a datetimelike "
+                             "index".format(self.window))
+
     @Substitution(name='rolling')
     @Appender(SelectionMixin._see_also_template)
     @Appender(SelectionMixin._agg_doc)
@@ -1175,6 +1181,25 @@ class RollingGroupby(_GroupByMixin, Rolling):
     def _constructor(self):
         return Rolling
 
+    def _gotitem(self, key, ndim, subset=None):
+
+        # we are setting the index on the actual object
+        # here so our index is carried thru to the selected obj
+        # when we do the splitting for the groupby
+        if self.on is not None:
+            self._groupby.obj = self._groupby.obj.set_index(self._on)
+            self.on = None
+        return super(RollingGroupby, self)._gotitem(key, ndim, subset=subset)
+
+    def _validate_monotonic(self):
+        """
+        validate that on is monotonic;
+        we don't care for groupby.rolling
+        because we have already validated at a higher
+        level
+        """
+        pass
+
 
 class Expanding(_Rolling_and_Expanding):
     """

pandas/indexes/category.py

@@ -255,6 +255,9 @@ def categories(self):
     def ordered(self):
         return self._data.ordered
 
+    def _reverse_indexer(self):
+        return self._data._reverse_indexer()
+
     def __contains__(self, key):
         hash(key)
         return key in self.values

pandas/io/tests/parser/c_parser_only.py

@@ -388,3 +388,10 @@ def test_read_nrows_large(self):
         df = self.read_csv(StringIO(test_input), sep='\t', nrows=1010)
 
         self.assertTrue(df.size == 1010 * 10)
+
+    def test_float_precision_round_trip_with_text(self):
+        # gh-15140 - This should not segfault on Python 2.7+
+        df = self.read_csv(StringIO('a'),
+                           float_precision='round_trip',
+                           header=None)
+        tm.assert_frame_equal(df, DataFrame({0: ['a']}))