Support Pandas 0.19.0

Author: mrocklin

.travis.yml

@@ -3,11 +3,11 @@ sudo: false
 
 env:
   matrix:
-    - PYTHON=2.7 NUMPY=1.10.4 PANDAS=0.18.0 COVERAGE='true' XTRATESTARGS=
+    - PYTHON=2.7 NUMPY=1.10.4 PANDAS=0.19.0 COVERAGE='true' XTRATESTARGS=
     - PYTHON=2.7 NUMPY=1.11.0 PANDAS=0.18.1 COVERAGE='false' PYTHONOPTIMIZE=2 XTRATESTARGS=--ignore=dask/diagnostics
     - PYTHON=3.3 NUMPY=1.9.2 PANDAS=0.18.1 COVERAGE='false' XTRATESTARGS=
     - PYTHON=3.4 NUMPY=1.10.4 PANDAS=0.18.0 COVERAGE='false' PYTHONOPTIMIZE=2 XTRATESTARGS=--ignore=dask/diagnostics
-    - PYTHON=3.5 NUMPY=1.11.0 PANDAS=0.18.1 COVERAGE='false' XTRATESTARGS=
+    - PYTHON=3.5 NUMPY=1.11.0 PANDAS=0.19.0 COVERAGE='false' XTRATESTARGS=
 
 addons:
     apt:

dask/dataframe/categorical.py

@@ -114,6 +114,10 @@ def strip_categories(df):
                               else df.index)
 
 
+def iscategorical(dt):
+    return isinstance(dt, pd.types.dtypes.CategoricalDtype)
+
+
 def get_categories(df):
     """
     Get Categories of dataframe

dask/dataframe/core.py

@@ -2,6 +2,7 @@
 
 from collections import Iterator
 from copy import copy
+from distutils.version import LooseVersion
 import operator
 from operator import getitem, setitem
 from pprint import pformat
@@ -1219,15 +1220,19 @@ def quantile(self, q=0.5, axis=0):
             0 or 'index' for row-wise, 1 or 'columns' for column-wise
         """
         axis = self._validate_axis(axis)
-        name = 'quantiles-concat--' + tokenize(self, q, axis)
+        keyname = 'quantiles-concat--' + tokenize(self, q, axis)
 
         if axis == 1:
             if isinstance(q, list):
                 # Not supported, the result will have current index as columns
                 raise ValueError("'q' must be scalar when axis=1 is specified")
-            meta = pd.Series([], dtype='f8')
+            if LooseVersion(pd.__version__) >= '0.19':
+                name = q
+            else:
+                name = None
+            meta = pd.Series([], dtype='f8', name=name)
             return map_partitions(M.quantile, self, q, axis,
-                                  token=name, meta=meta)
+                                  token=keyname, meta=meta)
         else:
             meta = self._meta.quantile(q, axis=axis)
             num = self._get_numeric_data()
@@ -1238,13 +1243,13 @@ def quantile(self, q=0.5, axis=0):
             qnames = [(_q._name, 0) for _q in quantiles]
 
             if isinstance(quantiles[0], Scalar):
-                dask[(name, 0)] = (pd.Series, (list, qnames), num.columns)
+                dask[(keyname, 0)] = (pd.Series, (list, qnames), num.columns)
                 divisions = (min(num.columns), max(num.columns))
-                return Series(dask, name, meta, divisions)
+                return Series(dask, keyname, meta, divisions)
             else:
                 from .multi import _pdconcat
-                dask[(name, 0)] = (_pdconcat, (list, qnames), 1)
-                return DataFrame(dask, name, meta, quantiles[0].divisions)
+                dask[(keyname, 0)] = (_pdconcat, (list, qnames), 1)
+                return DataFrame(dask, keyname, meta, quantiles[0].divisions)
 
     @derived_from(pd.DataFrame)
     def describe(self):

dask/dataframe/indexing.py

@@ -80,7 +80,6 @@ def _maybe_partial_time_string(index, indexer, kind):
     Convert indexer for partial string selection
     if data has DatetimeIndex/PeriodIndex
     """
-
     # do not pass dd.Index
     assert isinstance(index, pd.Index)
 

dask/dataframe/io.py

@@ -139,16 +139,16 @@ def from_pandas(data, npartitions=None, chunksize=None, sort=True, name=None):
     ...                   index=pd.date_range(start='20100101', periods=6))
     >>> ddf = from_pandas(df, npartitions=3)
     >>> ddf.divisions  # doctest: +NORMALIZE_WHITESPACE
-    (Timestamp('2010-01-01 00:00:00', offset='D'),
-     Timestamp('2010-01-03 00:00:00', offset='D'),
-     Timestamp('2010-01-05 00:00:00', offset='D'),
-     Timestamp('2010-01-06 00:00:00', offset='D'))
+    (Timestamp('2010-01-01 00:00:00', freq='D'),
+     Timestamp('2010-01-03 00:00:00', freq='D'),
+     Timestamp('2010-01-05 00:00:00', freq='D'),
+     Timestamp('2010-01-06 00:00:00', freq='D'))
     >>> ddf = from_pandas(df.a, npartitions=3)  # Works with Series too!
     >>> ddf.divisions  # doctest: +NORMALIZE_WHITESPACE
-    (Timestamp('2010-01-01 00:00:00', offset='D'),
-     Timestamp('2010-01-03 00:00:00', offset='D'),
-     Timestamp('2010-01-05 00:00:00', offset='D'),
-     Timestamp('2010-01-06 00:00:00', offset='D'))
+    (Timestamp('2010-01-01 00:00:00', freq='D'),
+     Timestamp('2010-01-03 00:00:00', freq='D'),
+     Timestamp('2010-01-05 00:00:00', freq='D'),
+     Timestamp('2010-01-06 00:00:00', freq='D'))
 
     Raises
     ------
@@ -554,7 +554,8 @@ def to_hdf(df, path_or_buf, key, mode='a', append=False, get=None,
 
 
 def _read_single_hdf(path, key, start=0, stop=None, columns=None,
-                     chunksize=int(1e6), sorted_index=False, lock=None, mode=None):
+                     chunksize=int(1e6), sorted_index=False, lock=None,
+                     mode='a'):
     """
     Read a single hdf file into a dask.dataframe. Used for each file in
     read_hdf.
@@ -652,7 +653,7 @@ def _pd_read_hdf(path, key, lock, kwargs):
 
 @wraps(pd.read_hdf)
 def read_hdf(pattern, key, start=0, stop=None, columns=None,
-             chunksize=1000000, sorted_index=False, lock=True, mode=None):
+             chunksize=1000000, sorted_index=False, lock=True, mode='a'):
     """
     Read hdf files into a dask dataframe. Like pandas.read_hdf, except it we
     can read multiple files, and read multiple keys from the same file by using

dask/dataframe/tests/test_io.py

@@ -1264,7 +1264,7 @@ def test_to_csv_simple():
         df.to_csv(dir)
         assert os.listdir(dir)
         result = dd.read_csv(os.path.join(dir, '*')).compute()
-    assert (result.x == df0.x).all()
+    assert (result.x.values == df0.x.values).all()
 
 
 def test_to_csv_series():