TST/CLN: Catch more warnings

Catches a handful of warnings from the dask.dataframe test suite. Remaining
warnings fall into a few classes:

1. ResoucreWarnings from cloudpickls (e.g.
  dask/dataframe/tests/test_arithmetics_reduction.py::test_frame_series_arithmetic_methods)
  Not sure what to do here. Probably a legitimate concern of dasks?
2. RuntimeWarnings from numpy (e.g.
  dask/dataframe/tests/test_arithmetics_reduction.py::test_reductions[False]
  I think dask should catch these, (followup PR)
3. ImportWarning from dependencies importing C code? e.g.
  dask/dataframe/tests/test_categorical.py::test_categorical_set_index[disk]
  I have a fix for partd (msgpack). Pandas has some too. Still investigating,
  may be a Cython issue
4. DeprecationWarning on regexes. e.g.
  dask/dataframe/tests/test_groupby.py::test_full_groupby_multilevel[grouper4]
  I think these are all pandas. xref
  pandas-dev/pandas#16481 (comment)
5. RuntimeWarning from pandas merge, e.g.
  dask/dataframe/tests/test_multi.py::test_merge_by_multiple_columns[disk-inner]
  Filing an issue on pandas

Author: TomAugspurger

dask/dataframe/groupby.py

@@ -911,7 +911,7 @@ def apply(self, func, meta=no_default):
                    "  Before: .apply(func)\n"
                    "  After:  .apply(func, meta={'x': 'f8', 'y': 'f8'}) for dataframe result\n"
                    "  or:     .apply(func, meta=('x', 'f8'))            for series result")
-            warnings.warn(msg)
+            warnings.warn(msg, stacklevel=2)
 
             with raise_on_meta_error("groupby.apply({0})".format(funcname(func))):
                 meta = self._meta_nonempty.apply(func)

dask/dataframe/io/tests/test_csv.py

@@ -436,11 +436,12 @@ def test_warn_non_seekable_files():
         assert 'blocksize=None' in msg
 
         with pytest.warns(None) as w:
-            df = dd.read_csv('2014-01-*.csv', compression='gzip', blocksize=None)
+            df = dd.read_csv('2014-01-*.csv', compression='gzip',
+                             blocksize=None)
         assert len(w) == 0
 
         with pytest.raises(NotImplementedError):
-            with pytest.warns(None):
+            with pytest.warns(UserWarning):  # needed for pytest
                 df = dd.read_csv('2014-01-*.csv', compression='foo')
 
 
@@ -730,8 +731,8 @@ def test_read_csv_sep():
     charlie###300""")
 
     with filetext(sep_text) as fn:
-        ddf = dd.read_csv(fn, sep="###")
-        df = pd.read_csv(fn, sep="###")
+        ddf = dd.read_csv(fn, sep="###", engine="python")
+        df = pd.read_csv(fn, sep="###", engine="python")
 
         assert (df.columns == ddf.columns).all()
         assert len(df) == len(ddf)

dask/dataframe/io/tests/test_io.py

@@ -310,12 +310,14 @@ def test_from_pandas_single_row():
     assert_eq(ddf, df)
 
 
+@pytest.mark.skipif(np.__version__ < '1.11',
+                    reason='datetime unit unsupported in NumPy < 1.11')
 def test_from_pandas_with_datetime_index():
     df = pd.DataFrame({"Date": ["2015-08-28", "2015-08-27", "2015-08-26",
                                 "2015-08-25", "2015-08-24", "2015-08-21",
                                 "2015-08-20", "2015-08-19", "2015-08-18"],
                        "Val": list(range(9))})
-    df.Date = df.Date.astype('datetime64')
+    df.Date = df.Date.astype('datetime64[ns]')
     ddf = dd.from_pandas(df, 2)
     assert_eq(df, ddf)
     ddf = dd.from_pandas(df, chunksize=2)

dask/dataframe/methods.py

@@ -1,5 +1,7 @@
 from __future__ import print_function, absolute_import, division
 
+import warnings
+
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_categorical_dtype
@@ -253,8 +255,11 @@ def concat(dfs, axis=0, join='outer', uniform=False):
             # concatenates.
             dfs3 = [df if isinstance(df, pd.DataFrame) else
                     df.to_frame().rename(columns={df.name: 0}) for df in dfs2]
-            cat_mask = pd.concat([(df.dtypes == 'category').to_frame().T
-                                  for df in dfs3], join=join).any()
+            # pandas may raise a RuntimeWarning for comparing ints and strs
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", RuntimeWarning)
+                cat_mask = pd.concat([(df.dtypes == 'category').to_frame().T
+                                      for df in dfs3], join=join).any()
 
         if cat_mask.any():
             not_cat = cat_mask[~cat_mask].index
@@ -280,7 +285,10 @@ def concat(dfs, axis=0, join='outer', uniform=False):
                 out[col] = union_categoricals(parts)
             out = out.reindex_axis(cat_mask.index, axis=1)
         else:
-            out = pd.concat(dfs3, join=join)
+            # pandas may raise a RuntimeWarning for comparing ints and strs
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", RuntimeWarning)
+                out = pd.concat(dfs3, join=join)
     else:
         if is_categorical_dtype(dfs2[0].dtype):
             if ind is None:

dask/dataframe/tests/test_arithmetics_reduction.py

@@ -635,9 +635,15 @@ def test_reductions(split_every):
         assert_eq(dds.min(split_every=split_every), pds.min())
         assert_eq(dds.max(split_every=split_every), pds.max())
         assert_eq(dds.count(split_every=split_every), pds.count())
-        assert_eq(dds.std(split_every=split_every), pds.std())
-        assert_eq(dds.var(split_every=split_every), pds.var())
-        assert_eq(dds.sem(split_every=split_every), pds.sem())
+        with pytest.warns(None):
+            # runtime warnings; https://github.com/dask/dask/issues/2381
+            assert_eq(dds.std(split_every=split_every), pds.std())
+        with pytest.warns(None):
+            # runtime warnings; https://github.com/dask/dask/issues/2381
+            assert_eq(dds.var(split_every=split_every), pds.var())
+        with pytest.warns(None):
+            # runtime warnings; https://github.com/dask/dask/issues/2381
+            assert_eq(dds.sem(split_every=split_every), pds.sem())
         assert_eq(dds.std(ddof=0, split_every=split_every), pds.std(ddof=0))
         assert_eq(dds.var(ddof=0, split_every=split_every), pds.var(ddof=0))
         assert_eq(dds.sem(ddof=0, split_every=split_every), pds.sem(ddof=0))

dask/dataframe/tests/test_dataframe.py

@@ -1381,9 +1381,10 @@ def test_eval():
         with pytest.raises(NotImplementedError):
             d.eval('z = x + y', inplace=True)
 
-        if p.eval('z = x + y', inplace=None) is None:
-            with pytest.raises(NotImplementedError):
-                d.eval('z = x + y', inplace=None)
+        with pytest.warns(None):
+            if p.eval('z = x + y', inplace=None) is None:
+                with pytest.raises(NotImplementedError):
+                    d.eval('z = x + y', inplace=None)
 
 
 @pytest.mark.parametrize('include, exclude', [
@@ -1702,7 +1703,7 @@ def test_apply():
     ddf = dd.from_pandas(df, npartitions=2)
 
     func = lambda row: row['x'] + row['y']
-    assert_eq(ddf.x.apply(lambda x: x + 1),
+    assert_eq(ddf.x.apply(lambda x: x + 1, meta=("x", int)),
               df.x.apply(lambda x: x + 1))
 
     # specify meta
@@ -1712,16 +1713,19 @@ def test_apply():
               df.apply(lambda xy: xy[0] + xy[1], axis='columns'))
 
     # inference
-    assert_eq(ddf.apply(lambda xy: xy[0] + xy[1], axis=1),
-              df.apply(lambda xy: xy[0] + xy[1], axis=1))
-    assert_eq(ddf.apply(lambda xy: xy, axis=1),
-              df.apply(lambda xy: xy, axis=1))
+    with pytest.warns(None):
+        assert_eq(ddf.apply(lambda xy: xy[0] + xy[1], axis=1),
+                  df.apply(lambda xy: xy[0] + xy[1], axis=1))
+    with pytest.warns(None):
+        assert_eq(ddf.apply(lambda xy: xy, axis=1),
+                  df.apply(lambda xy: xy, axis=1))
 
     # specify meta
     func = lambda x: pd.Series([x, x])
     assert_eq(ddf.x.apply(func, meta=[(0, int), (1, int)]), df.x.apply(func))
     # inference
-    assert_eq(ddf.x.apply(func), df.x.apply(func))
+    with pytest.warns(None):
+        assert_eq(ddf.x.apply(func), df.x.apply(func))
 
     # axis=0
     with pytest.raises(NotImplementedError):
@@ -1731,6 +1735,21 @@ def test_apply():
         ddf.apply(lambda xy: xy, axis='index')
 
 
+def test_apply_warns():
+    df = pd.DataFrame({'x': [1, 2, 3, 4], 'y': [10, 20, 30, 40]})
+    ddf = dd.from_pandas(df, npartitions=2)
+
+    func = lambda row: row['x'] + row['y']
+
+    with pytest.warns(UserWarning) as w:
+        ddf.apply(func, axis=1)
+    assert len(w) == 1
+
+    with pytest.warns(None) as w:
+        ddf.apply(func, axis=1, meta=(None, int))
+    assert len(w) == 0
+
+
 def test_applymap():
     df = pd.DataFrame({'x': [1, 2, 3, 4], 'y': [10, 20, 30, 40]})
     ddf = dd.from_pandas(df, npartitions=2)
@@ -1886,13 +1905,15 @@ def return_df(x):
         return pd.Series([x.sum(), x.mean()], index=['sum', 'mean'])
 
     # DataFrame to completely different DataFrame
-    result = ddf.apply(return_df, axis=1)
+    with pytest.warns(None):
+        result = ddf.apply(return_df, axis=1)
     assert isinstance(result, dd.DataFrame)
     tm.assert_index_equal(result.columns, pd.Index(['sum', 'mean']))
     assert_eq(result, df.apply(return_df, axis=1))
 
     # DataFrame to Series
-    result = ddf.apply(lambda x: 1, axis=1)
+    with pytest.warns(None):
+        result = ddf.apply(lambda x: 1, axis=1)
     assert isinstance(result, dd.Series)
     assert result.name is None
     assert_eq(result, df.apply(lambda x: 1, axis=1))
@@ -1901,13 +1922,15 @@ def return_df2(x):
         return pd.Series([x * 2, x * 3], index=['x2', 'x3'])
 
     # Series to completely different DataFrame
-    result = ddf.x.apply(return_df2)
+    with pytest.warns(None):
+        result = ddf.x.apply(return_df2)
     assert isinstance(result, dd.DataFrame)
     tm.assert_index_equal(result.columns, pd.Index(['x2', 'x3']))
     assert_eq(result, df.x.apply(return_df2))
 
     # Series to Series
-    result = ddf.x.apply(lambda x: 1)
+    with pytest.warns(None):
+        result = ddf.x.apply(lambda x: 1)
     assert isinstance(result, dd.Series)
     assert result.name == 'x'
     assert_eq(result, df.x.apply(lambda x: 1))