CLN: Assorted cleanups (#28848)

Author: jbrockmendel

ci/code_checks.sh

@@ -122,7 +122,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check for non-standard imports' ; echo $MSG
     invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas
     invgrep -R --include="*.py*" -E "from collections.abc import " pandas
-    # invgrep -R --include="*.py*" -E "from numpy import nan " pandas  # GH#24822 not yet implemented since the offending imports have not all been removed
+    invgrep -R --include="*.py*" -E "from numpy import nan " pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for use of exec' ; echo $MSG

pandas/core/internals/concat.py

@@ -287,7 +287,7 @@ def get_empty_dtype_and_na(join_units):
             return np.float64, np.nan
 
     if is_uniform_reindex(join_units):
-        # XXX: integrate property
+        # FIXME: integrate property
         empty_dtype = join_units[0].block.dtype
         upcasted_na = join_units[0].block.fill_value
         return empty_dtype, upcasted_na

pandas/core/internals/managers.py

@@ -2035,7 +2035,7 @@ def concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy):
             values = b.values
             if copy:
                 values = values.copy()
-            elif not copy:
+            else:
                 values = values.view()
             b = b.make_block_same_class(values, placement=placement)
         elif is_uniform_join_units(join_units):

pandas/core/ops/__init__.py

@@ -766,9 +766,9 @@ def f(self, other, axis=default_axis, level=None):
     return f
 
 
-def _comp_method_FRAME(cls, func, special):
-    str_rep = _get_opstr(func)
-    op_name = _get_op_name(func, special)
+def _comp_method_FRAME(cls, op, special):
+    str_rep = _get_opstr(op)
+    op_name = _get_op_name(op, special)
 
     @Appender("Wrapper for comparison method {name}".format(name=op_name))
     def f(self, other):
@@ -781,18 +781,18 @@ def f(self, other):
                 raise ValueError(
                     "Can only compare identically-labeled DataFrame objects"
                 )
-            new_data = dispatch_to_series(self, other, func, str_rep)
+            new_data = dispatch_to_series(self, other, op, str_rep)
             return self._construct_result(new_data)
 
         elif isinstance(other, ABCSeries):
             return _combine_series_frame(
-                self, other, func, fill_value=None, axis=None, level=None
+                self, other, op, fill_value=None, axis=None, level=None
             )
         else:
 
             # straight boolean comparisons we want to allow all columns
             # (regardless of dtype to pass thru) See #4537 for discussion.
-            new_data = dispatch_to_series(self, other, func)
+            new_data = dispatch_to_series(self, other, op)
             return self._construct_result(new_data)
 
     f.__name__ = op_name

pandas/core/ops/array_ops.py

@@ -118,14 +118,14 @@ def masked_arith_op(x, y, op):
     return result
 
 
-def define_na_arithmetic_op(op, str_rep, eval_kwargs):
+def define_na_arithmetic_op(op, str_rep: str, eval_kwargs):
     def na_op(x, y):
         return na_arithmetic_op(x, y, op, str_rep, eval_kwargs)
 
     return na_op
 
 
-def na_arithmetic_op(left, right, op, str_rep, eval_kwargs):
+def na_arithmetic_op(left, right, op, str_rep: str, eval_kwargs):
     """
     Return the result of evaluating op on the passed in values.
 
@@ -173,6 +173,7 @@ def arithmetic_op(
         Cannot be a DataFrame or Index.  Series is *not* excluded.
     op : {operator.add, operator.sub, ...}
         Or one of the reversed variants from roperator.
+    str_rep : str
 
     Returns
     -------
@@ -279,8 +280,16 @@ def comparison_op(
     return res_values
 
 
-def na_logical_op(x, y, op):
+def na_logical_op(x: np.ndarray, y, op):
     try:
+        # For exposition, write:
+        #  yarr = isinstance(y, np.ndarray)
+        #  yint = is_integer(y) or (yarr and y.dtype.kind == "i")
+        #  ybool = is_bool(y) or (yarr and y.dtype.kind == "b")
+        #  xint = x.dtype.kind == "i"
+        #  xbool = x.dtype.kind == "b"
+        # Then Cases where this goes through without raising include:
+        #  (xint or xbool) and (yint or bool)
         result = op(x, y)
     except TypeError:
         if isinstance(y, np.ndarray):
@@ -304,9 +313,9 @@ def na_logical_op(x, y, op):
                 NotImplementedError,
             ):
                 raise TypeError(
-                    "cannot compare a dtyped [{dtype}] array "
-                    "with a scalar of type [{typ}]".format(
-                        dtype=x.dtype, typ=type(y).__name__
+                    "Cannot perform '{op}' with a dtyped [{dtype}] array "
+                    "and scalar of type [{typ}]".format(
+                        op=op.__name__, dtype=x.dtype, typ=type(y).__name__
                     )
                 )
 

pandas/tests/groupby/test_bin_groupby.py

@@ -1,5 +1,4 @@
 import numpy as np
-from numpy import nan
 import pytest
 
 from pandas._libs import groupby, lib, reduction as libreduction
@@ -96,17 +95,17 @@ def _check(dtype):
 
         def _ohlc(group):
             if isna(group).all():
-                return np.repeat(nan, 4)
+                return np.repeat(np.nan, 4)
             return [group[0], group.max(), group.min(), group[-1]]
 
         expected = np.array([_ohlc(obj[:6]), _ohlc(obj[6:12]), _ohlc(obj[12:])])
 
         assert_almost_equal(out, expected)
         tm.assert_numpy_array_equal(counts, np.array([6, 6, 8], dtype=np.int64))
 
-        obj[:6] = nan
+        obj[:6] = np.nan
         func(out, counts, obj[:, None], labels)
-        expected[0] = nan
+        expected[0] = np.nan
         assert_almost_equal(out, expected)
 
     _check("float32")

pandas/tests/groupby/test_timegrouper.py

@@ -4,7 +4,6 @@
 from io import StringIO
 
 import numpy as np
-from numpy import nan
 import pytest
 import pytz
 
@@ -699,13 +698,13 @@ def test_first_last_max_min_on_time_data(self):
         df_test = DataFrame(
             {
                 "dt": [
-                    nan,
+                    np.nan,
                     "2015-07-24 10:10",
                     "2015-07-25 11:11",
                     "2015-07-23 12:12",
-                    nan,
+                    np.nan,
                 ],
-                "td": [nan, td(days=1), td(days=2), td(days=3), nan],
+                "td": [np.nan, td(days=1), td(days=2), td(days=3), np.nan],
             }
         )
         df_test.dt = pd.to_datetime(df_test.dt)

pandas/tests/io/excel/test_writers.py

@@ -4,7 +4,6 @@
 import os
 
 import numpy as np
-from numpy import nan
 import pytest
 
 from pandas.compat import PY36
@@ -323,7 +322,7 @@ def test_excel_writer_context_manager(self, frame, engine, ext):
 
     def test_roundtrip(self, engine, ext, frame):
         frame = frame.copy()
-        frame["A"][:5] = nan
+        frame["A"][:5] = np.nan
 
         frame.to_excel(self.path, "test1")
         frame.to_excel(self.path, "test1", columns=["A", "B"])
@@ -388,7 +387,7 @@ def test_ts_frame(self, tsframe, engine, ext):
 
     def test_basics_with_nan(self, engine, ext, frame):
         frame = frame.copy()
-        frame["A"][:5] = nan
+        frame["A"][:5] = np.nan
         frame.to_excel(self.path, "test1")
         frame.to_excel(self.path, "test1", columns=["A", "B"])
         frame.to_excel(self.path, "test1", header=False)
@@ -450,7 +449,7 @@ def test_inf_roundtrip(self, engine, ext):
 
     def test_sheets(self, engine, ext, frame, tsframe):
         frame = frame.copy()
-        frame["A"][:5] = nan
+        frame["A"][:5] = np.nan
 
         frame.to_excel(self.path, "test1")
         frame.to_excel(self.path, "test1", columns=["A", "B"])
@@ -473,7 +472,7 @@ def test_sheets(self, engine, ext, frame, tsframe):
 
     def test_colaliases(self, engine, ext, frame):
         frame = frame.copy()
-        frame["A"][:5] = nan
+        frame["A"][:5] = np.nan
 
         frame.to_excel(self.path, "test1")
         frame.to_excel(self.path, "test1", columns=["A", "B"])
@@ -491,7 +490,7 @@ def test_colaliases(self, engine, ext, frame):
 
     def test_roundtrip_indexlabels(self, merge_cells, engine, ext, frame):
         frame = frame.copy()
-        frame["A"][:5] = nan
+        frame["A"][:5] = np.nan
 
         frame.to_excel(self.path, "test1")
         frame.to_excel(self.path, "test1", columns=["A", "B"])

pandas/tests/io/parser/test_textreader.py

@@ -6,7 +6,6 @@
 import os
 
 import numpy as np
-from numpy import nan
 import pytest
 
 import pandas._libs.parsers as parser
@@ -309,10 +308,15 @@ def test_empty_field_eof(self):
         assert_array_dicts_equal(result, expected)
 
         # GH5664
-        a = DataFrame([["b"], [nan]], columns=["a"], index=["a", "c"])
+        a = DataFrame([["b"], [np.nan]], columns=["a"], index=["a", "c"])
         b = DataFrame([[1, 1, 1, 0], [1, 1, 1, 0]], columns=list("abcd"), index=[1, 1])
         c = DataFrame(
-            [[1, 2, 3, 4], [6, nan, nan, nan], [8, 9, 10, 11], [13, 14, nan, nan]],
+            [
+                [1, 2, 3, 4],
+                [6, np.nan, np.nan, np.nan],
+                [8, 9, 10, 11],
+                [13, 14, np.nan, np.nan],
+            ],
             columns=list("abcd"),
             index=[0, 5, 7, 12],
         )

pandas/tests/reshape/merge/test_merge.py

@@ -1,10 +1,9 @@
 from collections import OrderedDict
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 import random
 import re
 
 import numpy as np
-from numpy import nan
 import pytest
 
 from pandas.core.dtypes.common import is_categorical_dtype, is_object_dtype
@@ -565,9 +564,7 @@ def test_merge_all_na_column(self, series_of_dtype, series_of_dtype_all_na):
         assert_frame_equal(actual, expected)
 
     def test_merge_nosort(self):
-        # #2098, anything to do?
-
-        from datetime import datetime
+        # GH#2098, TODO: anything to do?
 
         d = {
             "var1": np.random.randint(0, 10, size=10),
@@ -621,9 +618,9 @@ def test_merge_nan_right(self):
         expected = DataFrame(
             {
                 "i1": {0: 0, 1: 1},
-                "i1_": {0: 0.0, 1: nan},
+                "i1_": {0: 0.0, 1: np.nan},
                 "i2": {0: 0.5, 1: 1.5},
-                "i3": {0: 0.69999999999999996, 1: nan},
+                "i3": {0: 0.69999999999999996, 1: np.nan},
             }
         )[["i1", "i2", "i1_", "i3"]]
         assert_frame_equal(result, expected)
@@ -640,21 +637,17 @@ def _constructor(self):
         assert isinstance(result, NotADataFrame)
 
     def test_join_append_timedeltas(self):
-
-        import datetime as dt
-        from pandas import NaT
-
         # timedelta64 issues with join/merge
         # GH 5695
 
-        d = {"d": dt.datetime(2013, 11, 5, 5, 56), "t": dt.timedelta(0, 22500)}
+        d = {"d": datetime(2013, 11, 5, 5, 56), "t": timedelta(0, 22500)}
         df = DataFrame(columns=list("dt"))
         df = df.append(d, ignore_index=True)
         result = df.append(d, ignore_index=True)
         expected = DataFrame(
             {
-                "d": [dt.datetime(2013, 11, 5, 5, 56), dt.datetime(2013, 11, 5, 5, 56)],
-                "t": [dt.timedelta(0, 22500), dt.timedelta(0, 22500)],
+                "d": [datetime(2013, 11, 5, 5, 56), datetime(2013, 11, 5, 5, 56)],
+                "t": [timedelta(0, 22500), timedelta(0, 22500)],
             }
         )
         assert_frame_equal(result, expected)
@@ -667,7 +660,7 @@ def test_join_append_timedeltas(self):
         expected = DataFrame(
             {
                 "0": Series([td, td], index=list("AB")),
-                "0r": Series([td, NaT], index=list("AB")),
+                "0r": Series([td, pd.NaT], index=list("AB")),
             }
         )
         assert_frame_equal(result, expected)

pandas/tests/reshape/merge/test_merge_ordered.py

@@ -1,4 +1,4 @@
-from numpy import nan
+import numpy as np
 import pytest
 
 import pandas as pd
@@ -17,8 +17,8 @@ def test_basic(self):
         expected = DataFrame(
             {
                 "key": ["a", "b", "c", "d", "e", "f"],
-                "lvalue": [1, nan, 2, nan, 3, nan],
-                "rvalue": [nan, 1, 2, 3, nan, 4],
+                "lvalue": [1, np.nan, 2, np.nan, 3, np.nan],
+                "rvalue": [np.nan, 1, 2, 3, np.nan, 4],
             }
         )
 
@@ -30,7 +30,7 @@ def test_ffill(self):
             {
                 "key": ["a", "b", "c", "d", "e", "f"],
                 "lvalue": [1.0, 1, 2, 2, 3, 3.0],
-                "rvalue": [nan, 1, 2, 3, 3, 4],
+                "rvalue": [np.nan, 1, 2, 3, 3, 4],
             }
         )
         assert_frame_equal(result, expected)
@@ -47,7 +47,7 @@ def test_multigroup(self):
             {
                 "key": ["a", "b", "c", "d", "e", "f"] * 2,
                 "lvalue": [1.0, 1, 2, 2, 3, 3.0] * 2,
-                "rvalue": [nan, 1, 2, 3, 3, 4] * 2,
+                "rvalue": [np.nan, 1, 2, 3, 3, 4] * 2,
             }
         )
         expected["group"] = ["a"] * 6 + ["b"] * 6
@@ -110,7 +110,7 @@ def test_doc_example(self):
                 "group": list("aaaaabbbbb"),
                 "key": ["a", "b", "c", "d", "e"] * 2,
                 "lvalue": [1, 1, 2, 2, 3] * 2,
-                "rvalue": [nan, 1, 2, 3, 3] * 2,
+                "rvalue": [np.nan, 1, 2, 3, 3] * 2,
             }
         )
 

pandas/tests/reshape/merge/test_multi.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict
 
 import numpy as np
-from numpy import nan
 from numpy.random import randn
 import pytest
 
@@ -311,11 +310,11 @@ def test_left_join_index_multi_match_multiindex(self):
             [
                 ["X", "Y", "C", "a", 6],
                 ["X", "Y", "C", "a", 9],
-                ["W", "Y", "C", "e", nan],
+                ["W", "Y", "C", "e", np.nan],
                 ["V", "Q", "A", "h", -3],
                 ["V", "R", "D", "i", 2],
                 ["V", "R", "D", "i", -1],
-                ["X", "Y", "D", "b", nan],
+                ["X", "Y", "D", "b", np.nan],
                 ["X", "Y", "A", "c", 1],
                 ["X", "Y", "A", "c", 4],
                 ["W", "Q", "B", "f", 3],
@@ -365,10 +364,10 @@ def test_left_join_index_multi_match(self):
                 ["c", 0, "x"],
                 ["c", 0, "r"],
                 ["c", 0, "s"],
-                ["b", 1, nan],
+                ["b", 1, np.nan],
                 ["a", 2, "v"],
                 ["a", 2, "z"],
-                ["b", 3, nan],
+                ["b", 3, np.nan],
             ],
             columns=["tag", "val", "char"],
             index=[2, 2, 2, 2, 0, 1, 1, 3],