Skip to content

TYP/CLN: assorted cleanups #38959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 5, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pandas/_libs/tslibs/offsets.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -3565,7 +3565,7 @@ cpdef to_offset(freq):
f"to_offset does not support tuples {freq}, pass as a string instead"
)

elif isinstance(freq, timedelta):
elif PyDelta_Check(freq):
return delta_to_tick(freq)

elif isinstance(freq, str):
Expand Down
10 changes: 1 addition & 9 deletions pandas/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -1002,14 +1002,6 @@ def tz_aware_fixture(request):
tz_aware_fixture2 = tz_aware_fixture


@pytest.fixture(scope="module")
def datetime_tz_utc():
"""
Yields the UTC timezone object from the datetime module.
"""
return timezone.utc


@pytest.fixture(params=["utc", "dateutil/UTC", utc, tzutc(), timezone.utc])
def utc_fixture(request):
"""
Expand Down Expand Up @@ -1189,7 +1181,7 @@ def any_nullable_int_dtype(request):


@pytest.fixture(params=tm.ALL_EA_INT_DTYPES + tm.FLOAT_EA_DTYPES)
def any_numeric_dtype(request):
def any_nullable_numeric_dtype(request):
"""
Parameterized fixture for any nullable integer dtype and
any float ea dtypes.
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -658,7 +658,7 @@ def _astype(self, dtype: DtypeObj, copy: bool) -> ArrayLike:
values = values.astype(dtype, copy=copy)

else:
values = astype_nansafe(values, dtype, copy=True)
values = astype_nansafe(values, dtype, copy=copy)

return values

Expand Down
2 changes: 1 addition & 1 deletion pandas/core/internals/concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ def _get_empty_dtype_and_na(join_units: Sequence[JoinUnit]) -> Tuple[DtypeObj, A
return np.dtype("M8[ns]"), np.datetime64("NaT", "ns")
elif "timedelta" in upcast_classes:
return np.dtype("m8[ns]"), np.timedelta64("NaT", "ns")
else: # pragma
else:
try:
common_dtype = np.find_common_type(upcast_classes, [])
except TypeError:
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -4494,7 +4494,7 @@ def replace(
method=method,
)

def _replace_single(self, to_replace, method, inplace, limit):
def _replace_single(self, to_replace, method: str, inplace: bool, limit):
"""
Replaces values in a Series using the fill method specified when no
replacement value is given in the replace method
Expand Down
46 changes: 25 additions & 21 deletions pandas/io/pytables.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@
if TYPE_CHECKING:
from tables import Col, File, Node

from pandas.core.internals import Block


# versioning attribute
_version = "0.15.2"
Expand Down Expand Up @@ -3860,20 +3862,17 @@ def _create_axes(
for a in new_non_index_axes:
obj = _reindex_axis(obj, a[0], a[1])

def get_blk_items(mgr, blocks):
return [mgr.items.take(blk.mgr_locs) for blk in blocks]

transposed = new_index.axis == 1

# figure out data_columns and get out blocks
data_columns = self.validate_data_columns(
data_columns, min_itemsize, new_non_index_axes
)

block_obj = self.get_object(obj, transposed)._consolidate()
frame = self.get_object(obj, transposed)._consolidate()

blocks, blk_items = self._get_blocks_and_items(
block_obj, table_exists, new_non_index_axes, self.values_axes, data_columns
frame, table_exists, new_non_index_axes, self.values_axes, data_columns
)

# add my values
Expand Down Expand Up @@ -3978,35 +3977,39 @@ def get_blk_items(mgr, blocks):

@staticmethod
def _get_blocks_and_items(
block_obj, table_exists, new_non_index_axes, values_axes, data_columns
frame: DataFrame,
table_exists: bool,
new_non_index_axes,
values_axes,
data_columns,
):
# Helper to clarify non-state-altering parts of _create_axes

def get_blk_items(mgr, blocks):
return [mgr.items.take(blk.mgr_locs) for blk in blocks]
def get_blk_items(mgr):
return [mgr.items.take(blk.mgr_locs) for blk in mgr.blocks]

blocks = block_obj._mgr.blocks
blk_items = get_blk_items(block_obj._mgr, blocks)
blocks: List["Block"] = list(frame._mgr.blocks)
blk_items: List[Index] = get_blk_items(frame._mgr)

if len(data_columns):
axis, axis_labels = new_non_index_axes[0]
new_labels = Index(axis_labels).difference(Index(data_columns))
mgr = block_obj.reindex(new_labels, axis=axis)._mgr
mgr = frame.reindex(new_labels, axis=axis)._mgr

blocks = list(mgr.blocks)
blk_items = get_blk_items(mgr, blocks)
blk_items = get_blk_items(mgr)
for c in data_columns:
mgr = block_obj.reindex([c], axis=axis)._mgr
mgr = frame.reindex([c], axis=axis)._mgr
blocks.extend(mgr.blocks)
blk_items.extend(get_blk_items(mgr, mgr.blocks))
blk_items.extend(get_blk_items(mgr))

# reorder the blocks in the same order as the existing table if we can
if table_exists:
by_items = {
tuple(b_items.tolist()): (b, b_items)
for b, b_items in zip(blocks, blk_items)
}
new_blocks = []
new_blocks: List["Block"] = []
new_blk_items = []
for ea in values_axes:
items = tuple(ea.values)
Expand Down Expand Up @@ -4875,7 +4878,7 @@ def _unconvert_index(


def _maybe_convert_for_string_atom(
name: str, block, existing_col, min_itemsize, nan_rep, encoding, errors
name: str, block: "Block", existing_col, min_itemsize, nan_rep, encoding, errors
):
if not block.is_object:
return block.values
Expand All @@ -4895,11 +4898,12 @@ def _maybe_convert_for_string_atom(
elif not (inferred_type == "string" or dtype_name == "object"):
return block.values

block = block.fillna(nan_rep, downcast=False)
if isinstance(block, list):
# Note: because block is always object dtype, fillna goes
# through a path such that the result is always a 1-element list
block = block[0]
blocks: List["Block"] = block.fillna(nan_rep, downcast=False)
# Note: because block is always object dtype, fillna goes
# through a path such that the result is always a 1-element list
assert len(blocks) == 1
block = blocks[0]

data = block.values

# see if we have a valid string type
Expand Down
1 change: 0 additions & 1 deletion pandas/tests/indexes/datetimes/test_timezones.py
Original file line number Diff line number Diff line change
Expand Up @@ -1159,7 +1159,6 @@ def test_dti_union_mixed(self):
@pytest.mark.parametrize(
"tz", [None, "UTC", "US/Central", dateutil.tz.tzoffset(None, -28800)]
)
@pytest.mark.usefixtures("datetime_tz_utc")
def test_iteration_preserves_nanoseconds(self, tz):
# GH 19603
index = DatetimeIndex(
Expand Down
8 changes: 0 additions & 8 deletions pandas/tests/indexes/timedeltas/test_partial_slicing.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import numpy as np
import pytest

from pandas import Series, timedelta_range
import pandas._testing as tm
Expand All @@ -22,13 +21,6 @@ def test_partial_slice(self):
expected = s.iloc[:134]
tm.assert_series_equal(result, expected)

result = s["6 days, 23:11:12"]
assert result == s.iloc[133]

msg = r"^Timedelta\('50 days 00:00:00'\)$"
with pytest.raises(KeyError, match=msg):
s["50 days"]

def test_partial_slice_high_reso(self):

# higher reso
Expand Down
18 changes: 18 additions & 0 deletions pandas/tests/series/indexing/test_getitem.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
Timestamp,
date_range,
period_range,
timedelta_range,
)
import pandas._testing as tm
from pandas.core.indexing import IndexingError
Expand Down Expand Up @@ -121,6 +122,23 @@ def test_getitem_scalar_categorical_index(self):
result = ser[cats[0]]
assert result == expected

def test_getitem_str_with_timedeltaindex(self):
rng = timedelta_range("1 day 10:11:12", freq="h", periods=500)
ser = Series(np.arange(len(rng)), index=rng)

key = "6 days, 23:11:12"
indexer = rng.get_loc(key)
assert indexer == 133

result = ser[key]
assert result == ser.iloc[133]

msg = r"^Timedelta\('50 days 00:00:00'\)$"
with pytest.raises(KeyError, match=msg):
rng.get_loc("50 days")
with pytest.raises(KeyError, match=msg):
ser["50 days"]


class TestSeriesGetitemSlices:
def test_getitem_partial_str_slice_with_datetimeindex(self):
Expand Down
16 changes: 8 additions & 8 deletions pandas/tests/series/indexing/test_setitem.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,19 +167,19 @@ def test_setitem_boolean_td64_values_cast_na(self, value):
expected = Series([NaT, 1, 2], dtype="timedelta64[ns]")
tm.assert_series_equal(series, expected)

def test_setitem_boolean_nullable_int_types(self, any_numeric_dtype):
def test_setitem_boolean_nullable_int_types(self, any_nullable_numeric_dtype):
# GH: 26468
ser = Series([5, 6, 7, 8], dtype=any_numeric_dtype)
ser[ser > 6] = Series(range(4), dtype=any_numeric_dtype)
expected = Series([5, 6, 2, 3], dtype=any_numeric_dtype)
ser = Series([5, 6, 7, 8], dtype=any_nullable_numeric_dtype)
ser[ser > 6] = Series(range(4), dtype=any_nullable_numeric_dtype)
expected = Series([5, 6, 2, 3], dtype=any_nullable_numeric_dtype)
tm.assert_series_equal(ser, expected)

ser = Series([5, 6, 7, 8], dtype=any_numeric_dtype)
ser.loc[ser > 6] = Series(range(4), dtype=any_numeric_dtype)
ser = Series([5, 6, 7, 8], dtype=any_nullable_numeric_dtype)
ser.loc[ser > 6] = Series(range(4), dtype=any_nullable_numeric_dtype)
tm.assert_series_equal(ser, expected)

ser = Series([5, 6, 7, 8], dtype=any_numeric_dtype)
loc_ser = Series(range(4), dtype=any_numeric_dtype)
ser = Series([5, 6, 7, 8], dtype=any_nullable_numeric_dtype)
loc_ser = Series(range(4), dtype=any_nullable_numeric_dtype)
ser.loc[ser > 6] = loc_ser.loc[loc_ser > 1]
tm.assert_series_equal(ser, expected)

Expand Down
28 changes: 14 additions & 14 deletions pandas/tests/series/test_constructors.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,8 @@ def test_constructor_index_dtype(self, dtype):
[
([1, 2]),
(["1", "2"]),
(list(pd.date_range("1/1/2011", periods=2, freq="H"))),
(list(pd.date_range("1/1/2011", periods=2, freq="H", tz="US/Eastern"))),
(list(date_range("1/1/2011", periods=2, freq="H"))),
(list(date_range("1/1/2011", periods=2, freq="H", tz="US/Eastern"))),
([Interval(left=0, right=5)]),
],
)
Expand Down Expand Up @@ -628,10 +628,10 @@ def test_constructor_copy(self):
@pytest.mark.parametrize(
"index",
[
pd.date_range("20170101", periods=3, tz="US/Eastern"),
pd.date_range("20170101", periods=3),
pd.timedelta_range("1 day", periods=3),
pd.period_range("2012Q1", periods=3, freq="Q"),
date_range("20170101", periods=3, tz="US/Eastern"),
date_range("20170101", periods=3),
timedelta_range("1 day", periods=3),
period_range("2012Q1", periods=3, freq="Q"),
Index(list("abc")),
pd.Int64Index([1, 2, 3]),
RangeIndex(0, 3),
Expand Down Expand Up @@ -1038,16 +1038,16 @@ def test_construction_consistency(self):

# make sure that we are not re-localizing upon construction
# GH 14928
s = Series(pd.date_range("20130101", periods=3, tz="US/Eastern"))
ser = Series(date_range("20130101", periods=3, tz="US/Eastern"))

result = Series(s, dtype=s.dtype)
tm.assert_series_equal(result, s)
result = Series(ser, dtype=ser.dtype)
tm.assert_series_equal(result, ser)

result = Series(s.dt.tz_convert("UTC"), dtype=s.dtype)
tm.assert_series_equal(result, s)
result = Series(ser.dt.tz_convert("UTC"), dtype=ser.dtype)
tm.assert_series_equal(result, ser)

result = Series(s.values, dtype=s.dtype)
tm.assert_series_equal(result, s)
result = Series(ser.values, dtype=ser.dtype)
tm.assert_series_equal(result, ser)

@pytest.mark.parametrize(
"data_constructor", [list, np.array], ids=["list", "ndarray[object]"]
Expand Down Expand Up @@ -1374,7 +1374,7 @@ def test_convert_non_ns(self):
# convert from a numpy array of non-ns timedelta64
arr = np.array([1, 2, 3], dtype="timedelta64[s]")
s = Series(arr)
expected = Series(pd.timedelta_range("00:00:01", periods=3, freq="s"))
expected = Series(timedelta_range("00:00:01", periods=3, freq="s"))
tm.assert_series_equal(s, expected)

# convert from a numpy array of non-ns datetime64
Expand Down