Skip to content

CLN refactor rest of core #37586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 38 commits into from
Jan 20, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
3f275aa
refactor rest of core
MarcoGorelli Nov 2, 2020
4a6c153
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 4, 2020
9db6e6c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 6, 2020
8794ceb
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 7, 2020
803f49b
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 8, 2020
4511a0c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 9, 2020
9c3598b
apply some suggestions
MarcoGorelli Nov 9, 2020
873d430
logicalnot -> ~
MarcoGorelli Nov 9, 2020
31d29af
parens
MarcoGorelli Nov 9, 2020
7a51410
parens
MarcoGorelli Nov 9, 2020
a31e89d
parens
MarcoGorelli Nov 9, 2020
a8c8c9c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 23, 2020
5e053d4
factor out _set_join_index
MarcoGorelli Nov 23, 2020
5f0a278
:label: type
MarcoGorelli Nov 23, 2020
142e837
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 29, 2020
8e18225
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 6, 2020
f181cb0
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 9, 2020
9383755
coverage, remove putmask
MarcoGorelli Dec 9, 2020
fe0e0ba
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 16, 2020
0d10a5b
pass dtype to maybe_castable directly
MarcoGorelli Dec 16, 2020
938b2d5
revert sourcery's change of moving message closer to usage
MarcoGorelli Dec 16, 2020
7fda4e7
avoid duplicated check
MarcoGorelli Dec 16, 2020
c36bf8d
remove redundant check
MarcoGorelli Dec 16, 2020
2c2868e
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 22, 2020
c6d3233
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 26, 2020
1f782fd
Dtype -> DtypeObj
MarcoGorelli Dec 26, 2020
b315629
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 3, 2021
84207d9
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 17, 2021
3f79955
remove redundant parens
MarcoGorelli Jan 17, 2021
9888383
use set to check for membership
MarcoGorelli Jan 17, 2021
0cd394f
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 17, 2021
2ec839e
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 18, 2021
5125e5f
revert change in generic.py
MarcoGorelli Jan 18, 2021
d693f58
revert _set_join_index refactoring
MarcoGorelli Jan 18, 2021
d4e3bd6
deduplicate from merge
MarcoGorelli Jan 18, 2021
5559352
:art:
MarcoGorelli Jan 18, 2021
4f14b81
some reversions
MarcoGorelli Jan 18, 2021
c03d876
revert maybe_castable
MarcoGorelli Jan 18, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions pandas/core/aggregation.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,6 @@ def validate_func_kwargs(
>>> validate_func_kwargs({'one': 'min', 'two': 'max'})
(['one', 'two'], ['min', 'max'])
"""
no_arg_message = "Must provide 'func' or named aggregation **kwargs."
tuple_given_message = "func is expected but received {} in **kwargs."
columns = list(kwargs)
func = []
Expand All @@ -397,6 +396,7 @@ def validate_func_kwargs(
raise TypeError(tuple_given_message.format(type(col_func).__name__))
func.append(col_func)
if not columns:
no_arg_message = "Must provide 'func' or named aggregation **kwargs."
raise TypeError(no_arg_message)
return columns, func

Expand Down Expand Up @@ -499,14 +499,14 @@ def transform_dict_like(
try:
results[name] = transform(colg, how, 0, *args, **kwargs)
except Exception as err:
if (
str(err) == "Function did not transform"
or str(err) == "No transform functions were provided"
):
if str(err) in {
"Function did not transform",
"No transform functions were provided",
}:
raise err

# combine results
if len(results) == 0:
if not results:
raise ValueError("Transform function failed")
return concat(results, axis=1)

Expand Down
20 changes: 7 additions & 13 deletions pandas/core/algorithms.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,12 +150,10 @@ def _ensure_data(
from pandas import PeriodIndex

values = PeriodIndex(values)._data
dtype = values.dtype
elif is_timedelta64_dtype(values.dtype) or is_timedelta64_dtype(dtype):
from pandas import TimedeltaIndex

values = TimedeltaIndex(values)._data
dtype = values.dtype
else:
# Datetime
if values.ndim > 1 and is_datetime64_ns_dtype(values.dtype):
Expand All @@ -169,8 +167,7 @@ def _ensure_data(
from pandas import DatetimeIndex

values = DatetimeIndex(values)._data
dtype = values.dtype

dtype = values.dtype
return values.asi8, dtype

elif is_categorical_dtype(values.dtype) and (
Expand Down Expand Up @@ -875,10 +872,9 @@ def value_counts_arraylike(values, dropna: bool):
keys, counts = f(values, dropna)

mask = isna(values)
if not dropna and mask.any():
if not isna(keys).any():
keys = np.insert(keys, 0, np.NaN)
counts = np.insert(counts, 0, mask.sum())
if not dropna and mask.any() and not isna(keys).any():
keys = np.insert(keys, 0, np.NaN)
counts = np.insert(counts, 0, mask.sum())

keys = _reconstruct_data(keys, original.dtype, original)

Expand Down Expand Up @@ -1741,9 +1737,8 @@ def take_nd(
dtype, fill_value = arr.dtype, arr.dtype.type()

flip_order = False
if arr.ndim == 2:
if arr.flags.f_contiguous:
flip_order = True
if arr.ndim == 2 and arr.flags.f_contiguous:
flip_order = True

if flip_order:
arr = arr.T
Expand Down Expand Up @@ -1915,8 +1910,7 @@ def searchsorted(arr, value, side="left", sorter=None) -> np.ndarray:
# and `value` is a pd.Timestamp, we may need to convert value
arr = ensure_wrapped_if_datetimelike(arr)

result = arr.searchsorted(value, side=side, sorter=sorter)
return result
return arr.searchsorted(value, side=side, sorter=sorter)


# ---- #
Expand Down
3 changes: 1 addition & 2 deletions pandas/core/array_algos/replace.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@ def _check_comparison_types(
if is_scalar(result) and isinstance(a, np.ndarray):
type_names = [type(a).__name__, type(b).__name__]

if isinstance(a, np.ndarray):
type_names[0] = f"ndarray(dtype={a.dtype})"
type_names[0] = f"ndarray(dtype={a.dtype})"

raise TypeError(
f"Cannot compare types {repr(type_names[0])} and {repr(type_names[1])}"
Expand Down
13 changes: 5 additions & 8 deletions pandas/core/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,10 +321,9 @@ def _try_aggregate_string_function(self, arg: str, *args, **kwargs):
return f

f = getattr(np, arg, None)
if f is not None:
if hasattr(self, "__array__"):
# in particular exclude Window
return f(self, *args, **kwargs)
if f is not None and hasattr(self, "__array__"):
# in particular exclude Window
return f(self, *args, **kwargs)

raise AttributeError(
f"'{arg}' is not a valid function for '{type(self).__name__}' object"
Expand Down Expand Up @@ -1046,15 +1045,14 @@ def value_counts(
1.0 1
dtype: int64
"""
result = value_counts(
return value_counts(
self,
sort=sort,
ascending=ascending,
normalize=normalize,
bins=bins,
dropna=dropna,
)
return result

def unique(self):
values = self._values
Expand Down Expand Up @@ -1317,8 +1315,7 @@ def drop_duplicates(self, keep="first"):
duplicated = self.duplicated(keep=keep)
# pandas\core\base.py:1507: error: Value of type "IndexOpsMixin" is not
# indexable [index]
result = self[np.logical_not(duplicated)] # type: ignore[index]
return result
return self[~duplicated] # type: ignore[index]

def duplicated(self, keep="first"):
return duplicated(self._values, keep=keep)
3 changes: 1 addition & 2 deletions pandas/core/construction.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,8 +343,7 @@ def array(
elif is_timedelta64_ns_dtype(dtype):
return TimedeltaArray._from_sequence(data, dtype=dtype, copy=copy)

result = PandasArray._from_sequence(data, dtype=dtype, copy=copy)
return result
return PandasArray._from_sequence(data, dtype=dtype, copy=copy)


def extract_array(obj: object, extract_numpy: bool = False) -> Union[Any, ArrayLike]:
Expand Down
8 changes: 4 additions & 4 deletions pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,7 @@ def _repr_fits_horizontal_(self, ignore_width: bool = False) -> bool:
# and to_string on entire frame may be expensive
d = self

if not (max_rows is None): # unlimited rows
if max_rows is not None: # unlimited rows
# min of two, where one may be None
d = d.iloc[: min(max_rows, len(d))]
else:
Expand Down Expand Up @@ -2029,10 +2029,10 @@ def to_records(
np.asarray(self.iloc[:, i]) for i in range(len(self.columns))
]

count = 0
index_names = list(self.index.names)

if isinstance(self.index, MultiIndex):
count = 0
for i, n in enumerate(index_names):
if n is None:
index_names[i] = f"level_{count}"
Expand Down Expand Up @@ -3334,7 +3334,7 @@ def _set_value(self, index, col, value, takeable: bool = False):
takeable : interpret the index/col as indexers, default False
"""
try:
if takeable is True:
if takeable:
series = self._ixs(col, axis=1)
series._set_value(index, value, takeable=True)
return
Expand Down Expand Up @@ -5041,7 +5041,7 @@ class max type

multi_col = isinstance(self.columns, MultiIndex)
for i, (lev, lab) in reversed(list(enumerate(to_insert))):
if not (level is None or i in level):
if level is not None and i not in level:
continue
name = names[i]
if multi_col:
Expand Down
91 changes: 44 additions & 47 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -773,8 +773,7 @@ def droplevel(self: FrameOrSeries, level, axis=0) -> FrameOrSeries:
"""
labels = self._get_axis(axis)
new_labels = labels.droplevel(level)
result = self.set_axis(new_labels, axis=axis, inplace=False)
return result
return self.set_axis(new_labels, axis=axis, inplace=False)

def pop(self, item: Hashable) -> Union[Series, Any]:
result = self[item]
Expand Down Expand Up @@ -1445,8 +1444,7 @@ def __invert__(self):
return self

new_data = self._mgr.apply(operator.invert)
result = self._constructor(new_data).__finalize__(self, method="__invert__")
return result
return self._constructor(new_data).__finalize__(self, method="__invert__")

@final
def __nonzero__(self):
Expand Down Expand Up @@ -2036,8 +2034,7 @@ def _repr_data_resource_(self):

as_json = data.to_json(orient="table")
as_json = cast(str, as_json)
payload = json.loads(as_json, object_pairs_hook=collections.OrderedDict)
return payload
return json.loads(as_json, object_pairs_hook=collections.OrderedDict)

# ----------------------------------------------------------------------
# I/O Methods
Expand Down Expand Up @@ -5342,11 +5339,11 @@ def sample(
"Replace has to be set to `True` when "
"upsampling the population `frac` > 1."
)
elif n is not None and frac is None and n % 1 != 0:
elif frac is None and n % 1 != 0:
raise ValueError("Only integers accepted as `n` values")
elif n is None and frac is not None:
n = round(frac * axis_length)
elif n is not None and frac is not None:
elif frac is not None:
raise ValueError("Please enter a value for `frac` OR `n`, not both")

# Check for negative sizes
Expand Down Expand Up @@ -5467,15 +5464,13 @@ def __getattr__(self, name: str):
# Note: obj.x will always call obj.__getattribute__('x') prior to
# calling obj.__getattr__('x').
if (
name in self._internal_names_set
or name in self._metadata
or name in self._accessors
name not in self._internal_names_set
and name not in self._metadata
and name not in self._accessors
and self._info_axis._can_hold_identifiers_and_holds_name(name)
):
return object.__getattribute__(self, name)
else:
if self._info_axis._can_hold_identifiers_and_holds_name(name):
return self[name]
return object.__getattribute__(self, name)
return self[name]
return object.__getattribute__(self, name)

def __setattr__(self, name: str, value) -> None:
"""
Expand Down Expand Up @@ -5585,17 +5580,16 @@ def _is_mixed_type(self) -> bool_t:
@final
def _check_inplace_setting(self, value) -> bool_t:
""" check whether we allow in-place setting with this type of value """
if self._is_mixed_type:
if not self._mgr.is_numeric_mixed_type:
if self._is_mixed_type and not self._mgr.is_numeric_mixed_type:

# allow an actual np.nan thru
if is_float(value) and np.isnan(value):
return True
# allow an actual np.nan thru
if is_float(value) and np.isnan(value):
return True

raise TypeError(
"Cannot do inplace boolean setting on "
"mixed-types with a non np.nan value"
)
raise TypeError(
"Cannot do inplace boolean setting on "
"mixed-types with a non np.nan value"
)

return True

Expand Down Expand Up @@ -6264,8 +6258,7 @@ def convert_dtypes(
)
for col_name, col in self.items()
]
result = concat(results, axis=1, copy=False)
return result
return concat(results, axis=1, copy=False)

# ----------------------------------------------------------------------
# Filling NA's
Expand Down Expand Up @@ -7443,9 +7436,13 @@ def clip(
upper = None

# GH 2747 (arguments were reversed)
if lower is not None and upper is not None:
if is_scalar(lower) and is_scalar(upper):
lower, upper = min(lower, upper), max(lower, upper)
if (
lower is not None
and upper is not None
and is_scalar(lower)
and is_scalar(upper)
):
lower, upper = min(lower, upper), max(lower, upper)

# fast-path for scalars
if (lower is None or (is_scalar(lower) and is_number(lower))) and (
Expand Down Expand Up @@ -8234,10 +8231,9 @@ def first(self: FrameOrSeries, offset) -> FrameOrSeries:
end_date = end = self.index[0] + offset

# Tick-like, e.g. 3 weeks
if isinstance(offset, Tick):
if end_date in self.index:
end = self.index.searchsorted(end_date, side="left")
return self.iloc[:end]
if isinstance(offset, Tick) and end_date in self.index:
end = self.index.searchsorted(end_date, side="left")
return self.iloc[:end]

return self.loc[:end]

Expand Down Expand Up @@ -8646,17 +8642,19 @@ def _align_frame(

is_series = isinstance(self, ABCSeries)

if axis is None or axis == 0:
if not self.index.equals(other.index):
join_index, ilidx, iridx = self.index.join(
other.index, how=join, level=level, return_indexers=True
)
if (axis is None or axis == 0) and not self.index.equals(other.index):
join_index, ilidx, iridx = self.index.join(
other.index, how=join, level=level, return_indexers=True
)

if axis is None or axis == 1:
if not is_series and not self.columns.equals(other.columns):
join_columns, clidx, cridx = self.columns.join(
other.columns, how=join, level=level, return_indexers=True
)
if (
(axis is None or axis == 1)
and not is_series
and not self.columns.equals(other.columns)
):
join_columns, clidx, cridx = self.columns.join(
other.columns, how=join, level=level, return_indexers=True
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think could de-duplicate these be using self.axes[axis] and other.axes[axis]


if is_series:
reindexers = {0: [join_index, ilidx]}
Expand Down Expand Up @@ -9526,9 +9524,8 @@ def truncate(
before = to_datetime(before)
after = to_datetime(after)

if before is not None and after is not None:
if before > after:
raise ValueError(f"Truncate: {after} must be after {before}")
if before is not None and after is not None and before > after:
raise ValueError(f"Truncate: {after} must be after {before}")

if len(ax) > 1 and ax.is_monotonic_decreasing:
before, after = after, before
Expand Down
11 changes: 5 additions & 6 deletions pandas/core/indexers.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,11 @@ def is_scalar_indexer(indexer, ndim: int) -> bool:
if ndim == 1 and is_integer(indexer):
# GH37748: allow indexer to be an integer for Series
return True
if isinstance(indexer, tuple):
if len(indexer) == ndim:
return all(
is_integer(x) or (isinstance(x, np.ndarray) and x.ndim == len(x) == 1)
for x in indexer
)
if isinstance(indexer, tuple) and len(indexer) == ndim:
return all(
is_integer(x) or (isinstance(x, np.ndarray) and x.ndim == len(x) == 1)
for x in indexer
)
return False


Expand Down
Loading