Skip to content

CLN refactor rest of core #37586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 38 commits into from
Jan 20, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
3f275aa
refactor rest of core
MarcoGorelli Nov 2, 2020
4a6c153
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 4, 2020
9db6e6c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 6, 2020
8794ceb
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 7, 2020
803f49b
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 8, 2020
4511a0c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 9, 2020
9c3598b
apply some suggestions
MarcoGorelli Nov 9, 2020
873d430
logicalnot -> ~
MarcoGorelli Nov 9, 2020
31d29af
parens
MarcoGorelli Nov 9, 2020
7a51410
parens
MarcoGorelli Nov 9, 2020
a31e89d
parens
MarcoGorelli Nov 9, 2020
a8c8c9c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 23, 2020
5e053d4
factor out _set_join_index
MarcoGorelli Nov 23, 2020
5f0a278
:label: type
MarcoGorelli Nov 23, 2020
142e837
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 29, 2020
8e18225
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 6, 2020
f181cb0
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 9, 2020
9383755
coverage, remove putmask
MarcoGorelli Dec 9, 2020
fe0e0ba
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 16, 2020
0d10a5b
pass dtype to maybe_castable directly
MarcoGorelli Dec 16, 2020
938b2d5
revert sourcery's change of moving message closer to usage
MarcoGorelli Dec 16, 2020
7fda4e7
avoid duplicated check
MarcoGorelli Dec 16, 2020
c36bf8d
remove redundant check
MarcoGorelli Dec 16, 2020
2c2868e
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 22, 2020
c6d3233
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 26, 2020
1f782fd
Dtype -> DtypeObj
MarcoGorelli Dec 26, 2020
b315629
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 3, 2021
84207d9
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 17, 2021
3f79955
remove redundant parens
MarcoGorelli Jan 17, 2021
9888383
use set to check for membership
MarcoGorelli Jan 17, 2021
0cd394f
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 17, 2021
2ec839e
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 18, 2021
5125e5f
revert change in generic.py
MarcoGorelli Jan 18, 2021
d693f58
revert _set_join_index refactoring
MarcoGorelli Jan 18, 2021
d4e3bd6
deduplicate from merge
MarcoGorelli Jan 18, 2021
5559352
:art:
MarcoGorelli Jan 18, 2021
4f14b81
some reversions
MarcoGorelli Jan 18, 2021
c03d876
revert maybe_castable
MarcoGorelli Jan 18, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions pandas/core/aggregation.py
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,6 @@ def validate_func_kwargs(
>>> validate_func_kwargs({'one': 'min', 'two': 'max'})
(['one', 'two'], ['min', 'max'])
"""
no_arg_message = "Must provide 'func' or named aggregation **kwargs."
tuple_given_message = "func is expected but received {} in **kwargs."
columns = list(kwargs)
func = []
Expand All @@ -396,6 +395,7 @@ def validate_func_kwargs(
raise TypeError(tuple_given_message.format(type(col_func).__name__))
func.append(col_func)
if not columns:
no_arg_message = "Must provide 'func' or named aggregation **kwargs."
raise TypeError(no_arg_message)
return columns, func

Expand Down Expand Up @@ -497,14 +497,14 @@ def transform_dict_like(
try:
results[name] = transform(colg, how, 0, *args, **kwargs)
except Exception as err:
if (
str(err) == "Function did not transform"
or str(err) == "No transform functions were provided"
):
if str(err) in [
"Function did not transform",
"No transform functions were provided",
]:
raise err

# combine results
if len(results) == 0:
if not results:
raise ValueError("Transform function failed")
return concat(results, axis=1)

Expand Down
30 changes: 13 additions & 17 deletions pandas/core/algorithms.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,13 @@ def _ensure_data(
values = extract_array(values, extract_numpy=True)

# we check some simple dtypes first
if is_object_dtype(dtype):
return ensure_object(np.asarray(values)), np.dtype("object")
elif is_object_dtype(values) and dtype is None:
if (
is_object_dtype(dtype)
or not is_object_dtype(dtype)
and is_object_dtype(values)
and dtype is None
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you put parens where appropriate to make this more obvious

):
return ensure_object(np.asarray(values)), np.dtype("object")

try:
if is_bool_dtype(values) or is_bool_dtype(dtype):
# we are actually coercing to uint64
Expand Down Expand Up @@ -137,12 +139,10 @@ def _ensure_data(
from pandas import PeriodIndex

values = PeriodIndex(values)
dtype = values.dtype
elif is_timedelta64_dtype(vals_dtype) or is_timedelta64_dtype(dtype):
from pandas import TimedeltaIndex

values = TimedeltaIndex(values)
dtype = values.dtype
else:
# Datetime
if values.ndim > 1 and is_datetime64_ns_dtype(vals_dtype):
Expand All @@ -156,8 +156,7 @@ def _ensure_data(
from pandas import DatetimeIndex

values = DatetimeIndex(values)
dtype = values.dtype

dtype = values.dtype
return values.asi8, dtype

elif is_categorical_dtype(vals_dtype) and (
Expand Down Expand Up @@ -821,10 +820,9 @@ def value_counts_arraylike(values, dropna: bool):
keys, counts = f(values, dropna)

mask = isna(values)
if not dropna and mask.any():
if not isna(keys).any():
keys = np.insert(keys, 0, np.NaN)
counts = np.insert(counts, 0, mask.sum())
if not dropna and mask.any() and not isna(keys).any():
keys = np.insert(keys, 0, np.NaN)
counts = np.insert(counts, 0, mask.sum())

keys = _reconstruct_data(keys, original.dtype, original)

Expand Down Expand Up @@ -1684,9 +1682,8 @@ def take_nd(
dtype, fill_value = arr.dtype, arr.dtype.type()

flip_order = False
if arr.ndim == 2:
if arr.flags.f_contiguous:
flip_order = True
if arr.ndim == 2 and arr.flags.f_contiguous:
flip_order = True

if flip_order:
arr = arr.T
Expand Down Expand Up @@ -1861,8 +1858,7 @@ def searchsorted(arr, value, side="left", sorter=None):
if isinstance(value, Timestamp) and value.tzinfo is None:
value = value.to_datetime64()

result = arr.searchsorted(value, side=side, sorter=sorter)
return result
return arr.searchsorted(value, side=side, sorter=sorter)


# ---- #
Expand Down
7 changes: 4 additions & 3 deletions pandas/core/apply.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,9 +377,10 @@ def wrap_results_for_axis(
else:
raise

if not isinstance(results[0], ABCSeries):
if len(result.index) == len(self.res_columns):
result.index = self.res_columns
if not isinstance(results[0], ABCSeries) and len(result.index) == len(
self.res_columns
):
result.index = self.res_columns

if len(result.columns) == len(res_index):
result.columns = res_index
Expand Down
3 changes: 1 addition & 2 deletions pandas/core/array_algos/replace.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,7 @@ def _check_comparison_types(
if is_scalar(result) and isinstance(a, np.ndarray):
type_names = [type(a).__name__, type(b).__name__]

if isinstance(a, np.ndarray):
type_names[0] = f"ndarray(dtype={a.dtype})"
type_names[0] = f"ndarray(dtype={a.dtype})"

raise TypeError(
f"Cannot compare types {repr(type_names[0])} and {repr(type_names[1])}"
Expand Down
17 changes: 7 additions & 10 deletions pandas/core/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,10 +284,9 @@ def _try_aggregate_string_function(self, arg: str, *args, **kwargs):
return f

f = getattr(np, arg, None)
if f is not None:
if hasattr(self, "__array__"):
# in particular exclude Window
return f(self, *args, **kwargs)
if f is not None and hasattr(self, "__array__"):
# in particular exclude Window
return f(self, *args, **kwargs)

raise AttributeError(
f"'{arg}' is not a valid function for '{type(self).__name__}' object"
Expand Down Expand Up @@ -570,8 +569,8 @@ def to_numpy(self, dtype=None, copy=False, na_value=lib.no_default, **kwargs):
# TODO(GH-24345): Avoid potential double copy
if copy or na_value is not lib.no_default:
result = result.copy()
if na_value is not lib.no_default:
result[self.isna()] = na_value
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why? its small but we potentially avoid a duplicate check

if na_value is not lib.no_default:
result[self.isna()] = na_value
return result

@property
Expand Down Expand Up @@ -974,15 +973,14 @@ def value_counts(
NaN 1
dtype: int64
"""
result = value_counts(
return value_counts(
self,
sort=sort,
ascending=ascending,
normalize=normalize,
bins=bins,
dropna=dropna,
)
return result

def unique(self):
values = self._values
Expand Down Expand Up @@ -1244,8 +1242,7 @@ def searchsorted(self, value, side="left", sorter=None) -> np.ndarray:

def drop_duplicates(self, keep="first"):
duplicated = self.duplicated(keep=keep)
result = self[np.logical_not(duplicated)]
return result
return self[np.logical_not(duplicated)]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

id prefer ~duplicated over logical_not


def duplicated(self, keep="first"):
return duplicated(self._values, keep=keep)
4 changes: 3 additions & 1 deletion pandas/core/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,10 @@ def is_bool_indexer(key: Any) -> bool:
key = np.asarray(key)

if not lib.is_bool_array(key):
na_msg = "Cannot mask with non-boolean array containing NA / NaN values"
if isna(key).any():
na_msg = (
"Cannot mask with non-boolean array containing NA / NaN values"
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this an improvement?

raise ValueError(na_msg)
return False
return True
Expand Down
21 changes: 13 additions & 8 deletions pandas/core/construction.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,8 +347,7 @@ def array(
elif is_timedelta64_ns_dtype(dtype):
return TimedeltaArray._from_sequence(data, dtype=dtype, copy=copy)

result = PandasArray._from_sequence(data, dtype=dtype, copy=copy)
return result
return PandasArray._from_sequence(data, dtype=dtype, copy=copy)


def extract_array(obj: AnyArrayLike, extract_numpy: bool = False) -> ArrayLike:
Expand Down Expand Up @@ -551,9 +550,13 @@ def _try_cast(arr, dtype: Optional[DtypeObj], copy: bool, raise_cast_failure: bo
Otherwise an object array is returned.
"""
# perf shortcut as this is the most common case
if isinstance(arr, np.ndarray):
if maybe_castable(arr) and not copy and dtype is None:
return arr
if (
isinstance(arr, np.ndarray)
and maybe_castable(arr)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should change maybe_castable to take arr.dtype directly

and not copy
and dtype is None
):
return arr

if isinstance(dtype, ExtensionDtype) and (dtype.kind != "M" or is_sparse(dtype)):
# create an extension array from its dtype
Expand All @@ -575,9 +578,11 @@ def _try_cast(arr, dtype: Optional[DtypeObj], copy: bool, raise_cast_failure: bo

# Take care in creating object arrays (but iterators are not
# supported):
if is_object_dtype(dtype) and (
is_list_like(subarr)
and not (is_iterator(subarr) or isinstance(subarr, np.ndarray))
if (
is_object_dtype(dtype)
and is_list_like(subarr)
and not is_iterator(subarr)
and not isinstance(subarr, np.ndarray)
):
subarr = construct_1d_object_array_from_listlike(subarr)
elif not is_extension_array_dtype(subarr):
Expand Down
8 changes: 4 additions & 4 deletions pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,7 @@ def _repr_fits_horizontal_(self, ignore_width: bool = False) -> bool:
# and to_string on entire frame may be expensive
d = self

if not (max_rows is None): # unlimited rows
if max_rows is not None: # unlimited rows
# min of two, where one may be None
d = d.iloc[: min(max_rows, len(d))]
else:
Expand Down Expand Up @@ -1934,10 +1934,10 @@ def to_records(
np.asarray(self.iloc[:, i]) for i in range(len(self.columns))
]

count = 0
index_names = list(self.index.names)

if isinstance(self.index, MultiIndex):
count = 0
for i, n in enumerate(index_names):
if n is None:
index_names[i] = f"level_{count}"
Expand Down Expand Up @@ -3179,7 +3179,7 @@ def _set_value(self, index, col, value, takeable: bool = False):
takeable : interpret the index/col as indexers, default False
"""
try:
if takeable is True:
if takeable:
series = self._ixs(col, axis=1)
series._set_value(index, value, takeable=True)
return
Expand Down Expand Up @@ -4913,7 +4913,7 @@ class max type

multi_col = isinstance(self.columns, MultiIndex)
for i, (lev, lab) in reversed(list(enumerate(to_insert))):
if not (level is None or i in level):
if level is not None and i not in level:
continue
name = names[i]
if multi_col:
Expand Down
Loading