Skip to content

PERF: Use generator expression for Blocks.replace_list #50778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Feb 28, 2023
30 changes: 30 additions & 0 deletions asv_bench/benchmarks/series_methods.py
Original file line number Diff line number Diff line change
Expand Up @@ -401,4 +401,34 @@ def time_to_numpy_copy(self):
self.ser.to_numpy(copy=True)


class Replace:

param_names = ["num_to_replace"]
params = [100, 1000]

def setup(self, num_to_replace):
N = 1_000_000
self.arr = np.random.randn(N)
self.arr1 = self.arr.copy()
np.random.shuffle(self.arr1)
self.ser = Series(self.arr)

self.to_replace_list = np.random.choice(self.arr, num_to_replace)
self.values_list = np.random.choice(self.arr1, num_to_replace)

self.replace_dict = dict(zip(self.to_replace_list, self.values_list))

def time_replace_dict(self, num_to_replace):
self.ser.replace(self.replace_dict)

def peakmem_replace_dict(self, num_to_replace):
self.ser.replace(self.replace_dict)

def time_replace_list(self, num_to_replace):
self.ser.replace(self.to_replace_list, self.values_list)

def peakmem_replace_list(self, num_to_replace):
self.ser.replace(self.to_replace_list, self.values_list)


from .pandas_vb_common import setup # noqa: F401 isort:skip
23 changes: 13 additions & 10 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -663,22 +663,25 @@ def replace_list(
if is_string_dtype(values.dtype):
# Calculate the mask once, prior to the call of comp
# in order to avoid repeating the same computations
mask = ~isna(values)
masks = [
compare_or_regex_search(values, s[0], regex=regex, mask=mask)
na_mask = ~isna(values)
masks = (
extract_bool_array(
compare_or_regex_search(values, s[0], regex=regex, mask=na_mask)
)
for s in pairs
]
)
else:
# GH#38086 faster if we know we dont need to check for regex
masks = [missing.mask_missing(values, s[0]) for s in pairs]
masks = (
extract_bool_array(missing.mask_missing(values, s[0])) for s in pairs
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there a chance of an API change here (not necessarily a bad one) with something like

src_list = [1, 2, 3]
dest_list = [2, 3, 4]
values = np.array([1, 2, 3])

IIUC in the status quo we end up with [2, 3, 4] but with this we'd end up with all-4s?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, looks like this would be invalid for inplace=True. Good catch!

I'll also add a test case, since it looks like this isn't caught by CI.

)

# error: Argument 1 to "extract_bool_array" has incompatible type
# "Union[ExtensionArray, ndarray, bool]"; expected "Union[ExtensionArray,
# ndarray]"
masks = [extract_bool_array(x) for x in masks] # type: ignore[arg-type]

# masks = [extract_bool_array(x) for x in masks] # type: ignore[arg-type]
rb = [self if inplace else self.copy()]
for i, (src, dest) in enumerate(pairs):
for i, ((src, dest), mask) in enumerate(zip(pairs, masks)):
convert = i == src_len # only convert once at the end
new_rb: list[Block] = []

Expand All @@ -687,9 +690,9 @@ def replace_list(
# where to index into the mask
for blk_num, blk in enumerate(rb):
if len(rb) == 1:
m = masks[i]
m = mask
else:
mib = masks[i]
mib = mask
assert not isinstance(mib, bool)
m = mib[blk_num : blk_num + 1]

Expand Down