Skip to content

Commit 4295b28

Browse files
committed
ENH: str.replace accepts a compiled expression
- .str.replace now accepts a compiled regular expression for `pat`. - Signature for .str.replace changed, but remains backwards compatible. See #15446
1 parent 211ecd5 commit 4295b28

File tree

4 files changed

+129
-16
lines changed

4 files changed

+129
-16
lines changed

doc/source/text.rst

+19
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
77
import numpy as np
88
import pandas as pd
9+
import re
910
randn = np.random.randn
1011
np.set_printoptions(precision=4, suppress=True)
1112
from pandas.compat import lrange
@@ -164,6 +165,24 @@ positional argument (a regex object) and return a string.
164165
repl = lambda m: m.group('two').swapcase()
165166
pd.Series(['Foo Bar Baz', np.nan]).str.replace(pat, repl)
166167
168+
The ``replace`` method also accepts a compiled regular expression object
169+
from :func:`re.compile` as a pattern. All flags should be included in the
170+
compiled regular expression object.
171+
172+
.. versionadded:: 0.20.0
173+
174+
.. ipython:: python
175+
176+
regex_pat = re.compile(r'^.a|dog', flags=re.IGNORECASE)
177+
s3.str.replace(regex_pat, 'XX-XX ')
178+
179+
Including a ``flags`` argument when calling ``replace`` with a compiled regular expression object will raise a ``ValueError``.
180+
181+
.. ipython::
182+
183+
@verbatim
184+
In [1]: s3.str.replace(regex_pat, 'XX-XX ', flags=re.IGNORECASE)
185+
ValueError: case and flags must be None when pat is a compiled regex
167186

168187
Indexing with ``.str``
169188
----------------------

doc/source/whatsnew/v0.20.0.txt

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ New features
2929
- Integration with the ``feather-format``, including a new top-level ``pd.read_feather()`` and ``DataFrame.to_feather()`` method, see :ref:`here <io.feather>`.
3030
- ``.str.replace`` now accepts a callable, as replacement, which is passed to ``re.sub`` (:issue:`15055`)
3131
- ``FrozenList`` has gained the ``.difference()`` setop method (:issue:`15475`)
32+
- ``.str.replace`` now accepts a compiled regular expression as a pattern (:issue:`15446`)
3233

3334

3435

pandas/core/strings.py

+53-16
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
is_string_like,
1010
is_list_like,
1111
is_scalar,
12-
is_integer)
12+
is_integer,
13+
is_re)
1314
from pandas.core.common import _values_from_object
1415

1516
from pandas.core.algorithms import take_1d
@@ -303,16 +304,20 @@ def str_endswith(arr, pat, na=np.nan):
303304
return _na_map(f, arr, na, dtype=bool)
304305

305306

306-
def str_replace(arr, pat, repl, n=-1, case=True, flags=0):
307+
def str_replace(arr, pat, repl, n=-1, case=None, flags=None):
307308
"""
308309
Replace occurrences of pattern/regex in the Series/Index with
309310
some other string. Equivalent to :meth:`str.replace` or
310311
:func:`re.sub`.
311312
312313
Parameters
313314
----------
314-
pat : string
315-
Character sequence or regular expression
315+
pat : string or compiled regex
316+
String can be a character sequence or regular expression.
317+
318+
.. versionadded:: 0.20.0
319+
`pat` also accepts a compiled regex.
320+
316321
repl : string or callable
317322
Replacement string or a callable. The callable is passed the regex
318323
match object and must return a replacement string to be used.
@@ -323,15 +328,23 @@ def str_replace(arr, pat, repl, n=-1, case=True, flags=0):
323328
324329
n : int, default -1 (all)
325330
Number of replacements to make from start
326-
case : boolean, default True
327-
If True, case sensitive
328-
flags : int, default 0 (no flags)
329-
re module flags, e.g. re.IGNORECASE
331+
case : boolean, optional
332+
- if False, case insensitive
333+
- Must be None if `pat` is a compiled regex
334+
flags : int, optional
335+
- re module flags, e.g. re.IGNORECASE
336+
- Must be None if `pat` is a compiled regex
330337
331338
Returns
332339
-------
333340
replaced : Series/Index of objects
334341
342+
Notes
343+
-----
344+
When `pat` is a compiled regex, all flags should be included in the
345+
compiled regex. Use of `case` or `flags` with a compiled regex will
346+
raise an error.
347+
335348
Examples
336349
--------
337350
When `repl` is a string, every `pat` is replaced as with
@@ -372,21 +385,45 @@ def str_replace(arr, pat, repl, n=-1, case=True, flags=0):
372385
0 tWO
373386
1 bAR
374387
dtype: object
388+
389+
Using a compiled regex with flags
390+
391+
>>> regex_pat = re.compile(r'FUZ', flags=re.IGNORECASE)
392+
>>> pd.Series(['foo', 'fuz', np.nan]).str.replace(regex_pat, 'bar')
393+
0 foo
394+
1 bar
395+
2 NaN
396+
dtype: object
375397
"""
376398

377399
# Check whether repl is valid (GH 13438, GH 15055)
378400
if not (is_string_like(repl) or callable(repl)):
379401
raise TypeError("repl must be a string or callable")
380-
use_re = not case or len(pat) > 1 or flags or callable(repl)
381402

382-
if use_re:
383-
if not case:
403+
is_compiled_re = is_re(pat)
404+
if is_compiled_re:
405+
if (case is not None) or (flags is not None):
406+
raise ValueError("case and flags must be None"
407+
" when pat is a compiled regex")
408+
flags = 0
409+
else:
410+
# not a compiled regex
411+
# set default case/flags
412+
if case is None:
413+
case = True
414+
if flags is None:
415+
flags = 0
416+
417+
# add case flag, if provided
418+
if case is False:
384419
flags |= re.IGNORECASE
385-
regex = re.compile(pat, flags=flags)
386-
n = n if n >= 0 else 0
387420

388-
def f(x):
389-
return regex.sub(repl, x, count=n)
421+
use_re = is_compiled_re or len(pat) > 1 or flags or callable(repl)
422+
423+
if use_re:
424+
n = n if n >= 0 else 0
425+
f = lambda x: re.sub(pattern=pat, repl=repl, string=x,
426+
count=n, flags=flags)
390427
else:
391428
f = lambda x: x.replace(pat, repl, n)
392429

@@ -1558,7 +1595,7 @@ def match(self, pat, case=True, flags=0, na=np.nan, as_indexer=False):
15581595
return self._wrap_result(result)
15591596

15601597
@copy(str_replace)
1561-
def replace(self, pat, repl, n=-1, case=True, flags=0):
1598+
def replace(self, pat, repl, n=-1, case=None, flags=None):
15621599
result = str_replace(self._data, pat, repl, n=n, case=case,
15631600
flags=flags)
15641601
return self._wrap_result(result)

pandas/tests/test_strings.py

+56
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,62 @@ def test_replace_callable(self):
469469
exp = Series(['bAR', NA])
470470
tm.assert_series_equal(result, exp)
471471

472+
def test_replace_compiled_regex(self):
473+
# GH 15446
474+
values = Series(['fooBAD__barBAD', NA])
475+
476+
# test with compiled regex
477+
pat = re.compile(r'BAD[_]*')
478+
result = values.str.replace(pat, '')
479+
exp = Series(['foobar', NA])
480+
tm.assert_series_equal(result, exp)
481+
482+
# mixed
483+
mixed = Series(['aBAD', NA, 'bBAD', True, datetime.today(), 'fooBAD',
484+
None, 1, 2.])
485+
486+
rs = Series(mixed).str.replace(pat, '')
487+
xp = Series(['a', NA, 'b', NA, NA, 'foo', NA, NA, NA])
488+
tm.assertIsInstance(rs, Series)
489+
tm.assert_almost_equal(rs, xp)
490+
491+
# unicode
492+
values = Series([u('fooBAD__barBAD'), NA])
493+
494+
result = values.str.replace(pat, '')
495+
exp = Series([u('foobar'), NA])
496+
tm.assert_series_equal(result, exp)
497+
498+
result = values.str.replace(pat, '', n=1)
499+
exp = Series([u('foobarBAD'), NA])
500+
tm.assert_series_equal(result, exp)
501+
502+
# flags + unicode
503+
values = Series([b"abcd,\xc3\xa0".decode("utf-8")])
504+
exp = Series([b"abcd, \xc3\xa0".decode("utf-8")])
505+
pat = re.compile(r"(?<=\w),(?=\w)", flags=re.UNICODE)
506+
result = values.str.replace(pat, ", ")
507+
tm.assert_series_equal(result, exp)
508+
509+
# case and flags provided to str.replace will have no effect
510+
# and will produce warnings
511+
values = Series(['fooBAD__barBAD__bad', NA])
512+
pat = re.compile(r'BAD[_]*')
513+
514+
with tm.assertRaisesRegexp(ValueError, "case and flags must be None"):
515+
result = values.str.replace(pat, '', flags=re.IGNORECASE)
516+
517+
with tm.assertRaisesRegexp(ValueError, "case and flags must be None"):
518+
result = values.str.replace(pat, '', case=False)
519+
520+
# test with callable
521+
values = Series(['fooBAD__barBAD', NA])
522+
repl = lambda m: m.group(0).swapcase()
523+
pat = re.compile('[a-z][A-Z]{2}')
524+
result = values.str.replace(pat, repl, n=2)
525+
exp = Series(['foObaD__baRbaD', NA])
526+
tm.assert_series_equal(result, exp)
527+
472528
def test_repeat(self):
473529
values = Series(['a', 'b', NA, 'c', NA, 'd'])
474530

0 commit comments

Comments
 (0)