GH 49633: special characters

Author: aram-cedarwood

pandas/core/computation/parsing.py

@@ -41,6 +41,16 @@ def create_valid_python_identifier(name: str) -> str:
     if name.isidentifier() and not iskeyword(name):
         return name
 
+    # Escape characters that fall outside the ASCII range (U+0001..U+007F).
+    # GH 49633
+    c_escaped_gen = (
+        "".join(chr(b) for b in c.encode("ascii", "backslashreplace")) for c in name
+    )
+    name = "".join(
+        c_escaped.replace("\\", "_UNICODE_" if c != c_escaped else "_BACKSLASH_")
+        for c, c_escaped in zip(name, c_escaped_gen)
+    )
+
     # Create a dict with the special characters and their replacement string.
     # EXACT_TOKEN_TYPES contains these special characters
     # token.tok_name contains a readable description of the replacement string.

pandas/core/frame.py

@@ -4556,12 +4556,8 @@ def query(self, expr: str, *, inplace: bool = False, **kwargs) -> DataFrame | No
         quoted string are replaced by strings that are allowed as a Python identifier.
         These characters include all operators in Python, the space character, the
         question mark, the exclamation mark, the dollar sign, and the euro sign.
-        For other characters that fall outside the ASCII range (U+0001..U+007F)
-        and those that are not further specified in PEP 3131,
-        the query parser will raise an error.
-        This excludes whitespace different than the space character,
-        but also the hashtag (as it is used for comments) and the backtick
-        itself (backtick can also not be escaped).
+
+        A backtick can be escaped by double backticks.
 
         See also the `Python documentation about lexical analysis
         <https://docs.python.org/3/reference/lexical_analysis.html>`__

pandas/tests/frame/test_query_eval.py

@@ -1246,6 +1246,8 @@ def df(self):
                 "it's": [6, 3, 1],
                 "that's": [9, 1, 8],
                 "☺": [8, 7, 6],
+                "xy （z）": [1, 2, 3],
+                "xy （z\\uff09": [4, 5, 6],
                 "foo#bar": [2, 4, 5],
                 1: [5, 7, 9],
             }
@@ -1346,10 +1348,22 @@ def test_quote(self, df):
         expect = df[df["it's"] > df["that's"]]
         tm.assert_frame_equal(res, expect)
 
-    def test_failing_character_outside_range(self, df):
-        msg = r"(Could not convert ).*( to a valid Python identifier.)"
-        with pytest.raises(SyntaxError, match=msg):
-            df.query("`☺` > 4")
+    def test_character_outside_range_smiley(self, df):
+        res = df.query("`☺` > 4")
+        expect = df[df["☺"] > 4]
+        tm.assert_frame_equal(res, expect)
+
+    def test_character_outside_range_2_byte_parens(self, df):
+        # GH 49633
+        res = df.query("`xy （z）` == 2")
+        expect = df[df["xy （z）"] == 2]
+        tm.assert_frame_equal(res, expect)
+
+    def test_character_outside_range_and_actual_backslash(self, df):
+        # GH 49633
+        res = df.query("`xy （z\\uff09` == 2")
+        expect = df[df["xy \uff08z\\uff09"] == 2]
+        tm.assert_frame_equal(res, expect)
 
     def test_hashtag(self, df):
         res = df.query("`foo#bar` > 4")