remove special handling of pypy offsets since modern pypy gets it right (#717)

Author: asottile

.github/workflows/test.yml

@@ -12,19 +12,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11.0-beta - 3.11.999", "pypy-3.7"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11.0-beta - 3.11.999", "pypy-3.9"]
         os: [ubuntu-latest]
         # Include minimum py3 + maximum py3 + pypy3 on Windows
         include:
           - { os: "windows-latest" , python-version: "3.6" }
           - { os: "windows-latest" , python-version: "3.10" }
-          - { os: "windows-latest" , python-version: "pypy-3.7" }
+          - { os: "windows-latest" , python-version: "pypy-3.9" }
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

pyflakes/api.py

@@ -37,25 +37,8 @@ def check(codeString, filename, reporter=None):
     # First, compile into an AST and handle syntax errors.
     try:
         tree = ast.parse(codeString, filename=filename)
-    except SyntaxError:
-        value = sys.exc_info()[1]
-        msg = value.args[0]
-
-        (lineno, offset, text) = value.lineno, value.offset, value.text
-
-        if checker.PYPY:
-            if text is None:
-                lines = codeString.splitlines()
-                if len(lines) >= lineno:
-                    text = lines[lineno - 1]
-                    if isinstance(text, bytes):
-                        try:
-                            text = text.decode('ascii')
-                        except UnicodeDecodeError:
-                            text = None
-            offset -= 1
-
-        reporter.syntaxError(filename, msg, lineno, offset, text)
+    except SyntaxError as e:
+        reporter.syntaxError(filename, e.args[0], e.lineno, e.offset, e.text)
         return 1
     except Exception:
         reporter.unexpectedError(filename, 'problem decoding source')
@@ -83,9 +66,8 @@ def checkPath(filename, reporter=None):
     try:
         with open(filename, 'rb') as f:
             codestr = f.read()
-    except OSError:
-        msg = sys.exc_info()[1]
-        reporter.unexpectedError(filename, msg.args[1])
+    except OSError as e:
+        reporter.unexpectedError(filename, e.args[1])
         return 1
     return check(codestr, filename, reporter)
 

pyflakes/checker.py

@@ -1393,10 +1393,7 @@ def handleDoctests(self, node):
         for example in examples:
             try:
                 tree = ast.parse(example.source, "<doctest>")
-            except SyntaxError:
-                e = sys.exc_info()[1]
-                if PYPY:
-                    e.offset += 1
+            except SyntaxError as e:
                 position = (node_lineno + example.lineno + e.lineno,
                             example.indent + 4 + (e.offset or 0))
                 self.report(messages.DoctestSyntaxError, node, position)

pyflakes/test/test_api.py

@@ -401,8 +401,7 @@ def evaluate(source):
             exec(source)
         try:
             evaluate(source)
-        except SyntaxError:
-            e = sys.exc_info()[1]
+        except SyntaxError as e:
             if not PYPY and sys.version_info < (3, 10):
                 self.assertTrue(e.text.count('\n') > 1)
         else:
@@ -416,7 +415,7 @@ def evaluate(source):
             else:
                 message = 'invalid syntax'
 
-            if sys.version_info >= (3, 10):
+            if PYPY or sys.version_info >= (3, 10):
                 column = 12
             elif sys.version_info >= (3, 8):
                 column = 8
@@ -443,9 +442,7 @@ def test_eofSyntaxError(self):
             else:
                 msg = 'unexpected EOF while parsing'
 
-            if PYPY:
-                column = 7
-            elif sys.version_info >= (3, 10):
+            if PYPY or sys.version_info >= (3, 10):
                 column = 8
             else:
                 column = 9
@@ -463,16 +460,13 @@ def test_eofSyntaxErrorWithTab(self):
         syntax error reflects the cause for the syntax error.
         """
         with self.makeTempFile("if True:\n\tfoo =") as sourcePath:
-            column = 6 if PYPY else 7
-            last_line = '\t    ^' if PYPY else '\t     ^'
-
             self.assertHasErrors(
                 sourcePath,
-                ["""\
-{}:2:{}: invalid syntax
+                [f"""\
+{sourcePath}:2:7: invalid syntax
 \tfoo =
-{}
-""".format(sourcePath, column, last_line)])
+\t     ^
+"""])
 
     def test_nonDefaultFollowsDefaultSyntaxError(self):
         """
@@ -485,8 +479,10 @@ def foo(bar=baz, bax):
     pass
 """
         with self.makeTempFile(source) as sourcePath:
-            if PYPY:
-                column = 7
+            if PYPY and sys.version_info >= (3, 9):
+                column = 18
+            elif PYPY:
+                column = 8
             elif sys.version_info >= (3, 10):
                 column = 18
             elif sys.version_info >= (3, 9):
@@ -514,11 +510,9 @@ def test_nonKeywordAfterKeywordSyntaxError(self):
 foo(bar=baz, bax)
 """
         with self.makeTempFile(source) as sourcePath:
-            if PYPY:
-                column = 12
-            elif sys.version_info >= (3, 9):
+            if sys.version_info >= (3, 9):
                 column = 17
-            elif sys.version_info >= (3, 8):
+            elif not PYPY and sys.version_info >= (3, 8):
                 column = 14
             else:
                 column = 13
@@ -541,8 +535,10 @@ def test_invalidEscape(self):
         # ValueError: invalid \x escape
         with self.makeTempFile(r"foo = '\xyz'") as sourcePath:
             position_end = 1
-            if PYPY:
-                column = 5
+            if PYPY and sys.version_info >= (3, 9):
+                column = 7
+            elif PYPY:
+                column = 6
             elif sys.version_info >= (3, 9):
                 column = 13
             else:
@@ -671,17 +667,17 @@ def test_stdinReportsErrors(self):
         self.assertEqual(count, 1)
         errlines = err.getvalue().split("\n")[:-1]
 
-        if PYPY:
+        if sys.version_info >= (3, 9):
             expected_error = [
-                "<stdin>:1:3: Generator expression must be parenthesized if not sole argument",  # noqa: E501
+                "<stdin>:1:5: Generator expression must be parenthesized",
                 "max(1 for i in range(10), key=lambda x: x+1)",
-                "  ^",
+                "    ^",
             ]
-        elif sys.version_info >= (3, 9):
+        elif PYPY:
             expected_error = [
-                "<stdin>:1:5: Generator expression must be parenthesized",
+                "<stdin>:1:4: Generator expression must be parenthesized if not sole argument",  # noqa: E501
                 "max(1 for i in range(10), key=lambda x: x+1)",
-                "    ^",
+                "   ^",
             ]
         elif sys.version_info >= (3, 8):
             expected_error = [
@@ -784,8 +780,8 @@ def test_errors_syntax(self):
         with open(self.tempfilepath, 'wb') as fd:
             fd.write(b"import")
         d = self.runPyflakes([self.tempfilepath])
-        error_msg = '{0}:1:{2}: invalid syntax{1}import{1}     {3}^{1}'.format(
-            self.tempfilepath, os.linesep, 6 if PYPY else 7, '' if PYPY else ' ')
+        error_msg = '{0}:1:7: invalid syntax{1}import{1}      ^{1}'.format(
+            self.tempfilepath, os.linesep)
         self.assertEqual(d, ('', error_msg, 1))
 
     def test_readFromStdin(self):

pyflakes/test/test_doctests.py

@@ -323,9 +323,7 @@ def doctest_stuff():
             m.DoctestSyntaxError).messages
         exc = exceptions[0]
         self.assertEqual(exc.lineno, 4)
-        if PYPY:
-            self.assertEqual(exc.col, 27)
-        elif sys.version_info >= (3, 8):
+        if not PYPY and sys.version_info >= (3, 8):
             self.assertEqual(exc.col, 18)
         else:
             self.assertEqual(exc.col, 26)
@@ -336,14 +334,12 @@ def doctest_stuff():
         exc = exceptions[1]
         self.assertEqual(exc.lineno, 5)
         if PYPY:
-            self.assertEqual(exc.col, 14)
+            self.assertEqual(exc.col, 13)
         else:
             self.assertEqual(exc.col, 16)
         exc = exceptions[2]
         self.assertEqual(exc.lineno, 6)
-        if PYPY:
-            self.assertEqual(exc.col, 14)
-        elif sys.version_info >= (3, 8):
+        if PYPY or sys.version_info >= (3, 8):
             self.assertEqual(exc.col, 13)
         else:
             self.assertEqual(exc.col, 18)
@@ -357,9 +353,7 @@ def doctest_stuff():
             """
         ''', m.DoctestSyntaxError).messages[0]
         self.assertEqual(exc.lineno, 5)
-        if PYPY:
-            self.assertEqual(exc.col, 14)
-        elif sys.version_info >= (3, 8):
+        if PYPY or sys.version_info >= (3, 8):
             self.assertEqual(exc.col, 13)
         else:
             self.assertEqual(exc.col, 16)
@@ -378,10 +372,7 @@ def doctest_stuff(arg1,
             m.DoctestSyntaxError,
             m.UndefinedName).messages
         self.assertEqual(exc1.lineno, 6)
-        if PYPY:
-            self.assertEqual(exc1.col, 20)
-        else:
-            self.assertEqual(exc1.col, 19)
+        self.assertEqual(exc1.col, 19)
         self.assertEqual(exc2.lineno, 7)
         self.assertEqual(exc2.col, 12)