[pyupgrade] Handle end-of-line comments for quoted-annotation (UP037) (#15824)

This PR uses the tokens of the parsed annotation available in the
`Checker`, instead of re-lexing (using `SimpleTokenizer`) the
annotation. This avoids some limitations of the `SimpleTokenizer`, such
as not being able to handle number and string literals.

Closes #15816 .

Author: dylwil3

crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_0.py

@@ -106,3 +106,17 @@ def foo(*, inplace):
 x = cast("str", x)
 
 X = List["MyClass"]
+
+# Handle end of line comment in string annotation
+# See https://github.com/astral-sh/ruff/issues/15816
+def f() -> "Literal[0]#":
+    return 0
+
+def g(x: "Literal['abc']#") -> None:
+    return
+
+def f() -> """Literal[0]
+    #
+    
+    """:
+    return 0

crates/ruff_linter/src/rules/pyupgrade/rules/quoted_annotation.rs

@@ -1,11 +1,11 @@
+use ruff_python_parser::TokenKind;
 use ruff_text_size::{TextLen, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;
 use ruff_diagnostics::{AlwaysFixableViolation, Diagnostic, Edit, Fix};
 use ruff_macros::{derive_message_formats, ViolationMetadata};
 use ruff_python_ast::Stmt;
 use ruff_python_semantic::SemanticModel;
-use ruff_python_trivia::{SimpleToken, SimpleTokenKind, SimpleTokenizer};
 use ruff_source_file::LineRanges;
 
 /// ## What it does
@@ -87,14 +87,12 @@ pub(crate) fn quoted_annotation(checker: &mut Checker, annotation: &str, range:
     let placeholder_range = TextRange::up_to(annotation.text_len());
     let spans_multiple_lines = annotation.contains_line_break(placeholder_range);
 
-    let tokenizer = SimpleTokenizer::new(annotation, placeholder_range);
-    let last_token_is_comment = matches!(
-        tokenizer.last(),
-        Some(SimpleToken {
-            kind: SimpleTokenKind::Comment,
-            ..
-        })
-    );
+    let last_token_is_comment = checker
+        .tokens()
+        // The actual last token will always be a logical newline,
+        // so we check the second to last
+        .get(checker.tokens().len().saturating_sub(2))
+        .is_some_and(|tok| tok.kind() == TokenKind::Comment);
 
     let new_content = match (spans_multiple_lines, last_token_is_comment) {
         (_, false) if in_parameter_annotation(range.start(), checker.semantic()) => {

crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP037_0.py.snap

@@ -554,4 +554,72 @@ UP037_0.py:67:45: UP037 [*] Remove quotes from type annotation
    67 |+x: NamedTuple(typename="X", fields=[("foo", int)])
 68 68 | 
 69 69 | X: MyCallable("X")
-70 70 |
+70 70 | 
+
+UP037_0.py:112:12: UP037 [*] Remove quotes from type annotation
+    |
+110 | # Handle end of line comment in string annotation
+111 | # See https://github.com/astral-sh/ruff/issues/15816
+112 | def f() -> "Literal[0]#":
+    |            ^^^^^^^^^^^^^ UP037
+113 |     return 0
+    |
+    = help: Remove quotes
+
+ℹ Safe fix
+109 109 | 
+110 110 | # Handle end of line comment in string annotation
+111 111 | # See https://github.com/astral-sh/ruff/issues/15816
+112     |-def f() -> "Literal[0]#":
+    112 |+def f() -> (Literal[0]#
+    113 |+):
+113 114 |     return 0
+114 115 | 
+115 116 | def g(x: "Literal['abc']#") -> None:
+
+UP037_0.py:115:10: UP037 [*] Remove quotes from type annotation
+    |
+113 |     return 0
+114 |
+115 | def g(x: "Literal['abc']#") -> None:
+    |          ^^^^^^^^^^^^^^^^^ UP037
+116 |     return
+    |
+    = help: Remove quotes
+
+ℹ Safe fix
+112 112 | def f() -> "Literal[0]#":
+113 113 |     return 0
+114 114 | 
+115     |-def g(x: "Literal['abc']#") -> None:
+    115 |+def g(x: (Literal['abc']#
+    116 |+)) -> None:
+116 117 |     return
+117 118 | 
+118 119 | def f() -> """Literal[0]
+
+UP037_0.py:118:12: UP037 [*] Remove quotes from type annotation
+    |
+116 |       return
+117 |
+118 |   def f() -> """Literal[0]
+    |  ____________^
+119 | |     #
+120 | |     
+121 | |     """:
+    | |_______^ UP037
+122 |       return 0
+    |
+    = help: Remove quotes
+
+ℹ Safe fix
+115 115 | def g(x: "Literal['abc']#") -> None:
+116 116 |     return
+117 117 | 
+118     |-def f() -> """Literal[0]
+    118 |+def f() -> (Literal[0]
+119 119 |     #
+120 120 |     
+121     |-    """:
+    121 |+    ):
+122 122 |     return 0