Rollup merge of #136829 - GuillaumeGomez:move-line-numbers-into-code, r=notriddle

[rustdoc] Move line numbers into the `<code>` directly

Fixes #84242.

This is the first for adding support for #127334 and also for another feature I'm working on.

A side-effect of this change is that it also fixes source code pages display in lynx since they're not directly in the source code.

To allow having code wrapping, the grid approach doesn't work as the line numbers are in their own container, so we need to move them into the code. Now with this, it becomes much simpler to do what we want (with CSS mostly). One downside: the highlighting became more complex and slow as we need to generate some extra HTML tags directly into the highlighting process. However that also allows to not have a huge HTML size increase.

You can test the result [here](https://rustdoc.crud.net/imperio/move-line-numbers-into-code/scrape_examples/fn.test_many.html) and [here](https://rustdoc.crud.net/imperio/move-line-numbers-into-code/src/scrape_examples/lib.rs.html#10).

The appearance should have close to no changes.

r? ``@notriddle``

Author: GuillaumeGomez

Diff for: src/librustdoc/html/highlight.rs

@@ -54,7 +54,7 @@ pub(crate) fn render_example_with_highlighting(
     extra_classes: &[String],
 ) {
     write_header(out, "rust-example-rendered", None, tooltip, extra_classes);
-    write_code(out, src, None, None);
+    write_code(out, src, None, None, None);
     write_footer(out, playground_button);
 }
 
@@ -150,6 +150,7 @@ struct TokenHandler<'a, 'tcx, F: Write> {
     /// used to generate links.
     pending_elems: Vec<(&'a str, Option<Class>)>,
     href_context: Option<HrefContext<'a, 'tcx>>,
+    write_line_number: fn(&mut F, u32, &'static str),
 }
 
 impl<F: Write> TokenHandler<'_, '_, F> {
@@ -182,7 +183,14 @@ impl<F: Write> TokenHandler<'_, '_, F> {
             && can_merge(current_class, Some(*parent_class), "")
         {
             for (text, class) in self.pending_elems.iter() {
-                string(self.out, EscapeBodyText(text), *class, &self.href_context, false);
+                string(
+                    self.out,
+                    EscapeBodyText(text),
+                    *class,
+                    &self.href_context,
+                    false,
+                    self.write_line_number,
+                );
             }
         } else {
             // We only want to "open" the tag ourselves if we have more than one pending and if the
@@ -204,6 +212,7 @@ impl<F: Write> TokenHandler<'_, '_, F> {
                     *class,
                     &self.href_context,
                     close_tag.is_none(),
+                    self.write_line_number,
                 );
             }
             if let Some(close_tag) = close_tag {
@@ -213,6 +222,11 @@ impl<F: Write> TokenHandler<'_, '_, F> {
         self.pending_elems.clear();
         true
     }
+
+    #[inline]
+    fn write_line_number(&mut self, line: u32, extra: &'static str) {
+        (self.write_line_number)(&mut self.out, line, extra);
+    }
 }
 
 impl<F: Write> Drop for TokenHandler<'_, '_, F> {
@@ -226,6 +240,43 @@ impl<F: Write> Drop for TokenHandler<'_, '_, F> {
     }
 }
 
+fn write_scraped_line_number(out: &mut impl Write, line: u32, extra: &'static str) {
+    // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#data-nosnippet-attr
+    // Do not show "1 2 3 4 5 ..." in web search results.
+    write!(out, "{extra}<span data-nosnippet>{line}</span>",).unwrap();
+}
+
+fn write_line_number(out: &mut impl Write, line: u32, extra: &'static str) {
+    // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#data-nosnippet-attr
+    // Do not show "1 2 3 4 5 ..." in web search results.
+    write!(out, "{extra}<a href=#{line} id={line} data-nosnippet>{line}</a>",).unwrap();
+}
+
+fn empty_line_number(out: &mut impl Write, _: u32, extra: &'static str) {
+    out.write_str(extra).unwrap();
+}
+
+#[derive(Clone, Copy)]
+pub(super) struct LineInfo {
+    pub(super) start_line: u32,
+    max_lines: u32,
+    pub(super) is_scraped_example: bool,
+}
+
+impl LineInfo {
+    pub(super) fn new(max_lines: u32) -> Self {
+        Self { start_line: 1, max_lines: max_lines + 1, is_scraped_example: false }
+    }
+
+    pub(super) fn new_scraped(max_lines: u32, start_line: u32) -> Self {
+        Self {
+            start_line: start_line + 1,
+            max_lines: max_lines + start_line + 1,
+            is_scraped_example: true,
+        }
+    }
+}
+
 /// Convert the given `src` source code into HTML by adding classes for highlighting.
 ///
 /// This code is used to render code blocks (in the documentation) as well as the source code pages.
@@ -242,6 +293,7 @@ pub(super) fn write_code(
     src: &str,
     href_context: Option<HrefContext<'_, '_>>,
     decoration_info: Option<&DecorationInfo>,
+    line_info: Option<LineInfo>,
 ) {
     // This replace allows to fix how the code source with DOS backline characters is displayed.
     let src = src.replace("\r\n", "\n");
@@ -252,6 +304,23 @@ pub(super) fn write_code(
         current_class: None,
         pending_elems: Vec::new(),
         href_context,
+        write_line_number: match line_info {
+            Some(line_info) => {
+                if line_info.is_scraped_example {
+                    write_scraped_line_number
+                } else {
+                    write_line_number
+                }
+            }
+            None => empty_line_number,
+        },
+    };
+
+    let (mut line, max_lines) = if let Some(line_info) = line_info {
+        token_handler.write_line_number(line_info.start_line, "");
+        (line_info.start_line, line_info.max_lines)
+    } else {
+        (0, u32::MAX)
     };
 
     Classifier::new(
@@ -282,7 +351,14 @@ pub(super) fn write_code(
                 if need_current_class_update {
                     token_handler.current_class = class.map(Class::dummy);
                 }
-                token_handler.pending_elems.push((text, class));
+                if text == "\n" {
+                    line += 1;
+                    if line < max_lines {
+                        token_handler.pending_elems.push((text, Some(Class::Backline(line))));
+                    }
+                } else {
+                    token_handler.pending_elems.push((text, class));
+                }
             }
             Highlight::EnterSpan { class } => {
                 let mut should_add = true;
@@ -348,6 +424,7 @@ enum Class {
     PreludeVal(Span),
     QuestionMark,
     Decoration(&'static str),
+    Backline(u32),
 }
 
 impl Class {
@@ -396,6 +473,7 @@ impl Class {
             Class::PreludeVal(_) => "prelude-val",
             Class::QuestionMark => "question-mark",
             Class::Decoration(kind) => kind,
+            Class::Backline(_) => "",
         }
     }
 
@@ -419,7 +497,8 @@ impl Class {
             | Self::Bool
             | Self::Lifetime
             | Self::QuestionMark
-            | Self::Decoration(_) => None,
+            | Self::Decoration(_)
+            | Self::Backline(_) => None,
         }
     }
 }
@@ -694,8 +773,13 @@ impl<'src> Classifier<'src> {
     ) {
         let lookahead = self.peek();
         let no_highlight = |sink: &mut dyn FnMut(_)| sink(Highlight::Token { text, class: None });
+        let whitespace = |sink: &mut dyn FnMut(_)| {
+            for part in text.split('\n').intersperse("\n").filter(|s| !s.is_empty()) {
+                sink(Highlight::Token { text: part, class: None });
+            }
+        };
         let class = match token {
-            TokenKind::Whitespace => return no_highlight(sink),
+            TokenKind::Whitespace => return whitespace(sink),
             TokenKind::LineComment { doc_style } | TokenKind::BlockComment { doc_style, .. } => {
                 if doc_style.is_some() {
                     Class::DocComment
@@ -716,7 +800,7 @@ impl<'src> Classifier<'src> {
             // or a reference or pointer type. Unless, of course, it looks like
             // a logical and or a multiplication operator: `&&` or `* `.
             TokenKind::Star => match self.tokens.peek() {
-                Some((TokenKind::Whitespace, _)) => return no_highlight(sink),
+                Some((TokenKind::Whitespace, _)) => return whitespace(sink),
                 Some((TokenKind::Ident, "mut")) => {
                     self.next();
                     sink(Highlight::Token { text: "*mut", class: Some(Class::RefKeyWord) });
@@ -740,7 +824,7 @@ impl<'src> Classifier<'src> {
                     sink(Highlight::Token { text: "&=", class: None });
                     return;
                 }
-                Some((TokenKind::Whitespace, _)) => return no_highlight(sink),
+                Some((TokenKind::Whitespace, _)) => return whitespace(sink),
                 Some((TokenKind::Ident, "mut")) => {
                     self.next();
                     sink(Highlight::Token { text: "&mut", class: Some(Class::RefKeyWord) });
@@ -887,7 +971,9 @@ impl<'src> Classifier<'src> {
         };
         // Anything that didn't return above is the simple case where we the
         // class just spans a single token, so we can use the `string` method.
-        sink(Highlight::Token { text, class: Some(class) });
+        for part in text.split('\n').intersperse("\n").filter(|s| !s.is_empty()) {
+            sink(Highlight::Token { text: part, class: Some(class) });
+        }
     }
 
     fn peek(&mut self) -> Option<TokenKind> {
@@ -939,14 +1025,18 @@ fn exit_span(out: &mut impl Write, closing_tag: &str) {
 /// Note that if `context` is not `None` and that the given `klass` contains a `Span`, the function
 /// will then try to find this `span` in the `span_correspondence_map`. If found, it'll then
 /// generate a link for this element (which corresponds to where its definition is located).
-fn string<T: Display>(
-    out: &mut impl Write,
+fn string<T: Display, W: Write>(
+    out: &mut W,
     text: T,
     klass: Option<Class>,
     href_context: &Option<HrefContext<'_, '_>>,
     open_tag: bool,
+    write_line_number_callback: fn(&mut W, u32, &'static str),
 ) {
-    if let Some(closing_tag) = string_without_closing_tag(out, text, klass, href_context, open_tag)
+    if let Some(Class::Backline(line)) = klass {
+        write_line_number_callback(out, line, "\n");
+    } else if let Some(closing_tag) =
+        string_without_closing_tag(out, text, klass, href_context, open_tag)
     {
         out.write_str(closing_tag).unwrap();
     }

Diff for: src/librustdoc/html/highlight/tests.rs

@@ -23,7 +23,7 @@ fn test_html_highlighting() {
         let src = include_str!("fixtures/sample.rs");
         let html = {
             let mut out = Buffer::new();
-            write_code(&mut out, src, None, None);
+            write_code(&mut out, src, None, None, None);
             format!("{STYLE}<pre><code>{}</code></pre>\n", out.into_inner())
         };
         expect_file!["fixtures/sample.html"].assert_eq(&html);
@@ -37,7 +37,7 @@ fn test_dos_backline() {
     println!(\"foo\");\r\n\
 }\r\n";
         let mut html = Buffer::new();
-        write_code(&mut html, src, None, None);
+        write_code(&mut html, src, None, None, None);
         expect_file!["fixtures/dos_line.html"].assert_eq(&html.into_inner());
     });
 }
@@ -51,7 +51,7 @@ let x = super::b::foo;
 let y = Self::whatever;";
 
         let mut html = Buffer::new();
-        write_code(&mut html, src, None, None);
+        write_code(&mut html, src, None, None, None);
         expect_file!["fixtures/highlight.html"].assert_eq(&html.into_inner());
     });
 }
@@ -61,7 +61,7 @@ fn test_union_highlighting() {
     create_default_session_globals_then(|| {
         let src = include_str!("fixtures/union.rs");
         let mut html = Buffer::new();
-        write_code(&mut html, src, None, None);
+        write_code(&mut html, src, None, None, None);
         expect_file!["fixtures/union.html"].assert_eq(&html.into_inner());
     });
 }
@@ -78,7 +78,7 @@ let a = 4;";
         decorations.insert("example2", vec![(22, 32)]);
 
         let mut html = Buffer::new();
-        write_code(&mut html, src, None, Some(&DecorationInfo(decorations)));
+        write_code(&mut html, src, None, Some(&DecorationInfo(decorations)), None);
         expect_file!["fixtures/decorations.html"].assert_eq(&html.into_inner());
     });
 }

Diff for: src/librustdoc/html/sources.rs

@@ -1,6 +1,5 @@
 use std::cell::RefCell;
 use std::ffi::OsStr;
-use std::ops::RangeInclusive;
 use std::path::{Component, Path, PathBuf};
 use std::{fmt, fs};
 
@@ -303,16 +302,16 @@ pub(crate) struct ScrapedInfo<'a> {
 #[template(path = "scraped_source.html")]
 struct ScrapedSource<'a, Code: std::fmt::Display> {
     info: ScrapedInfo<'a>,
-    lines: RangeInclusive<usize>,
     code_html: Code,
+    max_nb_digits: u32,
 }
 
 #[derive(Template)]
 #[template(path = "source.html")]
 struct Source<Code: std::fmt::Display> {
-    lines: RangeInclusive<usize>,
     code_html: Code,
     file_path: Option<(String, String)>,
+    max_nb_digits: u32,
 }
 
 pub(crate) enum SourceContext<'a> {
@@ -331,6 +330,15 @@ pub(crate) fn print_src(
     decoration_info: &highlight::DecorationInfo,
     source_context: SourceContext<'_>,
 ) {
+    let mut lines = s.lines().count();
+    let line_info = if let SourceContext::Embedded(ref info) = source_context {
+        highlight::LineInfo::new_scraped(lines as u32, info.offset as u32)
+    } else {
+        highlight::LineInfo::new(lines as u32)
+    };
+    if line_info.is_scraped_example {
+        lines += line_info.start_line as usize;
+    }
     let code = fmt::from_fn(move |fmt| {
         let current_href = context
             .href_from_span(clean::Span::new(file_span), false)
@@ -340,13 +348,13 @@ pub(crate) fn print_src(
             s,
             Some(highlight::HrefContext { context, file_span, root_path, current_href }),
             Some(decoration_info),
+            Some(line_info),
         );
         Ok(())
     });
-    let lines = s.lines().count();
+    let max_nb_digits = if lines > 0 { lines.ilog(10) + 1 } else { 1 };
     match source_context {
         SourceContext::Standalone { file_path } => Source {
-            lines: (1..=lines),
             code_html: code,
             file_path: if let Some(file_name) = file_path.file_name()
                 && let Some(file_path) = file_path.parent()
@@ -355,12 +363,14 @@ pub(crate) fn print_src(
             } else {
                 None
             },
+            max_nb_digits,
         }
         .render_into(&mut writer)
         .unwrap(),
         SourceContext::Embedded(info) => {
-            let lines = (1 + info.offset)..=(lines + info.offset);
-            ScrapedSource { info, lines, code_html: code }.render_into(&mut writer).unwrap();
+            ScrapedSource { info, code_html: code, max_nb_digits }
+                .render_into(&mut writer)
+                .unwrap();
         }
     };
 }