WIP: parse frontmatter

Author: fee1-dead

Diff for: compiler/rustc_lexer/src/cursor.rs

@@ -8,17 +8,19 @@ pub struct Cursor<'a> {
     len_remaining: usize,
     /// Iterator over chars. Slightly faster than a &str.
     chars: Chars<'a>,
+    pub(crate) frontmatter_allowed: bool,
     #[cfg(debug_assertions)]
     prev: char,
 }
 
 pub(crate) const EOF_CHAR: char = '\0';
 
 impl<'a> Cursor<'a> {
-    pub fn new(input: &'a str) -> Cursor<'a> {
+    pub fn new(input: &'a str, frontmatter_allowed: bool) -> Cursor<'a> {
         Cursor {
             len_remaining: input.len(),
             chars: input.chars(),
+            frontmatter_allowed,
             #[cfg(debug_assertions)]
             prev: EOF_CHAR,
         }
@@ -95,6 +97,11 @@ impl<'a> Cursor<'a> {
         Some(c)
     }
 
+    /// Moves to a substring by a number of bytes.
+    pub(crate) fn bump_bytes(&mut self, n: usize) {
+        self.chars = self.as_str()[n..].chars();
+    }
+
     /// Eats symbols while predicate returns true or until the end of file is reached.
     pub(crate) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
         // It was tried making optimized version of this for eg. line comments, but

Diff for: compiler/rustc_lexer/src/lib.rs

@@ -68,6 +68,11 @@ pub enum TokenKind {
     /// Any whitespace character sequence.
     Whitespace,
 
+    Frontmatter {
+        has_invalid_preceding_whitespace: bool,
+        invalid_infostring: bool,
+    },
+
     /// An identifier or keyword, e.g. `ident` or `continue`.
     Ident,
 
@@ -280,7 +285,7 @@ pub fn strip_shebang(input: &str) -> Option<usize> {
 #[inline]
 pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
     debug_assert!(!input.is_empty());
-    let mut cursor = Cursor::new(input);
+    let mut cursor = Cursor::new(input, false);
     // Move past the leading `r` or `br`.
     for _ in 0..prefix_len {
         cursor.bump().unwrap();
@@ -290,7 +295,7 @@ pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError>
 
 /// Creates an iterator that produces tokens from the input string.
 pub fn tokenize(input: &str) -> impl Iterator<Item = Token> {
-    let mut cursor = Cursor::new(input);
+    let mut cursor = Cursor::new(input, false);
     std::iter::from_fn(move || {
         let token = cursor.advance_token();
         if token.kind != TokenKind::Eof { Some(token) } else { None }
@@ -361,7 +366,27 @@ impl Cursor<'_> {
             Some(c) => c,
             None => return Token::new(TokenKind::Eof, 0),
         };
+
         let token_kind = match first_char {
+            c if self.frontmatter_allowed && is_whitespace(c) => {
+                let mut last = first_char;
+                while is_whitespace(self.first()) {
+                    let Some(c) = self.bump() else { break; };
+                    last = c;
+                }
+                // invalid frontmatter opening as whitespace preceding it isn't newline.
+                // combine the whitespace and the frontmatter to a single token as we shall
+                // error later.
+                if last != '\n' && self.as_str().starts_with("---") {
+                    self.frontmatter(true)
+                } else {
+                    Whitespace
+                }
+            }
+            _ if self.frontmatter_allowed && self.as_str().starts_with("---") => {
+                // happy path
+                self.frontmatter(false)
+            }
             // Slash, comment or block comment.
             '/' => match self.first() {
                 '/' => self.line_comment(),
@@ -464,11 +489,63 @@ impl Cursor<'_> {
             c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
             _ => Unknown,
         };
+        if self.frontmatter_allowed {
+            self.frontmatter_allowed = matches!(token_kind, Whitespace);
+        }
         let res = Token::new(token_kind, self.pos_within_token());
         self.reset_pos_within_token();
         res
     }
 
+    fn frontmatter(&mut self, has_invalid_preceding_whitespace: bool) -> TokenKind {
+        let pos = self.pos_within_token();
+        self.eat_while(|c| c == '-');
+        let length_opening = self.pos_within_token() - pos;
+
+        // must be ensured by the caller
+        debug_assert!(length_opening >= 3);
+
+        let s = self.as_str();
+        self.eat_identifier();
+        self.eat_while(|ch| ch != '\n' && is_whitespace(ch));
+        let invalid_infostring = self.first() != '\n';
+
+        if let Some(closing) = s.find(&"-".repeat(length_opening as usize)) {
+            self.bump_bytes(closing);
+            // in case like
+            // ---cargo
+            // --- blahblah
+            // or
+            // ---cargo
+            // ----
+            // combine those stuff into this frontmatter token such that it gets detected later.
+            self.eat_until(b'\n');
+        } else {
+            // recovery strategy:
+            // (1) a closing statement with precending whitespace/newline but not having enough characters
+            //     in this case we eat until there and report a mismatch
+            let mut potential_closing = None;
+            let mut rest = s;
+            while let Some(closing) = rest.find("---") {
+                let preceding_chars_start = rest[..closing].rfind("\n").map_or(0, |i| i + 1);
+                if rest[preceding_chars_start..closing].chars().all(is_whitespace) {
+                    // candidate found
+                    potential_closing = Some(closing);
+                } else {
+                    rest = &rest[closing + 3..];
+                }
+            }
+            if let Some(potential_closing) = potential_closing {
+                self.bump_bytes(potential_closing);
+            } else {
+                // eat everything.
+                self.eat_while(|_| true);
+            }
+        };
+
+        Frontmatter { has_invalid_preceding_whitespace, invalid_infostring }
+    }
+
     fn line_comment(&mut self) -> TokenKind {
         debug_assert!(self.prev() == '/' && self.first() == '/');
         self.bump();

Diff for: compiler/rustc_lexer/src/tests.rs

@@ -4,7 +4,7 @@ use super::*;
 
 fn check_raw_str(s: &str, expected: Result<u8, RawStrError>) {
     let s = &format!("r{}", s);
-    let mut cursor = Cursor::new(s);
+    let mut cursor = Cursor::new(s, false);
     cursor.bump();
     let res = cursor.raw_double_quoted_string(0);
     assert_eq!(res, expected);

Diff for: compiler/rustc_parse/messages.ftl

@@ -297,6 +297,19 @@ parse_forgot_paren = perhaps you forgot parentheses?
 parse_found_expr_would_be_stmt = expected expression, found `{$token}`
     .label = expected expression
 
+parse_frontmatter_extra_characters_after_close = extra characters after frontmatter close are not allowed
+parse_frontmatter_invalid_close_preceding_whitespace = invalid preceding whitespace for frontmatter close
+    .label = frontmatter close should not be preceded by whitespace
+parse_frontmatter_invalid_infostring = invalid infostring for frontmatter
+    .note = frontmatter infostrings must be an identifier immediately following the opening
+parse_frontmatter_invalid_opening_preceding_whitespace = invalid preceding whitespace for frontmatter opening
+    .label = frontmatter opening should not be preceded by whitespace
+parse_frontmatter_unclosed = unclosed frontmatter
+    .label = frontmatter opening here was not closed
+parse_frontmatter_length_mismatch = frontmatter close does not match the opening
+    .label_opening = the opening here has {$len_opening} dashes..
+    .label_close = ..while the close has {$len_close} dashes
+
 parse_function_body_equals_expr = function body cannot be `= expression;`
     .suggestion = surround the expression with `{"{"}` and `{"}"}` instead of `=` and `;`
 

Diff for: compiler/rustc_parse/src/errors.rs

@@ -730,6 +730,62 @@ pub(crate) struct FoundExprWouldBeStmt {
     pub suggestion: ExprParenthesesNeeded,
 }
 
+#[derive(Diagnostic)]
+#[diag(parse_frontmatter_extra_characters_after_close)]
+pub(crate) struct FrontmatterExtraCharactersAfterClose {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_frontmatter_invalid_infostring)]
+pub(crate) struct FrontmatterInvalidInfostring {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_frontmatter_invalid_opening_preceding_whitespace)]
+pub(crate) struct FrontmatterInvalidOpeningPrecedingWhitespace {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub label_span: Span,
+}
+
+
+#[derive(Diagnostic)]
+#[diag(parse_frontmatter_unclosed)]
+pub(crate) struct FrontmatterUnclosed {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub label_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_frontmatter_invalid_close_preceding_whitespace)]
+pub(crate) struct FrontmatterInvalidClosingPrecedingWhitespace {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub label_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_frontmatter_length_mismatch)]
+pub(crate) struct FrontmatterLengthMismatch {
+    #[primary_span]
+    pub span: Span,
+    #[label(parse_label_opening)]
+    pub opening: Span,
+    #[label(parse_label_close)]
+    pub close: Span,
+    pub len_opening: usize,
+    pub len_close: usize,
+}
+
+
 #[derive(Diagnostic)]
 #[diag(parse_leading_plus_not_supported)]
 pub(crate) struct LeadingPlusNotSupported {

Diff for: compiler/rustc_parse/src/lexer/mod.rs

@@ -6,7 +6,7 @@ use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::util::unicode::contains_text_flow_control_chars;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, DiagCtxtHandle, StashKey};
-use rustc_lexer::{Base, Cursor, DocStyle, LiteralKind, RawStrError};
+use rustc_lexer::{is_whitespace, Base, Cursor, DocStyle, LiteralKind, RawStrError};
 use rustc_literal_escaper::{EscapeError, Mode, unescape_mixed, unescape_unicode};
 use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::lint::builtin::{
@@ -55,7 +55,7 @@ pub(crate) fn lex_token_trees<'psess, 'src>(
         start_pos = start_pos + BytePos::from_usize(shebang_len);
     }
 
-    let cursor = Cursor::new(src);
+    let cursor = Cursor::new(src, true);
     let mut lexer = Lexer {
         psess,
         start_pos,
@@ -192,6 +192,11 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
                     let content = self.str_from_to(content_start, content_end);
                     self.cook_doc_comment(content_start, content, CommentKind::Block, doc_style)
                 }
+                rustc_lexer::TokenKind::Frontmatter { has_invalid_preceding_whitespace, invalid_infostring } => {
+                    self.validate_frontmatter(start, has_invalid_preceding_whitespace, invalid_infostring);
+                    preceded_by_whitespace = true;
+                    continue;
+                }
                 rustc_lexer::TokenKind::Whitespace => {
                     preceded_by_whitespace = true;
                     continue;
@@ -255,7 +260,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
                     // was consumed.
                     let lit_start = start + BytePos(prefix_len);
                     self.pos = lit_start;
-                    self.cursor = Cursor::new(&str_before[prefix_len as usize..]);
+                    self.cursor = Cursor::new(&str_before[prefix_len as usize..], false);
 
                     self.report_unknown_prefix(start);
                     let prefix_span = self.mk_sp(start, lit_start);
@@ -361,7 +366,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
                         // Reset the state so we just lex the `'r`.
                         let lt_start = start + BytePos(2);
                         self.pos = lt_start;
-                        self.cursor = Cursor::new(&str_before[2 as usize..]);
+                        self.cursor = Cursor::new(&str_before[2 as usize..], false);
 
                         let lifetime_name = self.str_from(start);
                         let ident = Symbol::intern(lifetime_name);
@@ -474,6 +479,64 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
         }
     }
 
+    fn validate_frontmatter(&self, start: BytePos, has_invalid_preceding_whitespace: bool, invalid_infostring: bool) {
+        let s = self.str_from(start);
+        let real_start = s.find("---").unwrap();
+        let frontmatter_opening_pos = BytePos(real_start as u32) + start;
+        let s_new = &s[real_start..];
+        let within = s_new.trim_start_matches('-');
+        let len_opening = s_new.len() - within.len();
+
+        let frontmatter_opening_end_pos = frontmatter_opening_pos + BytePos(len_opening as u32);
+        if has_invalid_preceding_whitespace {
+            let line_start = BytePos(s[..real_start].rfind("\n").map_or(0, |i| i as u32 + 1)) + start;
+            let span = self.mk_sp(line_start, frontmatter_opening_end_pos);
+            let label_span = self.mk_sp(line_start, frontmatter_opening_pos);
+            self.dcx().emit_err(errors::FrontmatterInvalidOpeningPrecedingWhitespace { span, label_span });
+        }
+
+        if invalid_infostring {
+            let line_end = s[real_start..].find('\n').unwrap_or(s[real_start..].len());
+            let span = self.mk_sp(frontmatter_opening_end_pos, frontmatter_opening_pos + BytePos(line_end as u32));
+            self.dcx().emit_err(errors::FrontmatterInvalidInfostring { span });
+        }
+
+        let last_line_start = within.rfind('\n').map_or(0, |i| i+1);
+        let last_line = &within[last_line_start..];
+        let last_line_trimmed = last_line.trim_start_matches(is_whitespace);
+        let last_line_start_pos = frontmatter_opening_end_pos + BytePos(last_line_start as u32);
+
+        if !last_line_trimmed.starts_with("---") {
+            let span = self.mk_sp(frontmatter_opening_pos, self.pos);
+            let label_span = self.mk_sp(frontmatter_opening_pos, frontmatter_opening_end_pos);
+            self.dcx().emit_err(errors::FrontmatterUnclosed { span, label_span });
+            return;
+        }
+
+        if last_line_trimmed.len() != last_line.len() {
+            let line_end = last_line_start_pos + BytePos(last_line.len() as u32);
+            let span = self.mk_sp(last_line_start_pos, line_end);
+            let whitespace_end = last_line_start_pos + BytePos((last_line.len() - last_line_trimmed.len()) as u32);
+            let label_span = self.mk_sp(last_line_start_pos, whitespace_end);
+            self.dcx().emit_err(errors::FrontmatterInvalidClosingPrecedingWhitespace { span, label_span });
+        }
+
+        let rest = last_line.trim_start_matches('-');
+        let len_close = last_line.len() - rest.len();
+        if len_close != len_opening {
+            let span = self.mk_sp(frontmatter_opening_pos, self.pos);
+            let opening = self.mk_sp(frontmatter_opening_pos, frontmatter_opening_end_pos);
+            let last_line_close_pos = last_line_start_pos + BytePos(len_close as u32);
+            let close = self.mk_sp(last_line_start_pos, last_line_close_pos);
+            self.dcx().emit_err(errors::FrontmatterLengthMismatch { span, opening, close, len_opening, len_close });
+        }
+
+        if !rest.trim_matches(is_whitespace).is_empty() {
+            let span = self.mk_sp(last_line_start_pos, self.pos);
+            self.dcx().emit_err(errors::FrontmatterExtraCharactersAfterClose { span });
+        }
+    }
+
     fn cook_doc_comment(
         &self,
         content_start: BytePos,
@@ -838,7 +901,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
         let space_pos = start + BytePos(1);
         let space_span = self.mk_sp(space_pos, space_pos);
 
-        let mut cursor = Cursor::new(str_before);
+        let mut cursor = Cursor::new(str_before, false);
 
         let (is_string, span, unterminated) = match cursor.guarded_double_quoted_string() {
             Some(rustc_lexer::GuardedStr { n_hashes, terminated, token_len }) => {
@@ -904,7 +967,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
             // For backwards compatibility, roll back to after just the first `#`
             // and return the `Pound` token.
             self.pos = start + BytePos(1);
-            self.cursor = Cursor::new(&str_before[1..]);
+            self.cursor = Cursor::new(&str_before[1..], false);
             token::Pound
         }
     }

Diff for: src/librustdoc/html/highlight.rs

@@ -638,7 +638,7 @@ impl<'src> Classifier<'src> {
     /// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
     /// file span which will be used later on by the `span_correspondence_map`.
     fn new(src: &'src str, file_span: Span, decoration_info: Option<&DecorationInfo>) -> Self {
-        let tokens = PeekIter::new(TokenIter { src, cursor: Cursor::new(src) });
+        let tokens = PeekIter::new(TokenIter { src, cursor: Cursor::new(src, true) });
         let decorations = decoration_info.map(Decorations::new);
         Classifier {
             tokens,
@@ -884,6 +884,7 @@ impl<'src> Classifier<'src> {
             | TokenKind::At
             | TokenKind::Tilde
             | TokenKind::Colon
+            | TokenKind::Frontmatter { .. }
             | TokenKind::Unknown => return no_highlight(sink),
 
             TokenKind::Question => Class::QuestionMark,