Unrolled build for #150099

Rollup merge of #150099 - GuillaumeGomez:field-handling, r=yotamofek

[rustdoc] Fix invalid handling of field followed by negated macro call

This is the bug uncovered in #150022. Once fixed Ill rebuild all compiler docs and see if we can enable the option for compiler docs. =D

It's a weird case where we extracted some tokens out of the iterator and then, when checking next items (from this iterator), it didn't find the `:` token, and therefore badly assumed the token kind.

The solution I came up with is to instead not extract tokens from the iterator and to count how many tokens are in the current path. So when iterate over the items, instead of having a mix of extracted tokens and tokens still inside the iterator, we now only iterate over the iterator.

The biggest change here is that `get_full_ident_path` will return an option instead of a `Vec`, and if it's contains `:` (one, not two), then it will return `None` and the `:` will be handled like any token and not like a path (which is more correct imo).

r? `@yotamofek`

Author: rust-timer

src/librustdoc/html/highlight.rs

@@ -832,6 +832,20 @@ impl<'a> PeekIter<'a> {
         .copied()
     }
 
+    fn peek_next_if<F: Fn((TokenKind, &'a str)) -> bool>(
+        &mut self,
+        f: F,
+    ) -> Option<(TokenKind, &'a str)> {
+        let next = self.peek_next()?;
+        if f(next) {
+            Some(next)
+        } else {
+            // We go one step back.
+            self.peek_pos -= 1;
+            None
+        }
+    }
+
     fn stop_peeking(&mut self) {
         self.peek_pos = 0;
     }
@@ -903,18 +917,17 @@ fn classify<'src>(
             }
         }
 
-        if let Some((TokenKind::Colon | TokenKind::Ident, _)) = classifier.tokens.peek() {
-            let tokens = classifier.get_full_ident_path();
-            for &(token, start, end) in &tokens {
-                let text = &classifier.src[start..end];
-                classifier.advance(token, text, sink, start as u32);
-                classifier.byte_pos += text.len() as u32;
-            }
-            if !tokens.is_empty() {
-                continue;
-            }
-        }
-        if let Some((token, text, before)) = classifier.next() {
+        if let Some((TokenKind::Colon | TokenKind::Ident, _)) = classifier.tokens.peek()
+            && let Some(nb_items) = classifier.get_full_ident_path()
+        {
+            let start = classifier.byte_pos as usize;
+            let len: usize = iter::from_fn(|| classifier.next())
+                .take(nb_items)
+                .map(|(_, text, _)| text.len())
+                .sum();
+            let text = &classifier.src[start..start + len];
+            classifier.advance(TokenKind::Ident, text, sink, start as u32);
+        } else if let Some((token, text, before)) = classifier.next() {
             classifier.advance(token, text, sink, before);
         } else {
             break;
@@ -957,47 +970,47 @@ impl<'src> Classifier<'src> {
     }
 
     /// Concatenate colons and idents as one when possible.
-    fn get_full_ident_path(&mut self) -> Vec<(TokenKind, usize, usize)> {
-        let start = self.byte_pos as usize;
-        let mut pos = start;
+    fn get_full_ident_path(&mut self) -> Option<usize> {
         let mut has_ident = false;
+        let mut nb_items = 0;
 
-        loop {
+        let ret = loop {
             let mut nb = 0;
-            while let Some((TokenKind::Colon, _)) = self.tokens.peek() {
-                self.tokens.next();
+            while self.tokens.peek_next_if(|(token, _)| token == TokenKind::Colon).is_some() {
                 nb += 1;
+                nb_items += 1;
             }
             // Ident path can start with "::" but if we already have content in the ident path,
             // the "::" is mandatory.
             if has_ident && nb == 0 {
-                return vec![(TokenKind::Ident, start, pos)];
+                break Some(nb_items);
             } else if nb != 0 && nb != 2 {
                 if has_ident {
-                    return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
+                    // Following `;` will be handled on its own.
+                    break Some(nb_items - 1);
                 } else {
-                    return vec![(TokenKind::Colon, start, pos + nb)];
+                    break None;
                 }
             }
 
-            if let Some((TokenKind::Ident, text)) = self.tokens.peek()
+            if let Some((TokenKind::Ident, text)) =
+                self.tokens.peek_next_if(|(token, _)| token == TokenKind::Ident)
                 && let symbol = Symbol::intern(text)
                 && (symbol.is_path_segment_keyword() || !is_keyword(symbol))
             {
-                // We only "add" the colon if there is an ident behind.
-                pos += text.len() + nb;
                 has_ident = true;
-                self.tokens.next();
+                nb_items += 1;
             } else if nb > 0 && has_ident {
-                return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
-            } else if nb > 0 {
-                return vec![(TokenKind::Colon, start, start + nb)];
+                // Following `;` will be handled on its own.
+                break Some(nb_items - 1);
             } else if has_ident {
-                return vec![(TokenKind::Ident, start, pos)];
+                break Some(nb_items);
             } else {
-                return Vec::new();
+                break None;
             }
-        }
+        };
+        self.tokens.stop_peeking();
+        ret
     }
 
     /// Wraps the tokens iteration to ensure that the `byte_pos` is always correct.
@@ -1243,7 +1256,6 @@ impl<'src> Classifier<'src> {
                 Class::MacroNonTerminal
             }
             TokenKind::Ident => {
-                let file_span = self.file_span;
                 let span = || new_span(before, text, file_span);
 
                 match text {

tests/rustdoc/macro-expansion/field-followed-by-exclamation.rs

@@ -0,0 +1,26 @@
+// This test ensures that the macro expansion is correctly handled in cases like:
+// `field: !f!`, because the `:` was simply not considered because of how paths
+// are handled.
+
+//@ compile-flags: -Zunstable-options --generate-macro-expansion
+
+#![crate_name = "foo"]
+
+//@ has 'src/foo/field-followed-by-exclamation.rs.html'
+
+struct Bar {
+    bla: bool,
+}
+
+macro_rules! f {
+    () => {{ false }}
+}
+
+const X: Bar = Bar {
+    //@ has - '//*[@class="expansion"]/*[@class="original"]/*[@class="macro"]' 'f!'
+    //@ has - '//*[@class="expansion"]/*[@class="original"]' 'f!()'
+    //@ has - '//*[@class="expansion"]/*[@class="expanded"]' '{ false }'
+    // It includes both original and expanded code.
+    //@ has - '//*[@class="expansion"]' '    bla: !{ false }f!()'
+    bla: !f!(),
+};