XXX: NtExpr/NtLiteral

Notes about tests:
- tests/ui/macros/stringify.rs: the `c2` macro is no longer needed,
  because the TokenStream pretty printer is now used for all cases.

- tests/ui/rfcs/rfc-2294-if-let-guard/feature-gate.rs: some messages are
  now duplicated due to repeated parsing.

- tests/ui/rfcs/rfc-2497-if-let-chains/disallowed-positions*.rs: ditto.

Getting a test failure here:
```
Building tool error_index_generator (stage1 -> stage2, x86_64-unknown-linux-gnu)
   Compiling cfg-if v1.0.0
    ...
   Compiling mdbook v0.4.37
error: internal compiler error: the following error was constructed but not emitted

error: unexpected token: keyword `self`
   --> /home/njn/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mdbook-0.4.37/src/book/summary.rs:280:31
    |
280 | ...   bail!(self.parse_error("Suffix chapters cannot be followed by a list"));
    |             ^^^^

thread 'rustc' panicked at /home/njn/dev/rust3/compiler/rustc_errors/src/diagnostic.rs:1375:17:
error was constructed but not emitted
```

Author: nnethercote

compiler/rustc_ast/src/ast_traits.rs

@@ -231,13 +231,11 @@ impl HasTokens for Attribute {
 impl HasTokens for Nonterminal {
     fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
             Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
             Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }

compiler/rustc_ast/src/mut_visit.rs

@@ -823,10 +823,8 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
 // multiple items there....
 pub fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
     match nt {
-        token::NtExpr(expr) => vis.visit_expr(expr),
         token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
         token::NtLifetime(ident) => vis.visit_ident(ident),
-        token::NtLiteral(expr) => vis.visit_expr(expr),
     }
 }
 

compiler/rustc_ast/src/token.rs

@@ -4,7 +4,6 @@ pub use Nonterminal::*;
 pub use TokenKind::*;
 
 use crate::ast;
-use crate::ptr::P;
 use crate::util::case::Case;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -134,17 +133,27 @@ impl Lit {
         }
     }
 
-    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` excluding unary negation.
+    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
+    /// `Parser::maybe_parse_token_lit` (excluding unary negation).
     pub fn from_token(token: &Token) -> Option<Lit> {
         match token.uninterpolate().kind {
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
             Literal(token_lit) => Some(token_lit),
-            // njn: deal with later, with NtExpr/NtLiteral
-            Interpolated(ref nt)
-                if let NtExpr(expr) | NtLiteral(expr) = &nt.0
-                    && let ast::ExprKind::Lit(token_lit) = expr.kind =>
-            {
-                Some(token_lit)
+            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Literal))) => {
+                panic!("njn: FROM_TOKEN (1)");
+                // if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                // && let ast::ExprKind::Lit(token_lit) = expr.kind =>
+                // {
+                //     Some(token_lit)
+                // }
+            }
+            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Expr))) => {
+                panic!("njn: FROM_TOKEN (2)");
+                // if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                // && let ast::ExprKind::Lit(token_lit) = expr.kind =>
+                // {
+                //     Some(token_lit)
+                // }
             }
             _ => None,
         }
@@ -464,6 +473,7 @@ impl Token {
         Token::new(Ident(ident.name, ident.is_raw_guess().into()), ident.span)
     }
 
+    /// njn: phase this out in favour of Parser::uninterpolated_span
     /// For interpolated tokens, returns a span of the fragment to which the interpolated
     /// token refers. For all other tokens this is just a regular span.
     /// It is particularly important to use this for identifiers and lifetimes
@@ -520,7 +530,6 @@ impl Token {
             PathSep                           | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) | NtExpr(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Block |
                 NonterminalKind::Expr |
@@ -547,7 +556,6 @@ impl Token {
             | DotDot | DotDotDot | DotDotEq      // ranges
             | Lt | BinOp(Shl)                    // associated path
             | PathSep => true,                   // global path
-            Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Block |
                 NonterminalKind::PatParam { .. } |
@@ -588,7 +596,6 @@ impl Token {
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
             OpenDelim(Delimiter::Brace) => true,
-            Interpolated(ref nt) => matches!(&nt.0, NtExpr(..) | NtLiteral(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Expr | NonterminalKind::Block | NonterminalKind::Literal,
             ))) => true,
@@ -631,22 +638,24 @@ impl Token {
     ///
     /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
     ///
-    /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
+    /// Keep this in sync with `Lit::from_token` and
+    /// `Parser::maybe_parse_token_lit` (excluding unary negation).
     pub fn can_begin_literal_maybe_minus(&self) -> bool {
         match self.uninterpolate().kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
-            Interpolated(ref nt) => match &nt.0 {
-                NtLiteral(_) => true,
-                NtExpr(e) => match &e.kind {
-                    ast::ExprKind::Lit(_) => true,
-                    ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
-                        matches!(&e.kind, ast::ExprKind::Lit(_))
-                    }
-                    _ => false,
-                },
-                _ => false,
-            },
+            // njn: fix up
+            // Interpolated(ref nt) => match &nt.0 {
+            //     NtLiteral(_) => true,
+            //     NtExpr(e) => match &e.kind {
+            //         ast::ExprKind::Lit(_) => true,
+            //         ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
+            //             matches!(&e.kind, ast::ExprKind::Lit(_))
+            //         }
+            //         _ => false,
+            //     },
+            //     _ => false,
+            // },
             // njn: too simple compared to what's above?
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 NonterminalKind::Literal | NonterminalKind::Expr,
@@ -667,7 +676,6 @@ impl Token {
                     Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
                 }
                 NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
-                _ => Cow::Borrowed(self),
             },
             _ => Cow::Borrowed(self),
         }
@@ -719,23 +727,19 @@ impl Token {
         self.ident().is_some_and(|(ident, _)| ident.name == name)
     }
 
-    /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
+    /// Would `maybe_reparse_metavar_expr` in `parser.rs` return `Ok(..)`?
     /// That is, is this a pre-parsed expression dropped into the token stream
     /// (which happens while parsing the result of macro expansion)?
-    pub fn is_whole_expr(&self) -> bool {
-        // njn: nothing needed, just deal with NtExpr/NtLiteral/NtPath/NtBlock later
-        if let Interpolated(nt) = &self.kind
-            && let NtExpr(_) | NtLiteral(_) = &nt.0
-        {
-            true
-        } else if matches!(
+    pub fn is_metavar_expr(&self) -> bool {
+        matches!(
             self.is_metavar_seq(),
-            Some(NonterminalKind::Block | NonterminalKind::Path)
-        ) {
-            true
-        } else {
-            false
-        }
+            Some(
+                NonterminalKind::Expr
+                    | NonterminalKind::Literal
+                    | NonterminalKind::Block
+                    | NonterminalKind::Path
+            )
+        )
     }
 
     /// Are we at a block from a metavar (`$b:block`)?
@@ -904,10 +908,8 @@ impl PartialEq<TokenKind> for Token {
 #[derive(Clone, Encodable, Decodable)]
 /// For interpolation during macro expansion.
 pub enum Nonterminal {
-    NtExpr(P<ast::Expr>),
     NtIdent(Ident, IdentIsRaw),
     NtLifetime(Ident),
-    NtLiteral(P<ast::Expr>),
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -991,15 +993,12 @@ impl fmt::Display for NonterminalKind {
 impl Nonterminal {
     pub fn use_span(&self) -> Span {
         match self {
-            NtExpr(expr) | NtLiteral(expr) => expr.span,
             NtIdent(ident, _) | NtLifetime(ident) => ident.span,
         }
     }
 
     pub fn descr(&self) -> &'static str {
         match self {
-            NtExpr(..) => "expression",
-            NtLiteral(..) => "literal",
             NtIdent(..) => "identifier",
             NtLifetime(..) => "lifetime",
         }
@@ -1025,9 +1024,7 @@ impl PartialEq for Nonterminal {
 impl fmt::Debug for Nonterminal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            NtExpr(..) => f.pad("NtExpr(..)"),
             NtIdent(..) => f.pad("NtIdent(..)"),
-            NtLiteral(..) => f.pad("NtLiteral(..)"),
             NtLifetime(..) => f.pad("NtLifetime(..)"),
         }
     }

compiler/rustc_ast/src/tokenstream.rs

@@ -483,7 +483,6 @@ impl TokenStream {
             Nonterminal::NtLifetime(ident) => {
                 TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
             }
-            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
         }
     }
 

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -844,10 +844,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
 
     fn nonterminal_to_string(&self, nt: &Nonterminal) -> String {
         match nt {
-            token::NtExpr(e) => self.expr_to_string(e),
             &token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(),
             token::NtLifetime(e) => e.to_string(),
-            token::NtLiteral(e) => self.expr_to_string(e),
         }
     }
 

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -254,6 +254,9 @@ pub(super) fn transcribe<'a>(
                         // Emit as a token stream within `Delimiter::Invisible` to maintain parsing
                         // priorities.
                         marker.visit_span(&mut sp);
+                        // njn: `sp` usage here means that both the open delim
+                        // and close delim end up with the same span, which
+                        // covers the `$foo` in the decl macro RHS
                         TokenTree::Delimited(
                             DelimSpan::from_single(sp),
                             DelimSpacing::new(Spacing::Alone, Spacing::Alone), // njn: ?
@@ -289,6 +292,12 @@ pub(super) fn transcribe<'a>(
                         MatchedSingle(ParseNtResult::PatWithOr(ref pat)) => {
                             mk_delimited(NonterminalKind::PatWithOr, TokenStream::from_ast(pat))
                         }
+                        MatchedSingle(ParseNtResult::Expr(ref expr)) => {
+                            mk_delimited(NonterminalKind::Expr, TokenStream::from_ast(expr))
+                        }
+                        MatchedSingle(ParseNtResult::Literal(ref expr)) => {
+                            mk_delimited(NonterminalKind::Literal, TokenStream::from_ast(expr))
+                        }
                         MatchedSingle(ParseNtResult::Ty(ref ty)) => {
                             mk_delimited(NonterminalKind::Ty, TokenStream::from_ast(ty))
                         }

compiler/rustc_index/src/bit_set/tests.rs

@@ -302,7 +302,7 @@ fn chunked_bitset() {
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x8000_0000_0000_0000
             ])),
-        ],
+        ], // njn: trailing comma here causes a crash in reparse_metavar_seq?
     );
     assert_eq!(b4096.count(), 2);
     b4096.assert_valid();
@@ -336,7 +336,7 @@ fn chunked_bitset() {
             ])),
             Zeros(2048),
             Zeros(1808),
-        ],
+        ], // njn: trailing comma here causes a crash in reparse_metavar_seq?
     );
     let mut b10000b = ChunkedBitSet::<usize>::new_empty(10000);
     b10000b.clone_from(&b10000);

compiler/rustc_parse/messages.ftl

@@ -803,7 +803,7 @@ parse_unexpected_parentheses_in_match_arm_pattern = unexpected parentheses surro
 parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
     .note = you cannot use `Self` as a generic parameter because it is reserved for associated items
 
-parse_unexpected_token_after_dot = unexpected token: `{$actual}`
+parse_unexpected_token_after_dot = unexpected token: {$actual}
 
 parse_unexpected_token_after_label = expected `while`, `for`, `loop` or `{"{"}` after a label
     .suggestion_remove_label = consider removing the label

compiler/rustc_parse/src/errors.rs

@@ -1600,10 +1600,10 @@ pub(crate) struct SelfArgumentPointer {
 
 #[derive(Diagnostic)]
 #[diag(parse_unexpected_token_after_dot)]
-pub struct UnexpectedTokenAfterDot<'a> {
+pub struct UnexpectedTokenAfterDot {
     #[primary_span]
     pub span: Span,
-    pub actual: Cow<'a, str>,
+    pub actual: String,
 }
 
 #[derive(Diagnostic)]