[WIP] Token-based outer attributes handling

compiler/rustc_ast/src/ast.rs

@@ -24,7 +24,7 @@ pub use UnsafeSource::*;
 
 use crate::ptr::P;
 use crate::token::{self, CommentKind, DelimToken};
-use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
+use crate::tokenstream::{DelimSpan, PreexpTokenStream, TokenStream, TokenTree};
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
@@ -926,10 +926,8 @@ impl Stmt {
         self.kind = match self.kind {
             StmtKind::Expr(expr) => StmtKind::Semi(expr),
             StmtKind::MacCall(mac) => {
-                StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs }| MacCallStmt {
-                    mac,
-                    style: MacStmtStyle::Semicolon,
-                    attrs,
+                StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs, tokens }| {
+                    MacCallStmt { mac, style: MacStmtStyle::Semicolon, attrs, tokens }
                 }))
             }
             kind => kind,
@@ -973,6 +971,7 @@ pub struct MacCallStmt {
     pub mac: MacCall,
     pub style: MacStmtStyle,
     pub attrs: AttrVec,
+    pub tokens: Option<PreexpTokenStream>,
 }
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
@@ -998,6 +997,7 @@ pub struct Local {
     pub init: Option<P<Expr>>,
     pub span: Span,
     pub attrs: AttrVec,
+    pub tokens: Option<PreexpTokenStream>,
 }
 
 /// An arm of a 'match'.
@@ -1066,7 +1066,7 @@ pub struct Expr {
     pub kind: ExprKind,
     pub span: Span,
     pub attrs: AttrVec,
-    pub tokens: Option<TokenStream>,
+    pub tokens: Option<PreexpTokenStream>,
 }
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
@@ -2436,6 +2436,7 @@ pub struct Attribute {
     /// or the construct this attribute is contained within (inner).
     pub style: AttrStyle,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -2582,7 +2583,7 @@ pub struct Item<K = ItemKind> {
     ///
     /// Note that the tokens here do not include the outer attributes, but will
     /// include inner attributes.
-    pub tokens: Option<TokenStream>,
+    pub tokens: Option<PreexpTokenStream>,
 }
 
 impl Item {

compiler/rustc_ast/src/attr/mod.rs

@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
 use crate::mut_visit::visit_clobber;
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Token};
-use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
+use crate::tokenstream::{DelimSpan, PreexpTokenStream, TokenStream, TokenTree, TreeAndSpacing};
 
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_span::source_map::{BytePos, Spanned};
@@ -334,7 +334,7 @@ pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attri
 }
 
 pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute {
-    Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span }
+    Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span, tokens: None }
 }
 
 /// Returns an inner attribute with the given value and span.
@@ -353,7 +353,13 @@ pub fn mk_doc_comment(
     data: Symbol,
     span: Span,
 ) -> Attribute {
-    Attribute { kind: AttrKind::DocComment(comment_kind, data), id: mk_attr_id(), style, span }
+    Attribute {
+        kind: AttrKind::DocComment(comment_kind, data),
+        id: mk_attr_id(),
+        style,
+        span,
+        tokens: None,
+    }
 }
 
 pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
@@ -585,6 +591,7 @@ impl NestedMetaItem {
 pub trait HasAttrs: Sized {
     fn attrs(&self) -> &[Attribute];
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));
+    fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream));
 }
 
 impl<T: HasAttrs> HasAttrs for Spanned<T> {
@@ -594,6 +601,9 @@ impl<T: HasAttrs> HasAttrs for Spanned<T> {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         self.node.visit_attrs(f);
     }
+    fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream)) {
+        self.node.visit_tokens(f)
+    }
 }
 
 impl HasAttrs for Vec<Attribute> {
@@ -603,6 +613,7 @@ impl HasAttrs for Vec<Attribute> {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         f(self)
     }
+    fn visit_tokens(&mut self, _f: impl FnOnce(&mut PreexpTokenStream)) {}
 }
 
 impl HasAttrs for AttrVec {
@@ -616,6 +627,7 @@ impl HasAttrs for AttrVec {
             vec.into()
         });
     }
+    fn visit_tokens(&mut self, _f: impl FnOnce(&mut PreexpTokenStream)) {}
 }
 
 impl<T: HasAttrs + 'static> HasAttrs for P<T> {
@@ -625,14 +637,30 @@ impl<T: HasAttrs + 'static> HasAttrs for P<T> {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         (**self).visit_attrs(f);
     }
+    fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream)) {
+        (**self).visit_tokens(f)
+    }
+}
+
+impl<T: HasAttrs> HasAttrs for Option<T> {
+    fn attrs(&self) -> &[Attribute] {
+        self.as_ref().map_or(&[], |inner| inner.attrs())
+    }
+    fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
+        self.as_mut().map(|inner| inner.visit_attrs(f));
+    }
+    fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream)) {
+        self.as_mut().map(|inner| inner.visit_tokens(f));
+    }
 }
 
 impl HasAttrs for StmtKind {
     fn attrs(&self) -> &[Attribute] {
         match *self {
             StmtKind::Local(ref local) => local.attrs(),
             StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.attrs(),
-            StmtKind::Empty | StmtKind::Item(..) => &[],
+            StmtKind::Item(..) => &[],
+            StmtKind::Empty => &[],
             StmtKind::MacCall(ref mac) => mac.attrs.attrs(),
         }
     }
@@ -641,12 +669,24 @@ impl HasAttrs for StmtKind {
         match self {
             StmtKind::Local(local) => local.visit_attrs(f),
             StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.visit_attrs(f),
-            StmtKind::Empty | StmtKind::Item(..) => {}
+            StmtKind::Item(item) => item.visit_attrs(f),
+            StmtKind::Empty => {}
             StmtKind::MacCall(mac) => {
                 mac.attrs.visit_attrs(f);
             }
         }
     }
+
+    fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream)) {
+        match self {
+            StmtKind::Local(local) => local.visit_tokens(f),
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.visit_tokens(f),
+            // FIXME: Is this correct?
+            StmtKind::Item(item) => item.visit_tokens(f),
+            StmtKind::Empty => {}
+            StmtKind::MacCall(mac) => mac.attrs.visit_tokens(f),
+        }
+    }
 }
 
 impl HasAttrs for Stmt {
@@ -657,6 +697,9 @@ impl HasAttrs for Stmt {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         self.kind.visit_attrs(f);
     }
+    fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream)) {
+        self.kind.visit_tokens(f);
+    }
 }
 
 macro_rules! derive_has_attrs {
@@ -669,11 +712,36 @@ macro_rules! derive_has_attrs {
             fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
                 self.attrs.visit_attrs(f);
             }
+            fn visit_tokens(&mut self, _f: impl FnOnce(&mut PreexpTokenStream)) {}
+        }
+    )* }
+}
+
+macro_rules! derive_has_attrs_with_tokens {
+    ($($ty:path),*) => { $(
+        impl HasAttrs for $ty {
+            fn attrs(&self) -> &[Attribute] {
+                &self.attrs
+            }
+
+            fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
+                self.attrs.visit_attrs(f);
+            }
+
+            fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream)) {
+                if let Some(tokens) = self.tokens.as_mut() {
+                    f(tokens)
+                }
+            }
         }
     )* }
 }
 
 derive_has_attrs! {
-    Item, Expr, Local, ast::AssocItem, ast::ForeignItem, ast::StructField, ast::Arm,
+    ast::StructField, ast::Arm,
     ast::Field, ast::FieldPat, ast::Variant, ast::Param, GenericParam
 }
+
+derive_has_attrs_with_tokens! {
+    Expr, Item, Local, ast::AssocItem, ast::ForeignItem
+}

compiler/rustc_ast/src/mut_visit.rs

@@ -567,7 +567,7 @@ pub fn noop_visit_parenthesized_parameter_data<T: MutVisitor>(
 }
 
 pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
-    let Local { id, pat, ty, init, span, attrs } = local.deref_mut();
+    let Local { id, pat, ty, init, span, attrs, tokens: _ } = local.deref_mut();
     vis.visit_id(id);
     vis.visit_pat(pat);
     visit_opt(ty, |ty| vis.visit_ty(ty));
@@ -577,7 +577,7 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
 }
 
 pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
-    let Attribute { kind, id: _, style: _, span } = attr;
+    let Attribute { kind, id: _, style: _, span, tokens: _ } = attr;
     match kind {
         AttrKind::Normal(AttrItem { path, args, tokens: _ }) => {
             vis.visit_path(path);
@@ -1311,7 +1311,7 @@ pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
         StmtKind::Semi(expr) => vis.filter_map_expr(expr).into_iter().map(StmtKind::Semi).collect(),
         StmtKind::Empty => smallvec![StmtKind::Empty],
         StmtKind::MacCall(mut mac) => {
-            let MacCallStmt { mac: mac_, style: _, attrs } = mac.deref_mut();
+            let MacCallStmt { mac: mac_, style: _, attrs, tokens: _ } = mac.deref_mut();
             vis.visit_mac(mac_);
             visit_thin_attrs(attrs, vis);
             smallvec![StmtKind::MacCall(mac)]

compiler/rustc_ast/src/tokenstream.rs

@@ -403,6 +403,10 @@ impl Cursor {
     pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
         self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone())
     }
+
+    pub fn index(&self) -> usize {
+        self.index
+    }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
@@ -428,3 +432,85 @@ impl DelimSpan {
         self.open.with_hi(self.close.hi())
     }
 }
+
+#[derive(Clone, Debug, Default, Encodable, Decodable)]
+pub struct PreexpTokenStream(pub Lrc<Vec<(PreexpTokenTree, Spacing)>>);
+
+#[derive(Clone, Debug, Encodable, Decodable)]
+pub enum PreexpTokenTree {
+    Token(Token),
+    Delimited(DelimSpan, DelimToken, PreexpTokenStream),
+    OuterAttributes(AttributesData),
+}
+
+impl PreexpTokenStream {
+    pub fn new(tokens: Vec<(PreexpTokenTree, Spacing)>) -> PreexpTokenStream {
+        PreexpTokenStream(Lrc::new(tokens))
+    }
+
+    pub fn replace_attributes(&mut self, f: impl FnOnce(&mut AttributesData)) {
+        if let &[(PreexpTokenTree::OuterAttributes(ref data), joint)] = &**self.0 {
+            let mut data = data.clone();
+            f(&mut data);
+            *self = PreexpTokenStream::new(vec![(PreexpTokenTree::OuterAttributes(data), joint)]);
+        } else {
+            panic!("Expected a single PreexpTokenTree::OuterAttributes, found {:?}", self);
+        }
+    }
+
+    pub fn to_tokenstream(self) -> TokenStream {
+        let trees: Vec<_> = self
+            .0
+            .iter()
+            .flat_map(|tree| match &tree.0 {
+                PreexpTokenTree::Token(inner) => {
+                    smallvec![(TokenTree::Token(inner.clone()), tree.1)].into_iter()
+                }
+                PreexpTokenTree::Delimited(span, delim, stream) => smallvec![(
+                    TokenTree::Delimited(*span, *delim, stream.clone().to_tokenstream()),
+                    tree.1,
+                )]
+                .into_iter(),
+                PreexpTokenTree::OuterAttributes(data) => {
+                    let flat: SmallVec<[_; 1]> = data
+                        .attrs
+                        .iter()
+                        .filter(|attr| attr.style == crate::AttrStyle::Outer)
+                        .flat_map(|attr| {
+                            attr.tokens.as_ref().expect("Missing tokens").0.iter().cloned()
+                        })
+                        .chain(data.tokens.clone().to_tokenstream().0.iter().cloned())
+                        .collect();
+                    flat.into_iter()
+                }
+            })
+            .collect();
+        TokenStream::new(trees)
+    }
+
+    pub fn from_tokenstream(stream: TokenStream) -> PreexpTokenStream {
+        let trees: Vec<_> = stream
+            .0
+            .iter()
+            .cloned()
+            .map(|tree| {
+                let new_tree = match tree.0 {
+                    TokenTree::Token(token) => PreexpTokenTree::Token(token),
+                    TokenTree::Delimited(sp, delim, inner) => PreexpTokenTree::Delimited(
+                        sp,
+                        delim,
+                        PreexpTokenStream::from_tokenstream(inner),
+                    ),
+                };
+                (new_tree, tree.1)
+            })
+            .collect();
+        PreexpTokenStream::new(trees)
+    }
+}
+
+#[derive(Clone, Debug, Encodable, Decodable)]
+pub struct AttributesData {
+    pub attrs: Vec<crate::Attribute>,
+    pub tokens: PreexpTokenStream,
+}

compiler/rustc_ast/src/visit.rs

@@ -692,7 +692,7 @@ pub fn walk_stmt<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Stmt) {
         StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => visitor.visit_expr(expr),
         StmtKind::Empty => {}
         StmtKind::MacCall(ref mac) => {
-            let MacCallStmt { ref mac, style: _, ref attrs } = **mac;
+            let MacCallStmt { ref mac, style: _, ref attrs, tokens: _ } = **mac;
             visitor.visit_mac(mac);
             for attr in attrs.iter() {
                 visitor.visit_attribute(attr);

compiler/rustc_ast_lowering/src/lib.rs

@@ -972,7 +972,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data),
         };
 
-        Attribute { kind, id: attr.id, style: attr.style, span: attr.span }
+        Attribute { kind, id: attr.id, style: attr.style, span: attr.span, tokens: None }
     }
 
     fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs {

compiler/rustc_builtin_macros/src/deriving/debug.rs

@@ -132,6 +132,7 @@ fn stmt_let_underscore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P<ast::Expr>) -> as
         id: ast::DUMMY_NODE_ID,
         span: sp,
         attrs: ast::AttrVec::new(),
+        tokens: None,
     });
     ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp, tokens: None }
 }

compiler/rustc_expand/src/base.rs

@@ -4,7 +4,7 @@ use crate::module::DirectoryOwnership;
 use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::ptr::P;
 use rustc_ast::token;
-use rustc_ast::tokenstream::{self, TokenStream};
+use rustc_ast::tokenstream::{self, PreexpTokenStream, TokenStream};
 use rustc_ast::visit::{AssocCtxt, Visitor};
 use rustc_ast::{self as ast, Attribute, NodeId, PatKind};
 use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
@@ -81,6 +81,24 @@ impl HasAttrs for Annotatable {
             Annotatable::Variant(v) => v.visit_attrs(f),
         }
     }
+
+    fn visit_tokens(&mut self, f: impl FnOnce(&mut PreexpTokenStream)) {
+        match self {
+            Annotatable::Item(item) => item.visit_tokens(f),
+            Annotatable::TraitItem(trait_item) => trait_item.visit_tokens(f),
+            Annotatable::ImplItem(impl_item) => impl_item.visit_tokens(f),
+            Annotatable::ForeignItem(foreign_item) => foreign_item.visit_tokens(f),
+            Annotatable::Stmt(stmt) => stmt.visit_tokens(f),
+            Annotatable::Expr(expr) => expr.visit_tokens(f),
+            Annotatable::Arm(arm) => arm.visit_tokens(f),
+            Annotatable::Field(field) => field.visit_tokens(f),
+            Annotatable::FieldPat(fp) => fp.visit_tokens(f),
+            Annotatable::GenericParam(gp) => gp.visit_tokens(f),
+            Annotatable::Param(p) => p.visit_tokens(f),
+            Annotatable::StructField(sf) => sf.visit_tokens(f),
+            Annotatable::Variant(v) => v.visit_tokens(f),
+        }
+    }
 }
 
 impl Annotatable {

compiler/rustc_expand/src/build.rs

@@ -180,6 +180,7 @@ impl<'a> ExtCtxt<'a> {
             id: ast::DUMMY_NODE_ID,
             span: sp,
             attrs: AttrVec::new(),
+            tokens: None,
         });
         ast::Stmt {
             id: ast::DUMMY_NODE_ID,
@@ -198,6 +199,7 @@ impl<'a> ExtCtxt<'a> {
             id: ast::DUMMY_NODE_ID,
             span,
             attrs: AttrVec::new(),
+            tokens: None,
         });
         ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span, tokens: None }
     }

compiler/rustc_expand/src/config.rs

@@ -3,7 +3,11 @@
 use rustc_ast::attr::HasAttrs;
 use rustc_ast::mut_visit::*;
 use rustc_ast::ptr::P;
-use rustc_ast::{self as ast, AttrItem, Attribute, MetaItem};
+use rustc_ast::token::{self, DelimToken, Token};
+use rustc_ast::tokenstream::{
+    DelimSpan, PreexpTokenStream, PreexpTokenTree, Spacing, TokenStream, TokenTree,
+};
+use rustc_ast::{self as ast, Attribute, MetaItem};
 use rustc_attr as attr;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::map_in_place::MapInPlace;
@@ -12,6 +16,7 @@ use rustc_feature::{Feature, Features, State as FeatureState};
 use rustc_feature::{
     ACCEPTED_FEATURES, ACTIVE_FEATURES, REMOVED_FEATURES, STABLE_REMOVED_FEATURES,
 };
+use rustc_parse::parser::attr::CfgAttrItem;
 use rustc_parse::{parse_in, validate_attr};
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
@@ -237,9 +242,20 @@ const CFG_ATTR_NOTE_REF: &str = "for more information, visit \
     #the-cfg_attr-attribute>";
 
 impl<'a> StripUnconfigured<'a> {
-    pub fn configure<T: HasAttrs>(&mut self, mut node: T) -> Option<T> {
+    pub fn configure_with_tokens<T: HasAttrs>(
+        &mut self,
+        mut node: T,
+        config_tokens: bool,
+    ) -> Option<T> {
         self.process_cfg_attrs(&mut node);
-        self.in_cfg(node.attrs()).then_some(node)
+        self.in_cfg(node.attrs().iter()).then(|| {
+            node.visit_tokens(|tokens| *tokens = self.configure_tokens(tokens, config_tokens));
+            node
+        })
+    }
+
+    pub fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
+        self.configure_with_tokens(node, true)
     }
 
     /// Parse and expand all `cfg_attr` attributes into a list of attributes
@@ -249,9 +265,41 @@ impl<'a> StripUnconfigured<'a> {
     /// attributes and is in the original source code. Gives compiler errors if
     /// the syntax of any `cfg_attr` is incorrect.
     pub fn process_cfg_attrs<T: HasAttrs>(&mut self, node: &mut T) {
-        node.visit_attrs(|attrs| {
-            attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
-        });
+        node.visit_attrs(|attrs| attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr)));
+    }
+
+    fn configure_tokens(
+        &mut self,
+        stream: &PreexpTokenStream,
+        config_tokens: bool,
+    ) -> PreexpTokenStream {
+        let trees: Vec<_> = stream
+            .0
+            .iter()
+            .flat_map(|tree| match tree.0.clone() {
+                PreexpTokenTree::OuterAttributes(mut data) => {
+                    data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
+
+                    if self.in_cfg(data.attrs.iter()) {
+                        if config_tokens {
+                            data.tokens = self.configure_tokens(&data.tokens, config_tokens);
+                        }
+                        Some((PreexpTokenTree::OuterAttributes(data), tree.1)).into_iter()
+                    } else {
+                        None.into_iter()
+                    }
+                }
+                PreexpTokenTree::Delimited(sp, delim, mut inner) => {
+                    if config_tokens {
+                        inner = self.configure_tokens(&inner, config_tokens);
+                    }
+                    Some((PreexpTokenTree::Delimited(sp, delim, inner), tree.1))
+                }
+                .into_iter(),
+                token_tree @ PreexpTokenTree::Token(_) => Some((token_tree, tree.1)).into_iter(),
+            })
+            .collect();
+        PreexpTokenStream::new(trees)
     }
 
     /// Parse and expand a single `cfg_attr` attribute into a list of attributes
@@ -288,14 +336,28 @@ impl<'a> StripUnconfigured<'a> {
         //  `#[cfg_attr(false, cfg_attr(true, some_attr))]`.
         expanded_attrs
             .into_iter()
-            .flat_map(|(item, span)| {
-                let attr = attr::mk_attr_from_item(attr.style, item, span);
+            .flat_map(|item| {
+                let mut attr = attr::mk_attr_from_item(attr.style, item.item, item.span);
+                // Construct synthetic `#[]` tokens
+                let tokens = vec![
+                    (TokenTree::Token(Token::new(token::Pound, attr.span)), Spacing::Alone),
+                    (
+                        TokenTree::Delimited(
+                            DelimSpan::from_single(attr.span),
+                            DelimToken::Bracket,
+                            item.tokens,
+                        ),
+                        Spacing::Alone,
+                    ),
+                ];
+
+                attr.tokens = Some(TokenStream::new(tokens));
                 self.process_cfg_attr(attr)
             })
             .collect()
     }
 
-    fn parse_cfg_attr(&self, attr: &Attribute) -> Option<(MetaItem, Vec<(AttrItem, Span)>)> {
+    fn parse_cfg_attr(&mut self, attr: &Attribute) -> Option<(MetaItem, Vec<CfgAttrItem>)> {
         match attr.get_normal_item().args {
             ast::MacArgs::Delimited(dspan, delim, ref tts) if !tts.is_empty() => {
                 let msg = "wrong `cfg_attr` delimiters";
@@ -316,7 +378,7 @@ impl<'a> StripUnconfigured<'a> {
         None
     }
 
-    fn error_malformed_cfg_attr_missing(&self, span: Span) {
+    fn error_malformed_cfg_attr_missing(&mut self, span: Span) {
         self.sess
             .parse_sess
             .span_diagnostic
@@ -332,8 +394,8 @@ impl<'a> StripUnconfigured<'a> {
     }
 
     /// Determines if a node with the given attributes should be included in this configuration.
-    pub fn in_cfg(&self, attrs: &[Attribute]) -> bool {
-        attrs.iter().all(|attr| {
+    pub fn in_cfg<'b>(&self, mut attrs: impl Iterator<Item = &'b Attribute>) -> bool {
+        attrs.all(|attr| {
             if !is_cfg(self.sess, attr) {
                 return true;
             }

compiler/rustc_expand/src/expand.rs

@@ -494,6 +494,22 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     let mut item = self.fully_configure(item);
                     item.visit_attrs(|attrs| attrs.retain(|a| !a.has_name(sym::derive)));
 
+                    if item.derive_allowed() {
+                        if let Annotatable::Item(item) = &mut item {
+                            let tokens = if let Some(tokens) = item.tokens.as_mut() {
+                                tokens
+                            } else {
+                                panic!("Missing tokens for {:?}", item);
+                            };
+                            tokens.replace_attributes(|data| {
+                                data.attrs.retain(|a| !a.has_name(sym::derive))
+                            });
+                        } else {
+                            panic!("Derive on non-item {:?}", item);
+                        }
+                    }
+                    tracing::debug!("item after: {:?}", item);
+
                     let mut derive_placeholders = Vec::with_capacity(derives.len());
                     invocations.reserve(derives.len());
                     for path in derives {
@@ -764,7 +780,13 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                     if *mark_used {
                         self.cx.sess.mark_attr_used(&attr);
                     }
-                    item.visit_attrs(|attrs| attrs.push(attr));
+
+                    item.visit_attrs(|attrs| attrs.push(attr.clone()));
+
+                    item.visit_tokens(|tokens| {
+                        tokens.replace_attributes(|data| data.attrs.push(attr));
+                    });
+
                     fragment_kind.expect_from_annotatables(iter::once(item))
                 }
                 _ => unreachable!(),
@@ -1053,19 +1075,26 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
 
     fn find_attr_invoc(
         &self,
-        attrs: &mut Vec<ast::Attribute>,
+        attr_target: &mut (impl HasAttrs + std::fmt::Debug),
         after_derive: &mut bool,
     ) -> Option<ast::Attribute> {
-        let attr = attrs
-            .iter()
-            .position(|a| {
-                if a.has_name(sym::derive) {
-                    *after_derive = true;
-                }
-                !self.cx.sess.is_attr_known(a) && !is_builtin_attr(a)
-            })
-            .map(|i| attrs.remove(i));
+        let mut attr = None;
+
+        attr_target.visit_attrs(|attrs| {
+            attrs
+                .iter()
+                .position(|a| {
+                    if a.has_name(sym::derive) {
+                        *after_derive = true;
+                    }
+                    !self.cx.sess.is_attr_known(a) && !is_builtin_attr(a)
+                })
+                .map(|i| attr = Some(attrs.remove(i)));
+        });
+
+        let mut has_inner = false;
         if let Some(attr) = &attr {
+            has_inner = attr.style == ast::AttrStyle::Inner;
             if !self.cx.ecfg.custom_inner_attributes()
                 && attr.style == ast::AttrStyle::Inner
                 && !attr.has_name(sym::test)
@@ -1079,18 +1108,56 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                 .emit();
             }
         }
+
+        let mut token_attr = None;
+        // Only attempt to replace tokens if we actually
+        // removed an attribute from the AST struct.
+        // We may end up collecting tokens for an AST struct
+        // even if `attrs_require_tokens` returns `false`, if
+        // a macro-rules matcher ends up forcing token collection.
+        // In this case, our collected tokens will not be a single
+        // `PreexpTokenStream::OuterAttributes`.
+        //
+        // To handle this case, we only attempt to modify the `TokenStream`
+        // if we actually need to do so - that is, if we need to remove
+        // a corresponding attribute that was removed from the parsed
+        // AST struct.
+        //
+        // FIXME: Support inner attributes.
+        // For now, we don't attempt to modify the TokenStream, which will
+        // cause us to use the pretty-print/retokenized stream later
+        // on due to the mismatch.
+        if attr.is_some() && !has_inner {
+            attr_target.visit_tokens(|tokens| {
+                tokens.replace_attributes(|data| {
+                    token_attr = data
+                        .attrs
+                        .iter()
+                        .position(|a| !self.cx.sess.is_attr_known(a) && !is_builtin_attr(a))
+                        .map(|i| data.attrs.remove(i));
+
+                    if token_attr.is_some() != attr.is_some() {
+                        panic!(
+                            "Mismatched AST and tokens: ast={:?} token_attr={:?}\ndata={:?}",
+                            attr, token_attr, data
+                        );
+                    }
+                });
+            });
+        }
+
         attr
     }
 
     /// If `item` is an attr invocation, remove and return the macro attribute and derive traits.
     fn classify_item(
         &mut self,
-        item: &mut impl HasAttrs,
+        item: &mut (impl HasAttrs + std::fmt::Debug),
     ) -> (Option<ast::Attribute>, Vec<Path>, /* after_derive */ bool) {
-        let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false);
+        let (mut traits, mut after_derive) = (Vec::new(), false);
 
+        let attr = self.find_attr_invoc(item, &mut after_derive);
         item.visit_attrs(|mut attrs| {
-            attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
             traits = collect_derives(&mut self.cx, &mut attrs);
         });
 
@@ -1102,19 +1169,17 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
     /// is a breaking change)
     fn classify_nonitem(
         &mut self,
-        nonitem: &mut impl HasAttrs,
+        nonitem: &mut (impl HasAttrs + std::fmt::Debug),
     ) -> (Option<ast::Attribute>, /* after_derive */ bool) {
-        let (mut attr, mut after_derive) = (None, false);
+        let mut after_derive = false;
 
-        nonitem.visit_attrs(|mut attrs| {
-            attr = self.find_attr_invoc(&mut attrs, &mut after_derive);
-        });
+        let attr = self.find_attr_invoc(nonitem, &mut after_derive);
 
         (attr, after_derive)
     }
 
     fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
-        self.cfg.configure(node)
+        self.cfg.configure_with_tokens(node, false)
     }
 
     // Detect use of feature-gated or invalid attributes on macro invocations
@@ -1357,7 +1422,8 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
         // we'll expand attributes on expressions separately
         if !stmt.is_expr() {
             let (attr, derives, after_derive) = if stmt.is_item() {
-                self.classify_item(&mut stmt)
+                (None, vec![], false)
+            //self.classify_item(&mut stmt)
             } else {
                 // ignore derives on non-item statements so it falls through
                 // to the unused-attributes lint
@@ -1379,7 +1445,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
         }
 
         if let StmtKind::MacCall(mac) = stmt.kind {
-            let MacCallStmt { mac, style, attrs } = mac.into_inner();
+            let MacCallStmt { mac, style, attrs, tokens: _ } = mac.into_inner();
             self.check_attributes(&attrs);
             let mut placeholder =
                 self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts).make_stmts();
@@ -1785,6 +1851,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                 span: at.span,
                 id: at.id,
                 style: at.style,
+                tokens: None,
             };
         } else {
             noop_visit_attribute(at, self)

compiler/rustc_expand/src/lib.rs

@@ -6,6 +6,7 @@
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_span)]
 #![feature(try_blocks)]
+#![recursion_limit = "256"]
 
 #[macro_use]
 extern crate rustc_macros;

compiler/rustc_expand/src/placeholders.rs

@@ -104,6 +104,7 @@ pub fn placeholder(
                 mac: mac_placeholder(),
                 style: ast::MacStmtStyle::Braces,
                 attrs: ast::AttrVec::new(),
+                tokens: None,
             });
             ast::Stmt { id, span, kind: ast::StmtKind::MacCall(mac), tokens: None }
         }]),

compiler/rustc_middle/src/ich/impls_syntax.rs

@@ -40,7 +40,7 @@ impl<'ctx> rustc_ast::HashStableContext for StableHashingContext<'ctx> {
         debug_assert!(!attr.ident().map_or(false, |ident| self.is_ignored_attr(ident.name)));
         debug_assert!(!attr.is_doc_comment());
 
-        let ast::Attribute { kind, id: _, style, span } = attr;
+        let ast::Attribute { kind, id: _, style, span, tokens: _ } = attr;
         if let ast::AttrKind::Normal(item) = kind {
             item.hash_stable(self, hasher);
             style.hash_stable(self, hasher);

compiler/rustc_parse/src/lib.rs

@@ -266,7 +266,8 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
     // before we fall back to the stringification.
     let tokens = match *nt {
         Nonterminal::NtItem(ref item) => {
-            prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
+            debug!("converting item tokenstream: {:?}", item.tokens);
+            item.tokens.clone().map(|t| t.to_tokenstream())
         }
         Nonterminal::NtBlock(ref block) => block.tokens.clone(),
         Nonterminal::NtStmt(ref stmt) => {
@@ -291,8 +292,10 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
         Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
             if expr.tokens.is_none() {
                 debug!("missing tokens for expr {:?}", expr);
+            } else {
+                debug!("converting expr tokenstream: {:?}", expr.tokens);
             }
-            prepend_attrs(sess, &expr.attrs, expr.tokens.as_ref(), span)
+            expr.tokens.clone().map(|t| t.to_tokenstream())
         }
     };
 
@@ -326,12 +329,16 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
     // tokens such as extra braces and commas, don't happen.
     if let Some(tokens) = tokens {
         if tokenstream_probably_equal_for_proc_macro(&tokens, &tokens_for_real, sess) {
+            debug!("matching tokens: {:?}", tokens);
+            debug!("reparsed tokens: {:?}", tokens_for_real);
             return tokens;
         }
         info!(
             "cached tokens found, but they're not \"probably equal\", \
                 going with stringified version"
         );
+        info!("cached stringified:   {:?}", pprust::tts_to_string(&tokens));
+        info!("reparsed stringified: {:?}", pprust::tts_to_string(&tokens_for_real));
         info!("cached tokens: {:?}", tokens);
         info!("reparsed tokens: {:?}", tokens_for_real);
     }
@@ -541,66 +548,3 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
         _ => panic!("forgot to add a token?"),
     }
 }
-
-fn prepend_attrs(
-    sess: &ParseSess,
-    attrs: &[ast::Attribute],
-    tokens: Option<&tokenstream::TokenStream>,
-    span: rustc_span::Span,
-) -> Option<tokenstream::TokenStream> {
-    let tokens = tokens?;
-    if attrs.is_empty() {
-        return Some(tokens.clone());
-    }
-    let mut builder = tokenstream::TokenStreamBuilder::new();
-    for attr in attrs {
-        assert_eq!(
-            attr.style,
-            ast::AttrStyle::Outer,
-            "inner attributes should prevent cached tokens from existing"
-        );
-
-        let source = pprust::attribute_to_string(attr);
-        let macro_filename = FileName::macro_expansion_source_code(&source);
-
-        let item = match attr.kind {
-            ast::AttrKind::Normal(ref item) => item,
-            ast::AttrKind::DocComment(..) => {
-                let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span));
-                builder.push(stream);
-                continue;
-            }
-        };
-
-        // synthesize # [ $path $tokens ] manually here
-        let mut brackets = tokenstream::TokenStreamBuilder::new();
-
-        // For simple paths, push the identifier directly
-        if item.path.segments.len() == 1 && item.path.segments[0].args.is_none() {
-            let ident = item.path.segments[0].ident;
-            let token = token::Ident(ident.name, ident.as_str().starts_with("r#"));
-            brackets.push(tokenstream::TokenTree::token(token, ident.span));
-
-        // ... and for more complicated paths, fall back to a reparse hack that
-        // should eventually be removed.
-        } else {
-            let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span));
-            brackets.push(stream);
-        }
-
-        brackets.push(item.args.outer_tokens());
-
-        // The span we list here for `#` and for `[ ... ]` are both wrong in
-        // that it encompasses more than each token, but it hopefully is "good
-        // enough" for now at least.
-        builder.push(tokenstream::TokenTree::token(token::Pound, attr.span));
-        let delim_span = tokenstream::DelimSpan::from_single(attr.span);
-        builder.push(tokenstream::TokenTree::Delimited(
-            delim_span,
-            token::DelimToken::Bracket,
-            brackets.build(),
-        ));
-    }
-    builder.push(tokens.clone());
-    Some(builder.build())
-}

compiler/rustc_parse/src/parser/generics.rs

@@ -73,67 +73,88 @@ impl<'a> Parser<'a> {
     pub(super) fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> {
         let mut params = Vec::new();
         loop {
-            let attrs = self.parse_outer_attributes()?;
-            if self.check_lifetime() {
-                let lifetime = self.expect_lifetime();
-                // Parse lifetime parameter.
-                let bounds =
-                    if self.eat(&token::Colon) { self.parse_lt_param_bounds() } else { Vec::new() };
-                params.push(ast::GenericParam {
-                    ident: lifetime.ident,
-                    id: lifetime.id,
-                    attrs: attrs.into(),
-                    bounds,
-                    kind: ast::GenericParamKind::Lifetime,
-                    is_placeholder: false,
-                });
-            } else if self.check_keyword(kw::Const) {
-                // Parse const parameter.
-                params.push(self.parse_const_param(attrs)?);
-            } else if self.check_ident() {
-                // Parse type parameter.
-                params.push(self.parse_ty_param(attrs)?);
-            } else if self.token.can_begin_type() {
-                // Trying to write an associated type bound? (#26271)
-                let snapshot = self.clone();
-                match self.parse_ty_where_predicate() {
-                    Ok(where_predicate) => {
-                        self.struct_span_err(
-                            where_predicate.span(),
-                            "bounds on associated types do not belong here",
-                        )
-                        .span_label(where_predicate.span(), "belongs in `where` clause")
-                        .emit();
-                    }
-                    Err(mut err) => {
-                        err.cancel();
-                        *self = snapshot;
-                        break;
-                    }
-                }
-            } else {
-                // Check for trailing attributes and stop parsing.
-                if !attrs.is_empty() {
-                    if !params.is_empty() {
-                        self.struct_span_err(
-                            attrs[0].span,
-                            "trailing attribute after generic parameter",
-                        )
-                        .span_label(attrs[0].span, "attributes must go before parameters")
-                        .emit();
+            let mut should_break = false;
+            let param = self.parse_outer_attributes(|this, attrs| {
+                let param = if this.check_lifetime() {
+                    let lifetime = this.expect_lifetime();
+                    // Parse lifetime parameter.
+                    let bounds = if this.eat(&token::Colon) {
+                        this.parse_lt_param_bounds()
                     } else {
-                        self.struct_span_err(attrs[0].span, "attribute without generic parameters")
+                        Vec::new()
+                    };
+                    Some(ast::GenericParam {
+                        ident: lifetime.ident,
+                        id: lifetime.id,
+                        attrs: attrs.into(),
+                        bounds,
+                        kind: ast::GenericParamKind::Lifetime,
+                        is_placeholder: false,
+                    })
+                } else if this.check_keyword(kw::Const) {
+                    // Parse const parameter.
+                    Some(this.parse_const_param(attrs)?)
+                } else if this.check_ident() {
+                    // Parse type parameter.
+                    Some(this.parse_ty_param(attrs)?)
+                } else if this.token.can_begin_type() {
+                    // Trying to write an associated type bound? (#26271)
+                    let snapshot = this.clone();
+                    match this.parse_ty_where_predicate() {
+                        Ok(where_predicate) => {
+                            this.struct_span_err(
+                                where_predicate.span(),
+                                "bounds on associated types do not belong here",
+                            )
+                            .span_label(where_predicate.span(), "belongs in `where` clause")
+                            .emit();
+
+                            None
+                        }
+                        Err(mut err) => {
+                            err.cancel();
+                            *this = snapshot;
+                            None
+                        }
+                    }
+                } else {
+                    // Check for trailing attributes and stop parsing.
+                    if !attrs.is_empty() {
+                        if !params.is_empty() {
+                            this.struct_span_err(
+                                attrs[0].span,
+                                "trailing attribute after generic parameter",
+                            )
+                            .span_label(attrs[0].span, "attributes must go before parameters")
+                            .emit();
+                        } else {
+                            this.struct_span_err(
+                                attrs[0].span,
+                                "attribute without generic parameters",
+                            )
                             .span_label(
                                 attrs[0].span,
                                 "attributes are only permitted when preceding parameters",
                             )
                             .emit();
+                        }
+                    }
+                    None
+                };
+                if param.is_some() {
+                    if !this.eat(&token::Comma) {
+                        should_break = true;
                     }
                 }
-                break;
-            }
+                Ok(param)
+            })?;
 
-            if !self.eat(&token::Comma) {
+            if let Some(param) = param {
+                params.push(param);
+                if should_break {
+                    break;
+                }
+            } else {
                 break;
             }
         }

compiler/rustc_parse/src/parser/mod.rs

@@ -15,9 +15,12 @@ pub use path::PathStyle;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, DelimToken, Token, TokenKind};
-use rustc_ast::tokenstream::{self, DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
+use rustc_ast::tokenstream::{
+    self, AttributesData, DelimSpan, PreexpTokenStream, PreexpTokenTree, Spacing, TokenStream,
+    TokenTree,
+};
 use rustc_ast::DUMMY_NODE_ID;
-use rustc_ast::{self as ast, AttrStyle, AttrVec, Const, CrateSugar, Extern, Unsafe};
+use rustc_ast::{self as ast, AttrStyle, Attribute, Const, CrateSugar, Extern, Unsafe};
 use rustc_ast::{Async, MacArgs, MacDelimiter, Mutability, StrLit, Visibility, VisibilityKind};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError, PResult};
@@ -106,6 +109,8 @@ pub struct Parser<'a> {
     pub last_type_ascription: Option<(Span, bool /* likely path typo */)>,
     /// If present, this `Parser` is not parsing Rust code but rather a macro call.
     subparser_name: Option<&'static str>,
+    pub is_rustdoc: bool,
+    in_derive: bool,
 }
 
 impl<'a> Drop for Parser<'a> {
@@ -118,8 +123,8 @@ impl<'a> Drop for Parser<'a> {
 struct TokenCursor {
     frame: TokenCursorFrame,
     stack: Vec<TokenCursorFrame>,
-    cur_token: Option<TreeAndSpacing>,
-    collecting: Option<Collecting>,
+    collecting_buf: Vec<(FlatToken, Spacing)>,
+    is_collecting: bool,
 }
 
 #[derive(Clone)]
@@ -129,24 +134,13 @@ struct TokenCursorFrame {
     open_delim: bool,
     tree_cursor: tokenstream::Cursor,
     close_delim: bool,
+    modified_stream: Vec<(PreexpTokenTree, Spacing)>,
 }
 
-/// Used to track additional state needed by `collect_tokens`
-#[derive(Clone, Debug)]
-struct Collecting {
-    /// Holds the current tokens captured during the most
-    /// recent call to `collect_tokens`
-    buf: Vec<TreeAndSpacing>,
-    /// The depth of the `TokenCursor` stack at the time
-    /// collection was started. When we encounter a `TokenTree::Delimited`,
-    /// we want to record the `TokenTree::Delimited` itself,
-    /// but *not* any of the inner tokens while we are inside
-    /// the new frame (this would cause us to record duplicate tokens).
-    ///
-    /// This `depth` fields tracks stack depth we are recording tokens.
-    /// Only tokens encountered at this depth will be recorded. See
-    /// `TokenCursor::next` for more details.
-    depth: usize,
+#[derive(Clone)]
+enum FlatToken {
+    Token(Token),
+    OuterAttributes(AttributesData),
 }
 
 impl TokenCursorFrame {
@@ -157,6 +151,7 @@ impl TokenCursorFrame {
             open_delim: delim == token::NoDelim,
             tree_cursor: tts.clone().into_trees(),
             close_delim: delim == token::NoDelim,
+            modified_stream: vec![],
         }
     }
 }
@@ -176,29 +171,19 @@ impl TokenCursor {
                 self.frame = frame;
                 continue;
             } else {
-                return Token::new(token::Eof, DUMMY_SP);
+                (TokenTree::Token(Token::new(token::Eof, DUMMY_SP)), Spacing::Alone)
             };
 
-            // Don't set an open delimiter as our current token - we want
-            // to leave it as the full `TokenTree::Delimited` from the previous
-            // iteration of this loop
-            if !matches!(tree.0, TokenTree::Token(Token { kind: TokenKind::OpenDelim(_), .. })) {
-                self.cur_token = Some(tree.clone());
-            }
-
-            if let Some(collecting) = &mut self.collecting {
-                if collecting.depth == self.stack.len() {
-                    debug!(
-                        "TokenCursor::next():  collected {:?} at depth {:?}",
-                        tree,
-                        self.stack.len()
-                    );
-                    collecting.buf.push(tree.clone())
+            match tree.0.clone() {
+                TokenTree::Token(token) => {
+                    if self.is_collecting {
+                        self.collecting_buf.push((FlatToken::Token(token.clone()), tree.1));
+                    } else {
+                        self.collecting_buf[0] = (FlatToken::Token(token.clone()), tree.1);
+                        debug_assert_eq!(self.collecting_buf.len(), 1);
+                    }
+                    return token;
                 }
-            }
-
-            match tree.0 {
-                TokenTree::Token(token) => return token,
                 TokenTree::Delimited(sp, delim, tts) => {
                     let frame = TokenCursorFrame::new(sp, delim, &tts);
                     self.stack.push(mem::replace(&mut self.frame, frame));
@@ -355,8 +340,12 @@ impl<'a> Parser<'a> {
             token_cursor: TokenCursor {
                 frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens),
                 stack: Vec::new(),
-                cur_token: None,
-                collecting: None,
+                // Will get overwritten when we bump the parser below
+                collecting_buf: vec![(
+                    FlatToken::Token(Token::new(token::Eof, DUMMY_SP)),
+                    Spacing::Alone,
+                )],
+                is_collecting: false,
             },
             desugar_doc_comments,
             unmatched_angle_bracket_count: 0,
@@ -365,6 +354,8 @@ impl<'a> Parser<'a> {
             last_unexpected_token_span: None,
             last_type_ascription: None,
             subparser_name,
+            is_rustdoc: false,
+            in_derive: false,
         };
 
         // Make parser point to the first token.
@@ -947,28 +938,23 @@ impl<'a> Parser<'a> {
         )
     }
 
-    fn parse_or_use_outer_attributes(
-        &mut self,
-        already_parsed_attrs: Option<AttrVec>,
-    ) -> PResult<'a, AttrVec> {
-        if let Some(attrs) = already_parsed_attrs {
-            Ok(attrs)
-        } else {
-            self.parse_outer_attributes().map(|a| a.into())
-        }
-    }
-
     /// Parses a single token tree from the input.
     pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
         match self.token.kind {
             token::OpenDelim(..) => {
-                let frame = mem::replace(
-                    &mut self.token_cursor.frame,
-                    self.token_cursor.stack.pop().unwrap(),
-                );
-                self.token = Token::new(TokenKind::CloseDelim(frame.delim), frame.span.close);
+                let depth = self.token_cursor.stack.len();
+                while !(depth == self.token_cursor.stack.len()
+                    && matches!(self.token.kind, token::CloseDelim(_)))
+                {
+                    self.bump();
+                }
+                let frame = &self.token_cursor.frame;
+                let stream = frame.tree_cursor.stream.clone();
+                let span = frame.span;
+                let delim = frame.delim;
+                // Consume close delimiter
                 self.bump();
-                TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream)
+                TokenTree::Delimited(span, delim, stream)
             }
             token::CloseDelim(_) | token::Eof => unreachable!(),
             _ => {
@@ -1157,6 +1143,13 @@ impl<'a> Parser<'a> {
         }
     }
 
+    pub fn collect_tokens_no_attrs<R>(
+        &mut self,
+        f: impl FnOnce(&mut Self) -> PResult<'a, R>,
+    ) -> PResult<'a, (R, PreexpTokenStream)> {
+        self.collect_tokens(|this| f(this).map(|res| (res, Vec::new())))
+    }
+
     /// Records all tokens consumed by the provided callback,
     /// including the current token. These tokens are collected
     /// into a `TokenStream`, and returned along with the result
@@ -1175,80 +1168,9 @@ impl<'a> Parser<'a> {
     /// a parsed AST item, which always has matching delimiters.
     pub fn collect_tokens<R>(
         &mut self,
-        f: impl FnOnce(&mut Self) -> PResult<'a, R>,
-    ) -> PResult<'a, (R, TokenStream)> {
-        // Record all tokens we parse when parsing this item.
-        let tokens: Vec<TreeAndSpacing> = self.token_cursor.cur_token.clone().into_iter().collect();
-        debug!("collect_tokens: starting with {:?}", tokens);
-
-        // We need special handling for the case where `collect_tokens` is called
-        // on an opening delimeter (e.g. '('). At this point, we have already pushed
-        // a new frame - however, we want to record the original `TokenTree::Delimited`,
-        // for consistency with the case where we start recording one token earlier.
-        // See `TokenCursor::next` to see how `cur_token` is set up.
-        let prev_depth =
-            if matches!(self.token_cursor.cur_token, Some((TokenTree::Delimited(..), _))) {
-                if self.token_cursor.stack.is_empty() {
-                    // There is nothing below us in the stack that
-                    // the function could consume, so the only thing it can legally
-                    // capture is the entire contents of the current frame.
-                    return Ok((f(self)?, TokenStream::new(tokens)));
-                }
-                // We have already recorded the full `TokenTree::Delimited` when we created
-                // our `tokens` vector at the start of this function. We are now inside
-                // a new frame corresponding to the `TokenTree::Delimited` we already recoreded.
-                // We don't want to record any of the tokens inside this frame, since they
-                // will be duplicates of the tokens nested inside the `TokenTree::Delimited`.
-                // Therefore, we set our recording depth to the *previous* frame. This allows
-                // us to record a sequence like: `(foo).bar()`: the `(foo)` will be recored
-                // as our initial `cur_token`, while the `.bar()` will be recored after we
-                // pop the `(foo)` frame.
-                self.token_cursor.stack.len() - 1
-            } else {
-                self.token_cursor.stack.len()
-            };
-        let prev_collecting =
-            self.token_cursor.collecting.replace(Collecting { buf: tokens, depth: prev_depth });
-
-        let ret = f(self);
-
-        let mut collected_tokens = if let Some(collecting) = self.token_cursor.collecting.take() {
-            collecting.buf
-        } else {
-            let msg = "our vector went away?";
-            debug!("collect_tokens: {}", msg);
-            self.sess.span_diagnostic.delay_span_bug(self.token.span, &msg);
-            // This can happen due to a bad interaction of two unrelated recovery mechanisms
-            // with mismatched delimiters *and* recovery lookahead on the likely typo
-            // `pub ident(` (#62895, different but similar to the case above).
-            return Ok((ret?, TokenStream::default()));
-        };
-
-        debug!("collect_tokens: got raw tokens {:?}", collected_tokens);
-
-        // If we're not at EOF our current token wasn't actually consumed by
-        // `f`, but it'll still be in our list that we pulled out. In that case
-        // put it back.
-        let extra_token = if self.token != token::Eof { collected_tokens.pop() } else { None };
-
-        if let Some(mut collecting) = prev_collecting {
-            // If we were previously collecting at the same depth,
-            // then the previous call to `collect_tokens` needs to see
-            // the tokens we just recorded.
-            //
-            // If we were previously recording at an lower `depth`,
-            // then the previous `collect_tokens` call already recorded
-            // this entire frame in the form of a `TokenTree::Delimited`,
-            // so there is nothing else for us to do.
-            if collecting.depth == prev_depth {
-                collecting.buf.extend(collected_tokens.iter().cloned());
-                collecting.buf.extend(extra_token);
-                debug!("collect_tokens: updating previous buf to {:?}", collecting);
-            }
-            self.token_cursor.collecting = Some(collecting)
-        }
-
-        Ok((ret?, TokenStream::new(collected_tokens)))
+        f: impl FnOnce(&mut Self) -> PResult<'a, (R, Vec<Attribute>)>,
+    ) -> PResult<'a, (R, PreexpTokenStream)> {
+        self.collect_tokens_keep_in_stream(true, f)
     }
 
     /// `::{` or `::*`

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -98,33 +98,38 @@ impl<'a> Parser<'a> {
         // in advance whether or not a proc-macro will be (transitively) invoked,
         // we always capture tokens for any `Nonterminal` which needs them.
         Ok(match kind {
-            NonterminalKind::Item => match self.collect_tokens(|this| this.parse_item())? {
-                (Some(mut item), tokens) => {
-                    // If we captured tokens during parsing (due to outer attributes),
-                    // use those.
-                    if item.tokens.is_none() {
-                        item.tokens = Some(tokens);
+            NonterminalKind::Item => {
+                match self.collect_tokens_no_attrs(|this| this.parse_item())? {
+                    (Some(mut item), tokens) => {
+                        // If we captured tokens during parsing (due to outer attributes),
+                        // use those.
+                        if item.tokens.is_none() {
+                            item.tokens = Some(tokens);
+                        }
+                        token::NtItem(item)
+                    }
+                    (None, _) => {
+                        return Err(
+                            self.struct_span_err(self.token.span, "expected an item keyword")
+                        );
                     }
-                    token::NtItem(item)
-                }
-                (None, _) => {
-                    return Err(self.struct_span_err(self.token.span, "expected an item keyword"));
                 }
-            },
+            }
             NonterminalKind::Block => {
-                let (mut block, tokens) = self.collect_tokens(|this| this.parse_block())?;
+                let (mut block, tokens) =
+                    self.collect_tokens_no_attrs(|this| this.parse_block())?;
                 // We have have eaten an NtBlock, which could already have tokens
                 if block.tokens.is_none() {
-                    block.tokens = Some(tokens);
+                    block.tokens = Some(tokens.to_tokenstream());
                 }
                 token::NtBlock(block)
             }
             NonterminalKind::Stmt => {
-                let (stmt, tokens) = self.collect_tokens(|this| this.parse_stmt())?;
+                let (stmt, tokens) = self.collect_tokens_no_attrs(|this| this.parse_stmt())?;
                 match stmt {
                     Some(mut s) => {
                         if s.tokens.is_none() {
-                            s.tokens = Some(tokens);
+                            s.tokens = Some(tokens.to_tokenstream());
                         }
                         token::NtStmt(s)
                     }
@@ -134,15 +139,16 @@ impl<'a> Parser<'a> {
                 }
             }
             NonterminalKind::Pat => {
-                let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
+                let (mut pat, tokens) =
+                    self.collect_tokens_no_attrs(|this| this.parse_pat(None))?;
                 // We have have eaten an NtPat, which could already have tokens
                 if pat.tokens.is_none() {
-                    pat.tokens = Some(tokens);
+                    pat.tokens = Some(tokens.to_tokenstream());
                 }
                 token::NtPat(pat)
             }
             NonterminalKind::Expr => {
-                let (mut expr, tokens) = self.collect_tokens(|this| this.parse_expr())?;
+                let (mut expr, tokens) = self.collect_tokens_no_attrs(|this| this.parse_expr())?;
                 // If we captured tokens during parsing (due to outer attributes),
                 // use those.
                 if expr.tokens.is_none() {
@@ -152,18 +158,18 @@ impl<'a> Parser<'a> {
             }
             NonterminalKind::Literal => {
                 let (mut lit, tokens) =
-                    self.collect_tokens(|this| this.parse_literal_maybe_minus())?;
+                    self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?;
                 // We have have eaten a nonterminal, which  could already have tokens
                 if lit.tokens.is_none() {
                     lit.tokens = Some(tokens);
                 }
                 token::NtLiteral(lit)
             }
             NonterminalKind::Ty => {
-                let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?;
+                let (mut ty, tokens) = self.collect_tokens_no_attrs(|this| this.parse_ty())?;
                 // We have an eaten an NtTy, which could already have tokens
                 if ty.tokens.is_none() {
-                    ty.tokens = Some(tokens);
+                    ty.tokens = Some(tokens.to_tokenstream());
                 }
                 token::NtTy(ty)
             }
@@ -180,28 +186,29 @@ impl<'a> Parser<'a> {
             }
             NonterminalKind::Path => {
                 let (mut path, tokens) =
-                    self.collect_tokens(|this| this.parse_path(PathStyle::Type))?;
+                    self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?;
                 // We have have eaten an NtPath, which could already have tokens
                 if path.tokens.is_none() {
-                    path.tokens = Some(tokens);
+                    path.tokens = Some(tokens.to_tokenstream());
                 }
                 token::NtPath(path)
             }
             NonterminalKind::Meta => {
-                let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?;
+                let (mut attr, tokens) =
+                    self.collect_tokens_no_attrs(|this| this.parse_attr_item())?;
                 // We may have eaten a nonterminal, which could already have tokens
                 if attr.tokens.is_none() {
-                    attr.tokens = Some(tokens);
+                    attr.tokens = Some(tokens.to_tokenstream());
                 }
                 token::NtMeta(P(attr))
             }
             NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
             NonterminalKind::Vis => {
-                let (mut vis, tokens) =
-                    self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?;
+                let (mut vis, tokens) = self
+                    .collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?;
                 // We may have etan an `NtVis`, which could already have tokens
                 if vis.tokens.is_none() {
-                    vis.tokens = Some(tokens);
+                    vis.tokens = Some(tokens.to_tokenstream());
                 }
                 token::NtVis(vis)
             }

compiler/rustc_parse/src/parser/pat.rs

@@ -831,102 +831,107 @@ impl<'a> Parser<'a> {
         let mut etc_span = None;
 
         while self.token != token::CloseDelim(token::Brace) {
-            let attrs = match self.parse_outer_attributes() {
-                Ok(attrs) => attrs,
-                Err(err) => {
-                    if let Some(mut delayed) = delayed_err {
+            let field_pat = self.parse_outer_attributes(|this, attrs| {
+                let lo = this.token.span;
+
+                // check that a comma comes after every field
+                if !ate_comma {
+                    let err = this.struct_span_err(this.prev_token.span, "expected `,`");
+                    if let Some(delayed) = delayed_err.as_mut() {
                         delayed.emit();
                     }
                     return Err(err);
                 }
-            };
-            let lo = self.token.span;
-
-            // check that a comma comes after every field
-            if !ate_comma {
-                let err = self.struct_span_err(self.prev_token.span, "expected `,`");
-                if let Some(mut delayed) = delayed_err {
-                    delayed.emit();
-                }
-                return Err(err);
-            }
-            ate_comma = false;
+                ate_comma = false;
 
-            if self.check(&token::DotDot) || self.token == token::DotDotDot {
-                etc = true;
-                let mut etc_sp = self.token.span;
+                if this.check(&token::DotDot) || this.token == token::DotDotDot {
+                    etc = true;
+                    let mut etc_sp = this.token.span;
 
-                self.recover_one_fewer_dotdot();
-                self.bump(); // `..` || `...`
+                    this.recover_one_fewer_dotdot();
+                    this.bump(); // `..` || `...`
 
-                if self.token == token::CloseDelim(token::Brace) {
-                    etc_span = Some(etc_sp);
-                    break;
-                }
-                let token_str = super::token_descr(&self.token);
-                let msg = &format!("expected `}}`, found {}", token_str);
-                let mut err = self.struct_span_err(self.token.span, msg);
-
-                err.span_label(self.token.span, "expected `}`");
-                let mut comma_sp = None;
-                if self.token == token::Comma {
-                    // Issue #49257
-                    let nw_span = self.sess.source_map().span_until_non_whitespace(self.token.span);
-                    etc_sp = etc_sp.to(nw_span);
-                    err.span_label(
-                        etc_sp,
-                        "`..` must be at the end and cannot have a trailing comma",
-                    );
-                    comma_sp = Some(self.token.span);
-                    self.bump();
-                    ate_comma = true;
-                }
-
-                etc_span = Some(etc_sp.until(self.token.span));
-                if self.token == token::CloseDelim(token::Brace) {
-                    // If the struct looks otherwise well formed, recover and continue.
-                    if let Some(sp) = comma_sp {
-                        err.span_suggestion_short(
-                            sp,
-                            "remove this comma",
-                            String::new(),
-                            Applicability::MachineApplicable,
-                        );
+                    if this.token == token::CloseDelim(token::Brace) {
+                        etc_span = Some(etc_sp);
+                        return Ok(None);
                     }
-                    err.emit();
-                    break;
-                } else if self.token.is_ident() && ate_comma {
-                    // Accept fields coming after `..,`.
-                    // This way we avoid "pattern missing fields" errors afterwards.
-                    // We delay this error until the end in order to have a span for a
-                    // suggested fix.
-                    if let Some(mut delayed_err) = delayed_err {
-                        delayed_err.emit();
-                        return Err(err);
-                    } else {
-                        delayed_err = Some(err);
+                    let token_str = super::token_descr(&this.token);
+                    let msg = &format!("expected `}}`, found {}", token_str);
+                    let mut err = this.struct_span_err(this.token.span, msg);
+
+                    err.span_label(this.token.span, "expected `}`");
+                    let mut comma_sp = None;
+                    if this.token == token::Comma {
+                        // Issue #49257
+                        let nw_span =
+                            this.sess.source_map().span_until_non_whitespace(this.token.span);
+                        etc_sp = etc_sp.to(nw_span);
+                        err.span_label(
+                            etc_sp,
+                            "`..` must be at the end and cannot have a trailing comma",
+                        );
+                        comma_sp = Some(this.token.span);
+                        this.bump();
+                        ate_comma = true;
                     }
-                } else {
-                    if let Some(mut err) = delayed_err {
+
+                    etc_span = Some(etc_sp.until(this.token.span));
+                    if this.token == token::CloseDelim(token::Brace) {
+                        // If the struct looks otherwise well formed, recover and continue.
+                        if let Some(sp) = comma_sp {
+                            err.span_suggestion_short(
+                                sp,
+                                "remove this comma",
+                                String::new(),
+                                Applicability::MachineApplicable,
+                            );
+                        }
                         err.emit();
+                        return Ok(None);
+                    } else if this.token.is_ident() && ate_comma {
+                        // Accept fields coming after `..,`.
+                        // This way we avoid "pattern missing fields" errors afterwards.
+                        // We delay this error until the end in order to have a span for a
+                        // suggested fix.
+                        if let Some(delayed_err) = delayed_err.as_mut() {
+                            delayed_err.emit();
+                            return Err(err);
+                        } else {
+                            delayed_err = Some(err);
+                        }
+                    } else {
+                        if let Some(err) = delayed_err.as_mut() {
+                            err.emit();
+                        }
+                        return Err(err);
                     }
-                    return Err(err);
                 }
-            }
 
-            fields.push(match self.parse_pat_field(lo, attrs) {
-                Ok(field) => field,
+                let field_pat = match this.parse_pat_field(lo, attrs) {
+                    Ok(field) => field,
+                    Err(err) => {
+                        if let Some(delayed_err) = delayed_err.as_mut() {
+                            delayed_err.emit();
+                        }
+                        return Err(err);
+                    }
+                };
+                ate_comma = this.eat(&token::Comma);
+                Ok(Some(field_pat))
+            });
+            match field_pat {
+                Ok(Some(field_pat)) => fields.push(field_pat),
+                Ok(None) => break,
                 Err(err) => {
-                    if let Some(mut delayed_err) = delayed_err {
-                        delayed_err.emit();
+                    if let Some(delayed) = delayed_err.as_mut() {
+                        delayed.emit();
                     }
                     return Err(err);
                 }
-            });
-            ate_comma = self.eat(&token::Comma);
+            }
         }
 
-        if let Some(mut err) = delayed_err {
+        if let Some(err) = delayed_err.as_mut() {
             if let Some(etc_span) = etc_span {
                 err.multipart_suggestion(
                     "move the `..` to the end of the field list",

compiler/rustc_parse/src/parser/stmt.rs

@@ -33,43 +33,68 @@ impl<'a> Parser<'a> {
     fn parse_stmt_without_recovery(&mut self) -> PResult<'a, Option<Stmt>> {
         maybe_whole!(self, NtStmt, |x| Some(x));
 
-        let attrs = self.parse_outer_attributes()?;
-        let lo = self.token.span;
-
-        let stmt = if self.eat_keyword(kw::Let) {
-            self.parse_local_mk(lo, attrs.into())?
-        } else if self.is_kw_followed_by_ident(kw::Mut) {
-            self.recover_stmt_local(lo, attrs.into(), "missing keyword", "let mut")?
-        } else if self.is_kw_followed_by_ident(kw::Auto) {
-            self.bump(); // `auto`
-            let msg = "write `let` instead of `auto` to introduce a new variable";
-            self.recover_stmt_local(lo, attrs.into(), msg, "let")?
-        } else if self.is_kw_followed_by_ident(sym::var) {
-            self.bump(); // `var`
-            let msg = "write `let` instead of `var` to introduce a new variable";
-            self.recover_stmt_local(lo, attrs.into(), msg, "let")?
-        } else if self.check_path() && !self.token.is_qpath_start() && !self.is_path_start_item() {
-            // We have avoided contextual keywords like `union`, items with `crate` visibility,
-            // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
-            // that starts like a path (1 token), but it fact not a path.
-            // Also, we avoid stealing syntax from `parse_item_`.
-            self.parse_stmt_path_start(lo, attrs)?
-        } else if let Some(item) = self.parse_item_common(attrs.clone(), false, true, |_| true)? {
-            // FIXME: Bad copy of attrs
-            self.mk_stmt(lo.to(item.span), StmtKind::Item(P(item)))
-        } else if self.eat(&token::Semi) {
-            // Do not attempt to parse an expression if we're done here.
-            self.error_outer_attrs(&attrs);
-            self.mk_stmt(lo, StmtKind::Empty)
-        } else if self.token != token::CloseDelim(token::Brace) {
-            // Remainder are line-expr stmts.
-            let e = self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs.into()))?;
-            self.mk_stmt(lo.to(e.span), StmtKind::Expr(e))
-        } else {
-            self.error_outer_attrs(&attrs);
-            return Ok(None);
+        let (mut stmt, tokens) = self.parse_outer_attributes_with_tokens(|this, attrs| {
+            let lo = this.token.span;
+
+            let stmt = if this.eat_keyword(kw::Let) {
+                this.parse_local_mk(lo, attrs.into())?
+            } else if this.is_kw_followed_by_ident(kw::Mut) {
+                this.recover_stmt_local(lo, attrs.into(), "missing keyword", "let mut")?
+            } else if this.is_kw_followed_by_ident(kw::Auto) {
+                this.bump(); // `auto`
+                let msg = "write `let` instead of `auto` to introduce a new variable";
+                this.recover_stmt_local(lo, attrs.into(), msg, "let")?
+            } else if this.is_kw_followed_by_ident(sym::var) {
+                this.bump(); // `var`
+                let msg = "write `let` instead of `var` to introduce a new variable";
+                this.recover_stmt_local(lo, attrs.into(), msg, "let")?
+            } else if this.check_path()
+                && !this.token.is_qpath_start()
+                && !this.is_path_start_item()
+            {
+                // We have avoided contextual keywords like `union`, items with `crate` visibility,
+                // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
+                // that starts like a path (1 token), but it fact not a path.
+                // Also, we avoid stealing syntax from `parse_item_`.
+                this.parse_stmt_path_start(lo, attrs)?
+            } else if let Some(item) =
+                this.parse_item_common(attrs.clone(), false, true, |_| true)?
+            {
+                // FIXME: Bad copy of attrs
+                this.mk_stmt(lo.to(item.span), StmtKind::Item(P(item)))
+            } else if this.eat(&token::Semi) {
+                // Do not attempt to parse an expression if we're done here.
+                this.error_outer_attrs(&attrs);
+                this.mk_stmt(lo, StmtKind::Empty)
+            } else if this.token != token::CloseDelim(token::Brace) {
+                // Remainder are line-expr stmts.
+                let e = this.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs.into()))?;
+                this.mk_stmt(lo.to(e.span), StmtKind::Expr(e))
+            } else {
+                this.error_outer_attrs(&attrs);
+                return Ok(None);
+            };
+            Ok(Some(stmt))
+        })?;
+        let old_tokens = match stmt.as_mut() {
+            Some(Stmt { kind: StmtKind::Local(local), .. }) => Some(&mut local.tokens),
+            Some(Stmt { kind: StmtKind::Item(item), .. }) => Some(&mut item.tokens),
+            Some(Stmt { kind: StmtKind::Expr(expr) | StmtKind::Semi(expr), .. }) => {
+                // FIXME: Attribute parsing should do this for us.
+                Some(&mut expr.tokens)
+            }
+            Some(Stmt { kind: StmtKind::Empty, .. }) => None,
+            Some(Stmt { kind: StmtKind::MacCall(mac), .. }) => Some(&mut mac.tokens),
+            None => None,
         };
-        Ok(Some(stmt))
+
+        if let Some(old_tokens) = old_tokens {
+            if old_tokens.is_none() {
+                *old_tokens = tokens;
+            }
+        }
+
+        Ok(stmt)
     }
 
     fn parse_stmt_path_start(&mut self, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, Stmt> {
@@ -107,7 +132,7 @@ impl<'a> Parser<'a> {
 
         let kind = if delim == token::Brace || self.token == token::Semi || self.token == token::Eof
         {
-            StmtKind::MacCall(P(MacCallStmt { mac, style, attrs }))
+            StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None }))
         } else {
             // Since none of the above applied, this is an expression statement macro.
             let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new());
@@ -219,7 +244,7 @@ impl<'a> Parser<'a> {
             }
         };
         let hi = if self.token == token::Semi { self.token.span } else { self.prev_token.span };
-        Ok(P(ast::Local { ty, pat, init, id: DUMMY_NODE_ID, span: lo.to(hi), attrs }))
+        Ok(P(ast::Local { ty, pat, init, id: DUMMY_NODE_ID, span: lo.to(hi), attrs, tokens: None }))
     }
 
     /// Parses the RHS of a local variable declaration (e.g., '= 14;').

src/librustdoc/doctest.rs

@@ -432,6 +432,8 @@ pub fn make_test(
                 }
             };
 
+            parser.is_rustdoc = true;
+
             loop {
                 match parser.parse_item() {
                     Ok(Some(item)) => {

src/test/mir-opt/unusual_item_types.Test-X-{{constructor}}.mir_map.0.32bit.mir

@@ -1,11 +1,11 @@
 // MIR for `Test::X` 0 mir_map
 
 fn Test::X(_1: usize) -> Test {
-    let mut _0: Test;                    // return place in scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
+    let mut _0: Test;                    // return place in scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
 
     bb0: {
-        ((_0 as X).0: usize) = move _1;  // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
-        discriminant(_0) = 0;            // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
-        return;                          // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
+        ((_0 as X).0: usize) = move _1;  // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
+        discriminant(_0) = 0;            // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
+        return;                          // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
     }
 }

src/test/mir-opt/unusual_item_types.Test-X-{{constructor}}.mir_map.0.64bit.mir

@@ -1,11 +1,11 @@
 // MIR for `Test::X` 0 mir_map
 
 fn Test::X(_1: usize) -> Test {
-    let mut _0: Test;                    // return place in scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
+    let mut _0: Test;                    // return place in scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
 
     bb0: {
-        ((_0 as X).0: usize) = move _1;  // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
-        discriminant(_0) = 0;            // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
-        return;                          // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:13
+        ((_0 as X).0: usize) = move _1;  // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
+        discriminant(_0) = 0;            // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
+        return;                          // scope 0 at $DIR/unusual-item-types.rs:16:5: 16:14
     }
 }

src/test/ui/ast-json/ast-json-noexpand-output.stdout

@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}}],"span":{"lo":0,"hi":0},"proc_macros":[]}

src/test/ui/ast-json/ast-json-output.stdout

@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0},"tokens":null}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0},"tokens":{"0":[[{"variant":"Token","fields":[{"kind":"Pound","span":{"lo":0,"hi":0}}]},"Joint"],[{"variant":"Token","fields":[{"kind":"Not","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Delimited","fields":[{"open":{"lo":0,"hi":0},"close":{"lo":0,"hi":0}},"Bracket",{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate_type",false]},"span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":"Eq","span":{"lo":0,"hi":0}}]},"Alone"],[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"Alone"]]}}],"span":{"lo":0,"hi":0},"proc_macros":[]}

src/test/ui/conditional-compilation/cfg-attr-parse.rs

@@ -2,10 +2,12 @@
 
 // Completely empty `cfg_attr` input
 #[cfg_attr()] //~ error: malformed `cfg_attr` attribute input
+//~| error: malformed `cfg_attr` attribute input
 struct NoConfigurationPredicate;
 
 // Zero attributes, zero trailing comma (comma manatory here)
 #[cfg_attr(all())] //~ error: expected `,`, found end of `cfg_attr`
+//~^ error: expected `,`, found end of `cfg_attr`
 struct A0C0;
 
 // Zero attributes, one trailing comma
@@ -14,6 +16,7 @@ struct A0C1;
 
 // Zero attributes, two trailing commas
 #[cfg_attr(all(),,)] //~ ERROR expected identifier
+//~| ERROR expected identifier
 struct A0C2;
 
 // One attribute, no trailing comma
@@ -26,6 +29,7 @@ struct A1C1;
 
 // One attribute, two trailing commas
 #[cfg_attr(all(), must_use,,)] //~ ERROR expected identifier
+//~| ERROR expected identifier
 struct A1C2;
 
 // Two attributes, no trailing comma
@@ -38,18 +42,23 @@ struct A2C1;
 
 // Two attributes, two trailing commas
 #[cfg_attr(all(), must_use, deprecated,,)] //~ ERROR expected identifier
+//~| ERROR expected identifier
 struct A2C2;
 
 // Wrong delimiter `[`
 #[cfg_attr[all(),,]]
-//~^ ERROR wrong `cfg_attr` delimiters
-//~| ERROR expected identifier, found `,`
+//~^ error wrong `cfg_attr` delimiters
+//~| error expected identifier, found `,`
+//~| error wrong `cfg_attr` delimiters
+//~| error expected identifier, found `,`
 struct BracketZero;
 
 // Wrong delimiter `{`
 #[cfg_attr{all(),,}]
-//~^ ERROR wrong `cfg_attr` delimiters
-//~| ERROR expected identifier, found `,`
+//~^ error wrong `cfg_attr` delimiters
+//~| error wrong `cfg_attr` delimiters
+//~| error expected identifier, found `,`
+//~| error expected identifier, found `,`
 struct BraceZero;
 
 fn main() {}

src/test/ui/conditional-compilation/cfg-attr-parse.stderr

@@ -6,17 +6,43 @@ LL | #[cfg_attr()]
    |
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
+error: malformed `cfg_attr` attribute input
+  --> $DIR/cfg-attr-parse.rs:4:1
+   |
+LL | #[cfg_attr()]
+   | ^^^^^^^^^^^^^ help: missing condition and attribute: `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   |
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
 error: expected `,`, found end of `cfg_attr` input
-  --> $DIR/cfg-attr-parse.rs:8:17
+  --> $DIR/cfg-attr-parse.rs:9:17
    |
 LL | #[cfg_attr(all())]
    |                 ^ expected `,`
    |
    = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
+error: expected `,`, found end of `cfg_attr` input
+  --> $DIR/cfg-attr-parse.rs:9:17
+   |
+LL | #[cfg_attr(all())]
+   |                 ^ expected `,`
+   |
+   = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
+error: expected identifier, found `,`
+  --> $DIR/cfg-attr-parse.rs:18:18
+   |
+LL | #[cfg_attr(all(),,)]
+   |                  ^ expected identifier
+   |
+   = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
 error: expected identifier, found `,`
-  --> $DIR/cfg-attr-parse.rs:16:18
+  --> $DIR/cfg-attr-parse.rs:18:18
    |
 LL | #[cfg_attr(all(),,)]
    |                  ^ expected identifier
@@ -25,7 +51,7 @@ LL | #[cfg_attr(all(),,)]
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
 error: expected identifier, found `,`
-  --> $DIR/cfg-attr-parse.rs:28:28
+  --> $DIR/cfg-attr-parse.rs:31:28
    |
 LL | #[cfg_attr(all(), must_use,,)]
    |                            ^ expected identifier
@@ -34,7 +60,25 @@ LL | #[cfg_attr(all(), must_use,,)]
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
 error: expected identifier, found `,`
-  --> $DIR/cfg-attr-parse.rs:40:40
+  --> $DIR/cfg-attr-parse.rs:31:28
+   |
+LL | #[cfg_attr(all(), must_use,,)]
+   |                            ^ expected identifier
+   |
+   = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
+error: expected identifier, found `,`
+  --> $DIR/cfg-attr-parse.rs:44:40
+   |
+LL | #[cfg_attr(all(), must_use, deprecated,,)]
+   |                                        ^ expected identifier
+   |
+   = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
+error: expected identifier, found `,`
+  --> $DIR/cfg-attr-parse.rs:44:40
    |
 LL | #[cfg_attr(all(), must_use, deprecated,,)]
    |                                        ^ expected identifier
@@ -43,7 +87,27 @@ LL | #[cfg_attr(all(), must_use, deprecated,,)]
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
 error: wrong `cfg_attr` delimiters
-  --> $DIR/cfg-attr-parse.rs:44:11
+  --> $DIR/cfg-attr-parse.rs:49:11
+   |
+LL | #[cfg_attr[all(),,]]
+   |           ^^^^^^^^^
+   |
+help: the delimiters should be `(` and `)`
+   |
+LL | #[cfg_attr(all(),,)]
+   |           ^       ^
+
+error: expected identifier, found `,`
+  --> $DIR/cfg-attr-parse.rs:49:18
+   |
+LL | #[cfg_attr[all(),,]]
+   |                  ^ expected identifier
+   |
+   = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
+error: wrong `cfg_attr` delimiters
+  --> $DIR/cfg-attr-parse.rs:49:11
    |
 LL | #[cfg_attr[all(),,]]
    |           ^^^^^^^^^
@@ -54,7 +118,7 @@ LL | #[cfg_attr(all(),,)]
    |           ^       ^
 
 error: expected identifier, found `,`
-  --> $DIR/cfg-attr-parse.rs:44:18
+  --> $DIR/cfg-attr-parse.rs:49:18
    |
 LL | #[cfg_attr[all(),,]]
    |                  ^ expected identifier
@@ -63,7 +127,27 @@ LL | #[cfg_attr[all(),,]]
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
 error: wrong `cfg_attr` delimiters
-  --> $DIR/cfg-attr-parse.rs:50:11
+  --> $DIR/cfg-attr-parse.rs:57:11
+   |
+LL | #[cfg_attr{all(),,}]
+   |           ^^^^^^^^^
+   |
+help: the delimiters should be `(` and `)`
+   |
+LL | #[cfg_attr(all(),,)]
+   |           ^       ^
+
+error: expected identifier, found `,`
+  --> $DIR/cfg-attr-parse.rs:57:18
+   |
+LL | #[cfg_attr{all(),,}]
+   |                  ^ expected identifier
+   |
+   = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
+error: wrong `cfg_attr` delimiters
+  --> $DIR/cfg-attr-parse.rs:57:11
    |
 LL | #[cfg_attr{all(),,}]
    |           ^^^^^^^^^
@@ -74,13 +158,13 @@ LL | #[cfg_attr(all(),,)]
    |           ^       ^
 
 error: expected identifier, found `,`
-  --> $DIR/cfg-attr-parse.rs:50:18
+  --> $DIR/cfg-attr-parse.rs:57:18
    |
 LL | #[cfg_attr{all(),,}]
    |                  ^ expected identifier
    |
    = help: the valid syntax is `#[cfg_attr(condition, attribute, other_attribute, ...)]`
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
-error: aborting due to 9 previous errors
+error: aborting due to 18 previous errors
 

src/test/ui/derive-uninhabited-enum-38885.stderr

@@ -2,7 +2,7 @@ warning: variant is never constructed: `Void`
   --> $DIR/derive-uninhabited-enum-38885.rs:13:5
    |
 LL |     Void(Void),
-   |     ^^^^^^^^^^
+   |     ^^^^^^^^^^^
    |
    = note: `-W dead-code` implied by `-W unused`
 

src/test/ui/discrim/discrim-overflow-2.stderr

@@ -2,63 +2,63 @@ error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:15:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 127
+   |         ^^^^^ overflowed on value after 127
    |
    = note: explicitly set `OhNo = -128` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:24:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 255
+   |         ^^^^^ overflowed on value after 255
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:33:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 32767
+   |         ^^^^^ overflowed on value after 32767
    |
    = note: explicitly set `OhNo = -32768` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:42:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 65535
+   |         ^^^^^ overflowed on value after 65535
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:51:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 2147483647
+   |         ^^^^^ overflowed on value after 2147483647
    |
    = note: explicitly set `OhNo = -2147483648` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:60:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 4294967295
+   |         ^^^^^ overflowed on value after 4294967295
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:69:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 9223372036854775807
+   |         ^^^^^ overflowed on value after 9223372036854775807
    |
    = note: explicitly set `OhNo = -9223372036854775808` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow-2.rs:78:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 18446744073709551615
+   |         ^^^^^ overflowed on value after 18446744073709551615
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 

src/test/ui/discrim/discrim-overflow.stderr

@@ -2,63 +2,63 @@ error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:13:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 127
+   |         ^^^^^ overflowed on value after 127
    |
    = note: explicitly set `OhNo = -128` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:24:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 255
+   |         ^^^^^ overflowed on value after 255
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:35:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 32767
+   |         ^^^^^ overflowed on value after 32767
    |
    = note: explicitly set `OhNo = -32768` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:46:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 65535
+   |         ^^^^^ overflowed on value after 65535
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:58:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 2147483647
+   |         ^^^^^ overflowed on value after 2147483647
    |
    = note: explicitly set `OhNo = -2147483648` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:70:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 4294967295
+   |         ^^^^^ overflowed on value after 4294967295
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:82:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 9223372036854775807
+   |         ^^^^^ overflowed on value after 9223372036854775807
    |
    = note: explicitly set `OhNo = -9223372036854775808` if that is desired outcome
 
 error[E0370]: enum discriminant overflowed
   --> $DIR/discrim-overflow.rs:94:9
    |
 LL |         OhNo,
-   |         ^^^^ overflowed on value after 18446744073709551615
+   |         ^^^^^ overflowed on value after 18446744073709551615
    |
    = note: explicitly set `OhNo = 0` if that is desired outcome
 

src/test/ui/empty/empty-struct-braces-pat-1.stderr

@@ -18,7 +18,7 @@ LL |         XE::XEmpty3 => ()
 LL |     XEmpty3 {},
    |     ------- `XE::XEmpty3` defined here
 LL |     XEmpty4,
-   |     ------- similarly named unit variant `XEmpty4` defined here
+   |     -------- similarly named unit variant `XEmpty4` defined here
    |
 help: use struct pattern syntax instead
    |

src/test/ui/empty/empty-struct-braces-pat-3.stderr

@@ -19,7 +19,7 @@ LL |     XEmpty3 {},
    |     ------- `XE::XEmpty3` defined here
 LL |     XEmpty4,
 LL |     XEmpty5(),
-   |     --------- similarly named tuple variant `XEmpty5` defined here
+   |     ---------- similarly named tuple variant `XEmpty5` defined here
    |
 help: use struct pattern syntax instead
    |
@@ -51,7 +51,7 @@ LL |     XEmpty3 {},
    |     ------- `XE::XEmpty3` defined here
 LL |     XEmpty4,
 LL |     XEmpty5(),
-   |     --------- similarly named tuple variant `XEmpty5` defined here
+   |     ---------- similarly named tuple variant `XEmpty5` defined here
    |
 help: use struct pattern syntax instead
    |

src/test/ui/empty/empty-struct-tuple-pat.stderr

@@ -34,9 +34,9 @@ LL |         XE::XEmpty5 => (),
   ::: $DIR/auxiliary/empty-struct.rs:7:5
    |
 LL |     XEmpty4,
-   |     ------- similarly named unit variant `XEmpty4` defined here
+   |     -------- similarly named unit variant `XEmpty4` defined here
 LL |     XEmpty5(),
-   |     --------- `XE::XEmpty5` defined here
+   |     ---------- `XE::XEmpty5` defined here
    |
 help: use the tuple variant pattern syntax instead
    |

src/test/ui/empty/empty-struct-unit-pat.stderr

@@ -59,7 +59,7 @@ LL |         XE::XEmpty4() => (),
   ::: $DIR/auxiliary/empty-struct.rs:8:5
    |
 LL |     XEmpty5(),
-   |     --------- similarly named tuple variant `XEmpty5` defined here
+   |     ---------- similarly named tuple variant `XEmpty5` defined here
 
 error[E0532]: expected tuple struct or tuple variant, found unit variant `E::Empty4`
   --> $DIR/empty-struct-unit-pat.rs:46:9
@@ -78,7 +78,7 @@ LL |         XE::XEmpty4(..) => (),
   ::: $DIR/auxiliary/empty-struct.rs:8:5
    |
 LL |     XEmpty5(),
-   |     --------- similarly named tuple variant `XEmpty5` defined here
+   |     ---------- similarly named tuple variant `XEmpty5` defined here
 
 error: aborting due to 8 previous errors
 

src/test/ui/enum-discriminant/feature-gate-arbitrary_enum_discriminant.stderr

@@ -23,10 +23,10 @@ LL |   Unit = 1,
    |          ^ disallowed custom discriminant
 LL |
 LL |   Tuple() = 2,
-   |   ----------- tuple variant defined here
+   |   ------------ tuple variant defined here
 LL |
 LL |   Struct{} = 3,
-   |   ------------ struct variant defined here
+   |   ------------- struct variant defined here
    |
    = note: see issue #60553 <https://github.com/rust-lang/rust/issues/60553> for more information
    = help: add `#![feature(arbitrary_enum_discriminant)]` to the crate attributes to enable

src/test/ui/enum/enum-size-variance.stderr

@@ -2,7 +2,7 @@ warning: enum variant is more than three times larger (32 bytes) than the next l
   --> $DIR/enum-size-variance.rs:18:5
    |
 LL |     L(i64, i64, i64, i64),
-   |     ^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^
    |
 note: the lint level is defined here
   --> $DIR/enum-size-variance.rs:3:9

src/test/ui/error-codes/E0023.stderr

@@ -2,7 +2,7 @@ error[E0023]: this pattern has 1 field, but the corresponding tuple variant has
   --> $DIR/E0023.rs:11:9
    |
 LL |     Apple(String, String),
-   |     --------------------- tuple variant defined here
+   |     ---------------------- tuple variant defined here
 ...
 LL |         Fruit::Apple(a) => {},
    |         ^^^^^^^^^^^^^^^ expected 2 fields, found 1
@@ -11,7 +11,7 @@ error[E0023]: this pattern has 3 fields, but the corresponding tuple variant has
   --> $DIR/E0023.rs:12:9
    |
 LL |     Apple(String, String),
-   |     --------------------- tuple variant defined here
+   |     ---------------------- tuple variant defined here
 ...
 LL |         Fruit::Apple(a, b, c) => {},
    |         ^^^^^^^^^^^^^^^^^^^^^ expected 2 fields, found 3
@@ -20,7 +20,7 @@ error[E0023]: this pattern has 2 fields, but the corresponding tuple variant has
   --> $DIR/E0023.rs:13:9
    |
 LL |     Pear(u32),
-   |     --------- tuple variant defined here
+   |     ---------- tuple variant defined here
 ...
 LL |         Fruit::Pear(1, 2) => {},
    |         ^^^^^^^^^^^^^^^^^ expected 1 field, found 2
@@ -29,7 +29,7 @@ error[E0023]: this pattern has 2 fields, but the corresponding tuple variant has
   --> $DIR/E0023.rs:14:9
    |
 LL |     Orange((String, String)),
-   |     ------------------------ tuple variant defined here
+   |     ------------------------- tuple variant defined here
 ...
 LL |         Fruit::Orange(a, b) => {},
    |         ^^^^^^^^^^^^^^^^^^^ expected 1 field, found 2
@@ -43,7 +43,7 @@ error[E0023]: this pattern has 0 fields, but the corresponding tuple variant has
   --> $DIR/E0023.rs:15:9
    |
 LL |     Banana(()),
-   |     ---------- tuple variant defined here
+   |     ----------- tuple variant defined here
 ...
 LL |         Fruit::Banana() => {},
    |         ^^^^^^^^^^^^^^^ expected 1 field, found 0

src/test/ui/error-codes/E0370.stderr

@@ -2,7 +2,7 @@ error[E0370]: enum discriminant overflowed
   --> $DIR/E0370.rs:7:5
    |
 LL |     Y,
-   |     ^ overflowed on value after 9223372036854775807
+   |     ^^ overflowed on value after 9223372036854775807
    |
    = note: explicitly set `Y = -9223372036854775808` if that is desired outcome
 

src/test/ui/error-codes/E0618.stderr

@@ -2,7 +2,7 @@ error[E0618]: expected function, found enum variant `X::Entry`
   --> $DIR/E0618.rs:6:5
    |
 LL |     Entry,
-   |     ----- `X::Entry` defined here
+   |     ------ `X::Entry` defined here
 ...
 LL |     X::Entry();
    |     ^^^^^^^^--

src/test/ui/feature-gates/feature-gate-cfg-target-thread-local.rs

@@ -8,6 +8,7 @@ extern crate cfg_target_thread_local;
 extern {
     #[cfg_attr(target_thread_local, thread_local)]
     //~^ `cfg(target_thread_local)` is experimental and subject to change
+    //~| `cfg(target_thread_local)` is experimental and subject to change
 
     static FOO: u32;
 }

src/test/ui/feature-gates/feature-gate-cfg-target-thread-local.stderr

@@ -7,6 +7,15 @@ LL |     #[cfg_attr(target_thread_local, thread_local)]
    = note: see issue #29594 <https://github.com/rust-lang/rust/issues/29594> for more information
    = help: add `#![feature(cfg_target_thread_local)]` to the crate attributes to enable
 
-error: aborting due to previous error
+error[E0658]: `cfg(target_thread_local)` is experimental and subject to change
+  --> $DIR/feature-gate-cfg-target-thread-local.rs:9:16
+   |
+LL |     #[cfg_attr(target_thread_local, thread_local)]
+   |                ^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #29594 <https://github.com/rust-lang/rust/issues/29594> for more information
+   = help: add `#![feature(cfg_target_thread_local)]` to the crate attributes to enable
+
+error: aborting due to 2 previous errors
 
 For more information about this error, try `rustc --explain E0658`.

src/test/ui/issues/issue-15524.stderr

@@ -13,7 +13,7 @@ LL |     A = 1,
    |         - first use of `1`
 ...
 LL |     D,
-   |     ^ enum already has `1`
+   |     ^^ enum already has `1`
 
 error[E0081]: discriminant value `1` already exists
   --> $DIR/issue-15524.rs:11:9

src/test/ui/issues/issue-32004.stderr

@@ -2,7 +2,7 @@ error[E0532]: expected unit struct, unit variant or constant, found tuple varian
   --> $DIR/issue-32004.rs:10:9
    |
 LL |     Bar(i32),
-   |     -------- `Foo::Bar` defined here
+   |     --------- `Foo::Bar` defined here
 LL |     Baz
    |     --- similarly named unit variant `Baz` defined here
 ...

src/test/ui/issues/issue-47706.stderr

@@ -11,7 +11,7 @@ error[E0593]: function is expected to take 0 arguments, but it takes 1 argument
   --> $DIR/issue-47706.rs:27:9
    |
 LL |     Bar(i32),
-   |     -------- takes 1 argument
+   |     --------- takes 1 argument
 ...
 LL | fn foo<F>(f: F)
    |    --- required by a bound in this

src/test/ui/issues/issue-5100.stderr

@@ -2,7 +2,7 @@ error[E0308]: mismatched types
   --> $DIR/issue-5100.rs:8:9
    |
 LL | enum A { B, C }
-   |          - unit variant defined here
+   |          -- unit variant defined here
 ...
 LL |     match (true, false) {
    |           ------------- this expression has type `(bool, bool)`

src/test/ui/issues/issue-63983.stderr

@@ -2,7 +2,7 @@ error[E0532]: expected unit struct, unit variant or constant, found tuple varian
   --> $DIR/issue-63983.rs:8:9
    |
 LL |     Tuple(i32),
-   |     ---------- `MyEnum::Tuple` defined here
+   |     ----------- `MyEnum::Tuple` defined here
 ...
 LL |         MyEnum::Tuple => "",
    |         ^^^^^^^^^^^^^ help: use the tuple variant pattern syntax instead: `MyEnum::Tuple(_)`
@@ -11,7 +11,7 @@ error[E0532]: expected unit struct, unit variant or constant, found struct varia
   --> $DIR/issue-63983.rs:10:9
    |
 LL |     Struct{ s: i32 },
-   |     ---------------- `MyEnum::Struct` defined here
+   |     ----------------- `MyEnum::Struct` defined here
 ...
 LL |         MyEnum::Struct => "",
    |         ^^^^^^^^^^^^^^ help: use struct pattern syntax instead: `MyEnum::Struct { s }`

src/test/ui/issues/issue-7867.stderr

@@ -2,7 +2,7 @@ error[E0308]: mismatched types
   --> $DIR/issue-7867.rs:7:9
    |
 LL | enum A { B, C }
-   |          - unit variant defined here
+   |          -- unit variant defined here
 ...
 LL |     match (true, false) {
    |           ------------- this expression has type `(bool, bool)`

src/test/ui/lint/dead-code/lint-dead-code-4.stderr

@@ -14,7 +14,7 @@ error: variant is never constructed: `X`
   --> $DIR/lint-dead-code-4.rs:15:5
    |
 LL |     X,
-   |     ^
+   |     ^^
 
 error: variant is never constructed: `Y`
   --> $DIR/lint-dead-code-4.rs:16:5
@@ -24,7 +24,7 @@ LL | |         a: String,
 LL | |         b: i32,
 LL | |         c: i32,
 LL | |     },
-   | |_____^
+   | |______^
 
 error: enum is never used: `ABC`
   --> $DIR/lint-dead-code-4.rs:24:6
@@ -36,7 +36,7 @@ error: variant is never constructed: `I`
   --> $DIR/lint-dead-code-4.rs:36:5
    |
 LL |     I,
-   |     ^
+   |     ^^
 
 error: field is never read: `b`
   --> $DIR/lint-dead-code-4.rs:39:9

src/test/ui/lint/dead-code/lint-dead-code-5.stderr

@@ -14,13 +14,13 @@ error: variant is never constructed: `Variant5`
   --> $DIR/lint-dead-code-5.rs:13:5
    |
 LL |     Variant5 { _x: isize },
-   |     ^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^
 
 error: variant is never constructed: `Variant6`
   --> $DIR/lint-dead-code-5.rs:14:5
    |
 LL |     Variant6(isize),
-   |     ^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^
 
 error: enum is never used: `Enum3`
   --> $DIR/lint-dead-code-5.rs:35:6

src/test/ui/lint/dead-code/unused-struct-variant.stderr

@@ -2,7 +2,7 @@ error: variant is never constructed: `Bar`
   --> $DIR/unused-struct-variant.rs:8:5
    |
 LL |     Bar(B),
-   |     ^^^^^^
+   |     ^^^^^^^
    |
 note: the lint level is defined here
   --> $DIR/unused-struct-variant.rs:1:9

src/test/ui/lint/dead-code/unused-variant.stderr

@@ -2,7 +2,7 @@ error: variant is never constructed: `Variant1`
   --> $DIR/unused-variant.rs:5:5
    |
 LL |     Variant1,
-   |     ^^^^^^^^
+   |     ^^^^^^^^^
    |
 note: the lint level is defined here
   --> $DIR/unused-variant.rs:1:9

src/test/ui/malformed/malformed-special-attrs.rs

@@ -1,7 +1,9 @@
 #[cfg_attr] //~ ERROR malformed `cfg_attr` attribute
+//~| ERROR malformed `cfg_attr` attribute
 struct S1;
 
 #[cfg_attr = ""] //~ ERROR malformed `cfg_attr` attribute
+//~| ERROR malformed `cfg_attr` attribute
 struct S2;
 
 #[derive] //~ ERROR malformed `derive` attribute

src/test/ui/malformed/malformed-special-attrs.stderr

@@ -7,24 +7,40 @@ LL | #[cfg_attr]
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
 error: malformed `cfg_attr` attribute input
-  --> $DIR/malformed-special-attrs.rs:4:1
+  --> $DIR/malformed-special-attrs.rs:1:1
+   |
+LL | #[cfg_attr]
+   | ^^^^^^^^^^^ help: missing condition and attribute: `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   |
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
+error: malformed `cfg_attr` attribute input
+  --> $DIR/malformed-special-attrs.rs:5:1
+   |
+LL | #[cfg_attr = ""]
+   | ^^^^^^^^^^^^^^^^ help: missing condition and attribute: `#[cfg_attr(condition, attribute, other_attribute, ...)]`
+   |
+   = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
+
+error: malformed `cfg_attr` attribute input
+  --> $DIR/malformed-special-attrs.rs:5:1
    |
 LL | #[cfg_attr = ""]
    | ^^^^^^^^^^^^^^^^ help: missing condition and attribute: `#[cfg_attr(condition, attribute, other_attribute, ...)]`
    |
    = note: for more information, visit <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>
 
 error: malformed `derive` attribute input
-  --> $DIR/malformed-special-attrs.rs:7:1
+  --> $DIR/malformed-special-attrs.rs:9:1
    |
 LL | #[derive]
    | ^^^^^^^^^ help: missing traits to be derived: `#[derive(Trait1, Trait2, ...)]`
 
 error: malformed `derive` attribute input
-  --> $DIR/malformed-special-attrs.rs:10:1
+  --> $DIR/malformed-special-attrs.rs:12:1
    |
 LL | #[derive = ""]
    | ^^^^^^^^^^^^^^ help: missing traits to be derived: `#[derive(Trait1, Trait2, ...)]`
 
-error: aborting due to 4 previous errors
+error: aborting due to 6 previous errors
 

src/test/ui/match/match-pattern-field-mismatch.stderr

@@ -2,7 +2,7 @@ error[E0023]: this pattern has 2 fields, but the corresponding tuple variant has
   --> $DIR/match-pattern-field-mismatch.rs:10:11
    |
 LL |         Rgb(usize, usize, usize),
-   |         ------------------------ tuple variant defined here
+   |         ------------------------- tuple variant defined here
 ...
 LL |           Color::Rgb(_, _) => { }
    |           ^^^^^^^^^^^^^^^^ expected 3 fields, found 2

src/test/ui/namespace/namespace-mix.stderr

@@ -46,9 +46,9 @@ error[E0423]: expected value, found struct variant `m7::V`
   --> $DIR/namespace-mix.rs:100:11
    |
 LL |         V {},
-   |         ---- `m7::V` defined here
+   |         ----- `m7::V` defined here
 LL |         TV(),
-   |         ---- similarly named tuple variant `TV` defined here
+   |         ----- similarly named tuple variant `TV` defined here
 ...
 LL |     check(m7::V);
    |           ^^^^^
@@ -79,7 +79,7 @@ LL |     check(xm7::V);
 LL |         V {},
    |         - `xm7::V` defined here
 LL |         TV(),
-   |         ---- similarly named tuple variant `TV` defined here
+   |         ----- similarly named tuple variant `TV` defined here
    |
 help: use struct literal syntax instead
    |

src/test/ui/parser/issue-62895.rs

@@ -5,7 +5,9 @@ mod _ { //~ ERROR expected identifier
 pub fn g() -> isizee { //~ ERROR cannot find type `isizee` in this scope
 mod _ { //~ ERROR expected identifier
 pub    g() -> is //~ ERROR missing `fn` for function definition
-(), w20);
+(), w20); //~ ERROR mismatched
 }
 (), w20); //~ ERROR expected item, found `;`
+//~^ ERROR mismatched
+
 }

src/test/ui/parser/issue-62895.stderr

@@ -27,6 +27,24 @@ error: expected item, found `;`
 LL | (), w20);
    |         ^ help: remove this semicolon
 
+error: mismatched closing delimiter: `)`
+  --> $DIR/issue-62895.rs:8:8
+   |
+LL | mod _ {
+   |       - unclosed delimiter
+LL | pub    g() -> is
+LL | (), w20);
+   |        ^ mismatched closing delimiter
+
+error: mismatched closing delimiter: `)`
+  --> $DIR/issue-62895.rs:10:8
+   |
+LL | mod _ {
+   |       - unclosed delimiter
+...
+LL | (), w20);
+   |        ^ mismatched closing delimiter
+
 error[E0412]: cannot find type `isizee` in this scope
   --> $DIR/issue-62895.rs:5:15
    |
@@ -41,7 +59,7 @@ LL | fn v() -> isize {
    |    |
    |    implicitly returns `()` as its body has no tail or `return` expression
 
-error: aborting due to 6 previous errors
+error: aborting due to 8 previous errors
 
 Some errors have detailed explanations: E0308, E0412.
 For more information about an error, try `rustc --explain E0308`.

src/test/ui/parser/issue-68890.rs

@@ -1,4 +1,9 @@
-enum e{A((?'a a+?+l))}
+enum E{A((?'a a+?+l))}
 //~^ ERROR `?` may only modify trait bounds, not lifetime bounds
 //~| ERROR expected one of `)`, `+`, or `,`
-//~| ERROR expected item, found `)`
+//~| ERROR expected identifier, found `)`
+//~| ERROR cannot find trait
+//~| ERROR use of undeclared lifetime
+//~| WARN trait objects without
+
+fn main() {}

src/test/ui/parser/issue-68890.stderr

@@ -1,20 +1,46 @@
 error: `?` may only modify trait bounds, not lifetime bounds
   --> $DIR/issue-68890.rs:1:11
    |
-LL | enum e{A((?'a a+?+l))}
+LL | enum E{A((?'a a+?+l))}
    |           ^
 
 error: expected one of `)`, `+`, or `,`, found `a`
   --> $DIR/issue-68890.rs:1:15
    |
-LL | enum e{A((?'a a+?+l))}
+LL | enum E{A((?'a a+?+l))}
    |               ^ expected one of `)`, `+`, or `,`
 
-error: expected item, found `)`
+error: expected identifier, found `)`
   --> $DIR/issue-68890.rs:1:21
    |
-LL | enum e{A((?'a a+?+l))}
-   |                     ^ expected item
+LL | enum E{A((?'a a+?+l))}
+   |                     ^ expected identifier
 
-error: aborting due to 3 previous errors
+error[E0405]: cannot find trait `l` in this scope
+  --> $DIR/issue-68890.rs:1:19
+   |
+LL | enum E{A((?'a a+?+l))}
+   |                   ^ not found in this scope
+
+warning: trait objects without an explicit `dyn` are deprecated
+  --> $DIR/issue-68890.rs:1:10
+   |
+LL | enum E{A((?'a a+?+l))}
+   |          ^^^^^^^^^^ help: use `dyn`: `dyn (?'a a+?+l`
+   |
+   = note: `#[warn(bare_trait_objects)]` on by default
+
+error[E0261]: use of undeclared lifetime name `'a`
+  --> $DIR/issue-68890.rs:1:12
+   |
+LL | enum E{A((?'a a+?+l))}
+   |       -    ^^ undeclared lifetime
+   |       |
+   |       help: consider introducing lifetime `'a` here: `<'a>`
+   |
+   = help: if you want to experiment with in-band lifetime bindings, add `#![feature(in_band_lifetimes)]` to the crate attributes
+
+error: aborting due to 5 previous errors; 1 warning emitted
 
+Some errors have detailed explanations: E0261, E0405.
+For more information about an error, try `rustc --explain E0261`.

src/test/ui/parser/recover-enum.rs

@@ -1,11 +1,11 @@
 fn main() {
     enum Test {
-        Very //~ HELP missing `,`
-        Bad(usize) //~ HELP missing `,`
-        //~^ ERROR expected one of `(`, `,`, `=`, `{`, or `}`, found `Bad`
-        Stuff { a: usize } //~ HELP missing `,`
-        //~^ ERROR expected one of `,`, `=`, or `}`, found `Stuff`
+        Very //~ HELP try adding a comma
+        Bad(usize) //~ HELP try adding a comma
+        //~^ ERROR expected one of `(`, `,`, `=`, or `{`, found `Bad`
+        Stuff { a: usize } //~ HELP try adding a comma
+        //~^ ERROR expected one of `,` or `=`, found `Stuff`
         Here
-        //~^ ERROR expected one of `,`, `=`, or `}`, found `Here`
+        //~^ ERROR expected one of `,` or `=`, found `Here`
     }
 }