Auto merge of #40597 - jseyfried:improve_span_expn_info, r=jseyfried

macros: improve `Span`'s expansion information

This PR improves `Span`'s expansion information. More specifically:
 - It refactors AST node span construction to preserve expansion information.
   - Today, we only use the underlying tokens' `BytePos`s, throwing away the `ExpnId`s.
   - This improves the accuracy of AST nodes' expansion information, fixing #30506.
 - It refactors `span.expn_id: ExpnId` to `span.ctxt: SyntaxContext` and removes `ExpnId`.
   - This gives all tokens as much hygiene information as `Ident`s.
   - This is groundwork for procedural macros 2.0 `TokenStream` API.
   - This is also groundwork for declarative macros 2.0, which will need this hygiene information for some non-`Ident` tokens.
 - It simplifies processing of spans' expansion information throughout the compiler.
 - It fixes #40649.
 - It fixes #39450 and fixes part of #23480.

r? @nrc

Author: bors

src/librustc/hir/lowering.rs

@@ -57,6 +57,7 @@ use std::mem;
 use syntax::attr;
 use syntax::ast::*;
 use syntax::errors;
+use syntax::ext::hygiene::{Mark, SyntaxContext};
 use syntax::ptr::P;
 use syntax::codemap::{self, respan, Spanned};
 use syntax::std_inject;
@@ -392,14 +393,16 @@ impl<'a> LoweringContext<'a> {
     }
 
     fn allow_internal_unstable(&self, reason: &'static str, mut span: Span) -> Span {
-        span.expn_id = self.sess.codemap().record_expansion(codemap::ExpnInfo {
+        let mark = Mark::fresh();
+        mark.set_expn_info(codemap::ExpnInfo {
             call_site: span,
             callee: codemap::NameAndSpan {
                 format: codemap::CompilerDesugaring(Symbol::intern(reason)),
                 span: Some(span),
                 allow_internal_unstable: true,
             },
         });
+        span.ctxt = SyntaxContext::empty().apply_mark(mark);
         span
     }
 
@@ -1993,7 +1996,7 @@ impl<'a> LoweringContext<'a> {
                     volatile: asm.volatile,
                     alignstack: asm.alignstack,
                     dialect: asm.dialect,
-                    expn_id: asm.expn_id,
+                    ctxt: asm.ctxt,
                 };
                 let outputs =
                     asm.outputs.iter().map(|out| self.lower_expr(&out.expr)).collect();

src/librustc/hir/map/def_collector.rs

@@ -92,7 +92,7 @@ impl<'a> DefCollector<'a> {
     fn visit_macro_invoc(&mut self, id: NodeId, const_expr: bool) {
         if let Some(ref mut visit) = self.visit_macro_invoc {
             visit(MacroInvocationData {
-                mark: Mark::from_placeholder_id(id),
+                mark: id.placeholder_to_mark(),
                 const_expr: const_expr,
                 def_index: self.parent_def.unwrap(),
             })

src/librustc/hir/mod.rs

@@ -33,11 +33,12 @@ use hir::def::Def;
 use hir::def_id::{DefId, DefIndex, CRATE_DEF_INDEX};
 use util::nodemap::{NodeMap, FxHashSet};
 
-use syntax_pos::{Span, ExpnId, DUMMY_SP};
+use syntax_pos::{Span, DUMMY_SP};
 use syntax::codemap::{self, Spanned};
 use syntax::abi::Abi;
 use syntax::ast::{Ident, Name, NodeId, DUMMY_NODE_ID, AsmDialect};
 use syntax::ast::{Attribute, Lit, StrStyle, FloatTy, IntTy, UintTy, MetaItem};
+use syntax::ext::hygiene::SyntaxContext;
 use syntax::ptr::P;
 use syntax::symbol::{Symbol, keywords};
 use syntax::tokenstream::TokenStream;
@@ -1367,7 +1368,7 @@ pub struct InlineAsm {
     pub volatile: bool,
     pub alignstack: bool,
     pub dialect: AsmDialect,
-    pub expn_id: ExpnId,
+    pub ctxt: SyntaxContext,
 }
 
 /// represents an argument in a function header

src/librustc/ich/caching_codemap_view.rs

@@ -47,10 +47,6 @@ impl<'tcx> CachingCodemapView<'tcx> {
         }
     }
 
-    pub fn codemap(&self) -> &'tcx CodeMap {
-        self.codemap
-    }
-
     pub fn byte_pos_to_line_and_col(&mut self,
                                     pos: BytePos)
                                     -> Option<(Rc<FileMap>, usize, BytePos)> {

src/librustc/middle/region.rs

@@ -236,7 +236,7 @@ impl CodeExtent {
                         // (This is the special case aluded to in the
                         // doc-comment for this method)
                         let stmt_span = blk.stmts[r.first_statement_index as usize].span;
-                        Some(Span { lo: stmt_span.hi, hi: blk.span.hi, expn_id: stmt_span.expn_id })
+                        Some(Span { lo: stmt_span.hi, hi: blk.span.hi, ctxt: stmt_span.ctxt })
                     }
                 }
             }

src/librustc/middle/resolve_lifetime.rs

@@ -29,7 +29,7 @@ use syntax::ast;
 use syntax::attr;
 use syntax::ptr::P;
 use syntax::symbol::keywords;
-use syntax_pos::{mk_sp, Span};
+use syntax_pos::Span;
 use errors::DiagnosticBuilder;
 use util::nodemap::{NodeMap, NodeSet, FxHashSet, FxHashMap, DefIdMap};
 use rustc_back::slice;
@@ -1468,8 +1468,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     self.resolve_lifetime_ref(bound);
                 } else {
                     self.insert_lifetime(bound, Region::Static);
-                    let full_span = mk_sp(lifetime_i.lifetime.span.lo, bound.span.hi);
-                    self.sess.struct_span_warn(full_span,
+                    self.sess.struct_span_warn(lifetime_i.lifetime.span.to(bound.span),
                         &format!("unnecessary lifetime parameter `{}`", lifetime_i.lifetime.name))
                         .help(&format!("you can use the `'static` lifetime directly, in place \
                                         of `{}`", lifetime_i.lifetime.name))

src/librustc/middle/stability.rs

@@ -467,7 +467,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     }
 
     pub fn check_stability(self, def_id: DefId, id: NodeId, span: Span) {
-        if self.sess.codemap().span_allows_unstable(span) {
+        if span.allows_unstable() {
             debug!("stability: \
                     skipping span={:?} since it is internal", span);
             return;

src/librustc_driver/driver.rs

@@ -580,7 +580,7 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
 
     krate = time(time_passes, "crate injection", || {
         let alt_std_name = sess.opts.alt_std_name.clone();
-        syntax::std_inject::maybe_inject_crates_ref(&sess.parse_sess, krate, alt_std_name)
+        syntax::std_inject::maybe_inject_crates_ref(krate, alt_std_name)
     });
 
     let mut addl_plugins = Some(addl_plugins);
@@ -798,7 +798,7 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
 
     // Discard hygiene data, which isn't required after lowering to HIR.
     if !keep_hygiene_data(sess) {
-        syntax::ext::hygiene::reset_hygiene_data();
+        syntax::ext::hygiene::clear_markings();
     }
 
     Ok(ExpansionResult {

src/librustc_errors/emitter.rs

@@ -10,7 +10,7 @@
 
 use self::Destination::*;
 
-use syntax_pos::{COMMAND_LINE_SP, DUMMY_SP, FileMap, Span, MultiSpan, CharPos};
+use syntax_pos::{DUMMY_SP, FileMap, Span, MultiSpan, CharPos};
 
 use {Level, CodeSuggestion, DiagnosticBuilder, SubDiagnostic, CodeMapper};
 use RenderSpan::*;
@@ -151,7 +151,7 @@ impl EmitterWriter {
 
         if let Some(ref cm) = self.cm {
             for span_label in msp.span_labels() {
-                if span_label.span == DUMMY_SP || span_label.span == COMMAND_LINE_SP {
+                if span_label.span == DUMMY_SP {
                     continue;
                 }
                 let lo = cm.lookup_char_pos(span_label.span.lo);
@@ -615,15 +615,15 @@ impl EmitterWriter {
         let mut max = 0;
         if let Some(ref cm) = self.cm {
             for primary_span in msp.primary_spans() {
-                if primary_span != &DUMMY_SP && primary_span != &COMMAND_LINE_SP {
+                if primary_span != &DUMMY_SP {
                     let hi = cm.lookup_char_pos(primary_span.hi);
                     if hi.line > max {
                         max = hi.line;
                     }
                 }
             }
             for span_label in msp.span_labels() {
-                if span_label.span != DUMMY_SP && span_label.span != COMMAND_LINE_SP {
+                if span_label.span != DUMMY_SP {
                     let hi = cm.lookup_char_pos(span_label.span.hi);
                     if hi.line > max {
                         max = hi.line;
@@ -659,20 +659,20 @@ impl EmitterWriter {
 
             // First, find all the spans in <*macros> and point instead at their use site
             for sp in span.primary_spans() {
-                if (*sp == COMMAND_LINE_SP) || (*sp == DUMMY_SP) {
+                if *sp == DUMMY_SP {
                     continue;
                 }
                 if cm.span_to_filename(sp.clone()).contains("macros>") {
-                    let v = cm.macro_backtrace(sp.clone());
+                    let v = sp.macro_backtrace();
                     if let Some(use_site) = v.last() {
                         before_after.push((sp.clone(), use_site.call_site.clone()));
                     }
                 }
-                for trace in cm.macro_backtrace(sp.clone()).iter().rev() {
+                for trace in sp.macro_backtrace().iter().rev() {
                     // Only show macro locations that are local
                     // and display them like a span_note
                     if let Some(def_site) = trace.def_site_span {
-                        if (def_site == COMMAND_LINE_SP) || (def_site == DUMMY_SP) {
+                        if def_site == DUMMY_SP {
                             continue;
                         }
                         // Check to make sure we're not in any <*macros>
@@ -689,11 +689,11 @@ impl EmitterWriter {
                 span.push_span_label(label_span, label_text);
             }
             for sp_label in span.span_labels() {
-                if (sp_label.span == COMMAND_LINE_SP) || (sp_label.span == DUMMY_SP) {
+                if sp_label.span == DUMMY_SP {
                     continue;
                 }
                 if cm.span_to_filename(sp_label.span.clone()).contains("macros>") {
-                    let v = cm.macro_backtrace(sp_label.span.clone());
+                    let v = sp_label.span.macro_backtrace();
                     if let Some(use_site) = v.last() {
                         before_after.push((sp_label.span.clone(), use_site.call_site.clone()));
                     }
@@ -848,7 +848,7 @@ impl EmitterWriter {
         // Make sure our primary file comes first
         let primary_lo = if let (Some(ref cm), Some(ref primary_span)) =
             (self.cm.as_ref(), msp.primary_span().as_ref()) {
-            if primary_span != &&DUMMY_SP && primary_span != &&COMMAND_LINE_SP {
+            if primary_span != &&DUMMY_SP {
                 cm.lookup_char_pos(primary_span.lo)
             } else {
                 emit_to_destination(&buffer.render(), level, &mut self.dst)?;

src/librustc_errors/lib.rs

@@ -48,7 +48,6 @@ pub mod styled_buffer;
 mod lock;
 
 use syntax_pos::{BytePos, Loc, FileLinesResult, FileName, MultiSpan, Span, NO_EXPANSION};
-use syntax_pos::MacroBacktrace;
 
 #[derive(Clone, Debug, PartialEq, RustcEncodable, RustcDecodable)]
 pub enum RenderSpan {
@@ -75,7 +74,6 @@ pub trait CodeMapper {
     fn span_to_lines(&self, sp: Span) -> FileLinesResult;
     fn span_to_string(&self, sp: Span) -> String;
     fn span_to_filename(&self, sp: Span) -> FileName;
-    fn macro_backtrace(&self, span: Span) -> Vec<MacroBacktrace>;
     fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option<Span>;
 }
 
@@ -120,7 +118,7 @@ impl CodeSuggestion {
         let bounding_span = Span {
             lo: lo,
             hi: hi,
-            expn_id: NO_EXPANSION,
+            ctxt: NO_EXPANSION,
         };
         let lines = cm.span_to_lines(bounding_span).unwrap();
         assert!(!lines.lines.is_empty());