Auto merge of #65324 - Centril:organize-syntax, r=petrochenkov

Split libsyntax apart

In this PR the general idea is to separate the AST, parser, and friends by a more data / logic structure (tho not fully realized!) by separating out the parser and macro expansion code from libsyntax. Specifically have now three crates instead of one (libsyntax):

- libsyntax:

   - concrete syntax tree (`syntax::ast`)

   - definition of tokens and token-streams (`syntax::{token, tokenstream}`) -- used by `syntax::ast`

   - visitors (`syntax::visit`, `syntax::mut_visit`)

   - shared definitions between `libsyntax_expand`

   - feature gating (`syntax::feature_gate`) -- we could possibly move this out to its own crater later.

   - attribute and meta item utilities, including used-marking (`syntax::attr`)

   - pretty printer (`syntax::print`) -- this should possibly be moved out later. For now I've reduced down the dependencies to a single essential one which could be broken via `ParseSess`. This entails that e.g. `Debug` impls for `Path` cannot reference the pretty printer.

   - definition of `ParseSess` (`syntax::sess`) -- this is used by `syntax::{attr, print, feature_gate}` and is a common definition used by the parser and other things like librustc.

   - the `syntax::source_map` -- this includes definitions used by `syntax::ast` and other things but could ostensibly be moved `syntax_pos` since that is more related to this module.

   - a smattering of misc utilities not sufficiently important to itemize -- some of these could be moved to where they are used (often a single place) but I wanted to limit the scope of this PR.

- librustc_parse:

   - parser (`rustc_parse::parser`) -- reading a file and such are defined in the crate root tho.

   - lexer (`rustc_parse::lexer`)

   - validation of meta grammar (post-expansion) in (`rustc_parse::validate_attr`)

- libsyntax_expand -- this defines the infra for macro expansion and conditional compilation but this is not libsyntax_ext; we might want to merge them later but currently libsyntax_expand is depended on by librustc_metadata which libsyntax_ext is not.

   - conditional compilation (`syntax_expand::config`) -- moved from `syntax::config` to here

   - the bulk of this crate is made up of the old `syntax::ext`

r? @estebank

Author: bors

Cargo.lock

@@ -3504,6 +3504,7 @@ dependencies = [
  "rustc_lint",
  "rustc_metadata",
  "rustc_mir",
+ "rustc_parse",
  "rustc_plugin",
  "rustc_plugin_impl",
  "rustc_resolve",
@@ -3572,6 +3573,7 @@ dependencies = [
  "rustc_lint",
  "rustc_metadata",
  "rustc_mir",
+ "rustc_parse",
  "rustc_passes",
  "rustc_plugin_impl",
  "rustc_privacy",
@@ -3649,6 +3651,7 @@ dependencies = [
  "rustc_data_structures",
  "rustc_errors",
  "rustc_index",
+ "rustc_parse",
  "rustc_target",
  "serialize",
  "smallvec 1.0.0",
@@ -3692,6 +3695,21 @@ dependencies = [
  "core",
 ]
 
+[[package]]
+name = "rustc_parse"
+version = "0.0.0"
+dependencies = [
+ "bitflags",
+ "log",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_lexer",
+ "rustc_target",
+ "smallvec 1.0.0",
+ "syntax",
+ "syntax_pos",
+]
+
 [[package]]
 name = "rustc_passes"
 version = "0.0.0"
@@ -3701,6 +3719,7 @@ dependencies = [
  "rustc_data_structures",
  "rustc_errors",
  "rustc_index",
+ "rustc_parse",
  "rustc_target",
  "syntax",
  "syntax_pos",
@@ -3763,6 +3782,7 @@ dependencies = [
  "rustc",
  "rustc_codegen_utils",
  "rustc_data_structures",
+ "rustc_parse",
  "serde_json",
  "syntax",
  "syntax_pos",
@@ -4372,14 +4392,11 @@ dependencies = [
 name = "syntax_expand"
 version = "0.0.0"
 dependencies = [
- "bitflags",
- "lazy_static 1.3.0",
  "log",
  "rustc_data_structures",
  "rustc_errors",
- "rustc_index",
  "rustc_lexer",
- "scoped-tls",
+ "rustc_parse",
  "serialize",
  "smallvec 1.0.0",
  "syntax",
@@ -4394,6 +4411,7 @@ dependencies = [
  "log",
  "rustc_data_structures",
  "rustc_errors",
+ "rustc_parse",
  "rustc_target",
  "smallvec 1.0.0",
  "syntax",

src/librustc/session/mod.rs

@@ -26,7 +26,7 @@ use syntax::expand::allocator::AllocatorKind;
 use syntax::feature_gate::{self, AttributeType};
 use syntax::json::JsonEmitter;
 use syntax::source_map;
-use syntax::sess::ParseSess;
+use syntax::sess::{ParseSess, ProcessCfgMod};
 use syntax::symbol::Symbol;
 use syntax_pos::{MultiSpan, Span};
 use crate::util::profiling::{SelfProfiler, SelfProfilerRef};
@@ -934,6 +934,7 @@ pub fn build_session(
     sopts: config::Options,
     local_crate_source_file: Option<PathBuf>,
     registry: errors::registry::Registry,
+    process_cfg_mod: ProcessCfgMod,
 ) -> Session {
     let file_path_mapping = sopts.file_path_mapping();
 
@@ -944,6 +945,7 @@ pub fn build_session(
         Lrc::new(source_map::SourceMap::new(file_path_mapping)),
         DiagnosticOutput::Default,
         Default::default(),
+        process_cfg_mod,
     )
 }
 
@@ -1022,6 +1024,7 @@ pub fn build_session_with_source_map(
     source_map: Lrc<source_map::SourceMap>,
     diagnostics_output: DiagnosticOutput,
     lint_caps: FxHashMap<lint::LintId, lint::Level>,
+    process_cfg_mod: ProcessCfgMod,
 ) -> Session {
     // FIXME: This is not general enough to make the warning lint completely override
     // normal diagnostic warnings, since the warning lint can also be denied and changed
@@ -1062,7 +1065,14 @@ pub fn build_session_with_source_map(
         },
     );
 
-    build_session_(sopts, local_crate_source_file, diagnostic_handler, source_map, lint_caps)
+    build_session_(
+        sopts,
+        local_crate_source_file,
+        diagnostic_handler,
+        source_map,
+        lint_caps,
+        process_cfg_mod,
+    )
 }
 
 fn build_session_(
@@ -1071,6 +1081,7 @@ fn build_session_(
     span_diagnostic: errors::Handler,
     source_map: Lrc<source_map::SourceMap>,
     driver_lint_caps: FxHashMap<lint::LintId, lint::Level>,
+    process_cfg_mod: ProcessCfgMod,
 ) -> Session {
     let self_profiler =
         if let SwitchWithOptPath::Enabled(ref d) = sopts.debugging_opts.self_profile {
@@ -1109,6 +1120,7 @@ fn build_session_(
     let parse_sess = ParseSess::with_span_handler(
         span_diagnostic,
         source_map,
+        process_cfg_mod,
     );
     let sysroot = match &sopts.maybe_sysroot {
         Some(sysroot) => sysroot.clone(),