Rollup merge of #63919 - matthewjasper:remove-gensymmed, r=petrochenkov

Use hygiene for AST passes

AST passes are now able to have resolve consider their expansions as if they were opaque macros defined either in some module in the current crate, or a fake empty module with `#[no_implicit_prelude]`.

* Add an ExpnKind for AST passes.
* Remove gensyms in AST passes.
* Remove gensyms in`#[test]`, `#[bench]` and `#[test_case]`.
* Allow opaque macros to define tests.
* Move tests for unit tests to their own directory.
* Remove `Ident::{gensym, is_gensymed}` - `Ident::gensym_if_underscore` still exists.

cc #60869, #61019

r? @petrochenkov

Author: Centril

src/librustc/ich/impls_syntax.rs

@@ -390,9 +390,17 @@ impl_stable_hash_for!(struct ::syntax_pos::hygiene::ExpnData {
 impl_stable_hash_for!(enum ::syntax_pos::hygiene::ExpnKind {
     Root,
     Macro(kind, descr),
+    AstPass(kind),
     Desugaring(kind)
 });
 
+impl_stable_hash_for!(enum ::syntax_pos::hygiene::AstPass {
+    StdImports,
+    TestHarness,
+    ProcMacroHarness,
+    PluginMacroDefs,
+});
+
 impl_stable_hash_for!(enum ::syntax_pos::hygiene::DesugaringKind {
     CondTemporary,
     Async,

src/librustc/lint/mod.rs

@@ -646,6 +646,30 @@ pub fn struct_lint_level<'a>(sess: &'a Session,
         (Level::Forbid, None) => sess.struct_err(msg),
     };
 
+    // Check for future incompatibility lints and issue a stronger warning.
+    let lints = sess.lint_store.borrow();
+    let lint_id = LintId::of(lint);
+    let future_incompatible = lints.future_incompatible(lint_id);
+
+    // If this code originates in a foreign macro, aka something that this crate
+    // did not itself author, then it's likely that there's nothing this crate
+    // can do about it. We probably want to skip the lint entirely.
+    if err.span.primary_spans().iter().any(|s| in_external_macro(sess, *s)) {
+        // Any suggestions made here are likely to be incorrect, so anything we
+        // emit shouldn't be automatically fixed by rustfix.
+        err.allow_suggestions(false);
+
+        // If this is a future incompatible lint it'll become a hard error, so
+        // we have to emit *something*. Also allow lints to whitelist themselves
+        // on a case-by-case basis for emission in a foreign macro.
+        if future_incompatible.is_none() && !lint.report_in_external_macro {
+            err.cancel();
+            // Don't continue further, since we don't want to have
+            // `diag_span_note_once` called for a diagnostic that isn't emitted.
+            return err;
+        }
+    }
+
     let name = lint.name_lower();
     match src {
         LintSource::Default => {
@@ -695,10 +719,6 @@ pub fn struct_lint_level<'a>(sess: &'a Session,
 
     err.code(DiagnosticId::Lint(name));
 
-    // Check for future incompatibility lints and issue a stronger warning.
-    let lints = sess.lint_store.borrow();
-    let lint_id = LintId::of(lint);
-    let future_incompatible = lints.future_incompatible(lint_id);
     if let Some(future_incompatible) = future_incompatible {
         const STANDARD_MESSAGE: &str =
             "this was previously accepted by the compiler but is being phased out; \
@@ -723,22 +743,6 @@ pub fn struct_lint_level<'a>(sess: &'a Session,
         err.note(&citation);
     }
 
-    // If this code originates in a foreign macro, aka something that this crate
-    // did not itself author, then it's likely that there's nothing this crate
-    // can do about it. We probably want to skip the lint entirely.
-    if err.span.primary_spans().iter().any(|s| in_external_macro(sess, *s)) {
-        // Any suggestions made here are likely to be incorrect, so anything we
-        // emit shouldn't be automatically fixed by rustfix.
-        err.allow_suggestions(false);
-
-        // If this is a future incompatible lint it'll become a hard error, so
-        // we have to emit *something*. Also allow lints to whitelist themselves
-        // on a case-by-case basis for emission in a foreign macro.
-        if future_incompatible.is_none() && !lint.report_in_external_macro {
-            err.cancel()
-        }
-    }
-
     return err
 }
 
@@ -868,7 +872,7 @@ pub fn in_external_macro(sess: &Session, span: Span) -> bool {
     let expn_data = span.ctxt().outer_expn_data();
     match expn_data.kind {
         ExpnKind::Root | ExpnKind::Desugaring(DesugaringKind::ForLoop) => false,
-        ExpnKind::Desugaring(_) => true, // well, it's "external"
+        ExpnKind::AstPass(_) | ExpnKind::Desugaring(_) => true, // well, it's "external"
         ExpnKind::Macro(MacroKind::Bang, _) => {
             if expn_data.def_site.is_dummy() {
                 // dummy span for the def_site means it's an external macro

src/librustc_interface/passes.rs

@@ -233,7 +233,7 @@ pub fn register_plugins<'a>(
         syntax::attr::inject(krate, &sess.parse_sess, &sess.opts.debugging_opts.crate_attr)
     });
 
-    let (mut krate, features) = syntax::config::features(
+    let (krate, features) = syntax::config::features(
         krate,
         &sess.parse_sess,
         sess.edition(),
@@ -268,16 +268,6 @@ pub fn register_plugins<'a>(
         middle::recursion_limit::update_limits(sess, &krate);
     });
 
-    krate = time(sess, "crate injection", || {
-        let alt_std_name = sess.opts.alt_std_name.as_ref().map(|s| &**s);
-        let (krate, name) =
-            syntax_ext::standard_library_imports::inject(krate, alt_std_name, sess.edition());
-        if let Some(name) = name {
-            sess.parse_sess.injected_crate_name.set(name);
-        }
-        krate
-    });
-
     let registrars = time(sess, "plugin loading", || {
         plugin::load::load_plugins(
             sess,
@@ -370,6 +360,21 @@ fn configure_and_expand_inner<'a>(
         &resolver_arenas,
     );
     syntax_ext::register_builtin_macros(&mut resolver, sess.edition());
+
+    krate = time(sess, "crate injection", || {
+        let alt_std_name = sess.opts.alt_std_name.as_ref().map(|s| Symbol::intern(s));
+        let (krate, name) = syntax_ext::standard_library_imports::inject(
+            krate,
+            &mut resolver,
+            &sess.parse_sess,
+            alt_std_name,
+        );
+        if let Some(name) = name {
+            sess.parse_sess.injected_crate_name.set(name);
+        }
+        krate
+    });
+
     syntax_ext::plugin_macro_defs::inject(
         &mut krate, &mut resolver, plugin_info.syntax_exts, sess.edition()
     );

src/librustc_resolve/build_reduced_graph.rs

@@ -126,7 +126,8 @@ impl<'a> Resolver<'a> {
     crate fn macro_def_scope(&mut self, expn_id: ExpnId) -> Module<'a> {
         let def_id = match self.macro_defs.get(&expn_id) {
             Some(def_id) => *def_id,
-            None => return self.graph_root,
+            None => return self.ast_transform_scopes.get(&expn_id)
+                .unwrap_or(&self.graph_root),
         };
         if let Some(id) = self.definitions.as_local_node_id(def_id) {
             self.local_macro_def_scopes[&id]

src/librustc_resolve/diagnostics.rs

@@ -604,6 +604,14 @@ impl<'a> Resolver<'a> {
         if lookup_ident.span.rust_2018() {
             let extern_prelude_names = self.extern_prelude.clone();
             for (ident, _) in extern_prelude_names.into_iter() {
+                if ident.span.from_expansion() {
+                    // Idents are adjusted to the root context before being
+                    // resolved in the extern prelude, so reporting this to the
+                    // user is no help. This skips the injected
+                    // `extern crate std` in the 2018 edition, which would
+                    // otherwise cause duplicate suggestions.
+                    continue;
+                }
                 if let Some(crate_id) = self.crate_loader.maybe_process_path_extern(ident.name,
                                                                                     ident.span) {
                     let crate_root = self.get_module(DefId {

src/librustc_resolve/lib.rs

@@ -880,6 +880,10 @@ pub struct Resolver<'a> {
     /// There will be an anonymous module created around `g` with the ID of the
     /// entry block for `f`.
     block_map: NodeMap<Module<'a>>,
+    /// A fake module that contains no definition and no prelude. Used so that
+    /// some AST passes can generate identifiers that only resolve to local or
+    /// language items.
+    empty_module: Module<'a>,
     module_map: FxHashMap<DefId, Module<'a>>,
     extern_module_map: FxHashMap<(DefId, bool /* MacrosOnly? */), Module<'a>>,
     binding_parent_modules: FxHashMap<PtrKey<'a, NameBinding<'a>>, Module<'a>>,
@@ -914,6 +918,7 @@ pub struct Resolver<'a> {
     non_macro_attrs: [Lrc<SyntaxExtension>; 2],
     macro_defs: FxHashMap<ExpnId, DefId>,
     local_macro_def_scopes: FxHashMap<NodeId, Module<'a>>,
+    ast_transform_scopes: FxHashMap<ExpnId, Module<'a>>,
     unused_macros: NodeMap<Span>,
     proc_macro_stubs: NodeSet,
     /// Traces collected during macro resolution and validated when it's complete.
@@ -1081,6 +1086,21 @@ impl<'a> Resolver<'a> {
             no_implicit_prelude: attr::contains_name(&krate.attrs, sym::no_implicit_prelude),
             ..ModuleData::new(None, root_module_kind, root_def_id, ExpnId::root(), krate.span)
         });
+        let empty_module_kind = ModuleKind::Def(
+            DefKind::Mod,
+            root_def_id,
+            kw::Invalid,
+        );
+        let empty_module = arenas.alloc_module(ModuleData {
+            no_implicit_prelude: true,
+            ..ModuleData::new(
+                Some(graph_root),
+                empty_module_kind,
+                root_def_id,
+                ExpnId::root(),
+                DUMMY_SP,
+            )
+        });
         let mut module_map = FxHashMap::default();
         module_map.insert(DefId::local(CRATE_DEF_INDEX), graph_root);
 
@@ -1140,10 +1160,12 @@ impl<'a> Resolver<'a> {
             label_res_map: Default::default(),
             export_map: FxHashMap::default(),
             trait_map: Default::default(),
+            empty_module,
             module_map,
             block_map: Default::default(),
             extern_module_map: FxHashMap::default(),
             binding_parent_modules: FxHashMap::default(),
+            ast_transform_scopes: FxHashMap::default(),
 
             glob_map: Default::default(),
 

src/librustc_resolve/macros.rs

@@ -8,6 +8,7 @@ use crate::{ModuleOrUniformRoot, KNOWN_TOOLS};
 use crate::Namespace::*;
 use crate::resolve_imports::ImportResolver;
 use rustc::hir::def::{self, DefKind, NonMacroAttrKind};
+use rustc::hir::def_id;
 use rustc::middle::stability;
 use rustc::{ty, lint, span_bug};
 use syntax::ast::{self, NodeId, Ident};
@@ -25,6 +26,7 @@ use syntax_pos::{Span, DUMMY_SP};
 
 use std::{mem, ptr};
 use rustc_data_structures::sync::Lrc;
+use syntax_pos::hygiene::AstPass;
 
 type Res = def::Res<NodeId>;
 
@@ -95,16 +97,6 @@ impl<'a> base::Resolver for Resolver<'a> {
         self.session.next_node_id()
     }
 
-    fn get_module_scope(&mut self, id: NodeId) -> ExpnId {
-        let expn_id = ExpnId::fresh(Some(ExpnData::default(
-            ExpnKind::Macro(MacroKind::Attr, sym::test_case), DUMMY_SP, self.session.edition()
-        )));
-        let module = self.module_map[&self.definitions.local_def_id(id)];
-        self.invocation_parent_scopes.insert(expn_id, ParentScope::module(module));
-        self.definitions.set_invocation_parent(expn_id, module.def_id().unwrap().index);
-        expn_id
-    }
-
     fn resolve_dollar_crates(&mut self) {
         hygiene::update_dollar_crate_names(|ctxt| {
             let ident = Ident::new(kw::DollarCrate, DUMMY_SP.with_ctxt(ctxt));
@@ -136,6 +128,37 @@ impl<'a> base::Resolver for Resolver<'a> {
         }
     }
 
+    // Create a new Expansion with a definition site of the provided module, or
+    // a fake empty `#[no_implicit_prelude]` module if no module is provided.
+    fn expansion_for_ast_pass(
+        &mut self,
+        call_site: Span,
+        pass: AstPass,
+        features: &[Symbol],
+        parent_module_id: Option<NodeId>,
+    ) -> ExpnId {
+        let expn_id = ExpnId::fresh(Some(ExpnData::allow_unstable(
+            ExpnKind::AstPass(pass),
+            call_site,
+            self.session.edition(),
+            features.into(),
+        )));
+
+        let parent_scope = if let Some(module_id) = parent_module_id {
+            let parent_def_id = self.definitions.local_def_id(module_id);
+            self.definitions.add_parent_module_of_macro_def(expn_id, parent_def_id);
+            self.module_map[&parent_def_id]
+        } else {
+            self.definitions.add_parent_module_of_macro_def(
+                expn_id,
+                def_id::DefId::local(def_id::CRATE_DEF_INDEX),
+            );
+            self.empty_module
+        };
+        self.ast_transform_scopes.insert(expn_id, parent_scope);
+        expn_id
+    }
+
     fn resolve_imports(&mut self) {
         ImportResolver { r: self }.resolve_imports()
     }

src/librustc_resolve/resolve_imports.rs

@@ -1307,12 +1307,11 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                 None => continue,
             };
 
-            // Filter away ambiguous and gensymed imports. Gensymed imports
-            // (e.g. implicitly injected `std`) cannot be properly encoded in metadata,
-            // so they can cause name conflict errors downstream.
-            let is_good_import = binding.is_import() && !binding.is_ambiguity() &&
-                                 // Note that as_str() de-gensyms the Symbol
-                                 !(ident.is_gensymed() && ident.name.as_str() != "_");
+            // Filter away ambiguous imports and anything that has def-site
+            // hygiene.
+            // FIXME: Implement actual cross-crate hygiene.
+            let is_good_import = binding.is_import() && !binding.is_ambiguity()
+                && !ident.span.modern().from_expansion();
             if is_good_import || binding.is_macro_def() {
                 let res = binding.res();
                 if res != Res::Err {

src/libsyntax/ext/base.rs

@@ -3,7 +3,7 @@ use crate::attr::{self, HasAttrs, Stability, Deprecation};
 use crate::source_map::SourceMap;
 use crate::edition::Edition;
 use crate::ext::expand::{self, AstFragment, Invocation};
-use crate::ext::hygiene::{ExpnId, Transparency};
+use crate::ext::hygiene::ExpnId;
 use crate::mut_visit::{self, MutVisitor};
 use crate::parse::{self, parser, ParseSess, DirectoryOwnership};
 use crate::parse::token;
@@ -16,7 +16,7 @@ use crate::visit::Visitor;
 use errors::{DiagnosticBuilder, DiagnosticId};
 use smallvec::{smallvec, SmallVec};
 use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP};
-use syntax_pos::hygiene::{ExpnData, ExpnKind};
+use syntax_pos::hygiene::{AstPass, ExpnData, ExpnKind};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
@@ -732,13 +732,19 @@ bitflags::bitflags! {
 pub trait Resolver {
     fn next_node_id(&mut self) -> NodeId;
 
-    fn get_module_scope(&mut self, id: NodeId) -> ExpnId;
-
     fn resolve_dollar_crates(&mut self);
     fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment,
                                             extra_placeholders: &[NodeId]);
     fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension);
 
+    fn expansion_for_ast_pass(
+        &mut self,
+        call_site: Span,
+        pass: AstPass,
+        features: &[Symbol],
+        parent_module_id: Option<NodeId>,
+    ) -> ExpnId;
+
     fn resolve_imports(&mut self);
 
     fn resolve_macro_invocation(
@@ -822,20 +828,20 @@ impl<'a> ExtCtxt<'a> {
     /// Equivalent of `Span::def_site` from the proc macro API,
     /// except that the location is taken from the span passed as an argument.
     pub fn with_def_site_ctxt(&self, span: Span) -> Span {
-        span.with_ctxt_from_mark(self.current_expansion.id, Transparency::Opaque)
+        span.with_def_site_ctxt(self.current_expansion.id)
     }
 
     /// Equivalent of `Span::call_site` from the proc macro API,
     /// except that the location is taken from the span passed as an argument.
     pub fn with_call_site_ctxt(&self, span: Span) -> Span {
-        span.with_ctxt_from_mark(self.current_expansion.id, Transparency::Transparent)
+        span.with_call_site_ctxt(self.current_expansion.id)
     }
 
     /// Span with a context reproducing `macro_rules` hygiene (hygienic locals, unhygienic items).
     /// FIXME: This should be eventually replaced either with `with_def_site_ctxt` (preferably),
     /// or with `with_call_site_ctxt` (where necessary).
     pub fn with_legacy_ctxt(&self, span: Span) -> Span {
-        span.with_ctxt_from_mark(self.current_expansion.id, Transparency::SemiTransparent)
+        span.with_legacy_ctxt(self.current_expansion.id)
     }
 
     /// Returns span for the macro which originally caused the current expansion to happen.

src/libsyntax/print/pprust.rs

@@ -128,10 +128,14 @@ pub fn print_crate<'a>(cm: &'a SourceMap,
         let fake_attr = attr::mk_attr_inner(list);
         s.print_attribute(&fake_attr);
 
-        // #![no_std]
-        let no_std_meta = attr::mk_word_item(ast::Ident::with_dummy_span(sym::no_std));
-        let fake_attr = attr::mk_attr_inner(no_std_meta);
-        s.print_attribute(&fake_attr);
+        // Currently on Rust 2018 we don't have `extern crate std;` at the crate
+        // root, so this is not needed, and actually breaks things.
+        if sess.edition == syntax_pos::edition::Edition::Edition2015 {
+            // #![no_std]
+            let no_std_meta = attr::mk_word_item(ast::Ident::with_dummy_span(sym::no_std));
+            let fake_attr = attr::mk_attr_inner(no_std_meta);
+            s.print_attribute(&fake_attr);
+        }
     }
 
     s.print_mod(&krate.module, &krate.attrs);