Do not get proc_macro from the sysroot in rustc

With the stage0 refactor the proc_macro version found in the sysroot
will no longer always match the proc_macro version that proc-macros get
compiled with by the rustc executable that uses this proc_macro. This
will cause problems as soon as the ABI of the bridge gets changed to
implement new features or change the way existing features work.

To fix this, this commit changes rustc crates to depend directly on the
local version of proc_macro which will also be used in the sysroot that
rustc will build.

Author: bjorn3

Cargo.lock

@@ -2893,6 +2893,13 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "proc_macro"
+version = "0.0.0"
+dependencies = [
+ "rustc-literal-escaper",
+]
+
 [[package]]
 name = "psm"
 version = "0.1.26"
@@ -3428,6 +3435,7 @@ dependencies = [
 name = "rustc_builtin_macros"
 version = "0.0.0"
 dependencies = [
+ "proc_macro",
  "rustc_ast",
  "rustc_ast_pretty",
  "rustc_attr_data_structures",
@@ -3719,6 +3727,7 @@ dependencies = [
 name = "rustc_expand"
 version = "0.0.0"
 dependencies = [
+ "proc_macro",
  "rustc_ast",
  "rustc_ast_passes",
  "rustc_ast_pretty",
@@ -4066,6 +4075,7 @@ dependencies = [
  "libc",
  "libloading",
  "odht",
+ "proc_macro",
  "rustc_abi",
  "rustc_ast",
  "rustc_attr_data_structures",

Cargo.toml

@@ -54,6 +54,8 @@ exclude = [
   "build",
   "compiler/rustc_codegen_cranelift",
   "compiler/rustc_codegen_gcc",
+  # Otherwise bootstrap complains that this crate is included in two separate workspaces
+  "library/proc_macro",
   "src/bootstrap",
   "tests/rustdoc-gui",
   # HACK(eddyb) This hardcodes the fact that our CI uses `/checkout/obj`.

compiler/rustc_builtin_macros/Cargo.toml

@@ -8,6 +8,9 @@ doctest = false
 
 [dependencies]
 # tidy-alphabetical-start
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+proc_macro = { path = "../../library/proc_macro", default-features = false }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }

compiler/rustc_builtin_macros/src/lib.rs

@@ -20,8 +20,6 @@
 #![recursion_limit = "256"]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 use std::sync::Arc;
 
 use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};

compiler/rustc_expand/Cargo.toml

@@ -9,6 +9,9 @@ doctest = false
 
 [dependencies]
 # tidy-alphabetical-start
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+proc_macro = { path = "../../library/proc_macro", default-features = false }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }

compiler/rustc_expand/src/lib.rs

@@ -14,8 +14,6 @@
 #![feature(yeet_expr)]
 // tidy-alphabetical-end
 
-extern crate proc_macro as pm;
-
 mod build;
 mod errors;
 // FIXME(Nilstrieb) Translate macro_rules diagnostics

compiler/rustc_expand/src/proc_macro.rs

@@ -1,11 +1,11 @@
-use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::ErrorGuaranteed;
 use rustc_parse::parser::{ForceCollect, Parser};
 use rustc_session::config::ProcMacroExecutionStrategy;
 use rustc_span::Span;
 use rustc_span::profiling::SpannedEventArgRecorder;
+use {proc_macro as pm, rustc_ast as ast};
 
 use crate::base::{self, *};
 use crate::{errors, proc_macro_server};

compiler/rustc_expand/src/proc_macro_server.rs

@@ -1,10 +1,10 @@
 use std::ops::{Bound, Range};
 
 use ast::token::IdentIsRaw;
-use pm::bridge::{
+use proc_macro::bridge::{
     DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
 };
-use pm::{Delimiter, Level};
+use proc_macro::{Delimiter, Level};
 use rustc_ast as ast;
 use rustc_ast::token;
 use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream};
@@ -66,7 +66,7 @@ impl FromInternal<token::LitKind> for LitKind {
             token::CStr => LitKind::CStr,
             token::CStrRaw(n) => LitKind::CStrRaw(n),
             token::Err(_guar) => {
-                // This is the only place a `pm::bridge::LitKind::ErrWithGuar`
+                // This is the only place a `proc_macro::bridge::LitKind::ErrWithGuar`
                 // is constructed. Note that an `ErrorGuaranteed` is available,
                 // as required. See the comment in `to_internal`.
                 LitKind::ErrWithGuar
@@ -149,7 +149,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     }
 
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::from_internal(delim),
+                        delimiter: proc_macro::Delimiter::from_internal(delim),
                         stream: Some(stream),
                         span: DelimSpan {
                             open: span.open,
@@ -270,7 +270,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     let stream =
                         TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span);
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::None,
+                        delimiter: proc_macro::Delimiter::None,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }))
@@ -302,7 +302,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                         trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
                     }
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::Bracket,
+                        delimiter: proc_macro::Delimiter::Bracket,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }));

compiler/rustc_metadata/Cargo.toml

@@ -8,6 +8,9 @@ edition = "2024"
 bitflags = "2.4.1"
 libloading = "0.8.0"
 odht = { version = "0.3.1", features = ["nightly"] }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+proc_macro = { path = "../../library/proc_macro", default-features = false }
 rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }

compiler/rustc_metadata/src/lib.rs

@@ -16,8 +16,6 @@
 #![feature(trusted_len)]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 pub use rmeta::provide;
 
 mod dependency_format;

library/proc_macro/Cargo.toml

@@ -4,9 +4,13 @@ version = "0.0.0"
 edition = "2024"
 
 [dependencies]
-std = { path = "../std" }
+std = { path = "../std", optional = true }
 # Workaround: when documenting this crate rustdoc will try to load crate named
 # `core` when resolving doc links. Without this line a different `core` will be
 # loaded from sysroot causing duplicate lang items and other similar errors.
-core = { path = "../core" }
-rustc-literal-escaper = { version = "0.0.2", features = ["rustc-dep-of-std"] }
+core = { path = "../core", optional = true }
+rustc-literal-escaper = "0.0.2"
+
+[features]
+default = ["rustc-dep-of-std"]
+rustc-dep-of-std = ["dep:core", "dep:std", "rustc-literal-escaper/rustc-dep-of-std"]

library/proc_macro/src/lib.rs

@@ -32,6 +32,7 @@
 #![recursion_limit = "256"]
 #![allow(internal_features)]
 #![deny(ffi_unwind_calls)]
+#![allow(rustc::internal)]
 #![warn(rustdoc::unescaped_backticks)]
 #![warn(unreachable_pub)]
 #![deny(unsafe_op_in_unsafe_fn)]
@@ -95,7 +96,7 @@ pub fn is_available() -> bool {
 ///
 /// This is both the input and output of `#[proc_macro]`, `#[proc_macro_attribute]`
 /// and `#[proc_macro_derive]` definitions.
-#[rustc_diagnostic_item = "TokenStream"]
+#[cfg_attr(feature = "rustc-dep-of-std", rustc_diagnostic_item = "TokenStream")]
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 #[derive(Clone)]
 pub struct TokenStream(Option<bridge::client::TokenStream>);

src/bootstrap/src/core/build_steps/dist.rs

@@ -776,7 +776,8 @@ impl Step for RustcDev {
         copy_src_dirs(
             builder,
             &builder.src,
-            &["compiler"],
+            // The compiler has a path dependency on proc_macro, so make sure to include it.
+            &["compiler", "library/proc_macro"],
             &[],
             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
         );