Do not get proc_macro from the sysroot in rustc

With the stage0 refactor the proc_macro version found in the sysroot
will no longer always match the proc_macro version that proc-macros get
compiled with by the rustc executable that uses this proc_macro. This
will cause problems as soon as the ABI of the bridge gets changed to
implement new features or change the way existing features work.

To fix this, this commit changes rustc crates to depend directly on the
local version of proc_macro which will also be used in the sysroot that
rustc will build.

Author: bjorn3

Cargo.lock

@@ -3444,6 +3444,7 @@ dependencies = [
  "rustc_macros",
  "rustc_parse",
  "rustc_parse_format",
+ "rustc_proc_macro",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -3733,6 +3734,7 @@ dependencies = [
  "rustc_lint_defs",
  "rustc_macros",
  "rustc_parse",
+ "rustc_proc_macro",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",
@@ -4081,6 +4083,7 @@ dependencies = [
  "rustc_index",
  "rustc_macros",
  "rustc_middle",
+ "rustc_proc_macro",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",
@@ -4337,6 +4340,13 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "rustc_proc_macro"
+version = "0.0.0"
+dependencies = [
+ "rustc-literal-escaper",
+]
+
 [[package]]
 name = "rustc_query_impl"
 version = "0.0.0"

compiler/rustc_builtin_macros/Cargo.toml

@@ -8,6 +8,9 @@ doctest = false
 
 [dependencies]
 # tidy-alphabetical-start
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }

compiler/rustc_builtin_macros/src/lib.rs

@@ -20,8 +20,6 @@
 #![recursion_limit = "256"]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 use std::sync::Arc;
 
 use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};
@@ -139,7 +137,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         CoercePointee: coerce_pointee::expand_deriving_coerce_pointee,
     }
 
-    let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
+    let client = rustc_proc_macro::bridge::client::Client::expand1(rustc_proc_macro::quote);
     register(sym::quote, SyntaxExtensionKind::Bang(Arc::new(BangProcMacro { client })));
     let requires = SyntaxExtensionKind::Attr(Arc::new(contracts::ExpandRequires));
     register(sym::contracts_requires, requires);

compiler/rustc_expand/Cargo.toml

@@ -9,6 +9,9 @@ doctest = false
 
 [dependencies]
 # tidy-alphabetical-start
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }

compiler/rustc_expand/src/lib.rs

@@ -14,8 +14,6 @@
 #![feature(yeet_expr)]
 // tidy-alphabetical-end
 
-extern crate proc_macro as pm;
-
 mod build;
 mod errors;
 // FIXME(Nilstrieb) Translate macro_rules diagnostics

compiler/rustc_expand/src/proc_macro.rs

@@ -1,11 +1,11 @@
-use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::ErrorGuaranteed;
 use rustc_parse::parser::{ForceCollect, Parser};
 use rustc_session::config::ProcMacroExecutionStrategy;
 use rustc_span::Span;
 use rustc_span::profiling::SpannedEventArgRecorder;
+use {rustc_ast as ast, rustc_proc_macro as pm};
 
 use crate::base::{self, *};
 use crate::{errors, proc_macro_server};

compiler/rustc_expand/src/proc_macro_server.rs

@@ -1,10 +1,6 @@
 use std::ops::{Bound, Range};
 
 use ast::token::IdentIsRaw;
-use pm::bridge::{
-    DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
-};
-use pm::{Delimiter, Level};
 use rustc_ast as ast;
 use rustc_ast::token;
 use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream};
@@ -15,6 +11,10 @@ use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
 use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::parser::Parser;
 use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
+use rustc_proc_macro::bridge::{
+    DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
+};
+use rustc_proc_macro::{Delimiter, Level};
 use rustc_session::parse::ParseSess;
 use rustc_span::def_id::CrateNum;
 use rustc_span::{BytePos, FileName, Pos, Span, Symbol, sym};
@@ -66,7 +66,7 @@ impl FromInternal<token::LitKind> for LitKind {
             token::CStr => LitKind::CStr,
             token::CStrRaw(n) => LitKind::CStrRaw(n),
             token::Err(_guar) => {
-                // This is the only place a `pm::bridge::LitKind::ErrWithGuar`
+                // This is the only place a `rustc_proc_macro::bridge::LitKind::ErrWithGuar`
                 // is constructed. Note that an `ErrorGuaranteed` is available,
                 // as required. See the comment in `to_internal`.
                 LitKind::ErrWithGuar
@@ -149,7 +149,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     }
 
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::from_internal(delim),
+                        delimiter: rustc_proc_macro::Delimiter::from_internal(delim),
                         stream: Some(stream),
                         span: DelimSpan {
                             open: span.open,
@@ -270,7 +270,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     let stream =
                         TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span);
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::None,
+                        delimiter: rustc_proc_macro::Delimiter::None,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }))
@@ -302,7 +302,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                         trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
                     }
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::Bracket,
+                        delimiter: rustc_proc_macro::Delimiter::Bracket,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }));

compiler/rustc_metadata/Cargo.toml

@@ -8,6 +8,9 @@ edition = "2024"
 bitflags = "2.4.1"
 libloading = "0.8.0"
 odht = { version = "0.3.1", features = ["nightly"] }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }

compiler/rustc_metadata/src/creader.rs

@@ -7,7 +7,6 @@ use std::str::FromStr;
 use std::time::Duration;
 use std::{cmp, env, iter};
 
-use proc_macro::bridge::client::ProcMacro;
 use rustc_ast::expand::allocator::{AllocatorKind, alloc_error_handler_name, global_fn_name};
 use rustc_ast::{self as ast, *};
 use rustc_data_structures::fx::FxHashSet;
@@ -23,6 +22,7 @@ use rustc_hir::definitions::Definitions;
 use rustc_index::IndexVec;
 use rustc_middle::bug;
 use rustc_middle::ty::{TyCtxt, TyCtxtFeed};
+use rustc_proc_macro::bridge::client::ProcMacro;
 use rustc_session::config::{
     self, CrateType, ExtendedTargetModifierInfo, ExternLocation, OptionsTargetModifiers,
     TargetModifier,

compiler/rustc_metadata/src/lib.rs

@@ -16,8 +16,6 @@
 #![feature(trusted_len)]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 pub use rmeta::provide;
 
 mod dependency_format;

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -6,7 +6,6 @@ use std::sync::{Arc, OnceLock};
 use std::{io, iter, mem};
 
 pub(super) use cstore_impl::provide;
-use proc_macro::bridge::client::ProcMacro;
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxIndexMap;
@@ -26,6 +25,7 @@ use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use rustc_middle::ty::Visibility;
 use rustc_middle::ty::codec::TyDecoder;
 use rustc_middle::{bug, implement_ty_decoder};
+use rustc_proc_macro::bridge::client::ProcMacro;
 use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::{Decodable, Decoder};
 use rustc_session::Session;

compiler/rustc_proc_macro/Cargo.toml

@@ -0,0 +1,19 @@
+# We need to use a separate crate including library/proc_macro as opposed to a
+# direct path dependency on library/proc_macro because doing the latter will
+# cause two copies of libproc_macro.rlib to end up in the sysroot, breaking
+# proc-macro crates. In addition it confuses the workspace_members function of
+# bootstrap.
+
+[package]
+name = "rustc_proc_macro"
+version = "0.0.0"
+edition = "2024"
+
+[lib]
+path = "../../library/proc_macro/src/lib.rs"
+
+[dependencies]
+rustc-literal-escaper = "0.0.2"
+
+[features]
+rustc-dep-of-std = []

library/proc_macro/Cargo.toml

@@ -10,3 +10,7 @@ std = { path = "../std" }
 # loaded from sysroot causing duplicate lang items and other similar errors.
 core = { path = "../core" }
 rustc-literal-escaper = { version = "0.0.2", features = ["rustc-dep-of-std"] }
+
+[features]
+default = ["rustc-dep-of-std"]
+rustc-dep-of-std = []

library/proc_macro/src/lib.rs

@@ -32,6 +32,7 @@
 #![recursion_limit = "256"]
 #![allow(internal_features)]
 #![deny(ffi_unwind_calls)]
+#![allow(rustc::internal)] // Can't use FxHashMap when compiled as part of the standard library
 #![warn(rustdoc::unescaped_backticks)]
 #![warn(unreachable_pub)]
 #![deny(unsafe_op_in_unsafe_fn)]
@@ -95,7 +96,7 @@ pub fn is_available() -> bool {
 ///
 /// This is both the input and output of `#[proc_macro]`, `#[proc_macro_attribute]`
 /// and `#[proc_macro_derive]` definitions.
-#[rustc_diagnostic_item = "TokenStream"]
+#[cfg_attr(feature = "rustc-dep-of-std", rustc_diagnostic_item = "TokenStream")]
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 #[derive(Clone)]
 pub struct TokenStream(Option<bridge::client::TokenStream>);

src/bootstrap/src/core/build_steps/dist.rs

@@ -776,7 +776,8 @@ impl Step for RustcDev {
         copy_src_dirs(
             builder,
             &builder.src,
-            &["compiler"],
+            // The compiler has a path dependency on proc_macro, so make sure to include it.
+            &["compiler", "library/proc_macro"],
             &[],
             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
         );