Auto merge of rust-lang#125419 - GuillaumeGomez:add-gcc-to-dist, r=Kobzol

[bootstrap] Add support for building gcc and libgccjit

As `@eholk` summarized below:

> From my understanding, this change would add libgccjit as an optional component to the Rust distribution. This library is licensed under GPLv2 and currently we do not have any other components under that license so it would be a new license, and one that is generally more restrictive than the other licenses we use.

It'll greatly improve the experience for anyone wanting to work on the GCC backend from the compiler.

Should help with rust-lang#124172.
Will unblock rust-lang#124353.

r? `@Kobzol`

Author: bors

Diff for: .gitmodules

@@ -51,3 +51,7 @@
 	path = src/tools/enzyme
 	url = https://github.com/EnzymeAD/Enzyme.git
 	shallow = true
+[submodule "src/gcc"]
+	path = src/gcc
+	url = https://github.com/rust-lang/gcc.git
+	shallow = true

Diff for: LICENSES/GCC-exception-3.1.txt

@@ -0,0 +1,30 @@
+GCC RUNTIME LIBRARY EXCEPTION
+
+Version 3.1, 31 March 2009
+
+Copyright © 2009 Free Software Foundation, Inc. <https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
+
+This GCC Runtime Library Exception ("Exception") is an additional permission under section 7 of the GNU General Public License, version 3 ("GPLv3"). It applies to a given file (the "Runtime Library") that bears a notice placed by the copyright holder of the file stating that the file is governed by GPLv3 along with this Exception.
+
+When you use GCC to compile a program, GCC may combine portions of certain GCC header files and runtime libraries with the compiled program. The purpose of this Exception is to allow compilation of non-GPL (including proprietary) programs to use, in this way, the header files and runtime libraries covered by this Exception.
+0. Definitions.
+
+A file is an "Independent Module" if it either requires the Runtime Library for execution after a Compilation Process, or makes use of an interface provided by the Runtime Library, but is not otherwise based on the Runtime Library.
+
+"GCC" means a version of the GNU Compiler Collection, with or without modifications, governed by version 3 (or a specified later version) of the GNU General Public License (GPL) with the option of using any subsequent versions published by the FSF.
+
+"GPL-compatible Software" is software whose conditions of propagation, modification and use would permit combination with GCC in accord with the license of GCC.
+
+"Target Code" refers to output from any compiler for a real or virtual target processor architecture, in executable form or suitable for input to an assembler, loader, linker and/or execution phase. Notwithstanding that, Target Code does not include data in any format that is used as a compiler intermediate representation, or used for producing a compiler intermediate representation.
+
+The "Compilation Process" transforms code entirely represented in non-intermediate languages designed for human-written code, and/or in Java Virtual Machine byte code, into Target Code. Thus, for example, use of source code generators and preprocessors need not be considered part of the Compilation Process, since the Compilation Process can be understood as starting with the output of the generators or preprocessors.
+
+A Compilation Process is "Eligible" if it is done using GCC, alone or with other GPL-compatible software, or if it is done without using any work based on GCC. For example, using non-GPL-compatible Software to optimize any GCC intermediate representations would not qualify as an Eligible Compilation Process.
+1. Grant of Additional Permission.
+
+You have permission to propagate a work of Target Code formed by combining the Runtime Library with Independent Modules, even if such propagation would otherwise violate the terms of GPLv3, provided that all Target Code was generated by Eligible Compilation Processes. You may then convey such a combination under terms of your choice, consistent with the licensing of the Independent Modules.
+2. No Weakening of GCC Copyleft.
+
+The availability of this Exception does not imply any general presumption that third-party software is unaffected by the copyleft requirements of the license of GCC.

Diff for: LICENSES/GPL-2.0-only.txt

@@ -0,0 +1,7 @@
+ISC License
+
+<copyright notice>
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

Diff for: LICENSES/GPL-3.0-or-later.txt

@@ -168,3 +168,37 @@ SPDX-FileCopyrightText = [
     "2003-2019 University of Illinois at Urbana-Champaign.",
 ]
 SPDX-License-Identifier = "NCSA AND Apache-2.0 WITH LLVM-exception"
+
+[[annotations]]
+path = "src/gcc/**"
+precedence = "override"
+SPDX-FileCopyrightText = [
+    "Copyright (C) 1997-2024 Free Software Foundation, Inc.",
+]
+SPDX-License-Identifier = "GPL-3.0-or-later"
+
+[[annotations]]
+path = "src/gcc/gcc/testsuite/**"
+precedence = "override"
+SPDX-FileCopyrightText = [
+    "Copyright (C) 2000-2024 Free Software Foundation, Inc.",
+]
+SPDX-License-Identifier = "GPL-2.0-only"
+
+[[annotations]]
+path = "src/gcc/gcc/testsuite/c-c++-common/analyzer/*.c"
+precedence = "override"
+SPDX-FileCopyrightText = [
+    "Copyright (c) 2007-2011 Atheros Communications Inc.",
+    "Copyright (c) 2011-2012,2017 Qualcomm Atheros, Inc.",
+    "Copyright (c) 2016-2017 Erik Stromdahl <[email protected]>",
+]
+SPDX-License-Identifier = "ISC"
+
+[[annotations]]
+path = "src/gcc/libstdc++-v3/config/os/aix/os_defines.h"
+precedence = "override"
+SPDX-FileCopyrightText = [
+    "Copyright (C) 2000-2024 Free Software Foundation, Inc.",
+]
+SPDX-License-Identifier = "GCC-exception-3.1"

Diff for: LICENSES/ISC.txt

@@ -45,6 +45,7 @@ ignore = [
     "src/tools/rust-analyzer",
     "src/tools/rustc-perf",
     "src/tools/rustfmt",
+    "src/gcc",
 
     # These are ignored by a standard cargo fmt run.
     "compiler/rustc_codegen_cranelift/scripts",

Diff for: REUSE.toml

@@ -0,0 +1,137 @@
+//! Compilation of native dependencies like GCC.
+//!
+//! Native projects like GCC unfortunately aren't suited just yet for
+//! compilation in build scripts that Cargo has. This is because the
+//! compilation takes a *very* long time but also because we don't want to
+//! compile GCC 3 times as part of a normal bootstrap (we want it cached).
+//!
+//! GCC and compiler-rt are essentially just wired up to everything else to
+//! ensure that they're always in place if needed.
+
+use std::fs;
+use std::path::PathBuf;
+use std::sync::OnceLock;
+
+use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::core::config::TargetSelection;
+use crate::utils::exec::command;
+use crate::utils::helpers::{self, t, HashStamp};
+use crate::{generate_smart_stamp_hash, Kind};
+
+pub struct Meta {
+    stamp: HashStamp,
+    out_dir: PathBuf,
+    install_dir: PathBuf,
+    root: PathBuf,
+}
+
+pub enum GccBuildStatus {
+    AlreadyBuilt,
+    ShouldBuild(Meta),
+}
+
+/// This returns whether we've already previously built GCC.
+///
+/// It's used to avoid busting caches during x.py check -- if we've already built
+/// GCC, it's fine for us to not try to avoid doing so.
+pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
+    // Initialize the gcc submodule if not initialized already.
+    builder.config.update_submodule("src/gcc");
+
+    // FIXME (GuillaumeGomez): To be done once gccjit has been built in the CI.
+    // builder.config.maybe_download_ci_gcc();
+
+    let root = builder.src.join("src/gcc");
+    let out_dir = builder.gcc_out(target).join("build");
+    let install_dir = builder.gcc_out(target).join("install");
+
+    static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
+    let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
+        generate_smart_stamp_hash(
+            builder,
+            &builder.config.src.join("src/gcc"),
+            builder.in_tree_gcc_info.sha().unwrap_or_default(),
+        )
+    });
+
+    let stamp = out_dir.join("gcc-finished-building");
+    let stamp = HashStamp::new(stamp, Some(smart_stamp_hash));
+
+    if stamp.is_done() {
+        if stamp.hash.is_none() {
+            builder.info(
+                "Could not determine the GCC submodule commit hash. \
+                     Assuming that an GCC rebuild is not necessary.",
+            );
+            builder.info(&format!(
+                "To force GCC to rebuild, remove the file `{}`",
+                stamp.path.display()
+            ));
+        }
+        return GccBuildStatus::AlreadyBuilt;
+    }
+
+    GccBuildStatus::ShouldBuild(Meta { stamp, out_dir, install_dir, root })
+}
+
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Gcc {
+    pub target: TargetSelection,
+}
+
+impl Step for Gcc {
+    type Output = bool;
+
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/gcc").alias("gcc")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Gcc { target: run.target });
+    }
+
+    /// Compile GCC for `target`.
+    fn run(self, builder: &Builder<'_>) -> bool {
+        let target = self.target;
+
+        // If GCC has already been built, we avoid building it again.
+        let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target)
+        {
+            GccBuildStatus::AlreadyBuilt => return true,
+            GccBuildStatus::ShouldBuild(m) => m,
+        };
+
+        let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
+        t!(stamp.remove());
+        let _time = helpers::timeit(builder);
+        t!(fs::create_dir_all(&out_dir));
+
+        if builder.config.dry_run() {
+            return true;
+        }
+
+        command(root.join("contrib/download_prerequisites")).current_dir(&root).run(builder);
+        command(root.join("configure"))
+            .current_dir(&out_dir)
+            .arg("--enable-host-shared")
+            .arg("--enable-languages=jit")
+            .arg("--enable-checking=release")
+            .arg("--disable-bootstrap")
+            .arg("--disable-multilib")
+            .arg(format!("--prefix={}", install_dir.display()))
+            .run(builder);
+        command("make").current_dir(&out_dir).arg(format!("-j{}", builder.jobs())).run(builder);
+        command("make").current_dir(&out_dir).arg("install").run(builder);
+
+        let lib_alias = install_dir.join("lib/libgccjit.so.0");
+        if !lib_alias.exists() {
+            t!(builder.symlink_file(install_dir.join("lib/libgccjit.so"), lib_alias,));
+        }
+
+        t!(stamp.write());
+
+        true
+    }
+}

Diff for: rustfmt.toml

@@ -8,12 +8,12 @@
 //! LLVM and compiler-rt are essentially just wired up to everything else to
 //! ensure that they're always in place if needed.
 
+use std::env;
 use std::env::consts::EXE_EXTENSION;
 use std::ffi::{OsStr, OsString};
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
-use std::{env, io};
 
 use build_helper::ci::CiEnv;
 use build_helper::git::get_closest_merge_commit;
@@ -23,7 +23,7 @@ use crate::core::config::{Config, TargetSelection};
 use crate::utils::channel;
 use crate::utils::exec::command;
 use crate::utils::helpers::{
-    self, exe, get_clang_cl_resource_dir, output, t, unhashed_basename, up_to_date,
+    self, exe, get_clang_cl_resource_dir, output, t, unhashed_basename, up_to_date, HashStamp,
 };
 use crate::{generate_smart_stamp_hash, CLang, GitRepo, Kind};
 
@@ -1242,44 +1242,6 @@ fn supported_sanitizers(
     }
 }
 
-struct HashStamp {
-    path: PathBuf,
-    hash: Option<Vec<u8>>,
-}
-
-impl HashStamp {
-    fn new(path: PathBuf, hash: Option<&str>) -> Self {
-        HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) }
-    }
-
-    fn is_done(&self) -> bool {
-        match fs::read(&self.path) {
-            Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(),
-            Err(e) if e.kind() == io::ErrorKind::NotFound => false,
-            Err(e) => {
-                panic!("failed to read stamp file `{}`: {}", self.path.display(), e);
-            }
-        }
-    }
-
-    fn remove(&self) -> io::Result<()> {
-        match fs::remove_file(&self.path) {
-            Ok(()) => Ok(()),
-            Err(e) => {
-                if e.kind() == io::ErrorKind::NotFound {
-                    Ok(())
-                } else {
-                    Err(e)
-                }
-            }
-        }
-    }
-
-    fn write(&self) -> io::Result<()> {
-        fs::write(&self.path, self.hash.as_deref().unwrap_or(b""))
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct CrtBeginEnd {
     pub target: TargetSelection,

Diff for: src/bootstrap/src/core/build_steps/gcc.rs

@@ -5,6 +5,7 @@ pub(crate) mod compile;
 pub(crate) mod dist;
 pub(crate) mod doc;
 pub(crate) mod format;
+pub(crate) mod gcc;
 pub(crate) mod install;
 pub(crate) mod llvm;
 pub(crate) mod perf;

Diff for: src/bootstrap/src/core/build_steps/llvm.rs

@@ -14,7 +14,7 @@ use clap::ValueEnum;
 
 use crate::core::build_steps::tool::{self, SourceType};
 use crate::core::build_steps::{
-    check, clean, clippy, compile, dist, doc, install, llvm, run, setup, test, vendor,
+    check, clean, clippy, compile, dist, doc, gcc, install, llvm, run, setup, test, vendor,
 };
 use crate::core::config::flags::{Color, Subcommand};
 use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection};
@@ -793,6 +793,7 @@ impl<'a> Builder<'a> {
                 tool::Clippy,
                 tool::CargoClippy,
                 llvm::Llvm,
+                gcc::Gcc,
                 llvm::Sanitizers,
                 tool::Rustfmt,
                 tool::Miri,

Diff for: src/bootstrap/src/core/build_steps/mod.rs

@@ -145,6 +145,7 @@ pub struct Build {
     rustfmt_info: GitInfo,
     enzyme_info: GitInfo,
     in_tree_llvm_info: GitInfo,
+    in_tree_gcc_info: GitInfo,
     local_rebuild: bool,
     fail_fast: bool,
     doc_tests: DocTests,
@@ -315,6 +316,7 @@ impl Build {
 
         // we always try to use git for LLVM builds
         let in_tree_llvm_info = GitInfo::new(false, &src.join("src/llvm-project"));
+        let in_tree_gcc_info = GitInfo::new(false, &src.join("src/gcc"));
 
         let initial_target_libdir_str = if config.dry_run() {
             "/dummy/lib/path/to/lib/".to_string()
@@ -407,6 +409,7 @@ impl Build {
             rustfmt_info,
             enzyme_info,
             in_tree_llvm_info,
+            in_tree_gcc_info,
             cc: RefCell::new(HashMap::new()),
             cxx: RefCell::new(HashMap::new()),
             ar: RefCell::new(HashMap::new()),
@@ -765,6 +768,10 @@ impl Build {
         self.out.join(&*target.triple).join("enzyme")
     }
 
+    fn gcc_out(&self, target: TargetSelection) -> PathBuf {
+        self.out.join(&*target.triple).join("gcc")
+    }
+
     fn lld_out(&self, target: TargetSelection) -> PathBuf {
         self.out.join(target).join("lld")
     }