Auto merge of rust-lang#129337 - EtomicBomb:rfc, r=notriddle

rustdoc rfc#3662 changes under unstable flags

* All new functionality is under unstable options
* Adds `--merge=shared|none|finalize` flags
* Adds `--parts-out-dir=<crate specific directory>` for `--merge=none`
to write cross-crate info file for a single crate
* Adds `--include-parts-dir=<previously specified directory>` for
`--merge=finalize` to write cross-crate info files
* `tests/rustdoc/` tests for the new flags

Author: bors

Diff for: src/librustdoc/config.rs

@@ -53,12 +53,18 @@ impl TryFrom<&str> for OutputFormat {
     }
 }
 
+/// Either an input crate, markdown file, or nothing (--merge=finalize).
+pub(crate) enum InputMode {
+    /// The `--merge=finalize` step does not need an input crate to rustdoc.
+    NoInputMergeFinalize,
+    /// A crate or markdown file.
+    HasFile(Input),
+}
+
 /// Configuration options for rustdoc.
 #[derive(Clone)]
 pub(crate) struct Options {
     // Basic options / Options passed directly to rustc
-    /// The crate root or Markdown file to load.
-    pub(crate) input: Input,
     /// The name of the crate being documented.
     pub(crate) crate_name: Option<String>,
     /// Whether or not this is a bin crate
@@ -179,7 +185,6 @@ impl fmt::Debug for Options {
         }
 
         f.debug_struct("Options")
-            .field("input", &self.input.source_name())
             .field("crate_name", &self.crate_name)
             .field("bin_crate", &self.bin_crate)
             .field("proc_macro_crate", &self.proc_macro_crate)
@@ -289,6 +294,12 @@ pub(crate) struct RenderOptions {
     /// This field is only used for the JSON output. If it's set to true, no file will be created
     /// and content will be displayed in stdout directly.
     pub(crate) output_to_stdout: bool,
+    /// Whether we should read or write rendered cross-crate info in the doc root.
+    pub(crate) should_merge: ShouldMerge,
+    /// Path to crate-info for external crates.
+    pub(crate) include_parts_dir: Vec<PathToParts>,
+    /// Where to write crate-info
+    pub(crate) parts_out_dir: Option<PathToParts>,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -348,7 +359,7 @@ impl Options {
         early_dcx: &mut EarlyDiagCtxt,
         matches: &getopts::Matches,
         args: Vec<String>,
-    ) -> Option<(Options, RenderOptions)> {
+    ) -> Option<(InputMode, Options, RenderOptions)> {
         // Check for unstable options.
         nightly_options::check_nightly_options(early_dcx, matches, &opts());
 
@@ -478,22 +489,34 @@ impl Options {
         let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(early_dcx, matches);
 
         let input = if describe_lints {
-            "" // dummy, this won't be used
+            InputMode::HasFile(make_input(early_dcx, ""))
         } else {
             match matches.free.as_slice() {
+                [] if matches.opt_str("merge").as_deref() == Some("finalize") => {
+                    InputMode::NoInputMergeFinalize
+                }
                 [] => dcx.fatal("missing file operand"),
-                [input] => input,
+                [input] => InputMode::HasFile(make_input(early_dcx, input)),
                 _ => dcx.fatal("too many file operands"),
             }
         };
-        let input = make_input(early_dcx, input);
 
         let externs = parse_externs(early_dcx, matches, &unstable_opts);
         let extern_html_root_urls = match parse_extern_html_roots(matches) {
             Ok(ex) => ex,
             Err(err) => dcx.fatal(err),
         };
 
+        let parts_out_dir =
+            match matches.opt_str("parts-out-dir").map(|p| PathToParts::from_flag(p)).transpose() {
+                Ok(parts_out_dir) => parts_out_dir,
+                Err(e) => dcx.fatal(e),
+            };
+        let include_parts_dir = match parse_include_parts_dir(matches) {
+            Ok(include_parts_dir) => include_parts_dir,
+            Err(e) => dcx.fatal(e),
+        };
+
         let default_settings: Vec<Vec<(String, String)>> = vec![
             matches
                 .opt_str("default-theme")
@@ -735,6 +758,10 @@ impl Options {
         let extern_html_root_takes_precedence =
             matches.opt_present("extern-html-root-takes-precedence");
         let html_no_source = matches.opt_present("html-no-source");
+        let should_merge = match parse_merge(matches) {
+            Ok(result) => result,
+            Err(e) => dcx.fatal(format!("--merge option error: {e}")),
+        };
 
         if generate_link_to_definition && (show_coverage || output_format != OutputFormat::Html) {
             dcx.struct_warn(
@@ -751,7 +778,6 @@ impl Options {
         let unstable_features =
             rustc_feature::UnstableFeatures::from_environment(crate_name.as_deref());
         let options = Options {
-            input,
             bin_crate,
             proc_macro_crate,
             error_format,
@@ -823,16 +849,17 @@ impl Options {
             no_emit_shared: false,
             html_no_source,
             output_to_stdout,
+            should_merge,
+            include_parts_dir,
+            parts_out_dir,
         };
-        Some((options, render_options))
+        Some((input, options, render_options))
     }
+}
 
-    /// Returns `true` if the file given as `self.input` is a Markdown file.
-    pub(crate) fn markdown_input(&self) -> Option<&Path> {
-        self.input
-            .opt_path()
-            .filter(|p| matches!(p.extension(), Some(e) if e == "md" || e == "markdown"))
-    }
+/// Returns `true` if the file given as `self.input` is a Markdown file.
+pub(crate) fn markdown_input(input: &Input) -> Option<&Path> {
+    input.opt_path().filter(|p| matches!(p.extension(), Some(e) if e == "md" || e == "markdown"))
 }
 
 fn parse_remap_path_prefix(
@@ -900,3 +927,71 @@ fn parse_extern_html_roots(
     }
     Ok(externs)
 }
+
+/// Path directly to crate-info file.
+///
+/// For example, `/home/user/project/target/doc.parts/<crate>/crate-info`.
+#[derive(Clone, Debug)]
+pub(crate) struct PathToParts(pub(crate) PathBuf);
+
+impl PathToParts {
+    fn from_flag(path: String) -> Result<PathToParts, String> {
+        let mut path = PathBuf::from(path);
+        // check here is for diagnostics
+        if path.exists() && !path.is_dir() {
+            Err(format!(
+                "--parts-out-dir and --include-parts-dir expect directories, found: {}",
+                path.display(),
+            ))
+        } else {
+            // if it doesn't exist, we'll create it. worry about that in write_shared
+            path.push("crate-info");
+            Ok(PathToParts(path))
+        }
+    }
+}
+
+/// Reports error if --include-parts-dir / crate-info is not a file
+fn parse_include_parts_dir(m: &getopts::Matches) -> Result<Vec<PathToParts>, String> {
+    let mut ret = Vec::new();
+    for p in m.opt_strs("include-parts-dir") {
+        let p = PathToParts::from_flag(p)?;
+        // this is just for diagnostic
+        if !p.0.is_file() {
+            return Err(format!("--include-parts-dir expected {} to be a file", p.0.display()));
+        }
+        ret.push(p);
+    }
+    Ok(ret)
+}
+
+/// Controls merging of cross-crate information
+#[derive(Debug, Clone)]
+pub(crate) struct ShouldMerge {
+    /// Should we append to existing cci in the doc root
+    pub(crate) read_rendered_cci: bool,
+    /// Should we write cci to the doc root
+    pub(crate) write_rendered_cci: bool,
+}
+
+/// Extracts read_rendered_cci and write_rendered_cci from command line arguments, or
+/// reports an error if an invalid option was provided
+fn parse_merge(m: &getopts::Matches) -> Result<ShouldMerge, &'static str> {
+    match m.opt_str("merge").as_deref() {
+        // default = read-write
+        None => Ok(ShouldMerge { read_rendered_cci: true, write_rendered_cci: true }),
+        Some("none") if m.opt_present("include-parts-dir") => {
+            Err("--include-parts-dir not allowed if --merge=none")
+        }
+        Some("none") => Ok(ShouldMerge { read_rendered_cci: false, write_rendered_cci: false }),
+        Some("shared") if m.opt_present("parts-out-dir") || m.opt_present("include-parts-dir") => {
+            Err("--parts-out-dir and --include-parts-dir not allowed if --merge=shared")
+        }
+        Some("shared") => Ok(ShouldMerge { read_rendered_cci: true, write_rendered_cci: true }),
+        Some("finalize") if m.opt_present("parts-out-dir") => {
+            Err("--parts-out-dir not allowed if --merge=finalize")
+        }
+        Some("finalize") => Ok(ShouldMerge { read_rendered_cci: false, write_rendered_cci: true }),
+        Some(_) => Err("argument to --merge must be `none`, `shared`, or `finalize`"),
+    }
+}

Diff for: src/librustdoc/core.rs

@@ -20,7 +20,7 @@ use rustc_interface::interface;
 use rustc_lint::{late_lint_mod, MissingDoc};
 use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::{ParamEnv, Ty, TyCtxt};
-use rustc_session::config::{self, CrateType, ErrorOutputType, ResolveDocLinks};
+use rustc_session::config::{self, CrateType, ErrorOutputType, Input, ResolveDocLinks};
 pub(crate) use rustc_session::config::{Options, UnstableOptions};
 use rustc_session::{lint, Session};
 use rustc_span::symbol::sym;
@@ -177,8 +177,8 @@ pub(crate) fn new_dcx(
 
 /// Parse, resolve, and typecheck the given crate.
 pub(crate) fn create_config(
+    input: Input,
     RustdocOptions {
-        input,
         crate_name,
         proc_macro_crate,
         error_format,

Diff for: src/librustdoc/doctest.rs

@@ -19,7 +19,7 @@ use rustc_errors::{ColorConfig, DiagCtxtHandle, ErrorGuaranteed, FatalError};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_hir::CRATE_HIR_ID;
 use rustc_interface::interface;
-use rustc_session::config::{self, CrateType, ErrorOutputType};
+use rustc_session::config::{self, CrateType, ErrorOutputType, Input};
 use rustc_session::lint;
 use rustc_span::edition::Edition;
 use rustc_span::symbol::sym;
@@ -88,7 +88,11 @@ fn get_doctest_dir() -> io::Result<TempDir> {
     TempFileBuilder::new().prefix("rustdoctest").tempdir()
 }
 
-pub(crate) fn run(dcx: DiagCtxtHandle<'_>, options: RustdocOptions) -> Result<(), ErrorGuaranteed> {
+pub(crate) fn run(
+    dcx: DiagCtxtHandle<'_>,
+    input: Input,
+    options: RustdocOptions,
+) -> Result<(), ErrorGuaranteed> {
     let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
 
     // See core::create_config for what's going on here.
@@ -135,7 +139,7 @@ pub(crate) fn run(dcx: DiagCtxtHandle<'_>, options: RustdocOptions) -> Result<()
         opts: sessopts,
         crate_cfg: cfgs,
         crate_check_cfg: options.check_cfgs.clone(),
-        input: options.input.clone(),
+        input: input.clone(),
         output_file: None,
         output_dir: None,
         file_loader: None,

Diff for: src/librustdoc/doctest/markdown.rs

@@ -3,6 +3,7 @@
 use std::fs::read_to_string;
 use std::sync::{Arc, Mutex};
 
+use rustc_session::config::Input;
 use rustc_span::FileName;
 use tempfile::tempdir;
 
@@ -69,17 +70,16 @@ impl DocTestVisitor for MdCollector {
 }
 
 /// Runs any tests/code examples in the markdown file `options.input`.
-pub(crate) fn test(options: Options) -> Result<(), String> {
-    use rustc_session::config::Input;
-    let input_str = match &options.input {
+pub(crate) fn test(input: &Input, options: Options) -> Result<(), String> {
+    let input_str = match input {
         Input::File(path) => {
             read_to_string(path).map_err(|err| format!("{}: {err}", path.display()))?
         }
         Input::Str { name: _, input } => input.clone(),
     };
 
     // Obviously not a real crate name, but close enough for purposes of doctests.
-    let crate_name = options.input.filestem().to_string();
+    let crate_name = input.filestem().to_string();
     let temp_dir =
         tempdir().map_err(|error| format!("failed to create temporary directory: {error:?}"))?;
     let args_file = temp_dir.path().join("rustdoc-cfgs");
@@ -96,8 +96,7 @@ pub(crate) fn test(options: Options) -> Result<(), String> {
     let mut md_collector = MdCollector {
         tests: vec![],
         cur_path: vec![],
-        filename: options
-            .input
+        filename: input
             .opt_path()
             .map(ToOwned::to_owned)
             .map(FileName::from)