diff --git a/.sqlx/query-43d0bb3b88356af3abdae506b7699ec762a6f1debbbda49a3479fddaa8917e17.json b/.sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json similarity index 54% rename from .sqlx/query-43d0bb3b88356af3abdae506b7699ec762a6f1debbbda49a3479fddaa8917e17.json rename to .sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json index 688a8d82a..00e60e300 100644 --- a/.sqlx/query-43d0bb3b88356af3abdae506b7699ec762a6f1debbbda49a3479fddaa8917e17.json +++ b/.sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, name, version, priority, registry\n FROM queue\n WHERE\n attempt < $1 AND\n (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2))\n ORDER BY priority ASC, attempt ASC, id ASC\n LIMIT 1\n FOR UPDATE SKIP LOCKED", + "query": "SELECT id, name, version, priority, registry, attempt\n FROM queue\n WHERE\n attempt < $1 AND\n (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2))\n ORDER BY priority ASC, attempt ASC, id ASC\n LIMIT 1\n FOR UPDATE SKIP LOCKED", "describe": { "columns": [ { @@ -27,6 +27,11 @@ "ordinal": 4, "name": "registry", "type_info": "Text" + }, + { + "ordinal": 5, + "name": "attempt", + "type_info": "Int4" } ], "parameters": { @@ -40,8 +45,9 @@ false, false, false, - true + true, + false ] }, - "hash": "43d0bb3b88356af3abdae506b7699ec762a6f1debbbda49a3479fddaa8917e17" + "hash": "4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b" } diff --git a/.sqlx/query-ebb47544b1090567139f3bdf2c22993c3a3aaef41c6520095a2c3bfaf6035da6.json b/.sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json similarity index 63% rename from .sqlx/query-ebb47544b1090567139f3bdf2c22993c3a3aaef41c6520095a2c3bfaf6035da6.json rename to .sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json index c7e4786cc..cbd84ed72 100644 --- a/.sqlx/query-ebb47544b1090567139f3bdf2c22993c3a3aaef41c6520095a2c3bfaf6035da6.json +++ b/.sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, name, version, priority, registry\n FROM queue\n WHERE attempt < $1\n ORDER BY priority ASC, attempt ASC, id ASC", + "query": "SELECT id, name, version, priority, registry, attempt\n FROM queue\n WHERE attempt < $1\n ORDER BY priority ASC, attempt ASC, id ASC", "describe": { "columns": [ { @@ -27,6 +27,11 @@ "ordinal": 4, "name": "registry", "type_info": "Text" + }, + { + "ordinal": 5, + "name": "attempt", + "type_info": "Int4" } ], "parameters": { @@ -39,8 +44,9 @@ false, false, false, - true + true, + false ] }, - "hash": "ebb47544b1090567139f3bdf2c22993c3a3aaef41c6520095a2c3bfaf6035da6" + "hash": "9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132" } diff --git a/docker-compose.yml b/docker-compose.yml index aabf44cc8..78ad3cbad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,9 +19,11 @@ services: - "/var/run/docker.sock:/var/run/docker.sock" - ".rustwide-docker:/opt/docsrs/rustwide" - "cratesio-index:/opt/docsrs/prefix/crates.io-index" + - "./ignored/cratesfyi-prefix/metrics:/opt/docsrs/prefix/metrics" - "./static:/opt/docsrs/static:ro" environment: DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide + DOCSRS_COMPILER_METRICS_PATH: /opt/docsrs/prefix/metrics DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db DOCSRS_STORAGE_BACKEND: s3 S3_ENDPOINT: http://s3:9000 diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 6642fe1f6..5e674b912 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -469,6 +469,7 @@ impl BuildSubcommand { .as_ref() .map(|s| PackageKind::Registry(s.as_str())) .unwrap_or(PackageKind::CratesIo), + true, ) .context("Building documentation failed")?; } diff --git a/src/build_queue.rs b/src/build_queue.rs index b81ccd57e..2c4b9e7dd 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -29,6 +29,7 @@ pub(crate) struct QueuedCrate { pub(crate) version: String, pub(crate) priority: i32, pub(crate) registry: Option, + pub(crate) attempt: i32, } #[derive(Debug)] @@ -165,7 +166,7 @@ impl AsyncBuildQueue { Ok(sqlx::query_as!( QueuedCrate, - "SELECT id, name, version, priority, registry + "SELECT id, name, version, priority, registry, attempt FROM queue WHERE attempt < $1 ORDER BY priority ASC, attempt ASC, id ASC", @@ -495,7 +496,7 @@ impl BuildQueue { let to_process = match self.runtime.block_on( sqlx::query_as!( QueuedCrate, - "SELECT id, name, version, priority, registry + "SELECT id, name, version, priority, registry, attempt FROM queue WHERE attempt < $1 AND @@ -646,7 +647,7 @@ impl BuildQueue { return Err(err); } - builder.build_package(&krate.name, &krate.version, kind) + builder.build_package(&krate.name, &krate.version, kind, krate.attempt == 0) })?; Ok(processed) diff --git a/src/config.rs b/src/config.rs index 40c885521..923823d52 100644 --- a/src/config.rs +++ b/src/config.rs @@ -78,6 +78,10 @@ pub struct Config { // for the remote archives? pub(crate) local_archive_cache_path: PathBuf, + // Where to collect metrics for the metrics initiative. + // When empty, we won't collect metrics. + pub(crate) compiler_metrics_collection_path: Option, + // Content Security Policy pub(crate) csp_report_only: bool, @@ -226,6 +230,8 @@ impl Config { prefix.join("archive_cache"), )?, + compiler_metrics_collection_path: maybe_env("DOCSRS_COMPILER_METRICS_PATH")?, + temp_dir, rustwide_workspace: env("DOCSRS_RUSTWIDE_WORKSPACE", PathBuf::from(".workspace"))?, diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 9dada2196..6cc60630d 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -303,7 +303,8 @@ impl RustwideBuilder { .run(|build| { let metadata = Metadata::from_crate_root(build.host_source_dir())?; - let res = self.execute_build(HOST_TARGET, true, build, &limits, &metadata, true)?; + let res = + self.execute_build(HOST_TARGET, true, build, &limits, &metadata, true, false)?; if !res.result.successful { bail!("failed to build dummy crate for {}", rustc_version); } @@ -351,7 +352,12 @@ impl RustwideBuilder { err.context(format!("failed to load local package {}", path.display())) })?; let package = metadata.root(); - self.build_package(&package.name, &package.version, PackageKind::Local(path)) + self.build_package( + &package.name, + &package.version, + PackageKind::Local(path), + false, + ) } #[instrument(name = "docbuilder.build_package", parent = None, skip(self, name), fields(krate=name))] @@ -360,6 +366,7 @@ impl RustwideBuilder { name: &str, version: &str, kind: PackageKind<'_>, + collect_metrics: bool, ) -> Result { let (crate_id, release_id, build_id) = self.runtime.block_on(async { let mut conn = self.db.get_async().await?; @@ -369,7 +376,15 @@ impl RustwideBuilder { Ok::<_, Error>((crate_id, release_id, build_id)) })?; - match self.build_package_inner(name, version, kind, crate_id, release_id, build_id) { + match self.build_package_inner( + name, + version, + kind, + crate_id, + release_id, + build_id, + collect_metrics, + ) { Ok(successful) => Ok(BuildPackageSummary { successful, should_reattempt: false, @@ -391,6 +406,7 @@ impl RustwideBuilder { } } + #[allow(clippy::too_many_arguments)] fn build_package_inner( &mut self, name: &str, @@ -399,6 +415,7 @@ impl RustwideBuilder { crate_id: CrateId, release_id: ReleaseId, build_id: BuildId, + collect_metrics: bool, ) -> Result { info!("building package {} {}", name, version); @@ -506,7 +523,7 @@ impl RustwideBuilder { // Perform an initial build let mut res = - self.execute_build(default_target, true, build, &limits, &metadata, false)?; + self.execute_build(default_target, true, build, &limits, &metadata, false, collect_metrics)?; // If the build fails with the lockfile given, try using only the dependencies listed in Cargo.toml. let cargo_lock = build.host_source_dir().join("Cargo.lock"); @@ -528,7 +545,7 @@ impl RustwideBuilder { .run_capture()?; } res = - self.execute_build(default_target, true, build, &limits, &metadata, false)?; + self.execute_build(default_target, true, build, &limits, &metadata, false, collect_metrics)?; } if res.result.successful { @@ -565,6 +582,7 @@ impl RustwideBuilder { local_storage.path(), &mut successful_targets, &metadata, + collect_metrics, )?; target_build_logs.insert(target, target_res.build_log); } @@ -730,6 +748,7 @@ impl RustwideBuilder { } #[instrument(skip(self, build))] + #[allow(clippy::too_many_arguments)] fn build_target( &self, target: &str, @@ -738,8 +757,17 @@ impl RustwideBuilder { local_storage: &Path, successful_targets: &mut Vec, metadata: &Metadata, + collect_metrics: bool, ) -> Result { - let target_res = self.execute_build(target, false, build, limits, metadata, false)?; + let target_res = self.execute_build( + target, + false, + build, + limits, + metadata, + false, + collect_metrics, + )?; if target_res.result.successful { // Cargo is not giving any error and not generating documentation of some crates // when we use a target compile options. Check documentation exists before @@ -782,7 +810,7 @@ impl RustwideBuilder { items_with_examples: 0, }; - self.prepare_command(build, target, metadata, limits, rustdoc_flags)? + self.prepare_command(build, target, metadata, limits, rustdoc_flags, false)? .process_lines(&mut |line, _| { if line.starts_with('{') && line.ends_with('}') { let parsed = match serde_json::from_str::>(line) { @@ -810,6 +838,7 @@ impl RustwideBuilder { } #[instrument(skip(self, build))] + #[allow(clippy::too_many_arguments)] fn execute_build( &self, target: &str, @@ -818,6 +847,7 @@ impl RustwideBuilder { limits: &Limits, metadata: &Metadata, create_essential_files: bool, + collect_metrics: bool, ) -> Result { let cargo_metadata = CargoMetadata::load_from_rustwide( &self.workspace, @@ -856,12 +886,34 @@ impl RustwideBuilder { let successful = { let _span = info_span!("cargo_build", target = %target, is_default_target).entered(); logging::capture(&storage, || { - self.prepare_command(build, target, metadata, limits, rustdoc_flags) - .and_then(|command| command.run().map_err(Error::from)) - .is_ok() + self.prepare_command( + build, + target, + metadata, + limits, + rustdoc_flags, + collect_metrics, + ) + .and_then(|command| command.run().map_err(Error::from)) + .is_ok() }) }; + if collect_metrics { + if let Some(compiler_metric_target_dir) = &self.config.compiler_metrics_collection_path + { + let metric_output = build.host_target_dir().join("metrics/"); + info!( + "found {} files in metric dir, copy over to {} (exists: {})", + fs::read_dir(&metric_output)?.count(), + &compiler_metric_target_dir.to_string_lossy(), + &compiler_metric_target_dir.exists(), + ); + copy_dir_all(&metric_output, compiler_metric_target_dir)?; + fs::remove_dir_all(&metric_output)?; + } + } + // For proc-macros, cargo will put the output in `target/doc`. // Move it to the target-specific directory for consistency with other builds. // NOTE: don't rename this if the build failed, because `target/doc` won't exist. @@ -899,6 +951,7 @@ impl RustwideBuilder { metadata: &Metadata, limits: &Limits, mut rustdoc_flags_extras: Vec, + collect_metrics: bool, ) -> Result> { // Add docs.rs specific arguments let mut cargo_args = vec![ @@ -945,7 +998,7 @@ impl RustwideBuilder { ]; rustdoc_flags_extras.extend(UNCONDITIONAL_ARGS.iter().map(|&s| s.to_owned())); - let cargo_args = metadata.cargo_args(&cargo_args, &rustdoc_flags_extras); + let mut cargo_args = metadata.cargo_args(&cargo_args, &rustdoc_flags_extras); // If the explicit target is not a tier one target, we need to install it. let has_build_std = cargo_args.windows(2).any(|args| { @@ -966,6 +1019,21 @@ impl RustwideBuilder { command = command.env(key, val); } + if collect_metrics && self.config.compiler_metrics_collection_path.is_some() { + // set the `./target/metrics/` directory inside the build container + // as a target directory for the metric files. + let flag = "-Zmetrics-dir=/opt/rustwide/target/metrics"; + + // this is how we can reach it from outside the container. + fs::create_dir_all(build.host_target_dir().join("metrics/"))?; + + let rustdocflags = toml::Value::try_from(vec![flag]) + .expect("serializing a string should never fail") + .to_string(); + cargo_args.push("--config".into()); + cargo_args.push(format!("build.rustdocflags={rustdocflags}")); + } + Ok(command.args(&cargo_args)) } @@ -1124,7 +1192,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1264,6 +1332,41 @@ mod tests { }) } + #[test] + #[ignore] + fn test_collect_metrics() { + wrapper(|env| { + let metrics_dir = tempfile::tempdir()?.into_path(); + + env.override_config(|cfg| { + cfg.compiler_metrics_collection_path = Some(metrics_dir.clone()); + cfg.include_default_targets = false; + }); + + let crate_ = DUMMY_CRATE_NAME; + let version = DUMMY_CRATE_VERSION; + + let mut builder = RustwideBuilder::init(env).unwrap(); + builder.update_toolchain()?; + assert!( + builder + .build_package(crate_, version, PackageKind::CratesIo, true)? + .successful + ); + + let metric_files: Vec<_> = fs::read_dir(&metrics_dir)? + .filter_map(|di| di.ok()) + .map(|di| di.path()) + .collect(); + + assert_eq!(metric_files.len(), 1); + + let _: serde_json::Value = serde_json::from_slice(&fs::read(&metric_files[0])?)?; + + Ok(()) + }) + } + #[test] #[ignore] fn test_build_binary_crate() { @@ -1282,7 +1385,7 @@ mod tests { builder.update_toolchain()?; assert!( !builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1402,7 +1505,7 @@ mod tests { assert!( // not successful build !builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1421,7 +1524,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1452,7 +1555,7 @@ mod tests { } assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1508,7 +1611,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1535,7 +1638,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1553,7 +1656,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); Ok(()) @@ -1576,7 +1679,7 @@ mod tests { // `Result` is `Ok`, but the build-result is `false` assert!( !builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1604,7 +1707,7 @@ mod tests { builder.update_toolchain()?; // `Result` is `Ok`, but the build-result is `false` - let summary = builder.build_package(crate_, version, PackageKind::CratesIo)?; + let summary = builder.build_package(crate_, version, PackageKind::CratesIo, false)?; assert!(!summary.successful); assert!(summary.should_reattempt); @@ -1648,7 +1751,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1673,7 +1776,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo)? + .build_package(crate_, version, PackageKind::CratesIo, false)? .successful ); @@ -1760,7 +1863,12 @@ mod tests { let mut builder = RustwideBuilder::init(env)?; assert!( builder - .build_package(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION, PackageKind::CratesIo)? + .build_package( + DUMMY_CRATE_NAME, + DUMMY_CRATE_VERSION, + PackageKind::CratesIo, + false + )? .successful ); assert_eq!(old_version, builder.rustc_version()?);