Switch to rustwide for sandboxing

src/db/migrate.rs

@@ -179,6 +179,20 @@ pub fn migrate(version: Option<Version>) -> CratesfyiResult<()> {
             // downgrade query
             "ALTER TABLE queue DROP COLUMN priority;"
         ),
+        migration!(
+            // version
+            3,
+            // description
+            "Added sandbox_overrides table",
+            // upgrade query
+            "CREATE TABLE sandbox_overrides (
+                 crate_name VARCHAR NOT NULL PRIMARY KEY,
+                 max_memory_bytes INTEGER,
+                 timeout_seconds INTEGER
+             );",
+            // downgrade query
+            "DROP TABLE sandbox_overrides;"
+        ),
     ];
 
     for migration in migrations {

src/docbuilder/limits.rs

@@ -0,0 +1,97 @@
+use error::Result;
+use postgres::Connection;
+use std::collections::BTreeMap;
+use std::time::Duration;
+
+const DEFAULT_MEMORY_LIMIT: usize = 3 * 1024 * 1024 * 1024; // 3 GB
+const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15 * 60); // 15 minutes
+const DEFAULT_NETWORKING: bool = false;
+const DEFAULT_MAX_LOG_SIZE: usize = 100 * 1024; // 100 KB
+
+pub(crate) struct Limits {
+    memory: usize,
+    timeout: Duration,
+    networking: bool,
+    max_log_size: usize,
+}
+
+impl Default for Limits {
+    fn default() -> Self {
+        Self {
+            memory: DEFAULT_MEMORY_LIMIT,
+            timeout: DEFAULT_TIMEOUT,
+            networking: DEFAULT_NETWORKING,
+            max_log_size: DEFAULT_MAX_LOG_SIZE,
+        }
+    }
+}
+
+impl Limits {
+    pub(crate) fn for_crate(conn: &Connection, name: &str) -> Result<Self> {
+        let mut limits = Self::default();
+
+        let res = conn.query(
+            "SELECT * FROM sandbox_overrides WHERE crate_name = $1;",
+            &[&name],
+        )?;
+        if !res.is_empty() {
+            let row = res.get(0);
+            if let Some(memory) = row.get::<_, Option<i32>>("max_memory_bytes") {
+                limits.memory = memory as usize;
+            }
+            if let Some(timeout) = row.get::<_, Option<i32>>("timeout_seconds") {
+                limits.timeout = Duration::from_secs(timeout as u64);
+            }
+        }
+
+        Ok(limits)
+    }
+
+    pub(crate) fn memory(&self) -> usize {
+        self.memory
+    }
+
+    pub(crate) fn timeout(&self) -> Duration {
+        self.timeout
+    }
+
+    pub(crate) fn networking(&self) -> bool {
+        self.networking
+    }
+
+    pub(crate) fn max_log_size(&self) -> usize {
+        self.max_log_size
+    }
+
+    pub(crate) fn for_website(&self) -> BTreeMap<String, String> {
+        let time_scale = |v| scale(v, 60, &["seconds", "minutes", "hours"]);
+        let size_scale = |v| scale(v, 1024, &["bytes", "KB", "MB", "GB"]);
+
+        let mut res = BTreeMap::new();
+        res.insert("Available RAM".into(), size_scale(self.memory));
+        res.insert(
+            "Maximum rustdoc execution time".into(),
+            time_scale(self.timeout.as_secs() as usize),
+        );
+        res.insert("Maximum size of a build log".into(), size_scale(self.max_log_size));
+        if self.networking {
+            res.insert("Network access".into(), "allowed".into());
+        } else {
+            res.insert("Network access".into(), "blocked".into());
+        }
+        res
+    }
+}
+
+fn scale(mut value: usize, interval: usize, labels: &[&str]) -> String {
+    let mut chosen_label = &labels[0];
+    for label in &labels[1..] {
+        if (value as f64) / (interval as f64) >= 1.0 {
+            chosen_label = label;
+            value = value / interval;
+        } else {
+            break;
+        }
+    }
+    format!("{} {}", value, chosen_label)
+}

src/docbuilder/mod.rs

@@ -1,12 +1,14 @@
 
 pub mod options;
 pub mod metadata;
+mod limits;
 mod rustwide_builder;
 mod crates;
 mod queue;
 
 pub use self::rustwide_builder::RustwideBuilder;
 pub(crate) use self::rustwide_builder::BuildResult;
+pub(crate) use self::limits::Limits;
 
 
 use std::fs;

src/docbuilder/rustwide_builder.rs

@@ -1,7 +1,7 @@
 use super::DocBuilder;
 use db::file::add_path_into_database;
 use db::{add_build_into_database, add_package_into_database, connect_db};
-use docbuilder::crates::crates_from_path;
+use docbuilder::{crates::crates_from_path, Limits};
 use error::Result;
 use failure::ResultExt;
 use log::LevelFilter;
@@ -11,18 +11,9 @@ use rustwide::cmd::{Command, SandboxBuilder};
 use rustwide::logging::{self, LogStorage};
 use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder};
 use std::path::Path;
-use std::time::Duration;
 use utils::{copy_doc_dir, parse_rustc_version, CargoMetadata};
 use Metadata;
 
-// TODO: 1GB might not be enough
-const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1024 * 1024; // 1GB
-const SANDBOX_NETWORKING: bool = false;
-const SANDBOX_MAX_LOG_SIZE: usize = 1024 * 1024; // 1MB
-const SANDBOX_MAX_LOG_LINES: usize = 10_000;
-const COMMAND_TIMEOUT: Option<Duration> = Some(Duration::from_secs(60 * 60)); // 1 hour
-const COMMAND_NO_OUTPUT_TIMEOUT: Option<Duration> = None;
-
 static USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)";
 static TARGETS: &[&str] = &[
     "i686-apple-darwin",
@@ -61,6 +52,9 @@ static ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[
     "SourceSerifPro-It.ttf.woff",
 ];
 
+static DUMMY_CRATE_NAME: &str = "acme-client";
+static DUMMY_CRATE_VERSION: &str = "0.0.0";
+
 pub struct RustwideBuilder {
     workspace: Workspace,
     toolchain: Toolchain,
@@ -69,10 +63,7 @@ pub struct RustwideBuilder {
 
 impl RustwideBuilder {
     pub fn init(workspace_path: &Path) -> Result<Self> {
-        let workspace = WorkspaceBuilder::new(workspace_path, USER_AGENT)
-            .command_timeout(COMMAND_TIMEOUT)
-            .command_no_output_timeout(COMMAND_NO_OUTPUT_TIMEOUT)
-            .init()?;
+        let workspace = WorkspaceBuilder::new(workspace_path, USER_AGENT).init()?;
         workspace.purge_all_build_dirs()?;
 
         let toolchain = Toolchain::Dist {
@@ -126,21 +117,24 @@ impl RustwideBuilder {
         info!("building a dummy crate to get essential files");
         let rustc_version = parse_rustc_version(&self.rustc_version)?;
 
+        let conn = connect_db()?;
+        let limits = Limits::for_crate(&conn, DUMMY_CRATE_NAME)?;
+
         let mut build_dir = self
             .workspace
             .build_dir(&format!("essential-files-{}", rustc_version));
         build_dir.purge()?;
 
         // acme-client-0.0.0 is an empty library crate and it will always build
-        let krate = Crate::crates_io("acme-client", "0.0.0");
+        let krate = Crate::crates_io(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION);
         krate.fetch(&self.workspace)?;
 
         let sandbox = SandboxBuilder::new()
-            .memory_limit(Some(SANDBOX_MEMORY_LIMIT))
-            .enable_networking(SANDBOX_NETWORKING);
+            .memory_limit(Some(limits.memory()))
+            .enable_networking(limits.networking());
 
         build_dir.build(&self.toolchain, &krate, sandbox, |build| {
-            let res = self.execute_build(None, build)?;
+            let res = self.execute_build(None, build, &limits)?;
             if !res.successful {
                 bail!("failed to build dummy crate for {}", self.rustc_version);
             }
@@ -171,7 +165,6 @@ impl RustwideBuilder {
                 })?;
             }
 
-            let conn = connect_db()?;
             add_path_into_database(&conn, "", &dest)?;
             conn.query(
                 "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \
@@ -218,6 +211,7 @@ impl RustwideBuilder {
         info!("building package {} {}", name, version);
 
         let conn = connect_db()?;
+        let limits = Limits::for_crate(&conn, name)?;
 
         let mut build_dir = self.workspace.build_dir(&format!("{}-{}", name, version));
         build_dir.purge()?;
@@ -226,16 +220,16 @@ impl RustwideBuilder {
         krate.fetch(&self.workspace)?;
 
         let sandbox = SandboxBuilder::new()
-            .memory_limit(Some(SANDBOX_MEMORY_LIMIT))
-            .enable_networking(SANDBOX_NETWORKING);
+            .memory_limit(Some(limits.memory()))
+            .enable_networking(limits.networking());
 
         let res = build_dir.build(&self.toolchain, &krate, sandbox, |build| {
             let mut files_list = None;
             let mut has_docs = false;
             let mut successful_targets = Vec::new();
 
             // Do an initial build and then copy the sources in the database
-            let res = self.execute_build(None, &build)?;
+            let res = self.execute_build(None, &build, &limits)?;
             if res.successful {
                 debug!("adding sources into database");
                 let prefix = format!("sources/{}/{}", name, version);
@@ -267,7 +261,7 @@ impl RustwideBuilder {
                 // Then build the documentation for all the targets
                 for target in TARGETS {
                     debug!("building package {} {} for {}", name, version, target);
-                    let target_res = self.execute_build(Some(target), &build)?;
+                    let target_res = self.execute_build(Some(target), &build, &limits)?;
                     if target_res.successful {
                         // Cargo is not giving any error and not generating documentation of some crates
                         // when we use a target compile options. Check documentation exists before
@@ -312,7 +306,7 @@ impl RustwideBuilder {
         Ok(res.successful)
     }
 
-    fn execute_build(&self, target: Option<&str>, build: &Build) -> Result<BuildResult> {
+    fn execute_build(&self, target: Option<&str>, build: &Build, limits: &Limits) -> Result<BuildResult> {
         let metadata = Metadata::from_source_dir(&build.host_source_dir())?;
         let cargo_metadata =
             CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?;
@@ -363,12 +357,13 @@ impl RustwideBuilder {
         }
 
         let mut storage = LogStorage::new(LevelFilter::Info);
-        storage.set_max_size(SANDBOX_MAX_LOG_SIZE);
-        storage.set_max_lines(SANDBOX_MAX_LOG_LINES);
+        storage.set_max_size(limits.max_log_size());
 
         let successful = logging::capture(&storage, || {
             build
                 .cargo()
+                .timeout(Some(limits.timeout()))
+                .no_output_timeout(None)
                 .env(
                     "RUSTFLAGS",
                     metadata

src/web/builds.rs

@@ -1,5 +1,6 @@
 
 
+use docbuilder::Limits;
 use std::collections::BTreeMap;
 use super::MetaData;
 use super::pool::Pool;
@@ -27,6 +28,7 @@ struct BuildsPage {
     metadata: Option<MetaData>,
     builds: Vec<Build>,
     build_details: Option<Build>,
+    limits: Limits,
 }
 
 
@@ -54,6 +56,7 @@ impl ToJson for BuildsPage {
         m.insert("metadata".to_owned(), self.metadata.to_json());
         m.insert("builds".to_owned(), self.builds.to_json());
         m.insert("build_details".to_owned(), self.build_details.to_json());
+        m.insert("limits".into(), self.limits.for_website().to_json());
         m.to_json()
     }
 }
@@ -67,6 +70,7 @@ pub fn build_list_handler(req: &mut Request) -> IronResult<Response> {
     let req_build_id: i32 = router.find("id").unwrap_or("0").parse().unwrap_or(0);
 
     let conn = extension!(req, Pool);
+    let limits = ctry!(Limits::for_crate(&conn, name));
 
     let mut build_list: Vec<Build> = Vec::new();
     let mut build_details = None;
@@ -131,6 +135,7 @@ pub fn build_list_handler(req: &mut Request) -> IronResult<Response> {
             metadata: MetaData::from_crate(&conn, &name, &version),
             builds: build_list,
             build_details: build_details,
+            limits,
         };
         Page::new(builds_page)
             .set_true("show_package_navigation")

src/web/sitemap.rs

@@ -1,7 +1,7 @@
 use std::collections::BTreeMap;
 use iron::prelude::*;
 use iron::headers::ContentType;
-use rustc_serialize::json::Json;
+use rustc_serialize::json::{Json, ToJson};
 use super::page::Page;
 use super::pool::Pool;
 use time;
@@ -40,10 +40,12 @@ pub fn about_handler(req: &mut Request) -> IronResult<Response> {
     if let Some(row) = res.iter().next() {
         if let Some(Ok::<Json, _>(res)) = row.get_opt(0) {
             if let Some(vers) = res.as_string() {
-                content.insert("rustc_version".to_string(), vers.to_string());
+                content.insert("rustc_version".to_string(), vers.to_json());
             }
         }
     }
 
+    content.insert("limits".to_string(), ::docbuilder::Limits::default().for_website().to_json());
+
     Page::new(content).title("About Docs.rs").to_resp("about")
 }

templates/about.hbs

@@ -31,6 +31,30 @@
   no README will be displayed.
   </p>
 
+  <h4>Global sandbox limits</h4>
+
+  <p>
+  All the builds on docs.rs are executed inside a sandbox with limited
+  resources. The current limits are the following:
+  </p>
+
+  <table class="pure-table pure-table-horizontal">
+    <tbody>
+      {{#each content.limits}}
+      <tr>
+        <td>{{{@key}}}</td>
+        <td>{{this}}</td>
+      </tr>
+      {{/each}}
+    </tbody>
+  </table>
+
+  <p>
+  If a build fails because it hit one of those limits please
+  <a href="https://github.com/rust-lang/docs.rs/issues/new">open an issue</a>
+  to get them increased for your crate.
+  </p>
+
   <h4>Redirections</h4>
 
   <p>