Skip to content

Switch to rustwide for sandboxing #407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 23 commits into from
Sep 30, 2019
Merged

Switch to rustwide for sandboxing #407

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9776e70
WIP rustwide
onur Aug 21, 2019
bbd19a5
temporarily switch to rustwide's git repository
pietroalbini Sep 11, 2019
8059fdc
correctly fetch rustwide's rustc version
pietroalbini Sep 11, 2019
c115545
move sandbox configuration to constants
pietroalbini Sep 11, 2019
4dd22ee
build docs with rustwide on different targets
pietroalbini Sep 11, 2019
0f037c3
use the cargo metadata cli instead of the library to resolve deps
pietroalbini Sep 11, 2019
844e2dc
remove last uses of the cargo library from rustwide
pietroalbini Sep 11, 2019
5965cc8
record the build log during rustwide builds
pietroalbini Sep 11, 2019
c004725
extract some code from chroot_builder into docbuilder
pietroalbini Sep 12, 2019
9d2b6b8
switch add_package to use cargo metadata
pietroalbini Sep 12, 2019
f23d0c1
replace chroot_builder with rustwide_builder
pietroalbini Sep 12, 2019
dcceaff
record the docsrs commit in rustwide builds
pietroalbini Sep 13, 2019
aa174e4
purge more rustwide stuff when it's not needed anymore
pietroalbini Sep 13, 2019
7137705
remove ignored and broken test
pietroalbini Sep 13, 2019
77372e2
remove outdated flags and environment variables
pietroalbini Sep 13, 2019
336fd5d
allow limits to be overridden and show them in the ui
pietroalbini Sep 13, 2019
6e7ab9f
make the rustwide workspace configurable via envvar
pietroalbini Sep 13, 2019
fd83121
show which environment variable is missing in the daemon
pietroalbini Sep 13, 2019
9e466a1
try to update to the latest nightly on every build
pietroalbini Sep 13, 2019
b9a2e28
compare parsed semver when adding a new version
pietroalbini Sep 16, 2019
c61f3a8
inline limits constants directly into the default impl
pietroalbini Sep 16, 2019
6043ff9
upgrade rustwide to 0.3.0
pietroalbini Sep 23, 2019
5027343
simplify option clone
pietroalbini Sep 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 62 additions & 74 deletions src/db/add_package.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@

use ChrootBuilderResult;
use Metadata;
use utils::source_path;
use utils::MetadataPackage;
use regex::Regex;

use std::io::prelude::*;
use std::io::BufReader;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::fs;

use cargo::core::{Package, TargetKind};
use cargo::core::dependency::Kind;
use rustc_serialize::json::{Json, ToJson};
use slug::slugify;
use reqwest::Client;
Expand All @@ -24,27 +22,28 @@ use failure::err_msg;
/// Adds a package into database.
///
/// Package must be built first.
pub fn add_package_into_database(conn: &Connection,
pkg: &Package,
pub(crate) fn add_package_into_database(conn: &Connection,
metadata_pkg: &MetadataPackage,
source_dir: &Path,
res: &ChrootBuilderResult,
files: Option<Json>,
doc_targets: Vec<String>)
-> Result<i32> {
debug!("Adding package into database");
let crate_id = try!(initialize_package_in_database(&conn, &pkg));
let dependencies = convert_dependencies(&pkg);
let rustdoc = get_rustdoc(&pkg).unwrap_or(None);
let readme = get_readme(&pkg).unwrap_or(None);
let (release_time, yanked, downloads) = try!(get_release_time_yanked_downloads(&pkg));
let is_library = match pkg.targets()[0].kind() {
&TargetKind::Lib(_) => true,
let crate_id = try!(initialize_package_in_database(&conn, metadata_pkg));
let dependencies = convert_dependencies(metadata_pkg);
let rustdoc = get_rustdoc(metadata_pkg, source_dir).unwrap_or(None);
let readme = get_readme(metadata_pkg, source_dir).unwrap_or(None);
let (release_time, yanked, downloads) = try!(get_release_time_yanked_downloads(metadata_pkg));
let is_library = match metadata_pkg.targets[0].kind.as_slice() {
&[ref kind] if kind == "lib" => true,
_ => false,
};
let metadata = Metadata::from_package(pkg)?;
let metadata = Metadata::from_source_dir(source_dir)?;

let release_id: i32 = {
let rows = try!(conn.query("SELECT id FROM releases WHERE crate_id = $1 AND version = $2",
&[&crate_id, &format!("{}", pkg.manifest().version())]));
&[&crate_id, &format!("{}", metadata_pkg.version)]));

if rows.len() == 0 {
let rows = try!(conn.query("INSERT INTO releases (
Expand All @@ -62,29 +61,29 @@ pub fn add_package_into_database(conn: &Connection,
)
RETURNING id",
&[&crate_id,
&format!("{}", pkg.manifest().version()),
&metadata_pkg.version,
&release_time,
&dependencies.to_json(),
&pkg.targets()[0].name().replace("-", "_"),
&metadata_pkg.targets[0].name.replace("-", "_"),
&yanked,
&res.build_success,
&res.have_doc,
&false, // TODO: Add test status somehow
&pkg.manifest().metadata().license,
&pkg.manifest().metadata().repository,
&pkg.manifest().metadata().homepage,
&pkg.manifest().metadata().description,
&metadata_pkg.license,
&metadata_pkg.repository,
&metadata_pkg.homepage,
&metadata_pkg.description,
&rustdoc,
&readme,
&pkg.manifest().metadata().authors.to_json(),
&pkg.manifest().metadata().keywords.to_json(),
&metadata_pkg.authors.to_json(),
&metadata_pkg.keywords.to_json(),
&res.have_examples,
&downloads,
&files,
&doc_targets.to_json(),
&is_library,
&res.rustc_version,
&pkg.manifest().metadata().documentation,
&metadata_pkg.documentation,
&metadata.default_target]));
// return id
rows.get(0).get(0)
Expand Down Expand Up @@ -116,38 +115,38 @@ pub fn add_package_into_database(conn: &Connection,
default_target = $25
WHERE crate_id = $1 AND version = $2",
&[&crate_id,
&format!("{}", pkg.manifest().version()),
&format!("{}", metadata_pkg.version),
&release_time,
&dependencies.to_json(),
&pkg.targets()[0].name().replace("-", "_"),
&metadata_pkg.targets[0].name.replace("-", "_"),
&yanked,
&res.build_success,
&res.have_doc,
&false, // TODO: Add test status somehow
&pkg.manifest().metadata().license,
&pkg.manifest().metadata().repository,
&pkg.manifest().metadata().homepage,
&pkg.manifest().metadata().description,
&metadata_pkg.license,
&metadata_pkg.repository,
&metadata_pkg.homepage,
&metadata_pkg.description,
&rustdoc,
&readme,
&pkg.manifest().metadata().authors.to_json(),
&pkg.manifest().metadata().keywords.to_json(),
&metadata_pkg.authors.to_json(),
&metadata_pkg.keywords.to_json(),
&res.have_examples,
&downloads,
&files,
&doc_targets.to_json(),
&is_library,
&res.rustc_version,
&pkg.manifest().metadata().documentation,
&metadata_pkg.documentation,
&metadata.default_target]));
rows.get(0).get(0)
}
};


try!(add_keywords_into_database(&conn, &pkg, &release_id));
try!(add_authors_into_database(&conn, &pkg, &release_id));
try!(add_owners_into_database(&conn, &pkg, &crate_id));
try!(add_keywords_into_database(&conn, &metadata_pkg, &release_id));
try!(add_authors_into_database(&conn, &metadata_pkg, &release_id));
try!(add_owners_into_database(&conn, &metadata_pkg, &crate_id));


// Update versions
Expand All @@ -159,13 +158,12 @@ pub fn add_package_into_database(conn: &Connection,
if let Some(versions_array) = versions.as_array_mut() {
let mut found = false;
for version in versions_array.clone() {
if &semver::Version::parse(version.as_string().unwrap()).unwrap() ==
pkg.manifest().version() {
if version.as_string().unwrap() == metadata_pkg.version {
found = true;
}
}
if !found {
versions_array.push(format!("{}", &pkg.manifest().version()).to_json());
versions_array.push(format!("{}", &metadata_pkg.version).to_json());
}
}
let _ = conn.query("UPDATE crates SET versions = $1 WHERE id = $2",
Expand All @@ -177,7 +175,7 @@ pub fn add_package_into_database(conn: &Connection,


/// Adds a build into database
pub fn add_build_into_database(conn: &Connection,
pub(crate) fn add_build_into_database(conn: &Connection,
release_id: &i32,
res: &ChrootBuilderResult)
-> Result<i32> {
Expand All @@ -196,40 +194,34 @@ pub fn add_build_into_database(conn: &Connection,
}


fn initialize_package_in_database(conn: &Connection, pkg: &Package) -> Result<i32> {
let mut rows = try!(conn.query("SELECT id FROM crates WHERE name = $1",
&[&pkg.manifest().name().as_str()]));
fn initialize_package_in_database(conn: &Connection, pkg: &MetadataPackage) -> Result<i32> {
let mut rows = try!(conn.query("SELECT id FROM crates WHERE name = $1", &[&pkg.name]));
// insert crate into database if it is not exists
if rows.len() == 0 {
rows = try!(conn.query("INSERT INTO crates (name) VALUES ($1) RETURNING id",
&[&pkg.manifest().name().as_str()]));
&[&pkg.name]));
}
Ok(rows.get(0).get(0))
}



/// Convert dependencies into Vec<(String, String, String)>
fn convert_dependencies(pkg: &Package) -> Vec<(String, String, String)> {
fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String)> {
let mut dependencies: Vec<(String, String, String)> = Vec::new();
for dependency in pkg.manifest().dependencies() {
let name = dependency.package_name().to_string();
let version = format!("{}", dependency.version_req());
let kind = match dependency.kind() {
Kind::Normal => "normal",
Kind::Development => "dev",
Kind::Build => "build",
};
for dependency in &pkg.dependencies {
let name = dependency.name.clone();
let version = dependency.req.clone();
let kind = dependency.kind.as_ref().map(|s| s.clone()).unwrap_or_else(|| "normal".into());
dependencies.push((name, version, kind.to_string()));
}
dependencies
}


/// Reads readme if there is any read defined in Cargo.toml of a Package
fn get_readme(pkg: &Package) -> Result<Option<String>> {
let readme_path = PathBuf::from(try!(source_path(&pkg).ok_or_else(|| err_msg("File not found"))))
.join(pkg.manifest().metadata().readme.clone().unwrap_or("README.md".to_owned()));
fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result<Option<String>> {
let readme_path = source_dir.join(pkg.readme.clone().unwrap_or("README.md".to_owned()));

if !readme_path.exists() {
return Ok(None);
Expand All @@ -249,14 +241,12 @@ fn get_readme(pkg: &Package) -> Result<Option<String>> {
}


fn get_rustdoc(pkg: &Package) -> Result<Option<String>> {
if let Some(src_path) = pkg.manifest().targets()[0].src_path().path() {
fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result<Option<String>> {
if let Some(src_path) = &pkg.targets[0].src_path {
if src_path.is_absolute() {
read_rust_doc(src_path)
} else {
let mut path = PathBuf::from(try!(source_path(&pkg).ok_or_else(|| err_msg("File not found"))));
path.push(src_path);
read_rust_doc(path.as_path())
read_rust_doc(&source_dir.join(src_path))
}
} else {
// FIXME: should we care about metabuild targets?
Expand Down Expand Up @@ -294,11 +284,10 @@ fn read_rust_doc(file_path: &Path) -> Result<Option<String>> {


/// Get release_time, yanked and downloads from crates.io
fn get_release_time_yanked_downloads
(pkg: &Package)
-> Result<(Option<time::Timespec>, Option<bool>, Option<i32>)> {
let url = format!("https://crates.io/api/v1/crates/{}/versions",
pkg.manifest().name());
fn get_release_time_yanked_downloads(
pkg: &MetadataPackage,
) -> Result<(Option<time::Timespec>, Option<bool>, Option<i32>)> {
let url = format!("https://crates.io/api/v1/crates/{}/versions", pkg.name);
// FIXME: There is probably better way to do this
// and so many unwraps...
let client = Client::new();
Expand All @@ -321,7 +310,7 @@ fn get_release_time_yanked_downloads
.and_then(|v| v.as_string())
.ok_or_else(|| err_msg("Not a JSON object")));

if &semver::Version::parse(version_num).unwrap() == pkg.manifest().version() {
if semver::Version::parse(version_num).unwrap().to_string() == pkg.version {
let release_time_raw = try!(version.get("created_at")
.and_then(|c| c.as_string())
.ok_or_else(|| err_msg("Not a JSON object")));
Expand All @@ -346,8 +335,8 @@ fn get_release_time_yanked_downloads


/// Adds keywords into database
fn add_keywords_into_database(conn: &Connection, pkg: &Package, release_id: &i32) -> Result<()> {
for keyword in &pkg.manifest().metadata().keywords {
fn add_keywords_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> {
for keyword in &pkg.keywords {
let slug = slugify(&keyword);
let keyword_id: i32 = {
let rows = try!(conn.query("SELECT id FROM keywords WHERE slug = $1", &[&slug]));
Expand All @@ -371,10 +360,10 @@ fn add_keywords_into_database(conn: &Connection, pkg: &Package, release_id: &i32


/// Adds authors into database
fn add_authors_into_database(conn: &Connection, pkg: &Package, release_id: &i32) -> Result<()> {
fn add_authors_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> {

let author_capture_re = Regex::new("^([^><]+)<*(.*?)>*$").unwrap();
for author in &pkg.manifest().metadata().authors {
for author in &pkg.authors {
if let Some(author_captures) = author_capture_re.captures(&author[..]) {
let author = author_captures.get(1).map(|m| m.as_str()).unwrap_or("").trim();
let email = author_captures.get(2).map(|m| m.as_str()).unwrap_or("").trim();
Expand Down Expand Up @@ -405,10 +394,9 @@ fn add_authors_into_database(conn: &Connection, pkg: &Package, release_id: &i32)


/// Adds owners into database
fn add_owners_into_database(conn: &Connection, pkg: &Package, crate_id: &i32) -> Result<()> {
fn add_owners_into_database(conn: &Connection, pkg: &MetadataPackage, crate_id: &i32) -> Result<()> {
// owners available in: https://crates.io/api/v1/crates/rand/owners
let owners_url = format!("https://crates.io/api/v1/crates/{}/owners",
&pkg.manifest().name());
let owners_url = format!("https://crates.io/api/v1/crates/{}/owners", pkg.name);
let client = Client::new();
let mut res = try!(client.get(&owners_url[..])
.header(ACCEPT, "application/json")
Expand Down
4 changes: 2 additions & 2 deletions src/db/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Database operations

pub use self::add_package::add_package_into_database;
pub use self::add_package::add_build_into_database;
pub(crate) use self::add_package::add_package_into_database;
pub(crate) use self::add_package::add_build_into_database;
pub use self::file::add_path_into_database;
pub use self::migrate::migrate;

Expand Down
18 changes: 12 additions & 6 deletions src/docbuilder/chroot_builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use super::DocBuilder;
use super::crates::crates_from_path;
use super::metadata::Metadata;
use utils::{get_package, source_path, copy_doc_dir,
update_sources, parse_rustc_version, command_result};
update_sources, parse_rustc_version, command_result, CargoMetadata};
use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database};
use cargo::core::Package;
use cargo::util::CargoResultExt;
Expand Down Expand Up @@ -99,11 +99,17 @@ impl DocBuilder {
Vec::new()
};

let release_id = try!(add_package_into_database(&conn,
&pkg,
&res,
Some(file_list),
successfully_targets));
let src_path = pkg.manifest_path()
.parent()
.ok_or_else(|| ::failure::err_msg("Source path not available"))?;
let release_id = try!(add_package_into_database(
&conn,
&CargoMetadata::from_cargo_lib_package(&pkg).root(),
src_path,
&res,
Some(file_list),
successfully_targets
));
try!(add_build_into_database(&conn, &release_id, &res));

// remove documentation, source and build directory after we are done
Expand Down
8 changes: 4 additions & 4 deletions src/utils/build_doc_rustwide.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,9 @@ pub fn build_doc_rustwide(
rustdoc_flags.push("--extern-html-root-url".to_string());
rustdoc_flags.push(format!(
"{}=https://docs.rs/{}/{}",
dep.name().replace("-", "_"),
dep.name(),
dep.version()
dep.name.replace("-", "_"),
dep.name,
dep.version
));
}

Expand Down Expand Up @@ -106,7 +106,7 @@ pub fn build_doc_rustwide(

// TODO: We need to return build result as well
Ok(BuildDocOutput {
package_version: cargo_metadata.root().version().to_string(),
package_version: cargo_metadata.root().version.to_string(),
build_log: storage.to_string(),
})
})?;
Expand Down
Loading