diff --git a/Cargo.lock b/Cargo.lock index d2e6f155b..190617c5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,6 +327,7 @@ dependencies = [ "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "magic 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "params 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "path-slash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1038,7 +1039,7 @@ name = "mime" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "unicase 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1054,13 +1055,11 @@ dependencies = [ [[package]] name = "mime_guess" -version = "2.0.0-alpha.6" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1776,7 +1775,7 @@ dependencies = [ "libflate 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2671,10 +2670,10 @@ dependencies = [ [[package]] name = "unicase" -version = "2.2.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2790,6 +2789,11 @@ name = "version_check" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "version_check" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "void" version = "1.0.2" @@ -3019,7 +3023,7 @@ dependencies = [ "checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" "checksum mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "4b082692d3f6cf41b453af73839ce3dfc212c4411cbb2441dff80a716e38bd79" "checksum mime_guess 1.8.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2d4c0961143b8efdcfa29c3ae63281601b446a4a668165454b6c90f8024954c5" -"checksum mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "30de2e4613efcba1ec63d8133f344076952090c122992a903359be5a4f99c3ed" +"checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" "checksum mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)" = "71646331f2619b1026cc302f87a2b8b648d5c6dd6937846a16cc8ce0f347f432" "checksum mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" @@ -3193,7 +3197,7 @@ dependencies = [ "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" -"checksum unicase 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d3218ea14b4edcaccfa0df0a64a3792a2c32cc706f1b336e48867f9d3147f90" +"checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" @@ -3212,6 +3216,7 @@ dependencies = [ "checksum vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "def296d3eb3b12371b2c7d0e83bfe1403e4db2d7a0bba324a12b21c4ee13143d" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" +"checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum walkdir 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c66c0b9792f0a765345452775f3adbd28dde9d33f30d13e5dcc5ae17cf6f3780" "checksum walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c7904a7e2bb3cdf0cf5e783f44204a85a37a93151738fa349f06680f59a98b45" diff --git a/Cargo.toml b/Cargo.toml index 8c94b6606..780e5443d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,11 +18,9 @@ reqwest = "0.9" semver = "0.9" slug = "=0.1.1" env_logger = "0.6" -magic = "0.12" r2d2 = "0.8" r2d2_postgres = "0.14" url = "1.4" -libc = "0.2" badge = { version = "0", path = "src/web/badge" } failure = "0.1.3" comrak = { version = "0.3", default-features = false } @@ -48,8 +46,13 @@ handlebars-iron = "0.22" params = "0.6" staticfile = { version = "0.4", features = [ "cache" ] } +[target.'cfg(not(windows))'.dependencies] +libc = "0.2" +magic = "0.12" + [target.'cfg(windows)'.dependencies] path-slash = "0.1.1" +mime_guess = "2.0.1" [dependencies.postgres] version = "0.15" diff --git a/src/db/file.rs b/src/db/file.rs index 4f7a2f763..1e0f1e4ff 100644 --- a/src/db/file.rs +++ b/src/db/file.rs @@ -16,7 +16,9 @@ use failure::err_msg; use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client}; use rusoto_core::region::Region; use rusoto_credential::DefaultCredentialsProvider; - +use std::ffi::OsStr; +#[cfg(not(windows))] +use magic::{Cookie, flags}; const MAX_CONCURRENT_UPLOADS: usize = 1000; @@ -155,12 +157,9 @@ pub fn add_path_into_database>(conn: &Connection, prefix: &str, path: P) -> Result { - use magic::{Cookie, flags}; - let cookie = Cookie::open(flags::MIME_TYPE)?; - cookie.load::<&str>(&[])?; - let trans = conn.transaction()?; - + #[cfg(not(windows))] + let mime_data = load_mime_data()?; use std::collections::HashMap; let mut file_paths_and_mimes: HashMap = HashMap::new(); use futures::future::Future; @@ -195,25 +194,10 @@ pub fn add_path_into_database>(conn: &Connection, #[cfg(not(windows))] let bucket_path = bucket_path.into_os_string().into_string().unwrap(); - let mime = { - let mime = cookie.buffer(&content)?; - // css's are causing some problem in browsers - // magic will return text/plain for css file types - // convert them to text/css - // do the same for javascript files - if mime == "text/plain" { - let e = file_path.extension().unwrap_or_default(); - if e == "css" { - "text/css".to_owned() - } else if e == "js" { - "application/javascript".to_owned() - } else { - mime.to_owned() - } - } else { - mime.to_owned() - } - }; + #[cfg(windows)] + let mime = detect_mime(&content, &file_path)?; + #[cfg(not(windows))] + let mime = detect_mime(&content, &file_path, &mime_data)?; if let Some(client) = &client { futures.push(client.put_object(PutObjectRequest { @@ -276,6 +260,42 @@ pub fn add_path_into_database>(conn: &Connection, file_list_to_json(file_list_with_mimes) } +#[cfg(not(windows))] +fn load_mime_data() -> Result { + let cookie = Cookie::open(flags::MIME_TYPE)?; + cookie.load::<&str>(&[])?; + Ok(cookie) +} + +#[cfg(not(windows))] +fn detect_mime(content: &Vec, file_path: &Path, cookie: &Cookie) -> Result { + let mime = cookie.buffer(&content)?; + correct_mime(&mime, &file_path) +} + +#[cfg(windows)] +fn detect_mime(_content: &Vec, file_path: &Path) -> Result { + let mime = mime_guess::from_path(file_path).first_raw().map(|m| m).unwrap_or("text/plain"); + correct_mime(&mime, &file_path) +} + +fn correct_mime(mime: &str, file_path: &Path) -> Result { + Ok(match mime { + "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => { + match file_path.extension().and_then(OsStr::to_str) { + Some("md") => "text/markdown", + Some("rs") => "text/rust", + Some("markdown") => "text/markdown", + Some("css") => "text/css", + Some("toml") => "text/toml", + Some("js") => "application/javascript", + Some("json") => "application/json", + _ => mime + } + } + _ => mime + }.to_owned()) +} fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result { @@ -343,7 +363,7 @@ pub fn move_to_s3(conn: &Connection, n: usize) -> Result { mod test { extern crate env_logger; use std::env; - use super::get_file_list; + use super::*; #[test] fn test_get_file_list() { @@ -356,4 +376,29 @@ mod test { let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap(); assert_eq!(files[0], std::path::Path::new("Cargo.toml")); } + #[test] + fn test_mime_types() { + check_mime("/ignored", ".gitignore", "text/plain"); + check_mime("[package]", "hello.toml","text/toml"); + check_mime(".ok { color:red; }", "hello.css","text/css"); + check_mime("var x = 1", "hello.js","application/javascript"); + check_mime("", "hello.html","text/html"); + check_mime("## HELLO", "hello.hello.md","text/markdown"); + check_mime("## WORLD", "hello.markdown","text/markdown"); + check_mime("{}", "hello.json","application/json"); + check_mime("hello world", "hello.txt","text/plain"); + check_mime("//! Simple module to ...", "file.rs", "text/rust"); + } + + fn check_mime(content: &str, path: &str, expected_mime: &str) { + #[cfg(not(windows))] + let mime_data = load_mime_data().unwrap(); + #[cfg(windows)] + let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path)); + #[cfg(not(windows))] + let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path), &mime_data); + let detected_mime = detected_mime.expect("no mime was given"); + assert_eq!(detected_mime, expected_mime); + } } + diff --git a/src/lib.rs b/src/lib.rs index a7c2da81d..afbb08304 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,6 @@ extern crate reqwest; extern crate time; extern crate semver; extern crate slug; -extern crate magic; extern crate iron; extern crate router; extern crate staticfile; @@ -26,7 +25,6 @@ extern crate r2d2; extern crate r2d2_postgres; extern crate url; extern crate params; -extern crate libc; extern crate badge; extern crate crates_index_diff; extern crate toml; @@ -42,8 +40,15 @@ extern crate systemstat; extern crate rustwide; extern crate tempdir; +#[cfg(not(windows))] +extern crate magic; +#[cfg(not(windows))] +extern crate libc; + #[cfg(windows)] extern crate path_slash; +#[cfg(windows)] +extern crate mime_guess; #[cfg(test)] extern crate once_cell;