diff --git a/Cargo.lock b/Cargo.lock index 0f23b5ecc..daeadee9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,11 +139,6 @@ dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "bitflags" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "bitflags" version = "1.0.4" @@ -326,7 +321,7 @@ dependencies = [ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "magic 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "params 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "postgres 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -947,24 +942,6 @@ name = "mac" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "magic" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", - "magic-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "magic-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "markup5ever" version = "0.7.2" @@ -1037,7 +1014,7 @@ name = "mime" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "unicase 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1053,13 +1030,11 @@ dependencies = [ [[package]] name = "mime_guess" -version = "2.0.0-alpha.6" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1770,7 +1745,7 @@ dependencies = [ "libflate 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2665,10 +2640,10 @@ dependencies = [ [[package]] name = "unicase" -version = "2.2.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2784,6 +2759,11 @@ name = "version_check" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "version_check" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "void" version = "1.0.2" @@ -2912,7 +2892,6 @@ dependencies = [ "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" -"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" @@ -3000,8 +2979,6 @@ dependencies = [ "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" "checksum mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" -"checksum magic 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "96f74caec41a12630bb8fd9546530a41addb720eb0c10593855f59ce96a779aa" -"checksum magic-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17442cc60e34d501588c95bc976da04b6a87c51ab02370e95e1c2893a52df16c" "checksum markup5ever 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfedc97d5a503e96816d10fedcd5b42f760b2e525ce2f7ec71f6a41780548475" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" @@ -3013,7 +2990,7 @@ dependencies = [ "checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" "checksum mime 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "4b082692d3f6cf41b453af73839ce3dfc212c4411cbb2441dff80a716e38bd79" "checksum mime_guess 1.8.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2d4c0961143b8efdcfa29c3ae63281601b446a4a668165454b6c90f8024954c5" -"checksum mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "30de2e4613efcba1ec63d8133f344076952090c122992a903359be5a4f99c3ed" +"checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" "checksum mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)" = "71646331f2619b1026cc302f87a2b8b648d5c6dd6937846a16cc8ce0f347f432" "checksum mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" @@ -3186,7 +3163,7 @@ dependencies = [ "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" -"checksum unicase 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d3218ea14b4edcaccfa0df0a64a3792a2c32cc706f1b336e48867f9d3147f90" +"checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" @@ -3205,6 +3182,7 @@ dependencies = [ "checksum vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "def296d3eb3b12371b2c7d0e83bfe1403e4db2d7a0bba324a12b21c4ee13143d" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" +"checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum walkdir 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c66c0b9792f0a765345452775f3adbd28dde9d33f30d13e5dcc5ae17cf6f3780" "checksum walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c7904a7e2bb3cdf0cf5e783f44204a85a37a93151738fa349f06680f59a98b45" diff --git a/Cargo.toml b/Cargo.toml index 62f9c00a2..c6c09f0d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ reqwest = "0.9" semver = "0.9" slug = "=0.1.1" env_logger = "0.6" -magic = "0.12" r2d2 = "0.8" r2d2_postgres = "0.14" url = "1.4" @@ -40,6 +39,7 @@ prometheus = { version = "0.7.0", default-features = false } lazy_static = "1.0.0" rustwide = "0.5.0" tempdir = "0.3" +mime_guess = "2.0.1" # iron dependencies iron = "0.5" diff --git a/src/db/file.rs b/src/db/file.rs index ad3609e46..1364e79ba 100644 --- a/src/db/file.rs +++ b/src/db/file.rs @@ -155,10 +155,6 @@ pub fn add_path_into_database>(conn: &Connection, prefix: &str, path: P) -> Result { - use magic::{Cookie, flags}; - let cookie = Cookie::open(flags::MIME_TYPE)?; - cookie.load::<&str>(&[])?; - let trans = conn.transaction()?; use std::collections::HashMap; @@ -191,32 +187,14 @@ pub fn add_path_into_database>(conn: &Connection, let bucket_path = Path::new(prefix).join(&file_path) .into_os_string().into_string().unwrap(); - let mime = { - let mime = cookie.buffer(&content)?; - // css's are causing some problem in browsers - // magic will return text/plain for css file types - // convert them to text/css - // do the same for javascript files - if mime == "text/plain" { - let e = file_path.extension().unwrap_or_default(); - if e == "css" { - "text/css".to_owned() - } else if e == "js" { - "application/javascript".to_owned() - } else { - mime.to_owned() - } - } else { - mime.to_owned() - } - }; + let mime = detect_mime(file_path); if let Some(client) = &client { futures.push(client.put_object(PutObjectRequest { bucket: S3_BUCKET_NAME.into(), key: bucket_path.clone(), body: Some(content.clone().into()), - content_type: Some(mime.clone()), + content_type: mime.clone(), ..Default::default() }).inspect(|_| { crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1); @@ -236,7 +214,7 @@ pub fn add_path_into_database>(conn: &Connection, } } - file_paths_and_mimes.insert(file_path.clone(), mime.clone()); + file_paths_and_mimes.insert(file_path.clone(), mime.unwrap_or_default()); } if !futures.is_empty() { @@ -272,7 +250,14 @@ pub fn add_path_into_database>(conn: &Connection, file_list_to_json(file_list_with_mimes) } - +fn detect_mime(path: &Path) -> Option { + let file_ext = path.extension().map(|ext| ext.to_str()).unwrap_or_default(); + match file_ext { + Some("md") => Some("text/markdown".to_owned()), + None => None, + _ => mime_guess::from_path(path).first_raw().map(|m| m.to_owned()) + } +} fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result { @@ -339,7 +324,7 @@ pub fn move_to_s3(conn: &Connection, n: usize) -> Result { mod test { extern crate env_logger; use std::env; - use super::get_file_list; + use super::*; #[test] fn test_get_file_list() { @@ -352,4 +337,17 @@ mod test { let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap(); assert_eq!(files[0], std::path::Path::new("Cargo.toml")); } + + #[test] + fn test_mime_types() { + assert_eq!(detect_mime(Path::new("hello.toml")).unwrap(), "text/x-toml"); + assert_eq!(detect_mime(Path::new("hello.css")).unwrap(), "text/css"); + assert_eq!(detect_mime(Path::new("hello.js")).unwrap(), "application/javascript"); + assert_eq!(detect_mime(Path::new("hello.html")).unwrap(), "text/html"); + assert_eq!(detect_mime(Path::new("hello.hello.md")).unwrap(), "text/markdown"); + assert_eq!(detect_mime(Path::new("hello.markdown")).unwrap(), "text/markdown"); + assert_eq!(detect_mime(Path::new("hello.json")).unwrap(), "application/json"); + assert_eq!(detect_mime(Path::new("hello.txt")).unwrap(), "text/plain"); + assert!(detect_mime(Path::new("blah")).is_none()); + } } diff --git a/src/lib.rs b/src/lib.rs index 6723147d2..d783751c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,7 @@ extern crate reqwest; extern crate time; extern crate semver; extern crate slug; -extern crate magic; +extern crate mime_guess; extern crate iron; extern crate router; extern crate staticfile; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 7e1dfa490..a1b4ab17b 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -5,12 +5,8 @@ use std::{env, thread}; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::process::exit; -use std::fs::File; -use std::io::Write; use std::time::Duration; use std::path::PathBuf; -use libc::fork; use time; use docbuilder::RustwideBuilder; use DocBuilderOptions; @@ -18,7 +14,13 @@ use DocBuilder; use utils::{update_release_activity, github_updater, pubsubhubbub}; use db::{connect_db, update_search_index}; - +#[cfg(not(target_os = "windows"))] +use ::{ + libc::fork, + std::process::exit, + std::fs::File, + std::io::Write +}; pub fn start_daemon(background: bool) { // first check required environment variables @@ -36,15 +38,22 @@ pub fn start_daemon(background: bool) { dbopts.check_paths().unwrap(); if background { - // fork the process - let pid = unsafe { fork() }; - if pid > 0 { - let mut file = File::create(dbopts.prefix.join("cratesfyi.pid")) - .expect("Failed to create pid file"); - writeln!(&mut file, "{}", pid).expect("Failed to write pid"); - - info!("cratesfyi {} daemon started on: {}", ::BUILD_VERSION, pid); - exit(0); + #[cfg(target_os = "windows")] + { + panic!("running in background not supported on windows"); + } + #[cfg(not(target_os = "windows"))] + { + // fork the process + let pid = unsafe { fork() }; + if pid > 0 { + let mut file = File::create(dbopts.prefix.join("cratesfyi.pid")) + .expect("Failed to create pid file"); + writeln!(&mut file, "{}", pid).expect("Failed to write pid"); + + info!("cratesfyi {} daemon started on: {}", ::BUILD_VERSION, pid); + exit(0); + } } }