Merge pull request #601 from integer32llc/local-upload

Local upload instead of S3 to enable publishing in development

Author: carols10cents

.env.sample

@@ -16,7 +16,6 @@ export TEST_DATABASE_URL=
 
 # Credentials for uploading packages to S3. You can leave these blank if
 # you're not publishing to s3 from your crates.io instance.
-# When running `cargo test`, set S3_BUCKET to `alexcrichton-test`.
 export S3_BUCKET=
 export S3_ACCESS_KEY=
 export S3_SECRET_KEY=

.travis.yml

@@ -45,7 +45,6 @@ env:
   global:
     - DATABASE_URL=postgres://postgres:@localhost/cargo_registry_test
     - TEST_DATABASE_URL=postgres://postgres:@localhost/cargo_registry_test
-    - S3_BUCKET=alexcrichton-test
 
 notifications:
   email:

README.md

@@ -102,16 +102,7 @@ After following the above instructions:
    export TEST_DATABASE_URL=postgres://postgres@localhost/cargo_registry_test
    ```
 
-2. In your `.env` file, set the s3 bucket to `alexcrichton-test`. No actual
-   requests to s3 will be made; the requests and responses are recorded in
-   files in `tests/http-data` and the s3 bucket name needs to match the
-   requests in the files.
-
-   ```
-   export S3_BUCKET=alexcrichton-test
-   ```
-
-3. Run the backend API server tests:
+2. Run the backend API server tests:
 
    ```
    cargo test

src/app.rs

@@ -7,7 +7,6 @@ use conduit_middleware::Middleware;
 use git2;
 use oauth2;
 use r2d2;
-use s3;
 use curl::easy::Easy;
 
 use {db, Config};
@@ -23,8 +22,7 @@ pub struct App {
 
     /// The GitHub OAuth2 configuration
     pub github: oauth2::Config,
-    pub bucket: s3::Bucket,
-    pub s3_proxy: Option<String>,
+
     pub session_key: String,
     pub git_repo: Mutex<git2::Repository>,
     pub git_repo_checkout: PathBuf,
@@ -64,12 +62,6 @@ impl App {
             database: db::pool(&config.db_url, db_config),
             diesel_database: db::diesel_pool(&config.db_url, diesel_db_config),
             github: github,
-            bucket: s3::Bucket::new(config.s3_bucket.clone(),
-                                    config.s3_region.clone(),
-                                    config.s3_access_key.clone(),
-                                    config.s3_secret_key.clone(),
-                                    config.api_protocol()),
-            s3_proxy: config.s3_proxy.clone(),
             session_key: config.session_key.clone(),
             git_repo: Mutex::new(repo),
             git_repo_checkout: config.git_repo_checkout.clone(),
@@ -79,7 +71,7 @@ impl App {
 
     pub fn handle(&self) -> Easy {
         let mut handle = Easy::new();
-        if let Some(ref proxy) = self.s3_proxy {
+        if let Some(ref proxy) = self.config.uploader.proxy() {
             handle.proxy(proxy).unwrap();
         }
         return handle

src/bin/fill-in-user-id.rs

@@ -14,26 +14,25 @@ extern crate rustc_serialize;
 
 use std::path::PathBuf;
 
-use cargo_registry::{http, env, App};
+use cargo_registry::{http, env, App, Replica};
 use cargo_registry::util::{CargoResult, human};
 
 #[allow(dead_code)]
 fn main() {
     git2::Repository::init("tmp/test").unwrap();
+    let api_protocol = String::from("https");
+    let uploader = cargo_registry::Uploader::NoOp;
     let config = cargo_registry::Config {
-        s3_bucket: String::new(),
-        s3_access_key: String::new(),
-        s3_secret_key: String::new(),
-        s3_region: None,
-        s3_proxy: None,
+        uploader: uploader,
         session_key: String::new(),
         git_repo_checkout: PathBuf::from("tmp/test"),
         gh_client_id: env("GH_CLIENT_ID"),
         gh_client_secret: env("GH_CLIENT_SECRET"),
         db_url: env("DATABASE_URL"),
         env: cargo_registry::Env::Production,
         max_upload_size: 0,
-        mirror: false,
+        mirror: Replica::Primary,
+        api_protocol: api_protocol,
     };
     let app = cargo_registry::App::new(&config);
     {

src/bin/server.rs

@@ -5,8 +5,9 @@ extern crate conduit_middleware;
 extern crate civet;
 extern crate git2;
 extern crate env_logger;
+extern crate s3;
 
-use cargo_registry::env;
+use cargo_registry::{env, Env, Uploader, Replica};
 use civet::Server;
 use std::env;
 use std::fs::{self, File};
@@ -38,26 +39,73 @@ fn main() {
     cfg.set_str("user.name", "bors").unwrap();
     cfg.set_str("user.email", "[email protected]").unwrap();
 
+    let api_protocol = String::from("https");
+    let mirror = if env::var("MIRROR").is_ok() {
+        Replica::ReadOnlyMirror
+    } else {
+        Replica::Primary
+    };
+
     let heroku = env::var("HEROKU").is_ok();
     let cargo_env = if heroku {
-        cargo_registry::Env::Production
+        Env::Production
     } else {
-        cargo_registry::Env::Development
+        Env::Development
     };
+
+    let uploader = match (cargo_env, mirror) {
+        (Env::Production, Replica::Primary) => {
+            // `env` panics if these vars are not set
+            Uploader::S3 {
+                bucket: s3::Bucket::new(env("S3_BUCKET"),
+                                        env::var("S3_REGION").ok(),
+                                        env("S3_ACCESS_KEY"),
+                                        env("S3_SECRET_KEY"),
+                                        &api_protocol),
+                proxy: None,
+            }
+        },
+        (Env::Production, Replica::ReadOnlyMirror) => {
+            // Read-only mirrors don't need access key or secret key,
+            // but they might have them. Definitely need bucket though.
+            Uploader::S3 {
+                bucket: s3::Bucket::new(env("S3_BUCKET"),
+                                        env::var("S3_REGION").ok(),
+                                        env::var("S3_ACCESS_KEY").unwrap_or(String::new()),
+                                        env::var("S3_SECRET_KEY").unwrap_or(String::new()),
+                                        &api_protocol),
+                proxy: None,
+            }
+        },
+        _ => {
+            if env::var("S3_BUCKET").is_ok() {
+                println!("Using S3 uploader");
+                Uploader::S3 {
+                    bucket: s3::Bucket::new(env("S3_BUCKET"),
+                                            env::var("S3_REGION").ok(),
+                                            env::var("S3_ACCESS_KEY").unwrap_or(String::new()),
+                                            env::var("S3_SECRET_KEY").unwrap_or(String::new()),
+                                            &api_protocol),
+                    proxy: None,
+                }
+            } else {
+                println!("Using local uploader, crate files will be in the dist directory");
+                Uploader::Local
+            }
+        },
+    };
+
     let config = cargo_registry::Config {
-        s3_bucket: env("S3_BUCKET"),
-        s3_access_key: env("S3_ACCESS_KEY"),
-        s3_secret_key: env("S3_SECRET_KEY"),
-        s3_region: env::var("S3_REGION").ok(),
-        s3_proxy: None,
+        uploader: uploader,
         session_key: env("SESSION_KEY"),
         git_repo_checkout: checkout,
         gh_client_id: env("GH_CLIENT_ID"),
         gh_client_secret: env("GH_CLIENT_SECRET"),
         db_url: env("DATABASE_URL"),
         env: cargo_env,
         max_upload_size: 10 * 1024 * 1024,
-        mirror: env::var("MIRROR").is_ok(),
+        mirror: mirror,
+        api_protocol: api_protocol,
     };
     let app = cargo_registry::App::new(&config);
     let app = cargo_registry::middleware(Arc::new(app));
@@ -69,7 +117,7 @@ fn main() {
     } else {
         env::var("PORT").ok().and_then(|s| s.parse().ok()).unwrap_or(8888)
     };
-    let threads = if cargo_env == cargo_registry::Env::Development {1} else {50};
+    let threads = if cargo_env == Env::Development {1} else {50};
     let mut cfg = civet::Config::new();
     cfg.port(port).threads(threads).keep_alive(true);
     let _a = Server::start(cfg, app);

src/config.rs

@@ -1,26 +1,16 @@
 use std::path::PathBuf;
+use {Uploader, Replica};
 
 #[derive(Clone)]
 pub struct Config {
-    pub s3_bucket: String,
-    pub s3_region: Option<String>,
-    pub s3_access_key: String,
-    pub s3_secret_key: String,
-    pub s3_proxy: Option<String>,
+    pub uploader: Uploader,
     pub session_key: String,
     pub git_repo_checkout: PathBuf,
     pub gh_client_id: String,
     pub gh_client_secret: String,
     pub db_url: String,
     pub env: ::Env,
     pub max_upload_size: u64,
-    pub mirror: bool,
-}
-
-impl Config {
-    pub fn api_protocol(&self) -> &'static str {
-        // When testing we route all API traffic over HTTP so we can
-        // sniff/record it, but everywhere else we use https
-        if self.env == ::Env::Test {"http"} else {"https"}
-    }
+    pub mirror: Replica,
+    pub api_protocol: String,
 }

src/http.rs

@@ -12,7 +12,7 @@ use std::str;
 /// parse_github_response to handle the "common" processing of responses.
 pub fn github(app: &App, url: &str, auth: &Token)
               -> Result<(Easy, Vec<u8>), curl::Error> {
-    let url = format!("{}://api.github.com{}", app.config.api_protocol(), url);
+    let url = format!("{}://api.github.com{}", app.config.api_protocol, url);
     info!("GITHUB HTTP: {}", url);
 
     let mut headers = List::new();

src/krate.rs

@@ -1,14 +1,9 @@
 use std::ascii::AsciiExt;
 use std::cmp;
 use std::collections::HashMap;
-use std::io::prelude::*;
-use std::io;
-use std::mem;
-use std::sync::Arc;
 
 use conduit::{Request, Response};
 use conduit_router::RequestParams;
-use curl::easy::Easy;
 use diesel::prelude::*;
 use diesel::pg::PgConnection;
 use diesel::pg::upsert::*;
@@ -23,7 +18,7 @@ use semver;
 use time::{Timespec, Duration};
 use url::Url;
 
-use {Model, User, Keyword, Version, Category, Badge};
+use {Model, User, Keyword, Version, Category, Badge, Replica};
 use app::{App, RequestApp};
 use db::RequestTransaction;
 use dependency::{Dependency, EncodableDependency};
@@ -36,7 +31,7 @@ use upload;
 use user::RequestUser;
 use owner::{EncodableOwner, Owner, Rights, OwnerKind, Team, rights, CrateOwner};
 use util::errors::NotFound;
-use util::{LimitErrorReader, HashingReader};
+use util::{read_le_u32, read_fill};
 use util::{RequestUtils, CargoResult, internal, ChainError, human};
 use version::EncodableVersion;
 use schema::*;
@@ -524,10 +519,6 @@ impl Crate {
         Ok(())
     }
 
-    pub fn s3_path(&self, version: &str) -> String {
-        format!("/crates/{}/{}-{}.crate", self.name, self.name, version)
-    }
-
     pub fn add_version(&mut self,
                        conn: &GenericConnection,
                        ver: &semver::Version,
@@ -890,45 +881,10 @@ pub fn new(req: &mut Request) -> CargoResult<Response> {
     )?;
     let max_version = krate.max_version(req.tx()?)?;
 
-    // Upload the crate to S3
-    let mut handle = req.app().handle();
-    let path = krate.s3_path(&vers.to_string());
-    let (response, cksum) = {
-        let length = read_le_u32(req.body())?;
-        let body = LimitErrorReader::new(req.body(), max);
-        let mut body = HashingReader::new(body);
-        let mut response = Vec::new();
-        {
-            let mut s3req = app.bucket.put(&mut handle, &path, &mut body,
-                                           "application/x-tar",
-                                           length as u64);
-            s3req.write_function(|data| {
-                response.extend(data);
-                Ok(data.len())
-            }).unwrap();
-            s3req.perform().chain_error(|| {
-                internal(format!("failed to upload to S3: `{}`", path))
-            })?;
-        }
-        (response, body.finalize())
-    };
-    if handle.response_code().unwrap() != 200 {
-        let response = String::from_utf8_lossy(&response);
-        return Err(internal(format!("failed to get a 200 response from S3: {}",
-                                    response)))
-    }
-
+    // Upload the crate, return way to delete the crate from the server
     // If the git commands fail below, we shouldn't keep the crate on the
     // server.
-    struct Bomb { app: Arc<App>, path: Option<String>, handle: Easy }
-    impl Drop for Bomb {
-        fn drop(&mut self) {
-            if let Some(ref path) = self.path {
-                drop(self.app.bucket.delete(&mut self.handle, &path).perform());
-            }
-        }
-    }
-    let mut bomb = Bomb { app: app.clone(), path: Some(path), handle: handle };
+    let (cksum, mut bomb) = app.config.uploader.upload(req, &krate, max, &vers)?;
 
     // Register this crate in our local git repo.
     let git_crate = git::Crate {
@@ -1003,28 +959,6 @@ fn parse_new_headers(req: &mut Request) -> CargoResult<(upload::NewCrate, User)>
     Ok((new, user.clone()))
 }
 
-fn read_le_u32<R: Read + ?Sized>(r: &mut R) -> io::Result<u32> {
-    let mut b = [0; 4];
-    read_fill(r, &mut b)?;
-    Ok(((b[0] as u32) <<  0) |
-       ((b[1] as u32) <<  8) |
-       ((b[2] as u32) << 16) |
-       ((b[3] as u32) << 24))
-}
-
-fn read_fill<R: Read + ?Sized>(r: &mut R, mut slice: &mut [u8])
-                               -> io::Result<()> {
-    while slice.len() > 0 {
-        let n = r.read(slice)?;
-        if n == 0 {
-            return Err(io::Error::new(io::ErrorKind::Other,
-                                      "end of file reached"))
-        }
-        slice = &mut mem::replace(&mut slice, &mut [])[n..];
-    }
-    Ok(())
-}
-
 /// Handles the `GET /crates/:crate_id/:version/download` route.
 pub fn download(req: &mut Request) -> CargoResult<Response> {
     let crate_name = &req.params()["crate_id"];
@@ -1034,15 +968,16 @@ pub fn download(req: &mut Request) -> CargoResult<Response> {
     // API-only mirrors won't have any crates in their database, and
     // incrementing the download count will look up the crate in the
     // database. Mirrors just want to pass along a redirect URL.
-    if req.app().config.mirror {
+    if req.app().config.mirror == Replica::ReadOnlyMirror {
         let _ = increment_download_counts(req, crate_name, version);
     } else {
         increment_download_counts(req, crate_name, version)?;
     }
 
-    let redirect_url = format!("https://{}/crates/{}/{}-{}.crate",
-                               req.app().bucket.host(),
-                               crate_name, crate_name, version);
+    let redirect_url = req.app().config.uploader
+        .crate_location(crate_name, version).ok_or_else(||
+            human("crate files not found")
+        )?;
 
     if req.wants_json() {
         #[derive(RustcEncodable)]