rustdoc: Add the ability to input json

This modifies the command-line usage of rustdoc to intake its own JSON output as
well as a rust source file. This also alters the command line from
`rustdoc input file`  to `rustdoc file` with the input/output formats specified
as -r and -w, respectively.

When using a JSON input, no passes or plugins are re-run over the json, instead
the output is generated directly from the JSON that was provided. Passes and
plugins are still run on rust source input, however.

Author: alexcrichton

mk/docs.mk

@@ -227,7 +227,7 @@ RUSTDOC = $(HBIN2_H_$(CFG_BUILD_TRIPLE))/rustdoc$(X_$(CFG_BUILD_TRIPLE))
 define libdoc
 doc/$(1)/index.html: $$(RUSTDOC) $$(TLIB2_T_$(3)_H_$(3))/$(CFG_STDLIB_$(3))
 	@$$(call E, rustdoc: $$@)
-	$(Q)$(RUSTDOC) html $(2)
+	$(Q)$(RUSTDOC) $(2)
 
 DOCS += doc/$(1)/index.html
 endef

src/librustdoc/html/render.rs

@@ -76,7 +76,7 @@ struct Cache {
 struct SourceCollector<'self> {
     seen: HashSet<~str>,
     dst: Path,
-    cx: &'self Context,
+    cx: &'self mut Context,
 }
 
 struct Item<'self> { cx: &'self Context, item: &'self clean::Item, }
@@ -179,15 +179,15 @@ pub fn run(mut crate: clean::Crate, dst: Path) {
         w.flush();
     }
 
-    if cx.include_sources {
+    {
         let dst = cx.dst.push("src");
         mkdir(&dst);
         let dst = dst.push(crate.name);
         mkdir(&dst);
         let mut folder = SourceCollector {
             dst: dst,
             seen: HashSet::new(),
-            cx: &cx,
+            cx: &mut cx,
         };
         crate = folder.fold_crate(crate);
     }
@@ -229,16 +229,28 @@ fn clean_srcpath(src: &str, f: &fn(&str)) {
 
 impl<'self> DocFolder for SourceCollector<'self> {
     fn fold_item(&mut self, item: clean::Item) -> Option<clean::Item> {
-        if !self.seen.contains(&item.source.filename) {
-            self.emit_source(item.source.filename);
+        if self.cx.include_sources && !self.seen.contains(&item.source.filename) {
+            // If it turns out that we couldn't read this file, then we probably
+            // can't read any of the files (generating html output from json or
+            // something like that), so just don't include sources for the
+            // entire crate. The other option is maintaining this mapping on a
+            // per-file basis, but that's probably not worth it...
+            self.cx.include_sources = self.emit_source(item.source.filename);
             self.seen.insert(item.source.filename.clone());
+
+            if !self.cx.include_sources {
+                println!("warning: source code was requested to be rendered, \
+                          but `{}` is a missing source file.",
+                         item.source.filename);
+                println!("         skipping rendering of source code");
+            }
         }
         self.fold_item_recur(item)
     }
 }
 
 impl<'self> SourceCollector<'self> {
-    fn emit_source(&self, filename: &str) {
+    fn emit_source(&mut self, filename: &str) -> bool {
         let p = Path(filename);
 
         // Read the contents of the file
@@ -251,7 +263,11 @@ impl<'self> SourceCollector<'self> {
             // If we couldn't open this file, then just returns because it
             // probably means that it's some standard library macro thing and we
             // can't have the source to it anyway.
-            let mut r = match r { Some(r) => r, None => return };
+            let mut r = match r {
+                Some(r) => r,
+                // eew macro hacks
+                None => return filename == "<std-macros>"
+            };
 
             // read everything
             loop {
@@ -283,6 +299,7 @@ impl<'self> SourceCollector<'self> {
         };
         layout::render(&mut w as &mut io::Writer, &self.cx.layout,
                        &page, &(""), &Source(contents.as_slice()));
+        return true;
     }
 }
 

src/librustdoc/rustdoc.rs

@@ -21,13 +21,15 @@ extern mod syntax;
 extern mod rustc;
 extern mod extra;
 
-use extra::serialize::Encodable;
-use extra::time;
-use extra::getopts::groups;
 use std::cell::Cell;
-use std::rt::io;
 use std::rt::io::Writer;
 use std::rt::io::file::FileInfo;
+use std::rt::io;
+use extra::getopts;
+use extra::getopts::groups;
+use extra::json;
+use extra::serialize::{Decodable, Encodable};
+use extra::time;
 
 pub mod clean;
 pub mod core;
@@ -70,9 +72,7 @@ static DEFAULT_PASSES: &'static [&'static str] = &[
 
 local_data_key!(pub ctxtkey: @core::DocContext)
 
-enum OutputFormat {
-    HTML, JSON
-}
+type Output = (clean::Crate, ~[plugins::PluginJson]);
 
 pub fn main() {
     std::os::set_exit_status(main_args(std::os::args()));
@@ -81,6 +81,12 @@ pub fn main() {
 pub fn opts() -> ~[groups::OptGroup] {
     use extra::getopts::groups::*;
     ~[
+        optflag("h", "help", "show this help message"),
+        optopt("r", "input-format", "the input type of the specified file",
+               "[rust|json]"),
+        optopt("w", "output-format", "the output type to write",
+               "[html|json]"),
+        optopt("o", "output", "where to place the output", "PATH"),
         optmulti("L", "library-path", "directory to add to crate search path",
                  "DIR"),
         optmulti("", "plugin-path", "directory to load plugins from", "DIR"),
@@ -89,32 +95,22 @@ pub fn opts() -> ~[groups::OptGroup] {
                  "PASSES"),
         optmulti("", "plugins", "space separated list of plugins to also load",
                  "PLUGINS"),
-        optflag("h", "help", "show this help message"),
         optflag("", "nodefaults", "don't run the default passes"),
-        optopt("o", "output", "where to place the output", "PATH"),
     ]
 }
 
 pub fn usage(argv0: &str) {
-    println(groups::usage(format!("{} [options] [html|json] <crate>",
-                                  argv0), opts()));
+    println(groups::usage(format!("{} [options] <input>", argv0), opts()));
 }
 
 pub fn main_args(args: &[~str]) -> int {
-    //use extra::getopts::groups::*;
-
     let matches = groups::getopts(args.tail(), opts()).unwrap();
-
     if matches.opt_present("h") || matches.opt_present("help") {
         usage(args[0]);
         return 0;
     }
 
-    let mut default_passes = !matches.opt_present("nodefaults");
-    let mut passes = matches.opt_strs("passes");
-    let mut plugins = matches.opt_strs("plugins");
-
-    if passes == ~[~"list"] {
+    if matches.opt_strs("passes") == ~[~"list"] {
         println("Available passes for running rustdoc:");
         for &(name, _, description) in PASSES.iter() {
             println!("{:>20s} - {}", name, description);
@@ -126,25 +122,68 @@ pub fn main_args(args: &[~str]) -> int {
         return 0;
     }
 
-    let (format, cratefile) = match matches.free.clone() {
-        [~"json", crate] => (JSON, crate),
-        [~"html", crate] => (HTML, crate),
-        [s, _] => {
-            println!("Unknown output format: `{}`", s);
-            usage(args[0]);
+    let (crate, res) = match acquire_input(&matches) {
+        Ok(pair) => pair,
+        Err(s) => {
+            println!("input error: {}", s);
             return 1;
         }
-        [_, .._] => {
-            println!("Expected exactly one crate to process");
-            usage(args[0]);
-            return 1;
+    };
+
+    info2!("going to format");
+    let started = time::precise_time_ns();
+    let output = matches.opt_str("o").map(|s| Path(*s));
+    match matches.opt_str("w") {
+        Some(~"html") | None => {
+            html::render::run(crate, output.unwrap_or(Path("doc")))
         }
-        _ => {
-            println!("Expected an output format and then one crate");
-            usage(args[0]);
+        Some(~"json") => {
+            json_output(crate, res, output.unwrap_or(Path("doc.json")))
+        }
+        Some(s) => {
+            println!("unknown output format: {}", s);
             return 1;
         }
-    };
+    }
+    let ended = time::precise_time_ns();
+    info2!("Took {:.03f}s", (ended as f64 - started as f64) / 1000000000f64);
+
+    return 0;
+}
+
+/// Looks inside the command line arguments to extract the relevant input format
+/// and files and then generates the necessary rustdoc output for formatting.
+fn acquire_input(matches: &getopts::Matches) -> Result<Output, ~str> {
+    if matches.free.len() == 0 {
+        return Err(~"expected an input file to act on");
+    } if matches.free.len() > 1 {
+        return Err(~"only one input file may be specified");
+    }
+
+    let input = matches.free[0].as_slice();
+    match matches.opt_str("r") {
+        Some(~"rust") => Ok(rust_input(input, matches)),
+        Some(~"json") => json_input(input),
+        Some(s) => Err("unknown input format: " + s),
+        None => {
+            if input.ends_with(".json") {
+                json_input(input)
+            } else {
+                Ok(rust_input(input, matches))
+            }
+        }
+    }
+}
+
+/// Interprets the input file as a rust source file, passing it through the
+/// compiler all the way through the analysis passes. The rustdoc output is then
+/// generated from the cleaned AST of the crate.
+///
+/// This form of input will run all of the plug/cleaning passes
+fn rust_input(cratefile: &str, matches: &getopts::Matches) -> Output {
+    let mut default_passes = !matches.opt_present("nodefaults");
+    let mut passes = matches.opt_strs("passes");
+    let mut plugins = matches.opt_strs("plugins");
 
     // First, parse the crate and extract all relevant information.
     let libs = Cell::new(matches.opt_strs("L").map(|s| Path(*s)));
@@ -206,45 +245,73 @@ pub fn main_args(args: &[~str]) -> int {
 
     // Run everything!
     info2!("Executing passes/plugins");
-    let (crate, res) = pm.run_plugins(crate);
+    return pm.run_plugins(crate);
+}
 
-    info2!("going to format");
-    let started = time::precise_time_ns();
-    let output = matches.opt_str("o").map(|s| Path(*s));
-    match format {
-        HTML => { html::render::run(crate, output.unwrap_or(Path("doc"))) }
-        JSON => { jsonify(crate, res, output.unwrap_or(Path("doc.json"))) }
+/// This input format purely deserializes the json output file. No passes are
+/// run over the deserialized output.
+fn json_input(input: &str) -> Result<Output, ~str> {
+    let input = match ::std::io::file_reader(&Path(input)) {
+        Ok(i) => i,
+        Err(s) => return Err(s),
+    };
+    match json::from_reader(input) {
+        Err(s) => Err(s.to_str()),
+        Ok(json::Object(obj)) => {
+            let mut obj = obj;
+            // Make sure the schema is what we expect
+            match obj.pop(&~"schema") {
+                Some(json::String(version)) => {
+                    if version.as_slice() != SCHEMA_VERSION {
+                        return Err(format!("sorry, but I only understand \
+                                            version {}", SCHEMA_VERSION))
+                    }
+                }
+                Some(*) => return Err(~"malformed json"),
+                None => return Err(~"expected a schema version"),
+            }
+            let crate = match obj.pop(&~"crate") {
+                Some(json) => {
+                    let mut d = json::Decoder(json);
+                    Decodable::decode(&mut d)
+                }
+                None => return Err(~"malformed json"),
+            };
+            // XXX: this should read from the "plugins" field, but currently
+            //      Json doesn't implement decodable...
+            let plugin_output = ~[];
+            Ok((crate, plugin_output))
+        }
+        Ok(*) => Err(~"malformed json input: expected an object at the top"),
     }
-    let ended = time::precise_time_ns();
-    info2!("Took {:.03f}s", (ended as f64 - started as f64) / 1000000000f64);
-
-    return 0;
 }
 
-fn jsonify(crate: clean::Crate, res: ~[plugins::PluginJson], dst: Path) {
+/// Outputs the crate/plugin json as a giant json blob at the specified
+/// destination.
+fn json_output(crate: clean::Crate, res: ~[plugins::PluginJson], dst: Path) {
     // {
     //   "schema": version,
     //   "crate": { parsed crate ... },
     //   "plugins": { output of plugins ... }
     // }
     let mut json = ~extra::treemap::TreeMap::new();
-    json.insert(~"schema", extra::json::String(SCHEMA_VERSION.to_owned()));
+    json.insert(~"schema", json::String(SCHEMA_VERSION.to_owned()));
     let plugins_json = ~res.move_iter().filter_map(|opt| opt).collect();
 
     // FIXME #8335: yuck, Rust -> str -> JSON round trip! No way to .encode
     // straight to the Rust JSON representation.
     let crate_json_str = do std::io::with_str_writer |w| {
-        crate.encode(&mut extra::json::Encoder(w));
+        crate.encode(&mut json::Encoder(w));
     };
-    let crate_json = match extra::json::from_str(crate_json_str) {
+    let crate_json = match json::from_str(crate_json_str) {
         Ok(j) => j,
         Err(_) => fail!("Rust generated JSON is invalid??")
     };
 
     json.insert(~"crate", crate_json);
-    json.insert(~"plugins", extra::json::Object(plugins_json));
+    json.insert(~"plugins", json::Object(plugins_json));
 
     let mut file = dst.open_writer(io::Create).unwrap();
-    let output = extra::json::Object(json).to_str();
+    let output = json::Object(json).to_str();
     file.write(output.as_bytes());
 }