rustdoc: (WIP) Allows the doc generation from compiled crates.

This commit uses an existing foreign inlining facility to implement
the doc generation from compiled crates. At the highest level,
this is a *huge* hack since the metadata doesn't provide everything
we need to generate docs.

Fixes rust-lang#2206. Will fix rust-lang#15309 once integrated with Makefile.

Author: lifthrasiir

src/librustdoc/clean/mod.rs

@@ -50,6 +50,7 @@ use rustc::middle::stability;
 use rustc::session::config;
 
 use std::rc::Rc;
+use std::cell::RefCell;
 use std::u32;
 use std::str::Str as StrTrait; // Conflicts with Str variant
 use std::char::Char as CharTrait; // Conflicts with Char variant
@@ -63,7 +64,7 @@ use visit_ast;
 /// Increment this when the `Crate` and related structures change.
 pub static SCHEMA_VERSION: &'static str = "0.8.3";
 
-mod inline;
+pub mod inline;
 
 // extract the stability index for a node from tcx, if possible
 fn get_stability(cx: &DocContext, def_id: ast::DefId) -> Option<Stability> {
@@ -124,19 +125,23 @@ pub struct Crate {
 
 impl<'a, 'tcx> Clean<Crate> for visit_ast::RustdocVisitor<'a, 'tcx> {
     fn clean(&self, cx: &DocContext) -> Crate {
-        let mut externs = Vec::new();
-        cx.sess().cstore.iter_crate_data(|n, meta| {
-            externs.push((n, meta.clean(cx)));
-        });
-        externs.sort_by(|&(a, _), &(b, _)| a.cmp(&b));
-
         // Figure out the name of this crate
         let input = config::Input::File(cx.src.clone());
         let name = link::find_crate_name(None, self.attrs.as_slice(), &input);
 
         // Clean the crate, translating the entire libsyntax AST to one that is
         // understood by rustdoc.
-        let mut module = self.module.clean(cx);
+        let module = self.module.clean(cx);
+
+        let postclean = RefCell::new(Some((name.to_string(), cx.src.clone(), module)));
+        postclean.clean(cx)
+    }
+}
+
+// post-cleaning processing consumes the cleaned results.
+impl Clean<Crate> for RefCell<Option<(String, FsPath, Item)>> {
+    fn clean(&self, cx: &DocContext) -> Crate {
+        let (name, src, mut module) = self.borrow_mut().take().unwrap();
 
         // Collect all inner modules which are tagged as implementations of
         // primitives.
@@ -194,9 +199,15 @@ impl<'a, 'tcx> Clean<Crate> for visit_ast::RustdocVisitor<'a, 'tcx> {
             m.items.extend(tmp.into_iter());
         }
 
+        let mut externs = Vec::new();
+        cx.sess().cstore.iter_crate_data(|n, meta| {
+            externs.push((n, meta.clean(cx)));
+        });
+        externs.sort_by(|&(a, _), &(b, _)| a.cmp(&b));
+
         Crate {
-            name: name.to_string(),
-            src: cx.src.clone(),
+            name: name,
+            src: src,
             module: Some(module),
             externs: externs,
             primitives: primitives,
@@ -1643,7 +1654,7 @@ pub struct Span {
 }
 
 impl Span {
-    fn empty() -> Span {
+    pub fn empty() -> Span {
         Span {
             filename: "".to_string(),
             loline: 0, locol: 0,

src/librustdoc/core.rs

@@ -11,11 +11,14 @@ pub use self::MaybeTyped::*;
 
 use rustc_driver::driver;
 use rustc::session::{mod, config};
+use rustc::metadata::decoder;
 use rustc::middle::{privacy, ty};
 use rustc::lint;
 use rustc_trans::back::link;
 
-use syntax::{ast, ast_map, codemap, diagnostic};
+use syntax::{ast, ast_map, ast_util, codemap, diagnostic};
+use syntax::parse::token;
+use syntax::ptr::P;
 
 use std::cell::RefCell;
 use std::collections::{HashMap, HashSet};
@@ -163,3 +166,184 @@ pub fn run_core(libs: Vec<Path>, cfgs: Vec<String>, externs: Externs,
     *analysis.inlined.borrow_mut() = map;
     (krate, analysis)
 }
+
+pub fn run_core_with_lib(libpath: &Path, srcpath: &Path) -> (clean::Crate, CrateAnalysis) {
+    let codemap = codemap::CodeMap::new();
+    let diagnostic_handler = diagnostic::default_handler(diagnostic::Auto, None);
+    let span_diagnostic_handler = diagnostic::mk_span_handler(diagnostic_handler, codemap);
+
+    let dummy_crate_name = "__crate";
+    let dummy_view_name = "__view";
+
+    let mut opts = config::basic_options();
+    opts.externs.insert(dummy_crate_name.into_string(),
+                        vec![libpath.as_str().unwrap().into_string()]);
+
+    let sess = session::build_session_(opts, None, span_diagnostic_handler);
+
+    // dummy AST to faciliate the crate loading
+    let dummy_crate_ident = ast::Ident::new(token::gensym(dummy_crate_name));
+    let dummy_view_ident = ast::Ident::new(token::gensym(dummy_view_name));
+    let krate = ast::Crate {
+        module: ast::Mod {
+            inner: codemap::DUMMY_SP,
+            view_items: vec![
+                ast::ViewItem {
+                    node: ast::ViewItemExternCrate(
+                        dummy_view_ident,
+                        Some((token::get_ident(dummy_crate_ident), ast::CookedStr)),
+                        ast::DUMMY_NODE_ID),
+                    attrs: Vec::new(),
+                    vis: ast::Inherited,
+                    span: codemap::DUMMY_SP,
+                },
+                ast::ViewItem {
+                    node: ast::ViewItemUse(
+                        P(codemap::dummy_spanned(ast::ViewPathSimple(
+                            dummy_crate_ident,
+                            ast::Path {
+                                span: codemap::DUMMY_SP,
+                                global: false,
+                                segments: vec![
+                                    ast::PathSegment {
+                                        identifier: dummy_view_ident, 
+                                        parameters: ast::PathParameters::none(),
+                                    },
+                                ],
+                            },
+                            ast::DUMMY_NODE_ID)
+                        ))
+                    ),
+                    attrs: Vec::new(),
+                    vis: ast::Public,
+                    span: codemap::DUMMY_SP,
+                },
+            ],
+            items: Vec::new(),
+        },
+        attrs: Vec::new(),
+        config: Vec::new(),
+        span: codemap::DUMMY_SP,
+        exported_macros: Vec::new(),
+    };
+
+    let mut forest = ast_map::Forest::new(krate);
+    let ast_map = driver::assign_node_ids_and_map(&sess, &mut forest);
+
+    let type_arena = TypedArena::new();
+    let ty::CrateAnalysis {
+        exported_items, public_items, ty_cx, ..
+    } = driver::phase_3_run_analysis_passes(sess, ast_map, &type_arena,
+                                            dummy_crate_name.into_string());
+
+    let ctxt = DocContext {
+        krate: ty_cx.map.krate(),
+        maybe_typed: Typed(ty_cx),
+        src: srcpath.clone(),
+        external_traits: RefCell::new(Some(HashMap::new())),
+        external_typarams: RefCell::new(Some(HashMap::new())),
+        external_paths: RefCell::new(Some(HashMap::new())),
+        inlined: RefCell::new(Some(HashSet::new())),
+        populated_crate_impls: RefCell::new(HashSet::new()),
+    };
+
+    // there should be only one Def available, namely reexport
+    let mut view_node_id = None;
+    for (id, _) in ctxt.tcx().def_map.borrow().iter() {
+        assert!(view_node_id.is_none(), "multiple Defs available");
+        view_node_id = Some(*id);
+    }
+    let view_node_id = view_node_id.expect("no Def available");
+
+    // we now have all necessary environments, try to inline.
+    let inlined = clean::inline::try_inline(&ctxt, view_node_id, None);
+    let inlined = inlined.expect("cannot inline crate");
+    if inlined.len() != 1 {
+        panic!("cannot inline crate");
+    }
+    let inlined = inlined.into_iter().next().unwrap();
+
+    // we still have to fill some gaps, so get the crate data in our hands
+    let crate_num = 1; // we don't have std injection so this should be the first
+    let crate_data = ctxt.sess().cstore.get_crate_data(crate_num);
+    let crate_name = crate_data.name();
+
+    // fix external_paths for the given crate_num
+    {
+        let mut external_paths = ctxt.external_paths.borrow_mut();
+        for (def_id, &(ref mut fqn, _)) in external_paths.as_mut().unwrap().iter_mut() {
+            if def_id.krate == crate_num {
+                assert_eq!(fqn.head().map(|s| s.as_slice()), Some(dummy_crate_name));
+                if fqn.len() == 1 {
+                    fqn[0] = "".into_string();
+                } else {
+                    fqn.remove(0);
+                }
+            }
+        }
+    }
+
+    let postclean = RefCell::new(Some((crate_name, srcpath.clone(), inlined)));
+    let mut krate = postclean.clean(&ctxt);
+
+    // why do we have both crate attributes and item attributes?!
+    let crate_attrs = decoder::get_crate_attributes(crate_data.data());
+    {
+        let mut attrs = &mut krate.module.as_mut().unwrap().attrs;
+        attrs.extend(crate_attrs.clean(&ctxt).into_iter());
+    }
+
+    // the reconstructed crate doesn't have exported macros (yet)
+    let macros = decoder::get_exported_macros(crate_data.data());
+    {
+        let mut module = match krate.module {
+            Some(clean::Item { inner: clean::ModuleItem(ref mut module), .. }) => module,
+            _ => panic!("unexpectedly cleaned crate")
+        };
+        for macro in macros.into_iter() {
+            // XXX okay, this is bad. the metadata doesn't have a direct macro name.
+            // for now we try to recognize `macro_rules!\s*([^/({\[]+)`.
+            // hope someone doesn't come up with `macro_rules! /*screw doc*/ foo()`...
+            let macname = {
+                let macro = macro.trim_left();
+                if macro.starts_with("macro_rules!") {
+                    let macro = macro.slice_from(12).trim_left();
+                    let sep = macro.find(['/', '(', '{', '['].as_slice());
+                    if let Some(sep) = sep {
+                        Some(format!("{}!", macro.slice_to(sep).trim_right()))
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                }
+            };
+            module.items.push(clean::Item {
+                name: macname,
+                attrs: Vec::new(),
+                source: clean::Span::empty(),
+                visibility: ast::Public.clean(&ctxt),
+                stability: None,
+                def_id: ast_util::local_def(ast::DUMMY_NODE_ID),
+                inner: clean::MacroItem(clean::Macro {
+                    source: macro,
+                }),
+            });
+        }
+    }
+
+    // we need the analysis for later uses
+    let DocContext {
+        external_paths, external_traits, external_typarams, inlined, ..
+    } = ctxt;
+    let analysis = CrateAnalysis {
+        exported_items: exported_items,
+        public_items: public_items,
+        external_paths: external_paths,
+        external_traits: external_traits,
+        external_typarams: external_typarams,
+        inlined: inlined,
+    };
+
+    (krate, analysis)
+}

src/librustdoc/lib.rs

@@ -282,10 +282,16 @@ fn acquire_input(input: &str,
     match matches.opt_str("r").as_ref().map(|s| s.as_slice()) {
         Some("rust") => Ok(rust_input(input, externs, matches)),
         Some("json") => json_input(input),
+        Some("lib") => Ok(lib_input(input)),
         Some(s) => Err(format!("unknown input format: {}", s)),
         None => {
             if input.ends_with(".json") {
                 json_input(input)
+            } else if input.ends_with(".rlib") ||
+                      input.ends_with(".so") ||
+                      input.ends_with(".dylib") ||
+                      input.ends_with(".dll") {
+                Ok(lib_input(input))
             } else {
                 Ok(rust_input(input, externs, matches))
             }
@@ -418,6 +424,27 @@ fn rust_input(cratefile: &str, externs: core::Externs, matches: &getopts::Matche
     return Output { krate: krate, json_plugins: json, passes: passes, };
 }
 
+/// This input format extracts the metadata from given rlib or dylib file.
+/// No passes are run over the deserialized output.
+fn lib_input(libfile: &str) -> Output {
+    let lib = Path::new(libfile);
+    info!("starting to run rustc");
+    let (krate, analysis) = std::task::try(proc() {
+        let lib = lib;
+        core::run_core_with_lib(&lib, &lib.clone())
+    }).map_err(|_| "rustc failed").unwrap();
+    info!("finished with rustc");
+    let mut analysis = Some(analysis);
+    ANALYSISKEY.with(|s| {
+        *s.borrow_mut() = analysis.take();
+    });
+
+    // FIXME: this should read from the "plugins" field, but currently
+    //      Json doesn't implement decodable...
+    let plugin_output = Vec::new();
+    Output { krate: krate, json_plugins: plugin_output, passes: Vec::new() }
+}
+
 /// This input format purely deserializes the json output file. No passes are
 /// run over the deserialized output.
 fn json_input(input: &str) -> Result<Output, String> {