Implement a web backend for rustdoc_ng

This large commit implements and `html` output option for rustdoc_ng. The
executable has been altered to be invoked as "rustdoc_ng html <crate>" and
it will dump everything into the local "doc" directory. JSON can still be
generated by changing 'html' to 'json'.

This also fixes a number of bugs in rustdoc_ng relating to comment stripping,
along with some other various issues that I found along the way.

The `make doc` command has been altered to generate the new documentation into
the `doc/ng/$(CRATE)` directories.

Author: alexcrichton

RELEASES.txt

@@ -142,6 +142,7 @@ Version 0.8 (October 2013)
       * The runtime uses jemalloc for allocations.
       * Segmented stacks are temporarily disabled as part of the transition to
         the new runtime. Stack overflows are possible!
+      * A new documentation backend, rustdoc_ng, is available for use
 
 Version 0.7 (July 2013)
 -----------------------

mk/docs.mk

@@ -213,6 +213,7 @@ else
 
 # The rustdoc executable
 RUSTDOC = $(HBIN2_H_$(CFG_BUILD_TRIPLE))/rustdoc$(X_$(CFG_BUILD_TRIPLE))
+RUSTDOC_NG = $(HBIN2_H_$(CFG_BUILD_TRIPLE))/rustdoc_ng$(X_$(CFG_BUILD_TRIPLE))
 
 # The library documenting macro
 # $(1) - The output directory
@@ -230,8 +231,22 @@ doc/$(1)/rust.css: rust.css
 DOCS += doc/$(1)/index.html
 endef
 
+# The library documenting macro
+# $(1) - The output directory
+# $(2) - The crate file
+# $(3) - The crate soruce files
+define libdocng
+doc/ng/$(1)/index.html: $(2) $(3) $$(RUSTDOC_NG)
+	@$$(call E, rustdoc_ng: $$@)
+	$(Q)$(RUSTDOC_NG) html $(2) -o doc/ng
+
+DOCS += doc/ng/$(1)/index.html
+endef
+
 $(eval $(call libdoc,std,$(STDLIB_CRATE),$(STDLIB_INPUTS)))
 $(eval $(call libdoc,extra,$(EXTRALIB_CRATE),$(EXTRALIB_INPUTS)))
+$(eval $(call libdocng,std,$(STDLIB_CRATE),$(STDLIB_INPUTS)))
+$(eval $(call libdocng,extra,$(EXTRALIB_CRATE),$(EXTRALIB_INPUTS)))
 endif
 
 

src/libextra/extra.rs

@@ -25,6 +25,10 @@ Rust extras are part of the standard Rust distribution.
        uuid = "122bed0b-c19b-4b82-b0b7-7ae8aead7297",
        url = "https://github.com/mozilla/rust/tree/master/src/libextra")];
 
+#[doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk.png",
+      html_favicon_url = "http://www.rust-lang.org/favicon.ico",
+      passes = "strip-hidden")];
+
 #[comment = "Rust extras"];
 #[license = "MIT/ASL2"];
 #[crate_type = "lib"];

src/libstd/macros.rs

@@ -9,6 +9,7 @@
 // except according to those terms.
 
 #[macro_escape];
+#[doc(hidden)];
 
 macro_rules! rterrln (
     ($( $arg:expr),+) => ( {

src/libstd/num/int_macros.rs

@@ -11,6 +11,7 @@
 // FIXME(#4375): this shouldn't have to be a nested module named 'generated'
 
 #[macro_escape];
+#[doc(hidden)];
 
 macro_rules! int_module (($T:ty, $bits:expr) => (mod generated {
 

src/libstd/num/uint_macros.rs

@@ -11,6 +11,7 @@
 // FIXME(#4375): this shouldn't have to be a nested module named 'generated'
 
 #[macro_escape];
+#[doc(hidden)];
 
 macro_rules! uint_module (($T:ty, $T_SIGNED:ty, $bits:expr) => (mod generated {
 

src/libstd/rt/mod.rs

@@ -54,7 +54,8 @@ Several modules in `core` are clients of `rt`:
 
 */
 
-#[doc(hidden)];
+// XXX: this should not be here.
+#[allow(missing_doc)];
 
 use cell::Cell;
 use clone::Clone;

src/libstd/std.rs

@@ -57,6 +57,10 @@ they contained the following prologue:
 #[license = "MIT/ASL2"];
 #[crate_type = "lib"];
 
+#[doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk.png",
+      html_favicon_url = "http://www.rust-lang.org/favicon.ico",
+      passes = "strip-hidden")];
+
 // Don't link to std. We are std.
 #[no_std];
 

src/libsyntax/ext/format.rs

@@ -262,7 +262,7 @@ impl Context {
                 let span = match self.names.find(&name) {
                     Some(e) => e.span,
                     None => {
-                        let msg = fmt!("There is no argument named `%s`", name);
+                        let msg = fmt!("there is no argument named `%s`", name);
                         self.ecx.span_err(self.fmtsp, msg);
                         return;
                     }

src/libsyntax/parse/comments.rs

@@ -107,9 +107,7 @@ pub fn strip_doc_comment_decoration(comment: &str) -> ~str {
     }
 
     if comment.starts_with("//") {
-        // FIXME #5475:
-        // return comment.slice(3u, comment.len()).to_owned();
-        let r = comment.slice(3u, comment.len()); return r.to_owned();
+        return comment.slice(3u, comment.len()).to_owned();
     }
 
     if comment.starts_with("/*") {

src/rustdoc_ng/clean.rs

@@ -15,6 +15,7 @@ use its = syntax::parse::token::ident_to_str;
 
 use syntax;
 use syntax::ast;
+use syntax::attr::AttributeMethods;
 
 use std;
 use doctree;
@@ -90,6 +91,48 @@ pub struct Item {
     id: ast::NodeId,
 }
 
+impl Item {
+    /// Finds the `doc` attribute as a List and returns the list of attributes
+    /// nested inside.
+    pub fn doc_list<'a>(&'a self) -> Option<&'a [Attribute]> {
+        for attr in self.attrs.iter() {
+            match *attr {
+                List(~"doc", ref list) => { return Some(list.as_slice()); }
+                _ => {}
+            }
+        }
+        return None;
+    }
+
+    /// Finds the `doc` attribute as a NameValue and returns the corresponding
+    /// value found.
+    pub fn doc_value<'a>(&'a self) -> Option<&'a str> {
+        for attr in self.attrs.iter() {
+            match *attr {
+                NameValue(~"doc", ref v) => { return Some(v.as_slice()); }
+                _ => {}
+            }
+        }
+        return None;
+    }
+
+    pub fn is_mod(&self) -> bool {
+        match self.inner { ModuleItem(*) => true, _ => false }
+    }
+    pub fn is_trait(&self) -> bool {
+        match self.inner { TraitItem(*) => true, _ => false }
+    }
+    pub fn is_struct(&self) -> bool {
+        match self.inner { StructItem(*) => true, _ => false }
+    }
+    pub fn is_enum(&self) -> bool {
+        match self.inner { EnumItem(*) => true, _ => false }
+    }
+    pub fn is_fn(&self) -> bool {
+        match self.inner { FunctionItem(*) => true, _ => false }
+    }
+}
+
 #[deriving(Clone, Encodable, Decodable)]
 pub enum ItemEnum {
     StructItem(Struct),
@@ -155,7 +198,7 @@ impl Clean<Attribute> for ast::MetaItem {
 
 impl Clean<Attribute> for ast::Attribute {
     fn clean(&self) -> Attribute {
-        self.node.value.clean()
+        self.desugar_doc().node.value.clean()
     }
 }
 
@@ -437,18 +480,24 @@ pub enum TraitMethod {
 }
 
 impl TraitMethod {
-    fn is_req(&self) -> bool {
+    pub fn is_req(&self) -> bool {
         match self {
             &Required(*) => true,
             _ => false,
         }
     }
-    fn is_def(&self) -> bool {
+    pub fn is_def(&self) -> bool {
         match self {
             &Provided(*) => true,
             _ => false,
         }
     }
+    pub fn item<'a>(&'a self) -> &'a Item {
+        match *self {
+            Required(ref item) => item,
+            Provided(ref item) => item,
+        }
+    }
 }
 
 impl Clean<TraitMethod> for ast::trait_method {

src/rustdoc_ng/fold.rs

@@ -91,21 +91,9 @@ pub trait DocFolder {
     }
 
     fn fold_crate(&mut self, mut c: Crate) -> Crate {
-        let mut mod_ = None;
-        std::util::swap(&mut mod_, &mut c.module);
-        let mod_ = mod_.unwrap();
-        c.module = self.fold_item(mod_);
-        let Crate { name, module } = c;
-        match module {
-            Some(Item { inner: ModuleItem(m), name: name_, attrs: attrs_,
-            source, visibility: vis, id }) => {
-                return Crate { module: Some(Item { inner:
-                                            ModuleItem(self.fold_mod(m)),
-                                            name: name_, attrs: attrs_,
-                                            source: source, id: id, visibility: vis }), name: name};
-            },
-            Some(_) => fail!("non-module item set as module of crate"),
-            None => return Crate { module: None, name: name},
-        }
+        c.module = match std::util::replace(&mut c.module, None) {
+            Some(module) => self.fold_item(module), None => None
+        };
+        return c;
     }
 }