rustdoc: Use sundown for markdown highlighting

This takes rendering times of documentation down from 30s to 0.5s. Kinda sad
that none of the parallelism is needed, but oh well!

Closes #7380
cc #3546

Author: alexcrichton

src/librustdoc/html/markdown.rs

@@ -8,47 +8,98 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#[allow(cstack)]; // each rendering task runs on a fixed stack segment.
+
 use std::fmt;
-use std::rt::io::Reader;
-use std::rt::io::pipe::PipeStream;
-use std::rt::io::process::{ProcessConfig, Process, CreatePipe};
+use std::libc;
 use std::rt::io;
+use std::vec;
 
 pub struct Markdown<'self>(&'self str);
 
+static OUTPUT_UNIT: libc::size_t = 64;
+
+type sd_markdown = libc::c_void;  // this is opaque to us
+
+// this is a large struct of callbacks we don't use
+type sd_callbacks = [libc::size_t, ..26];
+
+struct html_toc_data {
+    header_count: libc::c_int,
+    current_level: libc::c_int,
+    level_offset: libc::c_int,
+}
+
+struct html_renderopt {
+    toc_data: html_toc_data,
+    flags: libc::c_uint,
+    link_attributes: Option<extern "C" fn(*buf, *buf, *libc::c_void)>,
+}
+
+struct buf {
+    data: *u8,
+    size: libc::size_t,
+    asize: libc::size_t,
+    unit: libc::size_t,
+}
+
+// sundown FFI
+extern {
+    fn sdhtml_renderer(callbacks: *sd_callbacks,
+                       options_ptr: *html_renderopt,
+                       render_flags: libc::c_uint);
+    fn sd_markdown_new(extensions: libc::c_uint,
+                       max_nesting: libc::size_t,
+                       callbacks: *sd_callbacks,
+                       opaque: *libc::c_void) -> *sd_markdown;
+    fn sd_markdown_render(ob: *buf,
+                          document: *u8,
+                          doc_size: libc::size_t,
+                          md: *sd_markdown);
+    fn sd_markdown_free(md: *sd_markdown);
+
+    fn bufnew(unit: libc::size_t) -> *buf;
+    fn bufrelease(b: *buf);
+
+}
+
+fn render(w: &mut io::Writer, s: &str) {
+    // This code is all lifted from examples/sundown.c in the sundown repo
+    unsafe {
+        let ob = bufnew(OUTPUT_UNIT);
+        let options = html_renderopt {
+            toc_data: html_toc_data {
+                header_count: 0,
+                current_level: 0,
+                level_offset: 0,
+            },
+            flags: 0,
+            link_attributes: None,
+        };
+        let callbacks: sd_callbacks = [0, ..26];
+
+        sdhtml_renderer(&callbacks, &options, 0);
+        let markdown = sd_markdown_new(0, 16, &callbacks,
+                                       &options as *html_renderopt as *libc::c_void);
+
+        do s.as_imm_buf |data, len| {
+            sd_markdown_render(ob, data, len as libc::size_t, markdown);
+        }
+        sd_markdown_free(markdown);
+
+        do vec::raw::buf_as_slice((*ob).data, (*ob).size as uint) |buf| {
+            w.write(buf);
+        }
+
+        bufrelease(ob);
+    }
+}
+
 impl<'self> fmt::Default for Markdown<'self> {
     fn fmt(md: &Markdown<'self>, fmt: &mut fmt::Formatter) {
+        // This is actually common enough to special-case
         if md.len() == 0 { return; }
 
-        // Create the pandoc process
-        do io::io_error::cond.trap(|err| {
-            fail2!("Error executing `pandoc`: {}", err.desc);
-        }).inside {
-            let io = ~[CreatePipe(PipeStream::new().unwrap(), true, false),
-                       CreatePipe(PipeStream::new().unwrap(), false, true)];
-            let args = ProcessConfig {
-                program: "pandoc",
-                args: [],
-                env: None,
-                cwd: None,
-                io: io,
-            };
-            let mut p = Process::new(args).expect("couldn't fork for pandoc");
-
-            // Write the markdown to stdin and close it.
-            p.io[0].get_mut_ref().write(md.as_bytes());
-            p.io[0] = None;
-
-            // Ferry the output from pandoc over to the destination buffer.
-            let mut buf = [0, ..1024];
-            loop {
-                match p.io[1].get_mut_ref().read(buf) {
-                    None | Some(0) => { break }
-                    Some(n) => {
-                        fmt.buf.write(buf.slice_to(n));
-                    }
-                }
-            }
-        }
+        render(fmt.buf, md.as_slice());
     }
 }

src/librustdoc/html/render.rs

@@ -401,8 +401,16 @@ impl Context {
             let mut task = task::task();
             task.unlinked(); // we kill things manually
             task.name(format!("worker{}", i));
-            do task.spawn_with(cache.clone()) |cache| {
+            task.spawn_with(cache.clone(),
+                            |cache| worker(cache, &port, &chan, &prog_chan));
+
+            fn worker(cache: RWArc<Cache>,
+                      port: &SharedPort<Work>,
+                      chan: &SharedChan<Work>,
+                      prog_chan: &SharedChan<Progress>) {
+                #[fixed_stack_segment]; // we hit markdown FFI *a lot*
                 local_data::set(cache_key, cache);
+
                 loop {
                     match port.recv() {
                         Process(cx, item) => {
@@ -425,28 +433,20 @@ impl Context {
             }
         }
 
-        let watcher_chan = chan.clone();
-        let (done_port, done_chan) = comm::stream();
-        do task::spawn {
-            let mut jobs = 0;
-            loop {
-                match prog_port.recv() {
-                    JobNew => jobs += 1,
-                    JobDone => jobs -= 1,
-                }
-
-                if jobs == 0 { break }
+        chan.send(Process(self, item));
+        let mut jobs = 1;
+        loop {
+            match prog_port.recv() {
+                JobNew => jobs += 1,
+                JobDone => jobs -= 1,
             }
 
-            for _ in range(0, WORKERS) {
-                watcher_chan.send(Die);
-            }
-            done_chan.send(());
+            if jobs == 0 { break }
         }
 
-        prog_chan.send(JobNew);
-        chan.send(Process(self, item));
-        done_port.recv();
+        for _ in range(0, WORKERS) {
+            chan.send(Die);
+        }
     }
 
     fn item(&mut self, item: clean::Item, f: &fn(&mut Context, clean::Item)) {