Use sundown for markdown highlighting in rustdoc

This takes rendering times of documentation down from 30s to 0.5s. Kinda sad
that none of the parallelism is needed, but oh well!

Closes rust-lang#7380
cc rust-lang#3546

Author: alexcrichton

src/librustdoc/html/markdown.rs

@@ -8,47 +8,98 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#[allow(cstack)]; // each rendering task runs on a fixed stack segment.
+
 use std::fmt;
-use std::rt::io::Reader;
-use std::rt::io::pipe::PipeStream;
-use std::rt::io::process::{ProcessConfig, Process, CreatePipe};
+use std::libc;
 use std::rt::io;
+use std::vec;
 
 pub struct Markdown<'self>(&'self str);
 
+static OUTPUT_UNIT: libc::size_t = 64;
+
+type sd_markdown = libc::c_void;  // this is opaque to us
+
+// this is a large struct of callbacks we don't use
+type sd_callbacks = [libc::size_t, ..26];
+
+struct html_toc_data {
+    header_count: libc::c_int,
+    current_level: libc::c_int,
+    level_offset: libc::c_int,
+}
+
+struct html_renderopt {
+    toc_data: html_toc_data,
+    flags: libc::c_uint,
+    link_attributes: Option<extern "C" fn(*buf, *buf, *libc::c_void)>,
+}
+
+struct buf {
+    data: *u8,
+    size: libc::size_t,
+    asize: libc::size_t,
+    unit: libc::size_t,
+}
+
+// sundown FFI
+extern {
+    fn sdhtml_renderer(callbacks: *sd_callbacks,
+                       options_ptr: *html_renderopt,
+                       render_flags: libc::c_uint);
+    fn sd_markdown_new(extensions: libc::c_uint,
+                       max_nesting: libc::size_t,
+                       callbacks: *sd_callbacks,
+                       opaque: *libc::c_void) -> *sd_markdown;
+    fn sd_markdown_render(ob: *buf,
+                          document: *u8,
+                          doc_size: libc::size_t,
+                          md: *sd_markdown);
+    fn sd_markdown_free(md: *sd_markdown);
+
+    fn bufnew(unit: libc::size_t) -> *buf;
+    fn bufrelease(b: *buf);
+
+}
+
+fn render(w: &mut io::Writer, s: &str) {
+    // This code is all lifted from examples/sundown.c in the sundown repo
+    unsafe {
+        let ob = bufnew(OUTPUT_UNIT);
+        let options = html_renderopt {
+            toc_data: html_toc_data {
+                header_count: 0,
+                current_level: 0,
+                level_offset: 0,
+            },
+            flags: 0,
+            link_attributes: None,
+        };
+        let callbacks: sd_callbacks = [0, ..26];
+
+        sdhtml_renderer(&callbacks, &options, 0);
+        let markdown = sd_markdown_new(0, 16, &callbacks,
+                                       &options as *html_renderopt as *libc::c_void);
+
+        do s.as_imm_buf |data, len| {
+            sd_markdown_render(ob, data, len as libc::size_t, markdown);
+        }
+        sd_markdown_free(markdown);
+
+        do vec::raw::buf_as_slice((*ob).data, (*ob).size as uint) |buf| {
+            w.write(buf);
+        }
+
+        bufrelease(ob);
+    }
+}
+
 impl<'self> fmt::Default for Markdown<'self> {
     fn fmt(md: &Markdown<'self>, fmt: &mut fmt::Formatter) {
+        // This is actually common enough to special-case
         if md.len() == 0 { return; }
 
-        // Create the pandoc process
-        do io::io_error::cond.trap(|err| {
-            fail2!("Error executing `pandoc`: {}", err.desc);
-        }).inside {
-            let io = ~[CreatePipe(PipeStream::new().unwrap(), true, false),
-                       CreatePipe(PipeStream::new().unwrap(), false, true)];
-            let args = ProcessConfig {
-                program: "pandoc",
-                args: [],
-                env: None,
-                cwd: None,
-                io: io,
-            };
-            let mut p = Process::new(args).expect("couldn't fork for pandoc");
-
-            // Write the markdown to stdin and close it.
-            p.io[0].get_mut_ref().write(md.as_bytes());
-            p.io[0] = None;
-
-            // Ferry the output from pandoc over to the destination buffer.
-            let mut buf = [0, ..1024];
-            loop {
-                match p.io[1].get_mut_ref().read(buf) {
-                    None | Some(0) => { break }
-                    Some(n) => {
-                        fmt.buf.write(buf.slice_to(n));
-                    }
-                }
-            }
-        }
+        render(fmt.buf, md.as_slice());
     }
 }

src/librustdoc/html/render.rs

@@ -401,8 +401,16 @@ impl Context {
             let mut task = task::task();
             task.unlinked(); // we kill things manually
             task.name(format!("worker{}", i));
-            do task.spawn_with(cache.clone()) |cache| {
+            task.spawn_with(cache.clone(),
+                            |cache| worker(cache, &port, &chan, &prog_chan));
+
+            fn worker(cache: RWArc<Cache>,
+                      port: &SharedPort<Work>,
+                      chan: &SharedChan<Work>,
+                      prog_chan: &SharedChan<Progress>) {
+                #[fixed_stack_segment]; // we hit markdown FFI *a lot*
                 local_data::set(cache_key, cache);
+
                 loop {
                     match port.recv() {
                         Process(cx, item) => {
@@ -425,28 +433,20 @@ impl Context {
             }
         }
 
-        let watcher_chan = chan.clone();
-        let (done_port, done_chan) = comm::stream();
-        do task::spawn {
-            let mut jobs = 0;
-            loop {
-                match prog_port.recv() {
-                    JobNew => jobs += 1,
-                    JobDone => jobs -= 1,
-                }
-
-                if jobs == 0 { break }
+        chan.send(Process(self, item));
+        let mut jobs = 1;
+        loop {
+            match prog_port.recv() {
+                JobNew => jobs += 1,
+                JobDone => jobs -= 1,
             }
 
-            for _ in range(0, WORKERS) {
-                watcher_chan.send(Die);
-            }
-            done_chan.send(());
+            if jobs == 0 { break }
         }
 
-        prog_chan.send(JobNew);
-        chan.send(Process(self, item));
-        done_port.recv();
+        for _ in range(0, WORKERS) {
+            chan.send(Die);
+        }
     }
 
     fn item(&mut self, item: clean::Item, f: &fn(&mut Context, clean::Item)) {