Skip to content

Use rayon to parallelize the blog post rendering #1159

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 92 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ name = "blog"
path = "src/blog.rs"

[dependencies]
eyre = "0.6.8"
handlebars = { version = "3", features = ["dir_source"] }
lazy_static = "1.4.0"
serde = "1.0"
serde_derive = "1.0"
serde_yaml = "0.8"
serde_json = "1.0"
comrak = "0.13"
rayon = "1.8.0"
regex = "1.3"
sass-rs = "0.2"
chrono = "0.4"
Expand Down
4 changes: 1 addition & 3 deletions src/blog.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
use std::error::Error;

#[path = "lib.rs"]
mod lib;

pub fn main() -> Result<(), Box<dyn Error>> {
pub fn main() -> eyre::Result<()> {
lib::main()?;

println!("blog has been generated; you can now serve its content by running\n\
Expand Down
7 changes: 3 additions & 4 deletions src/blogs.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use super::posts::Post;
use serde_derive::{Deserialize, Serialize};
use std::error::Error;
use std::path::{Path, PathBuf};

static MANIFEST_FILE: &str = "blog.yml";
Expand Down Expand Up @@ -46,7 +45,7 @@ pub(crate) struct Blog {
}

impl Blog {
fn load(prefix: PathBuf, dir: &Path) -> Result<Self, Box<dyn Error>> {
fn load(prefix: PathBuf, dir: &Path) -> eyre::Result<Self> {
let manifest_content = std::fs::read_to_string(dir.join(MANIFEST_FILE))?;
let manifest: Manifest = serde_yaml::from_str(&manifest_content)?;

Expand Down Expand Up @@ -122,7 +121,7 @@ impl Blog {

/// Recursively load blogs in a directory. A blog is a directory with a `blog.yml`
/// file inside it.
pub(crate) fn load(base: &Path) -> Result<Vec<Blog>, Box<dyn Error>> {
pub(crate) fn load(base: &Path) -> eyre::Result<Vec<Blog>> {
let mut blogs = Vec::new();
load_recursive(base, base, &mut blogs)?;
Ok(blogs)
Expand All @@ -132,7 +131,7 @@ fn load_recursive(
base: &Path,
current: &Path,
blogs: &mut Vec<Blog>,
) -> Result<(), Box<dyn Error>> {
) -> eyre::Result<()> {
for entry in std::fs::read_dir(current)? {
let path = entry?.path();
let file_type = path.metadata()?.file_type();
Expand Down
30 changes: 14 additions & 16 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ use sass_rs::{compile_file, Options};
use serde_derive::Serialize;
use serde_json::json;
use std::convert::AsRef;
use std::error::Error;
use std::fs::{self, File};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use rayon::prelude::*;

struct Generator<'a> {
handlebars: Handlebars<'a>,
Expand Down Expand Up @@ -51,7 +51,7 @@ impl<'a> Generator<'a> {
fn new(
out_directory: impl AsRef<Path>,
posts_directory: impl AsRef<Path>,
) -> Result<Self, Box<dyn Error>> {
) -> eyre::Result<Self> {
let mut handlebars = Handlebars::new();
handlebars.set_strict_mode(true);
handlebars.register_templates_directory(".hbs", "templates")?;
Expand Down Expand Up @@ -80,7 +80,7 @@ impl<'a> Generator<'a> {
.replace(std::path::MAIN_SEPARATOR, "/")
}

fn render(&self) -> Result<(), Box<dyn Error>> {
fn render(&self) -> eyre::Result<()> {
// make sure our output directory exists
fs::create_dir_all(&self.out_directory)?;

Expand Down Expand Up @@ -116,7 +116,7 @@ impl<'a> Generator<'a> {
fs::write("./static/styles/vendor.css", &concatted).expect("couldn't write vendor css");
}

fn render_blog(&self, blog: &Blog) -> Result<(), Box<dyn Error>> {
fn render_blog(&self, blog: &Blog) -> eyre::Result<()> {
std::fs::create_dir_all(self.out_directory.join(blog.prefix()))?;

let path = self.render_index(blog)?;
Expand All @@ -126,17 +126,15 @@ impl<'a> Generator<'a> {
self.render_feed(blog)?;
self.render_releases_feed(blog)?;

for (i, post) in blog.posts().iter().enumerate() {
let path = self.render_post(blog, post)?;
if i == 0 {
println!("└─ Latest post: {}\n", self.file_url(&path));
}
let paths = blog.posts().par_iter().map(|post| self.render_post(blog, post)).collect::<Result<Vec<_>, _>>()?;
if let Some(path) = paths.first() {
println!("└─ Latest post: {}\n", self.file_url(path));
}

Ok(())
}

fn render_index(&self, blog: &Blog) -> Result<PathBuf, Box<dyn Error>> {
fn render_index(&self, blog: &Blog) -> eyre::Result<PathBuf> {
let other_blogs: Vec<_> = self
.blogs
.iter()
Expand All @@ -161,7 +159,7 @@ impl<'a> Generator<'a> {
Ok(path)
}

fn render_post(&self, blog: &Blog, post: &Post) -> Result<PathBuf, Box<dyn Error>> {
fn render_post(&self, blog: &Blog, post: &Post) -> eyre::Result<PathBuf> {
let path = blog
.prefix()
.join(format!("{:04}", &post.year))
Expand All @@ -186,7 +184,7 @@ impl<'a> Generator<'a> {
Ok(path)
}

fn render_feed(&self, blog: &Blog) -> Result<(), Box<dyn Error>> {
fn render_feed(&self, blog: &Blog) -> eyre::Result<()> {
let posts: Vec<_> = blog.posts().iter().take(10).collect();
let data = json!({
"blog": blog,
Expand All @@ -198,7 +196,7 @@ impl<'a> Generator<'a> {
Ok(())
}

fn render_releases_feed(&self, blog: &Blog) -> Result<(), Box<dyn Error>> {
fn render_releases_feed(&self, blog: &Blog) -> eyre::Result<()> {
let posts = blog.posts().iter().cloned().collect::<Vec<_>>();
let is_released: Vec<&Post> = posts.iter().filter(|post| post.release).collect();
let releases: Vec<ReleasePost> = is_released
Expand All @@ -223,7 +221,7 @@ impl<'a> Generator<'a> {
Ok(())
}

fn copy_static_files(&self) -> Result<(), Box<dyn Error>> {
fn copy_static_files(&self) -> eyre::Result<()> {
copy_dir("static/fonts", &self.out_directory)?;
copy_dir("static/images", &self.out_directory)?;
copy_dir("static/styles", &self.out_directory)?;
Expand All @@ -236,7 +234,7 @@ impl<'a> Generator<'a> {
name: impl AsRef<Path>,
template: &str,
data: serde_json::Value,
) -> Result<(), Box<dyn Error>> {
) -> eyre::Result<()> {
let out_file = self.out_directory.join(name.as_ref());
let file = File::create(out_file)?;
self.handlebars.render_to_write(template, &data, file)?;
Expand Down Expand Up @@ -264,7 +262,7 @@ fn copy_dir(source: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<(), io::
copy_inner(source, &dest)
}

pub fn main() -> Result<(), Box<dyn Error>> {
pub fn main() -> eyre::Result<()> {
let blog = Generator::new("site", "posts")?;

blog.render()?;
Expand Down
6 changes: 3 additions & 3 deletions src/posts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ use super::blogs::Manifest;
use comrak::{ComrakExtensionOptions, ComrakOptions, ComrakRenderOptions};
use regex::Regex;
use serde_derive::{Deserialize, Serialize};
use std::error::Error;
use std::path::{Path, PathBuf};
use eyre::eyre;

#[derive(Debug, PartialEq, Deserialize)]
struct YamlHeader {
Expand Down Expand Up @@ -36,7 +36,7 @@ pub(crate) struct Post {
}

impl Post {
pub(crate) fn open(path: &Path, manifest: &Manifest) -> Result<Self, Box<dyn Error>> {
pub(crate) fn open(path: &Path, manifest: &Manifest) -> eyre::Result<Self> {
// yeah this might blow up, but it won't
let filename = path.file_name().unwrap().to_str().unwrap();

Expand All @@ -52,7 +52,7 @@ impl Post {
let contents = std::fs::read_to_string(path)?;
if contents.len() < 5 {
return Err(
format!("{path:?} is empty, or too short to have valid front matter").into(),
eyre!("{path:?} is empty, or too short to have valid front matter")
);
}

Expand Down