perf(benchmarks): add a fs-text benchmark (#27004)

Author: pvdz

benchmarks/gabe-fs-text/.gitignore

@@ -0,0 +1,72 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# dotenv environment variable files
+.env*
+
+# gatsby files
+.cache/
+public
+
+# Mac files
+.DS_Store
+
+# Yarn
+yarn-error.log
+.pnp/
+.pnp.js
+# Yarn Integrity file
+.yarn-integrity
+
+generated_articles
+yarn.lock

benchmarks/gabe-fs-text/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

benchmarks/gabe-fs-text/README.md

@@ -0,0 +1,28 @@
+# Markdown Benchmark; fs+text version
+
+This is a baseline benchmark for tracking plaintext performance with individual files in the Gabe project.
+
+This will produce the same site as `gabe-fs-markdown` without using any markdown.
+
+The site can generate an arbitrary amount of super simple pages. Each page has a small header, a quote, and two small paragraphs of random text. No images, because that's a fixed cost we're not interested in.
+
+## Install
+
+Run `yarn` or `npm install`
+
+## Usage
+
+You can start a benchmark run like this:
+
+```shell
+N=1000 M=2 yarn bench
+```
+
+- `N=1000`: instructs the run to build a site of 1000 pages
+- `M=2`: instructs nodejs to use up to 2gb of memory for its long term storage
+- Deletes generates files from previous run
+- Generates `N` pages with pseudo-random content
+- Runs `gatsby clean`
+- Runs `gatsby build`
+
+The default `yarn bench` will build 512 pages with 1gb memory.

benchmarks/gabe-fs-text/gatsby-browser.js

@@ -0,0 +1,16 @@
+module.exports = {
+  siteMetadata: {
+    title: `Gatsby FS Text Benchmark for Gabe`,
+    description: "A blog like no other blog",
+    author: "Bob the Blogger",
+  },
+  plugins: [
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        name: `blog`,
+        path: `${__dirname}/generated_articles`,
+      },
+    },
+  ],
+}

benchmarks/gabe-fs-text/gatsby-config.js

@@ -0,0 +1,83 @@
+const fs = require("fs")
+const path = require(`path`)
+const { createFilePath } = require(`gatsby-source-filesystem`)
+
+exports.createPages = async ({ graphql, actions }) => {
+  const { createPage } = actions
+  const blogPost = path.resolve(`./src/templates/blog-post.js`)
+  const result = await graphql(
+    `
+      {
+        allTexto(sort: { fields: date, order: ASC }) {
+          edges {
+            node {
+              id
+              slug
+            }
+          }
+        }
+      }
+    `
+  )
+
+  if (result.errors) {
+    throw result.errors
+  }
+
+  // Create blog posts pages.
+  const posts = result.data.allTexto.edges
+
+  posts.forEach(({ node }, index) => {
+    const previous = index === posts.length - 1 ? null : posts[index + 1].node
+    const next = index === 0 ? null : posts[index - 1].node
+
+    createPage({
+      path: node.slug,
+      component: blogPost,
+      context: {
+        id: node.id,
+        slug: node.slug,
+        previous,
+        next,
+      },
+    })
+  })
+}
+
+exports.onCreateNode = ({ node, actions }) => {
+  if (node.internal.type === "File") {
+    // Do minimal processing to get some key pieces. This could be gatsby-transformer-text or -html :p
+
+    const html = fs.readFileSync(node.absolutePath, "utf8")
+
+    const base = path.basename(node.absolutePath)
+    const slug = base.slice(11, -5) // remove date prefix and `..txt` tail
+    const date = base.slice(0, 10)
+
+    const offset1 = html.indexOf("<h1>")
+    const title = html.slice(
+      offset1 + "<h1>".length,
+      html.indexOf("</h1>", offset1)
+    )
+
+    const offset2 = html.indexOf("<blockquote>", offset1)
+    const desc = html.slice(
+      offset2 + "<blockquote>".length,
+      html.indexOf("</blockquote>", offset2)
+    )
+
+    actions.createNode({
+      id: slug,
+      slug,
+      date,
+      title,
+      desc,
+      the_text: html,
+      internal: {
+        type: "Texto",
+        contentDigest: html,
+      },
+      parent: node.id, // Required otherwise the node is not cached and a warm build screws up
+    })
+  }
+}

benchmarks/gabe-fs-text/gatsby-node.js

@@ -0,0 +1,42 @@
+
+const fs = require("fs")
+const path = require("path")
+const faker = require(`faker`)
+
+const N = parseInt(process.env.N, 10) || 100
+
+let n = 0
+
+function createArticle(n, sentence, slug) {
+  const desc = faker.lorem.sentence();
+
+  return `
+<h1>${sentence.replace(/"/g, '\\"')}</h1>
+<blockquote>${desc}</blockquote>
+<p>${faker.lorem.paragraphs(1)}</p>
+<p>${faker.lorem.paragraphs(1)}</p>
+`
+}
+
+console.log("Start of gen")
+
+if (fs.existsSync("./generated_articles")) {
+  TODO // count existing folders. If they are less than given number, just amend to them. Otherwise abort and require a rimraf
+} else {
+  fs.mkdirSync("./generated_articles", { recursive: true })
+}
+
+console.log("Now generating " + N + " articles")
+for (let i = 0; i < N; ++i) {
+  const sentence = faker.lorem.sentence()
+  const slug = faker.helpers.slugify(sentence).toLowerCase()
+
+  const date = faker.date.recent(1000).toISOString().slice(0, 10)
+
+  fs.writeFileSync(
+    path.join("./generated_articles", date + '_' + slug + ".txt"),
+    createArticle(i, sentence, slug)
+  )
+}
+console.log("Finished generating " + N + " articles")
+console.log("End of gen")

benchmarks/gabe-fs-text/gen.js

@@ -0,0 +1,36 @@
+{
+  "name": "gabe-fs-text",
+  "private": true,
+  "description": "Benchmark site for testing baseline plaintext perf with individually generated files",
+  "author": "Peter van der Zee <pvdz@github>",
+  "version": "0.1.0",
+  "license": "MIT",
+  "scripts": {
+    "bench": "rm -rf generated_articles; gatsby clean; N=${N:-512} node gen.js; CI=1 node --max_old_space_size=${M:-2}000 node_modules/.bin/gatsby build",
+    "build": "gatsby build",
+    "clean": "gatsby clean",
+    "develop": "gatsby develop",
+    "format": "prettier --write \"**/*.{js,jsx,json,md}\""
+  },
+  "devDependencies": {
+    "prettier": "2.0.4"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gatsbyjs/gatsby/tree/master/benchmarks/"
+  },
+  "bugs": {
+    "url": "https://github.com/gatsbyjs/gatsby/issues"
+  },
+  "keywords": [
+    "gatsby",
+    "benchmark"
+  ],
+  "dependencies": {
+    "faker": "^4.1.0",
+    "gatsby": "^2",
+    "gatsby-source-filesystem": "^2",
+    "react": "^16.12.0",
+    "react-dom": "^16.12.0"
+  }
+}

benchmarks/gabe-fs-text/package.json

@@ -0,0 +1,30 @@
+/**
+ * Bio component that queries for data
+ * with Gatsby's useStaticQuery component
+ *
+ * See: https://www.gatsbyjs.org/docs/use-static-query/
+ */
+
+import React from "react"
+
+const Bio = () => {
+  return (
+    <div
+      style={{
+        display: `flex`,
+        marginBottom: '5px',
+      }}
+    >
+      <p>
+        Written by <strong>Bob</strong> who lives and works in Fan
+        Srancisco building useful things.
+        {` `}
+        <a href={`https://twitter.com/bob`}>
+          You should follow him on Twitter
+        </a>
+      </p>
+    </div>
+  )
+}
+
+export default Bio