chore - Allow a fixed amount of link checks to fail

Now the output is tidier as well.

Author: bebraw

components/organization/projects.json

@@ -256,7 +256,7 @@
 		"repo": "webpack/i18n-webpack-plugin",
 		"npm": "i18n-webpack-plugin",
 		"description": "Embed localization into your bundle.",
-		"maintainer": "EcutDavid"
+		"maintainer": ""
 	},
 	{
 		"repo": "webpack/json5-loader",

package.json

@@ -31,7 +31,7 @@
     "fetch": "scripts/fetch.sh",
     "init:generated": "mkdirp ./generated/loaders && mkdirp ./generated/plugins ",
     "lint": "run-s lint:*",
-    "lint:links": "hyperlink -r --exclude https://opencollective.com/webpack/*/*/website build/index.html",
+    "lint:links": "hyperlink -r --exclude https://opencollective.com/webpack/*/*/website build/index.html | ./scripts/check-links.js --skip 10",
     "lint:js": "eslint . --ext .js --ext .jsx",
     "lint:md": "eslint . --ext .md",
     "lint:markdown": "markdownlint --config ./.markdownlintrc **/*.md *.md ./content/**/*.md",
@@ -56,6 +56,7 @@
     "babel-preset-react": "^6.11.1",
     "copy-webpack-plugin": "^4.0.1",
     "css-loader": "^0.25.0",
+    "duplexer": "^0.1.1",
     "eslint": "3.6.0",
     "eslint-loader": "^1.5.0",
     "eslint-plugin-markdown": "^1.0.0-beta.2",
@@ -88,6 +89,8 @@
     "sitemap-static": "^0.3.1",
     "style-loader": "^0.13.1",
     "tap-min": "^1.1.0",
+    "tap-parser": "^5.3.3",
+    "through2": "^2.0.3",
     "url-loader": "^0.5.7",
     "webpack": "^1.13.2",
     "webpack-dev-server": "^1.16.1",

scripts/check-links.js

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+// Check piped links while allowing a fixed amount to fail
+// Adapted from tap-json.
+var parser = require('tap-parser');
+var through = require('through2');
+var duplexer = require('duplexer');
+var minimist = require('minimist');
+
+process.stdin
+  .pipe(checkLinks())
+  .pipe(process.stdout);
+
+function checkLinks(args) {
+  var argv = minimist(process.argv.slice(2));
+  var skip = argv.skip || 0;
+
+  var tap = parser();
+  var out = through.obj();
+  var dup = duplexer(tap, out);
+
+  var data = [];
+  var name = null;
+
+  tap.on('complete', function(res) {
+    const failures = res.failures;
+
+    if (failures.length > 0) {
+      console.log(formatFailures(failures));
+    }
+
+    // Fail hard only if over skip threshold
+    if (failures.length > skip) {
+      process.exit(1);
+    }
+  });
+
+  return dup;
+}
+
+function formatFailures(failures) {
+  return failures.map(function(failure) {
+    return failure.name + '\n' + failure.diag.actual + ' at ' + failure.diag.at;
+  }).join('\n\n');
+}

scripts/fetch.sh

@@ -7,11 +7,13 @@ cp -rf ./content/loaders/ ./generated/loaders
 mkdir -p ./generated/plugins
 cp -rf ./content/plugins/ ./generated/plugins
 
-# Fetches github.com/webpack/*-loader repositories
-./scripts/fetch_package_names.js "webpack" "-loader" | ./scripts/fetch_package_files.js "README.md" "./generated/loaders"
+# Fetches github.com/webpack/*-loader repositories.
+# Skipped because they are below contrib now.
+# ./scripts/fetch_package_names.js "webpack" "-loader" | ./scripts/fetch_package_files.js "README.md" "./generated/loaders"
 
-# Fetches github.com/webpack/*-webpack-plugin repositories
-./scripts/fetch_package_names.js "webpack" "-webpack-plugin" | ./scripts/fetch_package_files.js "README.md" "./generated/plugins"
+# Fetches github.com/webpack/*-webpack-plugin repositories.
+# Skipped because they are below contrib now.
+# ./scripts/fetch_package_names.js "webpack" "-webpack-plugin" | ./scripts/fetch_package_files.js "README.md" "./generated/plugins"
 
 # Fetches github.com/webpack-contrib/*-loader repositories
 ./scripts/fetch_package_names.js "webpack-contrib" "-loader" | ./scripts/fetch_package_files.js "README.md" "./generated/loaders"

yarn.lock

@@ -2024,7 +2024,7 @@ duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
 
-duplexer@~0.1.1:
+duplexer@^0.1.1, duplexer@~0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.1.tgz#ace6ff808c1ce66b57d1ebf97977acb02334cfc1"
 
@@ -5304,7 +5304,7 @@ [email protected], readable-stream@^1.0.33:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-stream@^2, readable-stream@^2.0.0, "readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.1.0, readable-stream@^2.2.2:
+readable-stream@^2, readable-stream@^2.0.0, "readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.1.0, readable-stream@^2.1.5, readable-stream@^2.2.2:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.3.tgz#9cf49463985df016c8ae8813097a9293a9b33729"
   dependencies:
@@ -6130,7 +6130,7 @@ tap-min@^1.1.0:
     readable-stream "^2.2.2"
     tap-parser "^5.3.2"
 
-tap-parser@^5.3.2:
+tap-parser@^5.3.2, tap-parser@^5.3.3:
   version "5.3.3"
   resolved "https://registry.yarnpkg.com/tap-parser/-/tap-parser-5.3.3.tgz#53ec8a90f275d6fff43f169e56a679502a741185"
   dependencies:
@@ -6139,9 +6139,9 @@ tap-parser@^5.3.2:
   optionalDependencies:
     readable-stream "^2"
 
-"tap-render@github:munter/tap-render#0.1.7-patch1":
+tap-render@Munter/tap-render#0.1.7-patch1:
   version "0.1.7"
-  resolved "https://codeload.github.com/munter/tap-render/tar.gz/35bf3ac21c4fd2776d8569d5e8a1ab62df1f6d4f"
+  resolved "https://codeload.github.com/Munter/tap-render/tar.gz/35bf3ac21c4fd2776d8569d5e8a1ab62df1f6d4f"
   dependencies:
     jsonify "0.0.0"
     pause-stream "0.0.7"
@@ -6205,6 +6205,13 @@ text-table@^0.2.0, text-table@~0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
 
+through2@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.3.tgz#0004569b37c7c74ba39c43f3ced78d1ad94140be"
+  dependencies:
+    readable-stream "^2.1.5"
+    xtend "~4.0.1"
+
 through@2, through@^2.3.6, through@~2.3, through@~2.3.1:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@@ -6873,7 +6880,7 @@ xmlhttprequest@1:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc"
 
-[email protected], "xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.0:
+[email protected], "xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.0, xtend@~4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"