chore(cfnspec): consume CloudFormation specification in parts (#18210)

Whenever there are errors in the CloudFormation specification, we
currently have to fail the build and can't consume anything.

To ensure we make some progress, apply the following strategy instead:

- Split the spec into fragments, on a per-service basis.
- Consume those per-service spec updates that are valid; if updates
  are invalid, we will leave them at the old version.

This will produce an always-building spec, of which certain parts
may be outdated. Report the outdated parts in the CHANGELOG.

Notifying the CloudFormation team about spec errors is an out-of-band
process, and out of scope of this PR.

As a side effect of this work, formalize the spec manipulation we do
with JSON and patch files into a mini-standard called "JSON Patch
Stacks", and add some tools to operate on them.


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/@aws-cdk/cfnspec/build-tools/build.ts

@@ -6,18 +6,18 @@
  */
 
 import * as path from 'path';
-import * as fs from 'fs-extra';
 import * as md5 from 'md5';
 import { schema } from '../lib';
-import { decorateResourceTypes, forEachSection, massageSpec, merge, normalize, patch } from './massage-spec';
+import { massageSpec, normalize } from './massage-spec';;
+import { writeSorted, applyPatchSet, applyAndWrite } from './patch-set';
 
 async function main() {
   const inputDir = path.join(process.cwd(), 'spec-source');
   const outDir = path.join(process.cwd(), 'spec');
 
   await generateResourceSpecification(inputDir, path.join(outDir, 'specification.json'));
-  await mergeSpecificationFromDirs(path.join(inputDir, 'cfn-lint'), path.join(outDir, 'cfn-lint.json'));
-  await fs.copyFile(path.join(inputDir, 'cfn-docs', 'cfn-docs.json'), path.join(outDir, 'cfn-docs.json'));
+  await applyAndWrite(path.join(outDir, 'cfn-lint.json'), path.join(inputDir, 'cfn-lint'));
+  await applyAndWrite(path.join(outDir, 'cfn-docs.json'), path.join(inputDir, 'cfn-docs'));
 }
 
 /**
@@ -26,64 +26,11 @@ async function main() {
 async function generateResourceSpecification(inputDir: string, outFile: string) {
   const spec: schema.Specification = { PropertyTypes: {}, ResourceTypes: {}, Fingerprint: '' };
 
-  const files = await fs.readdir(inputDir);
-  for (const file of files.filter(n => n.endsWith('.json')).sort()) {
-    const data = await fs.readJson(path.join(inputDir, file));
-    if (file.indexOf('patch') === -1) {
-      decorateResourceTypes(data);
-      forEachSection(spec, data, merge);
-    } else {
-      forEachSection(spec, data, patch);
-    }
-  }
-
+  Object.assign(spec, await applyPatchSet(path.join(inputDir, 'specification')));
   massageSpec(spec);
-
   spec.Fingerprint = md5(JSON.stringify(normalize(spec)));
 
-  await fs.mkdirp(path.dirname(outFile));
-  await fs.writeJson(outFile, spec, { spaces: 2 });
-}
-
-/**
- * Generate Cfnlint spec annotations from sources and patches
- */
-async function mergeSpecificationFromDirs(inputDir: string, outFile: string) {
-  const spec: any = {};
-
-  for (const child of await fs.readdir(inputDir)) {
-    const fullPath = path.join(inputDir, child);
-    if (!(await fs.stat(fullPath)).isDirectory()) { continue; }
-
-    const subspec = await loadMergedSpec(fullPath);
-    spec[child] = subspec;
-  }
-
-  await fs.mkdirp(path.dirname(outFile));
-  await fs.writeJson(outFile, spec, { spaces: 2 });
-}
-
-/**
- * Load all files in the given directory, merge them and apply patches in the order found
- *
- * The base structure is always an empty object
- */
-async function loadMergedSpec(inputDir: string) {
-  const structure: any = {};
-
-  const files = await fs.readdir(inputDir);
-  for (const file of files.filter(n => n.endsWith('.json')).sort()) {
-    const data = await fs.readJson(path.join(inputDir, file));
-    if (file.indexOf('patch') === -1) {
-      // Copy properties from current object into structure, adding/overwriting whatever is found
-      Object.assign(structure, data);
-    } else {
-      // Apply the loaded file as a patch onto the current structure
-      patch(structure, data);
-    }
-  }
-
-  return structure;
+  await writeSorted(outFile, spec);
 }
 
 main()

packages/@aws-cdk/cfnspec/build-tools/massage-spec.ts

@@ -1,4 +1,3 @@
-import * as fastJsonPatch from 'fast-json-patch';
 import { schema } from '../lib';
 import { detectScrutinyTypes } from './scrutiny';
 
@@ -8,22 +7,6 @@ export function massageSpec(spec: schema.Specification) {
   dropTypelessAttributes(spec);
 }
 
-export function forEachSection(spec: schema.Specification, data: any, cb: (spec: any, fragment: any, path: string[]) => void) {
-  cb(spec.PropertyTypes, data.PropertyTypes, ['PropertyTypes']);
-  cb(spec.ResourceTypes, data.ResourceTypes, ['ResourceTypes']);
-  // Per-resource specs are keyed on ResourceType (singular), but we want it in ResourceTypes (plural)
-  cb(spec.ResourceTypes, data.ResourceType, ['ResourceType']);
-}
-
-export function decorateResourceTypes(data: any) {
-  const requiredTransform = data.ResourceSpecificationTransform as string | undefined;
-  if (!requiredTransform) { return; }
-  const resourceTypes = data.ResourceTypes || data.ResourceType;
-  for (const name of Object.keys(resourceTypes)) {
-    resourceTypes[name].RequiredTransform = requiredTransform;
-  }
-}
-
 /**
  * Fix incomplete type definitions in PropertyTypes
  *
@@ -63,40 +46,6 @@ function dropTypelessAttributes(spec: schema.Specification) {
   });
 }
 
-export function merge(spec: any, fragment: any, jsonPath: string[]) {
-  if (!fragment) { return; }
-  for (const key of Object.keys(fragment)) {
-    if (key in spec) {
-      const specVal = spec[key];
-      const fragVal = fragment[key];
-      if (typeof specVal !== typeof fragVal) {
-        // eslint-disable-next-line max-len
-        throw new Error(`Attempted to merge ${JSON.stringify(fragVal)} into incompatible ${JSON.stringify(specVal)} at path ${jsonPath.join('/')}/${key}`);
-      }
-      if (typeof specVal !== 'object') {
-        // eslint-disable-next-line max-len
-        throw new Error(`Conflict when attempting to merge ${JSON.stringify(fragVal)} into ${JSON.stringify(specVal)} at path ${jsonPath.join('/')}/${key}`);
-      }
-      merge(specVal, fragVal, [...jsonPath, key]);
-    } else {
-      spec[key] = fragment[key];
-    }
-  }
-}
-
-export function patch(spec: any, fragment: any) {
-  if (!fragment) { return; }
-  if ('patch' in fragment) {
-    // eslint-disable-next-line no-console
-    console.log(`Applying patch: ${fragment.patch.description}`);
-    fastJsonPatch.applyPatch(spec, fragment.patch.operations);
-  } else {
-    for (const key of Object.keys(fragment)) {
-      patch(spec[key], fragment[key]);
-    }
-  }
-}
-
 /**
  * Modifies the provided specification so that ``ResourceTypes`` and ``PropertyTypes`` are listed in alphabetical order.
  *