chore(pipelines): write a GraphViz file with the pipeline structure (#24030)

This is a re-roll of #23908 which had to be reverted in  #24006 because there are some cases where change set approval steps that are shared between two stacks would cause cyclic dependencies between those stacks that caused them to be mutually unsortable.

Solve this by adding a mode to the toposort routine that will proceed even if there are cyclic dependencies (which is used purely for rendering).

----

Add a `pipeline.dot` file to the cloud assembly containing the graph structure of the pipeline.

This change is a `chore`, not a `feat`, as I want this to be a debugging aid, but I don't want to service feature requests on it.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts

@@ -1,10 +1,11 @@
+import * as fs from 'fs';
 import * as path from 'path';
 import * as cb from '@aws-cdk/aws-codebuild';
 import * as cp from '@aws-cdk/aws-codepipeline';
 import * as cpa from '@aws-cdk/aws-codepipeline-actions';
 import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
-import { Aws, CfnCapabilities, Duration, PhysicalName, Stack } from '@aws-cdk/core';
+import { Aws, CfnCapabilities, Duration, PhysicalName, Stack, Names } from '@aws-cdk/core';
 import * as cxapi from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
 import { AssetType, FileSet, IFileSetProducer, ManualApprovalStep, ShellStep, StackAsset, StackDeployment, Step } from '../blueprint';
@@ -423,6 +424,10 @@ export class CodePipeline extends PipelineBase {
     this._cloudAssemblyFileSet = graphFromBp.cloudAssemblyFileSet;
 
     this.pipelineStagesAndActionsFromGraph(graphFromBp);
+
+    // Write a dotfile for the pipeline layout
+    const dotFile = `${Names.uniqueId(this)}.dot`;
+    fs.writeFileSync(path.join(this.myCxAsmRoot, dotFile), graphFromBp.graph.renderDot().replace(/input\.dot/, dotFile), { encoding: 'utf-8' });
   }
 
   private get myCxAsmRoot(): string {

packages/@aws-cdk/pipelines/lib/helpers-internal/graph.ts

@@ -251,7 +251,7 @@ export class Graph<A> extends GraphNode<A> {
   /**
    * Return topologically sorted tranches of nodes at this graph level
    */
-  public sortedChildren(): GraphNode<A>[][] {
+  public sortedChildren(fail=true): GraphNode<A>[][] {
     // Project dependencies to current children
     const nodes = this.nodes;
     const projectedDependencies = projectDependencies(this.deepDependencies(), (node) => {
@@ -261,7 +261,7 @@ export class Graph<A> extends GraphNode<A> {
       return nodes.has(node) ? [node] : [];
     });
 
-    return topoSort(nodes, projectedDependencies);
+    return topoSort(nodes, projectedDependencies, fail);
   }
 
   /**
@@ -291,13 +291,21 @@ export class Graph<A> extends GraphNode<A> {
     return topoSort(new Set(projectedDependencies.keys()), projectedDependencies);
   }
 
-  public consoleLog(indent: number = 0) {
-    process.stdout.write(' '.repeat(indent) + this + depString(this) + '\n');
-    for (const node of this.nodes) {
-      if (node instanceof Graph) {
-        node.consoleLog(indent + 2);
-      } else {
-        process.stdout.write(' '.repeat(indent + 2) + node + depString(node) + '\n');
+  public render() {
+    const lines = new Array<string>();
+    recurse(this, '', true);
+    return lines.join('\n');
+
+    function recurse(x: GraphNode<A>, indent: string, last: boolean) {
+      const bullet = last ? '└─' : '├─';
+      const follow = last ? '  ' : '│ ';
+      lines.push(`${indent} ${bullet} ${x}${depString(x)}`);
+      if (x instanceof Graph) {
+        let i = 0;
+        const sortedNodes = Array.prototype.concat.call([], ...x.sortedChildren(false));
+        for (const child of sortedNodes) {
+          recurse(child, `${indent} ${follow} `, i++ == x.nodes.size - 1);
+        }
       }
     }
 
@@ -309,6 +317,79 @@ export class Graph<A> extends GraphNode<A> {
     }
   }
 
+  public renderDot() {
+    const lines = new Array<string>();
+
+    lines.push('digraph G {');
+    lines.push('  # Arrows represent an "unlocks" relationship (opposite of dependency). So chosen');
+    lines.push('  # because the layout looks more natural that way.');
+    lines.push('  # To represent subgraph dependencies, subgraphs are represented by BEGIN/END nodes.');
+    lines.push('  # To render: `dot -Tsvg input.dot > graph.svg`, open in a browser.');
+    lines.push('  node [shape="box"];');
+    for (const child of this.nodes) {
+      recurse(child);
+    }
+    lines.push('}');
+
+    return lines.join('\n');
+
+    function recurse(node: GraphNode<A>) {
+      let dependencySource;
+
+      if (node instanceof Graph) {
+        lines.push(`${graphBegin(node)} [shape="cds", style="filled", fillcolor="#b7deff"];`);
+        lines.push(`${graphEnd(node)} [shape="cds", style="filled", fillcolor="#b7deff"];`);
+        dependencySource = graphBegin(node);
+      } else {
+        dependencySource = nodeLabel(node);
+        lines.push(`${nodeLabel(node)};`);
+      }
+
+      for (const dep of node.dependencies) {
+        const dst = dep instanceof Graph ? graphEnd(dep) : nodeLabel(dep);
+        lines.push(`${dst} -> ${dependencySource};`);
+      }
+
+      if (node instanceof Graph && node.nodes.size > 0) {
+        for (const child of node.nodes) {
+          recurse(child);
+        }
+
+        // Add dependency arrows between the "subgraph begin" and the first rank of
+        // the children, and the last rank of the children and "subgraph end" nodes.
+        const sortedChildren = node.sortedChildren(false);
+        for (const first of sortedChildren[0]) {
+          const src = first instanceof Graph ? graphBegin(first) : nodeLabel(first);
+          lines.push(`${graphBegin(node)} -> ${src};`);
+        }
+        for (const last of sortedChildren[sortedChildren.length - 1]) {
+          const dst = last instanceof Graph ? graphEnd(last) : nodeLabel(last);
+          lines.push(`${dst} -> ${graphEnd(node)};`);
+        }
+      }
+    }
+
+    function id(node: GraphNode<A>) {
+      return node.rootPath().slice(1).map(n => n.id).join('.');
+    }
+
+    function nodeLabel(node: GraphNode<A>) {
+      return `"${id(node)}"`;
+    }
+
+    function graphBegin(node: Graph<A>) {
+      return `"BEGIN ${id(node)}"`;
+    }
+
+    function graphEnd(node: Graph<A>) {
+      return `"END ${id(node)}"`;
+    }
+  }
+
+  public consoleLog(_indent: number = 0) {
+    process.stdout.write(this.render() + '\n');
+  }
+
   /**
    * Return the union of all dependencies of the descendants of this graph
    */

packages/@aws-cdk/pipelines/lib/helpers-internal/toposort.ts

@@ -9,7 +9,7 @@ export function printDependencyMap<A>(dependencies: Map<GraphNode<A>, Set<GraphN
   console.log(lines.join('\n'));
 }
 
-export function topoSort<A>(nodes: Set<GraphNode<A>>, dependencies: Map<GraphNode<A>, Set<GraphNode<A>>>): GraphNode<A>[][] {
+export function topoSort<A>(nodes: Set<GraphNode<A>>, dependencies: Map<GraphNode<A>, Set<GraphNode<A>>>, fail=true): GraphNode<A>[][] {
   const remaining = new Set<GraphNode<A>>(nodes);
 
   const ret: GraphNode<A>[][] = [];
@@ -26,7 +26,14 @@ export function topoSort<A>(nodes: Set<GraphNode<A>>, dependencies: Map<GraphNod
     // If we didn't make any progress, we got stuck
     if (selectable.length === 0) {
       const cycle = findCycle(dependencies);
-      throw new Error(`Dependency cycle in graph: ${cycle.map(n => n.id).join(' => ')}`);
+
+      if (fail) {
+        throw new Error(`Dependency cycle in graph: ${cycle.map(n => n.id).join(' => ')}`);
+      }
+
+      // If we're trying not to fail, pick one at random from the cycle and treat it
+      // as selectable, then continue.
+      selectable.push(cycle[0]);
     }
 
     ret.push(selectable);

packages/@aws-cdk/pipelines/test/codepipeline/codepipeline.test.ts

@@ -392,6 +392,40 @@ test('action name is calculated properly if it has cross-stack dependencies', ()
   });
 });
 
+test('synths with change set approvers', () => {
+  // GIVEN
+  const pipelineStack = new cdk.Stack(app, 'PipelineStack', { env: PIPELINE_ENV });
+  const pipeline = new ModernTestGitHubNpmPipeline(pipelineStack, 'Cdk');
+
+  // WHEN
+  const csApproval = new cdkp.ManualApprovalStep('ChangeSetApproval');
+
+  // The issue we were diagnosing only manifests if the stacks don't have
+  // a dependency on each other
+  const stage = new TwoStackApp(app, 'TheApp', { withDependency: false });
+  pipeline.addStage(stage, {
+    stackSteps: [
+      { stack: stage.stack1, changeSet: [csApproval] },
+      { stack: stage.stack2, changeSet: [csApproval] },
+    ],
+  });
+
+  // THEN
+  const template = Template.fromStack(pipelineStack);
+  template.hasResourceProperties('AWS::CodePipeline::Pipeline', {
+    Stages: Match.arrayWith([{
+      Name: 'TheApp',
+      Actions: Match.arrayWith([
+        Match.objectLike({ Name: 'Stack1.Prepare', RunOrder: 1 }),
+        Match.objectLike({ Name: 'Stack2.Prepare', RunOrder: 1 }),
+        Match.objectLike({ Name: 'Stack1.ChangeSetApproval', RunOrder: 2 }),
+        Match.objectLike({ Name: 'Stack1.Deploy', RunOrder: 3 }),
+        Match.objectLike({ Name: 'Stack2.Deploy', RunOrder: 3 }),
+      ]),
+    }]),
+  });
+});
+
 interface ReuseCodePipelineStackProps extends cdk.StackProps {
   reuseCrossRegionSupportStacks?: boolean;
 }