fix(cli): asset existence check is slow for many assets (#25866)

When an application has many assets (think ~100) the existence check which seeks to scrub already existing assets from the work graph is starting to take a significant amount of time. Do those checks in parallel.

Also in this PR:

- Improve the printing of the work graph a little to make it slightly less unreadable if the graph gets large.
- Print the graphviz representation of the work graph to the trace log if the graph gets stuck, for debugging purposes.
- In the work graph builder, only add dependencies on stacks. Previously, it used to add dependencies on assets as if they were stacks (those were later on removed, but we didn't need to add them in the first place).

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/aws-cdk/lib/util/parallel.ts

@@ -0,0 +1,44 @@
+/**
+ * Run a number of promise generators with max parallelism
+ *
+ * Order is not maintained between the input and output.
+ */
+export async function parallelPromises<A>(n: number, promises: Array<() => Promise<A>>): Promise<Array<A>> {
+  const ret = new Array<A>();
+  let count = 0;
+  let error: Error | undefined;
+  const queue = [...promises];
+
+  return new Promise((ok, ko) => {
+    tick();
+
+    function tick() {
+      if (count === 0 && error) {
+        ko(error);
+        return;
+      }
+      if (count === 0 && queue.length === 0) {
+        ok(ret);
+        return;
+      }
+
+      while (count < n && queue.length > 0 && !error) {
+        const next = queue.shift();
+        if (next !== undefined) {
+          start(next);
+        }
+      }
+    }
+
+    function start(fn: () => Promise<A>) {
+      count += 1;
+      fn()
+        .then((result) => { ret.push(result); })
+        .catch((e) => { error = e; })
+        .finally(() => {
+          count -= 1;
+          tick();
+        });
+    }
+  });
+}

packages/aws-cdk/lib/util/work-graph-builder.ts

@@ -27,7 +27,7 @@ export class WorkGraphBuilder {
     this.graph.addNodes({
       type: 'stack',
       id: `${this.idPrefix}${artifact.id}`,
-      dependencies: new Set(this.getDepIds(artifact.dependencies)),
+      dependencies: new Set(this.getDepIds(onlyStacks(artifact.dependencies))),
       stack: artifact,
       deploymentState: DeploymentState.PENDING,
       priority: WorkGraphBuilder.PRIORITIES.stack,

packages/aws-cdk/lib/util/work-graph.ts

@@ -1,4 +1,6 @@
+import { parallelPromises } from './parallel';
 import { WorkNode, DeploymentState, StackNode, AssetBuildNode, AssetPublishNode } from './work-graph-types';
+import { debug, trace } from '../logging';
 
 export type Concurrency = number | Record<WorkNode['type'], number>;
 
@@ -215,15 +217,19 @@ export class WorkGraph {
     function renderNode(id: string, node: WorkNode): string[] {
       const ret = [];
       if (node.deploymentState === DeploymentState.COMPLETED) {
-        ret.push(`  "${id}" [style=filled,fillcolor=yellow];`);
+        ret.push(`  "${simplifyId(id)}" [style=filled,fillcolor=yellow];`);
       } else {
-        ret.push(`  "${id}";`);
+        ret.push(`  "${simplifyId(id)}";`);
       }
       for (const dep of node.dependencies) {
-        ret.push(`  "${id}" -> "${dep}";`);
+        ret.push(`  "${simplifyId(id)}" -> "${simplifyId(dep)}";`);
       }
       return ret;
     }
+
+    function simplifyId(id: string) {
+      return id.replace(/([0-9a-f]{6})[0-9a-f]{6,}/g, '$1');
+    }
   }
 
   /**
@@ -234,12 +240,8 @@ export class WorkGraph {
    */
   public removeUnavailableDependencies() {
     for (const node of Object.values(this.nodes)) {
-      const removeDeps = [];
-      for (const dep of node.dependencies) {
-        if (this.nodes[dep] === undefined) {
-          removeDeps.push(dep);
-        }
-      }
+      const removeDeps = Array.from(node.dependencies).filter((dep) => this.nodes[dep] === undefined);
+
       removeDeps.forEach((d) => {
         node.dependencies.delete(d);
       });
@@ -249,16 +251,25 @@ export class WorkGraph {
   /**
    * Remove all asset publishing steps for assets that are already published, and then build
    * that aren't used anymore.
+   *
+   * Do this in parallel, because there may be a lot of assets in an application (seen in practice: >100 assets)
    */
   public async removeUnnecessaryAssets(isUnnecessary: (x: AssetPublishNode) => Promise<boolean>) {
+    debug('Checking for previously published assets');
+
     const publishes = this.nodesOfType('asset-publish');
-    for (const assetNode of publishes) {
-      const unnecessary = await isUnnecessary(assetNode);
-      if (unnecessary) {
-        this.removeNode(assetNode);
-      }
+
+    const classifiedNodes = await parallelPromises(
+      8,
+      publishes.map((assetNode) => async() => [assetNode, await isUnnecessary(assetNode)] as const));
+
+    const alreadyPublished = classifiedNodes.filter(([_, unnecessary]) => unnecessary).map(([assetNode, _]) => assetNode);
+    for (const assetNode of alreadyPublished) {
+      this.removeNode(assetNode);
     }
 
+    debug(`${publishes.length} total assets, ${publishes.length - alreadyPublished.length} still need to be published`);
+
     // Now also remove any asset build steps that don't have any dependencies on them anymore
     const unusedBuilds = this.nodesOfType('asset-build').filter(build => this.dependees(build).length === 0);
     for (const unusedBuild of unusedBuilds) {
@@ -288,7 +299,8 @@ export class WorkGraph {
 
     if (this.readyPool.length === 0 && activeCount === 0 && pendingCount > 0) {
       const cycle = this.findCycle() ?? ['No cycle found!'];
-      throw new Error(`Unable to make progress anymore, dependency cycle between remaining artifacts: ${cycle.join(' -> ')}`);
+      trace(`Cycle ${cycle.join(' -> ')} in graph ${this}`);
+      throw new Error(`Unable to make progress anymore, dependency cycle between remaining artifacts: ${cycle.join(' -> ')} (run with -vv for full graph)`);
     }
   }
 

packages/aws-cdk/test/util/parallel.test.ts

@@ -0,0 +1,33 @@
+import { parallelPromises } from '../../lib/util/parallel';
+import { sleep } from '../util';
+
+test('parallelPromises', async () => {
+  const N = 4;
+  const J = 100;
+
+  let jobsDone = 0;
+  let concurrent = 0;
+  let maxConcurrent = 0;
+
+  const jobs = range(J).map(() => async () => {
+    concurrent += 1;
+    maxConcurrent = Math.max(concurrent, maxConcurrent);
+    await sleep(Math.round(Math.random() * 100));
+    concurrent -= 1;
+    jobsDone += 1;
+  });
+
+  await parallelPromises(N, jobs);
+
+  expect(maxConcurrent).toBeLessThanOrEqual(N);
+  expect(maxConcurrent).toBeGreaterThan(1);
+  expect(jobsDone).toEqual(J);
+});
+
+function range(n: number) {
+  const ret = new Array<number>();
+  for (let i = 0; i < n; i++) {
+    ret.push(i);
+  }
+  return ret;
+}