feat(toolkit-lib): emit marker messages during actions (#219)

Introduces the new concept of "Message Spans" to connect multiple
messages for toolkit sub-tasks like "synth" or "build-assets" together.
A span consists of a number of messages that have the same `span` value.

At a minimum, a span is a pair of a `start` and an `end` message. Every
end message also reports the elapsed time since the start of the span.

Message spans will allow integrators to more easily track the flow of
"tasks" completed in the Toolkit.

## Changes

- Refactors the previous `Timer` class to "Message Spans"
- Introduces a new registry for spans at the bottom of the existing
messages registry. A span consists of a start and an end message maker.
- This PR makes use of the new `IoHelper` to provide a nicer API
- There is some more of type magic to ensure correct payloads, I don't
think this can be tested further
- I also changed the message registry to be private again, we are not
ready to have it as part of the public interface (this doesn't affect
our packages)
- Made some improvements to the bundling of declaration files. It's
currently really slow because it ends up scanning through A LOT of SDK
types. Will look into this more in a follow-up PR.

Fixes aws/aws-cdk#33286

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

Author: mrgrain

.projenrc.ts

@@ -705,6 +705,7 @@ const tmpToolkitHelpers = configureProject(
       'archiver',
       'glob',
       'semver',
+      'uuid',
       'yaml@^1',
     ],
     tsconfig: {

packages/@aws-cdk/tmp-toolkit-helpers/.projen/deps.json

@@ -0,0 +1 @@
+export * from './stack-selector';

packages/@aws-cdk/tmp-toolkit-helpers/.projen/tasks.json

@@ -0,0 +1,100 @@
+/**
+ * Which stacks should be selected from a cloud assembly
+ */
+export enum StackSelectionStrategy {
+  /**
+   * Returns all stacks in the app regardless of patterns,
+   * including stacks inside nested assemblies.
+   */
+  ALL_STACKS = 'all-stacks',
+
+  /**
+   * Returns all stacks in the main (top level) assembly only.
+   */
+  MAIN_ASSEMBLY = 'main-assembly',
+
+  /**
+   * If the assembly includes a single stack, returns it.
+   * Otherwise throws an exception.
+   */
+  ONLY_SINGLE = 'only-single',
+
+  /**
+   * Return stacks matched by patterns.
+   * If no stacks are found, execution is halted successfully.
+   * Most likely you don't want to use this but `StackSelectionStrategy.MUST_MATCH_PATTERN`
+   */
+  PATTERN_MATCH = 'pattern-match',
+
+  /**
+   * Return stacks matched by patterns.
+   * Throws an exception if the patterns don't match at least one stack in the assembly.
+   */
+  PATTERN_MUST_MATCH = 'pattern-must-match',
+
+  /**
+   * Returns if exactly one stack is matched by the pattern(s).
+   * Throws an exception if no stack, or more than exactly one stack are matched.
+   */
+  PATTERN_MUST_MATCH_SINGLE = 'pattern-must-match-single',
+}
+
+/**
+ * When selecting stacks, what other stacks to include because of dependencies
+ */
+export enum ExpandStackSelection {
+  /**
+   * Don't select any extra stacks
+   */
+  NONE = 'none',
+
+  /**
+   * Include stacks that this stack depends on
+   */
+  UPSTREAM = 'upstream',
+
+  /**
+   * Include stacks that depend on this stack
+   */
+  DOWNSTREAM = 'downstream',
+
+  /**
+   * @TODO
+   * Include both directions.
+   * I.e. stacks that this stack depends on, and stacks that depend on this stack.
+   */
+  // FULL = 'full',
+}
+
+/**
+ * A specification of which stacks should be selected
+ */
+export interface StackSelector {
+  /**
+   * The behavior if if no selectors are provided.
+   */
+  strategy: StackSelectionStrategy;
+
+  /**
+   * A list of patterns to match the stack hierarchical ids
+   * Only used with `PATTERN_*` selection strategies.
+   */
+  patterns?: string[];
+
+  /**
+   * Expand the selection to upstream/downstream stacks.
+   * @default ExpandStackSelection.None only select the specified/matched stacks
+   */
+  expand?: ExpandStackSelection;
+
+  /**
+   * By default, we throw an exception if the assembly contains no stacks.
+   * Set to `false`, to halt execution for empty assemblies without error.
+   *
+   * Note that actions can still throw if a stack selection result is empty,
+   * but the assembly contains stacks in principle.
+   *
+   * @default true
+   */
+  failOnEmpty?: boolean;
+}

packages/@aws-cdk/tmp-toolkit-helpers/package.json

@@ -1,2 +1,3 @@
+export * from './cloud-assembly';
 export * from './io';
 export * from './toolkit-error';

packages/@aws-cdk/tmp-toolkit-helpers/src/api/cloud-assembly/index.ts

@@ -2,4 +2,3 @@ export * from './io-host';
 export * from './io-message';
 export * from './toolkit-action';
 export * from './payloads';
-export * from './messages';

packages/@aws-cdk/tmp-toolkit-helpers/src/api/cloud-assembly/stack-selector.ts

@@ -59,6 +59,16 @@ export interface IoMessage<T> {
    */
   readonly message: string;
 
+  /**
+   * Identifies the message span, this message belongs to.
+   *
+   * A message span, groups multiple messages together that semantically related to the same operation.
+   * This is an otherwise meaningless identifier.
+   *
+   * A message without a `spanId`, does not belong to a span.
+   */
+  readonly span?: string;
+
   /**
    * The data attached to the message.
    */

packages/@aws-cdk/tmp-toolkit-helpers/src/api/index.ts

@@ -1,4 +1,5 @@
 import type { CloudFormationStackArtifact } from '@aws-cdk/cx-api';
+import type { IManifestEntry } from 'cdk-assets';
 import type { PermissionChangeType } from './diff';
 import type { ConfirmationRequest } from './types';
 
@@ -55,3 +56,46 @@ export interface NeedRollbackFirstDeployStackResult {
 export interface ReplacementRequiresRollbackStackResult {
   readonly type: 'replacement-requires-rollback';
 }
+
+export interface StackDeployProgress {
+  /**
+   * The total number of stacks being deployed
+   */
+  readonly total: number;
+  /**
+   * The count of the stack currently attempted to be deployed
+   *
+   * This is counting value, not an identifier.
+   */
+  readonly current: number;
+  /**
+   * The stack that's currently being deployed
+   */
+  readonly stack: CloudFormationStackArtifact;
+}
+
+/**
+ * Payload for a yes/no confirmation in deploy. Includes information on
+ * what kind of change is being made.
+ */
+export interface DeployConfirmationRequest extends ConfirmationRequest {
+  /**
+   * The type of change being made to the IAM permissions.
+   */
+  readonly permissionChangeType: PermissionChangeType;
+}
+
+export interface BuildAsset {
+  /**
+   * The asset that is build
+   */
+  readonly asset: IManifestEntry;
+}
+
+export interface PublishAsset {
+
+  /**
+   * The asset that is published
+   */
+  readonly asset: IManifestEntry;
+}

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/index.ts

@@ -1,5 +1,12 @@
 import type { CloudFormationStackArtifact } from '@aws-cdk/cx-api';
 
+export interface StackDestroy {
+  /**
+   * The stacks that will be destroyed
+   */
+  readonly stacks: CloudFormationStackArtifact[];
+}
+
 export interface StackDestroyProgress {
   /**
    * The total number of stacks being destroyed

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/io-message.ts

@@ -6,6 +6,7 @@ export * from './sdk-trace';
 export * from './context';
 export * from './rollback';
 export * from './stack-activity';
+export * from './synth';
 export * from './types';
 export * from './progress';
 export * from './watch';

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/payloads/deploy.ts

@@ -1,5 +1,5 @@
-import { type MetadataEntry } from '@aws-cdk/cloud-assembly-schema';
-import { type CloudFormationStackArtifact } from '@aws-cdk/cx-api';
+import type { MetadataEntry } from '@aws-cdk/cloud-assembly-schema';
+import type { CloudFormationStackArtifact } from '@aws-cdk/cx-api';
 import type { StackEvent } from '@aws-sdk/client-cloudformation';
 import type { StackProgress } from './progress';
 

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/payloads/destroy.ts

@@ -0,0 +1,8 @@
+import type { StackSelector } from '../../cloud-assembly/stack-selector';
+
+export interface StackSelectionDetails {
+  /**
+   * The selected stacks, if any
+   */
+  readonly stacks: StackSelector;
+}

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/payloads/index.ts

@@ -1,4 +1,3 @@
-
 /**
  * The computed file watch settings
  */

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/payloads/stack-activity.ts

@@ -1,4 +1,6 @@
 export * from './io-helper';
 export * from './level-priority';
+export * from './span';
 export * from './message-maker';
+export * from './messages';
 export * from './types';

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/payloads/synth.ts

@@ -1,39 +1,59 @@
 import type { IIoHost } from '../io-host';
 import type { IoMessage, IoRequest } from '../io-message';
 import type { ToolkitAction } from '../toolkit-action';
+import type { SpanEnd, SpanDefinition } from './span';
+import { SpanMaker } from './span';
 
-export type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
-export type SimplifiedMessage<T> = Pick<IoMessage<T>, 'level' | 'code' | 'message' | 'data'>;
 export type ActionLessMessage<T> = Omit<IoMessage<T>, 'action'>;
 export type ActionLessRequest<T, U> = Omit<IoRequest<T, U>, 'action'>;
 
 /**
- * Helper for IO messaging.
- *
- * Wraps a client provided IoHost and provides additional features & services to toolkit internal classes.
+ * A class containing helper tools to interact with IoHost
  */
-export interface IoHelper extends IIoHost {
-  notify(msg: ActionLessMessage<unknown>): Promise<void>;
-  notify<T>(msg: ActionLessMessage<T>): Promise<void>;
-  requestResponse<T, U>(msg: ActionLessRequest<T, U>): Promise<U>;
+export class IoHelper implements IIoHost {
+  public static fromIoHost(ioHost: IIoHost, action: ToolkitAction) {
+    return new IoHelper(ioHost, action);
+  }
+
+  private readonly ioHost: IIoHost;
+  private readonly action: ToolkitAction;
+
+  private constructor(ioHost: IIoHost, action: ToolkitAction) {
+    this.ioHost = ioHost;
+    this.action = action;
+  }
+
+  /**
+   * Forward a message to the IoHost, while injection the current action
+   */
+  public notify(msg: ActionLessMessage<unknown>): Promise<void> {
+    return this.ioHost.notify({
+      ...msg,
+      action: this.action,
+    });
+  }
+
+  /**
+   * Forward a request to the IoHost, while injection the current action
+   */
+  public requestResponse<T, U>(msg: ActionLessRequest<T, U>): Promise<U> {
+    return this.ioHost.requestResponse({
+      ...msg,
+      action: this.action,
+    });
+  }
+
+  /**
+   * Create a new marker from a given registry entry
+   */
+  public span<S extends object, E extends SpanEnd>(definition: SpanDefinition<S, E>) {
+    return new SpanMaker(this, definition);
+  }
 }
 
 /**
  * Wraps an IoHost and creates an IoHelper from it
  */
 export function asIoHelper(ioHost: IIoHost, action: ToolkitAction): IoHelper {
-  return {
-    notify: async <T>(msg: Omit<IoMessage<T>, 'action'>) => {
-      await ioHost.notify({
-        ...msg,
-        action,
-      });
-    },
-    requestResponse: async <T, U>(msg: Omit<IoRequest<T, U>, 'action'>) => {
-      return ioHost.requestResponse({
-        ...msg,
-        action,
-      });
-    },
-  };
+  return IoHelper.fromIoHost(ioHost, action);
 }