refactor(cli): organize code that should only be used by the CLI into a folder (#33138)

### Reason for this change

This PR is separating the code that should only be used by the CLI into
a specific folder.
It's currently quite hard to reason about the existing api code as its
spread across many files and deep subpath imports.
The change deliberately gives up on potential feature reusability of
some helpers in order to create locality.

Similar to #33094

### Description of changes

Collating existing files and APIs that should only be used by the CLI
into a new `cli` folder.
Updated imports accordingly.
No functional changes.

### Describe any new or updated permissions being added

n/a

### Description of how you validated changes

exiting tests

### Checklist
- [x] My code adheres to the [CONTRIBUTING
GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and
[DESIGN
GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md)

----

*By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license*

Author: mrgrain

packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES

@@ -1,28 +1,39 @@
 /* eslint-disable import/no-restricted-paths */
-export { formatErrorMessage } from '../../../../aws-cdk/lib/util/error';
-export { DEFAULT_TOOLKIT_STACK_NAME, SdkProvider } from '../../../../aws-cdk/lib/api';
-export type { SuccessfulDeployStackResult } from '../../../../aws-cdk/lib/api';
-export { formatSdkLoggerContent } from '../../../../aws-cdk/lib/api/aws-auth/sdk-logger';
-export { CloudAssembly, sanitizePatterns, StackCollection, ExtendedStackSelection } from '../../../../aws-cdk/lib/api/cxapp/cloud-assembly';
-export { prepareDefaultEnvironment, prepareContext, spaceAvailableForContext } from '../../../../aws-cdk/lib/api/cxapp/exec';
-export { Deployments } from '../../../../aws-cdk/lib/api/deployments';
+
+// APIs
+export { formatSdkLoggerContent, SdkProvider } from '../../../../aws-cdk/lib/api/aws-auth';
+export { Context, PROJECT_CONTEXT } from '../../../../aws-cdk/lib/api/context';
+export { Deployments, type SuccessfulDeployStackResult } from '../../../../aws-cdk/lib/api/deployments';
+export { Settings } from '../../../../aws-cdk/lib/api/settings';
+export { tagsForStack } from '../../../../aws-cdk/lib/api/tags';
+export { DEFAULT_TOOLKIT_STACK_NAME } from '../../../../aws-cdk/lib/api/toolkit-info';
+
+// Context Providers
+export * as contextproviders from '../../../../aws-cdk/lib/context-providers';
+
+// @todo APIs not clean import
 export { HotswapMode } from '../../../../aws-cdk/lib/api/hotswap/common';
 export { StackActivityProgress } from '../../../../aws-cdk/lib/api/util/cloudformation/stack-activity-monitor';
-export { RWLock } from '../../../../aws-cdk/lib/api/util/rwlock';
-export type { ILock } from '../../../../aws-cdk/lib/api/util/rwlock';
+export { RWLock, type ILock } from '../../../../aws-cdk/lib/api/util/rwlock';
 export { formatTime } from '../../../../aws-cdk/lib/api/util/string-manipulation';
-export * as contextproviders from '../../../../aws-cdk/lib/context-providers';
+
+// @todo Not yet API probably should be
+export { formatErrorMessage } from '../../../../aws-cdk/lib/util/error';
 export { ResourceMigrator } from '../../../../aws-cdk/lib/migrator';
 export { obscureTemplate, serializeStructure } from '../../../../aws-cdk/lib/serialize';
-export { Context, Settings, PROJECT_CONTEXT } from '../../../../aws-cdk/lib/settings';
-export { tagsForStack } from '../../../../aws-cdk/lib/tags';
-export { CliIoHost } from '../../../../aws-cdk/lib/toolkit/cli-io-host';
 export { loadTree, some } from '../../../../aws-cdk/lib/tree';
 export { splitBySize } from '../../../../aws-cdk/lib/util';
 export { validateSnsTopicArn } from '../../../../aws-cdk/lib/util/validate-notification-arn';
 export { WorkGraph } from '../../../../aws-cdk/lib/util/work-graph';
 export type { Concurrency } from '../../../../aws-cdk/lib/util/work-graph';
 export { WorkGraphBuilder } from '../../../../aws-cdk/lib/util/work-graph-builder';
 export type { AssetBuildNode, AssetPublishNode, StackNode } from '../../../../aws-cdk/lib/util/work-graph-types';
-export { versionNumber } from '../../../../aws-cdk/lib/version';
+
+// @todo Cloud Assembly and Executable - this is a messy API right now
+export { CloudAssembly, sanitizePatterns, StackCollection, ExtendedStackSelection } from '../../../../aws-cdk/lib/api/cxapp/cloud-assembly';
+export { prepareDefaultEnvironment, prepareContext, spaceAvailableForContext } from '../../../../aws-cdk/lib/api/cxapp/exec';
 export { guessExecutable } from '../../../../aws-cdk/lib/api/cxapp/exec';
+
+// @todo Should not use! investigate how to replace
+export { versionNumber } from '../../../../aws-cdk/lib/cli/version';
+export { CliIoHost } from '../../../../aws-cdk/lib/toolkit/cli-io-host';

packages/@aws-cdk/toolkit/lib/api/aws-cdk.ts

@@ -1,2 +1,3 @@
 export * from './sdk';
 export * from './sdk-provider';
+export * from './sdk-logger';

packages/aws-cdk/lib/api/aws-auth/index.ts

@@ -1,5 +1,5 @@
 import { BootstrapSource } from './bootstrap-environment';
-import { Tag } from '../../tags';
+import { Tag } from '../tags';
 import { StringWithoutPlaceholders } from '../util/placeholders';
 
 export const BUCKET_NAME_OUTPUT = 'BucketName';

packages/aws-cdk/lib/api/bootstrap/bootstrap-props.ts

@@ -0,0 +1,108 @@
+import { Settings } from './settings';
+import { ToolkitError } from '../toolkit/error';
+
+export { TRANSIENT_CONTEXT_KEY } from './settings';
+export const PROJECT_CONTEXT = 'cdk.context.json';
+
+interface ContextBag {
+  /**
+   * The file name of the context. Will be used to potentially
+   * save new context back to the original file.
+   */
+  fileName?: string;
+
+  /**
+   * The context values.
+   */
+  bag: Settings;
+}
+
+/**
+ * Class that supports overlaying property bags
+ *
+ * Reads come from the first property bag that can has the given key,
+ * writes go to the first property bag that is not readonly. A write
+ * will remove the value from all property bags after the first
+ * writable one.
+ */
+export class Context {
+  private readonly bags: Settings[];
+  private readonly fileNames: (string | undefined)[];
+
+  constructor(...bags: ContextBag[]) {
+    this.bags = bags.length > 0 ? bags.map((b) => b.bag) : [new Settings()];
+    this.fileNames =
+      bags.length > 0 ? bags.map((b) => b.fileName) : ['default'];
+  }
+
+  public get keys(): string[] {
+    return Object.keys(this.all);
+  }
+
+  public has(key: string) {
+    return this.keys.indexOf(key) > -1;
+  }
+
+  public get all(): { [key: string]: any } {
+    let ret = new Settings();
+
+    // In reverse order so keys to the left overwrite keys to the right of them
+    for (const bag of [...this.bags].reverse()) {
+      ret = ret.merge(bag);
+    }
+
+    return ret.all;
+  }
+
+  public get(key: string): any {
+    for (const bag of this.bags) {
+      const v = bag.get([key]);
+      if (v !== undefined) {
+        return v;
+      }
+    }
+    return undefined;
+  }
+
+  public set(key: string, value: any) {
+    for (const bag of this.bags) {
+      if (bag.readOnly) {
+        continue;
+      }
+
+      // All bags past the first one have the value erased
+      bag.set([key], value);
+      value = undefined;
+    }
+  }
+
+  public unset(key: string) {
+    this.set(key, undefined);
+  }
+
+  public clear() {
+    for (const key of this.keys) {
+      this.unset(key);
+    }
+  }
+
+  /**
+   * Save a specific context file
+   */
+  public async save(fileName: string): Promise<this> {
+    const index = this.fileNames.indexOf(fileName);
+
+    // File not found, don't do anything in this scenario
+    if (index === -1) {
+      return this;
+    }
+
+    const bag = this.bags[index];
+    if (bag.readOnly) {
+      throw new ToolkitError(`Context file ${fileName} is read only!`);
+    }
+
+    await bag.save(fileName);
+    return this;
+  }
+}

packages/aws-cdk/lib/api/context.ts

@@ -1,8 +1,8 @@
 import * as cxapi from '@aws-cdk/cx-api';
 import { CloudAssembly } from './cloud-assembly';
+import { Configuration } from '../../cli/user-configuration';
 import * as contextproviders from '../../context-providers';
 import { debug } from '../../logging';
-import { Configuration } from '../../settings';
 import { ToolkitError } from '../../toolkit/error';
 import { SdkProvider } from '../aws-auth';
 

packages/aws-cdk/lib/api/cxapp/cloud-executable.ts

@@ -5,13 +5,14 @@ import * as cxschema from '@aws-cdk/cloud-assembly-schema';
 import * as cxapi from '@aws-cdk/cx-api';
 import * as fs from 'fs-extra';
 import * as semver from 'semver';
+import { Configuration, PROJECT_CONFIG, USER_DEFAULTS } from '../../cli/user-configuration';
+import { versionNumber } from '../../cli/version';
 import { debug, warning } from '../../logging';
-import { Configuration, PROJECT_CONFIG, Settings, USER_DEFAULTS } from '../../settings';
 import { ToolkitError } from '../../toolkit/error';
 import { loadTree, some } from '../../tree';
 import { splitBySize } from '../../util/objects';
-import { versionNumber } from '../../version';
 import { SdkProvider } from '../aws-auth';
+import { Settings } from '../settings';
 import { RWLock, ILock } from '../util/rwlock';
 
 export interface ExecProgramResult {

packages/aws-cdk/lib/api/cxapp/exec.ts

@@ -29,13 +29,13 @@ import {
   type RootTemplateWithNestedStacks,
 } from './nested-stack-helpers';
 import { debug, warning } from '../../logging';
-import type { Tag } from '../../tags';
 import { ToolkitError } from '../../toolkit/error';
 import { formatErrorMessage } from '../../util/error';
 import type { SdkProvider } from '../aws-auth/sdk-provider';
 import { EnvironmentAccess } from '../environment-access';
 import { type EnvironmentResources } from '../environment-resources';
 import { HotswapMode, HotswapPropertyOverrides } from '../hotswap/common';
+import type { Tag } from '../tags';
 import { DEFAULT_TOOLKIT_STACK_NAME } from '../toolkit-info';
 import { StackActivityMonitor, StackActivityProgress } from '../util/cloudformation/stack-activity-monitor';
 import { StackEventPoller } from '../util/cloudformation/stack-event-poller';

packages/aws-cdk/lib/api/deployments/deployments.ts

@@ -0,0 +1,171 @@
+import * as os from 'os';
+import * as fs_path from 'path';
+import * as fs from 'fs-extra';
+import { warning } from '../logging';
+import { ToolkitError } from '../toolkit/error';
+import * as util from '../util/objects';
+
+export type SettingsMap = { [key: string]: any };
+
+/**
+ * If a context value is an object with this key set to a truthy value, it won't be saved to cdk.context.json
+ */
+export const TRANSIENT_CONTEXT_KEY = '$dontSaveContext';
+
+/**
+ * A single bag of settings
+ */
+export class Settings {
+  public static mergeAll(...settings: Settings[]): Settings {
+    let ret = new Settings();
+    for (const setting of settings) {
+      ret = ret.merge(setting);
+    }
+    return ret;
+  }
+
+  constructor(
+    private settings: SettingsMap = {},
+    public readonly readOnly = false,
+  ) {}
+
+  public async load(fileName: string): Promise<this> {
+    if (this.readOnly) {
+      throw new ToolkitError(
+        `Can't load ${fileName}: settings object is readonly`,
+      );
+    }
+    this.settings = {};
+
+    const expanded = expandHomeDir(fileName);
+    if (await fs.pathExists(expanded)) {
+      this.settings = await fs.readJson(expanded);
+    }
+
+    // See https://github.com/aws/aws-cdk/issues/59
+    this.prohibitContextKey('default-account', fileName);
+    this.prohibitContextKey('default-region', fileName);
+    this.warnAboutContextKey('aws:', fileName);
+
+    return this;
+  }
+
+  public async save(fileName: string): Promise<this> {
+    const expanded = expandHomeDir(fileName);
+    await fs.writeJson(expanded, stripTransientValues(this.settings), {
+      spaces: 2,
+    });
+    return this;
+  }
+
+  public get all(): any {
+    return this.get([]);
+  }
+
+  public merge(other: Settings): Settings {
+    return new Settings(util.deepMerge(this.settings, other.settings));
+  }
+
+  public subSettings(keyPrefix: string[]) {
+    return new Settings(this.get(keyPrefix) || {}, false);
+  }
+
+  public makeReadOnly(): Settings {
+    return new Settings(this.settings, true);
+  }
+
+  public clear() {
+    if (this.readOnly) {
+      throw new ToolkitError('Cannot clear(): settings are readonly');
+    }
+    this.settings = {};
+  }
+
+  public get empty(): boolean {
+    return Object.keys(this.settings).length === 0;
+  }
+
+  public get(path: string[]): any {
+    return util.deepClone(util.deepGet(this.settings, path));
+  }
+
+  public set(path: string[], value: any): Settings {
+    if (this.readOnly) {
+      throw new ToolkitError(`Can't set ${path}: settings object is readonly`);
+    }
+    if (path.length === 0) {
+      // deepSet can't handle this case
+      this.settings = value;
+    } else {
+      util.deepSet(this.settings, path, value);
+    }
+    return this;
+  }
+
+  public unset(path: string[]) {
+    this.set(path, undefined);
+  }
+
+  private prohibitContextKey(key: string, fileName: string) {
+    if (!this.settings.context) {
+      return;
+    }
+    if (key in this.settings.context) {
+      // eslint-disable-next-line max-len
+      throw new ToolkitError(
+        `The 'context.${key}' key was found in ${fs_path.resolve(
+          fileName,
+        )}, but it is no longer supported. Please remove it.`,
+      );
+    }
+  }
+
+  private warnAboutContextKey(prefix: string, fileName: string) {
+    if (!this.settings.context) {
+      return;
+    }
+    for (const contextKey of Object.keys(this.settings.context)) {
+      if (contextKey.startsWith(prefix)) {
+        // eslint-disable-next-line max-len
+        warning(
+          `A reserved context key ('context.${prefix}') key was found in ${fs_path.resolve(
+            fileName,
+          )}, it might cause surprising behavior and should be removed.`,
+        );
+      }
+    }
+  }
+}
+
+function expandHomeDir(x: string) {
+  if (x.startsWith('~')) {
+    return fs_path.join(os.homedir(), x.slice(1));
+  }
+  return x;
+}
+
+/**
+ * Return all context value that are not transient context values
+ */
+function stripTransientValues(obj: { [key: string]: any }) {
+  const ret: any = {};
+  for (const [key, value] of Object.entries(obj)) {
+    if (!isTransientValue(value)) {
+      ret[key] = value;
+    }
+  }
+  return ret;
+}
+
+/**
+ * Return whether the given value is a transient context value
+ *
+ * Values that are objects with a magic key set to a truthy value are considered transient.
+ */
+function isTransientValue(value: any) {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    (value as any)[TRANSIENT_CONTEXT_KEY]
+  );
+}

packages/aws-cdk/lib/api/settings.ts

@@ -1,4 +1,4 @@
-import * as cxapi from '@aws-cdk/cx-api';
+import type * as cxapi from '@aws-cdk/cx-api';
 
 /**
  * @returns an array with the tags available in the stack metadata.