feat(clients): support util functions to consume response streams (#3977)

* feat(codegen): move util-stream-* package to aws dependency to ts dependency

* feat(types): add sdk stream mixin injector type

* fix(util-stream-browser): fix typo

* feat(util-stream-node): use TextDecoder

* feat(clients): support util functions to consume response streams

* feat(serde): use type mapping to describe stream mixin

Co-authored-by: AllanZhengYP <[email protected]>

Author: kuhe

clients/client-backupstorage/package.json

@@ -46,6 +46,8 @@
     "@aws-sdk/util-body-length-node": "*",
     "@aws-sdk/util-defaults-mode-browser": "*",
     "@aws-sdk/util-defaults-mode-node": "*",
+    "@aws-sdk/util-stream-browser": "*",
+    "@aws-sdk/util-stream-node": "*",
     "@aws-sdk/util-user-agent-browser": "*",
     "@aws-sdk/util-user-agent-node": "*",
     "@aws-sdk/util-utf8-browser": "*",

clients/client-backupstorage/src/BackupStorageClient.ts

@@ -48,6 +48,7 @@ import {
   Provider as __Provider,
   Provider,
   RegionInfoProvider,
+  SdkStreamMixinInjector as __SdkStreamMixinInjector,
   StreamCollector as __StreamCollector,
   UrlParser as __UrlParser,
   UserAgent as __UserAgent,
@@ -214,6 +215,12 @@ export interface ClientDefaults extends Partial<__SmithyResolvedConfiguration<__
    * The {@link DefaultsMode} that will be used to determine how certain default configuration options are resolved in the SDK.
    */
   defaultsMode?: DefaultsMode | Provider<DefaultsMode>;
+
+  /**
+   * The internal function that inject utilities to runtime-specific stream to help users consume the data
+   * @internal
+   */
+  sdkStreamMixin?: __SdkStreamMixinInjector;
 }
 
 type BackupStorageClientConfigType = Partial<__SmithyConfiguration<__HttpHandlerOptions>> &

clients/client-backupstorage/src/commands/GetChunkCommand.ts

@@ -9,7 +9,10 @@ import {
   HttpHandlerOptions as __HttpHandlerOptions,
   MetadataBearer as __MetadataBearer,
   MiddlewareStack,
+  SdkStream as __SdkStream,
+  SdkStreamSerdeContext as __SdkStreamSerdeContext,
   SerdeContext as __SerdeContext,
+  WithSdkStreamMixin as __WithSdkStreamMixin,
 } from "@aws-sdk/types";
 
 import { BackupStorageClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../BackupStorageClient";
@@ -25,7 +28,7 @@ import {
 } from "../protocols/Aws_restJson1";
 
 export interface GetChunkCommandInput extends GetChunkInput {}
-export interface GetChunkCommandOutput extends GetChunkOutput, __MetadataBearer {}
+export interface GetChunkCommandOutput extends __WithSdkStreamMixin<GetChunkOutput, "Data">, __MetadataBearer {}
 
 /**
  * Gets the specified object's chunk.
@@ -92,7 +95,10 @@ export class GetChunkCommand extends $Command<
     return serializeAws_restJson1GetChunkCommand(input, context);
   }
 
-  private deserialize(output: __HttpResponse, context: __SerdeContext): Promise<GetChunkCommandOutput> {
+  private deserialize(
+    output: __HttpResponse,
+    context: __SerdeContext & __SdkStreamSerdeContext
+  ): Promise<GetChunkCommandOutput> {
     return deserializeAws_restJson1GetChunkCommand(output, context);
   }
 

clients/client-backupstorage/src/commands/GetObjectMetadataCommand.ts

@@ -9,7 +9,10 @@ import {
   HttpHandlerOptions as __HttpHandlerOptions,
   MetadataBearer as __MetadataBearer,
   MiddlewareStack,
+  SdkStream as __SdkStream,
+  SdkStreamSerdeContext as __SdkStreamSerdeContext,
   SerdeContext as __SerdeContext,
+  WithSdkStreamMixin as __WithSdkStreamMixin,
 } from "@aws-sdk/types";
 
 import { BackupStorageClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../BackupStorageClient";
@@ -25,7 +28,9 @@ import {
 } from "../protocols/Aws_restJson1";
 
 export interface GetObjectMetadataCommandInput extends GetObjectMetadataInput {}
-export interface GetObjectMetadataCommandOutput extends GetObjectMetadataOutput, __MetadataBearer {}
+export interface GetObjectMetadataCommandOutput
+  extends __WithSdkStreamMixin<GetObjectMetadataOutput, "MetadataBlob">,
+    __MetadataBearer {}
 
 /**
  * Get metadata associated with an Object.
@@ -92,7 +97,10 @@ export class GetObjectMetadataCommand extends $Command<
     return serializeAws_restJson1GetObjectMetadataCommand(input, context);
   }
 
-  private deserialize(output: __HttpResponse, context: __SerdeContext): Promise<GetObjectMetadataCommandOutput> {
+  private deserialize(
+    output: __HttpResponse,
+    context: __SerdeContext & __SdkStreamSerdeContext
+  ): Promise<GetObjectMetadataCommandOutput> {
     return deserializeAws_restJson1GetObjectMetadataCommand(output, context);
   }
 

clients/client-backupstorage/src/protocols/Aws_restJson1.ts

@@ -15,6 +15,7 @@ import {
 import {
   Endpoint as __Endpoint,
   ResponseMetadata as __ResponseMetadata,
+  SdkStreamSerdeContext as __SdkStreamSerdeContext,
   SerdeContext as __SerdeContext,
 } from "@aws-sdk/types";
 
@@ -413,7 +414,7 @@ const deserializeAws_restJson1DeleteObjectCommandError = async (
 
 export const deserializeAws_restJson1GetChunkCommand = async (
   output: __HttpResponse,
-  context: __SerdeContext
+  context: __SerdeContext & __SdkStreamSerdeContext
 ): Promise<GetChunkCommandOutput> => {
   if (output.statusCode !== 200 && output.statusCode >= 300) {
     return deserializeAws_restJson1GetChunkCommandError(output, context);
@@ -428,6 +429,7 @@ export const deserializeAws_restJson1GetChunkCommand = async (
     ChecksumAlgorithm: [, output.headers["x-amz-checksum-algorithm"]],
   });
   const data: any = output.body;
+  context.sdkStreamMixin(data);
   contents.Data = data;
   return contents;
 };
@@ -476,7 +478,7 @@ const deserializeAws_restJson1GetChunkCommandError = async (
 
 export const deserializeAws_restJson1GetObjectMetadataCommand = async (
   output: __HttpResponse,
-  context: __SerdeContext
+  context: __SerdeContext & __SdkStreamSerdeContext
 ): Promise<GetObjectMetadataCommandOutput> => {
   if (output.statusCode !== 200 && output.statusCode >= 300) {
     return deserializeAws_restJson1GetObjectMetadataCommandError(output, context);
@@ -492,6 +494,7 @@ export const deserializeAws_restJson1GetObjectMetadataCommand = async (
     MetadataBlobChecksumAlgorithm: [, output.headers["x-amz-checksum-algorithm"]],
   });
   const data: any = output.body;
+  context.sdkStreamMixin(data);
   contents.MetadataBlob = data;
   return contents;
 };

clients/client-backupstorage/src/runtimeConfig.browser.ts

@@ -9,6 +9,7 @@ import { invalidProvider } from "@aws-sdk/invalid-dependency";
 import { DEFAULT_MAX_ATTEMPTS, DEFAULT_RETRY_MODE } from "@aws-sdk/middleware-retry";
 import { fromBase64, toBase64 } from "@aws-sdk/util-base64-browser";
 import { calculateBodyLength } from "@aws-sdk/util-body-length-browser";
+import { sdkStreamMixin } from "@aws-sdk/util-stream-browser";
 import { defaultUserAgent } from "@aws-sdk/util-user-agent-browser";
 import { fromUtf8, toUtf8 } from "@aws-sdk/util-utf8-browser";
 import { BackupStorageClientConfig } from "./BackupStorageClient";
@@ -40,6 +41,7 @@ export const getRuntimeConfig = (config: BackupStorageClientConfig) => {
     region: config?.region ?? invalidProvider("Region is missing"),
     requestHandler: config?.requestHandler ?? new RequestHandler(defaultConfigProvider),
     retryMode: config?.retryMode ?? (async () => (await defaultConfigProvider()).retryMode || DEFAULT_RETRY_MODE),
+    sdkStreamMixin: config?.sdkStreamMixin ?? sdkStreamMixin,
     sha256: config?.sha256 ?? Sha256,
     streamCollector: config?.streamCollector ?? streamCollector,
     useDualstackEndpoint: config?.useDualstackEndpoint ?? (() => Promise.resolve(DEFAULT_USE_DUALSTACK_ENDPOINT)),

clients/client-backupstorage/src/runtimeConfig.ts

@@ -20,6 +20,7 @@ import { loadConfig as loadNodeConfig } from "@aws-sdk/node-config-provider";
 import { NodeHttpHandler as RequestHandler, streamCollector } from "@aws-sdk/node-http-handler";
 import { fromBase64, toBase64 } from "@aws-sdk/util-base64-node";
 import { calculateBodyLength } from "@aws-sdk/util-body-length-node";
+import { sdkStreamMixin } from "@aws-sdk/util-stream-node";
 import { defaultUserAgent } from "@aws-sdk/util-user-agent-node";
 import { fromUtf8, toUtf8 } from "@aws-sdk/util-utf8-node";
 import { BackupStorageClientConfig } from "./BackupStorageClient";
@@ -58,6 +59,7 @@ export const getRuntimeConfig = (config: BackupStorageClientConfig) => {
         ...NODE_RETRY_MODE_CONFIG_OPTIONS,
         default: async () => (await defaultConfigProvider()).retryMode || DEFAULT_RETRY_MODE,
       }),
+    sdkStreamMixin: config?.sdkStreamMixin ?? sdkStreamMixin,
     sha256: config?.sha256 ?? Hash.bind(null, "sha256"),
     streamCollector: config?.streamCollector ?? streamCollector,
     useDualstackEndpoint: config?.useDualstackEndpoint ?? loadNodeConfig(NODE_USE_DUALSTACK_ENDPOINT_CONFIG_OPTIONS),

clients/client-codeartifact/package.json

@@ -46,6 +46,8 @@
     "@aws-sdk/util-body-length-node": "*",
     "@aws-sdk/util-defaults-mode-browser": "*",
     "@aws-sdk/util-defaults-mode-node": "*",
+    "@aws-sdk/util-stream-browser": "*",
+    "@aws-sdk/util-stream-node": "*",
     "@aws-sdk/util-user-agent-browser": "*",
     "@aws-sdk/util-user-agent-node": "*",
     "@aws-sdk/util-utf8-browser": "*",

clients/client-codeartifact/src/CodeartifactClient.ts

@@ -48,6 +48,7 @@ import {
   Provider as __Provider,
   Provider,
   RegionInfoProvider,
+  SdkStreamMixinInjector as __SdkStreamMixinInjector,
   StreamCollector as __StreamCollector,
   UrlParser as __UrlParser,
   UserAgent as __UserAgent,
@@ -361,6 +362,12 @@ export interface ClientDefaults extends Partial<__SmithyResolvedConfiguration<__
    * The {@link DefaultsMode} that will be used to determine how certain default configuration options are resolved in the SDK.
    */
   defaultsMode?: DefaultsMode | Provider<DefaultsMode>;
+
+  /**
+   * The internal function that inject utilities to runtime-specific stream to help users consume the data
+   * @internal
+   */
+  sdkStreamMixin?: __SdkStreamMixinInjector;
 }
 
 type CodeartifactClientConfigType = Partial<__SmithyConfiguration<__HttpHandlerOptions>> &

clients/client-codeartifact/src/commands/GetPackageVersionAssetCommand.ts

@@ -9,7 +9,10 @@ import {
   HttpHandlerOptions as __HttpHandlerOptions,
   MetadataBearer as __MetadataBearer,
   MiddlewareStack,
+  SdkStream as __SdkStream,
+  SdkStreamSerdeContext as __SdkStreamSerdeContext,
   SerdeContext as __SerdeContext,
+  WithSdkStreamMixin as __WithSdkStreamMixin,
 } from "@aws-sdk/types";
 
 import { CodeartifactClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../CodeartifactClient";
@@ -25,7 +28,9 @@ import {
 } from "../protocols/Aws_restJson1";
 
 export interface GetPackageVersionAssetCommandInput extends GetPackageVersionAssetRequest {}
-export interface GetPackageVersionAssetCommandOutput extends GetPackageVersionAssetResult, __MetadataBearer {}
+export interface GetPackageVersionAssetCommandOutput
+  extends __WithSdkStreamMixin<GetPackageVersionAssetResult, "asset">,
+    __MetadataBearer {}
 
 /**
  * <p>
@@ -96,7 +101,10 @@ export class GetPackageVersionAssetCommand extends $Command<
     return serializeAws_restJson1GetPackageVersionAssetCommand(input, context);
   }
 
-  private deserialize(output: __HttpResponse, context: __SerdeContext): Promise<GetPackageVersionAssetCommandOutput> {
+  private deserialize(
+    output: __HttpResponse,
+    context: __SerdeContext & __SdkStreamSerdeContext
+  ): Promise<GetPackageVersionAssetCommandOutput> {
     return deserializeAws_restJson1GetPackageVersionAssetCommand(output, context);
   }
 

clients/client-codeartifact/src/protocols/Aws_restJson1.ts

@@ -17,6 +17,7 @@ import {
 import {
   Endpoint as __Endpoint,
   ResponseMetadata as __ResponseMetadata,
+  SdkStreamSerdeContext as __SdkStreamSerdeContext,
   SerdeContext as __SerdeContext,
 } from "@aws-sdk/types";
 
@@ -2188,7 +2189,7 @@ const deserializeAws_restJson1GetDomainPermissionsPolicyCommandError = async (
 
 export const deserializeAws_restJson1GetPackageVersionAssetCommand = async (
   output: __HttpResponse,
-  context: __SerdeContext
+  context: __SerdeContext & __SdkStreamSerdeContext
 ): Promise<GetPackageVersionAssetCommandOutput> => {
   if (output.statusCode !== 200 && output.statusCode >= 300) {
     return deserializeAws_restJson1GetPackageVersionAssetCommandError(output, context);
@@ -2200,6 +2201,7 @@ export const deserializeAws_restJson1GetPackageVersionAssetCommand = async (
     packageVersionRevision: [, output.headers["x-packageversionrevision"]],
   });
   const data: any = output.body;
+  context.sdkStreamMixin(data);
   contents.asset = data;
   return contents;
 };

clients/client-codeartifact/src/runtimeConfig.browser.ts

@@ -9,6 +9,7 @@ import { invalidProvider } from "@aws-sdk/invalid-dependency";
 import { DEFAULT_MAX_ATTEMPTS, DEFAULT_RETRY_MODE } from "@aws-sdk/middleware-retry";
 import { fromBase64, toBase64 } from "@aws-sdk/util-base64-browser";
 import { calculateBodyLength } from "@aws-sdk/util-body-length-browser";
+import { sdkStreamMixin } from "@aws-sdk/util-stream-browser";
 import { defaultUserAgent } from "@aws-sdk/util-user-agent-browser";
 import { fromUtf8, toUtf8 } from "@aws-sdk/util-utf8-browser";
 import { CodeartifactClientConfig } from "./CodeartifactClient";
@@ -40,6 +41,7 @@ export const getRuntimeConfig = (config: CodeartifactClientConfig) => {
     region: config?.region ?? invalidProvider("Region is missing"),
     requestHandler: config?.requestHandler ?? new RequestHandler(defaultConfigProvider),
     retryMode: config?.retryMode ?? (async () => (await defaultConfigProvider()).retryMode || DEFAULT_RETRY_MODE),
+    sdkStreamMixin: config?.sdkStreamMixin ?? sdkStreamMixin,
     sha256: config?.sha256 ?? Sha256,
     streamCollector: config?.streamCollector ?? streamCollector,
     useDualstackEndpoint: config?.useDualstackEndpoint ?? (() => Promise.resolve(DEFAULT_USE_DUALSTACK_ENDPOINT)),

clients/client-codeartifact/src/runtimeConfig.ts

@@ -20,6 +20,7 @@ import { loadConfig as loadNodeConfig } from "@aws-sdk/node-config-provider";
 import { NodeHttpHandler as RequestHandler, streamCollector } from "@aws-sdk/node-http-handler";
 import { fromBase64, toBase64 } from "@aws-sdk/util-base64-node";
 import { calculateBodyLength } from "@aws-sdk/util-body-length-node";
+import { sdkStreamMixin } from "@aws-sdk/util-stream-node";
 import { defaultUserAgent } from "@aws-sdk/util-user-agent-node";
 import { fromUtf8, toUtf8 } from "@aws-sdk/util-utf8-node";
 import { CodeartifactClientConfig } from "./CodeartifactClient";
@@ -58,6 +59,7 @@ export const getRuntimeConfig = (config: CodeartifactClientConfig) => {
         ...NODE_RETRY_MODE_CONFIG_OPTIONS,
         default: async () => (await defaultConfigProvider()).retryMode || DEFAULT_RETRY_MODE,
       }),
+    sdkStreamMixin: config?.sdkStreamMixin ?? sdkStreamMixin,
     sha256: config?.sha256 ?? Hash.bind(null, "sha256"),
     streamCollector: config?.streamCollector ?? streamCollector,
     useDualstackEndpoint: config?.useDualstackEndpoint ?? loadNodeConfig(NODE_USE_DUALSTACK_ENDPOINT_CONFIG_OPTIONS),

clients/client-ebs/package.json

@@ -46,6 +46,8 @@
     "@aws-sdk/util-body-length-node": "*",
     "@aws-sdk/util-defaults-mode-browser": "*",
     "@aws-sdk/util-defaults-mode-node": "*",
+    "@aws-sdk/util-stream-browser": "*",
+    "@aws-sdk/util-stream-node": "*",
     "@aws-sdk/util-user-agent-browser": "*",
     "@aws-sdk/util-user-agent-node": "*",
     "@aws-sdk/util-utf8-browser": "*",

clients/client-ebs/src/EBSClient.ts

@@ -48,6 +48,7 @@ import {
   Provider as __Provider,
   Provider,
   RegionInfoProvider,
+  SdkStreamMixinInjector as __SdkStreamMixinInjector,
   StreamCollector as __StreamCollector,
   UrlParser as __UrlParser,
   UserAgent as __UserAgent,
@@ -202,6 +203,12 @@ export interface ClientDefaults extends Partial<__SmithyResolvedConfiguration<__
    * The {@link DefaultsMode} that will be used to determine how certain default configuration options are resolved in the SDK.
    */
   defaultsMode?: DefaultsMode | Provider<DefaultsMode>;
+
+  /**
+   * The internal function that inject utilities to runtime-specific stream to help users consume the data
+   * @internal
+   */
+  sdkStreamMixin?: __SdkStreamMixinInjector;
 }
 
 type EBSClientConfigType = Partial<__SmithyConfiguration<__HttpHandlerOptions>> &

clients/client-ebs/src/commands/GetSnapshotBlockCommand.ts

@@ -9,7 +9,10 @@ import {
   HttpHandlerOptions as __HttpHandlerOptions,
   MetadataBearer as __MetadataBearer,
   MiddlewareStack,
+  SdkStream as __SdkStream,
+  SdkStreamSerdeContext as __SdkStreamSerdeContext,
   SerdeContext as __SerdeContext,
+  WithSdkStreamMixin as __WithSdkStreamMixin,
 } from "@aws-sdk/types";
 
 import { EBSClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../EBSClient";
@@ -25,7 +28,9 @@ import {
 } from "../protocols/Aws_restJson1";
 
 export interface GetSnapshotBlockCommandInput extends GetSnapshotBlockRequest {}
-export interface GetSnapshotBlockCommandOutput extends GetSnapshotBlockResponse, __MetadataBearer {}
+export interface GetSnapshotBlockCommandOutput
+  extends __WithSdkStreamMixin<GetSnapshotBlockResponse, "BlockData">,
+    __MetadataBearer {}
 
 /**
  * <p>Returns the data in a block in an Amazon Elastic Block Store snapshot.</p>
@@ -92,7 +97,10 @@ export class GetSnapshotBlockCommand extends $Command<
     return serializeAws_restJson1GetSnapshotBlockCommand(input, context);
   }
 
-  private deserialize(output: __HttpResponse, context: __SerdeContext): Promise<GetSnapshotBlockCommandOutput> {
+  private deserialize(
+    output: __HttpResponse,
+    context: __SerdeContext & __SdkStreamSerdeContext
+  ): Promise<GetSnapshotBlockCommandOutput> {
     return deserializeAws_restJson1GetSnapshotBlockCommand(output, context);
   }
 

clients/client-ebs/src/protocols/Aws_restJson1.ts

@@ -18,6 +18,7 @@ import {
 import {
   Endpoint as __Endpoint,
   ResponseMetadata as __ResponseMetadata,
+  SdkStreamSerdeContext as __SdkStreamSerdeContext,
   SerdeContext as __SerdeContext,
 } from "@aws-sdk/types";
 import { v4 as generateIdempotencyToken } from "uuid";
@@ -298,7 +299,7 @@ const deserializeAws_restJson1CompleteSnapshotCommandError = async (
 
 export const deserializeAws_restJson1GetSnapshotBlockCommand = async (
   output: __HttpResponse,
-  context: __SerdeContext
+  context: __SerdeContext & __SdkStreamSerdeContext
 ): Promise<GetSnapshotBlockCommandOutput> => {
   if (output.statusCode !== 200 && output.statusCode >= 300) {
     return deserializeAws_restJson1GetSnapshotBlockCommandError(output, context);
@@ -313,6 +314,7 @@ export const deserializeAws_restJson1GetSnapshotBlockCommand = async (
     ChecksumAlgorithm: [, output.headers["x-amz-checksum-algorithm"]],
   });
   const data: any = output.body;
+  context.sdkStreamMixin(data);
   contents.BlockData = data;
   return contents;
 };

clients/client-ebs/src/runtimeConfig.browser.ts

@@ -9,6 +9,7 @@ import { invalidProvider } from "@aws-sdk/invalid-dependency";
 import { DEFAULT_MAX_ATTEMPTS, DEFAULT_RETRY_MODE } from "@aws-sdk/middleware-retry";
 import { fromBase64, toBase64 } from "@aws-sdk/util-base64-browser";
 import { calculateBodyLength } from "@aws-sdk/util-body-length-browser";
+import { sdkStreamMixin } from "@aws-sdk/util-stream-browser";
 import { defaultUserAgent } from "@aws-sdk/util-user-agent-browser";
 import { fromUtf8, toUtf8 } from "@aws-sdk/util-utf8-browser";
 import { EBSClientConfig } from "./EBSClient";
@@ -40,6 +41,7 @@ export const getRuntimeConfig = (config: EBSClientConfig) => {
     region: config?.region ?? invalidProvider("Region is missing"),
     requestHandler: config?.requestHandler ?? new RequestHandler(defaultConfigProvider),
     retryMode: config?.retryMode ?? (async () => (await defaultConfigProvider()).retryMode || DEFAULT_RETRY_MODE),
+    sdkStreamMixin: config?.sdkStreamMixin ?? sdkStreamMixin,
     sha256: config?.sha256 ?? Sha256,
     streamCollector: config?.streamCollector ?? streamCollector,
     useDualstackEndpoint: config?.useDualstackEndpoint ?? (() => Promise.resolve(DEFAULT_USE_DUALSTACK_ENDPOINT)),