feat(client-firehose): This release 1) Add configurable buffering hints for Snowflake as destination. 2) Add ReadFromTimestamp for MSK As Source. Firehose will start reading data from MSK Cluster using offset associated with this timestamp. 3) Gated public beta release to add Apache Iceberg tables as destination.

Author: awstools

clients/client-firehose/src/auth/httpAuthSchemeProvider.ts

@@ -60,7 +60,7 @@ function createAwsAuthSigv4HttpAuthOption(authParameters: FirehoseHttpAuthScheme
       name: "firehose",
       region: authParameters.region,
     },
-    propertiesExtractor: (config: FirehoseClientConfig, context) => ({
+    propertiesExtractor: (config: Partial<FirehoseClientConfig>, context) => ({
       /**
        * @internal
        */

clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts

@@ -540,6 +540,7 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       RoleARN: "STRING_VALUE", // required
  *       Connectivity: "PUBLIC" || "PRIVATE", // required
  *     },
+ *     ReadFromTimestamp: new Date("TIMESTAMP"),
  *   },
  *   SnowflakeDestinationConfiguration: { // SnowflakeDestinationConfiguration
  *     AccountUrl: "STRING_VALUE", // required
@@ -572,6 +573,34 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       RoleARN: "STRING_VALUE",
  *       Enabled: true || false, // required
  *     },
+ *     BufferingHints: { // SnowflakeBufferingHints
+ *       SizeInMBs: Number("int"),
+ *       IntervalInSeconds: Number("int"),
+ *     },
+ *   },
+ *   IcebergDestinationConfiguration: { // IcebergDestinationConfiguration
+ *     DestinationTableConfigurationList: [ // DestinationTableConfigurationList
+ *       { // DestinationTableConfiguration
+ *         DestinationTableName: "STRING_VALUE", // required
+ *         DestinationDatabaseName: "STRING_VALUE", // required
+ *         UniqueKeys: [
+ *           "STRING_VALUE",
+ *         ],
+ *         S3ErrorOutputPrefix: "STRING_VALUE",
+ *       },
+ *     ],
+ *     BufferingHints: "<BufferingHints>",
+ *     CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ *     ProcessingConfiguration: "<ProcessingConfiguration>",
+ *     S3BackupMode: "FailedDataOnly" || "AllData",
+ *     RetryOptions: {
+ *       DurationInSeconds: Number("int"),
+ *     },
+ *     RoleARN: "STRING_VALUE", // required
+ *     CatalogConfiguration: { // CatalogConfiguration
+ *       CatalogARN: "STRING_VALUE",
+ *     },
+ *     S3Configuration: "<S3DestinationConfiguration>", // required
  *   },
  * };
  * const command = new CreateDeliveryStreamCommand(input);

clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts

@@ -88,6 +88,7 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //           Connectivity: "PUBLIC" || "PRIVATE", // required
  * //         },
  * //         DeliveryStartTimestamp: new Date("TIMESTAMP"),
+ * //         ReadFromTimestamp: new Date("TIMESTAMP"),
  * //       },
  * //     },
  * //     Destinations: [ // DestinationDescriptionList // required
@@ -508,6 +509,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             RoleARN: "STRING_VALUE",
  * //             Enabled: true || false, // required
  * //           },
+ * //           BufferingHints: { // SnowflakeBufferingHints
+ * //             SizeInMBs: Number("int"),
+ * //             IntervalInSeconds: Number("int"),
+ * //           },
  * //         },
  * //         AmazonOpenSearchServerlessDestinationDescription: { // AmazonOpenSearchServerlessDestinationDescription
  * //           RoleARN: "STRING_VALUE",
@@ -535,6 +540,30 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             VpcId: "STRING_VALUE", // required
  * //           },
  * //         },
+ * //         IcebergDestinationDescription: { // IcebergDestinationDescription
+ * //           DestinationTableConfigurationList: [ // DestinationTableConfigurationList
+ * //             { // DestinationTableConfiguration
+ * //               DestinationTableName: "STRING_VALUE", // required
+ * //               DestinationDatabaseName: "STRING_VALUE", // required
+ * //               UniqueKeys: [
+ * //                 "STRING_VALUE",
+ * //               ],
+ * //               S3ErrorOutputPrefix: "STRING_VALUE",
+ * //             },
+ * //           ],
+ * //           BufferingHints: "<BufferingHints>",
+ * //           CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ * //           ProcessingConfiguration: "<ProcessingConfiguration>",
+ * //           S3BackupMode: "FailedDataOnly" || "AllData",
+ * //           RetryOptions: {
+ * //             DurationInSeconds: Number("int"),
+ * //           },
+ * //           RoleARN: "STRING_VALUE",
+ * //           CatalogConfiguration: { // CatalogConfiguration
+ * //             CatalogARN: "STRING_VALUE",
+ * //           },
+ * //           S3DestinationDescription: "<S3DestinationDescription>",
+ * //         },
  * //       },
  * //     ],
  * //     HasMoreDestinations: true || false, // required

clients/client-firehose/src/commands/UpdateDestinationCommand.ts

@@ -474,6 +474,43 @@ export interface UpdateDestinationCommandOutput extends UpdateDestinationOutput,
  *       RoleARN: "STRING_VALUE",
  *       Enabled: true || false, // required
  *     },
+ *     BufferingHints: { // SnowflakeBufferingHints
+ *       SizeInMBs: Number("int"),
+ *       IntervalInSeconds: Number("int"),
+ *     },
+ *   },
+ *   IcebergDestinationUpdate: { // IcebergDestinationUpdate
+ *     DestinationTableConfigurationList: [ // DestinationTableConfigurationList
+ *       { // DestinationTableConfiguration
+ *         DestinationTableName: "STRING_VALUE", // required
+ *         DestinationDatabaseName: "STRING_VALUE", // required
+ *         UniqueKeys: [
+ *           "STRING_VALUE",
+ *         ],
+ *         S3ErrorOutputPrefix: "STRING_VALUE",
+ *       },
+ *     ],
+ *     BufferingHints: "<BufferingHints>",
+ *     CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ *     ProcessingConfiguration: "<ProcessingConfiguration>",
+ *     S3BackupMode: "FailedDataOnly" || "AllData",
+ *     RetryOptions: {
+ *       DurationInSeconds: Number("int"),
+ *     },
+ *     RoleARN: "STRING_VALUE",
+ *     CatalogConfiguration: { // CatalogConfiguration
+ *       CatalogARN: "STRING_VALUE",
+ *     },
+ *     S3Configuration: { // S3DestinationConfiguration
+ *       RoleARN: "STRING_VALUE", // required
+ *       BucketARN: "STRING_VALUE", // required
+ *       Prefix: "STRING_VALUE",
+ *       ErrorOutputPrefix: "STRING_VALUE",
+ *       BufferingHints: "<BufferingHints>",
+ *       CompressionFormat: "UNCOMPRESSED" || "GZIP" || "ZIP" || "Snappy" || "HADOOP_SNAPPY",
+ *       EncryptionConfiguration: "<EncryptionConfiguration>",
+ *       CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ *     },
  *   },
  * };
  * const command = new UpdateDestinationCommand(input);