Skip to content

Commit f392d88

Browse files
author
awstools
committed
feat(client-firehose): Features : Adding support for new data ingestion source to Kinesis Firehose - AWS Managed Services Kafka.
1 parent 114788d commit f392d88

File tree

10 files changed

+344
-41
lines changed

10 files changed

+344
-41
lines changed

‎clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
118118
* const client = new FirehoseClient(config);
119119
* const input = { // CreateDeliveryStreamInput
120120
* DeliveryStreamName: "STRING_VALUE", // required
121-
* DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource",
121+
* DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource",
122122
* KinesisStreamSourceConfiguration: { // KinesisStreamSourceConfiguration
123123
* KinesisStreamARN: "STRING_VALUE", // required
124124
* RoleARN: "STRING_VALUE", // required
@@ -174,10 +174,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
174174
* Enabled: true || false,
175175
* Processors: [ // ProcessorList
176176
* { // Processor
177-
* Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
177+
* Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
178178
* Parameters: [ // ProcessorParameterList
179179
* { // ProcessorParameter
180-
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
180+
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
181181
* ParameterValue: "STRING_VALUE", // required
182182
* },
183183
* ],
@@ -306,10 +306,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
306306
* Enabled: true || false,
307307
* Processors: [
308308
* {
309-
* Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
309+
* Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
310310
* Parameters: [
311311
* {
312-
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
312+
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
313313
* ParameterValue: "STRING_VALUE", // required
314314
* },
315315
* ],
@@ -370,10 +370,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
370370
* Enabled: true || false,
371371
* Processors: [
372372
* {
373-
* Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
373+
* Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
374374
* Parameters: [
375375
* {
376-
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
376+
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
377377
* ParameterValue: "STRING_VALUE", // required
378378
* },
379379
* ],
@@ -414,10 +414,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
414414
* Enabled: true || false,
415415
* Processors: [
416416
* {
417-
* Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
417+
* Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
418418
* Parameters: [
419419
* {
420-
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
420+
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
421421
* ParameterValue: "STRING_VALUE", // required
422422
* },
423423
* ],
@@ -452,10 +452,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
452452
* Enabled: true || false,
453453
* Processors: [
454454
* {
455-
* Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
455+
* Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
456456
* Parameters: [
457457
* {
458-
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
458+
* ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
459459
* ParameterValue: "STRING_VALUE", // required
460460
* },
461461
* ],
@@ -523,6 +523,14 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
523523
* ],
524524
* },
525525
* },
526+
* MSKSourceConfiguration: { // MSKSourceConfiguration
527+
* MSKClusterARN: "STRING_VALUE", // required
528+
* TopicName: "STRING_VALUE", // required
529+
* AuthenticationConfiguration: { // AuthenticationConfiguration
530+
* RoleARN: "STRING_VALUE", // required
531+
* Connectivity: "PUBLIC" || "PRIVATE", // required
532+
* },
533+
* },
526534
* };
527535
* const command = new CreateDeliveryStreamCommand(input);
528536
* const response = await client.send(command);

‎clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
7979
* // Details: "STRING_VALUE", // required
8080
* // },
8181
* // },
82-
* // DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource", // required
82+
* // DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource", // required
8383
* // VersionId: "STRING_VALUE", // required
8484
* // CreateTimestamp: new Date("TIMESTAMP"),
8585
* // LastUpdateTimestamp: new Date("TIMESTAMP"),
@@ -89,6 +89,15 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
8989
* // RoleARN: "STRING_VALUE",
9090
* // DeliveryStartTimestamp: new Date("TIMESTAMP"),
9191
* // },
92+
* // MSKSourceDescription: { // MSKSourceDescription
93+
* // MSKClusterARN: "STRING_VALUE",
94+
* // TopicName: "STRING_VALUE",
95+
* // AuthenticationConfiguration: { // AuthenticationConfiguration
96+
* // RoleARN: "STRING_VALUE", // required
97+
* // Connectivity: "PUBLIC" || "PRIVATE", // required
98+
* // },
99+
* // DeliveryStartTimestamp: new Date("TIMESTAMP"),
100+
* // },
92101
* // },
93102
* // Destinations: [ // DestinationDescriptionList // required
94103
* // { // DestinationDescription
@@ -140,10 +149,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
140149
* // Enabled: true || false,
141150
* // Processors: [ // ProcessorList
142151
* // { // Processor
143-
* // Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
152+
* // Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
144153
* // Parameters: [ // ProcessorParameterList
145154
* // { // ProcessorParameter
146-
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
155+
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
147156
* // ParameterValue: "STRING_VALUE", // required
148157
* // },
149158
* // ],
@@ -271,10 +280,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
271280
* // Enabled: true || false,
272281
* // Processors: [
273282
* // {
274-
* // Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
283+
* // Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
275284
* // Parameters: [
276285
* // {
277-
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
286+
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
278287
* // ParameterValue: "STRING_VALUE", // required
279288
* // },
280289
* // ],
@@ -335,10 +344,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
335344
* // Enabled: true || false,
336345
* // Processors: [
337346
* // {
338-
* // Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
347+
* // Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
339348
* // Parameters: [
340349
* // {
341-
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
350+
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
342351
* // ParameterValue: "STRING_VALUE", // required
343352
* // },
344353
* // ],
@@ -380,10 +389,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
380389
* // Enabled: true || false,
381390
* // Processors: [
382391
* // {
383-
* // Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
392+
* // Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
384393
* // Parameters: [
385394
* // {
386-
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
395+
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
387396
* // ParameterValue: "STRING_VALUE", // required
388397
* // },
389398
* // ],
@@ -419,10 +428,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
419428
* // Enabled: true || false,
420429
* // Processors: [
421430
* // {
422-
* // Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
431+
* // Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
423432
* // Parameters: [
424433
* // {
425-
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
434+
* // ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
426435
* // ParameterValue: "STRING_VALUE", // required
427436
* // },
428437
* // ],

‎clients/client-firehose/src/commands/ListDeliveryStreamsCommand.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ export interface ListDeliveryStreamsCommandOutput extends ListDeliveryStreamsOut
5353
* const client = new FirehoseClient(config);
5454
* const input = { // ListDeliveryStreamsInput
5555
* Limit: Number("int"),
56-
* DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource",
56+
* DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource",
5757
* ExclusiveStartDeliveryStreamName: "STRING_VALUE",
5858
* };
5959
* const command = new ListDeliveryStreamsCommand(input);

‎clients/client-firehose/src/commands/PutRecordBatchCommand.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ export interface PutRecordBatchCommandOutput extends PutRecordBatchOutput, __Met
4141
* achieve higher throughput per producer than when writing single records. To write single
4242
* data records into a delivery stream, use <a>PutRecord</a>. Applications using
4343
* these operations are referred to as producers.</p>
44+
* <p>Kinesis Data Firehose accumulates and publishes a particular metric for a customer account in one minute intervals. It is possible that the bursts of incoming bytes/records ingested to a delivery stream last only for a few seconds. Due to this, the actual spikes in the traffic might not be fully visible in the customer's 1 minute CloudWatch metrics.</p>
4445
* <p>For information about service quota, see <a href="https://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon Kinesis Data Firehose
4546
* Quota</a>.</p>
4647
* <p>Each <a>PutRecordBatch</a> request supports up to 500 records. Each record
@@ -77,8 +78,11 @@ export interface PutRecordBatchCommandOutput extends PutRecordBatchOutput, __Met
7778
* duplicate records and also reduces the total bytes sent (and corresponding charges). We
7879
* recommend that you handle any duplicates at the destination.</p>
7980
* <p>If <a>PutRecordBatch</a> throws <code>ServiceUnavailableException</code>,
80-
* back off and retry. If the exception persists, it is possible that the throughput limits
81-
* have been exceeded for the delivery stream.</p>
81+
* the API is automatically reinvoked (retried) 3 times. If the exception persists, it is
82+
* possible that the throughput limits have been exceeded for the delivery stream.</p>
83+
* <p>Re-invoking the Put API operations (for example, PutRecord and PutRecordBatch) can
84+
* result in data duplicates. For larger data assets, allow for a longer time out before
85+
* retrying Put API operations.</p>
8286
* <p>Data records sent to Kinesis Data Firehose are stored for 24 hours from the time they
8387
* are added to a delivery stream as it attempts to send the records to the destination. If
8488
* the destination is unreachable for more than 24 hours, the data is no longer

‎clients/client-firehose/src/commands/PutRecordCommand.ts

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ export interface PutRecordCommandOutput extends PutRecordOutput, __MetadataBeare
4646
* operations for each delivery stream. For more information about limits and how to request
4747
* an increase, see <a href="https://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
4848
* Kinesis Data Firehose Limits</a>. </p>
49+
* <p>Kinesis Data Firehose accumulates and publishes a particular metric for a customer account in one minute intervals. It is possible that the bursts of incoming bytes/records ingested to a delivery stream last only for a few seconds. Due to this, the actual spikes in the traffic might not be fully visible in the customer's 1 minute CloudWatch metrics.</p>
4950
* <p>You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. The data record consists of a data blob that can be up to 1,000
5051
* KiB in size, and any kind of data. For example, it can be a segment from a log file,
5152
* geographic location data, website clickstream data, and so on.</p>
@@ -58,8 +59,12 @@ export interface PutRecordCommandOutput extends PutRecordOutput, __MetadataBeare
5859
* unique string assigned to each record. Producer applications can use this ID for purposes
5960
* such as auditability and investigation.</p>
6061
* <p>If the <code>PutRecord</code> operation throws a
61-
* <code>ServiceUnavailableException</code>, back off and retry. If the exception persists,
62-
* it is possible that the throughput limits have been exceeded for the delivery stream. </p>
62+
* <code>ServiceUnavailableException</code>, the API is automatically reinvoked (retried) 3
63+
* times. If the exception persists, it is possible that the throughput limits have been
64+
* exceeded for the delivery stream. </p>
65+
* <p>Re-invoking the Put API operations (for example, PutRecord and PutRecordBatch) can
66+
* result in data duplicates. For larger data assets, allow for a longer time out before
67+
* retrying Put API operations.</p>
6368
* <p>Data records sent to Kinesis Data Firehose are stored for 24 hours from the time they
6469
* are added to a delivery stream as it tries to send the records to the destination. If the
6570
* destination is unreachable for more than 24 hours, the data is no longer

‎clients/client-firehose/src/commands/StartDeliveryStreamEncryptionCommand.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ export interface StartDeliveryStreamEncryptionCommandOutput
6262
* CMK is of type <code>CUSTOMER_MANAGED_CMK</code>, Kinesis Data Firehose creates a grant
6363
* that enables it to use the new CMK to encrypt and decrypt data and to manage the
6464
* grant.</p>
65+
* <p>For the KMS grant creation to be successful, Kinesis Data Firehose APIs <code>StartDeliveryStreamEncryption</code> and <code>CreateDeliveryStream</code> should not be called with session credentials that are more than 6 hours old.</p>
6566
* <p>If a delivery stream already has encryption enabled and then you invoke this operation
6667
* to change the ARN of the CMK or both its type and ARN and you get
6768
* <code>ENABLING_FAILED</code>, this only means that the attempt to change the CMK failed.

0 commit comments

Comments
 (0)