Skip to content

Commit 6f03c03

Browse files
author
AWS
committed
Amazon Kinesis Firehose Update: Allow support for Snowflake as a Kinesis Data Firehose delivery destination.
1 parent 6c8b424 commit 6f03c03

File tree

2 files changed

+348
-1
lines changed

2 files changed

+348
-1
lines changed

‎.changes/next-release/feature-AmazonKinesisFirehose-be001cd.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "Amazon Kinesis Firehose",
4+
"contributor": "",
5+
"description": "Allow support for Snowflake as a Kinesis Data Firehose delivery destination."
6+
}

‎services/firehose/src/main/resources/codegen-resources/service-2.json

Lines changed: 342 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -783,7 +783,11 @@
783783
"shape":"AmazonOpenSearchServerlessDestinationConfiguration",
784784
"documentation":"<p>The destination in the Serverless offering for Amazon OpenSearch Service. You can specify only one destination.</p>"
785785
},
786-
"MSKSourceConfiguration":{"shape":"MSKSourceConfiguration"}
786+
"MSKSourceConfiguration":{"shape":"MSKSourceConfiguration"},
787+
"SnowflakeDestinationConfiguration":{
788+
"shape":"SnowflakeDestinationConfiguration",
789+
"documentation":"<p>Configure Snowflake destination</p>"
790+
}
787791
}
788792
},
789793
"CreateDeliveryStreamOutput":{
@@ -1115,6 +1119,10 @@
11151119
"shape":"HttpEndpointDestinationDescription",
11161120
"documentation":"<p>Describes the specified HTTP endpoint destination.</p>"
11171121
},
1122+
"SnowflakeDestinationDescription":{
1123+
"shape":"SnowflakeDestinationDescription",
1124+
"documentation":"<p>Optional description for the destination</p>"
1125+
},
11181126
"AmazonOpenSearchServerlessDestinationDescription":{
11191127
"shape":"AmazonOpenSearchServerlessDestinationDescription",
11201128
"documentation":"<p>The destination in the Serverless offering for Amazon OpenSearch Service.</p>"
@@ -3013,6 +3021,335 @@
30133021
"max":128,
30143022
"min":1
30153023
},
3024+
"SnowflakeAccountUrl":{
3025+
"type":"string",
3026+
"max":2048,
3027+
"min":24,
3028+
"pattern":".+?\\.snowflakecomputing\\.com",
3029+
"sensitive":true
3030+
},
3031+
"SnowflakeContentColumnName":{
3032+
"type":"string",
3033+
"max":255,
3034+
"min":1,
3035+
"sensitive":true
3036+
},
3037+
"SnowflakeDataLoadingOption":{
3038+
"type":"string",
3039+
"enum":[
3040+
"JSON_MAPPING",
3041+
"VARIANT_CONTENT_MAPPING",
3042+
"VARIANT_CONTENT_AND_METADATA_MAPPING"
3043+
]
3044+
},
3045+
"SnowflakeDatabase":{
3046+
"type":"string",
3047+
"max":255,
3048+
"min":1,
3049+
"sensitive":true
3050+
},
3051+
"SnowflakeDestinationConfiguration":{
3052+
"type":"structure",
3053+
"required":[
3054+
"AccountUrl",
3055+
"PrivateKey",
3056+
"User",
3057+
"Database",
3058+
"Schema",
3059+
"Table",
3060+
"RoleARN",
3061+
"S3Configuration"
3062+
],
3063+
"members":{
3064+
"AccountUrl":{
3065+
"shape":"SnowflakeAccountUrl",
3066+
"documentation":"<p>URL for accessing your Snowflake account. This URL must include your <a href=\"https://docs.snowflake.com/en/user-guide/admin-account-identifier\">account identifier</a>. Note that the protocol (https://) and port number are optional.</p>"
3067+
},
3068+
"PrivateKey":{
3069+
"shape":"SnowflakePrivateKey",
3070+
"documentation":"<p>The private key used to encrypt your Snowflake client. For information, see <a href=\"https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-configuration#using-key-pair-authentication-key-rotation\">Using Key Pair Authentication &amp; Key Rotation</a>.</p>"
3071+
},
3072+
"KeyPassphrase":{
3073+
"shape":"SnowflakeKeyPassphrase",
3074+
"documentation":"<p>Passphrase to decrypt the private key when the key is encrypted. For information, see <a href=\"https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-configuration#using-key-pair-authentication-key-rotation\">Using Key Pair Authentication &amp; Key Rotation</a>.</p>"
3075+
},
3076+
"User":{
3077+
"shape":"SnowflakeUser",
3078+
"documentation":"<p>User login name for the Snowflake account.</p>"
3079+
},
3080+
"Database":{
3081+
"shape":"SnowflakeDatabase",
3082+
"documentation":"<p>All data in Snowflake is maintained in databases.</p>"
3083+
},
3084+
"Schema":{
3085+
"shape":"SnowflakeSchema",
3086+
"documentation":"<p>Each database consists of one or more schemas, which are logical groupings of database objects, such as tables and views</p>"
3087+
},
3088+
"Table":{
3089+
"shape":"SnowflakeTable",
3090+
"documentation":"<p>All data in Snowflake is stored in database tables, logically structured as collections of columns and rows.</p>"
3091+
},
3092+
"SnowflakeRoleConfiguration":{
3093+
"shape":"SnowflakeRoleConfiguration",
3094+
"documentation":"<p>Optionally configure a Snowflake role. Otherwise the default user role will be used.</p>"
3095+
},
3096+
"DataLoadingOption":{
3097+
"shape":"SnowflakeDataLoadingOption",
3098+
"documentation":"<p>Choose to load JSON keys mapped to table column names or choose to split the JSON payload where content is mapped to a record content column and source metadata is mapped to a record metadata column.</p>"
3099+
},
3100+
"MetaDataColumnName":{
3101+
"shape":"SnowflakeMetaDataColumnName",
3102+
"documentation":"<p>The name of the record metadata column</p>"
3103+
},
3104+
"ContentColumnName":{
3105+
"shape":"SnowflakeContentColumnName",
3106+
"documentation":"<p>The name of the record content column</p>"
3107+
},
3108+
"SnowflakeVpcConfiguration":{
3109+
"shape":"SnowflakeVpcConfiguration",
3110+
"documentation":"<p>The VPCE ID for Firehose to privately connect with Snowflake. The ID format is com.amazonaws.vpce.[region].vpce-svc-&lt;[id]&gt;. For more information, see <a href=\"https://docs.snowflake.com/en/user-guide/admin-security-privatelink\">Amazon PrivateLink &amp; Snowflake</a> </p>"
3111+
},
3112+
"CloudWatchLoggingOptions":{"shape":"CloudWatchLoggingOptions"},
3113+
"ProcessingConfiguration":{"shape":"ProcessingConfiguration"},
3114+
"RoleARN":{
3115+
"shape":"RoleARN",
3116+
"documentation":"<p>The Amazon Resource Name (ARN) of the Snowflake role</p>"
3117+
},
3118+
"RetryOptions":{
3119+
"shape":"SnowflakeRetryOptions",
3120+
"documentation":"<p>The time period where Kinesis Data Firehose will retry sending data to the chosen HTTP endpoint.</p>"
3121+
},
3122+
"S3BackupMode":{
3123+
"shape":"SnowflakeS3BackupMode",
3124+
"documentation":"<p>Choose an S3 backup mode</p>"
3125+
},
3126+
"S3Configuration":{"shape":"S3DestinationConfiguration"}
3127+
},
3128+
"documentation":"<p>Configure Snowflake destination</p>"
3129+
},
3130+
"SnowflakeDestinationDescription":{
3131+
"type":"structure",
3132+
"members":{
3133+
"AccountUrl":{
3134+
"shape":"SnowflakeAccountUrl",
3135+
"documentation":"<p>URL for accessing your Snowflake account. This URL must include your <a href=\"https://docs.snowflake.com/en/user-guide/admin-account-identifier\">account identifier</a>. Note that the protocol (https://) and port number are optional.</p>"
3136+
},
3137+
"User":{
3138+
"shape":"SnowflakeUser",
3139+
"documentation":"<p>User login name for the Snowflake account.</p>"
3140+
},
3141+
"Database":{
3142+
"shape":"SnowflakeDatabase",
3143+
"documentation":"<p>All data in Snowflake is maintained in databases.</p>"
3144+
},
3145+
"Schema":{
3146+
"shape":"SnowflakeSchema",
3147+
"documentation":"<p>Each database consists of one or more schemas, which are logical groupings of database objects, such as tables and views</p>"
3148+
},
3149+
"Table":{
3150+
"shape":"SnowflakeTable",
3151+
"documentation":"<p>All data in Snowflake is stored in database tables, logically structured as collections of columns and rows.</p>"
3152+
},
3153+
"SnowflakeRoleConfiguration":{
3154+
"shape":"SnowflakeRoleConfiguration",
3155+
"documentation":"<p>Optionally configure a Snowflake role. Otherwise the default user role will be used.</p>"
3156+
},
3157+
"DataLoadingOption":{
3158+
"shape":"SnowflakeDataLoadingOption",
3159+
"documentation":"<p>Choose to load JSON keys mapped to table column names or choose to split the JSON payload where content is mapped to a record content column and source metadata is mapped to a record metadata column.</p>"
3160+
},
3161+
"MetaDataColumnName":{
3162+
"shape":"SnowflakeMetaDataColumnName",
3163+
"documentation":"<p>The name of the record metadata column</p>"
3164+
},
3165+
"ContentColumnName":{
3166+
"shape":"SnowflakeContentColumnName",
3167+
"documentation":"<p>The name of the record content column</p>"
3168+
},
3169+
"SnowflakeVpcConfiguration":{
3170+
"shape":"SnowflakeVpcConfiguration",
3171+
"documentation":"<p>The VPCE ID for Firehose to privately connect with Snowflake. The ID format is com.amazonaws.vpce.[region].vpce-svc-&lt;[id]&gt;. For more information, see <a href=\"https://docs.snowflake.com/en/user-guide/admin-security-privatelink\">Amazon PrivateLink &amp; Snowflake</a> </p>"
3172+
},
3173+
"CloudWatchLoggingOptions":{"shape":"CloudWatchLoggingOptions"},
3174+
"ProcessingConfiguration":{"shape":"ProcessingConfiguration"},
3175+
"RoleARN":{
3176+
"shape":"RoleARN",
3177+
"documentation":"<p>The Amazon Resource Name (ARN) of the Snowflake role</p>"
3178+
},
3179+
"RetryOptions":{
3180+
"shape":"SnowflakeRetryOptions",
3181+
"documentation":"<p>The time period where Kinesis Data Firehose will retry sending data to the chosen HTTP endpoint.</p>"
3182+
},
3183+
"S3BackupMode":{
3184+
"shape":"SnowflakeS3BackupMode",
3185+
"documentation":"<p>Choose an S3 backup mode</p>"
3186+
},
3187+
"S3DestinationDescription":{"shape":"S3DestinationDescription"}
3188+
},
3189+
"documentation":"<p>Optional Snowflake destination description</p>"
3190+
},
3191+
"SnowflakeDestinationUpdate":{
3192+
"type":"structure",
3193+
"members":{
3194+
"AccountUrl":{
3195+
"shape":"SnowflakeAccountUrl",
3196+
"documentation":"<p>URL for accessing your Snowflake account. This URL must include your <a href=\"https://docs.snowflake.com/en/user-guide/admin-account-identifier\">account identifier</a>. Note that the protocol (https://) and port number are optional.</p>"
3197+
},
3198+
"PrivateKey":{
3199+
"shape":"SnowflakePrivateKey",
3200+
"documentation":"<p>The private key used to encrypt your Snowflake client. For information, see <a href=\"https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-configuration#using-key-pair-authentication-key-rotation\">Using Key Pair Authentication &amp; Key Rotation</a>.</p>"
3201+
},
3202+
"KeyPassphrase":{
3203+
"shape":"SnowflakeKeyPassphrase",
3204+
"documentation":"<p>Passphrase to decrypt the private key when the key is encrypted. For information, see <a href=\"https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-configuration#using-key-pair-authentication-key-rotation\">Using Key Pair Authentication &amp; Key Rotation</a>.</p>"
3205+
},
3206+
"User":{
3207+
"shape":"SnowflakeUser",
3208+
"documentation":"<p>User login name for the Snowflake account.</p>"
3209+
},
3210+
"Database":{
3211+
"shape":"SnowflakeDatabase",
3212+
"documentation":"<p>All data in Snowflake is maintained in databases.</p>"
3213+
},
3214+
"Schema":{
3215+
"shape":"SnowflakeSchema",
3216+
"documentation":"<p>Each database consists of one or more schemas, which are logical groupings of database objects, such as tables and views</p>"
3217+
},
3218+
"Table":{
3219+
"shape":"SnowflakeTable",
3220+
"documentation":"<p>All data in Snowflake is stored in database tables, logically structured as collections of columns and rows.</p>"
3221+
},
3222+
"SnowflakeRoleConfiguration":{
3223+
"shape":"SnowflakeRoleConfiguration",
3224+
"documentation":"<p>Optionally configure a Snowflake role. Otherwise the default user role will be used.</p>"
3225+
},
3226+
"DataLoadingOption":{
3227+
"shape":"SnowflakeDataLoadingOption",
3228+
"documentation":"<p> JSON keys mapped to table column names or choose to split the JSON payload where content is mapped to a record content column and source metadata is mapped to a record metadata column.</p>"
3229+
},
3230+
"MetaDataColumnName":{
3231+
"shape":"SnowflakeMetaDataColumnName",
3232+
"documentation":"<p>The name of the record metadata column</p>"
3233+
},
3234+
"ContentColumnName":{
3235+
"shape":"SnowflakeContentColumnName",
3236+
"documentation":"<p>The name of the content metadata column</p>"
3237+
},
3238+
"CloudWatchLoggingOptions":{"shape":"CloudWatchLoggingOptions"},
3239+
"ProcessingConfiguration":{"shape":"ProcessingConfiguration"},
3240+
"RoleARN":{
3241+
"shape":"RoleARN",
3242+
"documentation":"<p>The Amazon Resource Name (ARN) of the Snowflake role</p>"
3243+
},
3244+
"RetryOptions":{
3245+
"shape":"SnowflakeRetryOptions",
3246+
"documentation":"<p>Specify how long Kinesis Data Firehose retries sending data to the New Relic HTTP endpoint. After sending data, Kinesis Data Firehose first waits for an acknowledgment from the HTTP endpoint. If an error occurs or the acknowledgment doesn’t arrive within the acknowledgment timeout period, Kinesis Data Firehose starts the retry duration counter. It keeps retrying until the retry duration expires. After that, Kinesis Data Firehose considers it a data delivery failure and backs up the data to your Amazon S3 bucket. Every time that Kinesis Data Firehose sends data to the HTTP endpoint (either the initial attempt or a retry), it restarts the acknowledgement timeout counter and waits for an acknowledgement from the HTTP endpoint. Even if the retry duration expires, Kinesis Data Firehose still waits for the acknowledgment until it receives it or the acknowledgement timeout period is reached. If the acknowledgment times out, Kinesis Data Firehose determines whether there's time left in the retry counter. If there is time left, it retries again and repeats the logic until it receives an acknowledgment or determines that the retry time has expired. If you don't want Kinesis Data Firehose to retry sending data, set this value to 0.</p>"
3247+
},
3248+
"S3BackupMode":{
3249+
"shape":"SnowflakeS3BackupMode",
3250+
"documentation":"<p>Choose an S3 backup mode</p>"
3251+
},
3252+
"S3Update":{"shape":"S3DestinationUpdate"}
3253+
},
3254+
"documentation":"<p>Update to configuration settings</p>"
3255+
},
3256+
"SnowflakeKeyPassphrase":{
3257+
"type":"string",
3258+
"max":255,
3259+
"min":7,
3260+
"sensitive":true
3261+
},
3262+
"SnowflakeMetaDataColumnName":{
3263+
"type":"string",
3264+
"max":255,
3265+
"min":1,
3266+
"sensitive":true
3267+
},
3268+
"SnowflakePrivateKey":{
3269+
"type":"string",
3270+
"max":4096,
3271+
"min":256,
3272+
"pattern":"^(?:[A-Za-z0-9+\\/]{4})*(?:[A-Za-z0-9+\\/]{2}==|[A-Za-z0-9+\\/]{3}=)?$",
3273+
"sensitive":true
3274+
},
3275+
"SnowflakePrivateLinkVpceId":{
3276+
"type":"string",
3277+
"max":255,
3278+
"min":47,
3279+
"pattern":"([a-zA-Z0-9\\-\\_]+\\.){2,3}vpce\\.[a-zA-Z0-9\\-]*\\.vpce-svc\\-[a-zA-Z0-9\\-]{17}$",
3280+
"sensitive":true
3281+
},
3282+
"SnowflakeRetryDurationInSeconds":{
3283+
"type":"integer",
3284+
"max":7200,
3285+
"min":0
3286+
},
3287+
"SnowflakeRetryOptions":{
3288+
"type":"structure",
3289+
"members":{
3290+
"DurationInSeconds":{
3291+
"shape":"SnowflakeRetryDurationInSeconds",
3292+
"documentation":"<p>the time period where Kinesis Data Firehose will retry sending data to the chosen HTTP endpoint.</p>"
3293+
}
3294+
},
3295+
"documentation":"<p>Specify how long Kinesis Data Firehose retries sending data to the New Relic HTTP endpoint. After sending data, Kinesis Data Firehose first waits for an acknowledgment from the HTTP endpoint. If an error occurs or the acknowledgment doesn’t arrive within the acknowledgment timeout period, Kinesis Data Firehose starts the retry duration counter. It keeps retrying until the retry duration expires. After that, Kinesis Data Firehose considers it a data delivery failure and backs up the data to your Amazon S3 bucket. Every time that Kinesis Data Firehose sends data to the HTTP endpoint (either the initial attempt or a retry), it restarts the acknowledgement timeout counter and waits for an acknowledgement from the HTTP endpoint. Even if the retry duration expires, Kinesis Data Firehose still waits for the acknowledgment until it receives it or the acknowledgement timeout period is reached. If the acknowledgment times out, Kinesis Data Firehose determines whether there's time left in the retry counter. If there is time left, it retries again and repeats the logic until it receives an acknowledgment or determines that the retry time has expired. If you don't want Kinesis Data Firehose to retry sending data, set this value to 0.</p>"
3296+
},
3297+
"SnowflakeRole":{
3298+
"type":"string",
3299+
"max":255,
3300+
"min":1,
3301+
"sensitive":true
3302+
},
3303+
"SnowflakeRoleConfiguration":{
3304+
"type":"structure",
3305+
"members":{
3306+
"Enabled":{
3307+
"shape":"BooleanObject",
3308+
"documentation":"<p>Enable Snowflake role</p>"
3309+
},
3310+
"SnowflakeRole":{
3311+
"shape":"SnowflakeRole",
3312+
"documentation":"<p>The Snowflake role you wish to configure</p>"
3313+
}
3314+
},
3315+
"documentation":"<p>Optionally configure a Snowflake role. Otherwise the default user role will be used.</p>"
3316+
},
3317+
"SnowflakeS3BackupMode":{
3318+
"type":"string",
3319+
"enum":[
3320+
"FailedDataOnly",
3321+
"AllData"
3322+
]
3323+
},
3324+
"SnowflakeSchema":{
3325+
"type":"string",
3326+
"max":255,
3327+
"min":1,
3328+
"sensitive":true
3329+
},
3330+
"SnowflakeTable":{
3331+
"type":"string",
3332+
"max":255,
3333+
"min":1,
3334+
"sensitive":true
3335+
},
3336+
"SnowflakeUser":{
3337+
"type":"string",
3338+
"max":255,
3339+
"min":1,
3340+
"sensitive":true
3341+
},
3342+
"SnowflakeVpcConfiguration":{
3343+
"type":"structure",
3344+
"required":["PrivateLinkVpceId"],
3345+
"members":{
3346+
"PrivateLinkVpceId":{
3347+
"shape":"SnowflakePrivateLinkVpceId",
3348+
"documentation":"<p>The VPCE ID for Firehose to privately connect with Snowflake. The ID format is com.amazonaws.vpce.[region].vpce-svc-&lt;[id]&gt;. For more information, see <a href=\"https://docs.snowflake.com/en/user-guide/admin-security-privatelink\">Amazon PrivateLink &amp; Snowflake</a> </p>"
3349+
}
3350+
},
3351+
"documentation":"<p>Configure a Snowflake VPC</p>"
3352+
},
30163353
"SourceDescription":{
30173354
"type":"structure",
30183355
"members":{
@@ -3399,6 +3736,10 @@
33993736
"AmazonOpenSearchServerlessDestinationUpdate":{
34003737
"shape":"AmazonOpenSearchServerlessDestinationUpdate",
34013738
"documentation":"<p>Describes an update for a destination in the Serverless offering for Amazon OpenSearch Service.</p>"
3739+
},
3740+
"SnowflakeDestinationUpdate":{
3741+
"shape":"SnowflakeDestinationUpdate",
3742+
"documentation":"<p>Update to the Snowflake destination condiguration settings</p>"
34023743
}
34033744
}
34043745
},

0 commit comments

Comments
 (0)