feat(aws-s3): create default bucket policy when required (under feature flag) (#20765)

Created a new feature flag `@aws-cdk/aws-s3:createDefaultLoggingPolicy`

There are certain types of S3 Buckets that AWS will automatically create a bucket policy for you if you do not create one. For example, if you create an S3 Bucket to be used as the destination for VPC Flow Logs and you  do not create a Bucket Policy, AWS will automatically create a bucket policy for you. The full list of resources can be found [here](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AWS-logs-and-resource-policy.html#AWS-logs-infrastructure-S3)

- [X] Vpc Flow Logs
~- [ ] AWS Network Firewall logs~ (No L2 support yet)
~- [ ] AWS Global Accelerator flow logs~ ([not currently possible](aws-cloudformation/cloudformation-coverage-roadmap#922)]
~- [ ] EC2 Spot Instance data feed~(no cloudformation support yet)
~- [ ] CloudFront access logs & streaming access logs~ (CloudFront uses bucket ACL _not_ bucket policy)
- [X] Network Load Balancer access logs (already done)
- [x] Amazon Managed Streaming for Apache Kafka broker logs

If we allow AWS to create these policies automatically, it prevents CDK from every managing that policy in the future. Since we know what the policy should be we should instead create the logging bucket with the required policy.

fixes #18816


----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [x] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: corymhall

packages/@aws-cdk/aws-ec2/README.md

@@ -1356,6 +1356,20 @@ new ec2.FlowLog(this, 'FlowLogWithKeyPrefix', {
 });
 ```
 
+When the S3 destination is configured, AWS will automatically create an S3 bucket policy
+that allows the service to write logs to the bucket. This makes it impossible to later update
+that bucket policy. To have CDK create the bucket policy so that future updates can be made,
+the `@aws-cdk/aws-s3:createDefaultLoggingPolicy` [feature flag](https://docs.aws.amazon.com/cdk/v2/guide/featureflags.html) can be used. This can be set
+in the `cdk.json` file.
+
+```json
+{
+  "context": {
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true
+  }
+}
+```
+
 ## User Data
 
 User data enables you to run a script when your instances start up.  In order to configure these scripts you can add commands directly to the script

packages/@aws-cdk/aws-ec2/lib/vpc-flow-logs.ts

@@ -1,15 +1,14 @@
 import * as iam from '@aws-cdk/aws-iam';
 import * as logs from '@aws-cdk/aws-logs';
 import * as s3 from '@aws-cdk/aws-s3';
-import { IResource, PhysicalName, RemovalPolicy, Resource } from '@aws-cdk/core';
+import { IResource, PhysicalName, RemovalPolicy, Resource, FeatureFlags, Stack } from '@aws-cdk/core';
+import { S3_CREATE_DEFAULT_LOGGING_POLICY } from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
 import { CfnFlowLog } from './ec2.generated';
 import { ISubnet, IVpc } from './vpc';
 
 /**
  * A FlowLog
- *
- *
  */
 export interface IFlowLog extends IResource {
   /**
@@ -22,8 +21,6 @@ export interface IFlowLog extends IResource {
 
 /**
  * The type of VPC traffic to log
- *
- *
  */
 export enum FlowLogTrafficType {
   /**
@@ -44,7 +41,6 @@ export enum FlowLogTrafficType {
 
 /**
  * The available destination types for Flow Logs
- *
  */
 export enum FlowLogDestinationType {
   /**
@@ -60,8 +56,6 @@ export enum FlowLogDestinationType {
 
 /**
  * The type of resource to create the flow log for
- *
- *
  */
 export abstract class FlowLogResourceType {
   /**
@@ -106,9 +100,29 @@ export abstract class FlowLogResourceType {
 }
 
 /**
- * The destination type for the flow log
- *
+ * Options for writing logs to a S3 destination
+ */
+export interface S3DestinationOptions {
+  /**
+   * Use Hive-compatible prefixes for flow logs
+   * stored in Amazon S3
+   *
+   * @default false
+   */
+  readonly hiveCompatiblePartitions?: boolean;
+}
+
+/**
+ * Options for writing logs to a destination
  *
+ * TODO: there are other destination options, currently they are
+ * only for s3 destinations (not sure if that will change)
+ */
+export interface DestinationOptions extends S3DestinationOptions { }
+
+
+/**
+ * The destination type for the flow log
  */
 export abstract class FlowLogDestination {
   /**
@@ -124,12 +138,18 @@ export abstract class FlowLogDestination {
 
   /**
    * Use S3 as the destination
+   *
+   * @param bucket optional s3 bucket to publish logs to. If one is not provided
+   * a default bucket will be created
+   * @param keyPrefix optional prefix within the bucket to write logs to
+   * @param options additional s3 destination options
    */
-  public static toS3(bucket?: s3.IBucket, keyPrefix?: string): FlowLogDestination {
+  public static toS3(bucket?: s3.IBucket, keyPrefix?: string, options?: S3DestinationOptions): FlowLogDestination {
     return new S3Destination({
       logDestinationType: FlowLogDestinationType.S3,
       s3Bucket: bucket,
       keyPrefix,
+      destinationOptions: options,
     });
   }
 
@@ -141,8 +161,6 @@ export abstract class FlowLogDestination {
 
 /**
  * Flow Log Destination configuration
- *
- *
  */
 export interface FlowLogDestinationConfig {
   /**
@@ -179,6 +197,13 @@ export interface FlowLogDestinationConfig {
    * @default - undefined
    */
   readonly keyPrefix?: string;
+
+  /**
+   * Options for writing flow logs to a supported destination
+   *
+   * @default - undefined
+   */
+  readonly destinationOptions?: DestinationOptions;
 }
 
 /**
@@ -196,13 +221,73 @@ class S3Destination extends FlowLogDestination {
         encryption: s3.BucketEncryption.UNENCRYPTED,
         removalPolicy: RemovalPolicy.RETAIN,
       });
+
     } else {
       s3Bucket = this.props.s3Bucket;
     }
+
+    // https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-s3.html#flow-logs-s3-permissions
+    if (FeatureFlags.of(scope).isEnabled(S3_CREATE_DEFAULT_LOGGING_POLICY)) {
+      const stack = Stack.of(scope);
+      let keyPrefix = this.props.keyPrefix ?? '';
+      if (keyPrefix && !keyPrefix.endsWith('/')) {
+        keyPrefix = keyPrefix + '/';
+      }
+      const prefix = this.props.destinationOptions?.hiveCompatiblePartitions
+        ? s3Bucket.arnForObjects(`${keyPrefix}AWSLogs/aws-account-id=${stack.account}/*`)
+        : s3Bucket.arnForObjects(`${keyPrefix}AWSLogs/${stack.account}/*`);
+
+      s3Bucket.addToResourcePolicy(new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        principals: [
+          new iam.ServicePrincipal('delivery.logs.amazonaws.com'),
+        ],
+        resources: [
+          prefix,
+        ],
+        actions: ['s3:PutObject'],
+        conditions: {
+          StringEquals: {
+            's3:x-amz-acl': 'bucket-owner-full-control',
+            'aws:SourceAccount': stack.account,
+          },
+          ArnLike: {
+            'aws:SourceArn': stack.formatArn({
+              service: 'logs',
+              resource: '*',
+            }),
+          },
+        },
+      }));
+
+      s3Bucket.addToResourcePolicy(new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        principals: [
+          new iam.ServicePrincipal('delivery.logs.amazonaws.com'),
+        ],
+        resources: [s3Bucket.bucketArn],
+        actions: [
+          's3:GetBucketAcl',
+          's3:ListBucket',
+        ],
+        conditions: {
+          StringEquals: {
+            'aws:SourceAccount': stack.account,
+          },
+          ArnLike: {
+            'aws:SourceArn': stack.formatArn({
+              service: 'logs',
+              resource: '*',
+            }),
+          },
+        },
+      }));
+    }
     return {
       logDestinationType: FlowLogDestinationType.S3,
       s3Bucket,
       keyPrefix: this.props.keyPrefix,
+      destinationOptions: this.props.destinationOptions,
     };
   }
 }
@@ -263,8 +348,6 @@ class CloudWatchLogsDestination extends FlowLogDestination {
 
 /**
  * Options to add a flow log to a VPC
- *
- *
  */
 export interface FlowLogOptions {
   /**
@@ -285,8 +368,6 @@ export interface FlowLogOptions {
 
 /**
  * Properties of a VPC Flow Log
- *
- *
  */
 export interface FlowLogProps extends FlowLogOptions {
   /**
@@ -307,8 +388,6 @@ export interface FlowLogProps extends FlowLogOptions {
 
 /**
  * The base class for a Flow Log
- *
- *
  */
 abstract class FlowLogBase extends Resource implements IFlowLog {
   /**
@@ -322,8 +401,6 @@ abstract class FlowLogBase extends Resource implements IFlowLog {
 /**
  * A VPC flow log.
  * @resource AWS::EC2::FlowLog
- *
- *
  */
 export class FlowLog extends FlowLogBase {
   /**
@@ -383,6 +460,7 @@ export class FlowLog extends FlowLogBase {
     }
 
     const flowLog = new CfnFlowLog(this, 'FlowLog', {
+      destinationOptions: destinationConfig.destinationOptions,
       deliverLogsPermissionArn: this.iamRole ? this.iamRole.roleArn : undefined,
       logDestinationType: destinationConfig.logDestinationType,
       logGroupName: this.logGroup ? this.logGroup.logGroupName : undefined,

packages/@aws-cdk/aws-ec2/package.json

@@ -83,6 +83,7 @@
     "@aws-cdk/assertions": "0.0.0",
     "@aws-cdk/cdk-build-tools": "0.0.0",
     "@aws-cdk/integ-runner": "0.0.0",
+    "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/cfn2ts": "0.0.0",
     "@aws-cdk/cloud-assembly-schema": "0.0.0",
     "@aws-cdk/cx-api": "0.0.0",

packages/@aws-cdk/aws-ec2/test/integ.vpc-flow-logs.ts

@@ -1,11 +1,35 @@
-/// !cdk-integ *
 import { PolicyStatement, Effect, ServicePrincipal } from '@aws-cdk/aws-iam';
 import * as s3 from '@aws-cdk/aws-s3';
 import { App, RemovalPolicy, Stack, StackProps } from '@aws-cdk/core';
-import { FlowLog, FlowLogDestination, FlowLogResourceType, Vpc } from '../lib';
+import { IntegTest, ExpectedResult, AssertionsProvider } from '@aws-cdk/integ-tests';
+import { FlowLog, FlowLogDestination, FlowLogResourceType, Vpc, Instance, InstanceType, InstanceClass, InstanceSize, MachineImage, AmazonLinuxGeneration } from '../lib';
 
 const app = new App();
 
+class FeatureFlagStack extends Stack {
+  public readonly bucketArn: string;
+  public readonly bucket: s3.IBucket;
+  constructor(scope: App, id: string, props?: StackProps) {
+    super(scope, id, props);
+
+    const vpc = new Vpc(this, 'VPC');
+
+    const flowLog = vpc.addFlowLog('FlowLogsS3', {
+      destination: FlowLogDestination.toS3(),
+    });
+    this.bucket = flowLog.bucket!;
+    this.bucketArn = this.exportValue(flowLog.bucket!.bucketArn);
+
+    new Instance(this, 'FlowLogsInstance', {
+      vpc,
+      instanceType: InstanceType.of(InstanceClass.T3, InstanceSize.SMALL),
+      machineImage: MachineImage.latestAmazonLinux({
+        generation: AmazonLinuxGeneration.AMAZON_LINUX_2,
+      }),
+    });
+  }
+}
+
 class TestStack extends Stack {
   constructor(scope: App, id: string, props?: StackProps) {
     super(scope, id, props);
@@ -66,6 +90,27 @@ class TestStack extends Stack {
   }
 }
 
-new TestStack(app, 'FlowLogsTestStack');
+const featureFlagTest = new FeatureFlagStack(app, 'FlowLogsFeatureFlag');
+
+const integ = new IntegTest(app, 'FlowLogs', {
+  testCases: [
+    new TestStack(app, 'FlowLogsTestStack'),
+    featureFlagTest,
+  ],
+});
+
+
+const objects = integ.assertions.awsApiCall('S3', 'listObjectsV2', {
+  Bucket: featureFlagTest.bucket.bucketName,
+  MaxKeys: 1,
+  Prefix: `AWSLogs/${featureFlagTest.account}/vpcflowlogs`,
+});
+const assertionProvider = objects.node.tryFindChild('SdkProvider') as AssertionsProvider;
+assertionProvider.addPolicyStatementFromSdkCall('s3', 'ListBucket', [featureFlagTest.bucketArn]);
+assertionProvider.addPolicyStatementFromSdkCall('s3', 'GetObject', [`${featureFlagTest.bucketArn}/*`]);
+
+objects.expect(ExpectedResult.objectLike({
+  KeyCount: 1,
+}));
 
 app.synth();