Turning off trailing checksums for SSE

.changes/next-release/bugfix-AmazonS3-22d85db.json

@@ -0,0 +1,5 @@
+{
+    "category": "Amazon S3", 
+    "type": "bugfix", 
+    "description": "Turns off trailing checksums when using SSE-C or SSE-KMS"
+}

services/s3/src/it/java/software/amazon/awssdk/services/s3/AsyncServerSideEncryptionIntegrationTest.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+package software.amazon.awssdk.services.s3;
+
+import static org.assertj.core.api.Fail.fail;
+import static software.amazon.awssdk.services.s3.model.ServerSideEncryption.AES256;
+import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.SecureRandom;
+import java.util.Base64;
+import java.util.UUID;
+import javax.crypto.KeyGenerator;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
+import software.amazon.awssdk.testutils.RandomTempFile;
+import software.amazon.awssdk.utils.Md5Utils;
+
+public class AsyncServerSideEncryptionIntegrationTest extends S3IntegrationTestBase {
+
+    private static final String BUCKET = temporaryBucketName(GetObjectIntegrationTest.class);
+
+    private static File file;
+
+    @BeforeClass
+    public static void setupFixture() throws IOException {
+        createBucket(BUCKET);
+        file = new RandomTempFile(10_000);
+    }
+
+    @AfterClass
+    public static void tearDownFixture() {
+        deleteBucketAndAllContents(BUCKET);
+        file.delete();
+    }
+
+    @Test
+    public void sse_AES256_succeeds() {
+        String key = UUID.randomUUID().toString();
+        PutObjectRequest request = PutObjectRequest.builder()
+                                                   .key(key)
+                                                   .bucket(BUCKET)
+                                                   .serverSideEncryption(AES256)
+                                                   .build();
+
+        s3Async.putObject(request, file.toPath()).join();
+
+        GetObjectRequest getObjectRequest = GetObjectRequest.builder()
+                                                            .key(key)
+                                                            .bucket(BUCKET)
+                                                            .build();
+
+        s3Async.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()).join();
+    }
+
+    @Test
+    public void sse_AWSKMS_succeeds() {
+        String key = UUID.randomUUID().toString();
+        PutObjectRequest request = PutObjectRequest.builder()
+                                                   .key(key)
+                                                   .bucket(BUCKET)
+                                                   .serverSideEncryption(ServerSideEncryption.AWS_KMS)
+                                                   .build();
+
+        s3Async.putObject(request, file.toPath()).join();
+
+        GetObjectRequest getObjectRequest = GetObjectRequest.builder()
+                                                            .key(key)
+                                                            .bucket(BUCKET)
+                                                            .build();
+
+        s3Async.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()).join();
+    }
+
+    @Test
+    public void sse_customerManaged_succeeds() {
+        String key = UUID.randomUUID().toString();
+        byte[] secretKey = generateSecretKey();
+        String b64Key = Base64.getEncoder().encodeToString(secretKey);
+        String b64KeyMd5 = Md5Utils.md5AsBase64(secretKey);
+
+        PutObjectRequest request = PutObjectRequest.builder()
+                                                   .key(key)
+                                                   .bucket(BUCKET)
+                                                   .sseCustomerKey(b64Key)
+                                                   .sseCustomerAlgorithm(AES256.name())
+                                                   .sseCustomerKeyMD5(b64KeyMd5)
+                                                   .build();
+
+        s3Async.putObject(request, file.toPath()).join();
+
+        GetObjectRequest getObjectRequest = GetObjectRequest.builder()
+                                                            .key(key)
+                                                            .bucket(BUCKET)
+                                                            .sseCustomerKey(b64Key)
+                                                            .sseCustomerAlgorithm(AES256.name())
+                                                            .sseCustomerKeyMD5(b64KeyMd5)
+                                                            .build();
+
+        s3Async.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()).join();
+    }
+
+    private static byte[] generateSecretKey() {
+        KeyGenerator generator;
+        try {
+            generator = KeyGenerator.getInstance("AES");
+            generator.init(256, new SecureRandom());
+            return generator.generateKey().getEncoded();
+        } catch (Exception e) {
+            fail("Unable to generate symmetric key: " + e.getMessage());
+            return null;
+        }
+    }    
+}

services/s3/src/it/java/software/amazon/awssdk/services/s3/SyncServerSideEncryptionIntegrationTest.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+package software.amazon.awssdk.services.s3;
+
+import static org.assertj.core.api.Fail.fail;
+import static software.amazon.awssdk.services.s3.model.ServerSideEncryption.AES256;
+import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.SecureRandom;
+import java.util.Base64;
+import java.util.UUID;
+import javax.crypto.KeyGenerator;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
+import software.amazon.awssdk.testutils.RandomTempFile;
+import software.amazon.awssdk.utils.IoUtils;
+import software.amazon.awssdk.utils.Md5Utils;
+
+public class SyncServerSideEncryptionIntegrationTest extends S3IntegrationTestBase {
+
+    private static final String BUCKET = temporaryBucketName(GetObjectIntegrationTest.class);
+
+    private static File file;
+
+    @BeforeClass
+    public static void setupFixture() throws IOException {
+        createBucket(BUCKET);
+        file = new RandomTempFile(10_000);
+    }
+
+    @AfterClass
+    public static void tearDownFixture() {
+        deleteBucketAndAllContents(BUCKET);
+        file.delete();
+    }
+
+    @Test
+    public void sse_AES256_succeeds() throws Exception {
+        String key = UUID.randomUUID().toString();
+        PutObjectRequest request = PutObjectRequest.builder()
+                                                   .key(key)
+                                                   .bucket(BUCKET)
+                                                   .serverSideEncryption(AES256)
+                                                   .build();
+
+        s3.putObject(request, file.toPath());
+
+        GetObjectRequest getObjectRequest = GetObjectRequest.builder()
+                                                            .key(key)
+                                                            .bucket(BUCKET)
+                                                            .build();
+
+        InputStream response = s3.getObject(getObjectRequest);
+        IoUtils.drainInputStream(response);
+        response.close();
+    }
+
+    @Test
+    public void sse_AWSKMS_succeeds() throws Exception {
+        String key = UUID.randomUUID().toString();
+        PutObjectRequest request = PutObjectRequest.builder()
+                                                   .key(key)
+                                                   .bucket(BUCKET)
+                                                   .serverSideEncryption(ServerSideEncryption.AWS_KMS)
+                                                   .build();
+
+        s3.putObject(request, file.toPath());
+
+        GetObjectRequest getObjectRequest = GetObjectRequest.builder()
+                                                            .key(key)
+                                                            .bucket(BUCKET)
+                                                            .build();
+
+        InputStream response = s3.getObject(getObjectRequest);
+        IoUtils.drainInputStream(response);
+        response.close();
+    }
+
+    @Test
+    public void sse_customerManaged_succeeds() throws Exception {
+        String key = UUID.randomUUID().toString();
+        byte[] secretKey = generateSecretKey();
+        String b64Key = Base64.getEncoder().encodeToString(secretKey);
+        String b64KeyMd5 = Md5Utils.md5AsBase64(secretKey);
+
+        PutObjectRequest request = PutObjectRequest.builder()
+                                                   .key(key)
+                                                   .bucket(BUCKET)
+                                                   .sseCustomerKey(b64Key)
+                                                   .sseCustomerAlgorithm(AES256.name())
+                                                   .sseCustomerKeyMD5(b64KeyMd5)
+                                                   .build();
+
+        s3.putObject(request, file.toPath());
+
+        GetObjectRequest getObjectRequest = GetObjectRequest.builder()
+                                                            .key(key)
+                                                            .bucket(BUCKET)
+                                                            .sseCustomerKey(b64Key)
+                                                            .sseCustomerAlgorithm(AES256.name())
+                                                            .sseCustomerKeyMD5(b64KeyMd5)
+                                                            .build();
+
+        InputStream response = s3.getObject(getObjectRequest);
+        IoUtils.drainInputStream(response);
+        response.close();
+    }
+
+    private static byte[] generateSecretKey() {
+        KeyGenerator generator;
+        try {
+            generator = KeyGenerator.getInstance("AES");
+            generator.init(256, new SecureRandom());
+            return generator.generateKey().getEncoded();
+        } catch (Exception e) {
+            fail("Unable to generate symmetric key: " + e.getMessage());
+            return null;
+        }
+    }
+}

services/s3/src/main/java/software/amazon/awssdk/services/s3/checksums/ChecksumConstant.java

@@ -40,6 +40,16 @@ public final class ChecksumConstant {
      */
     public static final String ENABLE_MD5_CHECKSUM_HEADER_VALUE = "append-md5";
 
+    /**
+     * Header value for specifying server side encryption.
+     */
+    public static final String SERVER_SIDE_ENCRYPTION_HEADER = "x-amz-server-side-encryption";
+
+    /**
+     * Header value for specifying server side encryption with a customer managed key.
+     */
+    public static final String SERVER_SIDE_CUSTOMER_ENCRYPTION_HEADER = "x-amz-server-side-encryption-customer-algorithm";
+
     /**
      * Length of an MD5 checksum in bytes.
      */

services/s3/src/main/java/software/amazon/awssdk/services/s3/checksums/ChecksumsEnabledValidator.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.checksums;
+
+import static software.amazon.awssdk.services.s3.checksums.ChecksumConstant.SERVER_SIDE_CUSTOMER_ENCRYPTION_HEADER;
+import static software.amazon.awssdk.services.s3.checksums.ChecksumConstant.SERVER_SIDE_ENCRYPTION_HEADER;
+import static software.amazon.awssdk.services.s3.model.ServerSideEncryption.AWS_KMS;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute;
+import software.amazon.awssdk.core.ClientType;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.internal.handlers.AsyncChecksumValidationInterceptor;
+import software.amazon.awssdk.services.s3.internal.handlers.SyncChecksumValidationInterceptor;
+
+/**
+ * Class used by {@link SyncChecksumValidationInterceptor} and
+ * {@link AsyncChecksumValidationInterceptor} to determine if trailing checksums
+ * should be enabled for a given request.
+ */
+@SdkInternalApi
+public final class ChecksumsEnabledValidator {
+
+    private ChecksumsEnabledValidator() {}
+
+    /**
+     * Validates that trailing checksums should be enabled based on {@link ClientType} and the presence
+     * or S3 specific headers.
+     *
+     * @param expectedClientType - The expected client type for enabling checksums
+     * @param executionAttributes - {@link ExecutionAttributes} to determine the actual client type
+     * @param headers A map of headers for a given request
+     * @return If trailing checksums should be enabled for this request.
+     */
+    public static boolean trailingChecksumsEnabled(ClientType expectedClientType,
+                                                   ExecutionAttributes executionAttributes,
+                                                   Map<String, List<String>> headers) {
+
+        // S3 doesn't support trailing checksums for customer encryption
+        if (headers.containsKey(SERVER_SIDE_CUSTOMER_ENCRYPTION_HEADER)) {
+            return false;
+        }
+
+        // S3 doesn't support trailing checksums for KMS encrypted objects
+        if (headers.getOrDefault(SERVER_SIDE_ENCRYPTION_HEADER, Collections.emptyList()).contains(AWS_KMS.toString())) {
+            return false;
+        }
+
+        ClientType actualClientType = executionAttributes.getAttribute(SdkExecutionAttribute.CLIENT_TYPE);
+
+        // Only validate checksums if the actual client type is the expected client type
+        if (expectedClientType.equals(actualClientType)) {
+            S3Configuration serviceConfiguration =
+                (S3Configuration) executionAttributes.getAttribute(AwsSignerExecutionAttribute.SERVICE_CONFIG);
+
+            return serviceConfiguration == null || serviceConfiguration.checksumValidationEnabled();
+        }
+
+        return false;
+    }
+}