Skip to content

Commit ff4ac00

Browse files
author
Bennett Lynch
committed
[S3] Add support for more user-friendly CopyObject source parameters
## Motivation and Context The current S3Client interface has a cumbersome API for invoking CopyObjectRequests. We require users to define the bucket name, key name, and version ID in a raw string format. We require that the string conform to the S3 API, which forces users to know the intricate details for how to join these values together. Additionally, portions (but not all) of the value must be URL encoded, further increasing the burden. https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html#API_CopyObject_RequestParameters In the Java SDK v1, users are given explicit parameters for the different copy source attributes. But in v2, parity for this support is clearly lacking. E.g., v1: ``` s3.copyObject(new CopyObjectRequest() .withSourceBucketName(SOURCE_BUCKET) .withSourceKey(key) .withSourceVersionId(versionId) .withDestinationBucketName(DESTINATION_BUCKET) .withDestinationKey(key)); ``` v2: ``` s3.copyObject(CopyObjectRequest.builder() .copySource(SOURCE_BUCKET + "/" + key + "?versionId=" + versionId) .destinationBucket(DESTINATION_BUCKET) .destinationKey(key) .build()); ``` The v1 SDK will also URL encode on the user's behalf, allowing users to use the same input values as they would for a PutObjectRequest. The v2 code snippet above may appear to work for most users until they run into unexpected source keys that require URL encoding, at which point they will typically be given `NoSuchKey` errors. This API deficiency has been called out by users in at least the following issues: * aws#1313 * aws#1452 * aws#1656 * awsdocs/aws-doc-sdk-examples#740 ## Description * For both CopyObjectRequest and UploadPartCopyRequest, add explicit parameters for: SourceBucket, SourceKey, SourceVersionId * If specified, these values will be used to construct a CopySource on the user's behalf, including URL encoding the relevant portions. * These values are introduced in a backwards compatible fashion. Users who are already using CopySource today will see no change in behavior, but these new fields may not be used in conjunction with CopySource. * A follow-up PR will be submitted to propose deprecating the current CopySource parameter. It is excluded from this PR since our current code gen configuration lacks appropriate support. * Add support for "DestinationBucket" & "DestinationKey" to UploadPartCopyRequest (this support already existed for CopyObjectRequest) * Utility function added to detect if an ARN is for a particular S3 resource type. This is to conform with the S3 API requirements of inserting "/object" in the path of these requests. ## Testing * New unit tests added * New integration tests added
1 parent 0f2e575 commit ff4ac00

File tree

1 file changed

+19
-17
lines changed

1 file changed

+19
-17
lines changed

services/s3/src/it/java/software/amazon/awssdk/services/s3/CopySourceIntegrationTest.java

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
import java.util.HashMap;
2626
import java.util.Map;
2727
import java.util.UUID;
28-
import org.junit.After;
29-
import org.junit.Before;
28+
import org.junit.AfterClass;
29+
import org.junit.BeforeClass;
3030
import org.junit.Test;
3131
import org.junit.runner.RunWith;
3232
import org.junit.runners.Parameterized;
@@ -50,18 +50,24 @@
5050
@RunWith(Parameterized.class)
5151
public class CopySourceIntegrationTest extends S3IntegrationTestBase {
5252

53-
private static final String SOURCE_BUCKET_NAME = temporaryBucketName("copy-source-integ-test-src");
53+
private static final String SOURCE_UNVERSIONED_BUCKET_NAME = temporaryBucketName("copy-source-integ-test-src");
54+
private static final String SOURCE_VERSIONED_BUCKET_NAME = temporaryBucketName("copy-source-integ-test-versioned-src");
5455
private static final String DESTINATION_BUCKET_NAME = temporaryBucketName("copy-source-integ-test-dest");
5556

56-
@Before
57-
public void initializeTestData() throws Exception {
58-
createBucket(SOURCE_BUCKET_NAME);
57+
@BeforeClass
58+
public static void initializeTestData() throws Exception {
59+
createBucket(SOURCE_UNVERSIONED_BUCKET_NAME);
60+
createBucket(SOURCE_VERSIONED_BUCKET_NAME);
61+
s3.putBucketVersioning(r -> r
62+
.bucket(SOURCE_VERSIONED_BUCKET_NAME)
63+
.versioningConfiguration(v -> v.status(BucketVersioningStatus.ENABLED)));
5964
createBucket(DESTINATION_BUCKET_NAME);
6065
}
6166

62-
@After
63-
public void tearDown() {
64-
deleteBucketAndAllContents(SOURCE_BUCKET_NAME);
67+
@AfterClass
68+
public static void tearDown() {
69+
deleteBucketAndAllContents(SOURCE_UNVERSIONED_BUCKET_NAME);
70+
deleteBucketAndAllContents(SOURCE_VERSIONED_BUCKET_NAME);
6571
deleteBucketAndAllContents(DESTINATION_BUCKET_NAME);
6672
}
6773

@@ -87,12 +93,12 @@ public void copyObject_WithoutVersion_AcceptsSameKeyAsPut() throws Exception {
8793
String originalContent = UUID.randomUUID().toString();
8894

8995
s3.putObject(PutObjectRequest.builder()
90-
.bucket(SOURCE_BUCKET_NAME)
96+
.bucket(SOURCE_UNVERSIONED_BUCKET_NAME)
9197
.key(key)
9298
.build(), RequestBody.fromString(originalContent, StandardCharsets.UTF_8));
9399

94100
s3.copyObject(CopyObjectRequest.builder()
95-
.sourceBucket(SOURCE_BUCKET_NAME)
101+
.sourceBucket(SOURCE_UNVERSIONED_BUCKET_NAME)
96102
.sourceKey(key)
97103
.destinationBucket(DESTINATION_BUCKET_NAME)
98104
.destinationKey(key)
@@ -113,16 +119,12 @@ public void copyObject_WithoutVersion_AcceptsSameKeyAsPut() throws Exception {
113119
*/
114120
@Test
115121
public void copyObject_WithVersion_AcceptsSameKeyAsPut() throws Exception {
116-
s3.putBucketVersioning(r -> r
117-
.bucket(SOURCE_BUCKET_NAME)
118-
.versioningConfiguration(v -> v.status(BucketVersioningStatus.ENABLED)));
119-
120122
Map<String, String> versionToContentMap = new HashMap<>();
121123
int numVersionsToCreate = 3;
122124
for (int i = 0; i < numVersionsToCreate; i++) {
123125
String originalContent = UUID.randomUUID().toString();
124126
PutObjectResponse response = s3.putObject(PutObjectRequest.builder()
125-
.bucket(SOURCE_BUCKET_NAME)
127+
.bucket(SOURCE_VERSIONED_BUCKET_NAME)
126128
.key(key)
127129
.build(),
128130
RequestBody.fromString(originalContent, StandardCharsets.UTF_8));
@@ -131,7 +133,7 @@ public void copyObject_WithVersion_AcceptsSameKeyAsPut() throws Exception {
131133

132134
versionToContentMap.forEach((versionId, originalContent) -> {
133135
s3.copyObject(CopyObjectRequest.builder()
134-
.sourceBucket(SOURCE_BUCKET_NAME)
136+
.sourceBucket(SOURCE_VERSIONED_BUCKET_NAME)
135137
.sourceKey(key)
136138
.sourceVersionId(versionId)
137139
.destinationBucket(DESTINATION_BUCKET_NAME)

0 commit comments

Comments
 (0)