Skip to content

Commit f72cee6

Browse files
joviegasjoviegas
committed
Fix for S3Client with httpCLient as url-connection-client fails with EOFException when executing HeadObjectRequest for gzip encoded objec
1 parent f3265c6 commit f72cee6

File tree

4 files changed

+242
-1
lines changed

4 files changed

+242
-1
lines changed

‎.changes/next-release/bugfix-URLConnectionHTTPClient-1acc5e0.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "bugfix",
3+
"category": "URL Connection HTTP Client",
4+
"contributor": "",
5+
"description": "Fix for S3Client with URL Connection http client fails with EOFException when executing HeadObjectRequest for gzip encodeFix to S3Client fails with EOFException."
6+
}

‎http-clients/url-connection-client/src/it/java/software/amazon/awssdk/http/urlconnection/HeadObjectIntegrationTest.java

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.http.urlconnection;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
20+
21+
import java.io.ByteArrayOutputStream;
22+
import java.io.IOException;
23+
import java.nio.charset.StandardCharsets;
24+
import java.util.zip.GZIPOutputStream;
25+
import org.junit.AfterClass;
26+
import org.junit.BeforeClass;
27+
import org.junit.Test;
28+
import software.amazon.awssdk.core.sync.RequestBody;
29+
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
30+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
31+
32+
public class HeadObjectIntegrationTest extends UrlHttpConnectionS3IntegrationTestBase {
33+
private static final String BUCKET = temporaryBucketName(HeadObjectIntegrationTest.class);
34+
35+
private static final String GZIPPED_KEY = "some-key";
36+
37+
@BeforeClass
38+
public static void setupFixture() throws IOException {
39+
createBucket(BUCKET);
40+
41+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
42+
GZIPOutputStream gzos = new GZIPOutputStream(baos);
43+
gzos.write("Test".getBytes(StandardCharsets.UTF_8));
44+
45+
s3.putObject(PutObjectRequest.builder()
46+
.bucket(BUCKET)
47+
.key(GZIPPED_KEY)
48+
.contentEncoding("gzip")
49+
.build(),
50+
RequestBody.fromBytes(baos.toByteArray()));
51+
}
52+
53+
@Test
54+
public void syncClientSupportsGzippedObjects() {
55+
HeadObjectResponse response = s3.headObject(r -> r.bucket(BUCKET).key(GZIPPED_KEY));
56+
assertThat(response.contentEncoding()).isEqualTo("gzip");
57+
}
58+
59+
@AfterClass
60+
public static void cleanup() {
61+
deleteBucketAndAllContents(BUCKET);
62+
}
63+
64+
}

‎http-clients/url-connection-client/src/it/java/software/amazon/awssdk/http/urlconnection/UrlHttpConnectionS3IntegrationTestBase.java

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.http.urlconnection;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
20+
import java.util.Iterator;
21+
import java.util.List;
22+
import org.junit.BeforeClass;
23+
import software.amazon.awssdk.core.ClientType;
24+
import software.amazon.awssdk.core.interceptor.Context;
25+
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
26+
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
27+
import software.amazon.awssdk.http.apache.ApacheHttpClient;
28+
import software.amazon.awssdk.regions.Region;
29+
import software.amazon.awssdk.services.s3.S3AsyncClient;
30+
import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
31+
import software.amazon.awssdk.services.s3.S3Client;
32+
import software.amazon.awssdk.services.s3.S3ClientBuilder;
33+
import software.amazon.awssdk.services.s3.model.BucketLocationConstraint;
34+
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
35+
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
36+
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
37+
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
38+
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
39+
import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
40+
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
41+
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
42+
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
43+
import software.amazon.awssdk.services.s3.model.S3Exception;
44+
import software.amazon.awssdk.services.s3.model.S3Object;
45+
import software.amazon.awssdk.testutils.Waiter;
46+
import software.amazon.awssdk.testutils.service.AwsTestBase;
47+
48+
/**
49+
* Base class for S3 integration tests. Loads AWS credentials from a properties
50+
* file and creates an S3 client for callers to use.
51+
*/
52+
public class UrlHttpConnectionS3IntegrationTestBase extends AwsTestBase {
53+
54+
protected static final Region DEFAULT_REGION = Region.US_WEST_2;
55+
/**
56+
* The S3 client for all tests to use.
57+
*/
58+
protected static S3Client s3;
59+
60+
/**
61+
* Loads the AWS account info for the integration tests and creates an S3
62+
* client for tests to use.
63+
*/
64+
@BeforeClass
65+
public static void setUp() throws Exception {
66+
s3 = s3ClientBuilder().build();
67+
}
68+
69+
protected static S3ClientBuilder s3ClientBuilder() {
70+
return S3Client.builder()
71+
.httpClient(UrlConnectionHttpClient.create())
72+
.region(DEFAULT_REGION)
73+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
74+
75+
}
76+
77+
protected static void createBucket(String bucketName) {
78+
createBucket(bucketName, 0);
79+
}
80+
81+
private static void createBucket(String bucketName, int retryCount) {
82+
try {
83+
s3.createBucket(
84+
CreateBucketRequest.builder()
85+
.bucket(bucketName)
86+
.createBucketConfiguration(
87+
CreateBucketConfiguration.builder()
88+
.locationConstraint(BucketLocationConstraint.US_WEST_2)
89+
.build())
90+
.build());
91+
} catch (S3Exception e) {
92+
System.err.println("Error attempting to create bucket: " + bucketName);
93+
if (e.awsErrorDetails().errorCode().equals("BucketAlreadyOwnedByYou")) {
94+
System.err.printf("%s bucket already exists, likely leaked by a previous run\n", bucketName);
95+
} else if (e.awsErrorDetails().errorCode().equals("TooManyBuckets")) {
96+
System.err.println("Printing all buckets for debug:");
97+
s3.listBuckets().buckets().forEach(System.err::println);
98+
if (retryCount < 2) {
99+
System.err.println("Retrying...");
100+
createBucket(bucketName, retryCount + 1);
101+
} else {
102+
throw e;
103+
}
104+
} else {
105+
throw e;
106+
}
107+
}
108+
}
109+
110+
protected static void deleteBucketAndAllContents(String bucketName) {
111+
deleteBucketAndAllContents(s3, bucketName);
112+
}
113+
114+
115+
public static void deleteBucketAndAllContents(S3Client s3, String bucketName) {
116+
try {
117+
System.out.println("Deleting S3 bucket: " + bucketName);
118+
ListObjectsResponse response = Waiter.run(() -> s3.listObjects(r -> r.bucket(bucketName)))
119+
.ignoringException(NoSuchBucketException.class)
120+
.orFail();
121+
List<S3Object> objectListing = response.contents();
122+
123+
if (objectListing != null) {
124+
while (true) {
125+
for (Iterator<?> iterator = objectListing.iterator(); iterator.hasNext(); ) {
126+
S3Object objectSummary = (S3Object) iterator.next();
127+
s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
128+
}
129+
130+
if (response.isTruncated()) {
131+
objectListing = s3.listObjects(ListObjectsRequest.builder()
132+
.bucket(bucketName)
133+
.marker(response.marker())
134+
.build())
135+
.contents();
136+
} else {
137+
break;
138+
}
139+
}
140+
}
141+
142+
143+
ListObjectVersionsResponse versions = s3
144+
.listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
145+
146+
if (versions.deleteMarkers() != null) {
147+
versions.deleteMarkers().forEach(v -> s3.deleteObject(DeleteObjectRequest.builder()
148+
.versionId(v.versionId())
149+
.bucket(bucketName)
150+
.key(v.key())
151+
.build()));
152+
}
153+
154+
if (versions.versions() != null) {
155+
versions.versions().forEach(v -> s3.deleteObject(DeleteObjectRequest.builder()
156+
.versionId(v.versionId())
157+
.bucket(bucketName)
158+
.key(v.key())
159+
.build()));
160+
}
161+
162+
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
163+
} catch (Exception e) {
164+
System.err.println("Failed to delete bucket: " + bucketName);
165+
e.printStackTrace();
166+
}
167+
}
168+
}

‎http-clients/url-connection-client/src/main/java/software/amazon/awssdk/http/urlconnection/UrlConnectionHttpClient.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,10 @@ public HttpExecuteResponse call() throws IOException {
302302
int responseCode = getResponseCodeSafely(connection);
303303
boolean isErrorResponse = HttpStatusFamily.of(responseCode).isOneOf(CLIENT_ERROR, SERVER_ERROR);
304304
Optional<InputStream> responseContent = isErrorResponse ? tryGetErrorStream() : tryGetInputStream();
305-
AbortableInputStream responseBody = responseContent.map(AbortableInputStream::create).orElse(null);
305+
306+
AbortableInputStream responseBody = responseHasNoContent()
307+
? null
308+
: responseContent.map(AbortableInputStream::create).orElse(null);
306309

307310
return HttpExecuteResponse.builder()
308311
.response(SdkHttpResponse.builder()

0 commit comments

Comments
 (0)