Skip to content

Commit 89fdb89

Browse files
zoewanggzoewangg
committed
Optimize file upload by reading from file in different offsets in parallel
1 parent ed0dcaa commit 89fdb89

File tree

13 files changed

+678
-73
lines changed

13 files changed

+678
-73
lines changed

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/FileRequestBodyConfiguration.java

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.core;
17+
18+
import java.nio.file.Path;
19+
import java.util.Objects;
20+
import software.amazon.awssdk.annotations.SdkPublicApi;
21+
import software.amazon.awssdk.core.async.AsyncRequestBody;
22+
import software.amazon.awssdk.utils.Validate;
23+
import software.amazon.awssdk.utils.builder.CopyableBuilder;
24+
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
25+
26+
/**
27+
* Configuration options for {@link AsyncRequestBody#fromFile(FileRequestBodyConfiguration)} to configure how the SDK
28+
* should read the file.
29+
*
30+
* @see #builder()
31+
*/
32+
@SdkPublicApi
33+
public final class FileRequestBodyConfiguration implements ToCopyableBuilder<FileRequestBodyConfiguration.Builder,
34+
FileRequestBodyConfiguration> {
35+
private final Integer chunkSizeInBytes;
36+
private final Long position;
37+
private final Long numBytesToRead;
38+
private final Path path;
39+
40+
private FileRequestBodyConfiguration(DefaultBuilder builder) {
41+
this.path = Validate.notNull(builder.path, "path");
42+
this.chunkSizeInBytes = Validate.isPositiveOrNull(builder.chunkSizeInBytes, "chunkSizeInBytes");
43+
this.position = Validate.isNotNegativeOrNull(builder.position, "position");
44+
this.numBytesToRead = Validate.isNotNegativeOrNull(builder.numBytesToRead, "numBytesToRead");
45+
}
46+
47+
/**
48+
* Create a {@link Builder}, used to create a {@link FileRequestBodyConfiguration}.
49+
*/
50+
public static Builder builder() {
51+
return new DefaultBuilder();
52+
}
53+
54+
/**
55+
* @return the size of each chunk to read from the file
56+
*/
57+
public Integer chunkSizeInBytes() {
58+
return chunkSizeInBytes;
59+
}
60+
61+
/**
62+
* @return the file position at which the request body begins.
63+
*/
64+
public Long position() {
65+
return position;
66+
}
67+
68+
/**
69+
* @return the number of bytes to read from this file.
70+
*/
71+
public Long numBytesToRead() {
72+
return numBytesToRead;
73+
}
74+
75+
/**
76+
* @return the file path
77+
*/
78+
public Path path() {
79+
return path;
80+
}
81+
82+
@Override
83+
public boolean equals(Object o) {
84+
if (this == o) {
85+
return true;
86+
}
87+
if (o == null || getClass() != o.getClass()) {
88+
return false;
89+
}
90+
91+
FileRequestBodyConfiguration that = (FileRequestBodyConfiguration) o;
92+
93+
if (!Objects.equals(chunkSizeInBytes, that.chunkSizeInBytes)) {
94+
return false;
95+
}
96+
if (!Objects.equals(position, that.position)) {
97+
return false;
98+
}
99+
if (!Objects.equals(numBytesToRead, that.numBytesToRead)) {
100+
return false;
101+
}
102+
return Objects.equals(path, that.path);
103+
}
104+
105+
@Override
106+
public int hashCode() {
107+
int result = chunkSizeInBytes != null ? chunkSizeInBytes.hashCode() : 0;
108+
result = 31 * result + (position != null ? position.hashCode() : 0);
109+
result = 31 * result + (numBytesToRead != null ? numBytesToRead.hashCode() : 0);
110+
result = 31 * result + (path != null ? path.hashCode() : 0);
111+
return result;
112+
}
113+
114+
@Override
115+
public Builder toBuilder() {
116+
return new DefaultBuilder(this);
117+
}
118+
119+
public interface Builder extends CopyableBuilder<Builder, FileRequestBodyConfiguration> {
120+
121+
/**
122+
* Sets the {@link Path} to the file containing data to send to the service
123+
*
124+
* @param path Path to file to read.
125+
* @return This builder for method chaining.
126+
*/
127+
Builder path(Path path);
128+
129+
/**
130+
* Sets the size of chunks read from the file. Increasing this will cause more data to be buffered into memory but
131+
* may yield better latencies. Decreasing this will reduce memory usage but may cause reduced latency. Setting this value
132+
* is very dependent on upload speed and requires some performance testing to tune.
133+
*
134+
* <p>The default chunk size is 16 KiB</p>
135+
*
136+
* @param chunkSize New chunk size in bytes.
137+
* @return This builder for method chaining.
138+
*/
139+
Builder chunkSizeInBytes(Integer chunkSize);
140+
141+
/**
142+
* Sets the file position at which the request body begins.
143+
*
144+
* <p>By default, it's 0, i.e., reading from the beginning.
145+
*
146+
* @param position the position of the file
147+
* @return The builder for method chaining.
148+
*/
149+
Builder position(Long position);
150+
151+
/**
152+
* Sets the number of bytes to read from this file.
153+
*
154+
* <p>By default, it's same as the file length.
155+
*
156+
* @param numBytesToRead number of bytes to read
157+
* @return The builder for method chaining.
158+
*/
159+
Builder numBytesToRead(Long numBytesToRead);
160+
}
161+
162+
private static final class DefaultBuilder implements Builder {
163+
private Long position;
164+
private Path path;
165+
private Integer chunkSizeInBytes;
166+
private Long numBytesToRead;
167+
168+
private DefaultBuilder(FileRequestBodyConfiguration configuration) {
169+
this.position = configuration.position;
170+
this.path = configuration.path;
171+
this.chunkSizeInBytes = configuration.chunkSizeInBytes;
172+
this.numBytesToRead = configuration.numBytesToRead;
173+
}
174+
175+
private DefaultBuilder() {
176+
177+
}
178+
179+
@Override
180+
public Builder path(Path path) {
181+
this.path = path;
182+
return this;
183+
}
184+
185+
@Override
186+
public Builder chunkSizeInBytes(Integer chunkSizeInBytes) {
187+
this.chunkSizeInBytes = chunkSizeInBytes;
188+
return this;
189+
}
190+
191+
@Override
192+
public Builder position(Long position) {
193+
this.position = position;
194+
return this;
195+
}
196+
197+
@Override
198+
public Builder numBytesToRead(Long numBytesToRead) {
199+
this.numBytesToRead = numBytesToRead;
200+
return this;
201+
}
202+
203+
@Override
204+
public FileRequestBodyConfiguration build() {
205+
return new FileRequestBodyConfiguration(this);
206+
}
207+
}
208+
209+
}

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncRequestBody.java

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.reactivestreams.Publisher;
3030
import org.reactivestreams.Subscriber;
3131
import software.amazon.awssdk.annotations.SdkPublicApi;
32+
import software.amazon.awssdk.core.FileRequestBodyConfiguration;
3233
import software.amazon.awssdk.core.internal.async.ByteBuffersAsyncRequestBody;
3334
import software.amazon.awssdk.core.internal.async.FileAsyncRequestBody;
3435
import software.amazon.awssdk.core.internal.async.InputStreamWithExecutorAsyncRequestBody;
@@ -112,16 +113,46 @@ static AsyncRequestBody fromFile(Path path) {
112113

113114
/**
114115
* Creates an {@link AsyncRequestBody} that produces data from the contents of a file. See
115-
* {@link FileAsyncRequestBody#builder} to create a customized body implementation.
116+
* {@link #fromFile(FileRequestBodyConfiguration)} to create a customized body implementation.
116117
*
117118
* @param file The file to read from.
118119
* @return Implementation of {@link AsyncRequestBody} that reads data from the specified file.
119-
* @see FileAsyncRequestBody
120120
*/
121121
static AsyncRequestBody fromFile(File file) {
122122
return FileAsyncRequestBody.builder().path(file.toPath()).build();
123123
}
124124

125+
/**
126+
* Creates an {@link AsyncRequestBody} that produces data from the contents of a file.
127+
*
128+
* @param configuration configuration for how the SDK should read the file
129+
* @return Implementation of {@link AsyncRequestBody} that reads data from the specified file.
130+
*/
131+
static AsyncRequestBody fromFile(FileRequestBodyConfiguration configuration) {
132+
Validate.notNull(configuration, "configuration");
133+
return FileAsyncRequestBody.builder()
134+
.path(configuration.path())
135+
.position(configuration.position())
136+
.chunkSizeInBytes(configuration.chunkSizeInBytes())
137+
.numBytesToRead(configuration.numBytesToRead())
138+
.build();
139+
}
140+
141+
/**
142+
* Creates an {@link AsyncRequestBody} that produces data from the contents of a file.
143+
*
144+
* <p>
145+
* This is a convenience method that creates an instance of the {@link FileRequestBodyConfiguration} builder,
146+
* avoiding the need to create one manually via {@link FileRequestBodyConfiguration#builder()}.
147+
*
148+
* @param configuration configuration for how the SDK should read the file
149+
* @return Implementation of {@link AsyncRequestBody} that reads data from the specified file.
150+
*/
151+
static AsyncRequestBody fromFile(Consumer<FileRequestBodyConfiguration.Builder> configuration) {
152+
Validate.notNull(configuration, "configuration");
153+
return fromFile(FileRequestBodyConfiguration.builder().applyMutation(configuration).build());
154+
}
155+
125156
/**
126157
* Creates an {@link AsyncRequestBody} that uses a single string as data.
127158
*

0 commit comments

Comments
 (0)