Skip to content

Commit df99fc3

Browse files
kuheAllanZhengYP
kuhe
and
AllanZhengYP
authored
feat(endpoint): endpoints 2.0 existing package changes (#3947)
* feat(endpoint): endpoints 2.0 existing package changes * feat(endpoint): enable middleware stack identification * feat(endpoint): update existing packages for endpoints v2 * feat(endpoint): address PR style comments * feat(endpoint): fixes to types * feat(endpoint): additional fixes from testing * feat(endpoint): s3 presigned post fixes for endpoints 2.0 * feat(endpoint): unit test fixes * feat(endpoint): type fixes * feat(endpoint): types and docs updates * feat(endpoint): s3 bucket name forcePathStyle * feat(endpoint): type fix * feat(endpoint): java code format * feat(endpoint): type fix Co-authored-by: AllanZhengYP <[email protected]>
1 parent f91b252 commit df99fc3

File tree

36 files changed

+566
-184
lines changed

36 files changed

+566
-184
lines changed

Diff for: ‎codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java

+81-79
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,8 @@ public final class AddS3Config implements TypeScriptIntegration {
7272
);
7373

7474
private static final String CRT_NOTIFICATION = "<p>Note: To supply the Multi-region Access Point (MRAP) to Bucket,"
75-
+ " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
76-
+ "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";
75+
+ " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
76+
+ "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";
7777

7878
@Override
7979
public Model preprocessModel(Model model, TypeScriptSettings settings) {
@@ -118,17 +118,19 @@ public void addConfigInterfaceFields(
118118
return;
119119
}
120120
writer.writeDocs("Whether to escape request path when signing the request.")
121-
.write("signingEscapePath?: boolean;\n");
121+
.write("signingEscapePath?: boolean;\n");
122122
writer.writeDocs(
123123
"Whether to override the request region with the region inferred from requested resource's ARN."
124-
+ " Defaults to false.")
125-
.addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName)
126-
.write("useArnRegion?: boolean | Provider<boolean>;");
124+
+ " Defaults to false.")
125+
.addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName)
126+
.write("useArnRegion?: boolean | Provider<boolean>;");
127127
}
128128

129129
@Override
130-
public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScriptSettings settings, Model model,
131-
SymbolProvider symbolProvider, LanguageTarget target) {
130+
public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(
131+
TypeScriptSettings settings, Model model,
132+
SymbolProvider symbolProvider, LanguageTarget target
133+
) {
132134
if (!isS3(settings.getService(model))) {
133135
return Collections.emptyMap();
134136
}
@@ -140,18 +142,18 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
140142
writer.write("false");
141143
}, "signerConstructor", writer -> {
142144
writer.addDependency(AwsDependency.SIGNATURE_V4_MULTIREGION)
143-
.addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion",
144-
AwsDependency.SIGNATURE_V4_MULTIREGION.packageName)
145-
.write("SignatureV4MultiRegion");
145+
.addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion",
146+
AwsDependency.SIGNATURE_V4_MULTIREGION.packageName)
147+
.write("SignatureV4MultiRegion");
146148
});
147149
case NODE:
148150
return MapUtils.of("useArnRegion", writer -> {
149151
writer.addDependency(AwsDependency.NODE_CONFIG_PROVIDER)
150-
.addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName)
151-
.addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE)
152-
.addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS",
153-
AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName)
154-
.write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)");
152+
.addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName)
153+
.addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE)
154+
.addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS",
155+
AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName)
156+
.write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)");
155157
});
156158
default:
157159
return Collections.emptyMap();
@@ -161,67 +163,67 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
161163
@Override
162164
public List<RuntimeClientPlugin> getClientPlugins() {
163165
return ListUtils.of(
164-
RuntimeClientPlugin.builder()
165-
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName",
166-
HAS_MIDDLEWARE)
167-
.servicePredicate((m, s) -> isS3(s))
168-
.build(),
169-
RuntimeClientPlugin.builder()
170-
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader",
171-
HAS_MIDDLEWARE)
172-
.operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject"))
173-
.build(),
174-
RuntimeClientPlugin.builder()
175-
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions",
176-
HAS_MIDDLEWARE)
177-
.operationPredicate(
178-
(m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s))
179-
&& isS3(s))
180-
.build(),
181-
RuntimeClientPlugin.builder()
182-
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency,
183-
"WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE)
184-
.operationPredicate((m, s, o) -> isS3(s)
185-
&& o.getId().getName(s).equals("WriteGetObjectResponse"))
186-
.build(),
187-
RuntimeClientPlugin.builder()
188-
.withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue",
189-
HAS_MIDDLEWARE)
190-
.servicePredicate((m, s) -> isS3(s))
191-
.build(),
192-
RuntimeClientPlugin.builder()
193-
.withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE)
194-
.operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS)
195-
&& isS3(s))
196-
.build(),
197-
RuntimeClientPlugin.builder()
198-
.withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint",
199-
HAS_MIDDLEWARE)
200-
.operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket")
201-
&& isS3(s))
202-
.build(),
203-
RuntimeClientPlugin.builder()
204-
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3",
205-
HAS_CONFIG)
206-
.servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s))
207-
.build(),
208-
/*
209-
* BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client.
210-
* The second applies the middleware to bucket endpoint operations.
211-
*/
212-
RuntimeClientPlugin.builder()
213-
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
214-
HAS_CONFIG)
215-
.servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s))
216-
.build(),
217-
RuntimeClientPlugin.builder()
218-
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
219-
HAS_MIDDLEWARE)
220-
.operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s))
221-
&& isS3(s)
222-
&& !isEndpointsV2Service(s)
223-
&& containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS))
224-
.build()
166+
RuntimeClientPlugin.builder()
167+
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName",
168+
HAS_MIDDLEWARE)
169+
.servicePredicate((m, s) -> isS3(s))
170+
.build(),
171+
RuntimeClientPlugin.builder()
172+
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader",
173+
HAS_MIDDLEWARE)
174+
.operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject"))
175+
.build(),
176+
RuntimeClientPlugin.builder()
177+
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions",
178+
HAS_MIDDLEWARE)
179+
.operationPredicate(
180+
(m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s))
181+
&& isS3(s))
182+
.build(),
183+
RuntimeClientPlugin.builder()
184+
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency,
185+
"WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE)
186+
.operationPredicate((m, s, o) -> isS3(s)
187+
&& o.getId().getName(s).equals("WriteGetObjectResponse"))
188+
.build(),
189+
RuntimeClientPlugin.builder()
190+
.withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue",
191+
HAS_MIDDLEWARE)
192+
.servicePredicate((m, s) -> isS3(s))
193+
.build(),
194+
RuntimeClientPlugin.builder()
195+
.withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE)
196+
.operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS)
197+
&& isS3(s))
198+
.build(),
199+
RuntimeClientPlugin.builder()
200+
.withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint",
201+
HAS_MIDDLEWARE)
202+
.operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket")
203+
&& isS3(s))
204+
.build(),
205+
RuntimeClientPlugin.builder()
206+
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3",
207+
HAS_CONFIG)
208+
.servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s))
209+
.build(),
210+
/*
211+
* BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client.
212+
* The second applies the middleware to bucket endpoint operations.
213+
*/
214+
RuntimeClientPlugin.builder()
215+
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
216+
HAS_CONFIG)
217+
.servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s))
218+
.build(),
219+
RuntimeClientPlugin.builder()
220+
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
221+
HAS_MIDDLEWARE)
222+
.operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s))
223+
&& isS3(s)
224+
&& !isEndpointsV2Service(s)
225+
&& containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS))
226+
.build()
225227
);
226228
}
227229

@@ -232,8 +234,8 @@ private static boolean containsInputMembers(
232234
) {
233235
OperationIndex operationIndex = OperationIndex.of(model);
234236
return operationIndex.getInput(operationShape)
235-
.filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains))
236-
.isPresent();
237+
.filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains))
238+
.isPresent();
237239
}
238240

239241
private static boolean isS3(Shape serviceShape) {

Diff for: ‎lib/lib-storage/package.json

+2
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
},
2525
"license": "Apache-2.0",
2626
"dependencies": {
27+
"@aws-sdk/middleware-endpoint": "*",
2728
"@aws-sdk/smithy-client": "*",
2829
"buffer": "5.6.0",
2930
"events": "3.3.0",
@@ -37,6 +38,7 @@
3738
"devDependencies": {
3839
"@aws-sdk/abort-controller": "*",
3940
"@aws-sdk/client-s3": "*",
41+
"@aws-sdk/types": "*",
4042
"@tsconfig/recommended": "1.0.1",
4143
"@types/node": "^14.11.2",
4244
"concurrently": "7.0.0",

Diff for: ‎lib/lib-storage/src/Upload.ts

+20-7
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,14 @@ import {
1313
Tag,
1414
UploadPartCommand,
1515
} from "@aws-sdk/client-s3";
16+
import {
17+
EndpointParameterInstructionsSupplier,
18+
getEndpointFromInstructions,
19+
toEndpointV1,
20+
} from "@aws-sdk/middleware-endpoint";
1621
import { HttpRequest } from "@aws-sdk/protocol-http";
1722
import { extendedEncodeURIComponent } from "@aws-sdk/smithy-client";
23+
import { Endpoint } from "@aws-sdk/types";
1824
import { EventEmitter } from "events";
1925

2026
import { byteLength } from "./bytelength";
@@ -101,7 +107,8 @@ export class Upload extends EventEmitter {
101107
this.isMultiPart = false;
102108
const params = { ...this.params, Body: dataPart.data };
103109

104-
const requestHandler = this.client.config.requestHandler;
110+
const clientConfig = this.client.config;
111+
const requestHandler = clientConfig.requestHandler;
105112
const eventEmitter: EventEmitter | null = requestHandler instanceof EventEmitter ? requestHandler : null;
106113
const uploadEventListener = (event: ProgressEvent) => {
107114
this.bytesUploadedSoFar = event.loaded;
@@ -120,14 +127,20 @@ export class Upload extends EventEmitter {
120127
eventEmitter.on("xhr.upload.progress", uploadEventListener);
121128
}
122129

123-
const [putResult, endpoint] = await Promise.all([
124-
this.client.send(new PutObjectCommand(params)),
125-
this.client.config?.endpoint?.(),
126-
]);
130+
const resolved = await Promise.all([this.client.send(new PutObjectCommand(params)), clientConfig?.endpoint?.()]);
131+
const putResult = resolved[0];
132+
let endpoint: Endpoint = resolved[1];
133+
134+
if (!endpoint) {
135+
endpoint = toEndpointV1(
136+
await getEndpointFromInstructions(params, PutObjectCommand as EndpointParameterInstructionsSupplier, {
137+
...clientConfig,
138+
})
139+
);
140+
}
127141

128142
if (!endpoint) {
129-
// TODO(endpointsv2): handle endpoint v2
130-
throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()".');
143+
throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()" nor EndpointsV2.');
131144
}
132145

133146
if (eventEmitter !== null) {

Diff for: ‎packages/config-resolver/src/endpointsConfig/resolveEndpointsConfig.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ import { getEndpointFromRegion } from "./utils/getEndpointFromRegion";
55

66
export interface EndpointsInputConfig {
77
/**
8-
* The fully qualified endpoint of the webservice. This is only required when using a custom endpoint (for example, when using a local version of S3).
8+
* The fully qualified endpoint of the webservice. This is only required when using
9+
* a custom endpoint (for example, when using a local version of S3).
910
*/
1011
endpoint?: string | Endpoint | Provider<Endpoint>;
1112

@@ -56,7 +57,7 @@ export const resolveEndpointsConfig = <T>(
5657
endpoint: endpoint
5758
? normalizeProvider(typeof endpoint === "string" ? urlParser(endpoint) : endpoint)
5859
: () => getEndpointFromRegion({ ...input, useDualstackEndpoint, useFipsEndpoint }),
59-
isCustomEndpoint: endpoint ? true : false,
60+
isCustomEndpoint: !!endpoint,
6061
useDualstackEndpoint,
6162
};
6263
};

Diff for: ‎packages/middleware-endpoint/package.json

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
66
"build:cjs": "tsc -p tsconfig.cjs.json",
77
"build:es": "tsc -p tsconfig.es.json",
8+
"build:include:deps": "lerna run --scope $npm_package_name --include-dependencies build",
89
"build:types": "tsc -p tsconfig.types.json",
910
"build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4",
1011
"clean": "rimraf ./dist-* && rimraf *.tsbuildinfo",
@@ -19,6 +20,7 @@
1920
},
2021
"license": "Apache-2.0",
2122
"dependencies": {
23+
"@aws-sdk/protocol-http": "*",
2224
"@aws-sdk/signature-v4": "*",
2325
"@aws-sdk/types": "*",
2426
"@aws-sdk/util-config-provider": "*",

Diff for: ‎packages/middleware-endpoint/src/adaptors/getEndpointFromInstructions.ts

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import { EndpointParameters, EndpointV2, HandlerExecutionContext } from "@aws-sdk/types";
2+
3+
import { EndpointResolvedConfig } from "../resolveEndpointConfig";
4+
import { EndpointParameterInstructions } from "../types";
5+
6+
export type EndpointParameterInstructionsSupplier = Partial<{
7+
getEndpointParameterInstructions(): EndpointParameterInstructions;
8+
}>;
9+
10+
/**
11+
* This step in the endpoint resolution process is exposed as a function
12+
* to allow packages such as signers, lib-upload, etc. to get
13+
* the V2 Endpoint associated to an instance of some api operation command
14+
* without needing to send it or resolve its middleware stack.
15+
*
16+
* @private
17+
* @param commandInput - the input of the Command in question.
18+
* @param instructionsSupplier - this is typically a Command constructor. A static function supplying the
19+
* endpoint parameter instructions will exist for commands in services
20+
* having an endpoints ruleset trait.
21+
* @param clientConfig - config of the service client.
22+
* @param context - optional context.
23+
*/
24+
export const getEndpointFromInstructions = async <
25+
T extends EndpointParameters,
26+
CommandInput extends Record<string, unknown>,
27+
Config extends Record<string, unknown>
28+
>(
29+
commandInput: CommandInput,
30+
instructionsSupplier: EndpointParameterInstructionsSupplier,
31+
clientConfig: Partial<EndpointResolvedConfig<T>> & Config,
32+
context?: HandlerExecutionContext
33+
): Promise<EndpointV2> => {
34+
const endpointParams: EndpointParameters = {};
35+
const instructions: EndpointParameterInstructions =
36+
(instructionsSupplier.getEndpointParameterInstructions || (() => null))() || {};
37+
38+
if (typeof clientConfig.endpointProvider !== "function") {
39+
throw new Error("config.endpointProvider is not set.");
40+
}
41+
42+
for (const [name, instruction] of Object.entries(instructions)) {
43+
switch (instruction.type) {
44+
case "staticContextParams":
45+
endpointParams[name] = instruction.value;
46+
break;
47+
case "contextParams":
48+
endpointParams[name] = commandInput[instruction.name] as string | boolean;
49+
break;
50+
case "clientContextParams":
51+
case "builtInParams":
52+
endpointParams[name] = await createConfigProvider<Config>(instruction.name, clientConfig)();
53+
break;
54+
default:
55+
throw new Error("Unrecognized endpoint parameter instruction: " + JSON.stringify(instruction));
56+
}
57+
}
58+
59+
const endpoint: EndpointV2 = clientConfig.endpointProvider!(endpointParams as T, context);
60+
61+
return endpoint;
62+
};
63+
64+
/**
65+
* Normalize some key of the client config to an async provider.
66+
* @private
67+
*/
68+
const createConfigProvider = <Config extends Record<string, unknown>>(configKey: string, config: Config) => {
69+
const configProvider = async () => {
70+
const configValue: unknown = config[configKey];
71+
if (typeof configValue === "function") {
72+
return configValue();
73+
}
74+
return configValue;
75+
};
76+
return configProvider;
77+
};

Diff for: ‎packages/middleware-endpoint/src/adaptors/index.ts

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export * from "./getEndpointFromInstructions";
2+
export * from "./toEndpointV1";

0 commit comments

Comments
 (0)