Skip to content

Commit 49da47b

Browse files
AllanZhengYPAllanZhengYP
authored
feat(s3): support generating endpoints from multi-region access point (#2742)
feat(s3): support generating endpoints from multi-region access point (#2742)
1 parent afeccd7 commit 49da47b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+2431
-269
lines changed

‎clients/client-s3/runtimeConfig.shared.ts

+2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { defaultRegionInfoProvider } from "./endpoints";
2+
import { S3SignatureV4 } from "@aws-sdk/middleware-sdk-s3";
23
import { Logger as __Logger } from "@aws-sdk/types";
34
import { parseUrl } from "@aws-sdk/url-parser";
45
import { S3ClientConfig } from "./S3Client";
@@ -12,6 +13,7 @@ export const getRuntimeConfig = (config: S3ClientConfig) => ({
1213
logger: config?.logger ?? ({} as __Logger),
1314
regionInfoProvider: config?.regionInfoProvider ?? defaultRegionInfoProvider,
1415
serviceId: config?.serviceId ?? "S3",
16+
signerConstructor: config?.signerConstructor ?? S3SignatureV4,
1517
signingEscapePath: config?.signingEscapePath ?? false,
1618
urlParser: config?.urlParser ?? parseUrl,
1719
useArnRegion: config?.useArnRegion ?? false,

‎codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java

+4
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,10 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
8787
writer.write("false");
8888
}, "useArnRegion", writer -> {
8989
writer.write("false");
90+
}, "signerConstructor", writer -> {
91+
writer.addDependency(AwsDependency.S3_MIDDLEWARE)
92+
.addImport("S3SignatureV4", "S3SignatureV4", AwsDependency.S3_MIDDLEWARE.packageName)
93+
.write("S3SignatureV4");
9094
});
9195
case NODE:
9296
return MapUtils.of("useArnRegion", writer -> {

‎packages/middleware-bucket-endpoint/src/bucketEndpointMiddleware.spec.ts

+1
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ describe("bucketEndpointMiddleware", () => {
134134
clientSigningRegion: mockRegion,
135135
useArnRegion: false,
136136
isCustomEndpoint: false,
137+
disableMultiregionAccessPoints: false,
137138
});
138139
expect(previouslyResolvedConfig.region).toBeCalled();
139140
expect(previouslyResolvedConfig.regionInfoProvider).toBeCalled();

‎packages/middleware-bucket-endpoint/src/bucketEndpointMiddleware.ts

+63-68
Original file line numberDiff line numberDiff line change
@@ -15,81 +15,76 @@ import { bucketHostname } from "./bucketHostname";
1515
import { getPseudoRegion } from "./bucketHostnameUtils";
1616
import { BucketEndpointResolvedConfig } from "./configurations";
1717

18-
export const bucketEndpointMiddleware =
19-
(options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> =>
20-
<Output extends MetadataBearer>(
21-
next: BuildHandler<any, Output>,
22-
context: HandlerExecutionContext
23-
): BuildHandler<any, Output> =>
24-
async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
25-
const { Bucket: bucketName } = args.input as { Bucket: string };
26-
let replaceBucketInPath = options.bucketEndpoint;
27-
const request = args.request;
28-
if (HttpRequest.isInstance(request)) {
29-
if (options.bucketEndpoint) {
30-
request.hostname = bucketName;
31-
} else if (validateArn(bucketName)) {
32-
const bucketArn = parseArn(bucketName);
33-
const clientRegion = getPseudoRegion(await options.region());
34-
const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
35-
const useArnRegion = await options.useArnRegion();
36-
const {
37-
hostname,
38-
bucketEndpoint,
39-
signingRegion: modifiedSigningRegion,
40-
signingService,
41-
} = bucketHostname({
42-
bucketName: bucketArn,
43-
baseHostname: request.hostname,
44-
accelerateEndpoint: options.useAccelerateEndpoint,
45-
dualstackEndpoint: options.useDualstackEndpoint,
46-
pathStyleEndpoint: options.forcePathStyle,
47-
tlsCompatible: request.protocol === "https:",
48-
useArnRegion,
49-
clientPartition: partition,
50-
clientSigningRegion: signingRegion,
51-
clientRegion: clientRegion,
52-
isCustomEndpoint: options.isCustomEndpoint,
53-
});
18+
export const bucketEndpointMiddleware = (options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> => <
19+
Output extends MetadataBearer
20+
>(
21+
next: BuildHandler<any, Output>,
22+
context: HandlerExecutionContext
23+
): BuildHandler<any, Output> => async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
24+
const { Bucket: bucketName } = args.input as { Bucket: string };
25+
let replaceBucketInPath = options.bucketEndpoint;
26+
const request = args.request;
27+
if (HttpRequest.isInstance(request)) {
28+
if (options.bucketEndpoint) {
29+
request.hostname = bucketName;
30+
} else if (validateArn(bucketName)) {
31+
const bucketArn = parseArn(bucketName);
32+
const clientRegion = getPseudoRegion(await options.region());
33+
const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
34+
const useArnRegion = await options.useArnRegion();
35+
const { hostname, bucketEndpoint, signingRegion: modifiedSigningRegion, signingService } = bucketHostname({
36+
bucketName: bucketArn,
37+
baseHostname: request.hostname,
38+
accelerateEndpoint: options.useAccelerateEndpoint,
39+
dualstackEndpoint: options.useDualstackEndpoint,
40+
pathStyleEndpoint: options.forcePathStyle,
41+
tlsCompatible: request.protocol === "https:",
42+
useArnRegion,
43+
clientPartition: partition,
44+
clientSigningRegion: signingRegion,
45+
clientRegion: clientRegion,
46+
isCustomEndpoint: options.isCustomEndpoint,
47+
disableMultiregionAccessPoints: await options.disableMultiregionAccessPoints(),
48+
});
5449

55-
// If the request needs to use a region or service name inferred from ARN that different from client region, we
56-
// need to set them in the handler context so the signer will use them
57-
if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
58-
context["signing_region"] = modifiedSigningRegion;
59-
}
60-
if (signingService && signingService !== "s3") {
61-
context["signing_service"] = signingService;
62-
}
50+
// If the request needs to use a region or service name inferred from ARN that different from client region, we
51+
// need to set them in the handler context so the signer will use them
52+
if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
53+
context["signing_region"] = modifiedSigningRegion;
54+
}
55+
if (signingService && signingService !== "s3") {
56+
context["signing_service"] = signingService;
57+
}
6358

64-
request.hostname = hostname;
65-
replaceBucketInPath = bucketEndpoint;
66-
} else {
67-
const clientRegion = getPseudoRegion(await options.region());
68-
const { hostname, bucketEndpoint } = bucketHostname({
69-
bucketName,
70-
clientRegion,
71-
baseHostname: request.hostname,
72-
accelerateEndpoint: options.useAccelerateEndpoint,
73-
dualstackEndpoint: options.useDualstackEndpoint,
74-
pathStyleEndpoint: options.forcePathStyle,
75-
tlsCompatible: request.protocol === "https:",
76-
isCustomEndpoint: options.isCustomEndpoint,
77-
});
59+
request.hostname = hostname;
60+
replaceBucketInPath = bucketEndpoint;
61+
} else {
62+
const clientRegion = getPseudoRegion(await options.region());
63+
const { hostname, bucketEndpoint } = bucketHostname({
64+
bucketName,
65+
clientRegion,
66+
baseHostname: request.hostname,
67+
accelerateEndpoint: options.useAccelerateEndpoint,
68+
dualstackEndpoint: options.useDualstackEndpoint,
69+
pathStyleEndpoint: options.forcePathStyle,
70+
tlsCompatible: request.protocol === "https:",
71+
isCustomEndpoint: options.isCustomEndpoint,
72+
});
7873

79-
request.hostname = hostname;
80-
replaceBucketInPath = bucketEndpoint;
81-
}
74+
request.hostname = hostname;
75+
replaceBucketInPath = bucketEndpoint;
76+
}
8277

83-
if (replaceBucketInPath) {
84-
request.path = request.path.replace(/^(\/)?[^\/]+/, "");
85-
if (request.path === "") {
86-
request.path = "/";
87-
}
78+
if (replaceBucketInPath) {
79+
request.path = request.path.replace(/^(\/)?[^\/]+/, "");
80+
if (request.path === "") {
81+
request.path = "/";
8882
}
8983
}
84+
}
9085

91-
return next({ ...args, request });
92-
};
86+
return next({ ...args, request });
87+
};
9388

9489
export const bucketEndpointMiddlewareOptions: RelativeMiddlewareOptions = {
9590
tags: ["BUCKET_ENDPOINT"],

‎packages/middleware-bucket-endpoint/src/bucketHostname.spec.ts

+116-4
Original file line numberDiff line numberDiff line change
@@ -424,10 +424,6 @@ describe("bucketHostname", () => {
424424
bucketArn: "arn:aws:s3:us-west-2:123456789012:bucket_name:mybucket",
425425
message: "ARN resource should begin with 'accesspoint:' or 'outpost:'",
426426
},
427-
{
428-
bucketArn: "arn:aws:s3::123456789012:accesspoint:myendpoint",
429-
message: "ARN region is empty",
430-
},
431427
{
432428
bucketArn: "arn:aws:s3:us-west-2::accesspoint:myendpoint",
433429
message: "Access point ARN accountID does not match regex '[0-9]{12}'",
@@ -481,6 +477,122 @@ describe("bucketHostname", () => {
481477
});
482478
});
483479

480+
describe("from Multi-region Access Point(MRAP) ARN", () => {
481+
["us-east-1", "us-west-2", "aws-global"].forEach((region) => {
482+
it(`should populate endpoint from MRAP ARN in region "${region}"`, () => {
483+
const { bucketEndpoint, hostname, signingRegion } = bucketHostname({
484+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
485+
baseHostname: `s3.${region}.amazonaws.com`,
486+
disableMultiregionAccessPoints: false,
487+
clientRegion: region,
488+
isCustomEndpoint: false,
489+
});
490+
expect(bucketEndpoint).toBe(true);
491+
expect(hostname).toBe("mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com");
492+
expect(signingRegion).toBe("*");
493+
});
494+
});
495+
496+
it('should populate endpoint from MRAP ARN in region "cn-north-2"', () => {
497+
const { bucketEndpoint, hostname, signingRegion } = bucketHostname({
498+
bucketName: parseArn("arn:aws-cn:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
499+
clientPartition: "aws-cn",
500+
baseHostname: `s3.${region}.amazonaws.com.cn`,
501+
disableMultiregionAccessPoints: false,
502+
clientRegion: region,
503+
isCustomEndpoint: false,
504+
});
505+
expect(bucketEndpoint).toBe(true);
506+
expect(hostname).toBe("mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com.cn");
507+
expect(signingRegion).toBe("*");
508+
});
509+
510+
it("should throw if MRAP ARN is supplied but disabled through options", () => {
511+
expect(() =>
512+
bucketHostname({
513+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
514+
baseHostname: `s3.us-west-2.amazonaws.com`,
515+
disableMultiregionAccessPoints: true,
516+
clientRegion: region,
517+
isCustomEndpoint: false,
518+
})
519+
).toThrow("SDK is attempting to use a MRAP ARN. Please enable to feature.");
520+
});
521+
522+
it("should throw if dualstack option is set", () => {
523+
expect(() =>
524+
bucketHostname({
525+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
526+
baseHostname: `s3.us-west-2.amazonaws.com`,
527+
dualstackEndpoint: true,
528+
clientRegion: region,
529+
isCustomEndpoint: false,
530+
})
531+
).toThrow("Dualstack endpoint is not supported with Outpost or Multi-region Access Point ARN.");
532+
});
533+
534+
it("should throw if accelerate endpoint option is set", () => {
535+
expect(() =>
536+
bucketHostname({
537+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
538+
baseHostname: `s3.us-west-2.amazonaws.com`,
539+
accelerateEndpoint: true,
540+
clientRegion: region,
541+
isCustomEndpoint: false,
542+
})
543+
).toThrow("Accelerate endpoint is not supported when bucket is an ARN");
544+
});
545+
546+
it("should throw if region is empty and disableMultiregionAccessPoints option is set", () => {
547+
expect(() =>
548+
bucketHostname({
549+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:myendpoint"),
550+
baseHostname: `s3.us-west-2.amazonaws.com`,
551+
disableMultiregionAccessPoints: true,
552+
clientRegion: region,
553+
isCustomEndpoint: false,
554+
})
555+
).toThrow("");
556+
});
557+
558+
it('should populate endpoint from MRAP ARN with access point name "myendpoint"', () => {
559+
const { bucketEndpoint, hostname } = bucketHostname({
560+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:myendpoint"),
561+
baseHostname: `s3.us-west-2.amazonaws.com`,
562+
disableMultiregionAccessPoints: false,
563+
clientRegion: region,
564+
isCustomEndpoint: false,
565+
});
566+
expect(bucketEndpoint).toBe(true);
567+
expect(hostname).toBe("myendpoint.accesspoint.s3-global.amazonaws.com");
568+
});
569+
570+
it('should populate endpoint from MRAP ARN with access point name "my.bucket"', () => {
571+
const { bucketEndpoint, hostname } = bucketHostname({
572+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:my.bucket"),
573+
baseHostname: `s3.us-west-2.amazonaws.com`,
574+
disableMultiregionAccessPoints: false,
575+
clientRegion: region,
576+
isCustomEndpoint: false,
577+
});
578+
expect(bucketEndpoint).toBe(true);
579+
expect(hostname).toBe("my.bucket.accesspoint.s3-global.amazonaws.com");
580+
});
581+
582+
it("should populate endpoint from MRAP ARN with custom endpoint", () => {
583+
const { bucketEndpoint, hostname, signingRegion } = bucketHostname({
584+
bucketName: parseArn("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"),
585+
baseHostname: "vpce-123-abc.vpce.s3-global.amazonaws.com",
586+
isCustomEndpoint: true,
587+
clientRegion: "us-west-2",
588+
disableMultiregionAccessPoints: false,
589+
});
590+
expect(bucketEndpoint).toBe(true);
591+
expect(hostname).toBe("mfzwi23gnjvgw.mrap.vpce-123-abc.vpce.s3-global.amazonaws.com");
592+
expect(signingRegion).toBe("*");
593+
});
594+
});
595+
484596
describe("from Outpost ARN", () => {
485597
describe("populates access point endpoint from ARN", () => {
486598
const s3Hostname = "s3.us-west-2.amazonaws.com";

0 commit comments

Comments
 (0)