Skip to content

Commit afce30a

Browse files
webratzwebratz
authored
feat(core): allow asset bundling on docker remote host / docker in docker (#23576)
Fixes #8799 This implements an alternative variant on how to put get files into bundling containers. This is more flexible in its use cases for complex Docker setup scenarios but more complex and slower. Therefore it is not enabled as a default, but as an additional option. For details to the approach please refer to the linked issue. ---- ### All Submissions: * [X] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) ### Adding new Construct Runtime Dependencies: * [ ] This PR adds new construct runtime dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-construct-runtime-dependencies) ### New Features * [X] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)? * [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 356a128 commit afce30a

35 files changed

+2668
-76
lines changed

Diff for: ‎packages/@aws-cdk/aws-lambda-go/README.md

+14
Original file line numberDiff line numberDiff line change
@@ -284,3 +284,17 @@ and Go only includes dependencies that are used in the executable. So in this ca
284284
if `cmd/api` used the `auth` & `middleware` packages, but `cmd/anotherApi` did not, then
285285
an update to `auth` or `middleware` would only trigger an update to the `cmd/api` Lambda
286286
Function.
287+
288+
## Docker based bundling in complex Docker configurations
289+
290+
By default the input and output of Docker based bundling is handled via bind mounts.
291+
In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations.
292+
293+
```ts
294+
new go.GoFunction(this, 'GoFunction', {
295+
entry: 'app/cmd/api',
296+
bundling: {
297+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
298+
},
299+
});
300+
```

Diff for: ‎packages/@aws-cdk/aws-lambda-go/lib/bundling.ts

+9
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,12 @@ export interface BundlingProps extends BundlingOptions {
6060
* The system architecture of the lambda function
6161
*/
6262
readonly architecture: Architecture;
63+
64+
/**
65+
* Which option to use to copy the source files to the docker container and output files back
66+
* @default - BundlingFileAccess.BIND_MOUNT
67+
*/
68+
readonly bundlingFileAccess?: cdk.BundlingFileAccess;
6369
}
6470

6571
/**
@@ -85,6 +91,7 @@ export class Bundling implements cdk.BundlingOptions {
8591
user: bundling.user,
8692
securityOpt: bundling.securityOpt,
8793
network: bundling.network,
94+
bundlingFileAccess: bundling.bundlingFileAccess,
8895
},
8996
});
9097
}
@@ -107,6 +114,7 @@ export class Bundling implements cdk.BundlingOptions {
107114
public readonly user?: string;
108115
public readonly securityOpt?: string;
109116
public readonly network?: string;
117+
public readonly bundlingFileAccess?: cdk.BundlingFileAccess;
110118

111119
private readonly relativeEntryPath: string;
112120

@@ -154,6 +162,7 @@ export class Bundling implements cdk.BundlingOptions {
154162
this.user = props.user;
155163
this.securityOpt = props.securityOpt;
156164
this.network = props.network;
165+
this.bundlingFileAccess = props.bundlingFileAccess;
157166

158167
// Local bundling
159168
if (!props.forcedDockerBundling) { // only if Docker is not forced

Diff for: ‎packages/@aws-cdk/aws-lambda-go/lib/types.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core';
1+
import { AssetHashType, BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core';
22

33
/**
44
* Bundling options
@@ -111,6 +111,12 @@ export interface BundlingOptions extends DockerRunOptions {
111111
* @default - Direct access
112112
*/
113113
readonly goProxies?: string[];
114+
115+
/**
116+
* Which option to use to copy the source files to the docker container and output files back
117+
* @default - BundlingFileAccess.BIND_MOUNT
118+
*/
119+
readonly bundlingFileAccess?: BundlingFileAccess;
114120
}
115121

116122
/**

Diff for: ‎packages/@aws-cdk/aws-lambda-go/rosetta/default.ts-fixture

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Fixture with packages imported, but nothing else
22
import { Construct } from 'constructs';
3-
import { DockerImage, Stack } from '@aws-cdk/core';
3+
import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core';
44
import * as go from '@aws-cdk/aws-lambda-go';
55

66
class Fixture extends Stack {

Diff for: ‎packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import * as child_process from 'child_process';
22
import * as os from 'os';
33
import * as path from 'path';
44
import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda';
5-
import { AssetHashType, DockerImage } from '@aws-cdk/core';
5+
import { AssetHashType, BundlingFileAccess, DockerImage } from '@aws-cdk/core';
66
import { Bundling } from '../lib/bundling';
77
import * as util from '../lib/util';
88

@@ -461,3 +461,21 @@ test('Custom bundling network', () => {
461461
}),
462462
});
463463
});
464+
465+
test('Custom bundling file copy variant', () => {
466+
Bundling.bundle({
467+
entry,
468+
moduleDir,
469+
runtime: Runtime.GO_1_X,
470+
architecture: Architecture.X86_64,
471+
forcedDockerBundling: true,
472+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
473+
});
474+
475+
expect(Code.fromAsset).toHaveBeenCalledWith('/project', {
476+
assetHashType: AssetHashType.OUTPUT,
477+
bundling: expect.objectContaining({
478+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
479+
}),
480+
});
481+
});

Diff for: ‎packages/@aws-cdk/aws-lambda-nodejs/README.md

+13
Original file line numberDiff line numberDiff line change
@@ -337,3 +337,16 @@ new nodejs.NodejsFunction(this, 'my-handler', {
337337

338338
If you chose to customize the hash, you will need to make sure it is updated every time the asset
339339
changes, or otherwise it is possible that some deployments will not be invalidated.
340+
341+
## Docker based bundling in complex Docker configurations
342+
343+
By default the input and output of Docker based bundling is handled via bind mounts.
344+
In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations.
345+
346+
```ts
347+
new nodejs.NodejsFunction(this, 'my-handler', {
348+
bundling: {
349+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
350+
},
351+
});
352+
```

Diff for: ‎packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts

+7
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ export interface BundlingProps extends BundlingOptions {
4343
*/
4444
readonly preCompilation?: boolean
4545

46+
/**
47+
* Which option to use to copy the source files to the docker container and output files back
48+
* @default - BundlingFileAccess.BIND_MOUNT
49+
*/
50+
readonly bundlingFileAccess?: cdk.BundlingFileAccess;
4651
}
4752

4853
/**
@@ -83,6 +88,7 @@ export class Bundling implements cdk.BundlingOptions {
8388
public readonly securityOpt?: string;
8489
public readonly network?: string;
8590
public readonly local?: cdk.ILocalBundling;
91+
public readonly bundlingFileAccess?: cdk.BundlingFileAccess;
8692

8793
private readonly projectRoot: string;
8894
private readonly relativeEntryPath: string;
@@ -154,6 +160,7 @@ export class Bundling implements cdk.BundlingOptions {
154160
this.user = props.user;
155161
this.securityOpt = props.securityOpt;
156162
this.network = props.network;
163+
this.bundlingFileAccess = props.bundlingFileAccess;
157164

158165
// Local bundling
159166
if (!props.forceDockerBundling) { // only if Docker is not forced

Diff for: ‎packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { DockerImage, DockerRunOptions } from '@aws-cdk/core';
1+
import { BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core';
22

33
/**
44
* Bundling options
@@ -301,6 +301,12 @@ export interface BundlingOptions extends DockerRunOptions {
301301
* @default - no code is injected
302302
*/
303303
readonly inject?: string[]
304+
305+
/**
306+
* Which option to use to copy the source files to the docker container and output files back
307+
* @default - BundlingFileAccess.BIND_MOUNT
308+
*/
309+
readonly bundlingFileAccess?: BundlingFileAccess;
304310
}
305311

306312
/**

Diff for: ‎packages/@aws-cdk/aws-lambda-nodejs/rosetta/default.ts-fixture

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Fixture with packages imported, but nothing else
22
import { Construct } from 'constructs';
3-
import { DockerImage, Stack } from '@aws-cdk/core';
3+
import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core';
44
import * as nodejs from '@aws-cdk/aws-lambda-nodejs';
55

66
class Fixture extends Stack {

Diff for: ‎packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts

+20-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import * as child_process from 'child_process';
22
import * as os from 'os';
33
import * as path from 'path';
44
import { Architecture, Code, Runtime, RuntimeFamily } from '@aws-cdk/aws-lambda';
5-
import { AssetHashType, DockerImage } from '@aws-cdk/core';
5+
import { AssetHashType, BundlingFileAccess, DockerImage } from '@aws-cdk/core';
66
import { version as delayVersion } from 'delay/package.json';
77
import { Bundling } from '../lib/bundling';
88
import { PackageInstallation } from '../lib/package-installation';
@@ -826,3 +826,22 @@ test('Custom bundling network', () => {
826826
}),
827827
});
828828
});
829+
830+
test('Custom bundling file copy variant', () => {
831+
Bundling.bundle({
832+
entry,
833+
projectRoot,
834+
depsLockFilePath,
835+
runtime: Runtime.NODEJS_14_X,
836+
architecture: Architecture.X86_64,
837+
forceDockerBundling: true,
838+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
839+
});
840+
841+
expect(Code.fromAsset).toHaveBeenCalledWith('/project', {
842+
assetHashType: AssetHashType.OUTPUT,
843+
bundling: expect.objectContaining({
844+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
845+
}),
846+
});
847+
});

Diff for: ‎packages/@aws-cdk/aws-lambda-python/README.md

+17
Original file line numberDiff line numberDiff line change
@@ -252,3 +252,20 @@ an array of commands to run. Commands are chained with `&&`.
252252

253253
The commands will run in the environment in which bundling occurs: inside the
254254
container for Docker bundling or on the host OS for local bundling.
255+
256+
## Docker based bundling in complex Docker configurations
257+
258+
By default the input and output of Docker based bundling is handled via bind mounts.
259+
In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations.
260+
261+
```ts
262+
const entry = '/path/to/function';
263+
264+
new python.PythonFunction(this, 'function', {
265+
entry,
266+
runtime: Runtime.PYTHON_3_8,
267+
bundling: {
268+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
269+
},
270+
});
271+
```

Diff for: ‎packages/@aws-cdk/aws-lambda-python/lib/bundling.ts

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import * as path from 'path';
22
import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda';
3-
import { AssetStaging, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core';
3+
import { AssetStaging, BundlingFileAccess, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core';
44
import { Packaging, DependenciesFile } from './packaging';
55
import { BundlingOptions, ICommandHooks } from './types';
66

@@ -41,6 +41,12 @@ export interface BundlingProps extends BundlingOptions {
4141
* @default - Does not skip bundling
4242
*/
4343
readonly skip?: boolean;
44+
45+
/**
46+
* Which option to use to copy the source files to the docker container and output files back
47+
* @default - BundlingFileAccess.BIND_MOUNT
48+
*/
49+
bundlingFileAccess?: BundlingFileAccess
4450
}
4551

4652
/**
@@ -66,6 +72,7 @@ export class Bundling implements CdkBundlingOptions {
6672
public readonly user?: string;
6773
public readonly securityOpt?: string;
6874
public readonly network?: string;
75+
public readonly bundlingFileAccess?: BundlingFileAccess;
6976

7077
constructor(props: BundlingProps) {
7178
const {
@@ -104,6 +111,7 @@ export class Bundling implements CdkBundlingOptions {
104111
this.user = props.user;
105112
this.securityOpt = props.securityOpt;
106113
this.network = props.network;
114+
this.bundlingFileAccess = props.bundlingFileAccess;
107115
}
108116

109117
private createBundlingCommand(options: BundlingCommandOptions): string[] {

Diff for: ‎packages/@aws-cdk/aws-lambda-python/lib/types.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core';
1+
import { AssetHashType, BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core';
22

33

44
/**
@@ -86,6 +86,12 @@ export interface BundlingOptions extends DockerRunOptions {
8686
* @default - do not run additional commands
8787
*/
8888
readonly commandHooks?: ICommandHooks;
89+
90+
/**
91+
* Which option to use to copy the source files to the docker container and output files back
92+
* @default - BundlingFileAccess.BIND_MOUNT
93+
*/
94+
readonly bundlingFileAccess?: BundlingFileAccess;
8995
}
9096

9197
/**

Diff for: ‎packages/@aws-cdk/aws-lambda-python/rosetta/default.ts-fixture

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Fixture with packages imported, but nothing else
22
import { Construct } from 'constructs';
3-
import { DockerImage, Stack } from '@aws-cdk/core';
3+
import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core';
44
import { Runtime } from '@aws-cdk/aws-lambda';
55
import * as python from '@aws-cdk/aws-lambda-python';
66

Diff for: ‎packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts

+17-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import * as fs from 'fs';
22
import * as path from 'path';
33
import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda';
4-
import { DockerImage } from '@aws-cdk/core';
4+
import { BundlingFileAccess, DockerImage } from '@aws-cdk/core';
55
import { Bundling } from '../lib/bundling';
66

77
jest.spyOn(Code, 'fromAsset');
@@ -374,6 +374,22 @@ test('Bundling with custom network', () => {
374374
}));
375375
});
376376

377+
test('Bundling with docker copy variant', () => {
378+
const entry = path.join(__dirname, 'lambda-handler');
379+
Bundling.bundle({
380+
entry: entry,
381+
runtime: Runtime.PYTHON_3_7,
382+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
383+
384+
});
385+
386+
expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
387+
bundling: expect.objectContaining({
388+
bundlingFileAccess: BundlingFileAccess.VOLUME_COPY,
389+
}),
390+
}));
391+
});
392+
377393
test('Do not build docker image when skipping bundling', () => {
378394
const entry = path.join(__dirname, 'lambda-handler');
379395
Bundling.bundle({

0 commit comments

Comments
 (0)