feat(lambda-python): add optional poetry bundling exclusion list parameter (#23670)

A summary of this change is: change from use of `cp` to `rsync --exclude='x'` in the bundling commands for `poetry` based lambdas.


The intention of this PR is to enable the bundling code for Poetry projects to exclude certain files and/or folders from the bundled assets. Currently, if developing a python lambda using either `virtualenv` itself or a toolchain that leverages virtual environments (re: `poetry`, specifically with `virtualenv.in-project = true`, which is strongly recommended for leveraging python tools in VSCode), the bundling code will copy the entire folder passed in. This leads to copying the entire `.venv` directory into the bundled assets, even though the directory is ignored. Ultimately this leads to inflating the assets by the size of unzipped dependencies (`numpy`, for instance, is 50Mb by itself).

I verified this concept works in another project I maintain which leverages `@aws-cdk/aws-lambda-python-alpha` by manually editing the bundling file (I mentioned it in more detail in the linked issue #22585), but this temporary approach requires manually editing files from `node_modules`, so it is not a proper fix.



fixes #22585 


----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Construct Runtime Dependencies:

~* [ ] This PR adds new construct runtime dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-construct-runtime-dependencies)~

### New Features

* [x] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

~~**Note**: I was unable to implement an integration test. My plan was to verify an asset was ignored from the existing poetry integration test sample directory `packages/@aws-cdk/aws-lambda-python/test/lambda-handler-poetry` by passing in `['.ignorefile']` and confirming that asset was excluded, but I was unable to get the test working due to SSM parameters missing. I wasn't sure if I could bootstrap this and get it working.~~


*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: ryanandonian

Diff for: packages/@aws-cdk/aws-lambda-python/README.md

@@ -109,6 +109,22 @@ Packaging is executed using the `Packaging` class, which:
 ├── poetry.lock # your poetry lock file has to be present at the entry path
 ```
 
+**Excluding source files**
+
+You can exclude files from being copied using the optional bundling string array parameter `assetExcludes`
+
+```ts
+new python.PythonFunction(this, 'function', {
+  entry: '/path/to/poetry-function',
+  runtime: Runtime.PYTHON_3_8,
+  bundling: {
+    // translates to `rsync --exclude='.venv'`
+    assetExcludes: ['.venv'],
+  },
+});
+```
+
+
 ## Custom Bundling
 
 Custom bundling can be performed by passing in additional build arguments that point to index URLs to private repos, or by using an entirely custom Docker images for bundling dependencies. The build args currently supported are:

Diff for: packages/@aws-cdk/aws-lambda-python/lib/bundling.ts

@@ -83,6 +83,7 @@ export class Bundling implements CdkBundlingOptions {
       image,
       poetryIncludeHashes,
       commandHooks,
+      assetExcludes = [],
     } = props;
 
     const outputPath = path.posix.join(AssetStaging.BUNDLING_OUTPUT_DIR, outputPathSuffix);
@@ -93,6 +94,7 @@ export class Bundling implements CdkBundlingOptions {
       outputDir: outputPath,
       poetryIncludeHashes,
       commandHooks,
+      assetExcludes,
     });
 
     this.image = image ?? DockerImage.fromBuild(path.join(__dirname, '../lib'), {
@@ -118,7 +120,10 @@ export class Bundling implements CdkBundlingOptions {
     const packaging = Packaging.fromEntry(options.entry, options.poetryIncludeHashes);
     let bundlingCommands: string[] = [];
     bundlingCommands.push(...options.commandHooks?.beforeBundling(options.inputDir, options.outputDir) ?? []);
-    bundlingCommands.push(`cp -rTL ${options.inputDir}/ ${options.outputDir}`);
+    const exclusionStr = options.assetExcludes?.map(item => `--exclude='${item}'`).join(' ');
+    bundlingCommands.push([
+      'rsync', '-rLv', exclusionStr ?? '', `${options.inputDir}/`, options.outputDir,
+    ].filter(item => item).join(' '));
     bundlingCommands.push(`cd ${options.outputDir}`);
     bundlingCommands.push(packaging.exportCommand ?? '');
     if (packaging.dependenciesFile) {
@@ -133,8 +138,9 @@ interface BundlingCommandOptions {
   readonly entry: string;
   readonly inputDir: string;
   readonly outputDir: string;
+  readonly assetExcludes?: string[];
   readonly poetryIncludeHashes?: boolean;
-  readonly commandHooks?: ICommandHooks
+  readonly commandHooks?: ICommandHooks;
 }
 
 /**

Diff for: packages/@aws-cdk/aws-lambda-python/lib/types.ts

@@ -15,6 +15,13 @@ export interface BundlingOptions extends DockerRunOptions {
    */
   readonly poetryIncludeHashes?: boolean;
 
+  /**
+ * List of file patterns to exclude when copying assets from source for bundling.
+ *
+ * @default - Empty list
+ */
+  readonly assetExcludes?: string[];
+
   /**
    * Output path suffix: the suffix for the directory into which the bundled output is written.
    *

Diff for: packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts

@@ -37,7 +37,7 @@ test('Bundling a function without dependencies', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output && cd /asset-output',
+        'rsync -rLv /asset-input/ /asset-output && cd /asset-output',
       ],
     }),
   }));
@@ -66,7 +66,32 @@ test('Bundling a function with requirements.txt', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output',
+        'rsync -rLv /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output',
+      ],
+    }),
+  }));
+
+  const files = fs.readdirSync(assetCode.path);
+  expect(files).toContain('index.py');
+  expect(files).toContain('requirements.txt');
+  expect(files).toContain('.ignorelist');
+});
+
+test('Bundling a function with requirements.txt using assetExcludes', () => {
+  const entry = path.join(__dirname, 'lambda-handler');
+  const assetCode = Bundling.bundle({
+    entry: entry,
+    runtime: Runtime.PYTHON_3_7,
+    architecture: Architecture.X86_64,
+    assetExcludes: ['.ignorelist'],
+  });
+
+  // Correctly bundles
+  expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
+    bundling: expect.objectContaining({
+      command: [
+        'bash', '-c',
+        "rsync -rLv --exclude='.ignorelist' /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output",
       ],
     }),
   }));
@@ -89,7 +114,26 @@ test('Bundling Python 2.7 with requirements.txt installed', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output',
+        'rsync -rLv /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output',
+      ],
+    }),
+  }));
+});
+
+test('Bundling Python 2.7 with requirements.txt installed', () => {
+  const entry = path.join(__dirname, 'lambda-handler');
+  Bundling.bundle({
+    entry: entry,
+    runtime: Runtime.PYTHON_2_7,
+    architecture: Architecture.X86_64,
+  });
+
+  // Correctly bundles with requirements.txt pip installed
+  expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
+    bundling: expect.objectContaining({
+      command: [
+        'bash', '-c',
+        'rsync -rLv /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output',
       ],
     }),
   }));
@@ -109,7 +153,7 @@ test('Bundling a layer with dependencies', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python',
+        'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python',
       ],
     }),
   }));
@@ -129,7 +173,7 @@ test('Bundling a python code layer', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python',
+        'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python',
       ],
     }),
   }));
@@ -149,7 +193,35 @@ test('Bundling a function with pipenv dependencies', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && PIPENV_VENV_IN_PROJECT=1 pipenv lock -r > requirements.txt && rm -rf .venv && python -m pip install -r requirements.txt -t /asset-output/python',
+        'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && PIPENV_VENV_IN_PROJECT=1 pipenv lock -r > requirements.txt && rm -rf .venv && python -m pip install -r requirements.txt -t /asset-output/python',
+      ],
+    }),
+  }));
+
+  const files = fs.readdirSync(assetCode.path);
+  expect(files).toContain('index.py');
+  expect(files).toContain('Pipfile');
+  expect(files).toContain('Pipfile.lock');
+  // Contains hidden files.
+  expect(files).toContain('.ignorefile');
+});
+
+test('Bundling a function with pipenv dependencies with assetExcludes', () => {
+  const entry = path.join(__dirname, 'lambda-handler-pipenv');
+
+  const assetCode = Bundling.bundle({
+    entry: path.join(entry, '.'),
+    runtime: Runtime.PYTHON_3_9,
+    architecture: Architecture.X86_64,
+    outputPathSuffix: 'python',
+    assetExcludes: ['.ignorefile'],
+  });
+
+  expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
+    bundling: expect.objectContaining({
+      command: [
+        'bash', '-c',
+        "rsync -rLv --exclude='.ignorefile' /asset-input/ /asset-output/python && cd /asset-output/python && PIPENV_VENV_IN_PROJECT=1 pipenv lock -r > requirements.txt && rm -rf .venv && python -m pip install -r requirements.txt -t /asset-output/python",
       ],
     }),
   }));
@@ -176,7 +248,7 @@ test('Bundling a function with poetry dependencies', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --without-hashes --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python',
+        'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --without-hashes --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python',
       ],
     }),
   }));
@@ -189,6 +261,48 @@ test('Bundling a function with poetry dependencies', () => {
   expect(files).toContain('.ignorefile');
 });
 
+test('Bundling a function with poetry and assetExcludes', () => {
+  const entry = path.join(__dirname, 'lambda-handler-poetry');
+
+  Bundling.bundle({
+    entry: path.join(entry, '.'),
+    runtime: Runtime.PYTHON_3_9,
+    architecture: Architecture.X86_64,
+    outputPathSuffix: 'python',
+    assetExcludes: ['.ignorefile'],
+  });
+
+  expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
+    bundling: expect.objectContaining({
+      command: [
+        'bash', '-c',
+        "rsync -rLv --exclude='.ignorefile' /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --without-hashes --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python",
+      ],
+    }),
+  }));
+
+});
+
+test('Bundling a function with poetry and no assetExcludes', () => {
+  const entry = path.join(__dirname, 'lambda-handler-poetry');
+
+  Bundling.bundle({
+    entry: path.join(entry, '.'),
+    runtime: Runtime.PYTHON_3_9,
+    architecture: Architecture.X86_64,
+    outputPathSuffix: 'python',
+  });
+
+  expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({
+    bundling: expect.objectContaining({
+      command: [
+        'bash', '-c',
+        expect.not.stringContaining('--exclude'),
+      ],
+    }),
+  }));
+});
+
 test('Bundling a function with poetry dependencies, with hashes', () => {
   const entry = path.join(__dirname, 'lambda-handler-poetry');
 
@@ -204,7 +318,7 @@ test('Bundling a function with poetry dependencies, with hashes', () => {
     bundling: expect.objectContaining({
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python',
+        'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python',
       ],
     }),
   }));
@@ -234,7 +348,7 @@ test('Bundling a function with custom bundling image', () => {
       image,
       command: [
         'bash', '-c',
-        'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python',
+        'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python',
       ],
     }),
   }));